mirror of
https://github.com/vladmandic/automatic
synced 2026-09-20 01:31:13 +02:00
Vladimir Mandic
GitHub
@@ -0,0 +1,26 @@
|
||||
# defaults
|
||||
.history
|
||||
.vscode/
|
||||
/__pycache__
|
||||
/.ruff_cache
|
||||
/cache
|
||||
/cache.json
|
||||
/config.json
|
||||
/extensions/*
|
||||
/html/extensions.json
|
||||
/html/themes.json
|
||||
/metadata.json
|
||||
/node_modules
|
||||
/outputs/*
|
||||
/package-lock.json
|
||||
/params.txt
|
||||
/pnpm-lock.yaml
|
||||
/styles.csv
|
||||
/tmp
|
||||
/ui-config.json
|
||||
/user.css
|
||||
/venv
|
||||
/webui-user.bat
|
||||
/webui-user.sh
|
||||
/*.log.*
|
||||
/*.log
|
||||
@@ -5,6 +5,7 @@
|
||||
"plugins": ["html", "json"],
|
||||
"extends": [
|
||||
"plugin:json/recommended",
|
||||
"plugin:node/recommended",
|
||||
"eslint:recommended",
|
||||
"airbnb-base"
|
||||
],
|
||||
|
||||
+1
-1
@@ -43,6 +43,7 @@ tunableop_results*.csv
|
||||
!webui.bat
|
||||
!webui.sh
|
||||
!package.json
|
||||
!requirements.txt
|
||||
|
||||
# pyinstaller
|
||||
*.spec
|
||||
@@ -74,4 +75,3 @@ dist/
|
||||
!/models/VAE-approx/model.pt
|
||||
!/models/Reference
|
||||
!/models/Reference/**/*
|
||||
|
||||
|
||||
@@ -8,29 +8,31 @@ fail-under=10
|
||||
ignore=CVS
|
||||
ignore-paths=/usr/lib/.*$,
|
||||
modules/apg,
|
||||
modules/consistory,
|
||||
modules/control/proc,
|
||||
modules/control/units,
|
||||
modules/ctrlx,
|
||||
modules/dcsolver,
|
||||
modules/dml,
|
||||
modules/ggml,
|
||||
modules/hidiffusion,
|
||||
modules/hijack,
|
||||
modules/instantir,
|
||||
modules/intel/ipex,
|
||||
modules/intel/openvino,
|
||||
modules/k-diffusion,
|
||||
modules/ldsr,
|
||||
modules/meissonic,
|
||||
modules/omnigen,
|
||||
modules/onnx_impl,
|
||||
modules/pag,
|
||||
modules/prompt_parser_xhinker.py,
|
||||
modules/pulid/eva_clip,
|
||||
modules/rife,
|
||||
modules/schedulers,
|
||||
modules/taesd,
|
||||
modules/todo,
|
||||
modules/unipc,
|
||||
modules/vdm,
|
||||
modules/xadapter,
|
||||
modules/meissonic,
|
||||
modules/omnigen,
|
||||
repositories,
|
||||
extensions-builtin/sd-webui-agent-scheduler,
|
||||
extensions-builtin/sd-extension-chainner/nodes,
|
||||
@@ -130,7 +132,8 @@ confidence=HIGH,
|
||||
INFERENCE_FAILURE,
|
||||
UNDEFINED
|
||||
# disable=C,R,W
|
||||
disable=bad-inline-option,
|
||||
disable=abstract-method,
|
||||
bad-inline-option,
|
||||
bare-except,
|
||||
broad-exception-caught,
|
||||
chained-comparison,
|
||||
@@ -174,6 +177,7 @@ disable=bad-inline-option,
|
||||
unnecessary-dict-index-lookup,
|
||||
unnecessary-dunder-call,
|
||||
unnecessary-lambda,
|
||||
unnecessary-lambda-assigment,
|
||||
use-dict-literal,
|
||||
use-symbolic-message-instead,
|
||||
unknown-option-value,
|
||||
|
||||
+6
-4
@@ -4,28 +4,30 @@ exclude = [
|
||||
".ruff_cache",
|
||||
".vscode",
|
||||
"modules/apg",
|
||||
"modules/consistory",
|
||||
"modules/control/proc",
|
||||
"modules/control/units",
|
||||
"modules/dcsolver",
|
||||
"modules/ggml",
|
||||
"modules/hidiffusion",
|
||||
"modules/hijack",
|
||||
"modules/instantir",
|
||||
"modules/intel/ipex",
|
||||
"modules/intel/openvino",
|
||||
"modules/k-diffusion",
|
||||
"modules/ldsr",
|
||||
"modules/meissonic",
|
||||
"modules/omnigen",
|
||||
"modules/pag",
|
||||
"modules/postprocess/aurasr_arch.py",
|
||||
"modules/prompt_parser_xhinker.py",
|
||||
"modules/pulid/eva_clip",
|
||||
"modules/rife",
|
||||
"modules/schedulers",
|
||||
"modules/segmoe",
|
||||
"modules/taesd",
|
||||
"modules/todo",
|
||||
"modules/unipc",
|
||||
"modules/vdm",
|
||||
"modules/xadapter",
|
||||
"modules/meissonic",
|
||||
"modules/omnigen",
|
||||
"repositories",
|
||||
"extensions-builtin/sd-extension-chainner/nodes",
|
||||
"extensions-builtin/sd-webui-agent-scheduler",
|
||||
|
||||
+145
-29
@@ -1,37 +1,156 @@
|
||||
# Change Log for SD.Next
|
||||
|
||||
## Update for 2024-11-02
|
||||
## Update for 2024-11-21
|
||||
|
||||
Smaller release just few days after the last one, but with some important fixes and improvements.
|
||||
### Highlights for 2024-11-21
|
||||
|
||||
Three weeks is a long time in Generative AI world - and we're back with ~140 commits worth of updates!
|
||||
|
||||
*What's New?*
|
||||
|
||||
First, a massive update to docs including new UI top-level **info** tab with access to [changelog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) and [wiki](https://github.com/vladmandic/automatic/wiki), many updates and new articles AND full **built-in documentation search** capabilities
|
||||
|
||||
#### New integrations
|
||||
|
||||
- [PuLID](https://github.com/ToTheBeginning/PuLID): Pure and Lightning ID Customization via Contrastive Alignment
|
||||
- [InstantX InstantIR](https://github.com/instantX-research/InstantIR): Blind Image Restoration with Instant Generative Reference
|
||||
- [nVidia Labs ConsiStory](https://github.com/NVlabs/consistory): Consistent Image Generation
|
||||
- [MiaoshouAI PromptGen v2.0](https://huggingface.co/MiaoshouAI/Florence-2-base-PromptGen-v2.0) VQA captioning
|
||||
|
||||
#### Workflow Improvements
|
||||
|
||||
- Native **Docker** support
|
||||
- **SD3x & Flux.1**: more ControlNets, all-in-one-safetensors, DPM samplers, skip-layer-guidance, etc.
|
||||
- **XYZ grid**: benchmarking, video creation, etc.
|
||||
- Enhanced **prompt** parsing
|
||||
- **UI** improvements
|
||||
- **Installer** self-healing `venv`
|
||||
|
||||
And quite a few more improvements and fixes since the last update!
|
||||
For full list and details see changelog...
|
||||
|
||||
[README](https://github.com/vladmandic/automatic/blob/master/README.md) | [CHANGELOG](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867)
|
||||
|
||||
### Details for 2024-11-21
|
||||
|
||||
- Docs:
|
||||
- new top-level **info** tab with access to [changelog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) and [wiki](https://github.com/vladmandic/automatic/wiki)
|
||||
- UI built-in [changelog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) search
|
||||
since changelog is the best up-to-date source of info
|
||||
go to info -> changelog and search/highligh/navigate directly in UI!
|
||||
- UI built-in [wiki](https://github.com/vladmandic/automatic/wiki)
|
||||
go to info -> wiki and search wiki pages directly in UI!
|
||||
- major [Wiki](https://github.com/vladmandic/automatic/wiki) and [Home](https://github.com/vladmandic/automatic) updates
|
||||
- updated API swagger docs for at `/docs`
|
||||
- Integrations:
|
||||
- [PuLID](https://github.com/ToTheBeginning/PuLID): Pure and Lightning ID Customization via Contrastive Alignment
|
||||
- advanced method of face id transfer with better quality as well as control over identity and appearance
|
||||
try it out, likely the best quality available for sdxl models
|
||||
- select in *scripts -> pulid*
|
||||
- compatible with *sdxl* for text-to-image, image-to-image, inpaint, refine, detailer workflows
|
||||
- can be used in xyz grid
|
||||
- *note*: this module contains several advanced features on top of original implementation
|
||||
- [InstantIR](https://github.com/instantX-research/InstantIR): Blind Image Restoration with Instant Generative Reference
|
||||
- alternative to traditional `img2img` with more control over restoration process
|
||||
- select in *image -> scripts -> instantir*
|
||||
- compatible with *sdxl*
|
||||
- *note*: after used once it cannot be unloaded without reloading base model
|
||||
- [ConsiStory](https://github.com/NVlabs/consistory): Consistent Image Generation
|
||||
- create consistent anchor image and then generate images that are consistent with anchor
|
||||
- select in *scripts -> consistory*
|
||||
- compatible with *sdxl*
|
||||
- *note*: very resource intensive and not compatible with model offloading
|
||||
- *note*: changing default parameters can lead to unexpected results and/or failures
|
||||
- *note*: after used once it cannot be unloaded without reloading base model
|
||||
- [MiaoshouAI PromptGen v2.0](https://huggingface.co/MiaoshouAI/Florence-2-base-PromptGen-v2.0) base and large:
|
||||
- *in process -> visual query*
|
||||
- caption modes:
|
||||
`<GENERATE_TAGS>` generate tags
|
||||
`<CAPTION>`, `<DETAILED_CAPTION>`, `<MORE_DETAILED_CAPTION>` caption image
|
||||
`<ANALYZE>` image composition
|
||||
`<MIXED_CAPTION>`, `<MIXED_CAPTION_PLUS>` detailed caption and tags with optional analyze
|
||||
|
||||
- Model improvements:
|
||||
- SD35: **ControlNets**:
|
||||
- *InstantX Canny, Pose, Depth, Tile*
|
||||
- *Alimama Inpainting, SoftEdge*
|
||||
- *note*: that just like with FLUX.1 or any large model, ControlNet are also large and can push your system over the limit
|
||||
e.g. SD3 controlnets vary from 1GB to over 4GB in size
|
||||
- SD35: **All-in-one** safetensors
|
||||
- *examples*: [large](https://civitai.com/models/882666/sd35-large-google-flan?modelVersionId=1003031), [medium](https://civitai.com/models/900327)
|
||||
- *note*: enable *bnb* on-the-fly quantization for even bigger gains
|
||||
- SD35: **skip-layer-guidance**
|
||||
- enable in *scripts -> slg*
|
||||
- allows for granular strength/start/stop control of guidance for each layer of the model
|
||||
- [NoobAI XL ControlNets](https://huggingface.co/collections/Eugeoter/controlnext-673161eae023f413e0432799), thanks @lbeltrame
|
||||
|
||||
- Workflow improvements:
|
||||
- Native Docker support with pre-defined [Dockerfile](https://github.com/vladmandic/automatic/blob/dev/Dockerfile)
|
||||
- Samplers:
|
||||
- **FlowMatch samplers**:
|
||||
- Applicable to SD 3.x and Flux.1 models
|
||||
- Complete family: *DPM2, DPM2a, DPM2++, DPM2++ 2M, DPM2++ 2S, DPM2++ SDE, DPM2++ 2M SDE, DPM2++ 3M SDE*
|
||||
- **Beta and Exponential** sigma method enabled for all samplers
|
||||
- **XYZ grid**:
|
||||
- optional time benchmark info to individual images
|
||||
- optional add params to individual images
|
||||
- create video from generated grid images
|
||||
supports all standard video types and interpolation
|
||||
- **Prompt parser**:
|
||||
- support for prompt scheduling
|
||||
- renamed parser options: `native`, `xhinker`, `compel`, `a1111`, `fixed`
|
||||
- parser options are available in xyz grid
|
||||
- improved caching
|
||||
- **UI**:
|
||||
- better gallery and networks sidebar sizing
|
||||
- add additional [hotkeys](https://github.com/vladmandic/automatic/wiki/Hotkeys)
|
||||
- add show networks on startup setting
|
||||
- better mapping of networks previews
|
||||
- optimize networks display load
|
||||
- Image2image:
|
||||
- integrated refine/upscale/hires workflow
|
||||
- Other:
|
||||
- **Installer**:
|
||||
- Log `venv` and package search paths
|
||||
- Auto-remove invalid packages from `venv/site-packages`
|
||||
e.g. packages starting with `~` which are left-over due to windows access violation
|
||||
- Requirements: update
|
||||
- Scripts:
|
||||
- More verbose descriptions for all scripts
|
||||
- Model loader:
|
||||
- Report modules included in safetensors when attempting to load a model
|
||||
- CLI:
|
||||
- refactor command line params
|
||||
run `webui.sh`/`webui.bat` with `--help` to see all options
|
||||
- added `cli/model-metadata.py` to display metadata in any safetensors file
|
||||
- added `cli/model-keys.py` to quicky display content of any safetensors file
|
||||
- Internal:
|
||||
- Auto pipeline switching coveres wrapper classes and nested pipelines
|
||||
- Full settings validation on load of `config.json`
|
||||
- Refactor of all params in main processing classes
|
||||
- Improve API scripts usage resiliency
|
||||
|
||||
- Fixes:
|
||||
- custom watermark add alphablending
|
||||
- fix xyz grid include images
|
||||
- fix xyz skip on interrupted
|
||||
- fix vqa models ignoring hfcache folder setting
|
||||
- fix network height in standard vs modern ui
|
||||
- fix k-diff enum on startup
|
||||
- fix text2video scripts
|
||||
- multiple xyz-grid fixes
|
||||
- dont uninstall flash-attn
|
||||
- ui css fixes
|
||||
|
||||
## Update for 2024-11-01
|
||||
|
||||
Smaller release just 3 days after the last one, but with some important fixes and improvements.
|
||||
This release can be considered an LTS release before we kick off the next round of major updates.
|
||||
|
||||
- Docs:
|
||||
- add built-in [changelog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) search
|
||||
since changelog is the best up-to-date source of info
|
||||
go to system -> changelog and search/highligh/navigate directly in UI!
|
||||
- SD3: ControlNets:
|
||||
- *InstantX Canny, Pose, Depth, Tile*
|
||||
- *Alimama Inpainting, SoftEdge*
|
||||
- *note*: that just like with FLUX.1 or any large model, ControlNet are also large and can push your system over the limit
|
||||
e.g. SD3 controlnets vary from 1GB to over 4GB in size
|
||||
- SD3: all-in-one safetensors
|
||||
- *examples*: [large](https://civitai.com/models/882666/sd35-large-google-flan?modelVersionId=1003031), [medium](https://civitai.com/models/900327)
|
||||
- *note*: enable *bnb* on-the-fly quantization for even bigger gains
|
||||
- UI:
|
||||
- add additional [hotkeys](https://github.com/vladmandic/automatic/wiki/Hotkeys)
|
||||
- add show networks on startup setting
|
||||
- better mapping of networks previews
|
||||
- optimize networks display load
|
||||
- XYZ grid:
|
||||
- optional per-image time benchmark info
|
||||
- CLI:
|
||||
- refactor command line params
|
||||
run `webui.sh`/`webui.bat` with `--help` to see all options
|
||||
- Other:
|
||||
- Repo: move screenshots to GH pages
|
||||
- Update requirements
|
||||
- Fixes:
|
||||
- custom watermark add alphablending
|
||||
- detailer min/max size as fractions of image size
|
||||
- ipadapter load on-demand
|
||||
- ipadapter face use correct yolo model
|
||||
@@ -40,10 +159,7 @@ This release can be considered an LTS release before we kick off the next round
|
||||
- fix diffusers load from folder
|
||||
- fix lora enum logging on windows
|
||||
- fix xyz grid with batch count
|
||||
- fix vqa models ignoring hfcache folder setting
|
||||
- fix network height in standard vs modern ui
|
||||
- fix k-diff enum on startup
|
||||
- move downloads of some auxillary models to hfcache instead of models folder
|
||||
- move dowwloads of some auxillary models to hfcache instead of models folder
|
||||
|
||||
## Update for 2024-10-29
|
||||
|
||||
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
# SD.Next Dockerfile
|
||||
# docs: <https://github.com/vladmandic/automatic/wiki/Docker>
|
||||
|
||||
# base image
|
||||
FROM pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime
|
||||
|
||||
# metadata
|
||||
LABEL org.opencontainers.image.vendor="SD.Next"
|
||||
LABEL org.opencontainers.image.authors="vladmandic"
|
||||
LABEL org.opencontainers.image.url="https://github.com/vladmandic/automatic/"
|
||||
LABEL org.opencontainers.image.documentation="https://github.com/vladmandic/automatic/wiki/Docker"
|
||||
LABEL org.opencontainers.image.source="https://github.com/vladmandic/automatic/"
|
||||
LABEL org.opencontainers.image.licenses="AGPL-3.0"
|
||||
LABEL org.opencontainers.image.title="SD.Next"
|
||||
LABEL org.opencontainers.image.description="SD.Next: Advanced Implementation of Stable Diffusion and other Diffusion-based generative image models"
|
||||
LABEL org.opencontainers.image.base.name="https://hub.docker.com/pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime"
|
||||
LABEL org.opencontainers.image.version="latest"
|
||||
|
||||
# minimum install
|
||||
RUN ["apt-get", "-y", "update"]
|
||||
RUN ["apt-get", "-y", "install", "git", "build-essential", "google-perftools", "curl"]
|
||||
# optional if full cuda-dev is required by some downstream library
|
||||
# RUN ["apt-get", "-y", "nvidia-cuda-toolkit"]
|
||||
RUN ["/usr/sbin/ldconfig"]
|
||||
|
||||
# copy sdnext
|
||||
COPY . /app
|
||||
WORKDIR /app
|
||||
|
||||
# stop pip and uv from caching
|
||||
ENV PIP_NO_CACHE_DIR=true
|
||||
ENV PIP_ROOT_USER_ACTION=ignore
|
||||
ENV UV_NO_CACHE=true
|
||||
# disable model hashing for faster startup
|
||||
ENV SD_NOHASHING=true
|
||||
# set data directories
|
||||
ENV SD_DATADIR="/mnt/data"
|
||||
ENV SD_MODELSDIR="/mnt/models"
|
||||
ENV SD_DOCKER=true
|
||||
|
||||
# tcmalloc is not required but it is highly recommended
|
||||
ENV LD_PRELOAD=libtcmalloc.so.4
|
||||
# sdnext will run all necessary pip install ops and then exit
|
||||
RUN ["python", "/app/launch.py", "--debug", "--uv", "--use-cuda", "--log", "sdnext.log", "--test", "--optional"]
|
||||
# preinstall additional packages to avoid installation during runtime
|
||||
|
||||
# actually run sdnext
|
||||
CMD ["python", "launch.py", "--debug", "--skip-all", "--listen", "--quick", "--api-log", "--log", "sdnext.log"]
|
||||
|
||||
# expose port
|
||||
EXPOSE 7860
|
||||
|
||||
# healthcheck function
|
||||
# HEALTHCHECK --interval=60s --timeout=10s --start-period=60s --retries=3 CMD curl --fail http://localhost:7860/sdapi/v1/status || exit 1
|
||||
|
||||
# stop signal
|
||||
STOPSIGNAL SIGINT
|
||||
@@ -1,12 +1,12 @@
|
||||
<div align="center">
|
||||
<img src="https://github.com/vladmandic/automatic/blob/dev/html/favicon.png" width=200 alt="SD.Next">
|
||||
<img src="https://github.com/vladmandic/automatic/blob/master/html/logo-transparent.png" width=200 alt="SD.Next">
|
||||
|
||||
**Stable Diffusion implementation with advanced features**
|
||||
**Image Diffusion implementation with advanced features**
|
||||
|
||||
[](https://github.com/sponsors/vladmandic)
|
||||
|
||||
|
||||
|
||||
[](https://discord.gg/VjvR2tabEX)
|
||||
[](https://github.com/sponsors/vladmandic)
|
||||
|
||||
[Wiki](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.gg/VjvR2tabEX) | [Changelog](CHANGELOG.md)
|
||||
|
||||
@@ -18,45 +18,36 @@
|
||||
- [SD.Next Features](#sdnext-features)
|
||||
- [Model support](#model-support)
|
||||
- [Platform support](#platform-support)
|
||||
- [Backend support](#backend-support)
|
||||
- [Examples](#examples)
|
||||
- [Install](#install)
|
||||
- [Notes](#notes)
|
||||
- [Getting started](#getting-started)
|
||||
|
||||
## SD.Next Features
|
||||
|
||||
All individual features are not listed here, instead check [ChangeLog](CHANGELOG.md) for full list of changes
|
||||
- Multiple backends!
|
||||
▹ **Diffusers | Original**
|
||||
- Multiple UIs!
|
||||
▹ **Standard | Modern**
|
||||
- Multiple diffusion models!
|
||||
▹ **Stable Diffusion 1.5/2.1/XL/3.0/3.5 | LCM | Lightning | Segmind | Kandinsky | Pixart-α | Pixart-Σ | Stable Cascade | FLUX.1 | AuraFlow | Würstchen | Alpha Lumina | Kwai Kolors | aMUSEd | DeepFloyd IF | UniDiffusion | SD-Distilled | BLiP Diffusion | KOALA | SDXS | Hyper-SD | HunyuanDiT | CogView | OmniGen | Meissonic | etc.**
|
||||
- Built-in Control for Text, Image, Batch and video processing!
|
||||
▹ **ControlNet | ControlNet XS | Control LLLite | T2I Adapters | IP Adapters**
|
||||
- Multiplatform!
|
||||
▹ **Windows | Linux | MacOS with CPU | nVidia | AMD | IntelArc/IPEX | DirectML | OpenVINO | ONNX+Olive | ZLUDA**
|
||||
- Platform specific autodetection and tuning performed on install
|
||||
▹ **Windows | Linux | MacOS | nVidia | AMD | IntelArc/IPEX | DirectML | OpenVINO | ONNX+Olive | ZLUDA**
|
||||
- Multiple backends!
|
||||
▹ **Diffusers | Original**
|
||||
- Platform specific autodetection and tuning performed on install
|
||||
- Optimized processing with latest `torch` developments with built-in support for `torch.compile`
|
||||
and multiple compile backends: *Triton, ZLUDA, StableFast, DeepCache, OpenVINO, NNCF, IPEX, OneDiff*
|
||||
- Improved prompt parser
|
||||
- Enhanced *Lora*/*LoCon*/*Lyco* code supporting latest trends in training
|
||||
- Built-in queue management
|
||||
- Enterprise level logging and hardened API
|
||||
- Built in installer with automatic updates and dependency management
|
||||
- Modernized UI with theme support and number of built-in themes *(dark and light)*
|
||||
- Mobile compatible
|
||||
- Mobile compatible
|
||||
|
||||
<br>
|
||||
|
||||
*Main interface using **StandardUI***:
|
||||
|
||||
|
||||
|
||||
*Main interface using **ModernUI***:
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
For screenshots and informations on other available themes, see [Themes Wiki](https://github.com/vladmandic/automatic/wiki/Themes)
|
||||
|
||||
@@ -65,12 +56,10 @@ For screenshots and informations on other available themes, see [Themes Wiki](ht
|
||||
## Model support
|
||||
|
||||
Additional models will be added as they become available and there is public interest in them
|
||||
See [models overview](https://github.com/vladmandic/automatic/wiki/Models) for details on each model, including their architecture, complexity and other info
|
||||
See [models overview](wiki/Models) for details on each model, including their architecture, complexity and other info
|
||||
|
||||
- [RunwayML Stable Diffusion](https://github.com/Stability-AI/stablediffusion/) 1.x and 2.x *(all variants)*
|
||||
- [StabilityAI Stable Diffusion XL](https://github.com/Stability-AI/generative-models)
|
||||
- [StabilityAI Stable Diffusion](https://stability.ai/news/stable-diffusion-3-medium)
|
||||
- [Stable Diffusion 3.x](https://huggingface.co/stabilityai/stable-diffusion-3.5-large) 3.0 Medium, 3.5 Medium, 3.5 Large, 3.5 Large Turbo
|
||||
- [StabilityAI Stable Diffusion XL](https://github.com/Stability-AI/generative-models), [StabilityAI Stable Diffusion 3.0](https://stability.ai/news/stable-diffusion-3-medium) Medium, [StabilityAI Stable Diffusion 3.5](https://huggingface.co/stabilityai/stable-diffusion-3.5-large) Medium, Large, Large Turbo
|
||||
- [StabilityAI Stable Video Diffusion](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid) Base, XT 1.0, XT 1.1
|
||||
- [StabilityAI Stable Cascade](https://github.com/Stability-AI/StableCascade) *Full* and *Lite*
|
||||
- [Black Forest Labs FLUX.1](https://blackforestlabs.ai/announcing-black-forest-labs/) Dev, Schnell
|
||||
@@ -84,13 +73,9 @@ See [models overview](https://github.com/vladmandic/automatic/wiki/Models) for d
|
||||
- [CogView 3+](https://huggingface.co/THUDM/CogView3-Plus-3B)
|
||||
- [LCM: Latent Consistency Models](https://github.com/openai/consistency_models)
|
||||
- [aMUSEd](https://huggingface.co/amused/amused-256) 256 and 512
|
||||
- [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega)
|
||||
- [Segmind SSD-1B](https://huggingface.co/segmind/SSD-1B)
|
||||
- [Segmind SegMoE](https://github.com/segmind/segmoe) *SD and SD-XL*
|
||||
- [Segmind SD Distilled](https://huggingface.co/blog/sd_distillation) *(all variants)*
|
||||
- [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega), [Segmind SSD-1B](https://huggingface.co/segmind/SSD-1B), [Segmind SegMoE](https://github.com/segmind/segmoe) *SD and SD-XL*, [Segmind SD Distilled](https://huggingface.co/blog/sd_distillation) *(all variants)*
|
||||
- [Kandinsky](https://github.com/ai-forever/Kandinsky-2) *2.1 and 2.2 and latest 3.0*
|
||||
- [PixArt-α XL 2](https://github.com/PixArt-alpha/PixArt-alpha) *Medium and Large*
|
||||
- [PixArt-Σ](https://github.com/PixArt-alpha/PixArt-sigma)
|
||||
- [PixArt-α XL 2](https://github.com/PixArt-alpha/PixArt-alpha) *Medium and Large*, [PixArt-Σ](https://github.com/PixArt-alpha/PixArt-sigma)
|
||||
- [Warp Wuerstchen](https://huggingface.co/blog/wuertschen)
|
||||
- [Tsinghua UniDiffusion](https://github.com/thu-ml/unidiffuser)
|
||||
- [DeepFloyd IF](https://github.com/deep-floyd/IF) *Medium and Large*
|
||||
@@ -101,15 +86,6 @@ See [models overview](https://github.com/vladmandic/automatic/wiki/Models) for d
|
||||
- [SDXS](https://github.com/IDKiro/sdxs)
|
||||
- [Hyper-SD](https://huggingface.co/ByteDance/Hyper-SD)
|
||||
|
||||
|
||||
Also supported are modifiers such as:
|
||||
- **LCM**, **Turbo** and **Lightning** (*adversarial diffusion distillation*) networks
|
||||
- All **LoRA** types such as LoCon, LyCORIS, HADA, IA3, Lokr, OFT
|
||||
- **IP-Adapters** for SD 1.5 and SD-XL
|
||||
- **InstantID**, **FaceSwap**, **FaceID**, **PhotoMerge**
|
||||
- **AnimateDiff** for SD 1.5
|
||||
- **MuLAN** multi-language support
|
||||
|
||||
## Platform support
|
||||
|
||||
- *nVidia* GPUs using **CUDA** libraries on both *Windows and Linux*
|
||||
@@ -121,6 +97,25 @@ Also supported are modifiers such as:
|
||||
- Any GPU or device compatible with **OpenVINO** libraries on both *Windows and Linux*
|
||||
- *Apple M1/M2* on *OSX* using built-in support in Torch with **MPS** optimizations
|
||||
- *ONNX/Olive*
|
||||
- *AMD* GPUs on Windows using **ZLUDA** libraries
|
||||
|
||||
## Getting started
|
||||
|
||||
- Get started with **SD.Next** by following the [installation instructions](wiki/Installation)
|
||||
- For more details, check out [advanced installation](wiki/Advanced-Install) guide
|
||||
- List and explanation of [command line arguments](wiki/CLI-Arguments)
|
||||
- Install walkthrough [video](https://www.youtube.com/watch?v=nWTnTyFTuAs)
|
||||
|
||||
> [!TIP]
|
||||
> And for platform specific information, check out
|
||||
> [WSL](wiki/WSL) | [Intel Arc](wiki/Intel-ARC) | [DirectML](wiki/DirectML) | [OpenVINO](wiki/OpenVINO) | [ONNX & Olive](wiki/ONNX-Runtime) | [ZLUDA](wiki/ZLUDA) | [AMD ROCm](wiki/AMD-ROCm) | [MacOS](wiki/MacOS-Python.md) | [nVidia](wiki/nVidia)
|
||||
|
||||
> [!WARNING]
|
||||
> If you run into issues, check out [troubleshooting](wiki/Troubleshooting) and [debugging](wiki/Debug) guides
|
||||
|
||||
> [!TIP]
|
||||
> All command line options can also be set via env variable
|
||||
> For example `--debug` is same as `set SD_DEBUG=true`
|
||||
|
||||
## Backend support
|
||||
|
||||
@@ -129,91 +124,11 @@ Also supported are modifiers such as:
|
||||
- **Diffusers**: Based on new [Huggingface Diffusers](https://huggingface.co/docs/diffusers/index) implementation
|
||||
Supports *all* models listed below
|
||||
This backend is set as default for new installations
|
||||
See [wiki article](https://github.com/vladmandic/automatic/wiki/Diffusers) for more information
|
||||
- **Original**: Based on [LDM](https://github.com/Stability-AI/stablediffusion) reference implementation and significantly expanded on by [A1111](https://github.com/AUTOMATIC1111/stable-diffusion-webui)
|
||||
This backend and is fully compatible with most existing functionality and extensions written for *A1111 SDWebUI*
|
||||
Supports **SD 1.x** and **SD 2.x** models
|
||||
All other model types such as *SD-XL, LCM, Stable Cascade, PixArt, Playground, Segmind, Kandinsky, etc.* require backend **Diffusers**
|
||||
|
||||
## Examples
|
||||
|
||||
*IP Adapters*:
|
||||
|
||||
|
||||
*Color grading*:
|
||||
|
||||
|
||||
*InstantID*:
|
||||
|
||||
|
||||
> [!IMPORTANT]
|
||||
> - Loading any model other than standard SD 1.x / SD 2.x requires use of backend **Diffusers**
|
||||
> - Loading any other models using **Original** backend is not supported
|
||||
> - Loading manually download model `.safetensors` files is supported for specified models only (typically SD 1.x / SD 2.x / SD-XL models only)
|
||||
> - For all other model types, use backend **Diffusers** and use built in Model downloader or
|
||||
select model from Networks -> Models -> Reference list in which case it will be auto-downloaded and loaded
|
||||
|
||||
## Install
|
||||
|
||||
- [Step-by-step install guide](https://github.com/vladmandic/automatic/wiki/Installation)
|
||||
- [Advanced install notes](https://github.com/vladmandic/automatic/wiki/Advanced-Install)
|
||||
- [Video: install and use](https://www.youtube.com/watch?v=nWTnTyFTuAs)
|
||||
- [Common installation errors](https://github.com/vladmandic/automatic/discussions/1627)
|
||||
- [FAQ](https://github.com/vladmandic/automatic/discussions/1011)
|
||||
|
||||
> [!TIP]
|
||||
> - If you can't run SD.Next locally, try cloud deployment using [RunDiffusion](https://rundiffusion.com?utm_source=github&utm_medium=referral&utm_campaign=SDNext)!
|
||||
> - Server can run with or without virtual environment,
|
||||
Recommended to use `VENV` to avoid library version conflicts with other applications
|
||||
> - **nVidia/CUDA** / **AMD/ROCm** / **Intel/OneAPI** are auto-detected if present and available,
|
||||
For any other use case such as **DirectML**, **ONNX/Olive**, **OpenVINO** specify required parameter explicitly
|
||||
or wrong packages may be installed as installer will assume CPU-only environment
|
||||
> - Full startup sequence is logged in `sdnext.log`,
|
||||
so if you encounter any issues, please check it first
|
||||
|
||||
### Run
|
||||
|
||||
Once SD.Next is installed, simply run `webui.ps1` or `webui.bat` (*Windows*) or `webui.sh` (*Linux or MacOS*)
|
||||
|
||||
For list of available command line options, run `webui --help` for the full & up-to-date list
|
||||
|
||||
> [!TIP]
|
||||
> All command line options can also be set via env variable
|
||||
> For example `--debug` is same as `set SD_DEBUG=true`
|
||||
|
||||
## Notes
|
||||
|
||||
> [!TIP]
|
||||
> If you don't want to use built-in `venv` support and prefer to run SD.Next in your own environment such as *Docker* container, *Conda* environment or any other virtual environment, you can skip `venv` create/activate and launch SD.Next directly using `python launch.py` (command line flags noted above still apply).
|
||||
|
||||
### Quantization
|
||||
|
||||
**SD.Next** comes with broad quantization support, including support for BitsAndBytes, Optimum.Quanto, TorchAO, NNCF and GGUF
|
||||
See [Quantization Wiki](https://github.com/vladmandic/automatic/wiki/Quantization)
|
||||
|
||||
### Control
|
||||
|
||||
**SD.Next** comes with built-in control for all types of text2image, image2image, video2video and batch processing
|
||||
|
||||
*Control interface*:
|
||||
|
||||
|
||||
*Control processors*:
|
||||
|
||||
|
||||
*Masking*:
|
||||
|
||||
|
||||
### Extensions
|
||||
|
||||
SD.Next comes with several extensions pre-installed:
|
||||
|
||||
- [System Info](https://github.com/vladmandic/sd-extension-system-info)
|
||||
- [chaiNNer](https://github.com/vladmandic/sd-extension-chainner)
|
||||
- [RemBg](https://github.com/vladmandic/sd-extension-rembg)
|
||||
- [Agent Scheduler](https://github.com/ArtVentureX/sd-webui-agent-scheduler)
|
||||
- [Modern UI](https://github.com/BinaryQuantumSoul/sdnext-modernui)
|
||||
|
||||
### Collab
|
||||
|
||||
- We'd love to have additional maintainers (with comes with full repo rights). If you're interested, ping us!
|
||||
@@ -242,12 +157,6 @@ This should be fully cross-platform, but we'd really love to have additional con
|
||||
If you're unsure how to use a feature, best place to start is [Wiki](https://github.com/vladmandic/automatic/wiki) and if its not there,
|
||||
check [ChangeLog](CHANGELOG.md) for when feature was first introduced as it will always have a short note on how to use it
|
||||
|
||||
- [Wiki](https://github.com/vladmandic/automatic/wiki)
|
||||
- [ReadMe](README.md)
|
||||
- [ToDo](TODO.md)
|
||||
- [ChangeLog](CHANGELOG.md)
|
||||
- [CLI Tools](cli/README.md)
|
||||
|
||||
### Sponsors
|
||||
|
||||
<div align="center">
|
||||
|
||||
@@ -4,11 +4,18 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
|
||||
|
||||
## Future Candidates
|
||||
|
||||
- async lowvram: <https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/14855>
|
||||
- fp8: <https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/14031>
|
||||
- ipadapter-negative: <https://github.com/huggingface/diffusers/discussions/7167>
|
||||
- include reference styles
|
||||
- SD35 IPAdapter: <https://github.com/huggingface/diffusers/issues/9966>
|
||||
- SD35 LoRA: <https://github.com/huggingface/diffusers/issues/9950>
|
||||
- Flux IPAdapter: <https://github.com/huggingface/diffusers/issues/9825>
|
||||
- Flux Fill/ControlNet/Redux: <https://github.com/huggingface/diffusers/pull/9985>
|
||||
- SANA: <https://github.com/huggingface/diffusers/pull/9982>
|
||||
|
||||
### Missing
|
||||
## Other
|
||||
|
||||
- control api scripts compatibility
|
||||
- IPAdapter negative: <https://github.com/huggingface/diffusers/discussions/7167>
|
||||
- Control API enhance scripts compatibility
|
||||
|
||||
## Workaround in place
|
||||
|
||||
- GGUF <https://github.com/huggingface/diffusers/issues/9487>
|
||||
- FlowMatch <https://github.com/huggingface/diffusers/issues/9607> <https://github.com/huggingface/diffusers/issues/9924>
|
||||
|
||||
-108
@@ -1,108 +0,0 @@
|
||||
# Stable-Diffusion Productivity Scripts
|
||||
|
||||
Note: All scripts have built-in `--help` parameter that can be used to get more information
|
||||
|
||||
<br>
|
||||
|
||||
## Main Scripts
|
||||
|
||||
### Generate
|
||||
|
||||
Text-to-image with all of the possible parameters
|
||||
Supports upsampling, face restoration and grid creation
|
||||
> python generate.py
|
||||
|
||||
By default uses parameters from `generate.json`
|
||||
|
||||
Parameters that are not specified will be randomized:
|
||||
|
||||
- Prompt will be dynamically created from template of random samples: `random.json`
|
||||
- Sampler/Scheduler will be randomly picked from available ones
|
||||
- CFG Scale set to 5-10
|
||||
|
||||
### Train
|
||||
|
||||
Combined pipeline for **embeddings**, **lora**, **lycoris**, **dreambooth** and **hypernetwork**
|
||||
Optionally runs several image processing steps before training:
|
||||
|
||||
- keep original image
|
||||
- detect and extract face
|
||||
- detect and extract body
|
||||
- detect blur
|
||||
- detect dynamic range
|
||||
- attempt to upscale low resolution images
|
||||
- attempt to restore quality of low quality images
|
||||
- automatically generate captions using interrogate
|
||||
- resize image
|
||||
- square image
|
||||
- run image segmentation to remove background
|
||||
|
||||
> python train.py
|
||||
|
||||
<br>
|
||||
|
||||
## Auxiliary Scripts
|
||||
|
||||
### Benchmark
|
||||
|
||||
> python run-benchmark.py
|
||||
|
||||
### Create Previews
|
||||
|
||||
Create previews for **embeddings**, **lora**, **lycoris**, **dreambooth** and **hypernetwork**
|
||||
|
||||
> python create-previews.py
|
||||
|
||||
## Image Grid
|
||||
|
||||
> python image-grid.py
|
||||
|
||||
### Image Watermark
|
||||
|
||||
Create invisible image watermark and remove existing EXIF tags
|
||||
|
||||
> python image-watermark.py
|
||||
|
||||
### Image Interrogate
|
||||
|
||||
Runs CLiP and Booru image interrogation
|
||||
|
||||
> python image-interrogate.py
|
||||
|
||||
### Palette Extract
|
||||
|
||||
Extract color palette from image(s)
|
||||
|
||||
> python image-palette.py
|
||||
|
||||
### Prompt Ideas
|
||||
|
||||
Generate complex prompt ideas
|
||||
|
||||
> python prompt-ideas.py
|
||||
|
||||
### Prompt Promptist
|
||||
|
||||
Attempts to beautify the provided prompt
|
||||
|
||||
> python prompt-promptist.py
|
||||
|
||||
### Video Extract
|
||||
|
||||
Extract frames from video files
|
||||
|
||||
> python video-extract.py
|
||||
|
||||
<br>
|
||||
|
||||
## Utility Scripts
|
||||
|
||||
### SDAPI
|
||||
|
||||
Utility module that handles async communication to Automatic API endpoints
|
||||
Note: Requires SD API
|
||||
|
||||
Can be used to manually execute specific commands:
|
||||
> python sdapi.py progress
|
||||
> python sdapi.py interrupt
|
||||
> python sdapi.py shutdown
|
||||
@@ -44,16 +44,15 @@ def encode(f):
|
||||
|
||||
|
||||
def detect(args): # pylint: disable=redefined-outer-name
|
||||
data = post('/sdapi/v1/faces', { 'image': encode(args.image) })
|
||||
for face in zip(data['images'], data['scores']):
|
||||
log.info(f'Face: score={face[1]}')
|
||||
image = Image.open(io.BytesIO(base64.b64decode(face[0])))
|
||||
image.save(f'/tmp/face_{face[1]}.jpg')
|
||||
data = post('/sdapi/v1/detect', { 'image': encode(args.image), 'model': args.model })
|
||||
for i in range(len(data['images'])):
|
||||
log.info(f"Item {i}: score={data['scores'][i]} cls={data['classes'][i]} box={data['boxes'][i]} label={data['labels'][i]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description = 'api-faces')
|
||||
parser.add_argument('--image', required=True, help='input image')
|
||||
parser.add_argument('--model', required=False, default='', help='model')
|
||||
args = parser.parse_args()
|
||||
log.info(f'api-faces: {args}')
|
||||
log.info(f'api-detect: {args}')
|
||||
detect(args)
|
||||
+2
-19
@@ -63,7 +63,7 @@ def generate(args): # pylint: disable=redefined-outer-name
|
||||
options['height'] = args.height
|
||||
options['face'] = {
|
||||
'mode': 'FaceID',
|
||||
'ip_model': 'FaceID Base',
|
||||
'ip_model': 'FaceID XL',
|
||||
'source_images': [encode(args.face)],
|
||||
}
|
||||
data = post('/sdapi/v1/txt2img', options)
|
||||
@@ -86,7 +86,7 @@ if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description = 'api-faceid')
|
||||
parser.add_argument('--width', required=False, default=512, help='image width')
|
||||
parser.add_argument('--height', required=False, default=512, help='image height')
|
||||
parser.add_argument('--face', required=False, help='face image')
|
||||
parser.add_argument('--face', required=True, help='face image')
|
||||
parser.add_argument('--prompt', required=False, default='', help='prompt text')
|
||||
parser.add_argument('--negative', required=False, default='', help='negative prompt text')
|
||||
parser.add_argument('--steps', required=False, default=20, help='number of steps')
|
||||
@@ -97,20 +97,3 @@ if __name__ == "__main__":
|
||||
args = parser.parse_args()
|
||||
log.info(f'api-faceid: {args}')
|
||||
generate(args)
|
||||
|
||||
"""
|
||||
request.face.mode,
|
||||
request.face.source_images,
|
||||
request.face.ip_model,
|
||||
request.face.ip_override_sampler,
|
||||
request.face.ip_cache_model,
|
||||
request.face.ip_strength,
|
||||
request.face.ip_structure,
|
||||
request.face.id_strength,
|
||||
request.face.id_conditioning,
|
||||
request.face.id_cache,
|
||||
request.face.pm_trigger,
|
||||
request.face.pm_strength,
|
||||
request.face.pm_start,
|
||||
request.face.fs_cache
|
||||
"""
|
||||
|
||||
+1
-1
@@ -45,7 +45,7 @@ if __name__ == "__main__":
|
||||
log.info(f'api-json: {args}')
|
||||
if os.path.isfile(args.json[0]):
|
||||
with open(args.json[0], 'r', encoding='ascii') as f:
|
||||
dct = json.load(f) # TODO fails with b64 encoded images inside json due to string encoding
|
||||
dct = json.load(f)
|
||||
else:
|
||||
dct = json.loads(args.json[0])
|
||||
res = post(endpoint=args.endpoint[0], payload=dct)
|
||||
|
||||
+5
-1
@@ -1,5 +1,9 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
check progress of last job and shutdown system if timeout reached
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import datetime
|
||||
@@ -16,7 +20,7 @@ opts = Dot({
|
||||
"timeout": 3600,
|
||||
"frequency": 60,
|
||||
"action": "sudo shutdown now",
|
||||
"url": "https://127.0.0.1:7860",
|
||||
"url": "http://127.0.0.1:7860",
|
||||
"user": "",
|
||||
"password": "",
|
||||
})
|
||||
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// simple nodejs script to test sdnext api
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const process = require('process');
|
||||
const argparse = require('argparse');
|
||||
|
||||
const sd_url = process.env.SDAPI_URL || 'http://127.0.0.1:7860';
|
||||
const sd_username = process.env.SDAPI_USR;
|
||||
const sd_password = process.env.SDAPI_PWD;
|
||||
|
||||
function b64(file) {
|
||||
const data = fs.readFileSync(file);
|
||||
const b64 = Buffer.from(data).toString('base64');
|
||||
const ext = path.extname(file).replace('.', '');
|
||||
str = `data:image/${ext};base64,${b64}`;
|
||||
// console.log('b64:', ext, b64.length);
|
||||
return str;
|
||||
}
|
||||
|
||||
function options() {
|
||||
const opt = {
|
||||
// first pass
|
||||
prompt: args.prompt || 'beautiful lady, in the steampunk style',
|
||||
negative_prompt: args.negative || 'foggy, blurry',
|
||||
seed: -1,
|
||||
steps: 20,
|
||||
batch_size: 1,
|
||||
n_iter: 1,
|
||||
cfg_scale: 6,
|
||||
width: args.width || 1024,
|
||||
height: args.height || 1024,
|
||||
// api return options
|
||||
save_images: false,
|
||||
send_images: true,
|
||||
};
|
||||
if (args.pulid) {
|
||||
const b64image = b64(args.pulid);
|
||||
opt.script_name = 'pulid';
|
||||
opt.script_args = [b64image, 0.9];
|
||||
}
|
||||
// console.log('options:', opt);
|
||||
return opt;
|
||||
}
|
||||
|
||||
function init() {
|
||||
const parser = new argparse.ArgumentParser({ description: 'SD.Next API' });
|
||||
parser.add_argument('--prompt', { type: 'str', help: 'prompt' });
|
||||
parser.add_argument('--negative', { type: 'str', help: 'negative' });
|
||||
parser.add_argument('--width', { type: 'int', help: 'width' });
|
||||
parser.add_argument('--height', { type: 'int', help: 'height' });
|
||||
parser.add_argument('--pulid', { type: 'str', help: 'pulid init image' });
|
||||
parser.add_argument('--output', { type: 'str', help: 'output path' });
|
||||
const args = parser.parse_args();
|
||||
return args
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const method = 'POST';
|
||||
const headers = new Headers();
|
||||
const opt = options();
|
||||
const body = JSON.stringify(opt);
|
||||
headers.set('Content-Type', 'application/json');
|
||||
if (sd_username && sd_password) headers.set({ Authorization: `Basic ${btoa('sd_username:sd_password')}` });
|
||||
const res = await fetch(`${sd_url}/sdapi/v1/txt2img`, { method, headers, body });
|
||||
|
||||
if (res.status !== 200) {
|
||||
console.log('Error', res.status);
|
||||
} else {
|
||||
const json = await res.json();
|
||||
console.log('result:', json.info);
|
||||
for (const i in json.images) { // eslint-disable-line guard-for-in
|
||||
const file = args.output || `/tmp/test-${i}.jpg`;
|
||||
const data = atob(json.images[i])
|
||||
fs.writeFileSync(file, data, 'binary');
|
||||
console.log('image saved:', file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const args = init();
|
||||
main();
|
||||
+1
-18
@@ -20,23 +20,6 @@ const sd_options = {
|
||||
cfg_scale: 6,
|
||||
width: 512,
|
||||
height: 512,
|
||||
/*
|
||||
// enable second pass
|
||||
enable_hr: true,
|
||||
// second pass: upscale
|
||||
hr_upscaler: 'SCUNet GAN',
|
||||
hr_scale: 2.0,
|
||||
// second pass: hires
|
||||
hr_force: true,
|
||||
hr_second_pass_steps: 20,
|
||||
hr_sampler_name: 'UniPC',
|
||||
denoising_strength: 0.5,
|
||||
// second pass: refiner
|
||||
refiner_steps: 5,
|
||||
refiner_start: 0.8,
|
||||
refiner_prompt: '',
|
||||
refiner_negative: '',
|
||||
*/
|
||||
// api return options
|
||||
save_images: false,
|
||||
send_images: true,
|
||||
@@ -55,7 +38,7 @@ async function main() {
|
||||
const json = await res.json();
|
||||
console.log('result:', json.info);
|
||||
for (const i in json.images) { // eslint-disable-line guard-for-in
|
||||
const f = `/tmp/test-{${i}.jpg`;
|
||||
const f = `/tmp/test-${i}.jpg`;
|
||||
fs.writeFileSync(f, atob(json.images[i]), 'binary');
|
||||
console.log('image saved:', f);
|
||||
}
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ def generate(args): # pylint: disable=redefined-outer-name
|
||||
options['sampler_name'] = args.sampler
|
||||
options['width'] = int(args.width)
|
||||
options['height'] = int(args.height)
|
||||
if args.faces:
|
||||
if args.detailer:
|
||||
options['detailer'] = args.detailer
|
||||
options['denoising_strength'] = 0.5
|
||||
options['hr_sampler_name'] = args.sampler
|
||||
|
||||
+3
-2
@@ -73,7 +73,8 @@ def upscale(args): # pylint: disable=redefined-outer-name
|
||||
if 'image' in data:
|
||||
b64 = data['image'].split(',',1)[0]
|
||||
image = Image.open(io.BytesIO(base64.b64decode(b64)))
|
||||
image.save(args.output)
|
||||
if args.output:
|
||||
image.save(args.output)
|
||||
log.info(f'received: image={image} file={args.output} time={t1-t0:.2f}')
|
||||
else:
|
||||
log.warning(f'no images received: {data}')
|
||||
@@ -82,7 +83,7 @@ def upscale(args): # pylint: disable=redefined-outer-name
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description = 'api-upscale')
|
||||
parser.add_argument('--input', required=True, help='input image')
|
||||
parser.add_argument('--output', required=True, help='output image')
|
||||
parser.add_argument('--output', required=False, help='output image')
|
||||
parser.add_argument('--upscaler', required=False, default='Nearest', help='upscaler name')
|
||||
parser.add_argument('--scale', required=False, default=2, help='upscaler scale')
|
||||
args = parser.parse_args()
|
||||
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
source venv/bin/activate
|
||||
echo image-exif
|
||||
python cli/api-info.py --input html/logo-bg-0.jpg
|
||||
echo txt2img
|
||||
python cli/api-txt2img.py --detailer --prompt "girl on a mountain" --seed 42 --sampler DEIS --width 1280 --height 800 --steps 10
|
||||
echo img2img
|
||||
python cli/api-img2img.py --init html/logo-bg-0.jpg --steps 10
|
||||
echo inpaint
|
||||
python cli/api-img2img.py --init html/logo-bg-0.jpg --mask html/logo-dark.png --steps 10
|
||||
echo upscale
|
||||
python cli/api-upscale.py --input html/logo-bg-0.jpg --upscaler "ESRGAN 4x Valar" --scale 4
|
||||
echo vqa
|
||||
python cli/api-vqa.py --input html/logo-bg-0.jpg
|
||||
echo detailer
|
||||
python cli/api-detect.py --image html/invoked.jpg
|
||||
echo faceid
|
||||
python cli/api-faceid.py --face html/simple-dark.jpg
|
||||
echo control-txt2img
|
||||
python cli/api-control.py --prompt "cute robot"
|
||||
echo control-img2img
|
||||
python cli/api-control.py --prompt "cute robot" --input html/logo-bg-0.jpg
|
||||
echo control-ipsadapter
|
||||
python cli/api-control.py --prompt "cute robot" --ipadapter "Base SDXL:html/logo-bg-0.jpg:0.8"
|
||||
echo control-preprocess
|
||||
python cli/api-preprocess.py --input html/logo-bg-0.jpg --model "Zoe Depth"
|
||||
echo control-controlnet
|
||||
python cli/api-control.py --prompt "cute robot" --input html/logo-bg-0.jpg --type controlnet --control "Zoe Depth:Xinsir Union XL:0.5"
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python
|
||||
import os
|
||||
import sys
|
||||
from rich import print as pprint
|
||||
|
||||
|
||||
def has(obj, attr, *args):
|
||||
import functools
|
||||
if not isinstance(obj, dict):
|
||||
return False
|
||||
def _getattr(obj, attr):
|
||||
return obj.get(attr, args) if isinstance(obj, dict) else False
|
||||
return functools.reduce(_getattr, [obj] + attr.split('.'))
|
||||
|
||||
|
||||
def remove_entries_after_depth(d, depth, current_depth=0):
|
||||
try:
|
||||
if current_depth >= depth:
|
||||
return None
|
||||
if isinstance(d, dict):
|
||||
return {k: remove_entries_after_depth(v, depth, current_depth + 1) for k, v in d.items() if remove_entries_after_depth(v, depth, current_depth + 1) is not None}
|
||||
except Exception:
|
||||
pass
|
||||
return d
|
||||
|
||||
|
||||
def list_to_dict(flat_list):
|
||||
result_dict = {}
|
||||
try:
|
||||
for item in flat_list:
|
||||
keys = item.split('.')
|
||||
d = result_dict
|
||||
for key in keys[:-1]:
|
||||
d = d.setdefault(key, {})
|
||||
d[keys[-1]] = None
|
||||
except Exception:
|
||||
pass
|
||||
return result_dict
|
||||
|
||||
|
||||
def guess_dct(dct: dict):
|
||||
# if has(dct, 'model.diffusion_model.input_blocks') and has(dct, 'model.diffusion_model.label_emb'):
|
||||
# return 'sdxl'
|
||||
if has(dct, 'model.diffusion_model.input_blocks') and len(list(has(dct, 'model.diffusion_model.input_blocks'))) == 12:
|
||||
return 'sd15'
|
||||
if has(dct, 'model.diffusion_model.input_blocks') and len(list(has(dct, 'model.diffusion_model.input_blocks'))) == 9:
|
||||
return 'sdxl'
|
||||
if has(dct, 'model.diffusion_model.joint_blocks') and len(list(has(dct, 'model.diffusion_model.joint_blocks'))) == 24:
|
||||
return 'sd35-medium'
|
||||
if has(dct, 'model.diffusion_model.joint_blocks') and len(list(has(dct, 'model.diffusion_model.joint_blocks'))) == 38:
|
||||
return 'sd35-large'
|
||||
if has(dct, 'model.diffusion_model.double_blocks') and len(list(has(dct, 'model.diffusion_model.double_blocks'))) == 19:
|
||||
return 'flux-dev'
|
||||
return None
|
||||
|
||||
|
||||
def read_keys(fn):
|
||||
if not fn.lower().endswith(".safetensors"):
|
||||
return
|
||||
from safetensors.torch import safe_open
|
||||
keys = []
|
||||
try:
|
||||
with safe_open(fn, framework="pt", device="cpu") as f:
|
||||
keys = f.keys()
|
||||
except Exception as e:
|
||||
pprint(e)
|
||||
dct = list_to_dict(keys)
|
||||
pprint(f'file: {fn}')
|
||||
pprint(remove_entries_after_depth(dct, 3))
|
||||
pprint(remove_entries_after_depth(dct, 6))
|
||||
guess = guess_dct(dct)
|
||||
pprint(f'guess: {guess}')
|
||||
return keys
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) == 0:
|
||||
print('metadata:', 'no files specified')
|
||||
for fn in sys.argv:
|
||||
if os.path.isfile(fn):
|
||||
read_keys(fn)
|
||||
elif os.path.isdir(fn):
|
||||
for root, _dirs, files in os.walk(fn):
|
||||
for file in files:
|
||||
read_keys(os.path.join(root, file))
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.argv.pop(0)
|
||||
main()
|
||||
@@ -26,7 +26,9 @@ def get_stepwise(param, step, steps):
|
||||
return v
|
||||
else:
|
||||
return m
|
||||
return calculate_weight(sorted_positions(param), step, steps)
|
||||
|
||||
stepwise = calculate_weight(sorted_positions(param), step, steps)
|
||||
return stepwise
|
||||
|
||||
|
||||
class ExtraNetworkLora(extra_networks.ExtraNetwork):
|
||||
@@ -145,7 +147,6 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
|
||||
if self.active and networks.debug:
|
||||
shared.log.debug(f"Network end: type=LoRA load={networks.timer['load']:.2f} apply={networks.timer['apply']:.2f} restore={networks.timer['restore']:.2f}")
|
||||
if self.errors:
|
||||
p.comment("Networks with errors: " + ", ".join(f"{k} ({v})" for k, v in self.errors.items()))
|
||||
for k, v in self.errors.items():
|
||||
shared.log.error(f'LoRA: name="{k}" errors={v}')
|
||||
self.errors.clear()
|
||||
|
||||
@@ -50,44 +50,45 @@ convert_diffusers_name_to_compvis = lora_convert.convert_diffusers_name_to_compv
|
||||
def assign_network_names_to_compvis_modules(sd_model):
|
||||
if sd_model is None:
|
||||
return
|
||||
sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility
|
||||
network_layer_mapping = {}
|
||||
if shared.native:
|
||||
if hasattr(shared.sd_model, 'text_encoder') and shared.sd_model.text_encoder is not None:
|
||||
for name, module in shared.sd_model.text_encoder.named_modules():
|
||||
prefix = "lora_te1_" if hasattr(shared.sd_model, 'text_encoder_2') else "lora_te_"
|
||||
if hasattr(sd_model, 'text_encoder') and sd_model.text_encoder is not None:
|
||||
for name, module in sd_model.text_encoder.named_modules():
|
||||
prefix = "lora_te1_" if hasattr(sd_model, 'text_encoder_2') else "lora_te_"
|
||||
network_name = prefix + name.replace(".", "_")
|
||||
network_layer_mapping[network_name] = module
|
||||
module.network_layer_name = network_name
|
||||
if hasattr(shared.sd_model, 'text_encoder_2'):
|
||||
for name, module in shared.sd_model.text_encoder_2.named_modules():
|
||||
if hasattr(sd_model, 'text_encoder_2'):
|
||||
for name, module in sd_model.text_encoder_2.named_modules():
|
||||
network_name = "lora_te2_" + name.replace(".", "_")
|
||||
network_layer_mapping[network_name] = module
|
||||
module.network_layer_name = network_name
|
||||
if hasattr(shared.sd_model, 'unet'):
|
||||
for name, module in shared.sd_model.unet.named_modules():
|
||||
if hasattr(sd_model, 'unet'):
|
||||
for name, module in sd_model.unet.named_modules():
|
||||
network_name = "lora_unet_" + name.replace(".", "_")
|
||||
network_layer_mapping[network_name] = module
|
||||
module.network_layer_name = network_name
|
||||
if hasattr(shared.sd_model, 'transformer'):
|
||||
for name, module in shared.sd_model.transformer.named_modules():
|
||||
if hasattr(sd_model, 'transformer'):
|
||||
for name, module in sd_model.transformer.named_modules():
|
||||
network_name = "lora_transformer_" + name.replace(".", "_")
|
||||
network_layer_mapping[network_name] = module
|
||||
if "norm" in network_name and "linear" not in network_name:
|
||||
continue
|
||||
module.network_layer_name = network_name
|
||||
else:
|
||||
if not hasattr(shared.sd_model, 'cond_stage_model'):
|
||||
if not hasattr(sd_model, 'cond_stage_model'):
|
||||
sd_model.network_layer_mapping = {}
|
||||
return
|
||||
for name, module in shared.sd_model.cond_stage_model.wrapped.named_modules():
|
||||
for name, module in sd_model.cond_stage_model.wrapped.named_modules():
|
||||
network_name = name.replace(".", "_")
|
||||
network_layer_mapping[network_name] = module
|
||||
module.network_layer_name = network_name
|
||||
for name, module in shared.sd_model.model.named_modules():
|
||||
for name, module in sd_model.model.named_modules():
|
||||
network_name = name.replace(".", "_")
|
||||
network_layer_mapping[network_name] = module
|
||||
module.network_layer_name = network_name
|
||||
sd_model.network_layer_mapping = network_layer_mapping
|
||||
shared.sd_model.network_layer_mapping = network_layer_mapping
|
||||
|
||||
|
||||
def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_default_multiplier) -> network.Network:
|
||||
@@ -226,6 +227,7 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No
|
||||
loaded_networks.clear()
|
||||
diffuser_loaded.clear()
|
||||
diffuser_scales.clear()
|
||||
|
||||
for i, (network_on_disk, name) in enumerate(zip(networks_on_disk, names)):
|
||||
net = None
|
||||
if network_on_disk is not None:
|
||||
@@ -261,14 +263,22 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No
|
||||
while len(lora_cache) > shared.opts.lora_in_memory_limit:
|
||||
name = next(iter(lora_cache))
|
||||
lora_cache.pop(name, None)
|
||||
|
||||
if len(diffuser_loaded) > 0:
|
||||
shared.log.debug(f'Load network: type=LoRA loaded={diffuser_loaded} scales={diffuser_scales}')
|
||||
shared.sd_model.set_adapters(adapter_names=diffuser_loaded, adapter_weights=diffuser_scales)
|
||||
if shared.opts.lora_fuse_diffusers:
|
||||
shared.sd_model.fuse_lora(adapter_names=diffuser_loaded, lora_scale=1.0, fuse_unet=True, fuse_text_encoder=True) # fuse uses fixed scale since later apply does the scaling
|
||||
shared.sd_model.unload_lora_weights()
|
||||
shared.log.debug(f'Load network: type=LoRA loaded={diffuser_loaded} available={shared.sd_model.get_list_adapters()} active={shared.sd_model.get_active_adapters()} scales={diffuser_scales}')
|
||||
try:
|
||||
shared.sd_model.set_adapters(adapter_names=diffuser_loaded, adapter_weights=diffuser_scales)
|
||||
if shared.opts.lora_fuse_diffusers:
|
||||
shared.sd_model.fuse_lora(adapter_names=diffuser_loaded, lora_scale=1.0, fuse_unet=True, fuse_text_encoder=True) # fuse uses fixed scale since later apply does the scaling
|
||||
shared.sd_model.unload_lora_weights()
|
||||
except Exception as e:
|
||||
shared.log.error(f'Load network: type=LoRA {e}')
|
||||
if debug:
|
||||
errors.display(e, 'LoRA')
|
||||
|
||||
if len(loaded_networks) > 0 and debug:
|
||||
shared.log.debug(f'Load network: type=LoRA loaded={len(loaded_networks)} cache={list(lora_cache)}')
|
||||
|
||||
devices.torch_gc()
|
||||
|
||||
if recompile_model:
|
||||
|
||||
@@ -5,7 +5,7 @@ import networks
|
||||
from modules import shared, ui_extra_networks
|
||||
|
||||
|
||||
debug = os.environ.get('SD_LOAD_DEBUG', None) is not None
|
||||
debug = os.environ.get('SD_LORA_DEBUG', None) is not None
|
||||
|
||||
|
||||
class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
|
||||
@@ -16,34 +16,18 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
|
||||
def refresh(self):
|
||||
networks.list_available_networks()
|
||||
|
||||
def create_item(self, name):
|
||||
l = networks.available_networks.get(name)
|
||||
if l is None:
|
||||
shared.log.warning(f'Networks: type=lora registered={len(list(networks.available_networks))} file="{name}" not registered')
|
||||
return None
|
||||
def get_tags(self, l, info):
|
||||
tags = {}
|
||||
try:
|
||||
# path, _ext = os.path.splitext(l.filename)
|
||||
name = os.path.splitext(os.path.relpath(l.filename, shared.cmd_opts.lora_dir))[0]
|
||||
item = {
|
||||
"type": 'Lora',
|
||||
"name": name,
|
||||
"filename": l.filename,
|
||||
"hash": l.shorthash,
|
||||
"prompt": json.dumps(f" <lora:{l.get_alias()}:{shared.opts.extra_networks_default_multiplier}>"),
|
||||
"metadata": json.dumps(l.metadata, indent=4) if l.metadata else None,
|
||||
"mtime": os.path.getmtime(l.filename),
|
||||
"size": os.path.getsize(l.filename),
|
||||
"version": l.sd_version,
|
||||
}
|
||||
info = self.find_info(l.filename)
|
||||
|
||||
tags = {}
|
||||
if l.metadata is not None:
|
||||
modelspec_tags = l.metadata.get('modelspec.tags', {})
|
||||
possible_tags = l.metadata.get('ss_tag_frequency', {}) # tags from model metedata
|
||||
possible_tags.update(modelspec_tags)
|
||||
if isinstance(possible_tags, str):
|
||||
possible_tags = {}
|
||||
if isinstance(modelspec_tags, str):
|
||||
modelspec_tags = {}
|
||||
if len(list(modelspec_tags)) > 0:
|
||||
possible_tags.update(modelspec_tags)
|
||||
for k, v in possible_tags.items():
|
||||
words = k.split('_', 1) if '_' in k else [v, k]
|
||||
words = [str(w).replace('.json', '') for w in words]
|
||||
@@ -80,20 +64,41 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
|
||||
tag = tag.strip().lower()
|
||||
if tag not in tags:
|
||||
tags[tag] = 0
|
||||
except Exception:
|
||||
pass
|
||||
bad_chars = [';', ':', '<', ">", "*", '?', '\'', '\"', '(', ')', '[', ']', '{', '}', '\\', '/']
|
||||
clean_tags = {}
|
||||
for k, v in tags.items():
|
||||
tag = ''.join(i for i in k if i not in bad_chars).strip()
|
||||
clean_tags[tag] = v
|
||||
|
||||
bad_chars = [';', ':', '<', ">", "*", '?', '\'', '\"', '(', ')', '[', ']', '{', '}', '\\', '/']
|
||||
clean_tags = {}
|
||||
for k, v in tags.items():
|
||||
tag = ''.join(i for i in k if i not in bad_chars).strip()
|
||||
clean_tags[tag] = v
|
||||
|
||||
clean_tags.pop('img', None)
|
||||
clean_tags.pop('dataset', None)
|
||||
clean_tags.pop('img', None)
|
||||
clean_tags.pop('dataset', None)
|
||||
return clean_tags
|
||||
|
||||
def create_item(self, name):
|
||||
l = networks.available_networks.get(name)
|
||||
if l is None:
|
||||
shared.log.warning(f'Networks: type=lora registered={len(list(networks.available_networks))} file="{name}" not registered')
|
||||
return None
|
||||
try:
|
||||
# path, _ext = os.path.splitext(l.filename)
|
||||
name = os.path.splitext(os.path.relpath(l.filename, shared.cmd_opts.lora_dir))[0]
|
||||
item = {
|
||||
"type": 'Lora',
|
||||
"name": name,
|
||||
"filename": l.filename,
|
||||
"hash": l.shorthash,
|
||||
"prompt": json.dumps(f" <lora:{l.get_alias()}:{shared.opts.extra_networks_default_multiplier}>"),
|
||||
"metadata": json.dumps(l.metadata, indent=4) if l.metadata else None,
|
||||
"mtime": os.path.getmtime(l.filename),
|
||||
"size": os.path.getsize(l.filename),
|
||||
"version": l.sd_version,
|
||||
}
|
||||
info = self.find_info(l.filename)
|
||||
item["info"] = info
|
||||
item["description"] = self.find_description(l.filename, info) # use existing info instead of double-read
|
||||
item["tags"] = clean_tags
|
||||
|
||||
item["tags"] = self.get_tags(l, info)
|
||||
return item
|
||||
except Exception as e:
|
||||
shared.log.error(f'Networks: type=lora file="{name}" {e}')
|
||||
|
||||
Submodule extensions-builtin/sdnext-modernui updated: 71bdbbd9c0...4647bd7f86
@@ -0,0 +1,868 @@
|
||||
.opblock {
|
||||
border-width: 0 !important;
|
||||
}
|
||||
.opblock-summary-operation-id {
|
||||
display: none !important;
|
||||
}
|
||||
.swagger-ui .models .json-schema-2020-12:not(.json-schema-2020-12--embedded)>.json-schema-2020-12-head .json-schema-2020-12__title:first-of-type {
|
||||
font-size: 14px;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.swagger-ui .json-schema-2020-12-keyword__name--primary {
|
||||
color: aqua;
|
||||
}
|
||||
|
||||
.swagger-ui .json-schema-2020-12-property .json-schema-2020-12__title {
|
||||
color: aqua;
|
||||
}
|
||||
|
||||
@media only screen and (prefers-color-scheme: dark) {
|
||||
|
||||
a { color: #8c8cfa; }
|
||||
|
||||
::-webkit-scrollbar-track-piece { background-color: rgba(255, 255, 255, .2) !important; }
|
||||
|
||||
::-webkit-scrollbar-track { background-color: rgba(255, 255, 255, .3) !important; }
|
||||
|
||||
::-webkit-scrollbar-thumb { background-color: rgba(255, 255, 255, .5) !important; }
|
||||
|
||||
embed[type="application/pdf"] { filter: invert(90%); }
|
||||
|
||||
html {
|
||||
background: #1f1f1f !important;
|
||||
box-sizing: border-box;
|
||||
filter: contrast(100%) brightness(100%) saturate(100%);
|
||||
overflow-y: scroll;
|
||||
}
|
||||
|
||||
body {
|
||||
background: #1f1f1f;
|
||||
background-color: #1f1f1f;
|
||||
background-image: none !important;
|
||||
}
|
||||
|
||||
button, input, select, textarea {
|
||||
background-color: #1f1f1f;
|
||||
color: #bfbfbf;
|
||||
}
|
||||
|
||||
font, html { color: #bfbfbf; }
|
||||
|
||||
.swagger-ui, .swagger-ui section h3 { color: #b5bac9; }
|
||||
|
||||
.swagger-ui a { background-color: transparent; }
|
||||
|
||||
.swagger-ui mark {
|
||||
background-color: #664b00;
|
||||
color: #bfbfbf;
|
||||
}
|
||||
|
||||
.swagger-ui legend { color: inherit; }
|
||||
|
||||
.swagger-ui .debug * { outline: #e6da99 solid 1px; }
|
||||
|
||||
.swagger-ui .debug-white * { outline: #fff solid 1px; }
|
||||
|
||||
.swagger-ui .debug-black * { outline: #bfbfbf solid 1px; }
|
||||
|
||||
.swagger-ui .debug-grid { background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMTExIDc5LjE1ODMyNSwgMjAxNS8wOS8xMC0wMToxMDoyMCAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6MTRDOTY4N0U2N0VFMTFFNjg2MzZDQjkwNkQ4MjgwMEIiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6MTRDOTY4N0Q2N0VFMTFFNjg2MzZDQjkwNkQ4MjgwMEIiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3NjcyQkQ3NjY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3NjcyQkQ3NzY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PsBS+GMAAAAjSURBVHjaYvz//z8DLsD4gcGXiYEAGBIKGBne//fFpwAgwAB98AaF2pjlUQAAAABJRU5ErkJggg==) 0 0; }
|
||||
|
||||
.swagger-ui .debug-grid-16 { background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMTExIDc5LjE1ODMyNSwgMjAxNS8wOS8xMC0wMToxMDoyMCAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6ODYyRjhERDU2N0YyMTFFNjg2MzZDQjkwNkQ4MjgwMEIiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6ODYyRjhERDQ2N0YyMTFFNjg2MzZDQjkwNkQ4MjgwMEIiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3NjcyQkQ3QTY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3NjcyQkQ3QjY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PvCS01IAAABMSURBVHjaYmR4/5+BFPBfAMFm/MBgx8RAGWCn1AAmSg34Q6kBDKMGMDCwICeMIemF/5QawEipAWwUhwEjMDvbAWlWkvVBwu8vQIABAEwBCph8U6c0AAAAAElFTkSuQmCC) 0 0; }
|
||||
|
||||
.swagger-ui .debug-grid-8-solid { background: url(data:image/jpeg;base64,/9j/4QAYRXhpZgAASUkqAAgAAAAAAAAAAAAAAP/sABFEdWNreQABAAQAAAAAAAD/4QMxaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wLwA8P3hwYWNrZXQgYmVnaW49Iu+7vyIgaWQ9Ilc1TTBNcENlaGlIenJlU3pOVGN6a2M5ZCI/PiA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJBZG9iZSBYTVAgQ29yZSA1LjYtYzExMSA3OS4xNTgzMjUsIDIwMTUvMDkvMTAtMDE6MTA6MjAgICAgICAgICI+IDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+IDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiIHhtbG5zOnhtcD0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wLyIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bXA6Q3JlYXRvclRvb2w9IkFkb2JlIFBob3Rvc2hvcCBDQyAyMDE1IChNYWNpbnRvc2gpIiB4bXBNTTpJbnN0YW5jZUlEPSJ4bXAuaWlkOkIxMjI0OTczNjdCMzExRTZCMkJDRTI0MDgxMDAyMTcxIiB4bXBNTTpEb2N1bWVudElEPSJ4bXAuZGlkOkIxMjI0OTc0NjdCMzExRTZCMkJDRTI0MDgxMDAyMTcxIj4gPHhtcE1NOkRlcml2ZWRGcm9tIHN0UmVmOmluc3RhbmNlSUQ9InhtcC5paWQ6QjEyMjQ5NzE2N0IzMTFFNkIyQkNFMjQwODEwMDIxNzEiIHN0UmVmOmRvY3VtZW50SUQ9InhtcC5kaWQ6QjEyMjQ5NzI2N0IzMTFFNkIyQkNFMjQwODEwMDIxNzEiLz4gPC9yZGY6RGVzY3JpcHRpb24+IDwvcmRmOlJERj4gPC94OnhtcG1ldGE+IDw/eHBhY2tldCBlbmQ9InIiPz7/7gAOQWRvYmUAZMAAAAAB/9sAhAAbGhopHSlBJiZBQi8vL0JHPz4+P0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHAR0pKTQmND8oKD9HPzU/R0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0f/wAARCAAIAAgDASIAAhEBAxEB/8QAWQABAQAAAAAAAAAAAAAAAAAAAAYBAQEAAAAAAAAAAAAAAAAAAAIEEAEBAAMBAAAAAAAAAAAAAAABADECA0ERAAEDBQAAAAAAAAAAAAAAAAARITFBUWESIv/aAAwDAQACEQMRAD8AoOnTV1QTD7JJshP3vSM3P//Z) 0 0 #1c1c21; }
|
||||
|
||||
.swagger-ui .debug-grid-16-solid { background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMTExIDc5LjE1ODMyNSwgMjAxNS8wOS8xMC0wMToxMDoyMCAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6NzY3MkJEN0U2N0M1MTFFNkIyQkNFMjQwODEwMDIxNzEiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6NzY3MkJEN0Y2N0M1MTFFNkIyQkNFMjQwODEwMDIxNzEiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3NjcyQkQ3QzY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3NjcyQkQ3RDY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/Pve6J3kAAAAzSURBVHjaYvz//z8D0UDsMwMjSRoYP5Gq4SPNbRjVMEQ1fCRDg+in/6+J1AJUxsgAEGAA31BAJMS0GYEAAAAASUVORK5CYII=) 0 0 #1c1c21; }
|
||||
|
||||
.swagger-ui .b--black { border-color: #000; }
|
||||
|
||||
.swagger-ui .b--near-black { border-color: #121212; }
|
||||
|
||||
.swagger-ui .b--dark-gray { border-color: #333; }
|
||||
|
||||
.swagger-ui .b--mid-gray { border-color: #545454; }
|
||||
|
||||
.swagger-ui .b--gray { border-color: #787878; }
|
||||
|
||||
.swagger-ui .b--silver { border-color: #999; }
|
||||
|
||||
.swagger-ui .b--light-silver { border-color: #6e6e6e; }
|
||||
|
||||
.swagger-ui .b--moon-gray { border-color: #4d4d4d; }
|
||||
|
||||
.swagger-ui .b--light-gray { border-color: #2b2b2b; }
|
||||
|
||||
.swagger-ui .b--near-white { border-color: #242424; }
|
||||
|
||||
.swagger-ui .b--white { border-color: #1c1c21; }
|
||||
|
||||
.swagger-ui .b--white-90 { border-color: rgba(28, 28, 33, .9); }
|
||||
|
||||
.swagger-ui .b--white-80 { border-color: rgba(28, 28, 33, .8); }
|
||||
|
||||
.swagger-ui .b--white-70 { border-color: rgba(28, 28, 33, .7); }
|
||||
|
||||
.swagger-ui .b--white-60 { border-color: rgba(28, 28, 33, .6); }
|
||||
|
||||
.swagger-ui .b--white-50 { border-color: rgba(28, 28, 33, .5); }
|
||||
|
||||
.swagger-ui .b--white-40 { border-color: rgba(28, 28, 33, .4); }
|
||||
|
||||
.swagger-ui .b--white-30 { border-color: rgba(28, 28, 33, .3); }
|
||||
|
||||
.swagger-ui .b--white-20 { border-color: rgba(28, 28, 33, .2); }
|
||||
|
||||
.swagger-ui .b--white-10 { border-color: rgba(28, 28, 33, .1); }
|
||||
|
||||
.swagger-ui .b--white-05 { border-color: rgba(28, 28, 33, .05); }
|
||||
|
||||
.swagger-ui .b--white-025 { border-color: rgba(28, 28, 33, .024); }
|
||||
|
||||
.swagger-ui .b--white-0125 { border-color: rgba(28, 28, 33, .01); }
|
||||
|
||||
.swagger-ui .b--black-90 { border-color: rgba(0, 0, 0, .9); }
|
||||
|
||||
.swagger-ui .b--black-80 { border-color: rgba(0, 0, 0, .8); }
|
||||
|
||||
.swagger-ui .b--black-70 { border-color: rgba(0, 0, 0, .7); }
|
||||
|
||||
.swagger-ui .b--black-60 { border-color: rgba(0, 0, 0, .6); }
|
||||
|
||||
.swagger-ui .b--black-50 { border-color: rgba(0, 0, 0, .5); }
|
||||
|
||||
.swagger-ui .b--black-40 { border-color: rgba(0, 0, 0, .4); }
|
||||
|
||||
.swagger-ui .b--black-30 { border-color: rgba(0, 0, 0, .3); }
|
||||
|
||||
.swagger-ui .b--black-20 { border-color: rgba(0, 0, 0, .2); }
|
||||
|
||||
.swagger-ui .b--black-10 { border-color: rgba(0, 0, 0, .1); }
|
||||
|
||||
.swagger-ui .b--black-05 { border-color: rgba(0, 0, 0, .05); }
|
||||
|
||||
.swagger-ui .b--black-025 { border-color: rgba(0, 0, 0, .024); }
|
||||
|
||||
.swagger-ui .b--black-0125 { border-color: rgba(0, 0, 0, .01); }
|
||||
|
||||
.swagger-ui .b--dark-red { border-color: #bc2f36; }
|
||||
|
||||
.swagger-ui .b--red { border-color: #c83932; }
|
||||
|
||||
.swagger-ui .b--light-red { border-color: #ab3c2b; }
|
||||
|
||||
.swagger-ui .b--orange { border-color: #cc6e33; }
|
||||
|
||||
.swagger-ui .b--purple { border-color: #5e2ca5; }
|
||||
|
||||
.swagger-ui .b--light-purple { border-color: #672caf; }
|
||||
|
||||
.swagger-ui .b--dark-pink { border-color: #ab2b81; }
|
||||
|
||||
.swagger-ui .b--hot-pink { border-color: #c03086; }
|
||||
|
||||
.swagger-ui .b--pink { border-color: #8f2464; }
|
||||
|
||||
.swagger-ui .b--light-pink { border-color: #721d4d; }
|
||||
|
||||
.swagger-ui .b--dark-green { border-color: #1c6e50; }
|
||||
|
||||
.swagger-ui .b--green { border-color: #279b70; }
|
||||
|
||||
.swagger-ui .b--light-green { border-color: #228762; }
|
||||
|
||||
.swagger-ui .b--navy { border-color: #0d1d35; }
|
||||
|
||||
.swagger-ui .b--dark-blue { border-color: #20497e; }
|
||||
|
||||
.swagger-ui .b--blue { border-color: #4380d0; }
|
||||
|
||||
.swagger-ui .b--light-blue { border-color: #20517e; }
|
||||
|
||||
.swagger-ui .b--lightest-blue { border-color: #143a52; }
|
||||
|
||||
.swagger-ui .b--washed-blue { border-color: #0c312d; }
|
||||
|
||||
.swagger-ui .b--washed-green { border-color: #0f3d2c; }
|
||||
|
||||
.swagger-ui .b--washed-red { border-color: #411010; }
|
||||
|
||||
.swagger-ui .b--transparent { border-color: transparent; }
|
||||
|
||||
.swagger-ui .b--gold, .swagger-ui .b--light-yellow, .swagger-ui .b--washed-yellow, .swagger-ui .b--yellow { border-color: #664b00; }
|
||||
|
||||
.swagger-ui .shadow-1 { box-shadow: rgba(0, 0, 0, .2) 0 0 4px 2px; }
|
||||
|
||||
.swagger-ui .shadow-2 { box-shadow: rgba(0, 0, 0, .2) 0 0 8px 2px; }
|
||||
|
||||
.swagger-ui .shadow-3 { box-shadow: rgba(0, 0, 0, .2) 2px 2px 4px 2px; }
|
||||
|
||||
.swagger-ui .shadow-4 { box-shadow: rgba(0, 0, 0, .2) 2px 2px 8px 0; }
|
||||
|
||||
.swagger-ui .shadow-5 { box-shadow: rgba(0, 0, 0, .2) 4px 4px 8px 0; }
|
||||
|
||||
@media screen and (min-width: 30em) {
|
||||
.swagger-ui .shadow-1-ns { box-shadow: rgba(0, 0, 0, .2) 0 0 4px 2px; }
|
||||
|
||||
.swagger-ui .shadow-2-ns { box-shadow: rgba(0, 0, 0, .2) 0 0 8px 2px; }
|
||||
|
||||
.swagger-ui .shadow-3-ns { box-shadow: rgba(0, 0, 0, .2) 2px 2px 4px 2px; }
|
||||
|
||||
.swagger-ui .shadow-4-ns { box-shadow: rgba(0, 0, 0, .2) 2px 2px 8px 0; }
|
||||
|
||||
.swagger-ui .shadow-5-ns { box-shadow: rgba(0, 0, 0, .2) 4px 4px 8px 0; }
|
||||
}
|
||||
|
||||
@media screen and (max-width: 60em) and (min-width: 30em) {
|
||||
.swagger-ui .shadow-1-m { box-shadow: rgba(0, 0, 0, .2) 0 0 4px 2px; }
|
||||
|
||||
.swagger-ui .shadow-2-m { box-shadow: rgba(0, 0, 0, .2) 0 0 8px 2px; }
|
||||
|
||||
.swagger-ui .shadow-3-m { box-shadow: rgba(0, 0, 0, .2) 2px 2px 4px 2px; }
|
||||
|
||||
.swagger-ui .shadow-4-m { box-shadow: rgba(0, 0, 0, .2) 2px 2px 8px 0; }
|
||||
|
||||
.swagger-ui .shadow-5-m { box-shadow: rgba(0, 0, 0, .2) 4px 4px 8px 0; }
|
||||
}
|
||||
|
||||
@media screen and (min-width: 60em) {
|
||||
.swagger-ui .shadow-1-l { box-shadow: rgba(0, 0, 0, .2) 0 0 4px 2px; }
|
||||
|
||||
.swagger-ui .shadow-2-l { box-shadow: rgba(0, 0, 0, .2) 0 0 8px 2px; }
|
||||
|
||||
.swagger-ui .shadow-3-l { box-shadow: rgba(0, 0, 0, .2) 2px 2px 4px 2px; }
|
||||
|
||||
.swagger-ui .shadow-4-l { box-shadow: rgba(0, 0, 0, .2) 2px 2px 8px 0; }
|
||||
|
||||
.swagger-ui .shadow-5-l { box-shadow: rgba(0, 0, 0, .2) 4px 4px 8px 0; }
|
||||
}
|
||||
|
||||
.swagger-ui .black-05 { color: rgba(191, 191, 191, .05); }
|
||||
|
||||
.swagger-ui .bg-black-05 { background-color: rgba(0, 0, 0, .05); }
|
||||
|
||||
.swagger-ui .black-90, .swagger-ui .hover-black-90:focus, .swagger-ui .hover-black-90:hover { color: rgba(191, 191, 191, .9); }
|
||||
|
||||
.swagger-ui .black-80, .swagger-ui .hover-black-80:focus, .swagger-ui .hover-black-80:hover { color: rgba(191, 191, 191, .8); }
|
||||
|
||||
.swagger-ui .black-70, .swagger-ui .hover-black-70:focus, .swagger-ui .hover-black-70:hover { color: rgba(191, 191, 191, .7); }
|
||||
|
||||
.swagger-ui .black-60, .swagger-ui .hover-black-60:focus, .swagger-ui .hover-black-60:hover { color: rgba(191, 191, 191, .6); }
|
||||
|
||||
.swagger-ui .black-50, .swagger-ui .hover-black-50:focus, .swagger-ui .hover-black-50:hover { color: rgba(191, 191, 191, .5); }
|
||||
|
||||
.swagger-ui .black-40, .swagger-ui .hover-black-40:focus, .swagger-ui .hover-black-40:hover { color: rgba(191, 191, 191, .4); }
|
||||
|
||||
.swagger-ui .black-30, .swagger-ui .hover-black-30:focus, .swagger-ui .hover-black-30:hover { color: rgba(191, 191, 191, .3); }
|
||||
|
||||
.swagger-ui .black-20, .swagger-ui .hover-black-20:focus, .swagger-ui .hover-black-20:hover { color: rgba(191, 191, 191, .2); }
|
||||
|
||||
.swagger-ui .black-10, .swagger-ui .hover-black-10:focus, .swagger-ui .hover-black-10:hover { color: rgba(191, 191, 191, .1); }
|
||||
|
||||
.swagger-ui .hover-white-90:focus, .swagger-ui .hover-white-90:hover, .swagger-ui .white-90 { color: rgba(255, 255, 255, .9); }
|
||||
|
||||
.swagger-ui .hover-white-80:focus, .swagger-ui .hover-white-80:hover, .swagger-ui .white-80 { color: rgba(255, 255, 255, .8); }
|
||||
|
||||
.swagger-ui .hover-white-70:focus, .swagger-ui .hover-white-70:hover, .swagger-ui .white-70 { color: rgba(255, 255, 255, .7); }
|
||||
|
||||
.swagger-ui .hover-white-60:focus, .swagger-ui .hover-white-60:hover, .swagger-ui .white-60 { color: rgba(255, 255, 255, .6); }
|
||||
|
||||
.swagger-ui .hover-white-50:focus, .swagger-ui .hover-white-50:hover, .swagger-ui .white-50 { color: rgba(255, 255, 255, .5); }
|
||||
|
||||
.swagger-ui .hover-white-40:focus, .swagger-ui .hover-white-40:hover, .swagger-ui .white-40 { color: rgba(255, 255, 255, .4); }
|
||||
|
||||
.swagger-ui .hover-white-30:focus, .swagger-ui .hover-white-30:hover, .swagger-ui .white-30 { color: rgba(255, 255, 255, .3); }
|
||||
|
||||
.swagger-ui .hover-white-20:focus, .swagger-ui .hover-white-20:hover, .swagger-ui .white-20 { color: rgba(255, 255, 255, .2); }
|
||||
|
||||
.swagger-ui .hover-white-10:focus, .swagger-ui .hover-white-10:hover, .swagger-ui .white-10 { color: rgba(255, 255, 255, .1); }
|
||||
|
||||
.swagger-ui .hover-moon-gray:focus, .swagger-ui .hover-moon-gray:hover, .swagger-ui .moon-gray { color: #ccc; }
|
||||
|
||||
.swagger-ui .hover-light-gray:focus, .swagger-ui .hover-light-gray:hover, .swagger-ui .light-gray { color: #ededed; }
|
||||
|
||||
.swagger-ui .hover-near-white:focus, .swagger-ui .hover-near-white:hover, .swagger-ui .near-white { color: #f5f5f5; }
|
||||
|
||||
.swagger-ui .dark-red, .swagger-ui .hover-dark-red:focus, .swagger-ui .hover-dark-red:hover { color: #e6999d; }
|
||||
|
||||
.swagger-ui .hover-red:focus, .swagger-ui .hover-red:hover, .swagger-ui .red { color: #e69d99; }
|
||||
|
||||
.swagger-ui .hover-light-red:focus, .swagger-ui .hover-light-red:hover, .swagger-ui .light-red { color: #e6a399; }
|
||||
|
||||
.swagger-ui .hover-orange:focus, .swagger-ui .hover-orange:hover, .swagger-ui .orange { color: #e6b699; }
|
||||
|
||||
.swagger-ui .gold, .swagger-ui .hover-gold:focus, .swagger-ui .hover-gold:hover { color: #e6d099; }
|
||||
|
||||
.swagger-ui .hover-yellow:focus, .swagger-ui .hover-yellow:hover, .swagger-ui .yellow { color: #e6da99; }
|
||||
|
||||
.swagger-ui .hover-light-yellow:focus, .swagger-ui .hover-light-yellow:hover, .swagger-ui .light-yellow { color: #ede6b6; }
|
||||
|
||||
.swagger-ui .hover-purple:focus, .swagger-ui .hover-purple:hover, .swagger-ui .purple { color: #b99ae4; }
|
||||
|
||||
.swagger-ui .hover-light-purple:focus, .swagger-ui .hover-light-purple:hover, .swagger-ui .light-purple { color: #bb99e6; }
|
||||
|
||||
.swagger-ui .dark-pink, .swagger-ui .hover-dark-pink:focus, .swagger-ui .hover-dark-pink:hover { color: #e699cc; }
|
||||
|
||||
.swagger-ui .hot-pink, .swagger-ui .hover-hot-pink:focus, .swagger-ui .hover-hot-pink:hover, .swagger-ui .hover-pink:focus, .swagger-ui .hover-pink:hover, .swagger-ui .pink { color: #e699c7; }
|
||||
|
||||
.swagger-ui .hover-light-pink:focus, .swagger-ui .hover-light-pink:hover, .swagger-ui .light-pink { color: #edb6d5; }
|
||||
|
||||
.swagger-ui .dark-green, .swagger-ui .green, .swagger-ui .hover-dark-green:focus, .swagger-ui .hover-dark-green:hover, .swagger-ui .hover-green:focus, .swagger-ui .hover-green:hover { color: #99e6c9; }
|
||||
|
||||
.swagger-ui .hover-light-green:focus, .swagger-ui .hover-light-green:hover, .swagger-ui .light-green { color: #a1e8ce; }
|
||||
|
||||
.swagger-ui .hover-navy:focus, .swagger-ui .hover-navy:hover, .swagger-ui .navy { color: #99b8e6; }
|
||||
|
||||
.swagger-ui .blue, .swagger-ui .dark-blue, .swagger-ui .hover-blue:focus, .swagger-ui .hover-blue:hover, .swagger-ui .hover-dark-blue:focus, .swagger-ui .hover-dark-blue:hover { color: #99bae6; }
|
||||
|
||||
.swagger-ui .hover-light-blue:focus, .swagger-ui .hover-light-blue:hover, .swagger-ui .light-blue { color: #a9cbea; }
|
||||
|
||||
.swagger-ui .hover-lightest-blue:focus, .swagger-ui .hover-lightest-blue:hover, .swagger-ui .lightest-blue { color: #d6e9f5; }
|
||||
|
||||
.swagger-ui .hover-washed-blue:focus, .swagger-ui .hover-washed-blue:hover, .swagger-ui .washed-blue { color: #f7fdfc; }
|
||||
|
||||
.swagger-ui .hover-washed-green:focus, .swagger-ui .hover-washed-green:hover, .swagger-ui .washed-green { color: #ebfaf4; }
|
||||
|
||||
.swagger-ui .hover-washed-yellow:focus, .swagger-ui .hover-washed-yellow:hover, .swagger-ui .washed-yellow { color: #fbf9ef; }
|
||||
|
||||
.swagger-ui .hover-washed-red:focus, .swagger-ui .hover-washed-red:hover, .swagger-ui .washed-red { color: #f9e7e7; }
|
||||
|
||||
.swagger-ui .color-inherit, .swagger-ui .hover-inherit:focus, .swagger-ui .hover-inherit:hover { color: inherit; }
|
||||
|
||||
.swagger-ui .bg-black-90, .swagger-ui .hover-bg-black-90:focus, .swagger-ui .hover-bg-black-90:hover { background-color: rgba(0, 0, 0, .9); }
|
||||
|
||||
.swagger-ui .bg-black-80, .swagger-ui .hover-bg-black-80:focus, .swagger-ui .hover-bg-black-80:hover { background-color: rgba(0, 0, 0, .8); }
|
||||
|
||||
.swagger-ui .bg-black-70, .swagger-ui .hover-bg-black-70:focus, .swagger-ui .hover-bg-black-70:hover { background-color: rgba(0, 0, 0, .7); }
|
||||
|
||||
.swagger-ui .bg-black-60, .swagger-ui .hover-bg-black-60:focus, .swagger-ui .hover-bg-black-60:hover { background-color: rgba(0, 0, 0, .6); }
|
||||
|
||||
.swagger-ui .bg-black-50, .swagger-ui .hover-bg-black-50:focus, .swagger-ui .hover-bg-black-50:hover { background-color: rgba(0, 0, 0, .5); }
|
||||
|
||||
.swagger-ui .bg-black-40, .swagger-ui .hover-bg-black-40:focus, .swagger-ui .hover-bg-black-40:hover { background-color: rgba(0, 0, 0, .4); }
|
||||
|
||||
.swagger-ui .bg-black-30, .swagger-ui .hover-bg-black-30:focus, .swagger-ui .hover-bg-black-30:hover { background-color: rgba(0, 0, 0, .3); }
|
||||
|
||||
.swagger-ui .bg-black-20, .swagger-ui .hover-bg-black-20:focus, .swagger-ui .hover-bg-black-20:hover { background-color: rgba(0, 0, 0, .2); }
|
||||
|
||||
.swagger-ui .bg-white-90, .swagger-ui .hover-bg-white-90:focus, .swagger-ui .hover-bg-white-90:hover { background-color: rgba(28, 28, 33, .9); }
|
||||
|
||||
.swagger-ui .bg-white-80, .swagger-ui .hover-bg-white-80:focus, .swagger-ui .hover-bg-white-80:hover { background-color: rgba(28, 28, 33, .8); }
|
||||
|
||||
.swagger-ui .bg-white-70, .swagger-ui .hover-bg-white-70:focus, .swagger-ui .hover-bg-white-70:hover { background-color: rgba(28, 28, 33, .7); }
|
||||
|
||||
.swagger-ui .bg-white-60, .swagger-ui .hover-bg-white-60:focus, .swagger-ui .hover-bg-white-60:hover { background-color: rgba(28, 28, 33, .6); }
|
||||
|
||||
.swagger-ui .bg-white-50, .swagger-ui .hover-bg-white-50:focus, .swagger-ui .hover-bg-white-50:hover { background-color: rgba(28, 28, 33, .5); }
|
||||
|
||||
.swagger-ui .bg-white-40, .swagger-ui .hover-bg-white-40:focus, .swagger-ui .hover-bg-white-40:hover { background-color: rgba(28, 28, 33, .4); }
|
||||
|
||||
.swagger-ui .bg-white-30, .swagger-ui .hover-bg-white-30:focus, .swagger-ui .hover-bg-white-30:hover { background-color: rgba(28, 28, 33, .3); }
|
||||
|
||||
.swagger-ui .bg-white-20, .swagger-ui .hover-bg-white-20:focus, .swagger-ui .hover-bg-white-20:hover { background-color: rgba(28, 28, 33, .2); }
|
||||
|
||||
.swagger-ui .bg-black, .swagger-ui .hover-bg-black:focus, .swagger-ui .hover-bg-black:hover { background-color: #000; }
|
||||
|
||||
.swagger-ui .bg-near-black, .swagger-ui .hover-bg-near-black:focus, .swagger-ui .hover-bg-near-black:hover { background-color: #121212; }
|
||||
|
||||
.swagger-ui .bg-dark-gray, .swagger-ui .hover-bg-dark-gray:focus, .swagger-ui .hover-bg-dark-gray:hover { background-color: #333; }
|
||||
|
||||
.swagger-ui .bg-mid-gray, .swagger-ui .hover-bg-mid-gray:focus, .swagger-ui .hover-bg-mid-gray:hover { background-color: #545454; }
|
||||
|
||||
.swagger-ui .bg-gray, .swagger-ui .hover-bg-gray:focus, .swagger-ui .hover-bg-gray:hover { background-color: #787878; }
|
||||
|
||||
.swagger-ui .bg-silver, .swagger-ui .hover-bg-silver:focus, .swagger-ui .hover-bg-silver:hover { background-color: #999; }
|
||||
|
||||
.swagger-ui .bg-white, .swagger-ui .hover-bg-white:focus, .swagger-ui .hover-bg-white:hover { background-color: #1c1c21; }
|
||||
|
||||
.swagger-ui .bg-transparent, .swagger-ui .hover-bg-transparent:focus, .swagger-ui .hover-bg-transparent:hover { background-color: transparent; }
|
||||
|
||||
.swagger-ui .bg-dark-red, .swagger-ui .hover-bg-dark-red:focus, .swagger-ui .hover-bg-dark-red:hover { background-color: #bc2f36; }
|
||||
|
||||
.swagger-ui .bg-red, .swagger-ui .hover-bg-red:focus, .swagger-ui .hover-bg-red:hover { background-color: #c83932; }
|
||||
|
||||
.swagger-ui .bg-light-red, .swagger-ui .hover-bg-light-red:focus, .swagger-ui .hover-bg-light-red:hover { background-color: #ab3c2b; }
|
||||
|
||||
.swagger-ui .bg-orange, .swagger-ui .hover-bg-orange:focus, .swagger-ui .hover-bg-orange:hover { background-color: #cc6e33; }
|
||||
|
||||
.swagger-ui .bg-gold, .swagger-ui .bg-light-yellow, .swagger-ui .bg-washed-yellow, .swagger-ui .bg-yellow, .swagger-ui .hover-bg-gold:focus, .swagger-ui .hover-bg-gold:hover, .swagger-ui .hover-bg-light-yellow:focus, .swagger-ui .hover-bg-light-yellow:hover, .swagger-ui .hover-bg-washed-yellow:focus, .swagger-ui .hover-bg-washed-yellow:hover, .swagger-ui .hover-bg-yellow:focus, .swagger-ui .hover-bg-yellow:hover { background-color: #664b00; }
|
||||
|
||||
.swagger-ui .bg-purple, .swagger-ui .hover-bg-purple:focus, .swagger-ui .hover-bg-purple:hover { background-color: #5e2ca5; }
|
||||
|
||||
.swagger-ui .bg-light-purple, .swagger-ui .hover-bg-light-purple:focus, .swagger-ui .hover-bg-light-purple:hover { background-color: #672caf; }
|
||||
|
||||
.swagger-ui .bg-dark-pink, .swagger-ui .hover-bg-dark-pink:focus, .swagger-ui .hover-bg-dark-pink:hover { background-color: #ab2b81; }
|
||||
|
||||
.swagger-ui .bg-hot-pink, .swagger-ui .hover-bg-hot-pink:focus, .swagger-ui .hover-bg-hot-pink:hover { background-color: #c03086; }
|
||||
|
||||
.swagger-ui .bg-pink, .swagger-ui .hover-bg-pink:focus, .swagger-ui .hover-bg-pink:hover { background-color: #8f2464; }
|
||||
|
||||
.swagger-ui .bg-light-pink, .swagger-ui .hover-bg-light-pink:focus, .swagger-ui .hover-bg-light-pink:hover { background-color: #721d4d; }
|
||||
|
||||
.swagger-ui .bg-dark-green, .swagger-ui .hover-bg-dark-green:focus, .swagger-ui .hover-bg-dark-green:hover { background-color: #1c6e50; }
|
||||
|
||||
.swagger-ui .bg-green, .swagger-ui .hover-bg-green:focus, .swagger-ui .hover-bg-green:hover { background-color: #279b70; }
|
||||
|
||||
.swagger-ui .bg-light-green, .swagger-ui .hover-bg-light-green:focus, .swagger-ui .hover-bg-light-green:hover { background-color: #228762; }
|
||||
|
||||
.swagger-ui .bg-navy, .swagger-ui .hover-bg-navy:focus, .swagger-ui .hover-bg-navy:hover { background-color: #0d1d35; }
|
||||
|
||||
.swagger-ui .bg-dark-blue, .swagger-ui .hover-bg-dark-blue:focus, .swagger-ui .hover-bg-dark-blue:hover { background-color: #20497e; }
|
||||
|
||||
.swagger-ui .bg-blue, .swagger-ui .hover-bg-blue:focus, .swagger-ui .hover-bg-blue:hover { background-color: #4380d0; }
|
||||
|
||||
.swagger-ui .bg-light-blue, .swagger-ui .hover-bg-light-blue:focus, .swagger-ui .hover-bg-light-blue:hover { background-color: #20517e; }
|
||||
|
||||
.swagger-ui .bg-lightest-blue, .swagger-ui .hover-bg-lightest-blue:focus, .swagger-ui .hover-bg-lightest-blue:hover { background-color: #143a52; }
|
||||
|
||||
.swagger-ui .bg-washed-blue, .swagger-ui .hover-bg-washed-blue:focus, .swagger-ui .hover-bg-washed-blue:hover { background-color: #0c312d; }
|
||||
|
||||
.swagger-ui .bg-washed-green, .swagger-ui .hover-bg-washed-green:focus, .swagger-ui .hover-bg-washed-green:hover { background-color: #0f3d2c; }
|
||||
|
||||
.swagger-ui .bg-washed-red, .swagger-ui .hover-bg-washed-red:focus, .swagger-ui .hover-bg-washed-red:hover { background-color: #411010; }
|
||||
|
||||
.swagger-ui .bg-inherit, .swagger-ui .hover-bg-inherit:focus, .swagger-ui .hover-bg-inherit:hover { background-color: inherit; }
|
||||
|
||||
.swagger-ui .shadow-hover { transition: all .5s cubic-bezier(.165, .84, .44, 1) 0s; }
|
||||
|
||||
.swagger-ui .shadow-hover::after {
|
||||
border-radius: inherit;
|
||||
box-shadow: rgba(0, 0, 0, .2) 0 0 16px 2px;
|
||||
content: "";
|
||||
height: 100%;
|
||||
left: 0;
|
||||
opacity: 0;
|
||||
position: absolute;
|
||||
top: 0;
|
||||
transition: opacity .5s cubic-bezier(.165, .84, .44, 1) 0s;
|
||||
width: 100%;
|
||||
z-index: -1;
|
||||
}
|
||||
|
||||
.swagger-ui .bg-animate, .swagger-ui .bg-animate:focus, .swagger-ui .bg-animate:hover { transition: background-color .15s ease-in-out 0s; }
|
||||
|
||||
.swagger-ui .nested-links a {
|
||||
color: #99bae6;
|
||||
transition: color .15s ease-in 0s;
|
||||
}
|
||||
|
||||
.swagger-ui .nested-links a:focus, .swagger-ui .nested-links a:hover {
|
||||
color: #a9cbea;
|
||||
transition: color .15s ease-in 0s;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock-tag {
|
||||
border-bottom: 1px solid rgba(58, 64, 80, .3);
|
||||
color: #b5bac9;
|
||||
transition: all .2s ease 0s;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock-tag svg, .swagger-ui section.models h4 svg { transition: all .4s ease 0s; }
|
||||
|
||||
.swagger-ui .opblock {
|
||||
border: 1px solid #000;
|
||||
border-radius: 4px;
|
||||
box-shadow: rgba(0, 0, 0, .19) 0 0 3px;
|
||||
margin: 0 0 15px;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock .tab-header .tab-item.active h4 span::after { background: gray; }
|
||||
|
||||
.swagger-ui .opblock.is-open .opblock-summary { border-bottom: 1px solid #000; }
|
||||
|
||||
.swagger-ui .opblock .opblock-section-header {
|
||||
background: rgba(28, 28, 33, .8);
|
||||
box-shadow: rgba(0, 0, 0, .1) 0 1px 2px;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock .opblock-section-header > label > span { padding: 0 10px 0 0; }
|
||||
|
||||
.swagger-ui .opblock .opblock-summary-method {
|
||||
background: #000;
|
||||
color: #fff;
|
||||
text-shadow: rgba(0, 0, 0, .1) 0 1px 0;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock.opblock-post {
|
||||
background: rgba(72, 203, 144, .1);
|
||||
border-color: #48cb90;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock.opblock-post .opblock-summary-method, .swagger-ui .opblock.opblock-post .tab-header .tab-item.active h4 span::after { background: #48cb90; }
|
||||
|
||||
.swagger-ui .opblock.opblock-post .opblock-summary { border-color: #48cb90; }
|
||||
|
||||
.swagger-ui .opblock.opblock-put {
|
||||
background: rgba(213, 157, 88, .1);
|
||||
border-color: #d59d58;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock.opblock-put .opblock-summary-method, .swagger-ui .opblock.opblock-put .tab-header .tab-item.active h4 span::after { background: #d59d58; }
|
||||
|
||||
.swagger-ui .opblock.opblock-put .opblock-summary { border-color: #d59d58; }
|
||||
|
||||
.swagger-ui .opblock.opblock-delete {
|
||||
background: rgba(200, 50, 50, .1);
|
||||
border-color: #c83232;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock.opblock-delete .opblock-summary-method, .swagger-ui .opblock.opblock-delete .tab-header .tab-item.active h4 span::after { background: #c83232; }
|
||||
|
||||
.swagger-ui .opblock.opblock-delete .opblock-summary { border-color: #c83232; }
|
||||
|
||||
.swagger-ui .opblock.opblock-get {
|
||||
background: rgba(42, 105, 167, .1);
|
||||
border-color: #2a69a7;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock.opblock-get .opblock-summary-method, .swagger-ui .opblock.opblock-get .tab-header .tab-item.active h4 span::after { background: #2a69a7; }
|
||||
|
||||
.swagger-ui .opblock.opblock-get .opblock-summary { border-color: #2a69a7; }
|
||||
|
||||
.swagger-ui .opblock.opblock-patch {
|
||||
background: rgba(92, 214, 188, .1);
|
||||
border-color: #5cd6bc;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock.opblock-patch .opblock-summary-method, .swagger-ui .opblock.opblock-patch .tab-header .tab-item.active h4 span::after { background: #5cd6bc; }
|
||||
|
||||
.swagger-ui .opblock.opblock-patch .opblock-summary { border-color: #5cd6bc; }
|
||||
|
||||
.swagger-ui .opblock.opblock-head {
|
||||
background: rgba(140, 63, 207, .1);
|
||||
border-color: #8c3fcf;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock.opblock-head .opblock-summary-method, .swagger-ui .opblock.opblock-head .tab-header .tab-item.active h4 span::after { background: #8c3fcf; }
|
||||
|
||||
.swagger-ui .opblock.opblock-head .opblock-summary { border-color: #8c3fcf; }
|
||||
|
||||
.swagger-ui .opblock.opblock-options {
|
||||
background: rgba(36, 89, 143, .1);
|
||||
border-color: #24598f;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock.opblock-options .opblock-summary-method, .swagger-ui .opblock.opblock-options .tab-header .tab-item.active h4 span::after { background: #24598f; }
|
||||
|
||||
.swagger-ui .opblock.opblock-options .opblock-summary { border-color: #24598f; }
|
||||
|
||||
.swagger-ui .opblock.opblock-deprecated {
|
||||
background: rgba(46, 46, 46, .1);
|
||||
border-color: #2e2e2e;
|
||||
opacity: .6;
|
||||
}
|
||||
|
||||
.swagger-ui .opblock.opblock-deprecated .opblock-summary-method, .swagger-ui .opblock.opblock-deprecated .tab-header .tab-item.active h4 span::after { background: #2e2e2e; }
|
||||
|
||||
.swagger-ui .opblock.opblock-deprecated .opblock-summary { border-color: #2e2e2e; }
|
||||
|
||||
.swagger-ui .filter .operation-filter-input { border: 2px solid #2b3446; }
|
||||
|
||||
.swagger-ui .tab li:first-of-type::after { background: rgba(0, 0, 0, .2); }
|
||||
|
||||
.swagger-ui .download-contents {
|
||||
background: #7c8192;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.swagger-ui .scheme-container {
|
||||
background: #1c1c21;
|
||||
box-shadow: rgba(0, 0, 0, .15) 0 1px 2px 0;
|
||||
}
|
||||
|
||||
.swagger-ui .loading-container .loading::before {
|
||||
animation: 1s linear 0s infinite normal none running rotation, .5s ease 0s 1 normal none running opacity;
|
||||
border-color: rgba(0, 0, 0, .6) rgba(84, 84, 84, .1) rgba(84, 84, 84, .1);
|
||||
}
|
||||
|
||||
.swagger-ui .response-control-media-type--accept-controller select { border-color: #196619; }
|
||||
|
||||
.swagger-ui .response-control-media-type__accept-message { color: #99e699; }
|
||||
|
||||
.swagger-ui .version-pragma__message code { background-color: #3b3b3b; }
|
||||
|
||||
.swagger-ui .btn {
|
||||
background: 0 0;
|
||||
border: 2px solid gray;
|
||||
box-shadow: rgba(0, 0, 0, .1) 0 1px 2px;
|
||||
color: #b5bac9;
|
||||
}
|
||||
|
||||
.swagger-ui .btn:hover { box-shadow: rgba(0, 0, 0, .3) 0 0 5px; }
|
||||
|
||||
.swagger-ui .btn.authorize, .swagger-ui .btn.cancel {
|
||||
background-color: transparent;
|
||||
border-color: #a72a2a;
|
||||
color: #e69999;
|
||||
}
|
||||
|
||||
.swagger-ui .btn.cancel:hover {
|
||||
background-color: #a72a2a;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.swagger-ui .btn.authorize {
|
||||
border-color: #48cb90;
|
||||
color: #9ce3c3;
|
||||
}
|
||||
|
||||
.swagger-ui .btn.authorize svg { fill: #9ce3c3; }
|
||||
|
||||
.btn.authorize.unlocked:hover {
|
||||
background-color: #48cb90;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.btn.authorize.unlocked:hover svg {
|
||||
fill: #fbfbfb;
|
||||
}
|
||||
|
||||
.swagger-ui .btn.execute {
|
||||
background-color: #5892d5;
|
||||
border-color: #5892d5;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.swagger-ui .copy-to-clipboard { background: #7c8192; }
|
||||
|
||||
.swagger-ui .copy-to-clipboard button { background: url("data:image/svg+xml;charset=utf-8,<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"16\" height=\"16\" aria-hidden=\"true\"><path fill=\"%23fff\" fill-rule=\"evenodd\" d=\"M2 13h4v1H2v-1zm5-6H2v1h5V7zm2 3V8l-3 3 3 3v-2h5v-2H9zM4.5 9H2v1h2.5V9zM2 12h2.5v-1H2v1zm9 1h1v2c-.02.28-.11.52-.3.7-.19.18-.42.28-.7.3H1c-.55 0-1-.45-1-1V4c0-.55.45-1 1-1h3c0-1.11.89-2 2-2 1.11 0 2 .89 2 2h3c.55 0 1 .45 1 1v5h-1V6H1v9h10v-2zM2 5h8c0-.55-.45-1-1-1H8c-.55 0-1-.45-1-1s-.45-1-1-1-1 .45-1 1-.45 1-1 1H3c-.55 0-1 .45-1 1z\"/></svg>") 50% center no-repeat; }
|
||||
|
||||
.swagger-ui select {
|
||||
background: url("data:image/svg+xml;charset=utf-8,<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 20 20\"><path d=\"M13.418 7.859a.695.695 0 01.978 0 .68.68 0 010 .969l-3.908 3.83a.697.697 0 01-.979 0l-3.908-3.83a.68.68 0 010-.969.695.695 0 01.978 0L10 11l3.418-3.141z\"/></svg>") right 10px center/20px no-repeat #212121;
|
||||
background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50cy8xLjEvIgogICB4bWxuczpjYz0iaHR0cDovL2NyZWF0aXZlY29tbW9ucy5vcmcvbnMjIgogICB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFREL3NvZGlwb2RpLTAuZHRkIgogICB4bWxuczppbmtzY2FwZT0iaHR0cDovL3d3dy5pbmtzY2FwZS5vcmcvbmFtZXNwYWNlcy9pbmtzY2FwZSIKICAgaW5rc2NhcGU6dmVyc2lvbj0iMS4wICg0MDM1YTRmYjQ5LCAyMDIwLTA1LTAxKSIKICAgc29kaXBvZGk6ZG9jbmFtZT0iZG93bmxvYWQuc3ZnIgogICBpZD0ic3ZnNCIKICAgdmVyc2lvbj0iMS4xIgogICB2aWV3Qm94PSIwIDAgMjAgMjAiPgogIDxtZXRhZGF0YQogICAgIGlkPSJtZXRhZGF0YTEwIj4KICAgIDxyZGY6UkRGPgogICAgICA8Y2M6V29yawogICAgICAgICByZGY6YWJvdXQ9IiI+CiAgICAgICAgPGRjOmZvcm1hdD5pbWFnZS9zdmcreG1sPC9kYzpmb3JtYXQ+CiAgICAgICAgPGRjOnR5cGUKICAgICAgICAgICByZGY6cmVzb3VyY2U9Imh0dHA6Ly9wdXJsLm9yZy9kYy9kY21pdHlwZS9TdGlsbEltYWdlIiAvPgogICAgICA8L2NjOldvcms+CiAgICA8L3JkZjpSREY+CiAgPC9tZXRhZGF0YT4KICA8ZGVmcwogICAgIGlkPSJkZWZzOCIgLz4KICA8c29kaXBvZGk6bmFtZWR2aWV3CiAgICAgaW5rc2NhcGU6Y3VycmVudC1sYXllcj0ic3ZnNCIKICAgICBpbmtzY2FwZTp3aW5kb3ctbWF4aW1pemVkPSIxIgogICAgIGlua3NjYXBlOndpbmRvdy15PSItOSIKICAgICBpbmtzY2FwZTp3aW5kb3cteD0iLTkiCiAgICAgaW5rc2NhcGU6Y3k9IjEwIgogICAgIGlua3NjYXBlOmN4PSIxMCIKICAgICBpbmtzY2FwZTp6b29tPSI0MS41IgogICAgIHNob3dncmlkPSJmYWxzZSIKICAgICBpZD0ibmFtZWR2aWV3NiIKICAgICBpbmtzY2FwZTp3aW5kb3ctaGVpZ2h0PSIxMDAxIgogICAgIGlua3NjYXBlOndpbmRvdy13aWR0aD0iMTkyMCIKICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIgogICAgIGlua3NjYXBlOnBhZ2VvcGFjaXR5PSIwIgogICAgIGd1aWRldG9sZXJhbmNlPSIxMCIKICAgICBncmlkdG9sZXJhbmNlPSIxMCIKICAgICBvYmplY3R0b2xlcmFuY2U9IjEwIgogICAgIGJvcmRlcm9wYWNpdHk9IjEiCiAgICAgYm9yZGVyY29sb3I9IiM2NjY2NjYiCiAgICAgcGFnZWNvbG9yPSIjZmZmZmZmIiAvPgogIDxwYXRoCiAgICAgc3R5bGU9ImZpbGw6I2ZmZmZmZiIKICAgICBpZD0icGF0aDIiCiAgICAgZD0iTTEzLjQxOCA3Ljg1OWEuNjk1LjY5NSAwIDAxLjk3OCAwIC42OC42OCAwIDAxMCAuOTY5bC0zLjkwOCAzLjgzYS42OTcuNjk3IDAgMDEtLjk3OSAwbC0zLjkwOC0zLjgzYS42OC42OCAwIDAxMC0uOTY5LjY5NS42OTUgMCAwMS45NzggMEwxMCAxMWwzLjQxOC0zLjE0MXoiIC8+Cjwvc3ZnPgo=) right 10px center/20px no-repeat #1c1c21;
|
||||
border: 2px solid #41444e;
|
||||
}
|
||||
|
||||
.swagger-ui select[multiple] { background: #212121; }
|
||||
|
||||
.swagger-ui button.invalid, .swagger-ui input[type=email].invalid, .swagger-ui input[type=file].invalid, .swagger-ui input[type=password].invalid, .swagger-ui input[type=search].invalid, .swagger-ui input[type=text].invalid, .swagger-ui select.invalid, .swagger-ui textarea.invalid {
|
||||
background: #390e0e;
|
||||
border-color: #c83232;
|
||||
}
|
||||
|
||||
.swagger-ui input[type=email], .swagger-ui input[type=file], .swagger-ui input[type=password], .swagger-ui input[type=search], .swagger-ui input[type=text], .swagger-ui textarea {
|
||||
background: #1c1c21;
|
||||
border: 1px solid #404040;
|
||||
}
|
||||
|
||||
.swagger-ui textarea {
|
||||
background: rgba(28, 28, 33, .8);
|
||||
color: #b5bac9;
|
||||
}
|
||||
|
||||
.swagger-ui input[disabled], .swagger-ui select[disabled] {
|
||||
background-color: #1f1f1f;
|
||||
color: #bfbfbf;
|
||||
}
|
||||
|
||||
.swagger-ui textarea[disabled] {
|
||||
background-color: #41444e;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.swagger-ui select[disabled] { border-color: #878787; }
|
||||
|
||||
.swagger-ui textarea:focus { border: 2px solid #2a69a7; }
|
||||
|
||||
.swagger-ui .checkbox input[type=checkbox] + label > .item {
|
||||
background: #303030;
|
||||
box-shadow: #303030 0 0 0 2px;
|
||||
}
|
||||
|
||||
.swagger-ui .checkbox input[type=checkbox]:checked + label > .item { background: url("data:image/svg+xml;charset=utf-8,<svg width=\"10\" height=\"8\" viewBox=\"3 7 10 8\" xmlns=\"http://www.w3.org/2000/svg\"><path fill=\"%2341474E\" fill-rule=\"evenodd\" d=\"M6.333 15L3 11.667l1.333-1.334 2 2L11.667 7 13 8.333z\"/></svg>") 50% center no-repeat #303030; }
|
||||
|
||||
.swagger-ui .dialog-ux .backdrop-ux { background: rgba(0, 0, 0, .8); }
|
||||
|
||||
.swagger-ui .dialog-ux .modal-ux {
|
||||
background: #1c1c21;
|
||||
border: 1px solid #2e2e2e;
|
||||
box-shadow: rgba(0, 0, 0, .2) 0 10px 30px 0;
|
||||
}
|
||||
|
||||
.swagger-ui .dialog-ux .modal-ux-header .close-modal { background: 0 0; }
|
||||
|
||||
.swagger-ui .model .deprecated span, .swagger-ui .model .deprecated td { color: #bfbfbf !important; }
|
||||
|
||||
.swagger-ui .model-toggle::after { background: url("data:image/svg+xml;charset=utf-8,<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"24\" height=\"24\"><path d=\"M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z\"/></svg>") 50% center/100% no-repeat; }
|
||||
|
||||
.swagger-ui .model-hint {
|
||||
background: rgba(0, 0, 0, .7);
|
||||
color: #ebebeb;
|
||||
}
|
||||
|
||||
.swagger-ui section.models { border: 1px solid rgba(58, 64, 80, .3); }
|
||||
|
||||
.swagger-ui section.models.is-open h4 { border-bottom: 1px solid rgba(58, 64, 80, .3); }
|
||||
|
||||
.swagger-ui section.models .model-container { background: rgba(0, 0, 0, .05); }
|
||||
|
||||
.swagger-ui section.models .model-container:hover { background: rgba(0, 0, 0, .07); }
|
||||
|
||||
.swagger-ui .model-box { background: rgba(0, 0, 0, .1); }
|
||||
|
||||
.swagger-ui .prop-type { color: #aaaad4; }
|
||||
|
||||
.swagger-ui table thead tr td, .swagger-ui table thead tr th {
|
||||
border-bottom: 1px solid rgba(58, 64, 80, .2);
|
||||
color: #b5bac9;
|
||||
}
|
||||
|
||||
.swagger-ui .parameter__name.required::after { color: rgba(230, 153, 153, .6); }
|
||||
|
||||
.swagger-ui .topbar .download-url-wrapper .select-label { color: #f0f0f0; }
|
||||
|
||||
.swagger-ui .topbar .download-url-wrapper .download-url-button {
|
||||
background: #63a040;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.swagger-ui .info .title small { background: #7c8492; }
|
||||
|
||||
.swagger-ui .info .title small.version-stamp { background-color: #7a9b27; }
|
||||
|
||||
.swagger-ui .auth-container .errors {
|
||||
background-color: #350d0d;
|
||||
color: #b5bac9;
|
||||
}
|
||||
|
||||
.swagger-ui .errors-wrapper {
|
||||
background: rgba(200, 50, 50, .1);
|
||||
border: 2px solid #c83232;
|
||||
}
|
||||
|
||||
.swagger-ui .markdown code, .swagger-ui .renderedmarkdown code {
|
||||
background: rgba(0, 0, 0, .05);
|
||||
color: #c299e6;
|
||||
}
|
||||
|
||||
.swagger-ui .model-toggle:after { background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50cy8xLjEvIgogICB4bWxuczpjYz0iaHR0cDovL2NyZWF0aXZlY29tbW9ucy5vcmcvbnMjIgogICB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFREL3NvZGlwb2RpLTAuZHRkIgogICB4bWxuczppbmtzY2FwZT0iaHR0cDovL3d3dy5pbmtzY2FwZS5vcmcvbmFtZXNwYWNlcy9pbmtzY2FwZSIKICAgaW5rc2NhcGU6dmVyc2lvbj0iMS4wICg0MDM1YTRmYjQ5LCAyMDIwLTA1LTAxKSIKICAgc29kaXBvZGk6ZG9jbmFtZT0iZG93bmxvYWQyLnN2ZyIKICAgaWQ9InN2ZzQiCiAgIHZlcnNpb249IjEuMSIKICAgaGVpZ2h0PSIyNCIKICAgd2lkdGg9IjI0Ij4KICA8bWV0YWRhdGEKICAgICBpZD0ibWV0YWRhdGExMCI+CiAgICA8cmRmOlJERj4KICAgICAgPGNjOldvcmsKICAgICAgICAgcmRmOmFib3V0PSIiPgogICAgICAgIDxkYzpmb3JtYXQ+aW1hZ2Uvc3ZnK3htbDwvZGM6Zm9ybWF0PgogICAgICAgIDxkYzp0eXBlCiAgICAgICAgICAgcmRmOnJlc291cmNlPSJodHRwOi8vcHVybC5vcmcvZGMvZGNtaXR5cGUvU3RpbGxJbWFnZSIgLz4KICAgICAgPC9jYzpXb3JrPgogICAgPC9yZGY6UkRGPgogIDwvbWV0YWRhdGE+CiAgPGRlZnMKICAgICBpZD0iZGVmczgiIC8+CiAgPHNvZGlwb2RpOm5hbWVkdmlldwogICAgIGlua3NjYXBlOmN1cnJlbnQtbGF5ZXI9InN2ZzQiCiAgICAgaW5rc2NhcGU6d2luZG93LW1heGltaXplZD0iMSIKICAgICBpbmtzY2FwZTp3aW5kb3cteT0iLTkiCiAgICAgaW5rc2NhcGU6d2luZG93LXg9Ii05IgogICAgIGlua3NjYXBlOmN5PSIxMiIKICAgICBpbmtzY2FwZTpjeD0iMTIiCiAgICAgaW5rc2NhcGU6em9vbT0iMzQuNTgzMzMzIgogICAgIHNob3dncmlkPSJmYWxzZSIKICAgICBpZD0ibmFtZWR2aWV3NiIKICAgICBpbmtzY2FwZTp3aW5kb3ctaGVpZ2h0PSIxMDAxIgogICAgIGlua3NjYXBlOndpbmRvdy13aWR0aD0iMTkyMCIKICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIgogICAgIGlua3NjYXBlOnBhZ2VvcGFjaXR5PSIwIgogICAgIGd1aWRldG9sZXJhbmNlPSIxMCIKICAgICBncmlkdG9sZXJhbmNlPSIxMCIKICAgICBvYmplY3R0b2xlcmFuY2U9IjEwIgogICAgIGJvcmRlcm9wYWNpdHk9IjEiCiAgICAgYm9yZGVyY29sb3I9IiM2NjY2NjYiCiAgICAgcGFnZWNvbG9yPSIjZmZmZmZmIiAvPgogIDxwYXRoCiAgICAgc3R5bGU9ImZpbGw6I2ZmZmZmZiIKICAgICBpZD0icGF0aDIiCiAgICAgZD0iTTEwIDZMOC41OSA3LjQxIDEzLjE3IDEybC00LjU4IDQuNTlMMTAgMThsNi02eiIgLz4KPC9zdmc+Cg==) 50% no-repeat; }
|
||||
|
||||
/* arrows for each operation and request are now white */
|
||||
.arrow, #large-arrow-up { fill: #fff; }
|
||||
|
||||
#unlocked { fill: #fff; }
|
||||
|
||||
::-webkit-scrollbar-track { background-color: #646464 !important; }
|
||||
|
||||
::-webkit-scrollbar-thumb {
|
||||
background-color: #242424 !important;
|
||||
border: 2px solid #3e4346 !important;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-button:vertical:start:decrement {
|
||||
background: linear-gradient(130deg, #696969 40%, rgba(255, 0, 0, 0) 41%), linear-gradient(230deg, #696969 40%, transparent 41%), linear-gradient(0deg, #696969 40%, transparent 31%);
|
||||
background-color: #b6b6b6;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-button:vertical:end:increment {
|
||||
background: linear-gradient(310deg, #696969 40%, transparent 41%), linear-gradient(50deg, #696969 40%, transparent 41%), linear-gradient(180deg, #696969 40%, transparent 31%);
|
||||
background-color: #b6b6b6;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-button:horizontal:end:increment {
|
||||
background: linear-gradient(210deg, #696969 40%, transparent 41%), linear-gradient(330deg, #696969 40%, transparent 41%), linear-gradient(90deg, #696969 30%, transparent 31%);
|
||||
background-color: #b6b6b6;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-button:horizontal:start:decrement {
|
||||
background: linear-gradient(30deg, #696969 40%, transparent 41%), linear-gradient(150deg, #696969 40%, transparent 41%), linear-gradient(270deg, #696969 30%, transparent 31%);
|
||||
background-color: #b6b6b6;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-button, ::-webkit-scrollbar-track-piece { background-color: #3e4346 !important; }
|
||||
|
||||
.swagger-ui .black, .swagger-ui .checkbox, .swagger-ui .dark-gray, .swagger-ui .download-url-wrapper .loading, .swagger-ui .errors-wrapper .errors small, .swagger-ui .fallback, .swagger-ui .filter .loading, .swagger-ui .gray, .swagger-ui .hover-black:focus, .swagger-ui .hover-black:hover, .swagger-ui .hover-dark-gray:focus, .swagger-ui .hover-dark-gray:hover, .swagger-ui .hover-gray:focus, .swagger-ui .hover-gray:hover, .swagger-ui .hover-light-silver:focus, .swagger-ui .hover-light-silver:hover, .swagger-ui .hover-mid-gray:focus, .swagger-ui .hover-mid-gray:hover, .swagger-ui .hover-near-black:focus, .swagger-ui .hover-near-black:hover, .swagger-ui .hover-silver:focus, .swagger-ui .hover-silver:hover, .swagger-ui .light-silver, .swagger-ui .markdown pre, .swagger-ui .mid-gray, .swagger-ui .model .property, .swagger-ui .model .property.primitive, .swagger-ui .model-title, .swagger-ui .near-black, .swagger-ui .parameter__extension, .swagger-ui .parameter__in, .swagger-ui .prop-format, .swagger-ui .renderedmarkdown pre, .swagger-ui .response-col_links .response-undocumented, .swagger-ui .response-col_status .response-undocumented, .swagger-ui .silver, .swagger-ui section.models h4, .swagger-ui section.models h5, .swagger-ui span.token-not-formatted, .swagger-ui span.token-string, .swagger-ui table.headers .header-example, .swagger-ui table.model tr.description, .swagger-ui table.model tr.extension { color: #bfbfbf; }
|
||||
|
||||
.swagger-ui .hover-white:focus, .swagger-ui .hover-white:hover, .swagger-ui .info .title small pre, .swagger-ui .topbar a, .swagger-ui .white { color: #fff; }
|
||||
|
||||
.swagger-ui .bg-black-10, .swagger-ui .hover-bg-black-10:focus, .swagger-ui .hover-bg-black-10:hover, .swagger-ui .stripe-dark:nth-child(2n + 1) { background-color: rgba(0, 0, 0, .1); }
|
||||
|
||||
.swagger-ui .bg-white-10, .swagger-ui .hover-bg-white-10:focus, .swagger-ui .hover-bg-white-10:hover, .swagger-ui .stripe-light:nth-child(2n + 1) { background-color: rgba(28, 28, 33, .1); }
|
||||
|
||||
.swagger-ui .bg-light-silver, .swagger-ui .hover-bg-light-silver:focus, .swagger-ui .hover-bg-light-silver:hover, .swagger-ui .striped--light-silver:nth-child(2n + 1) { background-color: #6e6e6e; }
|
||||
|
||||
.swagger-ui .bg-moon-gray, .swagger-ui .hover-bg-moon-gray:focus, .swagger-ui .hover-bg-moon-gray:hover, .swagger-ui .striped--moon-gray:nth-child(2n + 1) { background-color: #4d4d4d; }
|
||||
|
||||
.swagger-ui .bg-light-gray, .swagger-ui .hover-bg-light-gray:focus, .swagger-ui .hover-bg-light-gray:hover, .swagger-ui .striped--light-gray:nth-child(2n + 1) { background-color: #2b2b2b; }
|
||||
|
||||
.swagger-ui .bg-near-white, .swagger-ui .hover-bg-near-white:focus, .swagger-ui .hover-bg-near-white:hover, .swagger-ui .striped--near-white:nth-child(2n + 1) { background-color: #242424; }
|
||||
|
||||
.swagger-ui .opblock-tag:hover, .swagger-ui section.models h4:hover { background: rgba(0, 0, 0, .02); }
|
||||
|
||||
.swagger-ui .checkbox p, .swagger-ui .dialog-ux .modal-ux-content h4, .swagger-ui .dialog-ux .modal-ux-content p, .swagger-ui .dialog-ux .modal-ux-header h3, .swagger-ui .errors-wrapper .errors h4, .swagger-ui .errors-wrapper hgroup h4, .swagger-ui .info .base-url, .swagger-ui .info .title, .swagger-ui .info h1, .swagger-ui .info h2, .swagger-ui .info h3, .swagger-ui .info h4, .swagger-ui .info h5, .swagger-ui .info li, .swagger-ui .info p, .swagger-ui .info table, .swagger-ui .loading-container .loading::after, .swagger-ui .model, .swagger-ui .opblock .opblock-section-header h4, .swagger-ui .opblock .opblock-section-header > label, .swagger-ui .opblock .opblock-summary-description, .swagger-ui .opblock .opblock-summary-operation-id, .swagger-ui .opblock .opblock-summary-path, .swagger-ui .opblock .opblock-summary-path__deprecated, .swagger-ui .opblock-description-wrapper, .swagger-ui .opblock-description-wrapper h4, .swagger-ui .opblock-description-wrapper p, .swagger-ui .opblock-external-docs-wrapper, .swagger-ui .opblock-external-docs-wrapper h4, .swagger-ui .opblock-external-docs-wrapper p, .swagger-ui .opblock-tag small, .swagger-ui .opblock-title_normal, .swagger-ui .opblock-title_normal h4, .swagger-ui .opblock-title_normal p, .swagger-ui .parameter__name, .swagger-ui .parameter__type, .swagger-ui .response-col_links, .swagger-ui .response-col_status, .swagger-ui .responses-inner h4, .swagger-ui .responses-inner h5, .swagger-ui .scheme-container .schemes > label, .swagger-ui .scopes h2, .swagger-ui .servers > label, .swagger-ui .tab li, .swagger-ui label, .swagger-ui select, .swagger-ui table.headers td { color: #b5bac9; }
|
||||
|
||||
.swagger-ui .download-url-wrapper .failed, .swagger-ui .filter .failed, .swagger-ui .model-deprecated-warning, .swagger-ui .parameter__deprecated, .swagger-ui .parameter__name.required span, .swagger-ui table.model tr.property-row .star { color: #e69999; }
|
||||
|
||||
.swagger-ui .opblock-body pre.microlight, .swagger-ui textarea.curl {
|
||||
background: #41444e;
|
||||
border-radius: 4px;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.swagger-ui .expand-methods svg, .swagger-ui .expand-methods:hover svg { fill: #bfbfbf; }
|
||||
|
||||
.swagger-ui .auth-container, .swagger-ui .dialog-ux .modal-ux-header { border-bottom: 1px solid #2e2e2e; }
|
||||
|
||||
.swagger-ui .topbar .download-url-wrapper .select-label select, .swagger-ui .topbar .download-url-wrapper input[type=text] { border: 2px solid #63a040; }
|
||||
|
||||
.swagger-ui .info a, .swagger-ui .info a:hover, .swagger-ui .scopes h2 a { color: #99bde6; }
|
||||
|
||||
/* Dark Scrollbar */
|
||||
::-webkit-scrollbar {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-button {
|
||||
background-color: #3e4346 !important;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-track {
|
||||
background-color: #646464 !important;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-track-piece {
|
||||
background-color: #3e4346 !important;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-thumb {
|
||||
height: 50px;
|
||||
background-color: #242424 !important;
|
||||
border: 2px solid #3e4346 !important;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-corner {}
|
||||
|
||||
::-webkit-resizer {}
|
||||
|
||||
::-webkit-scrollbar-button:vertical:start:decrement {
|
||||
background:
|
||||
linear-gradient(130deg, #696969 40%, rgba(255, 0, 0, 0) 41%),
|
||||
linear-gradient(230deg, #696969 40%, rgba(0, 0, 0, 0) 41%),
|
||||
linear-gradient(0deg, #696969 40%, rgba(0, 0, 0, 0) 31%);
|
||||
background-color: #b6b6b6;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-button:vertical:end:increment {
|
||||
background:
|
||||
linear-gradient(310deg, #696969 40%, rgba(0, 0, 0, 0) 41%),
|
||||
linear-gradient(50deg, #696969 40%, rgba(0, 0, 0, 0) 41%),
|
||||
linear-gradient(180deg, #696969 40%, rgba(0, 0, 0, 0) 31%);
|
||||
background-color: #b6b6b6;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-button:horizontal:end:increment {
|
||||
background:
|
||||
linear-gradient(210deg, #696969 40%, rgba(0, 0, 0, 0) 41%),
|
||||
linear-gradient(330deg, #696969 40%, rgba(0, 0, 0, 0) 41%),
|
||||
linear-gradient(90deg, #696969 30%, rgba(0, 0, 0, 0) 31%);
|
||||
background-color: #b6b6b6;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-button:horizontal:start:decrement {
|
||||
background:
|
||||
linear-gradient(30deg, #696969 40%, rgba(0, 0, 0, 0) 41%),
|
||||
linear-gradient(150deg, #696969 40%, rgba(0, 0, 0, 0) 41%),
|
||||
linear-gradient(270deg, #696969 30%, rgba(0, 0, 0, 0) 31%);
|
||||
background-color: #b6b6b6;
|
||||
}
|
||||
}
|
||||
+82
-25
@@ -27,7 +27,7 @@ log_file = os.path.join(os.path.dirname(__file__), 'sdnext.log')
|
||||
log_rolled = False
|
||||
first_call = True
|
||||
quick_allowed = True
|
||||
errors = 0
|
||||
errors = []
|
||||
opts = {}
|
||||
args = Dot({
|
||||
'debug': False,
|
||||
@@ -110,6 +110,9 @@ def setup_logging():
|
||||
"traceback.border": "black",
|
||||
"traceback.border.syntax_error": "black",
|
||||
"inspect.value.border": "black",
|
||||
"logging.level.info": "blue_violet",
|
||||
"logging.level.debug": "purple4",
|
||||
"logging.level.trace": "dark_blue",
|
||||
}))
|
||||
logging.basicConfig(level=logging.ERROR, format='%(asctime)s | %(name)s | %(levelname)s | %(module)s | %(message)s', handlers=[logging.NullHandler()]) # redirect default logger to null
|
||||
pretty_install(console=console)
|
||||
@@ -277,8 +280,7 @@ def pip(arg: str, ignore: bool = False, quiet: bool = False, uv = True):
|
||||
txt = txt.strip()
|
||||
debug(f'Install {pipCmd}: {txt}')
|
||||
if result.returncode != 0 and not ignore:
|
||||
global errors # pylint: disable=global-statement
|
||||
errors += 1
|
||||
errors.append(f'pip: {package}')
|
||||
log.error(f'Install: {pipCmd}: {arg}')
|
||||
log.debug(f'Install: pip output {txt}')
|
||||
return txt
|
||||
@@ -321,8 +323,7 @@ def git(arg: str, folder: str = None, ignore: bool = False, optional: bool = Fal
|
||||
if result.returncode != 0 and not ignore:
|
||||
if "couldn't find remote ref" in txt: # not a git repo
|
||||
return txt
|
||||
global errors # pylint: disable=global-statement
|
||||
errors += 1
|
||||
errors.append(f'git: {folder}')
|
||||
log.error(f'Git: {folder} / {arg}')
|
||||
if 'or stash them' in txt:
|
||||
log.error(f'Git local changes detected: check details log="{log_file}"')
|
||||
@@ -377,10 +378,11 @@ def update(folder, keep_branch = False, rebase = True):
|
||||
else:
|
||||
res = git(f'pull origin {b} {arg}', folder)
|
||||
debug(f'Install update: folder={folder} branch={b} args={arg} {res}')
|
||||
commit = extensions_commit.get(os.path.basename(folder), None)
|
||||
if commit is not None:
|
||||
res = git(f'checkout {commit}', folder)
|
||||
debug(f'Install update: folder={folder} branch={b} args={arg} commit={commit} {res}')
|
||||
if not args.experimental:
|
||||
commit = extensions_commit.get(os.path.basename(folder), None)
|
||||
if commit is not None:
|
||||
res = git(f'checkout {commit}', folder)
|
||||
debug(f'Install update: folder={folder} branch={b} args={arg} commit={commit} {res}')
|
||||
return res
|
||||
|
||||
|
||||
@@ -410,14 +412,14 @@ def get_platform():
|
||||
else:
|
||||
release = platform.release()
|
||||
return {
|
||||
# 'host': platform.node(),
|
||||
'arch': platform.machine(),
|
||||
'cpu': platform.processor(),
|
||||
'system': platform.system(),
|
||||
'release': release,
|
||||
# 'platform': platform.platform(aliased = True, terse = False),
|
||||
# 'version': platform.version(),
|
||||
'python': platform.python_version(),
|
||||
'docker': os.environ.get('SD_INSTALL_DEBUG', None) is not None,
|
||||
# 'host': platform.node(),
|
||||
# 'version': platform.version(),
|
||||
}
|
||||
except Exception as e:
|
||||
return { 'error': e }
|
||||
@@ -455,12 +457,14 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None):
|
||||
|
||||
# check diffusers version
|
||||
def check_diffusers():
|
||||
sha = '0d1d267b12e47b40b0e8f265339c76e0f45f8c49'
|
||||
if args.skip_all or args.skip_requirements:
|
||||
return
|
||||
sha = 'f6f7afa1d7c6f45f8568c5603b1e6300d4583f04'
|
||||
pkg = pkg_resources.working_set.by_key.get('diffusers', None)
|
||||
minor = int(pkg.version.split('.')[1] if pkg is not None else 0)
|
||||
cur = opts.get('diffusers_version', '') if minor > 0 else ''
|
||||
if (minor == 0) or (cur != sha):
|
||||
log.debug(f'Diffusers {"install" if minor == 0 else "upgrade"}: package={pkg} current={cur} target={sha}')
|
||||
log.info(f'Diffusers {"install" if minor == 0 else "upgrade"}: package={pkg} current={cur} target={sha}')
|
||||
if minor > 0:
|
||||
pip('uninstall --yes diffusers', ignore=True, quiet=True, uv=False)
|
||||
pip(f'install --upgrade git+https://github.com/huggingface/diffusers@{sha}', ignore=False, quiet=True, uv=False)
|
||||
@@ -470,6 +474,8 @@ def check_diffusers():
|
||||
|
||||
# check onnx version
|
||||
def check_onnx():
|
||||
if args.skip_all or args.skip_requirements:
|
||||
return
|
||||
if not installed('onnx', quiet=True):
|
||||
install('onnx', 'onnx', ignore=True)
|
||||
if not installed('onnxruntime', quiet=True) and not (installed('onnxruntime-gpu', quiet=True) or installed('onnxruntime-openvino', quiet=True) or installed('onnxruntime-training', quiet=True)): # allow either
|
||||
@@ -477,6 +483,8 @@ def check_onnx():
|
||||
|
||||
|
||||
def check_torchao():
|
||||
if args.skip_all or args.skip_requirements:
|
||||
return
|
||||
if installed('torchao', quiet=True):
|
||||
ver = package_version('torchao')
|
||||
if ver != '0.5.0':
|
||||
@@ -488,14 +496,16 @@ def check_torchao():
|
||||
|
||||
def install_cuda():
|
||||
log.info('CUDA: nVidia toolkit detected')
|
||||
install('onnxruntime-gpu', 'onnxruntime-gpu', ignore=True, quiet=True)
|
||||
if not (args.skip_all or args.skip_requirements):
|
||||
install('onnxruntime-gpu', 'onnxruntime-gpu', ignore=True, quiet=True)
|
||||
# return os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/cu124')
|
||||
return os.environ.get('TORCH_COMMAND', 'torch==2.5.1+cu124 torchvision==0.20.1+cu124 --index-url https://download.pytorch.org/whl/cu124')
|
||||
|
||||
|
||||
def install_rocm_zluda():
|
||||
if args.skip_all or args.skip_requirements:
|
||||
return None
|
||||
from modules import rocm
|
||||
|
||||
if not rocm.is_installed:
|
||||
log.warning('ROCm: could not find ROCm toolkit installed')
|
||||
log.info('Using CPU-only torch')
|
||||
@@ -547,7 +557,6 @@ def install_rocm_zluda():
|
||||
os.environ['HIP_VISIBLE_DEVICES'] = args.device_id
|
||||
del args.device_id
|
||||
|
||||
log.warning("ZLUDA support: experimental")
|
||||
error = None
|
||||
from modules import zluda_installer
|
||||
zluda_installer.set_default_agent(device)
|
||||
@@ -595,11 +604,11 @@ def install_rocm_zluda():
|
||||
install(ort_package, 'onnxruntime-training')
|
||||
|
||||
if installed("torch") and device is not None:
|
||||
if 'Flash attention' in opts.get('sdp_options'):
|
||||
if 'Flash attention' in opts.get('sdp_options', ''):
|
||||
if not installed('flash-attn'):
|
||||
install(rocm.get_flash_attention_command(device), reinstall=True)
|
||||
elif not args.experimental:
|
||||
uninstall('flash-attn')
|
||||
#elif not args.experimental:
|
||||
# uninstall('flash-attn')
|
||||
|
||||
if device is not None and rocm.version != "6.2" and rocm.version == rocm.version_torch and rocm.get_blaslt_enabled():
|
||||
log.debug(f'ROCm hipBLASLt: arch={device.name} available={device.blaslt_supported}')
|
||||
@@ -657,7 +666,7 @@ def install_torch_addons():
|
||||
triton_command = os.environ.get('TRITON_COMMAND', 'triton') if sys.platform == 'linux' else None
|
||||
if 'xformers' in xformers_package:
|
||||
try:
|
||||
install(f'--no-deps {xformers_package}', ignore=True)
|
||||
install(xformers_package, ignore=True, no_deps=True)
|
||||
import torch # pylint: disable=unused-import
|
||||
import xformers # pylint: disable=unused-import
|
||||
except Exception as e:
|
||||
@@ -844,8 +853,7 @@ def run_extension_installer(folder):
|
||||
txt = result.stdout.decode(encoding="utf8", errors="ignore")
|
||||
debug(f'Extension installer: file="{path_installer}" {txt}')
|
||||
if result.returncode != 0:
|
||||
global errors # pylint: disable=global-statement
|
||||
errors += 1
|
||||
errors.append(f'ext: {os.path.basename(folder)}')
|
||||
if len(result.stderr) > 0:
|
||||
txt = txt + '\n' + result.stderr.decode(encoding="utf8", errors="ignore")
|
||||
log.error(f'Extension installer error: {path_installer}')
|
||||
@@ -986,6 +994,29 @@ def ensure_base_requirements():
|
||||
install('requests', 'requests', quiet=True)
|
||||
|
||||
|
||||
def install_optional():
|
||||
log.info('Installing optional requirements...')
|
||||
install('basicsr')
|
||||
install('gfpgan')
|
||||
install('clean-fid')
|
||||
install('optimum-quanto', ignore=True)
|
||||
install('bitsandbytes', ignore=True)
|
||||
install('pynvml', ignore=True)
|
||||
install('ultralytics', ignore=True)
|
||||
install('Cython', ignore=True)
|
||||
install('insightface', ignore=True) # problematic build
|
||||
install('nncf==2.7.0', ignore=True, no_deps=True) # requires older pandas
|
||||
# install('flash-attn', ignore=True) # requires cuda and nvcc to be installed
|
||||
install('gguf', ignore=True)
|
||||
try:
|
||||
import gguf
|
||||
scripts_dir = os.path.join(os.path.dirname(gguf.__file__), '..', 'scripts')
|
||||
if os.path.exists(scripts_dir):
|
||||
os.rename(scripts_dir, scripts_dir + '_gguf')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def install_requirements():
|
||||
if args.profile:
|
||||
pr = cProfile.Profile()
|
||||
@@ -995,10 +1026,13 @@ def install_requirements():
|
||||
if not installed('diffusers', quiet=True): # diffusers are not installed, so run initial installation
|
||||
global quick_allowed # pylint: disable=global-statement
|
||||
quick_allowed = False
|
||||
log.info('Installing requirements: this may take a while...')
|
||||
log.info('Install requirements: this may take a while...')
|
||||
pip('install -r requirements.txt')
|
||||
if args.optional:
|
||||
quick_allowed = False
|
||||
install_optional()
|
||||
installed('torch', reload=True) # reload packages cache
|
||||
log.info('Verifying requirements')
|
||||
log.info('Install: verifying requirements')
|
||||
with open('requirements.txt', 'r', encoding='utf8') as f:
|
||||
lines = [line.strip() for line in f.readlines() if line.strip() != '' and not line.startswith('#') and line is not None]
|
||||
for line in lines:
|
||||
@@ -1139,6 +1173,28 @@ def check_ui(ver):
|
||||
os.chdir(cwd)
|
||||
|
||||
|
||||
def check_venv():
|
||||
import site
|
||||
pkg_path = [os.path.relpath(p) for p in site.getsitepackages() if os.path.exists(p)]
|
||||
log.debug(f'Packages: venv={os.path.relpath(sys.prefix)} site={pkg_path}')
|
||||
for p in pkg_path:
|
||||
invalid = []
|
||||
for f in os.listdir(p):
|
||||
if f.startswith('~'):
|
||||
invalid.append(f)
|
||||
if len(invalid) > 0:
|
||||
log.warning(f'Packages: site="{p}" invalid={invalid} removing')
|
||||
for f in invalid:
|
||||
fn = os.path.join(p, f)
|
||||
try:
|
||||
if os.path.isdir(fn):
|
||||
shutil.rmtree(fn)
|
||||
elif os.path.isfile(fn):
|
||||
os.unlink(fn)
|
||||
except Exception as e:
|
||||
log.error(f'Packages: site={p} invalid={f} error={e}')
|
||||
|
||||
|
||||
# check version of the main repo and optionally upgrade it
|
||||
def check_version(offline=False, reset=True): # pylint: disable=unused-argument
|
||||
if args.skip_all:
|
||||
@@ -1239,6 +1295,7 @@ def add_args(parser):
|
||||
group_setup.add_argument('--upgrade', '--update', default = os.environ.get("SD_UPGRADE",False), action='store_true', help = "Upgrade main repository to latest version, default: %(default)s")
|
||||
group_setup.add_argument('--requirements', default = os.environ.get("SD_REQUIREMENTS",False), action='store_true', help = "Force re-check of requirements, default: %(default)s")
|
||||
group_setup.add_argument('--reinstall', default = os.environ.get("SD_REINSTALL",False), action='store_true', help = "Force reinstallation of all requirements, default: %(default)s")
|
||||
group_setup.add_argument('--optional', default = os.environ.get("SD_OPTIONAL",False), action='store_true', help = "Force installation of optional requirements, default: %(default)s")
|
||||
group_setup.add_argument('--uv', default = os.environ.get("SD_UV",False), action='store_true', help = "Use uv instead of pip to install the packages")
|
||||
|
||||
group_startup = parser.add_argument_group('Startup')
|
||||
|
||||
@@ -75,3 +75,10 @@ async function initChangelog() {
|
||||
};
|
||||
search.addEventListener('keyup', searchChangelog);
|
||||
}
|
||||
|
||||
function wikiSearch(txt) {
|
||||
log('wikiSearch', txt);
|
||||
const url = `https://github.com/search?q=repo%3Avladmandic%2Fautomatic+${encodeURIComponent(txt)}&type=wikis`;
|
||||
// window.open(url, '_blank').focus();
|
||||
return txt;
|
||||
}
|
||||
|
||||
@@ -481,11 +481,13 @@ function setupExtraNetworksForTab(tabname) {
|
||||
en.style.position = 'absolute';
|
||||
en.style.height = 'auto';
|
||||
en.style.width = `${window.opts.extra_networks_sidebar_width}vw`;
|
||||
en.style.maxWidth = '655px';
|
||||
en.style.right = '0';
|
||||
en.style.top = '13em';
|
||||
en.style.transition = 'width 0.3s ease';
|
||||
en.style.zIndex = 100;
|
||||
gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = `${100 - 2 - window.opts.extra_networks_sidebar_width}vw`;
|
||||
// gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = `${100 - 2 - window.opts.extra_networks_sidebar_width}vw`;
|
||||
gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = `calc(100vw - 2em - min(${window.opts.extra_networks_sidebar_width}vw, 655px))`;
|
||||
} else {
|
||||
en.style.position = 'relative';
|
||||
en.style.height = 'unset';
|
||||
|
||||
@@ -13,11 +13,12 @@ footer { display: none; margin-top: 0 !important;}
|
||||
table { overflow-x: auto !important; overflow-y: auto !important; }
|
||||
td { border-bottom: none !important; padding: 0 0.5em !important; }
|
||||
tr { border-bottom: none !important; padding: 0 0.5em !important; }
|
||||
td > div > span { overflow-y: auto; max-height: 3em; overflow-x: hidden; }
|
||||
textarea { overflow-y: auto !important; }
|
||||
span { font-size: var(--text-md) !important; }
|
||||
button { font-size: var(--text-lg) !important; }
|
||||
input[type='color'] { width: 64px; height: 32px; }
|
||||
td > div > span { overflow-y: auto; max-height: 3em; overflow-x: hidden; }
|
||||
input::-webkit-outer-spin-button, input::-webkit-inner-spin-button { margin-left: 4px; }
|
||||
|
||||
/* gradio elements */
|
||||
.block .padded:not(.gradio-accordion) { padding: 4px 0 0 0 !important; margin-right: 0; min-width: 90px !important; }
|
||||
@@ -326,6 +327,11 @@ div:has(>#tab-gallery-folders) { flex-grow: 0 !important; background-color: var(
|
||||
.changelog_arrow:hover { background-color: var(--button-primary-border-color-hover); }
|
||||
.changelog_highlight { background-color: var(--color-warning); }
|
||||
|
||||
/* wiki */
|
||||
#wiki_result > div > div { padding: 0.5em; margin-right: 2em; }
|
||||
#wiki_result li { display: block; }
|
||||
#wiki_result h3 { background-color: var(--background-fill-primary); margin: 0; padding: 0.3em; margin-bottom: 0.2em; }
|
||||
|
||||
/* loader */
|
||||
.splash { position: fixed; top: 0; left: 0; width: 100vw; height: 100vh; z-index: 1000; display: block; text-align: center; }
|
||||
.motd { margin-top: 2em; color: var(--body-text-color-subdued); font-family: monospace; font-variant: all-petite-caps; }
|
||||
|
||||
@@ -204,12 +204,13 @@ def main():
|
||||
installer.check_python()
|
||||
if args.reset:
|
||||
installer.git_reset()
|
||||
if args.skip_git:
|
||||
if args.skip_git or args.skip_all:
|
||||
installer.log.info('Skipping GIT operations')
|
||||
installer.check_version()
|
||||
installer.log.info(f'Platform: {installer.print_dict(installer.get_platform())}')
|
||||
installer.check_venv()
|
||||
installer.log.info(f'Args: {sys.argv[1:]}')
|
||||
if not args.skip_env:
|
||||
if not args.skip_env or args.skip_all:
|
||||
installer.set_environment()
|
||||
if args.uv:
|
||||
installer.install("uv", "uv")
|
||||
@@ -239,7 +240,7 @@ def main():
|
||||
installer.install_extensions()
|
||||
installer.install_requirements() # redo requirements since extensions may change them
|
||||
installer.update_wiki()
|
||||
if installer.errors == 0:
|
||||
if len(installer.errors) == 0:
|
||||
installer.log.debug(f'Setup complete without errors: {round(time.time())}')
|
||||
else:
|
||||
installer.log.warning(f'Setup complete with errors: {installer.errors}')
|
||||
@@ -257,9 +258,7 @@ def main():
|
||||
alive = False
|
||||
requests = 0
|
||||
if round(time.time()) % 120 == 0:
|
||||
state = f'job="{instance.state.job}" {instance.state.job_no}/{instance.state.job_count}' if instance.state.job != '' or instance.state.job_no != 0 or instance.state.job_count != 0 else 'idle'
|
||||
uptime = round(time.time() - instance.state.server_start)
|
||||
installer.log.debug(f'Server: alive={alive} jobs={instance.state.total_jobs} requests={requests} uptime={uptime} memory={get_memory_stats()} backend={instance.backend} state={state}')
|
||||
installer.log.debug(f'Server: alive={alive} requests={requests} memory={get_memory_stats()} {instance.state.status()}')
|
||||
if not alive:
|
||||
if uv is not None and uv.wants_restart:
|
||||
installer.log.info('Server restarting...')
|
||||
|
||||
+6
-3
@@ -5,7 +5,7 @@ from fastapi import FastAPI, APIRouter, Depends, Request
|
||||
from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
||||
from fastapi.exceptions import HTTPException
|
||||
from modules import errors, shared, postprocessing
|
||||
from modules.api import models, endpoints, script, helpers, server, nvml, generate, process, control, gallery
|
||||
from modules.api import models, endpoints, script, helpers, server, nvml, generate, process, control, gallery, docs
|
||||
|
||||
|
||||
errors.install()
|
||||
@@ -23,8 +23,10 @@ class Api:
|
||||
for line in file.readlines():
|
||||
user, password = line.split(":")
|
||||
self.credentials[user.replace('"', '').strip()] = password.replace('"', '').strip()
|
||||
|
||||
self.router = APIRouter()
|
||||
if shared.cmd_opts.docs:
|
||||
docs.create_docs(app)
|
||||
docs.create_redocs(app)
|
||||
self.app = app
|
||||
self.queue_lock = queue_lock
|
||||
self.generate = generate.APIGenerate(queue_lock)
|
||||
@@ -36,6 +38,7 @@ class Api:
|
||||
self.add_api_route("/sdapi/v1/log", server.get_log_buffer, methods=["GET"], response_model=List[str])
|
||||
self.add_api_route("/sdapi/v1/start", self.get_session_start, methods=["GET"])
|
||||
self.add_api_route("/sdapi/v1/version", server.get_version, methods=["GET"])
|
||||
self.add_api_route("/sdapi/v1/status", server.get_status, methods=["GET"], response_model=models.ResStatus)
|
||||
self.add_api_route("/sdapi/v1/platform", server.get_platform, methods=["GET"])
|
||||
self.add_api_route("/sdapi/v1/progress", server.get_progress, methods=["GET"], response_model=models.ResProgress)
|
||||
self.add_api_route("/sdapi/v1/interrupt", server.post_interrupt, methods=["POST"])
|
||||
@@ -55,7 +58,7 @@ class Api:
|
||||
self.add_api_route("/sdapi/v1/extra-batch-images", self.extras_batch_images_api, methods=["POST"], response_model=models.ResProcessBatch)
|
||||
self.add_api_route("/sdapi/v1/preprocess", self.process.post_preprocess, methods=["POST"])
|
||||
self.add_api_route("/sdapi/v1/mask", self.process.post_mask, methods=["POST"])
|
||||
self.add_api_route("/sdapi/v1/faces", self.process.post_face, methods=["POST"])
|
||||
self.add_api_route("/sdapi/v1/detect", self.process.post_detect, methods=["POST"])
|
||||
|
||||
# api dealing with optional scripts
|
||||
self.add_api_route("/sdapi/v1/scripts", script.get_scripts_list, methods=["GET"], response_model=models.ResScripts)
|
||||
|
||||
@@ -31,6 +31,7 @@ ReqControl = models.create_model_from_signature(
|
||||
{"key": "ip_adapter", "type": Optional[List[models.ItemIPAdapter]], "default": None, "exclude": True},
|
||||
{"key": "face", "type": Optional[models.ItemFace], "default": None, "exclude": True},
|
||||
{"key": "control", "type": Optional[List[ItemControl]], "default": [], "exclude": True},
|
||||
{"key": "extra", "type": Optional[dict], "default": {}, "exclude": True},
|
||||
]
|
||||
)
|
||||
|
||||
@@ -103,7 +104,7 @@ class APIControl():
|
||||
args['ip_adapter_scales'].append(ipadapter.scale)
|
||||
args['ip_adapter_starts'].append(ipadapter.start)
|
||||
args['ip_adapter_ends'].append(ipadapter.end)
|
||||
args['ip_adapter_crops'].append(ipadapter.end)
|
||||
args['ip_adapter_crops'].append(ipadapter.crop)
|
||||
args['ip_adapter_images'].append([helpers.decode_base64_to_image(x) for x in ipadapter.images])
|
||||
if ipadapter.masks:
|
||||
args['ip_adapter_masks'].append([helpers.decode_base64_to_image(x) for x in ipadapter.masks])
|
||||
@@ -159,6 +160,7 @@ class APIControl():
|
||||
output_processed = []
|
||||
output_info = ''
|
||||
run.control_set({ 'do_not_save_grid': not req.save_images, 'do_not_save_samples': not req.save_images, **self.prepare_ip_adapter(req) })
|
||||
run.control_set(getattr(req, "extra", {}))
|
||||
res = run.control_run(**args)
|
||||
for item in res:
|
||||
if len(item) > 0 and (isinstance(item[0], list) or item[0] is None): # output_images
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import json
|
||||
from starlette.responses import HTMLResponse
|
||||
from fastapi import FastAPI
|
||||
from fastapi.openapi.docs import get_redoc_html, swagger_ui_default_parameters
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
|
||||
|
||||
def get_swagger_ui_html(*,
|
||||
openapi_url: str,
|
||||
title: str,
|
||||
swagger_js_url: str = "https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui-bundle.js",
|
||||
swagger_css_url: str = "https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui.css",
|
||||
swagger_extra_css_url: str = None,
|
||||
swagger_favicon_url: str = "https://fastapi.tiangolo.com/img/favicon.png",
|
||||
oauth2_redirect_url: str = None,
|
||||
init_oauth: dict = None,
|
||||
swagger_ui_parameters: dict = None,
|
||||
) -> HTMLResponse:
|
||||
current_swagger_ui_parameters = swagger_ui_default_parameters.copy()
|
||||
if swagger_ui_parameters:
|
||||
current_swagger_ui_parameters.update(swagger_ui_parameters)
|
||||
html = f"""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<link type="text/css" rel="stylesheet" href="{swagger_css_url}">
|
||||
<link rel="shortcut icon" href="{swagger_favicon_url}">
|
||||
<title>{title}</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="swagger-ui"></div>
|
||||
<script src="{swagger_js_url}"></script>
|
||||
<script>
|
||||
const ui = SwaggerUIBundle({{
|
||||
url: '{openapi_url}',
|
||||
"""
|
||||
if swagger_extra_css_url is not None:
|
||||
html = html.replace('</head>', f'<link type="text/css" rel="stylesheet" href="{swagger_extra_css_url}"></head>')
|
||||
for key, value in current_swagger_ui_parameters.items():
|
||||
html += f"{json.dumps(key)}: {json.dumps(jsonable_encoder(value))},\n"
|
||||
if oauth2_redirect_url:
|
||||
html += f"oauth2RedirectUrl: window.location.origin + '{oauth2_redirect_url}',"
|
||||
html += """
|
||||
presets: [
|
||||
SwaggerUIBundle.presets.apis,
|
||||
SwaggerUIBundle.SwaggerUIStandalonePreset
|
||||
],
|
||||
})"""
|
||||
if init_oauth:
|
||||
html += f"""
|
||||
ui.initOAuth({json.dumps(jsonable_encoder(init_oauth))})
|
||||
"""
|
||||
html += """
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
return HTMLResponse(html)
|
||||
|
||||
|
||||
def create_docs(app: FastAPI):
|
||||
swagger_ui_parameters = {
|
||||
"displayOperationId": True,
|
||||
"layout": "BaseLayout",
|
||||
"showExtensions": True,
|
||||
"showCommonExtensions": True,
|
||||
"deepLinking": False,
|
||||
"dom_id": "#swagger-ui",
|
||||
}
|
||||
|
||||
@app.get("/docs", include_in_schema=True)
|
||||
async def custom_swagger_html():
|
||||
res = get_swagger_ui_html(
|
||||
title=f'{app.title}: Swagger UI',
|
||||
openapi_url=app.openapi_url,
|
||||
swagger_favicon_url='/file=html/favicon.svg',
|
||||
swagger_ui_parameters=swagger_ui_parameters,
|
||||
swagger_extra_css_url='file=html/swagger.css',
|
||||
)
|
||||
# res = inject_css(html.content, 'html/swagger.css')
|
||||
return res
|
||||
|
||||
|
||||
def create_redocs(app: FastAPI):
|
||||
@app.get("/redocs", include_in_schema=True)
|
||||
async def custom_redoc_html():
|
||||
res = get_redoc_html(
|
||||
title=f'{app.title}: ReDoc',
|
||||
openapi_url=app.openapi_url,
|
||||
redoc_favicon_url='/file=html/favicon.svg',
|
||||
)
|
||||
return res
|
||||
+20
-7
@@ -13,6 +13,7 @@ class APIGenerate():
|
||||
self.queue_lock = queue_lock
|
||||
self.default_script_arg_txt2img = []
|
||||
self.default_script_arg_img2img = []
|
||||
self.default_script_arg_control = []
|
||||
|
||||
def sanitize_args(self, args: dict):
|
||||
args = vars(args)
|
||||
@@ -40,7 +41,7 @@ class APIGenerate():
|
||||
sanitize_str(request.script_args)
|
||||
|
||||
def prepare_face_module(self, request):
|
||||
if hasattr(request, "face") and request.face and not request.script_name and (not request.alwayson_scripts or "face" not in request.alwayson_scripts.keys()):
|
||||
if getattr(request, "face", None) is not None and (not request.alwayson_scripts or "face" not in request.alwayson_scripts.keys()):
|
||||
request.script_name = "face"
|
||||
request.script_args = [
|
||||
request.face.mode,
|
||||
@@ -106,17 +107,23 @@ class APIGenerate():
|
||||
p.scripts = script_runner
|
||||
p.outpath_grids = shared.opts.outdir_grids or shared.opts.outdir_txt2img_grids
|
||||
p.outpath_samples = shared.opts.outdir_samples or shared.opts.outdir_txt2img_samples
|
||||
for key, value in getattr(txt2imgreq, "extra", {}).items():
|
||||
setattr(p, key, value)
|
||||
shared.state.begin('API TXT', api=True)
|
||||
script_args = script.init_script_args(p, txt2imgreq, self.default_script_arg_txt2img, selectable_scripts, selectable_script_idx, script_runner)
|
||||
p.script_args = tuple(script_args) # Need to pass args as tuple here
|
||||
if selectable_scripts is not None:
|
||||
processed = scripts.scripts_txt2img.run(p, *script_args) # Need to pass args as list here
|
||||
else:
|
||||
p.script_args = tuple(script_args) # Need to pass args as tuple here
|
||||
processed = process_images(p)
|
||||
shared.state.end(api=False)
|
||||
b64images = list(map(helpers.encode_pil_to_base64, processed.images)) if send_images else []
|
||||
if processed is None or processed.images is None or len(processed.images) == 0:
|
||||
b64images = []
|
||||
else:
|
||||
b64images = list(map(helpers.encode_pil_to_base64, processed.images)) if send_images else []
|
||||
self.sanitize_b64(txt2imgreq)
|
||||
return models.ResTxt2Img(images=b64images, parameters=vars(txt2imgreq), info=processed.js())
|
||||
info = processed.js() if processed else ''
|
||||
return models.ResTxt2Img(images=b64images, parameters=vars(txt2imgreq), info=info)
|
||||
|
||||
def post_img2img(self, img2imgreq: models.ReqImg2Img):
|
||||
self.prepare_face_module(img2imgreq)
|
||||
@@ -150,17 +157,23 @@ class APIGenerate():
|
||||
p.scripts = script_runner
|
||||
p.outpath_grids = shared.opts.outdir_img2img_grids
|
||||
p.outpath_samples = shared.opts.outdir_img2img_samples
|
||||
for key, value in getattr(img2imgreq, "extra", {}).items():
|
||||
setattr(p, key, value)
|
||||
shared.state.begin('API-IMG', api=True)
|
||||
script_args = script.init_script_args(p, img2imgreq, self.default_script_arg_img2img, selectable_scripts, selectable_script_idx, script_runner)
|
||||
p.script_args = tuple(script_args) # Need to pass args as tuple here
|
||||
if selectable_scripts is not None:
|
||||
processed = scripts.scripts_img2img.run(p, *script_args) # Need to pass args as list here
|
||||
else:
|
||||
p.script_args = tuple(script_args) # Need to pass args as tuple here
|
||||
processed = process_images(p)
|
||||
shared.state.end(api=False)
|
||||
b64images = list(map(helpers.encode_pil_to_base64, processed.images)) if send_images else []
|
||||
if processed is None or processed.images is None or len(processed.images) == 0:
|
||||
b64images = []
|
||||
else:
|
||||
b64images = list(map(helpers.encode_pil_to_base64, processed.images)) if send_images else []
|
||||
if not img2imgreq.include_init_images:
|
||||
img2imgreq.init_images = None
|
||||
img2imgreq.mask = None
|
||||
self.sanitize_b64(img2imgreq)
|
||||
return models.ResImg2Img(images=b64images, parameters=vars(img2imgreq), info=processed.js())
|
||||
info = processed.js() if processed else ''
|
||||
return models.ResImg2Img(images=b64images, parameters=vars(img2imgreq), info=info)
|
||||
|
||||
@@ -14,15 +14,21 @@ def validate_sampler_name(name):
|
||||
return name
|
||||
|
||||
|
||||
def decode_base64_to_image(encoding):
|
||||
def decode_base64_to_image(encoding, quiet=False):
|
||||
if encoding.startswith("data:image/"):
|
||||
encoding = encoding.split(";")[1].split(",")[1]
|
||||
try:
|
||||
image = Image.open(io.BytesIO(base64.b64decode(encoding)))
|
||||
decoded = base64.b64decode(encoding)
|
||||
data = io.BytesIO(decoded)
|
||||
image = Image.open(data)
|
||||
return image
|
||||
except Exception as e:
|
||||
shared.log.warning(f'API cannot decode image: {e}')
|
||||
raise HTTPException(status_code=500, detail="Invalid encoded image") from e
|
||||
from modules import errors
|
||||
errors.display(e, 'API cannot decode image')
|
||||
if not quiet:
|
||||
raise HTTPException(status_code=500, detail="Invalid encoded image") from e
|
||||
return None
|
||||
|
||||
|
||||
def encode_pil_to_base64(image):
|
||||
|
||||
@@ -90,4 +90,4 @@ def setup_middleware(app: FastAPI, cmd_opts):
|
||||
return handle_exception(req, e)
|
||||
|
||||
app.build_middleware_stack() # rebuild middleware stack on-the-fly
|
||||
log.debug(f'FastAPI middleware: {[m.__class__.__name__ for m in app.user_middleware]}')
|
||||
log.debug(f'API middleware: {[m.cls for m in app.user_middleware]}')
|
||||
|
||||
+28
-11
@@ -11,14 +11,6 @@ API_NOT_ALLOWED = [
|
||||
"sd_model",
|
||||
"outpath_samples",
|
||||
"outpath_grids",
|
||||
"sampler_index",
|
||||
"extra_generation_params",
|
||||
"overlay_images",
|
||||
"do_not_reload_embeddings",
|
||||
"seed_enable_extras",
|
||||
"prompt_for_display",
|
||||
"sampler_noise_scheduler_override",
|
||||
"ddim_discretize"
|
||||
]
|
||||
|
||||
class ModelDef(BaseModel):
|
||||
@@ -202,14 +194,17 @@ ReqTxt2Img = PydanticModelGenerator(
|
||||
"StableDiffusionProcessingTxt2Img",
|
||||
StableDiffusionProcessingTxt2Img,
|
||||
[
|
||||
{"key": "sampler_index", "type": str, "default": "UniPC"},
|
||||
{"key": "script_name", "type": str, "default": None},
|
||||
{"key": "sampler_index", "type": int, "default": 0},
|
||||
{"key": "sampler_name", "type": str, "default": "UniPC"},
|
||||
{"key": "hr_sampler_name", "type": str, "default": "Same as primary"},
|
||||
{"key": "script_name", "type": str, "default": "none"},
|
||||
{"key": "script_args", "type": list, "default": []},
|
||||
{"key": "send_images", "type": bool, "default": True},
|
||||
{"key": "save_images", "type": bool, "default": False},
|
||||
{"key": "alwayson_scripts", "type": dict, "default": {}},
|
||||
{"key": "ip_adapter", "type": Optional[List[ItemIPAdapter]], "default": None, "exclude": True},
|
||||
{"key": "face", "type": Optional[ItemFace], "default": None, "exclude": True},
|
||||
{"key": "extra", "type": Optional[dict], "default": {}, "exclude": True},
|
||||
]
|
||||
).generate_model()
|
||||
StableDiffusionTxt2ImgProcessingAPI = ReqTxt2Img
|
||||
@@ -223,7 +218,11 @@ ReqImg2Img = PydanticModelGenerator(
|
||||
"StableDiffusionProcessingImg2Img",
|
||||
StableDiffusionProcessingImg2Img,
|
||||
[
|
||||
{"key": "sampler_index", "type": str, "default": "UniPC"},
|
||||
{"key": "sampler_index", "type": int, "default": 0},
|
||||
{"key": "sampler_name", "type": str, "default": "UniPC"},
|
||||
{"key": "hr_sampler_name", "type": str, "default": "Same as primary"},
|
||||
{"key": "script_name", "type": str, "default": "none"},
|
||||
{"key": "script_args", "type": list, "default": []},
|
||||
{"key": "init_images", "type": list, "default": None},
|
||||
{"key": "denoising_strength", "type": float, "default": 0.5},
|
||||
{"key": "mask", "type": str, "default": None},
|
||||
@@ -235,6 +234,7 @@ ReqImg2Img = PydanticModelGenerator(
|
||||
{"key": "alwayson_scripts", "type": dict, "default": {}},
|
||||
{"key": "ip_adapter", "type": Optional[List[ItemIPAdapter]], "default": None, "exclude": True},
|
||||
{"key": "face_id", "type": Optional[ItemFace], "default": None, "exclude": True},
|
||||
{"key": "extra", "type": Optional[dict], "default": {}, "exclude": True},
|
||||
]
|
||||
).generate_model()
|
||||
StableDiffusionImg2ImgProcessingAPI = ReqImg2Img
|
||||
@@ -300,6 +300,23 @@ class ResProgress(BaseModel):
|
||||
current_image: str = Field(default=None, title="Current image", description="The current image in base64 format. opts.show_progress_every_n_steps is required for this to work.")
|
||||
textinfo: str = Field(default=None, title="Info text", description="Info text used by WebUI.")
|
||||
|
||||
class ResStatus(BaseModel):
|
||||
status: str = Field(title="Status", description="Current status")
|
||||
task: str = Field(title="Task", description="Current task")
|
||||
timestamp: Optional[str] = Field(title="Timestamp", description="Timestamp of the current job")
|
||||
id: str = Field(title="ID", description="ID of the current task")
|
||||
job: int = Field(title="Job", description="Current job")
|
||||
jobs: int = Field(title="Jobs", description="Total jobs")
|
||||
total: int = Field(title="Total Jobs", description="Total jobs")
|
||||
step: int = Field(title="Step", description="Current step")
|
||||
steps: int = Field(title="Steps", description="Total steps")
|
||||
queued: int = Field(title="Queued", description="Number of queued tasks")
|
||||
uptime: int = Field(title="Uptime", description="Uptime of the server")
|
||||
elapsed: Optional[float] = Field(title="Elapsed time")
|
||||
eta: Optional[float] = Field(title="ETA in secs")
|
||||
progress: Optional[float] = Field(title="Progress", description="The progress with a range of 0 to 1")
|
||||
|
||||
|
||||
class ReqInterrogate(BaseModel):
|
||||
image: str = Field(default="", title="Image", description="Image to work on, must be a Base64 string containing the image's data.")
|
||||
clip_model: str = Field(default="", title="CLiP Model", description="The interrogate model used.")
|
||||
|
||||
+17
-7
@@ -28,8 +28,12 @@ class ReqMask(BaseModel):
|
||||
|
||||
class ReqFace(BaseModel):
|
||||
image: str = Field(title="Image", description="The base64 encoded image")
|
||||
model: Optional[str] = Field(title="Model", description="The model to use for detection")
|
||||
|
||||
class ResFace(BaseModel):
|
||||
classes: List[int] = Field(title="Class", description="The class of detected item")
|
||||
labels: List[str] = Field(title="Label", description="The label of detected item")
|
||||
boxes: List[List[int]] = Field(title="Box", description="The bounding box of detected item")
|
||||
images: List[str] = Field(title="Image", description="The base64 encoded images of detected faces")
|
||||
scores: List[float] = Field(title="Scores", description="The scores of the detected faces")
|
||||
|
||||
@@ -106,16 +110,22 @@ class APIProcess():
|
||||
image = encode_pil_to_base64(processed)
|
||||
return ResMask(mask=image)
|
||||
|
||||
def post_face(self, req: ReqFace):
|
||||
from shared import yolo # pylint: disable=no-name-in-module
|
||||
def post_detect(self, req: ReqFace):
|
||||
from modules.shared import yolo # pylint: disable=no-name-in-module
|
||||
image = decode_base64_to_image(req.image)
|
||||
shared.state.begin('API-FACE', api=True)
|
||||
images = []
|
||||
scores = []
|
||||
classes = []
|
||||
boxes = []
|
||||
labels = []
|
||||
with self.queue_lock:
|
||||
faces = yolo.predict('face-yolo8n', image)
|
||||
for face in faces:
|
||||
images.append(encode_pil_to_base64(face.item))
|
||||
scores.append(face.score)
|
||||
items = yolo.predict(req.model, image)
|
||||
for item in items:
|
||||
images.append(encode_pil_to_base64(item.item))
|
||||
scores.append(item.score)
|
||||
classes.append(item.cls)
|
||||
labels.append(item.label)
|
||||
boxes.append(item.box)
|
||||
shared.state.end(api=False)
|
||||
return ResFace(images=images, scores=scores)
|
||||
return ResFace(classes=classes, labels=labels, scores=scores, boxes=boxes, images=images)
|
||||
|
||||
+25
-7
@@ -3,27 +3,39 @@ from fastapi.exceptions import HTTPException
|
||||
import gradio as gr
|
||||
from modules.api import models
|
||||
from modules import scripts
|
||||
from modules.errors import log
|
||||
|
||||
|
||||
def script_name_to_index(name, scripts_list):
|
||||
try:
|
||||
return [script.title().lower() for script in scripts_list].index(name.lower())
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=422, detail=f"Script '{name}' not found") from e
|
||||
if name is None or len(name) == 0 or name == 'none':
|
||||
return None
|
||||
available = [script.title().lower() for script in scripts_list]
|
||||
if name.lower() in available:
|
||||
return available.index(name.lower())
|
||||
short = [available.split(':')[0] for available in available]
|
||||
if name.lower() in short:
|
||||
return short.index(name.lower())
|
||||
log.error(f'API: script={name} available={available} not found')
|
||||
return None
|
||||
|
||||
|
||||
def get_selectable_script(script_name, script_runner):
|
||||
if script_name is None or script_name == "":
|
||||
if script_name is None or script_name == "" or script_name == 'none':
|
||||
return None, None
|
||||
script_idx = script_name_to_index(script_name, script_runner.selectable_scripts)
|
||||
if script_idx is None:
|
||||
return None, None
|
||||
script = script_runner.selectable_scripts[script_idx]
|
||||
return script, script_idx
|
||||
|
||||
|
||||
def get_scripts_list():
|
||||
t2ilist = [script.name for script in scripts.scripts_txt2img.scripts if script.name is not None]
|
||||
i2ilist = [script.name for script in scripts.scripts_img2img.scripts if script.name is not None]
|
||||
control = [script.name for script in scripts.scripts_control.scripts if script.name is not None]
|
||||
return models.ResScripts(txt2img = t2ilist, img2img = i2ilist, control = control)
|
||||
|
||||
|
||||
def get_script_info(script_name: Optional[str] = None):
|
||||
res = []
|
||||
for script_list in [scripts.scripts_txt2img.scripts, scripts.scripts_img2img.scripts, scripts.scripts_control.scripts]:
|
||||
@@ -32,12 +44,16 @@ def get_script_info(script_name: Optional[str] = None):
|
||||
res.append(script.api_info)
|
||||
return res
|
||||
|
||||
|
||||
def get_script(script_name, script_runner):
|
||||
if script_name is None or script_name == "":
|
||||
if script_name is None or script_name == "" or script_name == 'none':
|
||||
return None, None
|
||||
script_idx = script_name_to_index(script_name, script_runner.scripts)
|
||||
if script_idx is None:
|
||||
return None
|
||||
return script_runner.scripts[script_idx]
|
||||
|
||||
|
||||
def init_default_script_args(script_runner):
|
||||
# find max idx from the scripts in runner and generate a none array to init script_args
|
||||
last_arg_index = 1
|
||||
@@ -60,11 +76,13 @@ def init_default_script_args(script_runner):
|
||||
script_args[script.args_from:script.args_to] = ui_default_values
|
||||
return script_args
|
||||
|
||||
|
||||
def init_script_args(p, request, default_script_args, selectable_scripts, selectable_script_idx, script_runner):
|
||||
script_args = default_script_args.copy()
|
||||
# position 0 in script_arg is the idx+1 of the selectable script that is going to be run when using scripts.scripts_*2img.run()
|
||||
if selectable_scripts:
|
||||
script_args[selectable_scripts.args_from:selectable_scripts.args_to] = request.script_args
|
||||
for idx in range(len(request.script_args)):
|
||||
script_args[selectable_scripts.args_from + idx] = request.script_args[idx]
|
||||
script_args[0] = selectable_script_idx + 1
|
||||
# Now check for always on scripts
|
||||
if request.alwayson_scripts and (len(request.alwayson_scripts) > 0):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import time
|
||||
from typing import Any, Dict
|
||||
from fastapi import Depends
|
||||
from modules import shared
|
||||
@@ -66,7 +67,6 @@ def get_cmd_flags():
|
||||
return vars(shared.cmd_opts)
|
||||
|
||||
def get_progress(req: models.ReqProgress = Depends()):
|
||||
import time
|
||||
if shared.state.job_count == 0:
|
||||
return models.ResProgress(progress=0, eta_relative=0, state=shared.state.dict(), textinfo=shared.state.textinfo)
|
||||
shared.state.do_set_current_image()
|
||||
@@ -85,6 +85,9 @@ def get_progress(req: models.ReqProgress = Depends()):
|
||||
res = models.ResProgress(progress=progress, eta_relative=eta_relative, state=shared.state.dict(), current_image=current_image, textinfo=shared.state.textinfo)
|
||||
return res
|
||||
|
||||
def get_status():
|
||||
return shared.state.status()
|
||||
|
||||
def post_interrupt():
|
||||
shared.state.interrupt()
|
||||
return {}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
"""
|
||||
original code from <https://github.com/NVlabs/consistory>
|
||||
"""
|
||||
from .consistory_pipeline import ConsistoryExtendAttnSDXLPipeline
|
||||
from .consistory_unet_sdxl import ConsistorySDXLUNet2DConditionModel
|
||||
from .consistory_run import run_anchor_generation, run_extra_generation
|
||||
@@ -0,0 +1,287 @@
|
||||
# Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Not a contribution
|
||||
# Changes made by NVIDIA CORPORATION & AFFILIATES enabling ConsiStory or otherwise documented as NVIDIA-proprietary
|
||||
# are not a contribution and subject to the license under the LICENSE file located at the root directory.
|
||||
|
||||
|
||||
from typing import Callable, Optional
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from diffusers.utils import USE_PEFT_BACKEND
|
||||
from diffusers.models.attention_processor import Attention
|
||||
from .consistory_utils import AnchorCache, FeatureInjector, QueryStore
|
||||
|
||||
|
||||
class ConsistoryAttnStoreProcessor:
|
||||
def __init__(self, attnstore, place_in_unet):
|
||||
super().__init__()
|
||||
self.attnstore = attnstore
|
||||
self.place_in_unet = place_in_unet
|
||||
|
||||
def __call__(self, attn: Attention, hidden_states, encoder_hidden_states=None, attention_mask=None, record_attention=True, **kwargs):
|
||||
batch_size, sequence_length, _ = hidden_states.shape
|
||||
attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
|
||||
|
||||
query = attn.to_q(hidden_states)
|
||||
|
||||
is_cross = encoder_hidden_states is not None
|
||||
encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states
|
||||
key = attn.to_k(encoder_hidden_states)
|
||||
value = attn.to_v(encoder_hidden_states)
|
||||
|
||||
query = attn.head_to_batch_dim(query)
|
||||
key = attn.head_to_batch_dim(key)
|
||||
value = attn.head_to_batch_dim(value)
|
||||
|
||||
attention_probs = attn.get_attention_scores(query, key, attention_mask)
|
||||
|
||||
# only need to store attention maps during the Attend and Excite process
|
||||
# if attention_probs.requires_grad:
|
||||
if record_attention:
|
||||
self.attnstore(attention_probs, is_cross, self.place_in_unet, attn.heads)
|
||||
|
||||
hidden_states = torch.bmm(attention_probs, value)
|
||||
hidden_states = attn.batch_to_head_dim(hidden_states)
|
||||
|
||||
# linear proj
|
||||
hidden_states = attn.to_out[0](hidden_states)
|
||||
# dropout
|
||||
hidden_states = attn.to_out[1](hidden_states)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
class ConsistoryExtendedAttnXFormersAttnProcessor:
|
||||
r"""
|
||||
Processor for implementing memory efficient attention using xFormers.
|
||||
|
||||
Args:
|
||||
attention_op (`Callable`, *optional*, defaults to `None`):
|
||||
The base
|
||||
[operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to
|
||||
use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best
|
||||
operator.
|
||||
"""
|
||||
|
||||
def __init__(self, place_in_unet, attnstore, extended_attn_kwargs, attention_op: Optional[Callable] = None):
|
||||
self.attention_op = attention_op
|
||||
self.t_range = extended_attn_kwargs.get('t_range', [])
|
||||
self.extend_kv_unet_parts = extended_attn_kwargs.get('extend_kv_unet_parts', ['down', 'mid', 'up'])
|
||||
|
||||
self.place_in_unet = place_in_unet
|
||||
self.curr_unet_part = self.place_in_unet.split('_')[0]
|
||||
self.attnstore = attnstore
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
attn: Attention,
|
||||
hidden_states: torch.FloatTensor,
|
||||
encoder_hidden_states: Optional[torch.FloatTensor] = None,
|
||||
attention_mask: Optional[torch.FloatTensor] = None,
|
||||
temb: Optional[torch.FloatTensor] = None,
|
||||
scale: float = 1.0,
|
||||
perform_extend_attn: bool = False,
|
||||
query_store: Optional[QueryStore] = None,
|
||||
feature_injector: Optional[FeatureInjector] = None,
|
||||
anchors_cache: Optional[AnchorCache] = None,
|
||||
**kwargs
|
||||
) -> torch.FloatTensor:
|
||||
residual = hidden_states
|
||||
|
||||
args = () if USE_PEFT_BACKEND else (scale,)
|
||||
|
||||
if attn.spatial_norm is not None:
|
||||
hidden_states = attn.spatial_norm(hidden_states, temb)
|
||||
|
||||
input_ndim = hidden_states.ndim
|
||||
|
||||
if input_ndim == 4:
|
||||
batch_size, channel, height, width = hidden_states.shape
|
||||
hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
|
||||
else:
|
||||
batch_size, wh, channel = hidden_states.shape
|
||||
height = width = int(wh ** 0.5)
|
||||
|
||||
is_cross = encoder_hidden_states is not None
|
||||
perform_extend_attn = perform_extend_attn and (not is_cross) and \
|
||||
any([self.attnstore.curr_iter >= x[0] and self.attnstore.curr_iter <= x[1] for x in self.t_range]) and \
|
||||
self.curr_unet_part in self.extend_kv_unet_parts
|
||||
|
||||
batch_size, key_tokens, _ = (
|
||||
hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
|
||||
)
|
||||
|
||||
attention_mask = attn.prepare_attention_mask(attention_mask, key_tokens, batch_size)
|
||||
if attention_mask is not None:
|
||||
# expand our mask's singleton query_tokens dimension:
|
||||
# [batch*heads, 1, key_tokens] ->
|
||||
# [batch*heads, query_tokens, key_tokens]
|
||||
# so that it can be added as a bias onto the attention scores that xformers computes:
|
||||
# [batch*heads, query_tokens, key_tokens]
|
||||
# we do this explicitly because xformers doesn't broadcast the singleton dimension for us.
|
||||
_, query_tokens, _ = hidden_states.shape
|
||||
attention_mask = attention_mask.expand(-1, query_tokens, -1)
|
||||
|
||||
if attn.group_norm is not None:
|
||||
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
|
||||
|
||||
query = attn.to_q(hidden_states, *args)
|
||||
|
||||
if (self.curr_unet_part in self.extend_kv_unet_parts) and query_store and query_store.mode == 'cache':
|
||||
query_store.cache_query(query, self.place_in_unet)
|
||||
elif perform_extend_attn and query_store and query_store.mode == 'inject':
|
||||
query = query_store.inject_query(query, self.place_in_unet, self.attnstore.curr_iter)
|
||||
|
||||
if encoder_hidden_states is None:
|
||||
encoder_hidden_states = hidden_states
|
||||
elif attn.norm_cross:
|
||||
encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
|
||||
|
||||
key = attn.to_k(encoder_hidden_states, *args)
|
||||
value = attn.to_v(encoder_hidden_states, *args)
|
||||
|
||||
query = attn.head_to_batch_dim(query).contiguous()
|
||||
|
||||
if perform_extend_attn:
|
||||
# Anchor Caching
|
||||
if anchors_cache and anchors_cache.is_cache_mode():
|
||||
if self.place_in_unet not in anchors_cache.input_h_cache:
|
||||
anchors_cache.input_h_cache[self.place_in_unet] = {}
|
||||
|
||||
# Hidden states inside the mask, for uncond (index 0) and cond (index 1) prompts
|
||||
subjects_hidden_states = torch.stack([x[self.attnstore.last_mask_dropout[width]] for x in hidden_states.chunk(2)])
|
||||
anchors_cache.input_h_cache[self.place_in_unet][self.attnstore.curr_iter] = subjects_hidden_states
|
||||
|
||||
if anchors_cache and anchors_cache.is_inject_mode():
|
||||
# We make extended key and value by concatenating the original key and value with the query.
|
||||
anchors_hidden_states = anchors_cache.input_h_cache[self.place_in_unet][self.attnstore.curr_iter]
|
||||
|
||||
anchors_keys = attn.to_k(anchors_hidden_states, *args)
|
||||
anchors_values = attn.to_v(anchors_hidden_states, *args)
|
||||
|
||||
extended_key = torch.cat([torch.cat([key.chunk(2, dim=0)[x], anchors_keys[x].unsqueeze(0)], dim=1) for x in range(2)])
|
||||
extended_value = torch.cat([torch.cat([value.chunk(2, dim=0)[x], anchors_values[x].unsqueeze(0)], dim=1) for x in range(2)])
|
||||
|
||||
extended_key = attn.head_to_batch_dim(extended_key).contiguous()
|
||||
extended_value = attn.head_to_batch_dim(extended_value).contiguous()
|
||||
|
||||
# attn_masks needs to be of shape [batch_size, query_tokens, key_tokens]
|
||||
# hidden_states = xformers.ops.memory_efficient_attention(query, extended_key, extended_value, op=self.attention_op, scale=attn.scale)
|
||||
hidden_states = F.scaled_dot_product_attention(query, extended_key, extended_value, scale=attn.scale)
|
||||
else:
|
||||
# # We make extended key and value by concatenating the original key and value with the query.
|
||||
# attention_mask_bias = self.attnstore.get_attn_mask_bias(tgt_size = width, bsz = batch_size)
|
||||
|
||||
# if attention_mask_bias is not None:
|
||||
# attention_mask_bias = torch.cat([x.unsqueeze(0).expand(attn.heads, -1, -1) for x in attention_mask_bias])
|
||||
|
||||
# Pre-allocate the output tensor
|
||||
ex_out = torch.empty_like(query)
|
||||
|
||||
for i in range(batch_size):
|
||||
start_idx = i * attn.heads
|
||||
end_idx = start_idx + attn.heads
|
||||
|
||||
attention_mask = self.attnstore.get_extended_attn_mask_instance(width, i%(batch_size//2))
|
||||
|
||||
curr_q = query[start_idx:end_idx]
|
||||
|
||||
if i < batch_size//2:
|
||||
curr_k = key[:batch_size//2]
|
||||
curr_v = value[:batch_size//2]
|
||||
else:
|
||||
curr_k = key[batch_size//2:]
|
||||
curr_v = value[batch_size//2:]
|
||||
|
||||
curr_k = curr_k.flatten(0,1)[attention_mask].unsqueeze(0)
|
||||
curr_v = curr_v.flatten(0,1)[attention_mask].unsqueeze(0)
|
||||
|
||||
curr_k = attn.head_to_batch_dim(curr_k).contiguous()
|
||||
curr_v = attn.head_to_batch_dim(curr_v).contiguous()
|
||||
|
||||
# hidden_states = xformers.ops.memory_efficient_attention(curr_q, curr_k, curr_v, op=self.attention_op, scale=attn.scale)
|
||||
hidden_states = F.scaled_dot_product_attention(curr_q, curr_k, curr_v, scale=attn.scale)
|
||||
|
||||
ex_out[start_idx:end_idx] = hidden_states
|
||||
|
||||
hidden_states = ex_out
|
||||
else:
|
||||
key = attn.head_to_batch_dim(key).contiguous()
|
||||
value = attn.head_to_batch_dim(value).contiguous()
|
||||
|
||||
# attn_masks needs to be of shape [batch_size, query_tokens, key_tokens]
|
||||
# hidden_states = xformers.ops.memory_efficient_attention(query, key, value, op=self.attention_op, scale=attn.scale)
|
||||
hidden_states = F.scaled_dot_product_attention(query, key, value, scale=attn.scale)
|
||||
|
||||
hidden_states = hidden_states.to(query.dtype)
|
||||
hidden_states = attn.batch_to_head_dim(hidden_states)
|
||||
|
||||
# linear proj
|
||||
hidden_states = attn.to_out[0](hidden_states, *args)
|
||||
# dropout
|
||||
hidden_states = attn.to_out[1](hidden_states)
|
||||
|
||||
if feature_injector is not None:
|
||||
output_res = int(hidden_states.shape[1] ** 0.5)
|
||||
|
||||
if anchors_cache and anchors_cache.is_inject_mode():
|
||||
hidden_states[batch_size//2:] = feature_injector.inject_anchors(hidden_states[batch_size//2:], self.attnstore.curr_iter, output_res, self.attnstore.extended_mapping, self.place_in_unet, anchors_cache)
|
||||
else:
|
||||
hidden_states[batch_size//2:] = feature_injector.inject_outputs(hidden_states[batch_size//2:], self.attnstore.curr_iter, output_res, self.attnstore.extended_mapping, self.place_in_unet, anchors_cache)
|
||||
|
||||
if input_ndim == 4:
|
||||
hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
|
||||
|
||||
if attn.residual_connection:
|
||||
hidden_states = hidden_states + residual
|
||||
|
||||
hidden_states = hidden_states / attn.rescale_output_factor
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
def register_extended_self_attn(unet, attnstore, extended_attn_kwargs):
|
||||
DICT_PLACE_TO_RES = {'down_0': 64, 'down_1': 64, 'down_2': 64, 'down_3': 64, 'down_4': 64, 'down_5': 64, 'down_6': 64, 'down_7': 64,
|
||||
'down_8': 32, 'down_9': 32, 'down_10': 32, 'down_11': 32, 'down_12': 32, 'down_13': 32, 'down_14': 32, 'down_15': 32,
|
||||
'down_16': 32, 'down_17': 32, 'down_18': 32, 'down_19': 32, 'down_20': 32, 'down_21': 32, 'down_22': 32, 'down_23': 32,
|
||||
'down_24': 32, 'down_25': 32, 'down_26': 32, 'down_27': 32, 'down_28': 32, 'down_29': 32, 'down_30': 32, 'down_31': 32,
|
||||
'down_32': 32, 'down_33': 32, 'down_34': 32, 'down_35': 32, 'down_36': 32, 'down_37': 32, 'down_38': 32, 'down_39': 32,
|
||||
'down_40': 32, 'down_41': 32, 'down_42': 32, 'down_43': 32, 'down_44': 32, 'down_45': 32, 'down_46': 32, 'down_47': 32,
|
||||
'mid_120': 32, 'mid_121': 32, 'mid_122': 32, 'mid_123': 32, 'mid_124': 32, 'mid_125': 32, 'mid_126': 32, 'mid_127': 32,
|
||||
'mid_128': 32, 'mid_129': 32, 'mid_130': 32, 'mid_131': 32, 'mid_132': 32, 'mid_133': 32, 'mid_134': 32, 'mid_135': 32,
|
||||
'mid_136': 32, 'mid_137': 32, 'mid_138': 32, 'mid_139': 32, 'up_49': 32, 'up_51': 32, 'up_53': 32, 'up_55': 32, 'up_57': 32,
|
||||
'up_59': 32, 'up_61': 32, 'up_63': 32, 'up_65': 32, 'up_67': 32, 'up_69': 32, 'up_71': 32, 'up_73': 32, 'up_75': 32,
|
||||
'up_77': 32, 'up_79': 32, 'up_81': 32, 'up_83': 32, 'up_85': 32, 'up_87': 32, 'up_89': 32, 'up_91': 32, 'up_93': 32,
|
||||
'up_95': 32, 'up_97': 32, 'up_99': 32, 'up_101': 32, 'up_103': 32, 'up_105': 32, 'up_107': 32, 'up_109': 64, 'up_111': 64,
|
||||
'up_113': 64, 'up_115': 64, 'up_117': 64, 'up_119': 64}
|
||||
attn_procs = {}
|
||||
for i, name in enumerate(unet.attn_processors.keys()):
|
||||
is_self_attn = i % 2 == 0
|
||||
if name.startswith("mid_block"):
|
||||
place_in_unet = f"mid_{i}"
|
||||
elif name.startswith("up_blocks"):
|
||||
place_in_unet = f"up_{i}"
|
||||
elif name.startswith("down_blocks"):
|
||||
place_in_unet = f"down_{i}"
|
||||
else:
|
||||
continue
|
||||
|
||||
if is_self_attn:
|
||||
attn_procs[name] = ConsistoryExtendedAttnXFormersAttnProcessor(place_in_unet, attnstore, extended_attn_kwargs)
|
||||
else:
|
||||
attn_procs[name] = ConsistoryAttnStoreProcessor(attnstore, place_in_unet)
|
||||
|
||||
unet.set_attn_processor(attn_procs)
|
||||
@@ -0,0 +1,519 @@
|
||||
# Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Not a contribution
|
||||
# Changes made by NVIDIA CORPORATION & AFFILIATES enabling ConsiStory or otherwise documented as NVIDIA-proprietary
|
||||
# are not a contribution and subject to the license under the LICENSE file located at the root directory.
|
||||
|
||||
import torch
|
||||
from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput
|
||||
from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl import StableDiffusionXLPipeline, \
|
||||
rescale_noise_cfg, EXAMPLE_DOC_STRING
|
||||
from diffusers.utils import (
|
||||
deprecate,
|
||||
is_torch_xla_available,
|
||||
logging,
|
||||
replace_example_docstring,
|
||||
)
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from .attention_processor import register_extended_self_attn
|
||||
from .consistory_utils import FeatureInjector, AnchorCache, QueryStore
|
||||
from .utils.ptp_utils import AttentionStore
|
||||
|
||||
if is_torch_xla_available():
|
||||
# import torch_xla.core.xla_model as xm
|
||||
|
||||
XLA_AVAILABLE = True
|
||||
else:
|
||||
XLA_AVAILABLE = False
|
||||
|
||||
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
||||
|
||||
T = torch.Tensor
|
||||
|
||||
class ConsistoryExtendAttnSDXLPipeline(
|
||||
StableDiffusionXLPipeline
|
||||
):
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
@replace_example_docstring(EXAMPLE_DOC_STRING)
|
||||
def __call__(
|
||||
self,
|
||||
prompt: Union[str, List[str]] = None,
|
||||
prompt_2: Optional[Union[str, List[str]]] = None,
|
||||
height: Optional[int] = None,
|
||||
width: Optional[int] = None,
|
||||
num_inference_steps: int = 50,
|
||||
denoising_end: Optional[float] = None,
|
||||
guidance_scale: float = 5.0,
|
||||
negative_prompt: Optional[Union[str, List[str]]] = None,
|
||||
negative_prompt_2: Optional[Union[str, List[str]]] = None,
|
||||
num_images_per_prompt: Optional[int] = 1,
|
||||
eta: float = 0.0,
|
||||
generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
|
||||
latents: Optional[torch.FloatTensor] = None,
|
||||
prompt_embeds: Optional[torch.FloatTensor] = None,
|
||||
negative_prompt_embeds: Optional[torch.FloatTensor] = None,
|
||||
pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
|
||||
negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
|
||||
output_type: Optional[str] = "pil",
|
||||
return_dict: bool = True,
|
||||
cross_attention_kwargs: Optional[Dict[str, Any]] = None,
|
||||
guidance_rescale: float = 0.0,
|
||||
original_size: Optional[Tuple[int, int]] = None,
|
||||
crops_coords_top_left: Tuple[int, int] = (0, 0),
|
||||
target_size: Optional[Tuple[int, int]] = None,
|
||||
negative_original_size: Optional[Tuple[int, int]] = None,
|
||||
negative_crops_coords_top_left: Tuple[int, int] = (0, 0),
|
||||
negative_target_size: Optional[Tuple[int, int]] = None,
|
||||
clip_skip: Optional[int] = None,
|
||||
callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
|
||||
callback_on_step_end_tensor_inputs: List[str] = ["latents"],
|
||||
|
||||
attention_store_kwargs: Optional[Dict] = None,
|
||||
extended_attn_kwargs: Optional[Dict] = None,
|
||||
share_queries: bool = False,
|
||||
query_store_kwargs: Optional[Dict] = {},
|
||||
feature_injector: Optional[FeatureInjector] = None,
|
||||
anchors_cache: Optional[AnchorCache] = None,
|
||||
|
||||
instance_latents: Optional[torch.FloatTensor] = None,
|
||||
**kwargs,
|
||||
):
|
||||
r"""
|
||||
Function invoked when calling the pipeline for generation.
|
||||
|
||||
Args:
|
||||
prompt (`str` or `List[str]`, *optional*):
|
||||
The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
|
||||
instead.
|
||||
prompt_2 (`str` or `List[str]`, *optional*):
|
||||
The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
|
||||
used in both text-encoders
|
||||
height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
|
||||
The height in pixels of the generated image. This is set to 1024 by default for the best results.
|
||||
Anything below 512 pixels won't work well for
|
||||
[stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
|
||||
and checkpoints that are not specifically fine-tuned on low resolutions.
|
||||
width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
|
||||
The width in pixels of the generated image. This is set to 1024 by default for the best results.
|
||||
Anything below 512 pixels won't work well for
|
||||
[stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
|
||||
and checkpoints that are not specifically fine-tuned on low resolutions.
|
||||
num_inference_steps (`int`, *optional*, defaults to 50):
|
||||
The number of denoising steps. More denoising steps usually lead to a higher quality image at the
|
||||
expense of slower inference.
|
||||
denoising_end (`float`, *optional*):
|
||||
When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
|
||||
completed before it is intentionally prematurely terminated. As a result, the returned sample will
|
||||
still retain a substantial amount of noise as determined by the discrete timesteps selected by the
|
||||
scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a
|
||||
"Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image
|
||||
Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output)
|
||||
guidance_scale (`float`, *optional*, defaults to 5.0):
|
||||
Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
|
||||
`guidance_scale` is defined as `w` of equation 2. of [Imagen
|
||||
Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
|
||||
1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
|
||||
usually at the expense of lower image quality.
|
||||
negative_prompt (`str` or `List[str]`, *optional*):
|
||||
The prompt or prompts not to guide the image generation. If not defined, one has to pass
|
||||
`negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
|
||||
less than `1`).
|
||||
negative_prompt_2 (`str` or `List[str]`, *optional*):
|
||||
The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
|
||||
`text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
|
||||
num_images_per_prompt (`int`, *optional*, defaults to 1):
|
||||
The number of images to generate per prompt.
|
||||
eta (`float`, *optional*, defaults to 0.0):
|
||||
Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
|
||||
[`schedulers.DDIMScheduler`], will be ignored for others.
|
||||
generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
|
||||
One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
|
||||
to make generation deterministic.
|
||||
latents (`torch.FloatTensor`, *optional*):
|
||||
Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
|
||||
generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
|
||||
tensor will ge generated by sampling using the supplied random `generator`.
|
||||
prompt_embeds (`torch.FloatTensor`, *optional*):
|
||||
Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
|
||||
provided, text embeddings will be generated from `prompt` input argument.
|
||||
negative_prompt_embeds (`torch.FloatTensor`, *optional*):
|
||||
Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
|
||||
weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
|
||||
argument.
|
||||
pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
|
||||
Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
|
||||
If not provided, pooled text embeddings will be generated from `prompt` input argument.
|
||||
negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
|
||||
Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
|
||||
weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
|
||||
input argument.
|
||||
output_type (`str`, *optional*, defaults to `"pil"`):
|
||||
The output format of the generate image. Choose between
|
||||
[PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
|
||||
return_dict (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead
|
||||
of a plain tuple.
|
||||
cross_attention_kwargs (`dict`, *optional*):
|
||||
A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
|
||||
`self.processor` in
|
||||
[diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
|
||||
guidance_rescale (`float`, *optional*, defaults to 0.0):
|
||||
Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
|
||||
Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of
|
||||
[Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf).
|
||||
Guidance rescale factor should fix overexposure when using zero terminal SNR.
|
||||
original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
|
||||
If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
|
||||
`original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as
|
||||
explained in section 2.2 of
|
||||
[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
|
||||
crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):
|
||||
`crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
|
||||
`crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
|
||||
`crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
|
||||
[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
|
||||
target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
|
||||
For most cases, `target_size` should be set to the desired height and width of the generated image. If
|
||||
not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in
|
||||
section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
|
||||
negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
|
||||
To negatively condition the generation process based on a specific image resolution. Part of SDXL's
|
||||
micro-conditioning as explained in section 2.2 of
|
||||
[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
|
||||
information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
|
||||
negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):
|
||||
To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's
|
||||
micro-conditioning as explained in section 2.2 of
|
||||
[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
|
||||
information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
|
||||
negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
|
||||
To negatively condition the generation process based on a target image resolution. It should be as same
|
||||
as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of
|
||||
[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
|
||||
information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
|
||||
callback_on_step_end (`Callable`, *optional*):
|
||||
A function that calls at the end of each denoising steps during the inference. The function is called
|
||||
with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
|
||||
callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
|
||||
`callback_on_step_end_tensor_inputs`.
|
||||
callback_on_step_end_tensor_inputs (`List`, *optional*):
|
||||
The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
|
||||
will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
|
||||
`._callback_tensor_inputs` attribute of your pipeine class.
|
||||
|
||||
Examples:
|
||||
|
||||
Returns:
|
||||
[`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`:
|
||||
[`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a
|
||||
`tuple`. When returning a tuple, the first element is a list with the generated images.
|
||||
"""
|
||||
callback = kwargs.pop("callback", None)
|
||||
callback_steps = kwargs.pop("callback_steps", None)
|
||||
|
||||
if callback is not None:
|
||||
deprecate(
|
||||
"callback",
|
||||
"1.0.0",
|
||||
"Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
|
||||
)
|
||||
if callback_steps is not None:
|
||||
deprecate(
|
||||
"callback_steps",
|
||||
"1.0.0",
|
||||
"Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
|
||||
)
|
||||
|
||||
# 0. Default height and width to unet
|
||||
height = height or self.default_sample_size * self.vae_scale_factor
|
||||
width = width or self.default_sample_size * self.vae_scale_factor
|
||||
|
||||
original_size = original_size or (height, width)
|
||||
target_size = target_size or (height, width)
|
||||
|
||||
# 1. Check inputs. Raise error if not correct
|
||||
self.check_inputs(
|
||||
prompt,
|
||||
prompt_2,
|
||||
height,
|
||||
width,
|
||||
callback_steps,
|
||||
negative_prompt,
|
||||
negative_prompt_2,
|
||||
prompt_embeds,
|
||||
negative_prompt_embeds,
|
||||
pooled_prompt_embeds,
|
||||
negative_pooled_prompt_embeds,
|
||||
callback_on_step_end_tensor_inputs,
|
||||
)
|
||||
|
||||
self._guidance_scale = guidance_scale
|
||||
self._guidance_rescale = guidance_rescale
|
||||
self._clip_skip = clip_skip
|
||||
self._cross_attention_kwargs = cross_attention_kwargs
|
||||
self._denoising_end = denoising_end
|
||||
|
||||
# 2. Define call parameters
|
||||
if prompt is not None and isinstance(prompt, str):
|
||||
batch_size = 1
|
||||
elif prompt is not None and isinstance(prompt, list):
|
||||
batch_size = len(prompt)
|
||||
else:
|
||||
batch_size = prompt_embeds.shape[0]
|
||||
|
||||
device = self._execution_device
|
||||
|
||||
# 3. Encode input prompt
|
||||
lora_scale = (
|
||||
self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
|
||||
)
|
||||
|
||||
(
|
||||
prompt_embeds,
|
||||
negative_prompt_embeds,
|
||||
pooled_prompt_embeds,
|
||||
negative_pooled_prompt_embeds,
|
||||
) = self.encode_prompt(
|
||||
prompt=prompt,
|
||||
prompt_2=prompt_2,
|
||||
device=device,
|
||||
num_images_per_prompt=num_images_per_prompt,
|
||||
do_classifier_free_guidance=self.do_classifier_free_guidance,
|
||||
negative_prompt=negative_prompt,
|
||||
negative_prompt_2=negative_prompt_2,
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds,
|
||||
pooled_prompt_embeds=pooled_prompt_embeds,
|
||||
negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
|
||||
lora_scale=lora_scale,
|
||||
clip_skip=self.clip_skip,
|
||||
)
|
||||
|
||||
# 4. Prepare timesteps
|
||||
self.scheduler.set_timesteps(num_inference_steps, device=device)
|
||||
|
||||
timesteps = self.scheduler.timesteps
|
||||
|
||||
# 5. Prepare latent variables
|
||||
num_channels_latents = self.unet.config.in_channels
|
||||
latents = self.prepare_latents(
|
||||
batch_size * num_images_per_prompt,
|
||||
num_channels_latents,
|
||||
height,
|
||||
width,
|
||||
prompt_embeds.dtype,
|
||||
device,
|
||||
generator,
|
||||
latents,
|
||||
)
|
||||
|
||||
# 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
|
||||
extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
|
||||
|
||||
if share_queries:
|
||||
query_store = QueryStore(**query_store_kwargs)
|
||||
else:
|
||||
query_store = None
|
||||
|
||||
self.attention_store = AttentionStore(attention_store_kwargs)
|
||||
register_extended_self_attn(self.unet, self.attention_store, extended_attn_kwargs)
|
||||
|
||||
# 7. Prepare added time ids & embeddings
|
||||
add_text_embeds = pooled_prompt_embeds
|
||||
if self.text_encoder_2 is None:
|
||||
text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
|
||||
else:
|
||||
text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
|
||||
|
||||
add_time_ids = self._get_add_time_ids(
|
||||
original_size,
|
||||
crops_coords_top_left,
|
||||
target_size,
|
||||
dtype=prompt_embeds.dtype,
|
||||
text_encoder_projection_dim=text_encoder_projection_dim,
|
||||
)
|
||||
if negative_original_size is not None and negative_target_size is not None:
|
||||
negative_add_time_ids = self._get_add_time_ids(
|
||||
negative_original_size,
|
||||
negative_crops_coords_top_left,
|
||||
negative_target_size,
|
||||
dtype=prompt_embeds.dtype,
|
||||
text_encoder_projection_dim=text_encoder_projection_dim,
|
||||
)
|
||||
else:
|
||||
negative_add_time_ids = add_time_ids
|
||||
|
||||
if self.do_classifier_free_guidance:
|
||||
prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
|
||||
add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)
|
||||
add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0)
|
||||
|
||||
prompt_embeds = prompt_embeds.to(device)
|
||||
add_text_embeds = add_text_embeds.to(device)
|
||||
add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)
|
||||
|
||||
# 8. Denoising loop
|
||||
num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
|
||||
|
||||
# 8.1 Apply denoising_end
|
||||
if (
|
||||
self.denoising_end is not None
|
||||
and isinstance(self.denoising_end, float)
|
||||
and self.denoising_end > 0
|
||||
and self.denoising_end < 1
|
||||
):
|
||||
discrete_timestep_cutoff = int(
|
||||
round(
|
||||
self.scheduler.config.num_train_timesteps
|
||||
- (self.denoising_end * self.scheduler.config.num_train_timesteps)
|
||||
)
|
||||
)
|
||||
num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
|
||||
timesteps = timesteps[:num_inference_steps]
|
||||
|
||||
# 9. Optionally get Guidance Scale Embedding
|
||||
timestep_cond = None
|
||||
if self.unet.config.time_cond_proj_dim is not None:
|
||||
guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
|
||||
timestep_cond = self.get_guidance_scale_embedding(
|
||||
guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
|
||||
).to(device=device, dtype=latents.dtype)
|
||||
|
||||
self._num_timesteps = len(timesteps)
|
||||
|
||||
if instance_latents is not None:
|
||||
n_instances = instance_latents.shape[0]
|
||||
instance_noise = latents[:n_instances].clone()
|
||||
|
||||
with self.progress_bar(total=num_inference_steps) as progress_bar:
|
||||
for i, t in enumerate(timesteps):
|
||||
self.attention_store.curr_iter = i
|
||||
|
||||
if instance_latents is not None:
|
||||
noised_instances = self.scheduler.add_noise(instance_latents, instance_noise, t.repeat(n_instances).long())
|
||||
latents[:n_instances] = noised_instances
|
||||
|
||||
# expand the latents if we are doing classifier free guidance
|
||||
latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
|
||||
latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
|
||||
|
||||
# predict the noise residual
|
||||
added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
|
||||
|
||||
if share_queries and (i >= query_store.t_range[0] and i <= query_store.t_range[1]):
|
||||
query_store.set_mode('cache')
|
||||
noise_pred_vanilla = self.unet(
|
||||
latent_model_input,
|
||||
t,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
timestep_cond=timestep_cond,
|
||||
cross_attention_kwargs={'query_store': query_store,
|
||||
'perform_extend_attn': False,
|
||||
'record_attention': False},
|
||||
added_cond_kwargs=added_cond_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
|
||||
query_store.set_mode('inject')
|
||||
|
||||
noise_pred = self.unet(
|
||||
latent_model_input,
|
||||
t,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
timestep_cond=timestep_cond,
|
||||
cross_attention_kwargs={'query_store': query_store,
|
||||
'perform_extend_attn': True,
|
||||
'record_attention': True,
|
||||
'feature_injector': feature_injector,
|
||||
'anchors_cache': anchors_cache},
|
||||
added_cond_kwargs=added_cond_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
|
||||
# perform guidance
|
||||
if self.do_classifier_free_guidance:
|
||||
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
|
||||
noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
|
||||
|
||||
if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:
|
||||
# Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
|
||||
noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale)
|
||||
|
||||
# compute the previous noisy sample x_t -> x_t-1
|
||||
latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
|
||||
|
||||
if callback_on_step_end is not None:
|
||||
callback_kwargs = {}
|
||||
for k in callback_on_step_end_tensor_inputs:
|
||||
callback_kwargs[k] = locals()[k]
|
||||
callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
|
||||
|
||||
latents = callback_outputs.pop("latents", latents)
|
||||
prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
|
||||
negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
|
||||
add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)
|
||||
negative_pooled_prompt_embeds = callback_outputs.pop(
|
||||
"negative_pooled_prompt_embeds", negative_pooled_prompt_embeds
|
||||
)
|
||||
add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
|
||||
negative_add_time_ids = callback_outputs.pop("negative_add_time_ids", negative_add_time_ids)
|
||||
|
||||
# call the callback, if provided
|
||||
if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
|
||||
progress_bar.update()
|
||||
if callback is not None and i % callback_steps == 0:
|
||||
step_idx = i // getattr(self.scheduler, "order", 1)
|
||||
callback(step_idx, t, latents)
|
||||
|
||||
if XLA_AVAILABLE:
|
||||
# xm.mark_step()
|
||||
pass
|
||||
|
||||
# Update attention store mask
|
||||
self.attention_store.aggregate_last_steps_attention()
|
||||
|
||||
if not output_type == "latent":
|
||||
# make sure the VAE is in float32 mode, as it overflows in float16
|
||||
needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
|
||||
|
||||
if needs_upcasting:
|
||||
self.upcast_vae()
|
||||
latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
|
||||
|
||||
image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]
|
||||
|
||||
# cast back to fp16 if needed
|
||||
if needs_upcasting:
|
||||
self.vae.to(dtype=torch.float16)
|
||||
else:
|
||||
image = latents
|
||||
|
||||
if not output_type == "latent":
|
||||
# apply watermark if available
|
||||
if self.watermark is not None:
|
||||
image = self.watermark.apply_watermark(image)
|
||||
|
||||
image = self.image_processor.postprocess(image, output_type=output_type)
|
||||
|
||||
# Offload all models
|
||||
self.maybe_free_model_hooks()
|
||||
|
||||
if not return_dict:
|
||||
return (image,)
|
||||
|
||||
return StableDiffusionXLPipelineOutput(images=image)
|
||||
@@ -0,0 +1,260 @@
|
||||
# Copyright (C) 2024 NVIDIA Corporation. All rights reserved.
|
||||
#
|
||||
# This work is licensed under the LICENSE file
|
||||
# located at the root directory.
|
||||
|
||||
import torch
|
||||
from diffusers import DDIMScheduler
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
from .consistory_unet_sdxl import ConsistorySDXLUNet2DConditionModel
|
||||
from .consistory_pipeline import ConsistoryExtendAttnSDXLPipeline
|
||||
from .consistory_utils import FeatureInjector, AnchorCache
|
||||
# from .utils.general_utils import *
|
||||
from .utils.general_utils import gaussian_smooth, cyclic_nn_map, anchor_nn_map
|
||||
|
||||
|
||||
LATENT_RESOLUTIONS = [32, 64]
|
||||
|
||||
|
||||
def load_pipeline(gpu_id=0):
|
||||
float_type = torch.float16
|
||||
sd_id = "stabilityai/stable-diffusion-xl-base-1.0"
|
||||
device = torch.device(f'cuda:{gpu_id}') if torch.cuda.is_available() else torch.device('cpu')
|
||||
unet = ConsistorySDXLUNet2DConditionModel.from_pretrained(sd_id, subfolder="unet", torch_dtype=float_type)
|
||||
scheduler = DDIMScheduler.from_pretrained(sd_id, subfolder="scheduler")
|
||||
story_pipeline = ConsistoryExtendAttnSDXLPipeline.from_pretrained(sd_id, unet=unet, torch_dtype=float_type, variant="fp16", use_safetensors=True, scheduler=scheduler).to(device)
|
||||
story_pipeline.enable_freeu(s1=0.6, s2=0.4, b1=1.1, b2=1.2)
|
||||
return story_pipeline
|
||||
|
||||
|
||||
def create_anchor_mapping(bsz, anchor_indices=[0]):
|
||||
anchor_mapping = torch.eye(bsz, dtype=torch.bool)
|
||||
for anchor_idx in anchor_indices:
|
||||
anchor_mapping[:, anchor_idx] = True
|
||||
return anchor_mapping
|
||||
|
||||
|
||||
def create_token_indices(prompts, batch_size, concept_token, tokenizer):
|
||||
if isinstance(concept_token, str):
|
||||
concept_token = [concept_token]
|
||||
concept_token_id = [tokenizer.encode(x, add_special_tokens=False)[0] for x in concept_token]
|
||||
tokens = tokenizer.batch_encode_plus(prompts, padding=True, return_tensors='pt')['input_ids']
|
||||
token_indices = torch.full((len(concept_token), batch_size), -1, dtype=torch.int64)
|
||||
for i, token_id in enumerate(concept_token_id):
|
||||
batch_loc, token_loc = torch.where(tokens == token_id)
|
||||
token_indices[i, batch_loc] = token_loc
|
||||
return token_indices
|
||||
|
||||
|
||||
def create_latents(story_pipeline, seed, batch_size, same_latent, device, float_type):
|
||||
# if seed is int
|
||||
if isinstance(seed, int):
|
||||
g = torch.Generator('cuda').manual_seed(seed)
|
||||
shape = (batch_size, story_pipeline.unet.config.in_channels, 128, 128)
|
||||
latents = randn_tensor(shape, generator=g, device=device, dtype=float_type)
|
||||
elif isinstance(seed, list):
|
||||
shape = (batch_size, story_pipeline.unet.config.in_channels, 128, 128)
|
||||
latents = torch.empty(shape, device=device, dtype=float_type)
|
||||
for i, seed_i in enumerate(seed):
|
||||
g = torch.Generator('cuda').manual_seed(seed_i)
|
||||
curr_latent = randn_tensor(shape, generator=g, device=device, dtype=float_type)
|
||||
latents[i] = curr_latent[i]
|
||||
if same_latent:
|
||||
latents = latents[:1].repeat(batch_size, 1, 1, 1)
|
||||
return latents, g
|
||||
|
||||
|
||||
# Batch inference
|
||||
def run_batch_generation(story_pipeline, prompts, concept_token,
|
||||
seed=40, n_steps=50, mask_dropout=0.5,
|
||||
same_latent=False, share_queries=True,
|
||||
perform_sdsa=True, perform_injection=True,
|
||||
inject_range_alpha=(10,20,0.8),
|
||||
n_achors=2):
|
||||
device = story_pipeline.device
|
||||
tokenizer = story_pipeline.tokenizer
|
||||
float_type = story_pipeline.dtype
|
||||
unet = story_pipeline.unet
|
||||
batch_size = len(prompts)
|
||||
token_indices = create_token_indices(prompts, batch_size, concept_token, tokenizer)
|
||||
anchor_mappings = create_anchor_mapping(batch_size, anchor_indices=list(range(n_achors)))
|
||||
default_attention_store_kwargs = {
|
||||
'token_indices': token_indices,
|
||||
'mask_dropout': mask_dropout,
|
||||
'extended_mapping': anchor_mappings
|
||||
}
|
||||
default_extended_attn_kwargs = {'extend_kv_unet_parts': ['up']}
|
||||
query_store_kwargs= {'t_range': [0,n_steps//10], 'strength_start': 0.9, 'strength_end': 0.81836735}
|
||||
latents, g = create_latents(story_pipeline, seed, batch_size, same_latent, device, float_type)
|
||||
|
||||
# ------------------ #
|
||||
# Extended attention First Run #
|
||||
if perform_sdsa:
|
||||
extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': [(1, n_steps)]}
|
||||
else:
|
||||
extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': []}
|
||||
out = story_pipeline(prompt=prompts, generator=g, latents=latents,
|
||||
attention_store_kwargs=default_attention_store_kwargs,
|
||||
extended_attn_kwargs=extended_attn_kwargs,
|
||||
share_queries=share_queries,
|
||||
query_store_kwargs=query_store_kwargs,
|
||||
num_inference_steps=n_steps)
|
||||
last_masks = story_pipeline.attention_store.last_mask
|
||||
dift_features = unet.latent_store.dift_features['261_0'][batch_size:]
|
||||
dift_features = torch.stack([gaussian_smooth(x, kernel_size=3, sigma=1) for x in dift_features], dim=0)
|
||||
nn_map, nn_distances = cyclic_nn_map(dift_features, last_masks, LATENT_RESOLUTIONS, device)
|
||||
|
||||
# ------------------ #
|
||||
# Extended attention with nn_map #
|
||||
if perform_injection:
|
||||
feature_injector = FeatureInjector(
|
||||
nn_map,
|
||||
nn_distances,
|
||||
last_masks,
|
||||
inject_range_alpha=[inject_range_alpha],
|
||||
swap_strategy='min', inject_unet_parts=['up', 'down'], dist_thr='dynamic')
|
||||
out = story_pipeline(prompt=prompts, generator=g, latents=latents,
|
||||
attention_store_kwargs=default_attention_store_kwargs,
|
||||
extended_attn_kwargs=extended_attn_kwargs,
|
||||
share_queries=share_queries,
|
||||
query_store_kwargs=query_store_kwargs,
|
||||
feature_injector=feature_injector,
|
||||
num_inference_steps=n_steps)
|
||||
# display_attn_maps(story_pipeline.attention_store.last_mask, out.images)
|
||||
return out.images
|
||||
|
||||
|
||||
# Anchors
|
||||
def run_anchor_generation(story_pipeline, prompts, concept_token,
|
||||
seed=40, n_steps=50, mask_dropout=0.5,
|
||||
inject_range_alpha=(10,20,0.8),
|
||||
same_latent=False, share_queries=True,
|
||||
perform_sdsa=True, perform_injection=True):
|
||||
device = story_pipeline.device
|
||||
tokenizer = story_pipeline.tokenizer
|
||||
float_type = story_pipeline.dtype
|
||||
unet = story_pipeline.unet
|
||||
batch_size = len(prompts)
|
||||
token_indices = create_token_indices(prompts, batch_size, concept_token, tokenizer)
|
||||
default_attention_store_kwargs = {
|
||||
'token_indices': token_indices,
|
||||
'mask_dropout': mask_dropout
|
||||
}
|
||||
default_extended_attn_kwargs = {'extend_kv_unet_parts': ['up']}
|
||||
query_store_kwargs={'t_range': [0,n_steps//10], 'strength_start': 0.9, 'strength_end': 0.81836735}
|
||||
latents, g = create_latents(story_pipeline, seed, batch_size, same_latent, device, float_type)
|
||||
anchor_cache_first_stage = AnchorCache()
|
||||
anchor_cache_second_stage = AnchorCache()
|
||||
|
||||
# ------------------ #
|
||||
# Extended attention First Run #
|
||||
if perform_sdsa:
|
||||
extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': [(1, n_steps)]}
|
||||
else:
|
||||
extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': []}
|
||||
out = story_pipeline(prompt=prompts, generator=g, latents=latents,
|
||||
attention_store_kwargs=default_attention_store_kwargs,
|
||||
extended_attn_kwargs=extended_attn_kwargs,
|
||||
share_queries=share_queries,
|
||||
query_store_kwargs=query_store_kwargs,
|
||||
anchors_cache=anchor_cache_first_stage,
|
||||
num_inference_steps=n_steps)
|
||||
last_masks = story_pipeline.attention_store.last_mask
|
||||
dift_features = unet.latent_store.dift_features['261_0'][batch_size:]
|
||||
dift_features = torch.stack([gaussian_smooth(x, kernel_size=3, sigma=1) for x in dift_features], dim=0)
|
||||
anchor_cache_first_stage.dift_cache = dift_features
|
||||
anchor_cache_first_stage.anchors_last_mask = last_masks
|
||||
nn_map, nn_distances = cyclic_nn_map(dift_features, last_masks, LATENT_RESOLUTIONS, device)
|
||||
|
||||
# ------------------ #
|
||||
# Extended attention with nn_map #
|
||||
if perform_injection:
|
||||
feature_injector = FeatureInjector(
|
||||
nn_map,
|
||||
nn_distances,
|
||||
last_masks,
|
||||
inject_range_alpha=[inject_range_alpha],
|
||||
swap_strategy='min',
|
||||
inject_unet_parts=['up', 'down'],
|
||||
dist_thr='dynamic')
|
||||
out = story_pipeline(prompt=prompts, generator=g, latents=latents,
|
||||
attention_store_kwargs=default_attention_store_kwargs,
|
||||
extended_attn_kwargs=extended_attn_kwargs,
|
||||
share_queries=share_queries,
|
||||
query_store_kwargs=query_store_kwargs,
|
||||
feature_injector=feature_injector,
|
||||
anchors_cache=anchor_cache_second_stage,
|
||||
num_inference_steps=n_steps)
|
||||
# display_attn_maps(story_pipeline.attention_store.last_mask, out.images)
|
||||
anchor_cache_second_stage.dift_cache = dift_features
|
||||
anchor_cache_second_stage.anchors_last_mask = last_masks
|
||||
return out.images, anchor_cache_first_stage, anchor_cache_second_stage
|
||||
|
||||
|
||||
def run_extra_generation(story_pipeline, prompts, concept_token,
|
||||
anchor_cache_first_stage, anchor_cache_second_stage,
|
||||
seed=40, n_steps=50, mask_dropout=0.5,
|
||||
inject_range_alpha=(10,20,0.8),
|
||||
same_latent=False, share_queries=True,
|
||||
perform_sdsa=True, perform_injection=True):
|
||||
device = story_pipeline.device
|
||||
tokenizer = story_pipeline.tokenizer
|
||||
float_type = story_pipeline.dtype
|
||||
unet = story_pipeline.unet
|
||||
batch_size = len(prompts)
|
||||
token_indices = create_token_indices(prompts, batch_size, concept_token, tokenizer)
|
||||
default_attention_store_kwargs = {
|
||||
'token_indices': token_indices,
|
||||
'mask_dropout': mask_dropout
|
||||
}
|
||||
default_extended_attn_kwargs = {'extend_kv_unet_parts': ['up']}
|
||||
query_store_kwargs={'t_range': [0,n_steps//10], 'strength_start': 0.9, 'strength_end': 0.81836735}
|
||||
extra_batch_size = batch_size + 2
|
||||
if isinstance(seed, list):
|
||||
seed = [seed[0], seed[0], *seed]
|
||||
latents, g = create_latents(story_pipeline, seed, extra_batch_size, same_latent, device, float_type)
|
||||
latents = latents[2:]
|
||||
anchor_cache_first_stage.set_mode_inject()
|
||||
anchor_cache_second_stage.set_mode_inject()
|
||||
|
||||
# ------------------ #
|
||||
# Extended attention First Run #
|
||||
if perform_sdsa:
|
||||
extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': [(1, n_steps)]}
|
||||
else:
|
||||
extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': []}
|
||||
out = story_pipeline(prompt=prompts, generator=g, latents=latents,
|
||||
attention_store_kwargs=default_attention_store_kwargs,
|
||||
extended_attn_kwargs=extended_attn_kwargs,
|
||||
share_queries=share_queries,
|
||||
query_store_kwargs=query_store_kwargs,
|
||||
anchors_cache=anchor_cache_first_stage,
|
||||
num_inference_steps=n_steps)
|
||||
last_masks = story_pipeline.attention_store.last_mask
|
||||
dift_features = unet.latent_store.dift_features['261_0'][batch_size:]
|
||||
dift_features = torch.stack([gaussian_smooth(x, kernel_size=3, sigma=1) for x in dift_features], dim=0)
|
||||
anchor_dift_features = anchor_cache_first_stage.dift_cache
|
||||
anchor_last_masks = anchor_cache_first_stage.anchors_last_mask
|
||||
nn_map, nn_distances = anchor_nn_map(dift_features, anchor_dift_features, last_masks, anchor_last_masks, LATENT_RESOLUTIONS, device)
|
||||
|
||||
# ------------------ #
|
||||
# Extended attention with nn_map #
|
||||
if perform_injection:
|
||||
feature_injector = FeatureInjector(
|
||||
nn_map,
|
||||
nn_distances,
|
||||
last_masks,
|
||||
inject_range_alpha=[inject_range_alpha],
|
||||
swap_strategy='min',
|
||||
inject_unet_parts=['up', 'down'],
|
||||
dist_thr='dynamic')
|
||||
out = story_pipeline(prompt=prompts, generator=g, latents=latents,
|
||||
attention_store_kwargs=default_attention_store_kwargs,
|
||||
extended_attn_kwargs=extended_attn_kwargs,
|
||||
share_queries=share_queries,
|
||||
query_store_kwargs=query_store_kwargs,
|
||||
feature_injector=feature_injector,
|
||||
anchors_cache=anchor_cache_second_stage,
|
||||
num_inference_steps=n_steps)
|
||||
# display_attn_maps(story_pipeline.attention_store.last_mask, out.images)
|
||||
return out.images
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,192 @@
|
||||
# Copyright (C) 2024 NVIDIA Corporation. All rights reserved.
|
||||
#
|
||||
# This work is licensed under the LICENSE file
|
||||
# located at the root directory.
|
||||
|
||||
from typing import List
|
||||
from collections import defaultdict
|
||||
import numpy as np
|
||||
import torch
|
||||
from .utils.general_utils import get_dynamic_threshold
|
||||
|
||||
|
||||
class FeatureInjector:
|
||||
def __init__(self, nn_map, nn_distances, attn_masks, inject_range_alpha=[(10,20,0.8)], swap_strategy='min', dist_thr='dynamic', inject_unet_parts=['up']):
|
||||
self.nn_map = nn_map
|
||||
self.nn_distances = nn_distances
|
||||
self.attn_masks = attn_masks
|
||||
self.inject_range_alpha = inject_range_alpha if isinstance(inject_range_alpha, list) else [inject_range_alpha]
|
||||
self.swap_strategy = swap_strategy # 'min / 'mean' / 'first'
|
||||
self.dist_thr = dist_thr
|
||||
self.inject_unet_parts = inject_unet_parts
|
||||
self.inject_res = [64]
|
||||
|
||||
def inject_outputs(self, output, curr_iter, output_res, extended_mapping, place_in_unet, anchors_cache=None):
|
||||
curr_unet_part = place_in_unet.split('_')[0]
|
||||
|
||||
# Inject only in the specified unet parts (up, mid, down)
|
||||
if (curr_unet_part not in self.inject_unet_parts) or output_res not in self.inject_res:
|
||||
return output
|
||||
|
||||
bsz = output.shape[0]
|
||||
nn_map = self.nn_map[output_res]
|
||||
nn_distances = self.nn_distances[output_res]
|
||||
attn_masks = self.attn_masks[output_res]
|
||||
vector_dim = output_res**2
|
||||
|
||||
alpha = next((alpha for min_range, max_range, alpha in self.inject_range_alpha if min_range <= curr_iter <= max_range), None)
|
||||
if alpha:
|
||||
old_output = output#.clone()
|
||||
for i in range(bsz):
|
||||
other_outputs = []
|
||||
|
||||
if self.swap_strategy == 'min':
|
||||
curr_mapping = extended_mapping[i]
|
||||
|
||||
# If the current image is not mapped to any other image, skip
|
||||
if not torch.any(torch.cat([curr_mapping[:i], curr_mapping[i+1:]])):
|
||||
continue
|
||||
|
||||
min_dists = nn_distances[i][curr_mapping].argmin(dim=0)
|
||||
curr_nn_map = nn_map[i][curr_mapping][min_dists, torch.arange(vector_dim)]
|
||||
|
||||
curr_nn_distances = nn_distances[i][curr_mapping][min_dists, torch.arange(vector_dim)]
|
||||
dist_thr = get_dynamic_threshold(curr_nn_distances) if self.dist_thr == 'dynamic' else self.dist_thr
|
||||
dist_mask = curr_nn_distances < dist_thr
|
||||
final_mask_tgt = attn_masks[i] & dist_mask
|
||||
|
||||
other_outputs = old_output[curr_mapping][min_dists, curr_nn_map][final_mask_tgt]
|
||||
|
||||
output[i][final_mask_tgt] = alpha * other_outputs + (1 - alpha)*old_output[i][final_mask_tgt]
|
||||
|
||||
if anchors_cache and anchors_cache.is_cache_mode():
|
||||
if place_in_unet not in anchors_cache.h_out_cache:
|
||||
anchors_cache.h_out_cache[place_in_unet] = {}
|
||||
|
||||
anchors_cache.h_out_cache[place_in_unet][curr_iter] = output
|
||||
|
||||
return output
|
||||
|
||||
def inject_anchors(self, output, curr_iter, output_res, extended_mapping, place_in_unet, anchors_cache):
|
||||
curr_unet_part = place_in_unet.split('_')[0]
|
||||
|
||||
# Inject only in the specified unet parts (up, mid, down)
|
||||
if (curr_unet_part not in self.inject_unet_parts) or output_res not in self.inject_res:
|
||||
return output
|
||||
|
||||
bsz = output.shape[0]
|
||||
nn_map = self.nn_map[output_res]
|
||||
nn_distances = self.nn_distances[output_res]
|
||||
attn_masks = self.attn_masks[output_res]
|
||||
vector_dim = output_res**2
|
||||
|
||||
alpha = next((alpha for min_range, max_range, alpha in self.inject_range_alpha if min_range <= curr_iter <= max_range), None)
|
||||
if alpha:
|
||||
|
||||
anchor_outputs = anchors_cache.h_out_cache[place_in_unet][curr_iter]
|
||||
|
||||
old_output = output#.clone()
|
||||
for i in range(bsz):
|
||||
other_outputs = []
|
||||
|
||||
if self.swap_strategy == 'min':
|
||||
min_dists = nn_distances[i].argmin(dim=0)
|
||||
curr_nn_map = nn_map[i][min_dists, torch.arange(vector_dim)]
|
||||
|
||||
curr_nn_distances = nn_distances[i][min_dists, torch.arange(vector_dim)]
|
||||
dist_thr = get_dynamic_threshold(curr_nn_distances) if self.dist_thr == 'dynamic' else self.dist_thr
|
||||
dist_mask = curr_nn_distances < dist_thr
|
||||
final_mask_tgt = attn_masks[i] & dist_mask
|
||||
|
||||
other_outputs = anchor_outputs[min_dists, curr_nn_map][final_mask_tgt]
|
||||
|
||||
output[i][final_mask_tgt] = alpha * other_outputs + (1 - alpha)*old_output[i][final_mask_tgt]
|
||||
|
||||
return output
|
||||
|
||||
|
||||
class AnchorCache:
|
||||
def __init__(self):
|
||||
self.input_h_cache = {} # place_in_unet, iter, h_in
|
||||
self.h_out_cache = {} # place_in_unet, iter, h_out
|
||||
self.anchors_last_mask = None
|
||||
self.dift_cache = None
|
||||
|
||||
self.mode = 'cache' # mode can be 'cache' or 'inject'
|
||||
|
||||
def set_mode(self, mode):
|
||||
self.mode = mode
|
||||
|
||||
def set_mode_inject(self):
|
||||
self.mode = 'inject'
|
||||
|
||||
def set_mode_cache(self):
|
||||
self.mode = 'cache'
|
||||
|
||||
def is_inject_mode(self):
|
||||
return self.mode == 'inject'
|
||||
|
||||
def is_cache_mode(self):
|
||||
return self.mode == 'cache'
|
||||
|
||||
|
||||
def to_device(self, device):
|
||||
for key, value in self.input_h_cache.items():
|
||||
self.input_h_cache[key] = {k: v.to(device) for k, v in value.items()}
|
||||
|
||||
for key, value in self.h_out_cache.items():
|
||||
self.h_out_cache[key] = {k: v.to(device) for k, v in value.items()}
|
||||
|
||||
if self.anchors_last_mask:
|
||||
self.anchors_last_mask = {k: v.to(device) for k, v in self.anchors_last_mask.items()}
|
||||
|
||||
if self.dift_cache is not None:
|
||||
self.dift_cache = self.dift_cache.to(device)
|
||||
|
||||
|
||||
class QueryStore:
|
||||
def __init__(self, mode='store', t_range=[0, 1000], strength_start=1, strength_end=1):
|
||||
"""
|
||||
Initialize an empty ActivationsStore
|
||||
"""
|
||||
self.query_store = defaultdict(list)
|
||||
self.mode = mode
|
||||
self.t_range = t_range
|
||||
self.strengthes = np.linspace(strength_start, strength_end, (t_range[1] - t_range[0])+1)
|
||||
|
||||
def set_mode(self, mode): # mode can be 'cache' or 'inject'
|
||||
self.mode = mode
|
||||
|
||||
def cache_query(self, query, place_in_unet: str):
|
||||
self.query_store[place_in_unet] = query
|
||||
|
||||
def inject_query(self, query, place_in_unet, t):
|
||||
if t >= self.t_range[0] and t <= self.t_range[1]:
|
||||
relative_t = t - self.t_range[0]
|
||||
strength = self.strengthes[relative_t]
|
||||
new_query = strength * self.query_store[place_in_unet] + (1 - strength) * query
|
||||
else:
|
||||
new_query = query
|
||||
|
||||
return new_query
|
||||
|
||||
class DIFTLatentStore:
|
||||
def __init__(self, steps: List[int], up_ft_indices: List[int]):
|
||||
self.steps = steps
|
||||
self.up_ft_indices = up_ft_indices
|
||||
self.dift_features = {}
|
||||
|
||||
def __call__(self, features: torch.Tensor, t: int, layer_index: int):
|
||||
if t in self.steps and layer_index in self.up_ft_indices:
|
||||
self.dift_features[f'{int(t)}_{layer_index}'] = features
|
||||
|
||||
def copy(self):
|
||||
copy_dift = DIFTLatentStore(self.steps, self.up_ft_indices)
|
||||
|
||||
for key, value in self.dift_features.items():
|
||||
copy_dift.dift_features[key] = value.clone()
|
||||
|
||||
return copy_dift
|
||||
|
||||
def reset(self):
|
||||
self.dift_features = {}
|
||||
@@ -0,0 +1,117 @@
|
||||
# Copyright (C) 2024 NVIDIA Corporation. All rights reserved.
|
||||
#
|
||||
# This work is licensed under the LICENSE file
|
||||
# located at the root directory.
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import numpy as np
|
||||
from skimage import filters
|
||||
|
||||
|
||||
## Attention Utils
|
||||
def get_dynamic_threshold(tensor):
|
||||
return filters.threshold_otsu(tensor.float().cpu().numpy())
|
||||
|
||||
|
||||
def attn_map_to_binary(attention_map, scaler=1.):
|
||||
attention_map_np = attention_map.float().cpu().numpy()
|
||||
threshold_value = filters.threshold_otsu(attention_map_np) * scaler
|
||||
binary_mask = (attention_map_np > threshold_value).astype(np.uint8)
|
||||
|
||||
return binary_mask
|
||||
|
||||
|
||||
## Features
|
||||
|
||||
def gaussian_smooth(input_tensor, kernel_size=3, sigma=1):
|
||||
"""
|
||||
Function to apply Gaussian smoothing on each 2D slice of a 3D tensor.
|
||||
"""
|
||||
kernel = np.fromfunction(
|
||||
lambda x, y: (1/ (2 * np.pi * sigma ** 2)) *
|
||||
np.exp(-((x - (kernel_size - 1) / 2) ** 2 + (y - (kernel_size - 1) / 2) ** 2) / (2 * sigma ** 2)),
|
||||
(kernel_size, kernel_size)
|
||||
)
|
||||
kernel = torch.Tensor(kernel / kernel.sum()).to(input_tensor.dtype).to(input_tensor.device)
|
||||
# Add batch and channel dimensions to the kernel
|
||||
kernel = kernel.unsqueeze(0).unsqueeze(0)
|
||||
# Iterate over each 2D slice and apply convolution
|
||||
smoothed_slices = []
|
||||
for i in range(input_tensor.size(0)):
|
||||
slice_tensor = input_tensor[i, :, :]
|
||||
slice_tensor = F.conv2d(slice_tensor.unsqueeze(0).unsqueeze(0), kernel, padding=kernel_size // 2)[0, 0]
|
||||
smoothed_slices.append(slice_tensor)
|
||||
# Stack the smoothed slices to get the final tensor
|
||||
smoothed_tensor = torch.stack(smoothed_slices, dim=0)
|
||||
return smoothed_tensor
|
||||
|
||||
|
||||
## Dense correspondence utils
|
||||
|
||||
def cos_dist(a, b):
|
||||
a_norm = F.normalize(a, dim=-1)
|
||||
b_norm = F.normalize(b, dim=-1)
|
||||
res = a_norm @ b_norm.T
|
||||
return 1 - res
|
||||
|
||||
|
||||
def gen_nn_map(src_features, src_mask, tgt_features, tgt_mask, device, batch_size=100, tgt_size=768):
|
||||
resized_src_features = F.interpolate(src_features.unsqueeze(0), size=tgt_size, mode='bilinear', align_corners=False).squeeze(0)
|
||||
resized_src_features = resized_src_features.permute(1,2,0).view(tgt_size**2, -1)
|
||||
resized_tgt_features = F.interpolate(tgt_features.unsqueeze(0), size=tgt_size, mode='bilinear', align_corners=False).squeeze(0)
|
||||
resized_tgt_features = resized_tgt_features.permute(1,2,0).view(tgt_size**2, -1)
|
||||
nearest_neighbor_indices = torch.zeros(tgt_size**2, dtype=torch.long, device=device)
|
||||
nearest_neighbor_distances = torch.zeros(tgt_size**2, dtype=src_features.dtype, device=device)
|
||||
if not batch_size:
|
||||
batch_size = tgt_size**2
|
||||
for i in range(0, tgt_size**2, batch_size):
|
||||
distances = cos_dist(resized_src_features, resized_tgt_features[i:i+batch_size])
|
||||
distances[~src_mask] = 2.
|
||||
min_distances, min_indices = torch.min(distances, dim=0)
|
||||
nearest_neighbor_indices[i:i+batch_size] = min_indices
|
||||
nearest_neighbor_distances[i:i+batch_size] = min_distances
|
||||
return nearest_neighbor_indices, nearest_neighbor_distances
|
||||
|
||||
|
||||
def cyclic_nn_map(features, masks, latent_resolutions, device):
|
||||
bsz = features.shape[0]
|
||||
nn_map_dict = {}
|
||||
nn_distances_dict = {}
|
||||
|
||||
for tgt_size in latent_resolutions:
|
||||
nn_map = torch.empty(bsz, bsz, tgt_size**2, dtype=torch.long, device=device)
|
||||
nn_distances = torch.full((bsz, bsz, tgt_size**2), float('inf'), dtype=features.dtype, device=device)
|
||||
|
||||
for i in range(bsz):
|
||||
for j in range(bsz):
|
||||
if i != j:
|
||||
nearest_neighbor_indices, nearest_neighbor_distances = gen_nn_map(features[j], masks[tgt_size][j], features[i], masks[tgt_size][i], device, batch_size=None, tgt_size=tgt_size)
|
||||
nn_map[i,j] = nearest_neighbor_indices
|
||||
nn_distances[i,j] = nearest_neighbor_distances
|
||||
|
||||
nn_map_dict[tgt_size] = nn_map
|
||||
nn_distances_dict[tgt_size] = nn_distances
|
||||
|
||||
return nn_map_dict, nn_distances_dict
|
||||
|
||||
|
||||
def anchor_nn_map(features, anchor_features, masks, anchor_masks, latent_resolutions, device):
|
||||
bsz = features.shape[0]
|
||||
anchor_bsz = anchor_features.shape[0]
|
||||
nn_map_dict = {}
|
||||
nn_distances_dict = {}
|
||||
|
||||
for tgt_size in latent_resolutions:
|
||||
nn_map = torch.empty(bsz, anchor_bsz, tgt_size**2, dtype=torch.long, device=device)
|
||||
nn_distances = torch.full((bsz, anchor_bsz, tgt_size**2), float('inf'), dtype=features.dtype, device=device)
|
||||
|
||||
for i in range(bsz):
|
||||
for j in range(anchor_bsz):
|
||||
nearest_neighbor_indices, nearest_neighbor_distances = gen_nn_map(anchor_features[j], anchor_masks[tgt_size][j], features[i], masks[tgt_size][i], device, batch_size=None, tgt_size=tgt_size)
|
||||
nn_map[i,j] = nearest_neighbor_indices
|
||||
nn_distances[i,j] = nearest_neighbor_distances
|
||||
nn_map_dict[tgt_size] = nn_map
|
||||
nn_distances_dict[tgt_size] = nn_distances
|
||||
|
||||
return nn_map_dict, nn_distances_dict
|
||||
@@ -0,0 +1,194 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2023 AttendAndExcite
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Not a contribution
|
||||
# Changes made by NVIDIA CORPORATION & AFFILIATES enabling ConsiStory or otherwise documented as NVIDIA-proprietary
|
||||
# are not a contribution and subject to the license under the LICENSE file located at the root directory.
|
||||
|
||||
import torch
|
||||
from collections import defaultdict
|
||||
import numpy as np
|
||||
from typing import Union, List
|
||||
from PIL import Image
|
||||
|
||||
from modules.consistory.utils.general_utils import attn_map_to_binary
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
class AttentionStore:
|
||||
def __init__(self, attention_store_kwargs):
|
||||
"""
|
||||
Initialize an empty AttentionStore :param step_index: used to visualize only a specific step in the diffusion
|
||||
process
|
||||
"""
|
||||
self.attn_res = attention_store_kwargs.get('attn_res', (32,32))
|
||||
self.token_indices = attention_store_kwargs['token_indices']
|
||||
bsz = self.token_indices.size(1)
|
||||
self.mask_background_query = attention_store_kwargs.get('mask_background_query', False)
|
||||
self.original_attn_masks = attention_store_kwargs.get('original_attn_masks', None)
|
||||
self.extended_mapping = attention_store_kwargs.get('extended_mapping', torch.ones(bsz, bsz).bool())
|
||||
self.mask_dropout = attention_store_kwargs.get('mask_dropout', 0.0)
|
||||
torch.manual_seed(0) # For dropout mask reproducibility
|
||||
|
||||
self.curr_iter = 0
|
||||
self.ALL_RES = [32, 64]
|
||||
self.step_store = defaultdict(list)
|
||||
self.attn_masks = {res: None for res in self.ALL_RES}
|
||||
self.last_mask = {res: None for res in self.ALL_RES}
|
||||
self.last_mask_dropout = {res: None for res in self.ALL_RES}
|
||||
|
||||
def __call__(self, attn, is_cross: bool, place_in_unet: str, attn_heads: int):
|
||||
if is_cross and attn.shape[1] == np.prod(self.attn_res):
|
||||
guidance_attention = attn[attn.size(0)//2:]
|
||||
batched_guidance_attention = guidance_attention.reshape([guidance_attention.shape[0]//attn_heads, attn_heads, *guidance_attention.shape[1:]])
|
||||
batched_guidance_attention = batched_guidance_attention.mean(dim=1)
|
||||
self.step_store[place_in_unet].append(batched_guidance_attention)
|
||||
|
||||
def reset(self):
|
||||
self.step_store = defaultdict(list)
|
||||
self.attn_masks = {res: None for res in self.ALL_RES}
|
||||
self.last_mask = {res: None for res in self.ALL_RES}
|
||||
self.last_mask_dropout = {res: None for res in self.ALL_RES}
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
def aggregate_last_steps_attention(self) -> torch.Tensor:
|
||||
"""Aggregates the attention across the different layers and heads at the specified resolution."""
|
||||
attention_maps = torch.cat([torch.stack(x[-20:]) for x in self.step_store.values()]).mean(dim=0)
|
||||
bsz, wh, _ = attention_maps.shape
|
||||
|
||||
# Create attention maps for each concept token, for each batch item
|
||||
agg_attn_maps = []
|
||||
for i in range(bsz):
|
||||
curr_prompt_indices = []
|
||||
|
||||
for concept_token_indices in self.token_indices:
|
||||
if concept_token_indices[i] != -1:
|
||||
curr_prompt_indices.append(attention_maps[i, :, concept_token_indices[i]].view(*self.attn_res))
|
||||
|
||||
agg_attn_maps.append(torch.stack(curr_prompt_indices))
|
||||
|
||||
# Upsample the attention maps to the target resolution
|
||||
# and create the attention masks, unifying masks across the different concepts
|
||||
for tgt_size in self.ALL_RES:
|
||||
pixels = tgt_size ** 2
|
||||
tgt_agg_attn_maps = [F.interpolate(x.unsqueeze(1), size=tgt_size, mode='bilinear').squeeze(1) for x in agg_attn_maps]
|
||||
|
||||
attn_masks = []
|
||||
for batch_item_map in tgt_agg_attn_maps:
|
||||
concept_attn_masks = []
|
||||
|
||||
for concept_maps in batch_item_map:
|
||||
concept_attn_masks.append(torch.from_numpy(attn_map_to_binary(concept_maps, 1.)).to(attention_maps.device).bool().view(-1))
|
||||
|
||||
concept_attn_masks = torch.stack(concept_attn_masks, dim=0).max(dim=0).values
|
||||
attn_masks.append(concept_attn_masks)
|
||||
|
||||
attn_masks = torch.stack(attn_masks)
|
||||
self.last_mask[tgt_size] = attn_masks.clone()
|
||||
|
||||
# Add mask dropout
|
||||
if self.curr_iter < 1000:
|
||||
rand_mask = (torch.rand_like(attn_masks.float()) < self.mask_dropout)
|
||||
attn_masks[rand_mask] = False
|
||||
|
||||
self.last_mask_dropout[tgt_size] = attn_masks.clone()
|
||||
|
||||
# # Create subject driven extended self attention masks
|
||||
# output_attn_mask = torch.zeros((bsz, tgt_size**2, attn_masks.view(-1).size(0)), device=attn_masks.device).bool()
|
||||
|
||||
# for i in range(bsz):
|
||||
# for j in range(bsz):
|
||||
# if i==j:
|
||||
# output_attn_mask[i, :, j*pixels:(j+1)*pixels] = 1
|
||||
# else:
|
||||
# if self.extended_mapping[i,j]:
|
||||
# if not self.mask_background_query:
|
||||
# output_attn_mask[i, :, j*pixels:(j+1)*pixels] = attn_masks[j].unsqueeze(0).expand(pixels, -1)
|
||||
# else:
|
||||
# output_attn_mask[i, attn_masks[i], j*pixels:(j+1)*pixels] = attn_masks[j].unsqueeze(0).expand(attn_masks[i].sum(), -1)
|
||||
|
||||
# self.attn_masks[tgt_size] = output_attn_mask
|
||||
|
||||
def get_attn_mask_bias(self, tgt_size, bsz=None):
|
||||
attn_mask = self.attn_masks[tgt_size] if self.original_attn_masks is None else self.original_attn_masks[tgt_size]
|
||||
|
||||
if attn_mask is None:
|
||||
return None
|
||||
|
||||
attn_bias = torch.zeros_like(attn_mask, dtype=torch.float16)
|
||||
attn_bias[~attn_mask] = float('-inf')
|
||||
|
||||
if bsz and bsz != attn_bias.shape[0]:
|
||||
attn_bias = attn_bias.repeat(bsz // attn_bias.shape[0], 1, 1)
|
||||
|
||||
return attn_bias
|
||||
|
||||
def get_extended_attn_mask_instance(self, width, i):
|
||||
attn_mask = self.last_mask_dropout[width]
|
||||
if attn_mask is None:
|
||||
return None
|
||||
|
||||
n_patches = width**2
|
||||
|
||||
|
||||
output_attn_mask = torch.zeros((attn_mask.shape[0] * attn_mask.shape[1],), device=attn_mask.device, dtype=torch.bool)
|
||||
for j in range(attn_mask.shape[0]):
|
||||
if i==j:
|
||||
output_attn_mask[j*n_patches:(j+1)*n_patches] = 1
|
||||
else:
|
||||
if self.extended_mapping[i,j]:
|
||||
if not self.mask_background_query:
|
||||
output_attn_mask[j*n_patches:(j+1)*n_patches] = attn_mask[j].unsqueeze(0) #.expand(n_patches, -1)
|
||||
else:
|
||||
raise NotImplementedError('mask_background_query is not supported anymore')
|
||||
output_attn_mask[0, attn_mask[i], k*n_patches:(k+1)*n_patches] = attn_mask[j].unsqueeze(0).expand(attn_mask[i].sum(), -1)
|
||||
|
||||
return output_attn_mask
|
||||
@@ -66,11 +66,11 @@ def control_run(state: str = '',
|
||||
resize_mode_before: int = 0, resize_name_before: str = 'None', resize_context_before: str = 'None', width_before: int = 512, height_before: int = 512, scale_by_before: float = 1.0, selected_scale_tab_before: int = 0,
|
||||
resize_mode_after: int = 0, resize_name_after: str = 'None', resize_context_after: str = 'None', width_after: int = 0, height_after: int = 0, scale_by_after: float = 1.0, selected_scale_tab_after: int = 0,
|
||||
resize_mode_mask: int = 0, resize_name_mask: str = 'None', resize_context_mask: str = 'None', width_mask: int = 0, height_mask: int = 0, scale_by_mask: float = 1.0, selected_scale_tab_mask: int = 0,
|
||||
denoising_strength: float = 0, batch_count: int = 1, batch_size: int = 1,
|
||||
enable_hr: bool = False, hr_sampler_index: int = None, hr_denoising_strength: float = 0.3, hr_resize_mode: int = 0, hr_resize_context: str = 'None', hr_upscaler: str = None, hr_force: bool = False, hr_second_pass_steps: int = 20,
|
||||
denoising_strength: float = 0.3, batch_count: int = 1, batch_size: int = 1,
|
||||
enable_hr: bool = False, hr_sampler_index: int = None, hr_denoising_strength: float = 0.0, hr_resize_mode: int = 0, hr_resize_context: str = 'None', hr_upscaler: str = None, hr_force: bool = False, hr_second_pass_steps: int = 20,
|
||||
hr_scale: float = 1.0, hr_resize_x: int = 0, hr_resize_y: int = 0, refiner_steps: int = 5, refiner_start: float = 0.0, refiner_prompt: str = '', refiner_negative: str = '',
|
||||
video_skip_frames: int = 0, video_type: str = 'None', video_duration: float = 2.0, video_loop: bool = False, video_pad: int = 0, video_interpolate: int = 0,
|
||||
*input_script_args
|
||||
*input_script_args,
|
||||
):
|
||||
# handle optional initialization via ui
|
||||
for u in units:
|
||||
|
||||
@@ -56,6 +56,12 @@ predefined_sdxl = {
|
||||
'Xinsir Depth XL': 'xinsir/controlnet-depth-sdxl-1.0',
|
||||
'Xinsir Scribble XL': 'xinsir/controlnet-scribble-sdxl-1.0',
|
||||
'Xinsir Anime Painter XL': 'xinsir/anime-painter',
|
||||
'NoobAI Canny XL': 'Eugeoter/noob-sdxl-controlnet-canny',
|
||||
'NoobAI Lineart Anime XL': 'Eugeoter/noob-sdxl-controlnet-lineart_anime',
|
||||
'NoobAI Depth XL': 'Eugeoter/noob-sdxl-controlnet-depth',
|
||||
'NoobAI Normal XL': 'Eugeoter/noob-sdxl-controlnet-normal',
|
||||
'NoobAI SoftEdge XL': 'Eugeoter/noob-sdxl-controlnet-softedge_hed',
|
||||
'NoobAI OpenPose XL': 'einar77/noob-openpose',
|
||||
# 'StabilityAI Canny R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-canny-rank128.safetensors',
|
||||
# 'StabilityAI Depth R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-depth-rank128.safetensors',
|
||||
# 'StabilityAI Recolor R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-recolor-rank128.safetensors',
|
||||
@@ -156,17 +162,21 @@ class ControlNet():
|
||||
def get_class(self):
|
||||
import modules.shared
|
||||
if modules.shared.sd_model_type == 'sd':
|
||||
from diffusers import ControlNetModel as model_class # pylint: disable=reimported
|
||||
from diffusers import ControlNetModel as cls # pylint: disable=reimported
|
||||
config = 'lllyasviel/control_v11p_sd15_canny'
|
||||
elif modules.shared.sd_model_type == 'sdxl':
|
||||
from diffusers import ControlNetModel as model_class # pylint: disable=reimported # sdxl shares same model class
|
||||
from diffusers import ControlNetModel as cls # pylint: disable=reimported # sdxl shares same model class
|
||||
config = 'Eugeoter/noob-sdxl-controlnet-canny'
|
||||
elif modules.shared.sd_model_type == 'f1':
|
||||
from diffusers import FluxControlNetModel as model_class
|
||||
from diffusers import FluxControlNetModel as cls
|
||||
config = 'InstantX/FLUX.1-dev-Controlnet-Union'
|
||||
elif modules.shared.sd_model_type == 'sd3':
|
||||
from diffusers import SD3ControlNetModel as model_class
|
||||
from diffusers import SD3ControlNetModel as cls
|
||||
config = 'InstantX/SD3-Controlnet-Canny'
|
||||
else:
|
||||
log.error(f'Control {what}: type={modules.shared.sd_model_type} unsupported model')
|
||||
return None
|
||||
return model_class
|
||||
return None, None
|
||||
return cls, config
|
||||
|
||||
def load_safetensors(self, model_path):
|
||||
name = os.path.splitext(model_path)[0]
|
||||
@@ -193,8 +203,11 @@ class ControlNet():
|
||||
config_path = f'{name}.json'
|
||||
if config_path is not None:
|
||||
self.load_config['original_config_file '] = config_path
|
||||
cls = self.get_class()
|
||||
self.model = cls.from_single_file(model_path, **self.load_config)
|
||||
cls, config = self.get_class()
|
||||
if cls is None:
|
||||
log.error(f'Control {what} model load failed: unknown base model')
|
||||
else:
|
||||
self.model = cls.from_single_file(model_path, config=config, **self.load_config)
|
||||
|
||||
def load(self, model_id: str = None, force: bool = True) -> str:
|
||||
try:
|
||||
@@ -219,14 +232,19 @@ class ControlNet():
|
||||
if model_path.endswith('.safetensors'):
|
||||
self.load_safetensors(model_path)
|
||||
else:
|
||||
kwargs = {}
|
||||
if '/bin' in model_path:
|
||||
model_path = model_path.replace('/bin', '')
|
||||
self.load_config['use_safetensors'] = False
|
||||
cls = self.get_class()
|
||||
cls, _config = self.get_class()
|
||||
if cls is None:
|
||||
log.error(f'Control {what} model load failed: id="{model_id}" unknown base model')
|
||||
return
|
||||
self.model = cls.from_pretrained(model_path, **self.load_config)
|
||||
if 'Eugeoter' in model_path:
|
||||
kwargs['variant'] = 'fp16'
|
||||
self.model = cls.from_pretrained(model_path, **self.load_config, **kwargs)
|
||||
if self.model is None:
|
||||
return
|
||||
if self.dtype is not None:
|
||||
self.model.to(self.dtype)
|
||||
if "ControlNet" in opts.nncf_compress_weights:
|
||||
|
||||
+2
-2
@@ -59,7 +59,7 @@ def exception(suppress=[]):
|
||||
console.print_exception(show_locals=False, max_frames=16, extra_lines=2, suppress=suppress, theme="ansi_dark", word_wrap=False, width=min([console.width, 200]))
|
||||
|
||||
|
||||
def profile(profiler, msg: str):
|
||||
def profile(profiler, msg: str, n: int = 5):
|
||||
profiler.disable()
|
||||
import io
|
||||
import pstats
|
||||
@@ -83,7 +83,7 @@ def profile(profiler, msg: str):
|
||||
and 'rich' not in x
|
||||
and x.strip() != ''
|
||||
]
|
||||
txt = '\n'.join(lines[:min(5, len(lines))])
|
||||
txt = '\n'.join(lines[:min(n, len(lines))])
|
||||
log.debug(f'Profile {msg}: {txt}')
|
||||
|
||||
|
||||
|
||||
@@ -102,8 +102,9 @@ def activate(p, extra_network_data, step=0):
|
||||
except Exception as e:
|
||||
errors.display(e, f"Activating network: type={extra_network_name}")
|
||||
|
||||
p.extra_network_data = extra_network_data
|
||||
if stepwise:
|
||||
p.extra_network_data = extra_network_data
|
||||
p.stepwise_lora = True
|
||||
shared.opts.data['lora_functional'] = functional
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ debug = shared.log.trace if os.environ.get('SD_FACE_DEBUG', None) is not None el
|
||||
|
||||
class Script(scripts.Script):
|
||||
def title(self):
|
||||
return 'Face'
|
||||
return 'Face: Multiple ID Transfers'
|
||||
|
||||
def show(self, is_img2img):
|
||||
return True if shared.native else False
|
||||
@@ -28,10 +28,10 @@ class Script(scripts.Script):
|
||||
elif hasattr(file, 'name'):
|
||||
image = Image.open(file.name) # _TemporaryFileWrapper from gr.Files
|
||||
else:
|
||||
raise ValueError(f'PhotoMaker unknown input: {file}')
|
||||
raise ValueError(f'Face: unknown input: {file}')
|
||||
init_images.append(image)
|
||||
except Exception as e:
|
||||
shared.log.warning(f'PhotoMaker failed to load image: {e}')
|
||||
shared.log.warning(f'Face: failed to load image: {e}')
|
||||
return init_images
|
||||
|
||||
def mode_change(self, mode):
|
||||
@@ -45,7 +45,7 @@ class Script(scripts.Script):
|
||||
# return signature is array of gradio components
|
||||
def ui(self, _is_img2img):
|
||||
with gr.Row():
|
||||
gr.HTML("<span>  Face module</span><br>")
|
||||
gr.HTML("<span>  Face: Multiple ID Transfers</span><br>")
|
||||
with gr.Row():
|
||||
mode = gr.Dropdown(label='Mode', choices=['None', 'FaceID', 'FaceSwap', 'InstantID', 'PhotoMaker'], value='None')
|
||||
with gr.Group(visible=False) as cfg_faceid:
|
||||
|
||||
@@ -68,7 +68,7 @@ def instant_id(p: processing.StableDiffusionProcessing, app, source_images, stre
|
||||
processing.process_init(p)
|
||||
p.init(p.all_prompts, p.all_seeds, p.all_subseeds)
|
||||
orig_prompt_attention = shared.opts.prompt_attention
|
||||
shared.opts.data['prompt_attention'] = 'Fixed attention' # otherwise need to deal with class_tokens_mask
|
||||
shared.opts.data['prompt_attention'] = 'fixed' # otherwise need to deal with class_tokens_mask
|
||||
p.task_args['image_embeds'] = face_embeds[0].shape # placeholder
|
||||
p.task_args['image'] = face_images[0]
|
||||
p.task_args['controlnet_conditioning_scale'] = float(conditioning)
|
||||
|
||||
@@ -49,7 +49,7 @@ def photo_maker(p: processing.StableDiffusionProcessing, input_images, trigger,
|
||||
shared.sd_model.to(dtype=devices.dtype)
|
||||
|
||||
orig_prompt_attention = shared.opts.prompt_attention
|
||||
shared.opts.data['prompt_attention'] = 'Fixed attention' # otherwise need to deal with class_tokens_mask
|
||||
shared.opts.data['prompt_attention'] = 'fixed' # otherwise need to deal with class_tokens_mask
|
||||
p.task_args['input_id_images'] = input_images
|
||||
p.task_args['start_merge_step'] = int(start * p.steps)
|
||||
p.task_args['prompt'] = p.all_prompts[0] if p.all_prompts is not None else p.prompt
|
||||
|
||||
+46
-42
@@ -40,8 +40,6 @@ def atomically_save_image():
|
||||
except Exception:
|
||||
shared.log.warning(f'Save: unknown image format: {extension}')
|
||||
image_format = 'JPEG'
|
||||
if shared.opts.image_watermark_enabled or (shared.opts.image_watermark_position != 'none' and shared.opts.image_watermark_image != ''):
|
||||
image = set_watermark(image, shared.opts.image_watermark)
|
||||
exifinfo = (exifinfo or "") if shared.opts.image_metadata else ""
|
||||
# additional metadata saved in files
|
||||
if shared.opts.save_txt and len(exifinfo) > 0:
|
||||
@@ -153,6 +151,11 @@ def save_image(image,
|
||||
info = image.info.get(pnginfo_section_name, '')
|
||||
if info is not None:
|
||||
pnginfo[pnginfo_section_name] = info
|
||||
|
||||
wm_text = getattr(p, 'watermark_text', shared.opts.image_watermark)
|
||||
wm_image = getattr(p, 'watermark_image', shared.opts.image_watermark_image)
|
||||
image = set_watermark(image, wm_text, wm_image)
|
||||
|
||||
params = script_callbacks.ImageSaveParams(image, p, filename, pnginfo)
|
||||
params.filename = namegen.sanitize(filename)
|
||||
dirname = os.path.dirname(params.filename)
|
||||
@@ -361,53 +364,54 @@ def flatten(img, bgcolor):
|
||||
return img.convert('RGB')
|
||||
|
||||
|
||||
def draw_overlay(im, text):
|
||||
def draw_overlay(im, text: str = '', y_offset: int = 0):
|
||||
d = ImageDraw.Draw(im)
|
||||
fontsize = (im.width + im.height) // 50
|
||||
font = get_font(fontsize)
|
||||
d.text((fontsize//2, fontsize//2), text, font=font, fill=shared.opts.font_color)
|
||||
d.text((fontsize//2, fontsize//2 + y_offset), text, font=font, fill=shared.opts.font_color)
|
||||
return im
|
||||
|
||||
|
||||
def set_watermark(image, watermark):
|
||||
if shared.opts.image_watermark_position != 'none': # visible watermark
|
||||
wm_image = None
|
||||
try:
|
||||
wm_image = Image.open(shared.opts.image_watermark_image)
|
||||
if wm_image.mode != 'RGBA':
|
||||
wm_image = wm_image.convert('RGBA')
|
||||
except Exception as e:
|
||||
shared.log.warning(f'Set image watermark: fn="{shared.opts.image_watermark_image}" {e}')
|
||||
if wm_image is not None:
|
||||
if shared.opts.image_watermark_position == 'top/left':
|
||||
position = (0, 0)
|
||||
elif shared.opts.image_watermark_position == 'top/right':
|
||||
position = (image.width - wm_image.width, 0)
|
||||
elif shared.opts.image_watermark_position == 'bottom/left':
|
||||
position = (0, image.height - wm_image.height)
|
||||
elif shared.opts.image_watermark_position == 'bottom/right':
|
||||
position = (image.width - wm_image.width, image.height - wm_image.height)
|
||||
elif shared.opts.image_watermark_position == 'center':
|
||||
position = ((image.width - wm_image.width) // 2, (image.height - wm_image.height) // 2)
|
||||
else:
|
||||
position = (random.randint(0, image.width - wm_image.width), random.randint(0, image.height - wm_image.height))
|
||||
def set_watermark(image, wm_text: str = None, wm_image: Image.Image = None):
|
||||
if shared.opts.image_watermark_position != 'none' and wm_image is not None: # visible watermark
|
||||
if isinstance(wm_image, str):
|
||||
try:
|
||||
for x in range(wm_image.width):
|
||||
for y in range(wm_image.height):
|
||||
rgba = wm_image.getpixel((x, y))
|
||||
orig = image.getpixel((x+position[0], y+position[1]))
|
||||
# alpha blend
|
||||
a = rgba[3] / 255
|
||||
r = int(rgba[0] * a + orig[0] * (1 - a))
|
||||
g = int(rgba[1] * a + orig[1] * (1 - a))
|
||||
b = int(rgba[2] * a + orig[2] * (1 - a))
|
||||
if not a == 0:
|
||||
image.putpixel((x+position[0], y+position[1]), (r, g, b))
|
||||
shared.log.debug(f'Set image watermark: fn="{shared.opts.image_watermark_image}" image={wm_image} position={position}')
|
||||
wm_image = Image.open(wm_image)
|
||||
except Exception as e:
|
||||
shared.log.warning(f'Set image watermark: image={wm_image} {e}')
|
||||
return image
|
||||
if isinstance(wm_image, Image.Image):
|
||||
if wm_image.mode != 'RGBA':
|
||||
wm_image = wm_image.convert('RGBA')
|
||||
if shared.opts.image_watermark_position == 'top/left':
|
||||
position = (0, 0)
|
||||
elif shared.opts.image_watermark_position == 'top/right':
|
||||
position = (image.width - wm_image.width, 0)
|
||||
elif shared.opts.image_watermark_position == 'bottom/left':
|
||||
position = (0, image.height - wm_image.height)
|
||||
elif shared.opts.image_watermark_position == 'bottom/right':
|
||||
position = (image.width - wm_image.width, image.height - wm_image.height)
|
||||
elif shared.opts.image_watermark_position == 'center':
|
||||
position = ((image.width - wm_image.width) // 2, (image.height - wm_image.height) // 2)
|
||||
else:
|
||||
position = (random.randint(0, image.width - wm_image.width), random.randint(0, image.height - wm_image.height))
|
||||
try:
|
||||
for x in range(wm_image.width):
|
||||
for y in range(wm_image.height):
|
||||
rgba = wm_image.getpixel((x, y))
|
||||
orig = image.getpixel((x+position[0], y+position[1]))
|
||||
# alpha blend
|
||||
a = rgba[3] / 255
|
||||
r = int(rgba[0] * a + orig[0] * (1 - a))
|
||||
g = int(rgba[1] * a + orig[1] * (1 - a))
|
||||
b = int(rgba[2] * a + orig[2] * (1 - a))
|
||||
if not a == 0:
|
||||
image.putpixel((x+position[0], y+position[1]), (r, g, b))
|
||||
shared.log.debug(f'Set image watermark: image={wm_image} position={position}')
|
||||
except Exception as e:
|
||||
shared.log.warning(f'Set image watermark: image={wm_image} {e}')
|
||||
|
||||
if shared.opts.image_watermark_enabled: # invisible watermark
|
||||
if shared.opts.image_watermark_enabled and wm_text is not None: # invisible watermark
|
||||
from imwatermark import WatermarkEncoder
|
||||
wm_type = 'bytes'
|
||||
wm_method = 'dwtDctSvd'
|
||||
@@ -416,16 +420,16 @@ def set_watermark(image, watermark):
|
||||
info = image.info
|
||||
data = np.asarray(image)
|
||||
encoder = WatermarkEncoder()
|
||||
text = f"{watermark:<{length}}"[:length]
|
||||
text = f"{wm_text:<{length}}"[:length]
|
||||
bytearr = text.encode(encoding='ascii', errors='ignore')
|
||||
try:
|
||||
encoder.set_watermark(wm_type, bytearr)
|
||||
encoded = encoder.encode(data, wm_method)
|
||||
image = Image.fromarray(encoded)
|
||||
image.info = info
|
||||
shared.log.debug(f'Set invisible watermark: {watermark} method={wm_method} bits={wm_length}')
|
||||
shared.log.debug(f'Set invisible watermark: {wm_text} method={wm_method} bits={wm_length}')
|
||||
except Exception as e:
|
||||
shared.log.warning(f'Set invisible watermark error: {watermark} method={wm_method} bits={wm_length} {e}')
|
||||
shared.log.warning(f'Set invisible watermark error: {wm_text} method={wm_method} bits={wm_length} {e}')
|
||||
|
||||
return image
|
||||
|
||||
|
||||
@@ -34,10 +34,10 @@ class FilenameGenerator:
|
||||
'timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp),
|
||||
'job_timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp),
|
||||
|
||||
'model': lambda self: shared.sd_model.sd_checkpoint_info.title if shared.sd_loaded else '',
|
||||
'model_shortname': lambda self: shared.sd_model.sd_checkpoint_info.model_name if shared.sd_loaded else '',
|
||||
'model_name': lambda self: shared.sd_model.sd_checkpoint_info.model_name if shared.sd_loaded else '',
|
||||
'model_hash': lambda self: shared.sd_model.sd_checkpoint_info.shorthash if shared.sd_loaded else '',
|
||||
'model': lambda self: shared.sd_model.sd_checkpoint_info.title if shared.sd_loaded and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None else '',
|
||||
'model_shortname': lambda self: shared.sd_model.sd_checkpoint_info.model_name if shared.sd_loaded and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None else '',
|
||||
'model_name': lambda self: shared.sd_model.sd_checkpoint_info.model_name if shared.sd_loaded and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None else '',
|
||||
'model_hash': lambda self: shared.sd_model.sd_checkpoint_info.shorthash if shared.sd_loaded and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None else '',
|
||||
|
||||
'prompt': lambda self: self.prompt_full(),
|
||||
'prompt_no_styles': lambda self: self.prompt_no_style(),
|
||||
|
||||
+19
-1
@@ -137,6 +137,7 @@ def img2img(id_task: str, state: str, mode: int,
|
||||
inpaint_full_res, inpaint_full_res_padding, inpainting_mask_invert,
|
||||
img2img_batch_files, img2img_batch_input_dir, img2img_batch_output_dir, img2img_batch_inpaint_mask_dir,
|
||||
hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio,
|
||||
enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, hr_refiner_start, refiner_prompt, refiner_negative,
|
||||
override_settings_texts,
|
||||
*args): # pylint: disable=unused-argument
|
||||
|
||||
@@ -214,7 +215,6 @@ def img2img(id_task: str, state: str, mode: int,
|
||||
subseed_strength=subseed_strength,
|
||||
seed_resize_from_h=seed_resize_from_h,
|
||||
seed_resize_from_w=seed_resize_from_w,
|
||||
seed_enable_extras=True,
|
||||
sampler_name = processing.get_sampler_name(sampler_index, img=True),
|
||||
batch_size=batch_size,
|
||||
n_iter=n_iter,
|
||||
@@ -247,6 +247,23 @@ def img2img(id_task: str, state: str, mode: int,
|
||||
inpainting_mask_invert=inpainting_mask_invert,
|
||||
hdr_mode=hdr_mode, hdr_brightness=hdr_brightness, hdr_color=hdr_color, hdr_sharpen=hdr_sharpen, hdr_clamp=hdr_clamp,
|
||||
hdr_boundary=hdr_boundary, hdr_threshold=hdr_threshold, hdr_maximize=hdr_maximize, hdr_max_center=hdr_max_center, hdr_max_boundry=hdr_max_boundry, hdr_color_picker=hdr_color_picker, hdr_tint_ratio=hdr_tint_ratio,
|
||||
# refiner
|
||||
enable_hr=enable_hr,
|
||||
hr_denoising_strength=hr_denoising_strength,
|
||||
hr_scale=hr_scale,
|
||||
hr_resize_mode=hr_resize_mode,
|
||||
hr_resize_context=hr_resize_context,
|
||||
hr_upscaler=hr_upscaler,
|
||||
hr_force=hr_force,
|
||||
hr_second_pass_steps=hr_second_pass_steps,
|
||||
hr_resize_x=hr_resize_x,
|
||||
hr_resize_y=hr_resize_y,
|
||||
hr_sampler_name = processing.get_sampler_name(hr_sampler_index),
|
||||
refiner_steps=refiner_steps,
|
||||
hr_refiner_start=hr_refiner_start,
|
||||
refiner_prompt=refiner_prompt,
|
||||
refiner_negative=refiner_negative,
|
||||
# override
|
||||
override_settings=override_settings,
|
||||
)
|
||||
p.scripts = modules.scripts.scripts_img2img
|
||||
@@ -267,6 +284,7 @@ def img2img(id_task: str, state: str, mode: int,
|
||||
processed = modules.scripts.scripts_img2img.run(p, *args)
|
||||
if processed is None:
|
||||
processed = processing.process_images(p)
|
||||
processed = modules.scripts.scripts_img2img.after(p, processed, *args)
|
||||
p.close()
|
||||
generation_info_js = processed.js() if processed is not None else ''
|
||||
if processed is None:
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .sdxl_instantir import InstantIRPipeline
|
||||
from .lcm_single_step_scheduler import LCMSingleStepScheduler
|
||||
from .ip_adapter.utils import init_adapter_in_unet, load_adapter_to_pipe
|
||||
@@ -0,0 +1,983 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
|
||||
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
||||
from diffusers.loaders.single_file_model import FromOriginalModelMixin
|
||||
from diffusers.utils import BaseOutput, logging
|
||||
from diffusers.models.attention_processor import (
|
||||
ADDED_KV_ATTENTION_PROCESSORS,
|
||||
CROSS_ATTENTION_PROCESSORS,
|
||||
AttentionProcessor,
|
||||
AttnAddedKVProcessor,
|
||||
AttnProcessor,
|
||||
)
|
||||
from diffusers.models.embeddings import TextImageProjection, TextImageTimeEmbedding, TextTimeEmbedding, TimestepEmbedding, Timesteps
|
||||
from diffusers.models.modeling_utils import ModelMixin
|
||||
from diffusers.models.unets.unet_2d_blocks import (
|
||||
CrossAttnDownBlock2D,
|
||||
DownBlock2D,
|
||||
UNetMidBlock2D,
|
||||
UNetMidBlock2DCrossAttn,
|
||||
get_down_block,
|
||||
)
|
||||
from diffusers.models.unets.unet_2d_condition import UNet2DConditionModel
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
||||
|
||||
|
||||
class ZeroConv(nn.Module):
|
||||
def __init__(self, label_nc, norm_nc, mask=False):
|
||||
super().__init__()
|
||||
self.zero_conv = zero_module(nn.Conv2d(label_nc+norm_nc, norm_nc, 1, 1, 0))
|
||||
self.mask = mask
|
||||
|
||||
def forward(self, hidden_states, h_ori=None):
|
||||
# with torch.cuda.amp.autocast(enabled=False, dtype=torch.float32):
|
||||
c, h = hidden_states
|
||||
if not self.mask:
|
||||
h = self.zero_conv(torch.cat([c, h], dim=1))
|
||||
else:
|
||||
h = self.zero_conv(torch.cat([c, h], dim=1)) * torch.zeros_like(h)
|
||||
if h_ori is not None:
|
||||
h = torch.cat([h_ori, h], dim=1)
|
||||
return h
|
||||
|
||||
|
||||
class SFT(nn.Module):
|
||||
def __init__(self, label_nc, norm_nc, mask=False):
|
||||
super().__init__()
|
||||
|
||||
# param_free_norm_type = str(parsed.group(1))
|
||||
ks = 3
|
||||
pw = ks // 2
|
||||
|
||||
self.mask = mask
|
||||
|
||||
nhidden = 128
|
||||
|
||||
self.mlp_shared = nn.Sequential(
|
||||
nn.Conv2d(label_nc, nhidden, kernel_size=ks, padding=pw),
|
||||
nn.SiLU()
|
||||
)
|
||||
self.mul = nn.Conv2d(nhidden, norm_nc, kernel_size=ks, padding=pw)
|
||||
self.add = nn.Conv2d(nhidden, norm_nc, kernel_size=ks, padding=pw)
|
||||
|
||||
def forward(self, hidden_states, mask=False):
|
||||
|
||||
c, h = hidden_states
|
||||
mask = mask or self.mask
|
||||
assert mask is False
|
||||
|
||||
actv = self.mlp_shared(c)
|
||||
gamma = self.mul(actv)
|
||||
beta = self.add(actv)
|
||||
|
||||
if self.mask:
|
||||
gamma = gamma * torch.zeros_like(gamma)
|
||||
beta = beta * torch.zeros_like(beta)
|
||||
# gamma_ori, gamma_res = torch.split(gamma, [h_ori_c, h_c], dim=1)
|
||||
# beta_ori, beta_res = torch.split(beta, [h_ori_c, h_c], dim=1)
|
||||
# print(gamma_ori.mean(), gamma_res.mean(), beta_ori.mean(), beta_res.mean())
|
||||
h = h * (gamma + 1) + beta
|
||||
# sample_ori, sample_res = torch.split(h, [h_ori_c, h_c], dim=1)
|
||||
# print(sample_ori.mean(), sample_res.mean())
|
||||
|
||||
return h
|
||||
|
||||
|
||||
@dataclass
|
||||
class AggregatorOutput(BaseOutput):
|
||||
"""
|
||||
The output of [`Aggregator`].
|
||||
|
||||
Args:
|
||||
down_block_res_samples (`tuple[torch.Tensor]`):
|
||||
A tuple of downsample activations at different resolutions for each downsampling block. Each tensor should
|
||||
be of shape `(batch_size, channel * resolution, height //resolution, width // resolution)`. Output can be
|
||||
used to condition the original UNet's downsampling activations.
|
||||
mid_down_block_re_sample (`torch.Tensor`):
|
||||
The activation of the midde block (the lowest sample resolution). Each tensor should be of shape
|
||||
`(batch_size, channel * lowest_resolution, height // lowest_resolution, width // lowest_resolution)`.
|
||||
Output can be used to condition the original UNet's middle block activation.
|
||||
"""
|
||||
|
||||
down_block_res_samples: Tuple[torch.Tensor]
|
||||
mid_block_res_sample: torch.Tensor
|
||||
|
||||
|
||||
class ConditioningEmbedding(nn.Module):
|
||||
"""
|
||||
Quoting from https://arxiv.org/abs/2302.05543: "Stable Diffusion uses a pre-processing method similar to VQ-GAN
|
||||
[11] to convert the entire dataset of 512 × 512 images into smaller 64 × 64 “latent images” for stabilized
|
||||
training. This requires ControlNets to convert image-based conditions to 64 × 64 feature space to match the
|
||||
convolution size. We use a tiny network E(·) of four convolution layers with 4 × 4 kernels and 2 × 2 strides
|
||||
(activated by ReLU, channels are 16, 32, 64, 128, initialized with Gaussian weights, trained jointly with the full
|
||||
model) to encode image-space conditions ... into feature maps ..."
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
conditioning_embedding_channels: int,
|
||||
conditioning_channels: int = 3,
|
||||
block_out_channels: Tuple[int, ...] = (16, 32, 96, 256),
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.conv_in = nn.Conv2d(conditioning_channels, block_out_channels[0], kernel_size=3, padding=1)
|
||||
|
||||
self.blocks = nn.ModuleList([])
|
||||
|
||||
for i in range(len(block_out_channels) - 1):
|
||||
channel_in = block_out_channels[i]
|
||||
channel_out = block_out_channels[i + 1]
|
||||
self.blocks.append(nn.Conv2d(channel_in, channel_in, kernel_size=3, padding=1))
|
||||
self.blocks.append(nn.Conv2d(channel_in, channel_out, kernel_size=3, padding=1, stride=2))
|
||||
|
||||
self.conv_out = zero_module(
|
||||
nn.Conv2d(block_out_channels[-1], conditioning_embedding_channels, kernel_size=3, padding=1)
|
||||
)
|
||||
|
||||
def forward(self, conditioning):
|
||||
embedding = self.conv_in(conditioning)
|
||||
embedding = F.silu(embedding)
|
||||
|
||||
for block in self.blocks:
|
||||
embedding = block(embedding)
|
||||
embedding = F.silu(embedding)
|
||||
|
||||
embedding = self.conv_out(embedding)
|
||||
|
||||
return embedding
|
||||
|
||||
|
||||
class Aggregator(ModelMixin, ConfigMixin, FromOriginalModelMixin):
|
||||
"""
|
||||
Aggregator model.
|
||||
|
||||
Args:
|
||||
in_channels (`int`, defaults to 4):
|
||||
The number of channels in the input sample.
|
||||
flip_sin_to_cos (`bool`, defaults to `True`):
|
||||
Whether to flip the sin to cos in the time embedding.
|
||||
freq_shift (`int`, defaults to 0):
|
||||
The frequency shift to apply to the time embedding.
|
||||
down_block_types (`tuple[str]`, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`):
|
||||
The tuple of downsample blocks to use.
|
||||
only_cross_attention (`Union[bool, Tuple[bool]]`, defaults to `False`):
|
||||
block_out_channels (`tuple[int]`, defaults to `(320, 640, 1280, 1280)`):
|
||||
The tuple of output channels for each block.
|
||||
layers_per_block (`int`, defaults to 2):
|
||||
The number of layers per block.
|
||||
downsample_padding (`int`, defaults to 1):
|
||||
The padding to use for the downsampling convolution.
|
||||
mid_block_scale_factor (`float`, defaults to 1):
|
||||
The scale factor to use for the mid block.
|
||||
act_fn (`str`, defaults to "silu"):
|
||||
The activation function to use.
|
||||
norm_num_groups (`int`, *optional*, defaults to 32):
|
||||
The number of groups to use for the normalization. If None, normalization and activation layers is skipped
|
||||
in post-processing.
|
||||
norm_eps (`float`, defaults to 1e-5):
|
||||
The epsilon to use for the normalization.
|
||||
cross_attention_dim (`int`, defaults to 1280):
|
||||
The dimension of the cross attention features.
|
||||
transformer_layers_per_block (`int` or `Tuple[int]`, *optional*, defaults to 1):
|
||||
The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for
|
||||
[`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],
|
||||
[`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`].
|
||||
encoder_hid_dim (`int`, *optional*, defaults to None):
|
||||
If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim`
|
||||
dimension to `cross_attention_dim`.
|
||||
encoder_hid_dim_type (`str`, *optional*, defaults to `None`):
|
||||
If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text
|
||||
embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`.
|
||||
attention_head_dim (`Union[int, Tuple[int]]`, defaults to 8):
|
||||
The dimension of the attention heads.
|
||||
use_linear_projection (`bool`, defaults to `False`):
|
||||
class_embed_type (`str`, *optional*, defaults to `None`):
|
||||
The type of class embedding to use which is ultimately summed with the time embeddings. Choose from None,
|
||||
`"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`.
|
||||
addition_embed_type (`str`, *optional*, defaults to `None`):
|
||||
Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or
|
||||
"text". "text" will use the `TextTimeEmbedding` layer.
|
||||
num_class_embeds (`int`, *optional*, defaults to 0):
|
||||
Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing
|
||||
class conditioning with `class_embed_type` equal to `None`.
|
||||
upcast_attention (`bool`, defaults to `False`):
|
||||
resnet_time_scale_shift (`str`, defaults to `"default"`):
|
||||
Time scale shift config for ResNet blocks (see `ResnetBlock2D`). Choose from `default` or `scale_shift`.
|
||||
projection_class_embeddings_input_dim (`int`, *optional*, defaults to `None`):
|
||||
The dimension of the `class_labels` input when `class_embed_type="projection"`. Required when
|
||||
`class_embed_type="projection"`.
|
||||
controlnet_conditioning_channel_order (`str`, defaults to `"rgb"`):
|
||||
The channel order of conditional image. Will convert to `rgb` if it's `bgr`.
|
||||
conditioning_embedding_out_channels (`tuple[int]`, *optional*, defaults to `(16, 32, 96, 256)`):
|
||||
The tuple of output channel for each block in the `conditioning_embedding` layer.
|
||||
global_pool_conditions (`bool`, defaults to `False`):
|
||||
TODO(Patrick) - unused parameter.
|
||||
addition_embed_type_num_heads (`int`, defaults to 64):
|
||||
The number of heads to use for the `TextTimeEmbedding` layer.
|
||||
"""
|
||||
|
||||
_supports_gradient_checkpointing = True
|
||||
|
||||
@register_to_config
|
||||
def __init__(
|
||||
self,
|
||||
in_channels: int = 4,
|
||||
conditioning_channels: int = 3,
|
||||
flip_sin_to_cos: bool = True,
|
||||
freq_shift: int = 0,
|
||||
down_block_types: Tuple[str, ...] = (
|
||||
"CrossAttnDownBlock2D",
|
||||
"CrossAttnDownBlock2D",
|
||||
"CrossAttnDownBlock2D",
|
||||
"DownBlock2D",
|
||||
),
|
||||
mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn",
|
||||
only_cross_attention: Union[bool, Tuple[bool]] = False,
|
||||
block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280),
|
||||
layers_per_block: int = 2,
|
||||
downsample_padding: int = 1,
|
||||
mid_block_scale_factor: float = 1,
|
||||
act_fn: str = "silu",
|
||||
norm_num_groups: Optional[int] = 32,
|
||||
norm_eps: float = 1e-5,
|
||||
cross_attention_dim: int = 1280,
|
||||
transformer_layers_per_block: Union[int, Tuple[int, ...]] = 1,
|
||||
encoder_hid_dim: Optional[int] = None,
|
||||
encoder_hid_dim_type: Optional[str] = None,
|
||||
attention_head_dim: Union[int, Tuple[int, ...]] = 8,
|
||||
num_attention_heads: Optional[Union[int, Tuple[int, ...]]] = None,
|
||||
use_linear_projection: bool = False,
|
||||
class_embed_type: Optional[str] = None,
|
||||
addition_embed_type: Optional[str] = None,
|
||||
addition_time_embed_dim: Optional[int] = None,
|
||||
num_class_embeds: Optional[int] = None,
|
||||
upcast_attention: bool = False,
|
||||
resnet_time_scale_shift: str = "default",
|
||||
projection_class_embeddings_input_dim: Optional[int] = None,
|
||||
controlnet_conditioning_channel_order: str = "rgb",
|
||||
conditioning_embedding_out_channels: Optional[Tuple[int, ...]] = (16, 32, 96, 256),
|
||||
global_pool_conditions: bool = False,
|
||||
addition_embed_type_num_heads: int = 64,
|
||||
pad_concat: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
# If `num_attention_heads` is not defined (which is the case for most models)
|
||||
# it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
|
||||
# The reason for this behavior is to correct for incorrectly named variables that were introduced
|
||||
# when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
|
||||
# Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
|
||||
# which is why we correct for the naming here.
|
||||
num_attention_heads = num_attention_heads or attention_head_dim
|
||||
self.pad_concat = pad_concat
|
||||
|
||||
# Check inputs
|
||||
if len(block_out_channels) != len(down_block_types):
|
||||
raise ValueError(
|
||||
f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}."
|
||||
)
|
||||
|
||||
if not isinstance(only_cross_attention, bool) and len(only_cross_attention) != len(down_block_types):
|
||||
raise ValueError(
|
||||
f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}."
|
||||
)
|
||||
|
||||
if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types):
|
||||
raise ValueError(
|
||||
f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}."
|
||||
)
|
||||
|
||||
if isinstance(transformer_layers_per_block, int):
|
||||
transformer_layers_per_block = [transformer_layers_per_block] * len(down_block_types)
|
||||
|
||||
# input
|
||||
conv_in_kernel = 3
|
||||
conv_in_padding = (conv_in_kernel - 1) // 2
|
||||
self.conv_in = nn.Conv2d(
|
||||
in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding
|
||||
)
|
||||
|
||||
# time
|
||||
time_embed_dim = block_out_channels[0] * 4
|
||||
self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift)
|
||||
timestep_input_dim = block_out_channels[0]
|
||||
self.time_embedding = TimestepEmbedding(
|
||||
timestep_input_dim,
|
||||
time_embed_dim,
|
||||
act_fn=act_fn,
|
||||
)
|
||||
|
||||
if encoder_hid_dim_type is None and encoder_hid_dim is not None:
|
||||
encoder_hid_dim_type = "text_proj"
|
||||
self.register_to_config(encoder_hid_dim_type=encoder_hid_dim_type)
|
||||
logger.info("encoder_hid_dim_type defaults to 'text_proj' as `encoder_hid_dim` is defined.")
|
||||
|
||||
if encoder_hid_dim is None and encoder_hid_dim_type is not None:
|
||||
raise ValueError(
|
||||
f"`encoder_hid_dim` has to be defined when `encoder_hid_dim_type` is set to {encoder_hid_dim_type}."
|
||||
)
|
||||
|
||||
if encoder_hid_dim_type == "text_proj":
|
||||
self.encoder_hid_proj = nn.Linear(encoder_hid_dim, cross_attention_dim)
|
||||
elif encoder_hid_dim_type == "text_image_proj":
|
||||
# image_embed_dim DOESN'T have to be `cross_attention_dim`. To not clutter the __init__ too much
|
||||
# they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use
|
||||
# case when `addition_embed_type == "text_image_proj"` (Kandinsky 2.1)`
|
||||
self.encoder_hid_proj = TextImageProjection(
|
||||
text_embed_dim=encoder_hid_dim,
|
||||
image_embed_dim=cross_attention_dim,
|
||||
cross_attention_dim=cross_attention_dim,
|
||||
)
|
||||
|
||||
elif encoder_hid_dim_type is not None:
|
||||
raise ValueError(
|
||||
f"encoder_hid_dim_type: {encoder_hid_dim_type} must be None, 'text_proj' or 'text_image_proj'."
|
||||
)
|
||||
else:
|
||||
self.encoder_hid_proj = None
|
||||
|
||||
# class embedding
|
||||
if class_embed_type is None and num_class_embeds is not None:
|
||||
self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)
|
||||
elif class_embed_type == "timestep":
|
||||
self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)
|
||||
elif class_embed_type == "identity":
|
||||
self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)
|
||||
elif class_embed_type == "projection":
|
||||
if projection_class_embeddings_input_dim is None:
|
||||
raise ValueError(
|
||||
"`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set"
|
||||
)
|
||||
# The projection `class_embed_type` is the same as the timestep `class_embed_type` except
|
||||
# 1. the `class_labels` inputs are not first converted to sinusoidal embeddings
|
||||
# 2. it projects from an arbitrary input dimension.
|
||||
#
|
||||
# Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations.
|
||||
# When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings.
|
||||
# As a result, `TimestepEmbedding` can be passed arbitrary vectors.
|
||||
self.class_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim)
|
||||
else:
|
||||
self.class_embedding = None
|
||||
|
||||
if addition_embed_type == "text":
|
||||
if encoder_hid_dim is not None:
|
||||
text_time_embedding_from_dim = encoder_hid_dim
|
||||
else:
|
||||
text_time_embedding_from_dim = cross_attention_dim
|
||||
|
||||
self.add_embedding = TextTimeEmbedding(
|
||||
text_time_embedding_from_dim, time_embed_dim, num_heads=addition_embed_type_num_heads
|
||||
)
|
||||
elif addition_embed_type == "text_image":
|
||||
# text_embed_dim and image_embed_dim DON'T have to be `cross_attention_dim`. To not clutter the __init__ too much
|
||||
# they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use
|
||||
# case when `addition_embed_type == "text_image"` (Kandinsky 2.1)`
|
||||
self.add_embedding = TextImageTimeEmbedding(
|
||||
text_embed_dim=cross_attention_dim, image_embed_dim=cross_attention_dim, time_embed_dim=time_embed_dim
|
||||
)
|
||||
elif addition_embed_type == "text_time":
|
||||
self.add_time_proj = Timesteps(addition_time_embed_dim, flip_sin_to_cos, freq_shift)
|
||||
self.add_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim)
|
||||
|
||||
elif addition_embed_type is not None:
|
||||
raise ValueError(f"addition_embed_type: {addition_embed_type} must be None, 'text' or 'text_image'.")
|
||||
|
||||
# control net conditioning embedding
|
||||
self.ref_conv_in = nn.Conv2d(
|
||||
in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding
|
||||
)
|
||||
|
||||
self.down_blocks = nn.ModuleList([])
|
||||
self.controlnet_down_blocks = nn.ModuleList([])
|
||||
|
||||
if isinstance(only_cross_attention, bool):
|
||||
only_cross_attention = [only_cross_attention] * len(down_block_types)
|
||||
|
||||
if isinstance(attention_head_dim, int):
|
||||
attention_head_dim = (attention_head_dim,) * len(down_block_types)
|
||||
|
||||
if isinstance(num_attention_heads, int):
|
||||
num_attention_heads = (num_attention_heads,) * len(down_block_types)
|
||||
|
||||
# down
|
||||
output_channel = block_out_channels[0]
|
||||
|
||||
# controlnet_block = ZeroConv(output_channel, output_channel)
|
||||
controlnet_block = nn.Sequential(
|
||||
SFT(output_channel, output_channel),
|
||||
zero_module(nn.Conv2d(output_channel, output_channel, kernel_size=1))
|
||||
)
|
||||
self.controlnet_down_blocks.append(controlnet_block)
|
||||
|
||||
for i, down_block_type in enumerate(down_block_types):
|
||||
input_channel = output_channel
|
||||
output_channel = block_out_channels[i]
|
||||
is_final_block = i == len(block_out_channels) - 1
|
||||
|
||||
down_block = get_down_block(
|
||||
down_block_type,
|
||||
num_layers=layers_per_block,
|
||||
transformer_layers_per_block=transformer_layers_per_block[i],
|
||||
in_channels=input_channel,
|
||||
out_channels=output_channel,
|
||||
temb_channels=time_embed_dim,
|
||||
add_downsample=not is_final_block,
|
||||
resnet_eps=norm_eps,
|
||||
resnet_act_fn=act_fn,
|
||||
resnet_groups=norm_num_groups,
|
||||
cross_attention_dim=cross_attention_dim,
|
||||
num_attention_heads=num_attention_heads[i],
|
||||
attention_head_dim=attention_head_dim[i] if attention_head_dim[i] is not None else output_channel,
|
||||
downsample_padding=downsample_padding,
|
||||
use_linear_projection=use_linear_projection,
|
||||
only_cross_attention=only_cross_attention[i],
|
||||
upcast_attention=upcast_attention,
|
||||
resnet_time_scale_shift=resnet_time_scale_shift,
|
||||
)
|
||||
self.down_blocks.append(down_block)
|
||||
|
||||
for _ in range(layers_per_block):
|
||||
# controlnet_block = ZeroConv(output_channel, output_channel)
|
||||
controlnet_block = nn.Sequential(
|
||||
SFT(output_channel, output_channel),
|
||||
zero_module(nn.Conv2d(output_channel, output_channel, kernel_size=1))
|
||||
)
|
||||
self.controlnet_down_blocks.append(controlnet_block)
|
||||
|
||||
if not is_final_block:
|
||||
# controlnet_block = ZeroConv(output_channel, output_channel)
|
||||
controlnet_block = nn.Sequential(
|
||||
SFT(output_channel, output_channel),
|
||||
zero_module(nn.Conv2d(output_channel, output_channel, kernel_size=1))
|
||||
)
|
||||
self.controlnet_down_blocks.append(controlnet_block)
|
||||
|
||||
# mid
|
||||
mid_block_channel = block_out_channels[-1]
|
||||
|
||||
# controlnet_block = ZeroConv(mid_block_channel, mid_block_channel)
|
||||
controlnet_block = nn.Sequential(
|
||||
SFT(mid_block_channel, mid_block_channel),
|
||||
zero_module(nn.Conv2d(mid_block_channel, mid_block_channel, kernel_size=1))
|
||||
)
|
||||
self.controlnet_mid_block = controlnet_block
|
||||
|
||||
if mid_block_type == "UNetMidBlock2DCrossAttn":
|
||||
self.mid_block = UNetMidBlock2DCrossAttn(
|
||||
transformer_layers_per_block=transformer_layers_per_block[-1],
|
||||
in_channels=mid_block_channel,
|
||||
temb_channels=time_embed_dim,
|
||||
resnet_eps=norm_eps,
|
||||
resnet_act_fn=act_fn,
|
||||
output_scale_factor=mid_block_scale_factor,
|
||||
resnet_time_scale_shift=resnet_time_scale_shift,
|
||||
cross_attention_dim=cross_attention_dim,
|
||||
num_attention_heads=num_attention_heads[-1],
|
||||
resnet_groups=norm_num_groups,
|
||||
use_linear_projection=use_linear_projection,
|
||||
upcast_attention=upcast_attention,
|
||||
)
|
||||
elif mid_block_type == "UNetMidBlock2D":
|
||||
self.mid_block = UNetMidBlock2D(
|
||||
in_channels=block_out_channels[-1],
|
||||
temb_channels=time_embed_dim,
|
||||
num_layers=0,
|
||||
resnet_eps=norm_eps,
|
||||
resnet_act_fn=act_fn,
|
||||
output_scale_factor=mid_block_scale_factor,
|
||||
resnet_groups=norm_num_groups,
|
||||
resnet_time_scale_shift=resnet_time_scale_shift,
|
||||
add_attention=False,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"unknown mid_block_type : {mid_block_type}")
|
||||
|
||||
@classmethod
|
||||
def from_unet(
|
||||
cls,
|
||||
unet: UNet2DConditionModel,
|
||||
controlnet_conditioning_channel_order: str = "rgb",
|
||||
conditioning_embedding_out_channels: Optional[Tuple[int, ...]] = (16, 32, 96, 256),
|
||||
load_weights_from_unet: bool = True,
|
||||
conditioning_channels: int = 3,
|
||||
):
|
||||
r"""
|
||||
Instantiate a [`ControlNetModel`] from [`UNet2DConditionModel`].
|
||||
|
||||
Parameters:
|
||||
unet (`UNet2DConditionModel`):
|
||||
The UNet model weights to copy to the [`ControlNetModel`]. All configuration options are also copied
|
||||
where applicable.
|
||||
"""
|
||||
transformer_layers_per_block = (
|
||||
unet.config.transformer_layers_per_block if "transformer_layers_per_block" in unet.config else 1
|
||||
)
|
||||
encoder_hid_dim = unet.config.encoder_hid_dim if "encoder_hid_dim" in unet.config else None
|
||||
encoder_hid_dim_type = unet.config.encoder_hid_dim_type if "encoder_hid_dim_type" in unet.config else None
|
||||
addition_embed_type = unet.config.addition_embed_type if "addition_embed_type" in unet.config else None
|
||||
addition_time_embed_dim = (
|
||||
unet.config.addition_time_embed_dim if "addition_time_embed_dim" in unet.config else None
|
||||
)
|
||||
|
||||
controlnet = cls(
|
||||
encoder_hid_dim=encoder_hid_dim,
|
||||
encoder_hid_dim_type=encoder_hid_dim_type,
|
||||
addition_embed_type=addition_embed_type,
|
||||
addition_time_embed_dim=addition_time_embed_dim,
|
||||
transformer_layers_per_block=transformer_layers_per_block,
|
||||
in_channels=unet.config.in_channels,
|
||||
flip_sin_to_cos=unet.config.flip_sin_to_cos,
|
||||
freq_shift=unet.config.freq_shift,
|
||||
down_block_types=unet.config.down_block_types,
|
||||
only_cross_attention=unet.config.only_cross_attention,
|
||||
block_out_channels=unet.config.block_out_channels,
|
||||
layers_per_block=unet.config.layers_per_block,
|
||||
downsample_padding=unet.config.downsample_padding,
|
||||
mid_block_scale_factor=unet.config.mid_block_scale_factor,
|
||||
act_fn=unet.config.act_fn,
|
||||
norm_num_groups=unet.config.norm_num_groups,
|
||||
norm_eps=unet.config.norm_eps,
|
||||
cross_attention_dim=unet.config.cross_attention_dim,
|
||||
attention_head_dim=unet.config.attention_head_dim,
|
||||
num_attention_heads=unet.config.num_attention_heads,
|
||||
use_linear_projection=unet.config.use_linear_projection,
|
||||
class_embed_type=unet.config.class_embed_type,
|
||||
num_class_embeds=unet.config.num_class_embeds,
|
||||
upcast_attention=unet.config.upcast_attention,
|
||||
resnet_time_scale_shift=unet.config.resnet_time_scale_shift,
|
||||
projection_class_embeddings_input_dim=unet.config.projection_class_embeddings_input_dim,
|
||||
mid_block_type=unet.config.mid_block_type,
|
||||
controlnet_conditioning_channel_order=controlnet_conditioning_channel_order,
|
||||
conditioning_embedding_out_channels=conditioning_embedding_out_channels,
|
||||
conditioning_channels=conditioning_channels,
|
||||
)
|
||||
|
||||
if load_weights_from_unet:
|
||||
controlnet.conv_in.load_state_dict(unet.conv_in.state_dict())
|
||||
controlnet.ref_conv_in.load_state_dict(unet.conv_in.state_dict())
|
||||
controlnet.time_proj.load_state_dict(unet.time_proj.state_dict())
|
||||
controlnet.time_embedding.load_state_dict(unet.time_embedding.state_dict())
|
||||
|
||||
if controlnet.class_embedding:
|
||||
controlnet.class_embedding.load_state_dict(unet.class_embedding.state_dict())
|
||||
|
||||
if hasattr(controlnet, "add_embedding"):
|
||||
controlnet.add_embedding.load_state_dict(unet.add_embedding.state_dict())
|
||||
|
||||
controlnet.down_blocks.load_state_dict(unet.down_blocks.state_dict())
|
||||
controlnet.mid_block.load_state_dict(unet.mid_block.state_dict())
|
||||
|
||||
return controlnet
|
||||
|
||||
@property
|
||||
# Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors
|
||||
def attn_processors(self) -> Dict[str, AttentionProcessor]:
|
||||
r"""
|
||||
Returns:
|
||||
`dict` of attention processors: A dictionary containing all attention processors used in the model with
|
||||
indexed by its weight name.
|
||||
"""
|
||||
# set recursively
|
||||
processors = {}
|
||||
|
||||
def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]):
|
||||
if hasattr(module, "get_processor"):
|
||||
processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True)
|
||||
|
||||
for sub_name, child in module.named_children():
|
||||
fn_recursive_add_processors(f"{name}.{sub_name}", child, processors)
|
||||
|
||||
return processors
|
||||
|
||||
for name, module in self.named_children():
|
||||
fn_recursive_add_processors(name, module, processors)
|
||||
|
||||
return processors
|
||||
|
||||
# Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor
|
||||
def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]):
|
||||
r"""
|
||||
Sets the attention processor to use to compute attention.
|
||||
|
||||
Parameters:
|
||||
processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):
|
||||
The instantiated processor class or a dictionary of processor classes that will be set as the processor
|
||||
for **all** `Attention` layers.
|
||||
|
||||
If `processor` is a dict, the key needs to define the path to the corresponding cross attention
|
||||
processor. This is strongly recommended when setting trainable attention processors.
|
||||
|
||||
"""
|
||||
count = len(self.attn_processors.keys())
|
||||
|
||||
if isinstance(processor, dict) and len(processor) != count:
|
||||
raise ValueError(
|
||||
f"A dict of processors was passed, but the number of processors {len(processor)} does not match the"
|
||||
f" number of attention layers: {count}. Please make sure to pass {count} processor classes."
|
||||
)
|
||||
|
||||
def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor):
|
||||
if hasattr(module, "set_processor"):
|
||||
if not isinstance(processor, dict):
|
||||
module.set_processor(processor)
|
||||
else:
|
||||
module.set_processor(processor.pop(f"{name}.processor"))
|
||||
|
||||
for sub_name, child in module.named_children():
|
||||
fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor)
|
||||
|
||||
for name, module in self.named_children():
|
||||
fn_recursive_attn_processor(name, module, processor)
|
||||
|
||||
# Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor
|
||||
def set_default_attn_processor(self):
|
||||
"""
|
||||
Disables custom attention processors and sets the default attention implementation.
|
||||
"""
|
||||
if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()):
|
||||
processor = AttnAddedKVProcessor()
|
||||
elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()):
|
||||
processor = AttnProcessor()
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}"
|
||||
)
|
||||
|
||||
self.set_attn_processor(processor)
|
||||
|
||||
# Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attention_slice
|
||||
def set_attention_slice(self, slice_size: Union[str, int, List[int]]) -> None:
|
||||
r"""
|
||||
Enable sliced attention computation.
|
||||
|
||||
When this option is enabled, the attention module splits the input tensor in slices to compute attention in
|
||||
several steps. This is useful for saving some memory in exchange for a small decrease in speed.
|
||||
|
||||
Args:
|
||||
slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`):
|
||||
When `"auto"`, input to the attention heads is halved, so attention is computed in two steps. If
|
||||
`"max"`, maximum amount of memory is saved by running only one slice at a time. If a number is
|
||||
provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim`
|
||||
must be a multiple of `slice_size`.
|
||||
"""
|
||||
sliceable_head_dims = []
|
||||
|
||||
def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module):
|
||||
if hasattr(module, "set_attention_slice"):
|
||||
sliceable_head_dims.append(module.sliceable_head_dim)
|
||||
|
||||
for child in module.children():
|
||||
fn_recursive_retrieve_sliceable_dims(child)
|
||||
|
||||
# retrieve number of attention layers
|
||||
for module in self.children():
|
||||
fn_recursive_retrieve_sliceable_dims(module)
|
||||
|
||||
num_sliceable_layers = len(sliceable_head_dims)
|
||||
|
||||
if slice_size == "auto":
|
||||
# half the attention head size is usually a good trade-off between
|
||||
# speed and memory
|
||||
slice_size = [dim // 2 for dim in sliceable_head_dims]
|
||||
elif slice_size == "max":
|
||||
# make smallest slice possible
|
||||
slice_size = num_sliceable_layers * [1]
|
||||
|
||||
slice_size = num_sliceable_layers * [slice_size] if not isinstance(slice_size, list) else slice_size
|
||||
|
||||
if len(slice_size) != len(sliceable_head_dims):
|
||||
raise ValueError(
|
||||
f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different"
|
||||
f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}."
|
||||
)
|
||||
|
||||
for i in range(len(slice_size)):
|
||||
size = slice_size[i]
|
||||
dim = sliceable_head_dims[i]
|
||||
if size is not None and size > dim:
|
||||
raise ValueError(f"size {size} has to be smaller or equal to {dim}.")
|
||||
|
||||
# Recursively walk through all the children.
|
||||
# Any children which exposes the set_attention_slice method
|
||||
# gets the message
|
||||
def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]):
|
||||
if hasattr(module, "set_attention_slice"):
|
||||
module.set_attention_slice(slice_size.pop())
|
||||
|
||||
for child in module.children():
|
||||
fn_recursive_set_attention_slice(child, slice_size)
|
||||
|
||||
reversed_slice_size = list(reversed(slice_size))
|
||||
for module in self.children():
|
||||
fn_recursive_set_attention_slice(module, reversed_slice_size)
|
||||
|
||||
def process_encoder_hidden_states(
|
||||
self, encoder_hidden_states: torch.Tensor, added_cond_kwargs: Dict[str, Any]
|
||||
) -> torch.Tensor:
|
||||
if self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "text_proj":
|
||||
encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states)
|
||||
elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "text_image_proj":
|
||||
# Kandinsky 2.1 - style
|
||||
if "image_embeds" not in added_cond_kwargs:
|
||||
raise ValueError(
|
||||
f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'text_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`"
|
||||
)
|
||||
|
||||
image_embeds = added_cond_kwargs.get("image_embeds")
|
||||
encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states, image_embeds)
|
||||
elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "image_proj":
|
||||
# Kandinsky 2.2 - style
|
||||
if "image_embeds" not in added_cond_kwargs:
|
||||
raise ValueError(
|
||||
f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`"
|
||||
)
|
||||
image_embeds = added_cond_kwargs.get("image_embeds")
|
||||
encoder_hidden_states = self.encoder_hid_proj(image_embeds)
|
||||
elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "ip_image_proj":
|
||||
if "image_embeds" not in added_cond_kwargs:
|
||||
raise ValueError(
|
||||
f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'ip_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`"
|
||||
)
|
||||
image_embeds = added_cond_kwargs.get("image_embeds")
|
||||
image_embeds = self.encoder_hid_proj(image_embeds)
|
||||
encoder_hidden_states = (encoder_hidden_states, image_embeds)
|
||||
return encoder_hidden_states
|
||||
|
||||
def _set_gradient_checkpointing(self, module, value: bool = False) -> None:
|
||||
if isinstance(module, (CrossAttnDownBlock2D, DownBlock2D)):
|
||||
module.gradient_checkpointing = value
|
||||
|
||||
def forward(
|
||||
self,
|
||||
sample: torch.FloatTensor,
|
||||
timestep: Union[torch.Tensor, float, int],
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
controlnet_cond: torch.FloatTensor,
|
||||
cat_dim: int = -2,
|
||||
conditioning_scale: float = 1.0,
|
||||
class_labels: Optional[torch.Tensor] = None,
|
||||
timestep_cond: Optional[torch.Tensor] = None,
|
||||
attention_mask: Optional[torch.Tensor] = None,
|
||||
added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,
|
||||
cross_attention_kwargs: Optional[Dict[str, Any]] = None,
|
||||
return_dict: bool = True,
|
||||
) -> Union[AggregatorOutput, Tuple[Tuple[torch.FloatTensor, ...], torch.FloatTensor]]:
|
||||
"""
|
||||
The [`Aggregator`] forward method.
|
||||
|
||||
Args:
|
||||
sample (`torch.FloatTensor`):
|
||||
The noisy input tensor.
|
||||
timestep (`Union[torch.Tensor, float, int]`):
|
||||
The number of timesteps to denoise an input.
|
||||
encoder_hidden_states (`torch.Tensor`):
|
||||
The encoder hidden states.
|
||||
controlnet_cond (`torch.FloatTensor`):
|
||||
The conditional input tensor of shape `(batch_size, sequence_length, hidden_size)`.
|
||||
conditioning_scale (`float`, defaults to `1.0`):
|
||||
The scale factor for ControlNet outputs.
|
||||
class_labels (`torch.Tensor`, *optional*, defaults to `None`):
|
||||
Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings.
|
||||
timestep_cond (`torch.Tensor`, *optional*, defaults to `None`):
|
||||
Additional conditional embeddings for timestep. If provided, the embeddings will be summed with the
|
||||
timestep_embedding passed through the `self.time_embedding` layer to obtain the final timestep
|
||||
embeddings.
|
||||
attention_mask (`torch.Tensor`, *optional*, defaults to `None`):
|
||||
An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask
|
||||
is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large
|
||||
negative values to the attention scores corresponding to "discard" tokens.
|
||||
added_cond_kwargs (`dict`):
|
||||
Additional conditions for the Stable Diffusion XL UNet.
|
||||
cross_attention_kwargs (`dict[str]`, *optional*, defaults to `None`):
|
||||
A kwargs dictionary that if specified is passed along to the `AttnProcessor`.
|
||||
return_dict (`bool`, defaults to `True`):
|
||||
Whether or not to return a [`~models.controlnet.ControlNetOutput`] instead of a plain tuple.
|
||||
|
||||
Returns:
|
||||
[`~models.controlnet.ControlNetOutput`] **or** `tuple`:
|
||||
If `return_dict` is `True`, a [`~models.controlnet.ControlNetOutput`] is returned, otherwise a tuple is
|
||||
returned where the first element is the sample tensor.
|
||||
"""
|
||||
# check channel order
|
||||
channel_order = self.config.controlnet_conditioning_channel_order
|
||||
|
||||
if channel_order == "rgb":
|
||||
# in rgb order by default
|
||||
...
|
||||
else:
|
||||
raise ValueError(f"unknown `controlnet_conditioning_channel_order`: {channel_order}")
|
||||
|
||||
# prepare attention_mask
|
||||
if attention_mask is not None:
|
||||
attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0
|
||||
attention_mask = attention_mask.unsqueeze(1)
|
||||
|
||||
# 1. time
|
||||
timesteps = timestep
|
||||
if not torch.is_tensor(timesteps):
|
||||
# TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can
|
||||
# This would be a good case for the `match` statement (Python 3.10+)
|
||||
is_mps = sample.device.type == "mps"
|
||||
if isinstance(timestep, float):
|
||||
dtype = torch.float32 if is_mps else torch.float64
|
||||
else:
|
||||
dtype = torch.int32 if is_mps else torch.int64
|
||||
timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)
|
||||
elif len(timesteps.shape) == 0:
|
||||
timesteps = timesteps[None].to(sample.device)
|
||||
|
||||
# broadcast to batch dimension in a way that's compatible with ONNX/Core ML
|
||||
timesteps = timesteps.expand(sample.shape[0])
|
||||
|
||||
t_emb = self.time_proj(timesteps)
|
||||
|
||||
# timesteps does not contain any weights and will always return f32 tensors
|
||||
# but time_embedding might actually be running in fp16. so we need to cast here.
|
||||
# there might be better ways to encapsulate this.
|
||||
t_emb = t_emb.to(dtype=sample.dtype)
|
||||
|
||||
emb = self.time_embedding(t_emb, timestep_cond)
|
||||
aug_emb = None
|
||||
|
||||
if self.class_embedding is not None:
|
||||
if class_labels is None:
|
||||
raise ValueError("class_labels should be provided when num_class_embeds > 0")
|
||||
|
||||
if self.config.class_embed_type == "timestep":
|
||||
class_labels = self.time_proj(class_labels)
|
||||
|
||||
class_emb = self.class_embedding(class_labels).to(dtype=self.dtype)
|
||||
emb = emb + class_emb
|
||||
|
||||
if self.config.addition_embed_type is not None:
|
||||
if self.config.addition_embed_type == "text":
|
||||
aug_emb = self.add_embedding(encoder_hidden_states)
|
||||
|
||||
elif self.config.addition_embed_type == "text_time":
|
||||
if "text_embeds" not in added_cond_kwargs:
|
||||
raise ValueError(
|
||||
f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`"
|
||||
)
|
||||
text_embeds = added_cond_kwargs.get("text_embeds")
|
||||
if "time_ids" not in added_cond_kwargs:
|
||||
raise ValueError(
|
||||
f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`"
|
||||
)
|
||||
time_ids = added_cond_kwargs.get("time_ids")
|
||||
time_embeds = self.add_time_proj(time_ids.flatten())
|
||||
time_embeds = time_embeds.reshape((text_embeds.shape[0], -1))
|
||||
|
||||
add_embeds = torch.concat([text_embeds, time_embeds], dim=-1)
|
||||
add_embeds = add_embeds.to(emb.dtype)
|
||||
aug_emb = self.add_embedding(add_embeds)
|
||||
|
||||
emb = emb + aug_emb if aug_emb is not None else emb
|
||||
|
||||
encoder_hidden_states = self.process_encoder_hidden_states(
|
||||
encoder_hidden_states=encoder_hidden_states, added_cond_kwargs=added_cond_kwargs
|
||||
)
|
||||
|
||||
# 2. prepare input
|
||||
cond_latent = self.conv_in(sample)
|
||||
ref_latent = self.ref_conv_in(controlnet_cond)
|
||||
batch_size, channel, height, width = cond_latent.shape
|
||||
if self.pad_concat:
|
||||
if cat_dim == -2 or cat_dim == 2:
|
||||
concat_pad = torch.zeros(batch_size, channel, 1, width)
|
||||
elif cat_dim == -1 or cat_dim == 3:
|
||||
concat_pad = torch.zeros(batch_size, channel, height, 1)
|
||||
else:
|
||||
raise ValueError(f"Aggregator shall concat along spatial dimension, but is asked to concat dim: {cat_dim}.")
|
||||
concat_pad = concat_pad.to(cond_latent.device, dtype=cond_latent.dtype)
|
||||
sample = torch.cat([cond_latent, concat_pad, ref_latent], dim=cat_dim)
|
||||
else:
|
||||
sample = torch.cat([cond_latent, ref_latent], dim=cat_dim)
|
||||
|
||||
# 3. down
|
||||
down_block_res_samples = (sample,)
|
||||
for downsample_block in self.down_blocks:
|
||||
sample, res_samples = downsample_block(
|
||||
hidden_states=sample,
|
||||
temb=emb,
|
||||
cross_attention_kwargs=cross_attention_kwargs,
|
||||
)
|
||||
|
||||
# rebuild sample: split and concat
|
||||
if self.pad_concat:
|
||||
batch_size, channel, height, width = sample.shape
|
||||
if cat_dim == -2 or cat_dim == 2:
|
||||
cond_latent = sample[:, :, :height//2, :]
|
||||
ref_latent = sample[:, :, -(height//2):, :]
|
||||
concat_pad = torch.zeros(batch_size, channel, 1, width)
|
||||
elif cat_dim == -1 or cat_dim == 3:
|
||||
cond_latent = sample[:, :, :, :width//2]
|
||||
ref_latent = sample[:, :, :, -(width//2):]
|
||||
concat_pad = torch.zeros(batch_size, channel, height, 1)
|
||||
concat_pad = concat_pad.to(cond_latent.device, dtype=cond_latent.dtype)
|
||||
sample = torch.cat([cond_latent, concat_pad, ref_latent], dim=cat_dim)
|
||||
res_samples = res_samples[:-1] + (sample,)
|
||||
|
||||
down_block_res_samples += res_samples
|
||||
|
||||
# 4. mid
|
||||
if self.mid_block is not None:
|
||||
sample = self.mid_block(
|
||||
sample,
|
||||
emb,
|
||||
cross_attention_kwargs=cross_attention_kwargs,
|
||||
)
|
||||
|
||||
# 5. split samples and SFT.
|
||||
controlnet_down_block_res_samples = ()
|
||||
for down_block_res_sample, controlnet_block in zip(down_block_res_samples, self.controlnet_down_blocks):
|
||||
batch_size, channel, height, width = down_block_res_sample.shape
|
||||
if cat_dim == -2 or cat_dim == 2:
|
||||
cond_latent = down_block_res_sample[:, :, :height//2, :]
|
||||
ref_latent = down_block_res_sample[:, :, -(height//2):, :]
|
||||
elif cat_dim == -1 or cat_dim == 3:
|
||||
cond_latent = down_block_res_sample[:, :, :, :width//2]
|
||||
ref_latent = down_block_res_sample[:, :, :, -(width//2):]
|
||||
down_block_res_sample = controlnet_block((cond_latent, ref_latent), )
|
||||
controlnet_down_block_res_samples = controlnet_down_block_res_samples + (down_block_res_sample,)
|
||||
|
||||
down_block_res_samples = controlnet_down_block_res_samples
|
||||
|
||||
batch_size, channel, height, width = sample.shape
|
||||
if cat_dim == -2 or cat_dim == 2:
|
||||
cond_latent = sample[:, :, :height//2, :]
|
||||
ref_latent = sample[:, :, -(height//2):, :]
|
||||
elif cat_dim == -1 or cat_dim == 3:
|
||||
cond_latent = sample[:, :, :, :width//2]
|
||||
ref_latent = sample[:, :, :, -(width//2):]
|
||||
mid_block_res_sample = self.controlnet_mid_block((cond_latent, ref_latent), )
|
||||
|
||||
# 6. scaling
|
||||
down_block_res_samples = [sample*conditioning_scale for sample in down_block_res_samples]
|
||||
mid_block_res_sample = mid_block_res_sample*conditioning_scale
|
||||
|
||||
if self.config.global_pool_conditions:
|
||||
down_block_res_samples = [
|
||||
torch.mean(sample, dim=(2, 3), keepdim=True) for sample in down_block_res_samples
|
||||
]
|
||||
mid_block_res_sample = torch.mean(mid_block_res_sample, dim=(2, 3), keepdim=True)
|
||||
|
||||
if not return_dict:
|
||||
return (down_block_res_samples, mid_block_res_sample)
|
||||
|
||||
return AggregatorOutput(
|
||||
down_block_res_samples=down_block_res_samples, mid_block_res_sample=mid_block_res_sample
|
||||
)
|
||||
|
||||
|
||||
def zero_module(module):
|
||||
for p in module.parameters():
|
||||
nn.init.zeros_(p)
|
||||
return module
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,236 @@
|
||||
import os
|
||||
import torch
|
||||
from typing import List
|
||||
from collections import namedtuple, OrderedDict
|
||||
|
||||
def is_torch2_available():
|
||||
return hasattr(torch.nn.functional, "scaled_dot_product_attention")
|
||||
|
||||
if is_torch2_available():
|
||||
from .attention_processor import (
|
||||
AttnProcessor2_0 as AttnProcessor,
|
||||
)
|
||||
from .attention_processor import (
|
||||
CNAttnProcessor2_0 as CNAttnProcessor,
|
||||
)
|
||||
from .attention_processor import (
|
||||
IPAttnProcessor2_0 as IPAttnProcessor,
|
||||
)
|
||||
from .attention_processor import (
|
||||
TA_IPAttnProcessor2_0 as TA_IPAttnProcessor,
|
||||
)
|
||||
else:
|
||||
from .attention_processor import AttnProcessor, CNAttnProcessor, IPAttnProcessor, TA_IPAttnProcessor
|
||||
|
||||
|
||||
class ImageProjModel(torch.nn.Module):
|
||||
"""Projection Model"""
|
||||
|
||||
def __init__(self, cross_attention_dim=2048, clip_embeddings_dim=1280, clip_extra_context_tokens=4):
|
||||
super().__init__()
|
||||
|
||||
self.cross_attention_dim = cross_attention_dim
|
||||
self.clip_extra_context_tokens = clip_extra_context_tokens
|
||||
self.proj = torch.nn.Linear(clip_embeddings_dim, self.clip_extra_context_tokens * cross_attention_dim)
|
||||
self.norm = torch.nn.LayerNorm(cross_attention_dim)
|
||||
|
||||
def forward(self, image_embeds):
|
||||
embeds = image_embeds
|
||||
clip_extra_context_tokens = self.proj(embeds).reshape(
|
||||
-1, self.clip_extra_context_tokens, self.cross_attention_dim
|
||||
)
|
||||
clip_extra_context_tokens = self.norm(clip_extra_context_tokens)
|
||||
return clip_extra_context_tokens
|
||||
|
||||
|
||||
class MLPProjModel(torch.nn.Module):
|
||||
"""SD model with image prompt"""
|
||||
def __init__(self, cross_attention_dim=2048, clip_embeddings_dim=1280):
|
||||
super().__init__()
|
||||
|
||||
self.proj = torch.nn.Sequential(
|
||||
torch.nn.Linear(clip_embeddings_dim, clip_embeddings_dim),
|
||||
torch.nn.GELU(),
|
||||
torch.nn.Linear(clip_embeddings_dim, cross_attention_dim),
|
||||
torch.nn.LayerNorm(cross_attention_dim)
|
||||
)
|
||||
|
||||
def forward(self, image_embeds):
|
||||
clip_extra_context_tokens = self.proj(image_embeds)
|
||||
return clip_extra_context_tokens
|
||||
|
||||
|
||||
class MultiIPAdapterImageProjection(torch.nn.Module):
|
||||
def __init__(self, IPAdapterImageProjectionLayers):
|
||||
super().__init__()
|
||||
self.image_projection_layers = torch.nn.ModuleList(IPAdapterImageProjectionLayers)
|
||||
|
||||
def forward(self, image_embeds: List[torch.FloatTensor]):
|
||||
projected_image_embeds = []
|
||||
|
||||
# currently, we accept `image_embeds` as
|
||||
# 1. a tensor (deprecated) with shape [batch_size, embed_dim] or [batch_size, sequence_length, embed_dim]
|
||||
# 2. list of `n` tensors where `n` is number of ip-adapters, each tensor can hae shape [batch_size, num_images, embed_dim] or [batch_size, num_images, sequence_length, embed_dim]
|
||||
if not isinstance(image_embeds, list):
|
||||
image_embeds = [image_embeds.unsqueeze(1)]
|
||||
|
||||
if len(image_embeds) != len(self.image_projection_layers):
|
||||
raise ValueError(
|
||||
f"image_embeds must have the same length as image_projection_layers, got {len(image_embeds)} and {len(self.image_projection_layers)}"
|
||||
)
|
||||
|
||||
for image_embed, image_projection_layer in zip(image_embeds, self.image_projection_layers):
|
||||
batch_size, num_images = image_embed.shape[0], image_embed.shape[1]
|
||||
image_embed = image_embed.reshape((batch_size * num_images,) + image_embed.shape[2:])
|
||||
image_embed = image_projection_layer(image_embed)
|
||||
# image_embed = image_embed.reshape((batch_size, num_images) + image_embed.shape[1:])
|
||||
|
||||
projected_image_embeds.append(image_embed)
|
||||
|
||||
return projected_image_embeds
|
||||
|
||||
|
||||
class IPAdapter(torch.nn.Module):
|
||||
"""IP-Adapter"""
|
||||
def __init__(self, unet, image_proj_model, adapter_modules, ckpt_path=None):
|
||||
super().__init__()
|
||||
self.unet = unet
|
||||
self.image_proj = image_proj_model
|
||||
self.ip_adapter = adapter_modules
|
||||
|
||||
if ckpt_path is not None:
|
||||
self.load_from_checkpoint(ckpt_path)
|
||||
|
||||
def forward(self, noisy_latents, timesteps, encoder_hidden_states, image_embeds):
|
||||
ip_tokens = self.image_proj(image_embeds)
|
||||
encoder_hidden_states = torch.cat([encoder_hidden_states, ip_tokens], dim=1)
|
||||
# Predict the noise residual
|
||||
noise_pred = self.unet(noisy_latents, timesteps, encoder_hidden_states).sample
|
||||
return noise_pred
|
||||
|
||||
def load_from_checkpoint(self, ckpt_path: str):
|
||||
# Calculate original checksums
|
||||
orig_ip_proj_sum = torch.sum(torch.stack([torch.sum(p) for p in self.image_proj.parameters()]))
|
||||
orig_adapter_sum = torch.sum(torch.stack([torch.sum(p) for p in self.ip_adapter.parameters()]))
|
||||
|
||||
state_dict = torch.load(ckpt_path, map_location="cpu")
|
||||
keys = list(state_dict.keys())
|
||||
if keys != ["image_proj", "ip_adapter"]:
|
||||
state_dict = revise_state_dict(state_dict)
|
||||
|
||||
# Load state dict for image_proj_model and adapter_modules
|
||||
self.image_proj.load_state_dict(state_dict["image_proj"], strict=True)
|
||||
self.ip_adapter.load_state_dict(state_dict["ip_adapter"], strict=True)
|
||||
|
||||
# Calculate new checksums
|
||||
new_ip_proj_sum = torch.sum(torch.stack([torch.sum(p) for p in self.image_proj.parameters()]))
|
||||
new_adapter_sum = torch.sum(torch.stack([torch.sum(p) for p in self.ip_adapter.parameters()]))
|
||||
|
||||
# Verify if the weights have changed
|
||||
assert orig_ip_proj_sum != new_ip_proj_sum, "Weights of image_proj_model did not change!"
|
||||
assert orig_adapter_sum != new_adapter_sum, "Weights of adapter_modules did not change!"
|
||||
|
||||
|
||||
class IPAdapterPlus(torch.nn.Module):
|
||||
"""IP-Adapter"""
|
||||
def __init__(self, unet, image_proj_model, adapter_modules, ckpt_path=None):
|
||||
super().__init__()
|
||||
self.unet = unet
|
||||
self.image_proj = image_proj_model
|
||||
self.ip_adapter = adapter_modules
|
||||
|
||||
if ckpt_path is not None:
|
||||
self.load_from_checkpoint(ckpt_path)
|
||||
|
||||
def forward(self, noisy_latents, timesteps, encoder_hidden_states, image_embeds):
|
||||
ip_tokens = self.image_proj(image_embeds)
|
||||
encoder_hidden_states = torch.cat([encoder_hidden_states, ip_tokens], dim=1)
|
||||
# Predict the noise residual
|
||||
noise_pred = self.unet(noisy_latents, timesteps, encoder_hidden_states).sample
|
||||
return noise_pred
|
||||
|
||||
def load_from_checkpoint(self, ckpt_path: str):
|
||||
# Calculate original checksums
|
||||
orig_ip_proj_sum = torch.sum(torch.stack([torch.sum(p) for p in self.image_proj.parameters()]))
|
||||
orig_adapter_sum = torch.sum(torch.stack([torch.sum(p) for p in self.ip_adapter.parameters()]))
|
||||
org_unet_sum = []
|
||||
for attn_name, attn_proc in self.unet.attn_processors.items():
|
||||
if isinstance(attn_proc, (TA_IPAttnProcessor, IPAttnProcessor)):
|
||||
org_unet_sum.append(torch.sum(torch.stack([torch.sum(p) for p in attn_proc.parameters()])))
|
||||
org_unet_sum = torch.sum(torch.stack(org_unet_sum))
|
||||
|
||||
state_dict = torch.load(ckpt_path, map_location="cpu")
|
||||
keys = list(state_dict.keys())
|
||||
if keys != ["image_proj", "ip_adapter"]:
|
||||
state_dict = revise_state_dict(state_dict)
|
||||
|
||||
# Check if 'latents' exists in both the saved state_dict and the current model's state_dict
|
||||
strict_load_image_proj_model = True
|
||||
if "latents" in state_dict["image_proj"] and "latents" in self.image_proj.state_dict():
|
||||
# Check if the shapes are mismatched
|
||||
if state_dict["image_proj"]["latents"].shape != self.image_proj.state_dict()["latents"].shape:
|
||||
print(f"Shapes of 'image_proj.latents' in checkpoint {ckpt_path} and current model do not match.")
|
||||
print("Removing 'latents' from checkpoint and loading the rest of the weights.")
|
||||
del state_dict["image_proj"]["latents"]
|
||||
strict_load_image_proj_model = False
|
||||
|
||||
# Load state dict for image_proj_model and adapter_modules
|
||||
self.image_proj.load_state_dict(state_dict["image_proj"], strict=strict_load_image_proj_model)
|
||||
missing_key, unexpected_key = self.ip_adapter.load_state_dict(state_dict["ip_adapter"], strict=False)
|
||||
if len(missing_key) > 0:
|
||||
for ms in missing_key:
|
||||
if "ln" not in ms:
|
||||
raise ValueError(f"Missing key in adapter_modules: {len(missing_key)}")
|
||||
if len(unexpected_key) > 0:
|
||||
raise ValueError(f"Unexpected key in adapter_modules: {len(unexpected_key)}")
|
||||
|
||||
# Calculate new checksums
|
||||
new_ip_proj_sum = torch.sum(torch.stack([torch.sum(p) for p in self.image_proj.parameters()]))
|
||||
new_adapter_sum = torch.sum(torch.stack([torch.sum(p) for p in self.ip_adapter.parameters()]))
|
||||
|
||||
# Verify if the weights loaded to unet
|
||||
unet_sum = []
|
||||
for attn_name, attn_proc in self.unet.attn_processors.items():
|
||||
if isinstance(attn_proc, (TA_IPAttnProcessor, IPAttnProcessor)):
|
||||
unet_sum.append(torch.sum(torch.stack([torch.sum(p) for p in attn_proc.parameters()])))
|
||||
unet_sum = torch.sum(torch.stack(unet_sum))
|
||||
|
||||
assert org_unet_sum != unet_sum, "Weights of adapter_modules in unet did not change!"
|
||||
assert (unet_sum - new_adapter_sum < 1e-4), "Weights of adapter_modules did not load to unet!"
|
||||
|
||||
# Verify if the weights have changed
|
||||
assert orig_ip_proj_sum != new_ip_proj_sum, "Weights of image_proj_model did not change!"
|
||||
assert orig_adapter_sum != new_adapter_sum, "Weights of adapter_mod`ules did not change!"
|
||||
|
||||
|
||||
class IPAdapterXL(IPAdapter):
|
||||
"""SDXL"""
|
||||
|
||||
def forward(self, noisy_latents, timesteps, encoder_hidden_states, unet_added_cond_kwargs, image_embeds):
|
||||
ip_tokens = self.image_proj(image_embeds)
|
||||
encoder_hidden_states = torch.cat([encoder_hidden_states, ip_tokens], dim=1)
|
||||
# Predict the noise residual
|
||||
noise_pred = self.unet(noisy_latents, timesteps, encoder_hidden_states, added_cond_kwargs=unet_added_cond_kwargs).sample
|
||||
return noise_pred
|
||||
|
||||
|
||||
class IPAdapterPlusXL(IPAdapterPlus):
|
||||
"""IP-Adapter with fine-grained features"""
|
||||
|
||||
def forward(self, noisy_latents, timesteps, encoder_hidden_states, unet_added_cond_kwargs, image_embeds):
|
||||
ip_tokens = self.image_proj(image_embeds)
|
||||
encoder_hidden_states = torch.cat([encoder_hidden_states, ip_tokens], dim=1)
|
||||
# Predict the noise residual
|
||||
noise_pred = self.unet(noisy_latents, timesteps, encoder_hidden_states, added_cond_kwargs=unet_added_cond_kwargs).sample
|
||||
return noise_pred
|
||||
|
||||
|
||||
class IPAdapterFull(IPAdapterPlus):
|
||||
"""IP-Adapter with full features"""
|
||||
|
||||
def init_proj(self):
|
||||
image_proj_model = MLPProjModel(
|
||||
cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
|
||||
clip_embeddings_dim=self.image_encoder.config.hidden_size,
|
||||
).to(self.device, dtype=torch.float16)
|
||||
return image_proj_model
|
||||
@@ -0,0 +1,158 @@
|
||||
# modified from https://github.com/mlfoundations/open_flamingo/blob/main/open_flamingo/src/helpers.py
|
||||
# and https://github.com/lucidrains/imagen-pytorch/blob/main/imagen_pytorch/imagen_pytorch.py
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
from einops.layers.torch import Rearrange
|
||||
|
||||
|
||||
# FFN
|
||||
def FeedForward(dim, mult=4):
|
||||
inner_dim = int(dim * mult)
|
||||
return nn.Sequential(
|
||||
nn.LayerNorm(dim),
|
||||
nn.Linear(dim, inner_dim, bias=False),
|
||||
nn.GELU(),
|
||||
nn.Linear(inner_dim, dim, bias=False),
|
||||
)
|
||||
|
||||
|
||||
def reshape_tensor(x, heads):
|
||||
bs, length, width = x.shape
|
||||
# (bs, length, width) --> (bs, length, n_heads, dim_per_head)
|
||||
x = x.view(bs, length, heads, -1)
|
||||
# (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
|
||||
x = x.transpose(1, 2)
|
||||
# (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
|
||||
x = x.reshape(bs, heads, length, -1)
|
||||
return x
|
||||
|
||||
|
||||
class PerceiverAttention(nn.Module):
|
||||
def __init__(self, *, dim, dim_head=64, heads=8):
|
||||
super().__init__()
|
||||
self.scale = dim_head**-0.5
|
||||
self.dim_head = dim_head
|
||||
self.heads = heads
|
||||
inner_dim = dim_head * heads
|
||||
|
||||
self.norm1 = nn.LayerNorm(dim)
|
||||
self.norm2 = nn.LayerNorm(dim)
|
||||
|
||||
self.to_q = nn.Linear(dim, inner_dim, bias=False)
|
||||
self.to_kv = nn.Linear(dim, inner_dim * 2, bias=False)
|
||||
self.to_out = nn.Linear(inner_dim, dim, bias=False)
|
||||
|
||||
def forward(self, x, latents):
|
||||
"""
|
||||
Args:
|
||||
x (torch.Tensor): image features
|
||||
shape (b, n1, D)
|
||||
latent (torch.Tensor): latent features
|
||||
shape (b, n2, D)
|
||||
"""
|
||||
x = self.norm1(x)
|
||||
latents = self.norm2(latents)
|
||||
|
||||
b, l, _ = latents.shape
|
||||
|
||||
q = self.to_q(latents)
|
||||
kv_input = torch.cat((x, latents), dim=-2)
|
||||
k, v = self.to_kv(kv_input).chunk(2, dim=-1)
|
||||
|
||||
q = reshape_tensor(q, self.heads)
|
||||
k = reshape_tensor(k, self.heads)
|
||||
v = reshape_tensor(v, self.heads)
|
||||
|
||||
# attention
|
||||
scale = 1 / math.sqrt(math.sqrt(self.dim_head))
|
||||
weight = (q * scale) @ (k * scale).transpose(-2, -1) # More stable with f16 than dividing afterwards
|
||||
weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype)
|
||||
out = weight @ v
|
||||
|
||||
out = out.permute(0, 2, 1, 3).reshape(b, l, -1)
|
||||
|
||||
return self.to_out(out)
|
||||
|
||||
|
||||
class Resampler(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
dim=1280,
|
||||
depth=4,
|
||||
dim_head=64,
|
||||
heads=20,
|
||||
num_queries=64,
|
||||
embedding_dim=768,
|
||||
output_dim=1024,
|
||||
ff_mult=4,
|
||||
max_seq_len: int = 257, # CLIP tokens + CLS token
|
||||
apply_pos_emb: bool = False,
|
||||
num_latents_mean_pooled: int = 0, # number of latents derived from mean pooled representation of the sequence
|
||||
):
|
||||
super().__init__()
|
||||
self.pos_emb = nn.Embedding(max_seq_len, embedding_dim) if apply_pos_emb else None
|
||||
|
||||
self.latents = nn.Parameter(torch.randn(1, num_queries, dim) / dim**0.5)
|
||||
|
||||
self.proj_in = nn.Linear(embedding_dim, dim)
|
||||
|
||||
self.proj_out = nn.Linear(dim, output_dim)
|
||||
self.norm_out = nn.LayerNorm(output_dim)
|
||||
|
||||
self.to_latents_from_mean_pooled_seq = (
|
||||
nn.Sequential(
|
||||
nn.LayerNorm(dim),
|
||||
nn.Linear(dim, dim * num_latents_mean_pooled),
|
||||
Rearrange("b (n d) -> b n d", n=num_latents_mean_pooled),
|
||||
)
|
||||
if num_latents_mean_pooled > 0
|
||||
else None
|
||||
)
|
||||
|
||||
self.layers = nn.ModuleList([])
|
||||
for _ in range(depth):
|
||||
self.layers.append(
|
||||
nn.ModuleList(
|
||||
[
|
||||
PerceiverAttention(dim=dim, dim_head=dim_head, heads=heads),
|
||||
FeedForward(dim=dim, mult=ff_mult),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
if self.pos_emb is not None:
|
||||
n, device = x.shape[1], x.device
|
||||
pos_emb = self.pos_emb(torch.arange(n, device=device))
|
||||
x = x + pos_emb
|
||||
|
||||
latents = self.latents.repeat(x.size(0), 1, 1)
|
||||
|
||||
x = self.proj_in(x)
|
||||
|
||||
if self.to_latents_from_mean_pooled_seq:
|
||||
meanpooled_seq = masked_mean(x, dim=1, mask=torch.ones(x.shape[:2], device=x.device, dtype=torch.bool))
|
||||
meanpooled_latents = self.to_latents_from_mean_pooled_seq(meanpooled_seq)
|
||||
latents = torch.cat((meanpooled_latents, latents), dim=-2)
|
||||
|
||||
for attn, ff in self.layers:
|
||||
latents = attn(x, latents) + latents
|
||||
latents = ff(latents) + latents
|
||||
|
||||
latents = self.proj_out(latents)
|
||||
return self.norm_out(latents)
|
||||
|
||||
|
||||
def masked_mean(t, *, dim, mask=None):
|
||||
if mask is None:
|
||||
return t.mean(dim=dim)
|
||||
|
||||
denom = mask.sum(dim=dim, keepdim=True)
|
||||
mask = rearrange(mask, "b n -> b n 1")
|
||||
masked_t = t.masked_fill(~mask, 0.0)
|
||||
|
||||
return masked_t.sum(dim=dim) / denom.clamp(min=1e-5)
|
||||
@@ -0,0 +1,248 @@
|
||||
import torch
|
||||
from collections import namedtuple, OrderedDict
|
||||
from safetensors import safe_open
|
||||
from .attention_processor import init_attn_proc
|
||||
from .ip_adapter import MultiIPAdapterImageProjection
|
||||
from .resampler import Resampler
|
||||
from transformers import (
|
||||
AutoModel, AutoImageProcessor,
|
||||
CLIPVisionModelWithProjection, CLIPImageProcessor)
|
||||
|
||||
|
||||
def init_adapter_in_unet(
|
||||
unet,
|
||||
image_proj_model=None,
|
||||
pretrained_model_path_or_dict=None,
|
||||
adapter_tokens=64,
|
||||
embedding_dim=None,
|
||||
use_lcm=False,
|
||||
use_adaln=True,
|
||||
):
|
||||
device = unet.device
|
||||
dtype = unet.dtype
|
||||
if image_proj_model is None:
|
||||
assert embedding_dim is not None, "embedding_dim must be provided if image_proj_model is None."
|
||||
image_proj_model = Resampler(
|
||||
embedding_dim=embedding_dim,
|
||||
output_dim=unet.config.cross_attention_dim,
|
||||
num_queries=adapter_tokens,
|
||||
)
|
||||
if pretrained_model_path_or_dict is not None:
|
||||
if not isinstance(pretrained_model_path_or_dict, dict):
|
||||
if pretrained_model_path_or_dict.endswith(".safetensors"):
|
||||
state_dict = {"image_proj": {}, "ip_adapter": {}}
|
||||
with safe_open(pretrained_model_path_or_dict, framework="pt", device=unet.device) as f:
|
||||
for key in f.keys():
|
||||
if key.startswith("image_proj."):
|
||||
state_dict["image_proj"][key.replace("image_proj.", "")] = f.get_tensor(key)
|
||||
elif key.startswith("ip_adapter."):
|
||||
state_dict["ip_adapter"][key.replace("ip_adapter.", "")] = f.get_tensor(key)
|
||||
else:
|
||||
state_dict = torch.load(pretrained_model_path_or_dict, map_location=unet.device)
|
||||
else:
|
||||
state_dict = pretrained_model_path_or_dict
|
||||
keys = list(state_dict.keys())
|
||||
if "image_proj" not in keys and "ip_adapter" not in keys:
|
||||
state_dict = revise_state_dict(state_dict)
|
||||
|
||||
# Creat IP cross-attention in unet.
|
||||
attn_procs = init_attn_proc(unet, adapter_tokens, use_lcm, use_adaln)
|
||||
unet.set_attn_processor(attn_procs)
|
||||
|
||||
# Load pretrinaed model if needed.
|
||||
if pretrained_model_path_or_dict is not None:
|
||||
if "ip_adapter" in state_dict.keys():
|
||||
adapter_modules = torch.nn.ModuleList(unet.attn_processors.values())
|
||||
missing, unexpected = adapter_modules.load_state_dict(state_dict["ip_adapter"], strict=False)
|
||||
for mk in missing:
|
||||
if "ln" not in mk:
|
||||
raise ValueError(f"Missing keys in adapter_modules: {missing}")
|
||||
if "image_proj" in state_dict.keys():
|
||||
image_proj_model.load_state_dict(state_dict["image_proj"])
|
||||
|
||||
# Load image projectors into iterable ModuleList.
|
||||
image_projection_layers = []
|
||||
image_projection_layers.append(image_proj_model)
|
||||
unet.encoder_hid_proj = MultiIPAdapterImageProjection(image_projection_layers)
|
||||
|
||||
# Adjust unet config to handle addtional ip hidden states.
|
||||
unet.config.encoder_hid_dim_type = "ip_image_proj"
|
||||
unet.to(dtype=dtype, device=device)
|
||||
|
||||
|
||||
def load_adapter_to_pipe(
|
||||
pipe,
|
||||
pretrained_model_path_or_dict,
|
||||
image_encoder_or_path=None,
|
||||
feature_extractor_or_path=None,
|
||||
use_clip_encoder=False,
|
||||
adapter_tokens=64,
|
||||
use_lcm=False,
|
||||
use_adaln=True,
|
||||
):
|
||||
|
||||
if not isinstance(pretrained_model_path_or_dict, dict):
|
||||
if pretrained_model_path_or_dict.endswith(".safetensors"):
|
||||
state_dict = {"image_proj": {}, "ip_adapter": {}}
|
||||
with safe_open(pretrained_model_path_or_dict, framework="pt", device=pipe.device) as f:
|
||||
for key in f.keys():
|
||||
if key.startswith("image_proj."):
|
||||
state_dict["image_proj"][key.replace("image_proj.", "")] = f.get_tensor(key)
|
||||
elif key.startswith("ip_adapter."):
|
||||
state_dict["ip_adapter"][key.replace("ip_adapter.", "")] = f.get_tensor(key)
|
||||
else:
|
||||
state_dict = torch.load(pretrained_model_path_or_dict, map_location=pipe.device)
|
||||
else:
|
||||
state_dict = pretrained_model_path_or_dict
|
||||
keys = list(state_dict.keys())
|
||||
if "image_proj" not in keys and "ip_adapter" not in keys:
|
||||
state_dict = revise_state_dict(state_dict)
|
||||
|
||||
# load CLIP image encoder here if it has not been registered to the pipeline yet
|
||||
if image_encoder_or_path is not None:
|
||||
if isinstance(image_encoder_or_path, str):
|
||||
feature_extractor_or_path = image_encoder_or_path if feature_extractor_or_path is None else feature_extractor_or_path
|
||||
|
||||
image_encoder_or_path = (
|
||||
CLIPVisionModelWithProjection.from_pretrained(
|
||||
image_encoder_or_path
|
||||
) if use_clip_encoder else
|
||||
AutoModel.from_pretrained(image_encoder_or_path)
|
||||
)
|
||||
|
||||
if feature_extractor_or_path is not None:
|
||||
if isinstance(feature_extractor_or_path, str):
|
||||
feature_extractor_or_path = (
|
||||
CLIPImageProcessor() if use_clip_encoder else
|
||||
AutoImageProcessor.from_pretrained(feature_extractor_or_path)
|
||||
)
|
||||
|
||||
# create image encoder if it has not been registered to the pipeline yet
|
||||
if hasattr(pipe, "image_encoder") and getattr(pipe, "image_encoder", None) is None:
|
||||
image_encoder = image_encoder_or_path.to(pipe.device, dtype=pipe.dtype)
|
||||
pipe.register_modules(image_encoder=image_encoder)
|
||||
else:
|
||||
image_encoder = pipe.image_encoder
|
||||
|
||||
# create feature extractor if it has not been registered to the pipeline yet
|
||||
if hasattr(pipe, "feature_extractor") and getattr(pipe, "feature_extractor", None) is None:
|
||||
feature_extractor = feature_extractor_or_path
|
||||
pipe.register_modules(feature_extractor=feature_extractor)
|
||||
else:
|
||||
feature_extractor = pipe.feature_extractor
|
||||
|
||||
# load adapter into unet
|
||||
unet = getattr(pipe, pipe.unet_name) if not hasattr(pipe, "unet") else pipe.unet
|
||||
attn_procs = init_attn_proc(unet, adapter_tokens, use_lcm, use_adaln)
|
||||
unet.set_attn_processor(attn_procs)
|
||||
image_proj_model = Resampler(
|
||||
embedding_dim=image_encoder.config.hidden_size,
|
||||
output_dim=unet.config.cross_attention_dim,
|
||||
num_queries=adapter_tokens,
|
||||
)
|
||||
|
||||
# Load pretrinaed model if needed.
|
||||
if "ip_adapter" in state_dict.keys():
|
||||
adapter_modules = torch.nn.ModuleList(unet.attn_processors.values())
|
||||
missing, unexpected = adapter_modules.load_state_dict(state_dict["ip_adapter"], strict=False)
|
||||
for mk in missing:
|
||||
if "ln" not in mk:
|
||||
raise ValueError(f"Missing keys in adapter_modules: {missing}")
|
||||
if "image_proj" in state_dict.keys():
|
||||
image_proj_model.load_state_dict(state_dict["image_proj"])
|
||||
|
||||
# convert IP-Adapter Image Projection layers to diffusers
|
||||
image_projection_layers = []
|
||||
image_projection_layers.append(image_proj_model)
|
||||
unet.encoder_hid_proj = MultiIPAdapterImageProjection(image_projection_layers)
|
||||
|
||||
# Adjust unet config to handle addtional ip hidden states.
|
||||
unet.config.encoder_hid_dim_type = "ip_image_proj"
|
||||
unet.to(dtype=pipe.dtype, device=pipe.device)
|
||||
|
||||
|
||||
def revise_state_dict(old_state_dict_or_path, map_location="cpu"):
|
||||
new_state_dict = OrderedDict()
|
||||
new_state_dict["image_proj"] = OrderedDict()
|
||||
new_state_dict["ip_adapter"] = OrderedDict()
|
||||
if isinstance(old_state_dict_or_path, str):
|
||||
old_state_dict = torch.load(old_state_dict_or_path, map_location=map_location)
|
||||
else:
|
||||
old_state_dict = old_state_dict_or_path
|
||||
for name, weight in old_state_dict.items():
|
||||
if name.startswith("image_proj_model."):
|
||||
new_state_dict["image_proj"][name[len("image_proj_model."):]] = weight
|
||||
elif name.startswith("adapter_modules."):
|
||||
new_state_dict["ip_adapter"][name[len("adapter_modules."):]] = weight
|
||||
return new_state_dict
|
||||
|
||||
|
||||
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image
|
||||
def encode_image(image_encoder, feature_extractor, image, device, num_images_per_prompt, output_hidden_states=None):
|
||||
dtype = next(image_encoder.parameters()).dtype
|
||||
|
||||
if not isinstance(image, torch.Tensor):
|
||||
image = feature_extractor(image, return_tensors="pt").pixel_values
|
||||
|
||||
image = image.to(device=device, dtype=dtype)
|
||||
if output_hidden_states:
|
||||
image_enc_hidden_states = image_encoder(image, output_hidden_states=True).hidden_states[-2]
|
||||
image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)
|
||||
return image_enc_hidden_states
|
||||
else:
|
||||
if isinstance(image_encoder, CLIPVisionModelWithProjection):
|
||||
# CLIP image encoder.
|
||||
image_embeds = image_encoder(image).image_embeds
|
||||
else:
|
||||
# DINO image encoder.
|
||||
image_embeds = image_encoder(image).last_hidden_state
|
||||
image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)
|
||||
return image_embeds
|
||||
|
||||
|
||||
def prepare_training_image_embeds(
|
||||
image_encoder, feature_extractor,
|
||||
ip_adapter_image, ip_adapter_image_embeds,
|
||||
device, drop_rate, output_hidden_state, idx_to_replace=None
|
||||
):
|
||||
if ip_adapter_image_embeds is None:
|
||||
if not isinstance(ip_adapter_image, list):
|
||||
ip_adapter_image = [ip_adapter_image]
|
||||
|
||||
# if len(ip_adapter_image) != len(unet.encoder_hid_proj.image_projection_layers):
|
||||
# raise ValueError(
|
||||
# f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(unet.encoder_hid_proj.image_projection_layers)} IP Adapters."
|
||||
# )
|
||||
|
||||
image_embeds = []
|
||||
for single_ip_adapter_image in ip_adapter_image:
|
||||
if idx_to_replace is None:
|
||||
idx_to_replace = torch.rand(len(single_ip_adapter_image)) < drop_rate
|
||||
zero_ip_adapter_image = torch.zeros_like(single_ip_adapter_image)
|
||||
single_ip_adapter_image[idx_to_replace] = zero_ip_adapter_image[idx_to_replace]
|
||||
single_image_embeds = encode_image(
|
||||
image_encoder, feature_extractor, single_ip_adapter_image, device, 1, output_hidden_state
|
||||
)
|
||||
single_image_embeds = torch.stack([single_image_embeds], dim=1) # FIXME
|
||||
|
||||
image_embeds.append(single_image_embeds)
|
||||
else:
|
||||
repeat_dims = [1]
|
||||
image_embeds = []
|
||||
for single_image_embeds in ip_adapter_image_embeds:
|
||||
if do_classifier_free_guidance:
|
||||
single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2)
|
||||
single_image_embeds = single_image_embeds.repeat(
|
||||
num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:]))
|
||||
)
|
||||
single_negative_image_embeds = single_negative_image_embeds.repeat(
|
||||
num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:]))
|
||||
)
|
||||
single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds])
|
||||
else:
|
||||
single_image_embeds = single_image_embeds.repeat(
|
||||
num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:]))
|
||||
)
|
||||
image_embeds.append(single_image_embeds)
|
||||
|
||||
return image_embeds
|
||||
@@ -0,0 +1,537 @@
|
||||
# Copyright 2023 Stanford University Team and The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion
|
||||
# and https://github.com/hojonathanho/diffusion
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
||||
from diffusers.utils import BaseOutput, logging
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
from diffusers.schedulers.scheduling_utils import SchedulerMixin
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
||||
|
||||
|
||||
@dataclass
|
||||
class LCMSingleStepSchedulerOutput(BaseOutput):
|
||||
"""
|
||||
Output class for the scheduler's `step` function output.
|
||||
|
||||
Args:
|
||||
pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):
|
||||
The predicted denoised sample `(x_{0})` based on the model output from the current timestep.
|
||||
`pred_original_sample` can be used to preview progress or for guidance.
|
||||
"""
|
||||
|
||||
denoised: Optional[torch.FloatTensor] = None
|
||||
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar
|
||||
def betas_for_alpha_bar(
|
||||
num_diffusion_timesteps,
|
||||
max_beta=0.999,
|
||||
alpha_transform_type="cosine",
|
||||
):
|
||||
"""
|
||||
Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of
|
||||
(1-beta) over time from t = [0,1].
|
||||
|
||||
Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up
|
||||
to that part of the diffusion process.
|
||||
|
||||
|
||||
Args:
|
||||
num_diffusion_timesteps (`int`): the number of betas to produce.
|
||||
max_beta (`float`): the maximum beta to use; use values lower than 1 to
|
||||
prevent singularities.
|
||||
alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar.
|
||||
Choose from `cosine` or `exp`
|
||||
|
||||
Returns:
|
||||
betas (`np.ndarray`): the betas used by the scheduler to step the model outputs
|
||||
"""
|
||||
if alpha_transform_type == "cosine":
|
||||
|
||||
def alpha_bar_fn(t):
|
||||
return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2
|
||||
|
||||
elif alpha_transform_type == "exp":
|
||||
|
||||
def alpha_bar_fn(t):
|
||||
return math.exp(t * -12.0)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}")
|
||||
|
||||
betas = []
|
||||
for i in range(num_diffusion_timesteps):
|
||||
t1 = i / num_diffusion_timesteps
|
||||
t2 = (i + 1) / num_diffusion_timesteps
|
||||
betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta))
|
||||
return torch.tensor(betas, dtype=torch.float32)
|
||||
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr
|
||||
def rescale_zero_terminal_snr(betas: torch.FloatTensor) -> torch.FloatTensor:
|
||||
"""
|
||||
Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1)
|
||||
|
||||
|
||||
Args:
|
||||
betas (`torch.FloatTensor`):
|
||||
the betas that the scheduler is being initialized with.
|
||||
|
||||
Returns:
|
||||
`torch.FloatTensor`: rescaled betas with zero terminal SNR
|
||||
"""
|
||||
# Convert betas to alphas_bar_sqrt
|
||||
alphas = 1.0 - betas
|
||||
alphas_cumprod = torch.cumprod(alphas, dim=0)
|
||||
alphas_bar_sqrt = alphas_cumprod.sqrt()
|
||||
|
||||
# Store old values.
|
||||
alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone()
|
||||
alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone()
|
||||
|
||||
# Shift so the last timestep is zero.
|
||||
alphas_bar_sqrt -= alphas_bar_sqrt_T
|
||||
|
||||
# Scale so the first timestep is back to the old value.
|
||||
alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T)
|
||||
|
||||
# Convert alphas_bar_sqrt to betas
|
||||
alphas_bar = alphas_bar_sqrt**2 # Revert sqrt
|
||||
alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod
|
||||
alphas = torch.cat([alphas_bar[0:1], alphas])
|
||||
betas = 1 - alphas
|
||||
|
||||
return betas
|
||||
|
||||
|
||||
class LCMSingleStepScheduler(SchedulerMixin, ConfigMixin):
|
||||
"""
|
||||
`LCMSingleStepScheduler` extends the denoising procedure introduced in denoising diffusion probabilistic models (DDPMs) with
|
||||
non-Markovian guidance.
|
||||
|
||||
This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. [`~ConfigMixin`] takes care of storing all config
|
||||
attributes that are passed in the scheduler's `__init__` function, such as `num_train_timesteps`. They can be
|
||||
accessed via `scheduler.config.num_train_timesteps`. [`SchedulerMixin`] provides general loading and saving
|
||||
functionality via the [`SchedulerMixin.save_pretrained`] and [`~SchedulerMixin.from_pretrained`] functions.
|
||||
|
||||
Args:
|
||||
num_train_timesteps (`int`, defaults to 1000):
|
||||
The number of diffusion steps to train the model.
|
||||
beta_start (`float`, defaults to 0.0001):
|
||||
The starting `beta` value of inference.
|
||||
beta_end (`float`, defaults to 0.02):
|
||||
The final `beta` value.
|
||||
beta_schedule (`str`, defaults to `"linear"`):
|
||||
The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from
|
||||
`linear`, `scaled_linear`, or `squaredcos_cap_v2`.
|
||||
trained_betas (`np.ndarray`, *optional*):
|
||||
Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`.
|
||||
original_inference_steps (`int`, *optional*, defaults to 50):
|
||||
The default number of inference steps used to generate a linearly-spaced timestep schedule, from which we
|
||||
will ultimately take `num_inference_steps` evenly spaced timesteps to form the final timestep schedule.
|
||||
clip_sample (`bool`, defaults to `True`):
|
||||
Clip the predicted sample for numerical stability.
|
||||
clip_sample_range (`float`, defaults to 1.0):
|
||||
The maximum magnitude for sample clipping. Valid only when `clip_sample=True`.
|
||||
set_alpha_to_one (`bool`, defaults to `True`):
|
||||
Each diffusion step uses the alphas product value at that step and at the previous one. For the final step
|
||||
there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`,
|
||||
otherwise it uses the alpha value at step 0.
|
||||
steps_offset (`int`, defaults to 0):
|
||||
An offset added to the inference steps. You can use a combination of `offset=1` and
|
||||
`set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
|
||||
Diffusion.
|
||||
prediction_type (`str`, defaults to `epsilon`, *optional*):
|
||||
Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process),
|
||||
`sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen
|
||||
Video](https://imagen.research.google/video/paper.pdf) paper).
|
||||
thresholding (`bool`, defaults to `False`):
|
||||
Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such
|
||||
as Stable Diffusion.
|
||||
dynamic_thresholding_ratio (`float`, defaults to 0.995):
|
||||
The ratio for the dynamic thresholding method. Valid only when `thresholding=True`.
|
||||
sample_max_value (`float`, defaults to 1.0):
|
||||
The threshold value for dynamic thresholding. Valid only when `thresholding=True`.
|
||||
timestep_spacing (`str`, defaults to `"leading"`):
|
||||
The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
|
||||
Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
|
||||
timestep_scaling (`float`, defaults to 10.0):
|
||||
The factor the timesteps will be multiplied by when calculating the consistency model boundary conditions
|
||||
`c_skip` and `c_out`. Increasing this will decrease the approximation error (although the approximation
|
||||
error at the default of `10.0` is already pretty small).
|
||||
rescale_betas_zero_snr (`bool`, defaults to `False`):
|
||||
Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and
|
||||
dark samples instead of limiting it to samples with medium brightness. Loosely related to
|
||||
[`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506).
|
||||
"""
|
||||
|
||||
order = 1
|
||||
|
||||
@register_to_config
|
||||
def __init__(
|
||||
self,
|
||||
num_train_timesteps: int = 1000,
|
||||
beta_start: float = 0.00085,
|
||||
beta_end: float = 0.012,
|
||||
beta_schedule: str = "scaled_linear",
|
||||
trained_betas: Optional[Union[np.ndarray, List[float]]] = None,
|
||||
original_inference_steps: int = 50,
|
||||
clip_sample: bool = False,
|
||||
clip_sample_range: float = 1.0,
|
||||
set_alpha_to_one: bool = True,
|
||||
steps_offset: int = 0,
|
||||
prediction_type: str = "epsilon",
|
||||
thresholding: bool = False,
|
||||
dynamic_thresholding_ratio: float = 0.995,
|
||||
sample_max_value: float = 1.0,
|
||||
timestep_spacing: str = "leading",
|
||||
timestep_scaling: float = 10.0,
|
||||
rescale_betas_zero_snr: bool = False,
|
||||
):
|
||||
if trained_betas is not None:
|
||||
self.betas = torch.tensor(trained_betas, dtype=torch.float32)
|
||||
elif beta_schedule == "linear":
|
||||
self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32)
|
||||
elif beta_schedule == "scaled_linear":
|
||||
# this schedule is very specific to the latent diffusion model.
|
||||
self.betas = (
|
||||
torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2
|
||||
)
|
||||
elif beta_schedule == "squaredcos_cap_v2":
|
||||
# Glide cosine schedule
|
||||
self.betas = betas_for_alpha_bar(num_train_timesteps)
|
||||
else:
|
||||
raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}")
|
||||
|
||||
# Rescale for zero SNR
|
||||
if rescale_betas_zero_snr:
|
||||
self.betas = rescale_zero_terminal_snr(self.betas)
|
||||
|
||||
self.alphas = 1.0 - self.betas
|
||||
self.alphas_cumprod = torch.cumprod(self.alphas, dim=0)
|
||||
|
||||
# At every step in ddim, we are looking into the previous alphas_cumprod
|
||||
# For the final step, there is no previous alphas_cumprod because we are already at 0
|
||||
# `set_alpha_to_one` decides whether we set this parameter simply to one or
|
||||
# whether we use the final alpha of the "non-previous" one.
|
||||
self.final_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0]
|
||||
|
||||
# standard deviation of the initial noise distribution
|
||||
self.init_noise_sigma = 1.0
|
||||
|
||||
# setable values
|
||||
self.num_inference_steps = None
|
||||
self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy().astype(np.int64))
|
||||
|
||||
self._step_index = None
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index
|
||||
def _init_step_index(self, timestep):
|
||||
if isinstance(timestep, torch.Tensor):
|
||||
timestep = timestep.to(self.timesteps.device)
|
||||
|
||||
index_candidates = (self.timesteps == timestep).nonzero()
|
||||
|
||||
# The sigma index that is taken for the **very** first `step`
|
||||
# is always the second index (or the last index if there is only 1)
|
||||
# This way we can ensure we don't accidentally skip a sigma in
|
||||
# case we start in the middle of the denoising schedule (e.g. for image-to-image)
|
||||
if len(index_candidates) > 1:
|
||||
step_index = index_candidates[1]
|
||||
else:
|
||||
step_index = index_candidates[0]
|
||||
|
||||
self._step_index = step_index.item()
|
||||
|
||||
@property
|
||||
def step_index(self):
|
||||
return self._step_index
|
||||
|
||||
def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor:
|
||||
"""
|
||||
Ensures interchangeability with schedulers that need to scale the denoising model input depending on the
|
||||
current timestep.
|
||||
|
||||
Args:
|
||||
sample (`torch.FloatTensor`):
|
||||
The input sample.
|
||||
timestep (`int`, *optional*):
|
||||
The current timestep in the diffusion chain.
|
||||
Returns:
|
||||
`torch.FloatTensor`:
|
||||
A scaled input sample.
|
||||
"""
|
||||
return sample
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample
|
||||
def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor:
|
||||
"""
|
||||
"Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the
|
||||
prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by
|
||||
s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing
|
||||
pixels from saturation at each step. We find that dynamic thresholding results in significantly better
|
||||
photorealism as well as better image-text alignment, especially when using very large guidance weights."
|
||||
|
||||
https://arxiv.org/abs/2205.11487
|
||||
"""
|
||||
dtype = sample.dtype
|
||||
batch_size, channels, *remaining_dims = sample.shape
|
||||
|
||||
if dtype not in (torch.float32, torch.float64):
|
||||
sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half
|
||||
|
||||
# Flatten sample for doing quantile calculation along each image
|
||||
sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))
|
||||
|
||||
abs_sample = sample.abs() # "a certain percentile absolute pixel value"
|
||||
|
||||
s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1)
|
||||
s = torch.clamp(
|
||||
s, min=1, max=self.config.sample_max_value
|
||||
) # When clamped to min=1, equivalent to standard clipping to [-1, 1]
|
||||
s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0
|
||||
sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s"
|
||||
|
||||
sample = sample.reshape(batch_size, channels, *remaining_dims)
|
||||
sample = sample.to(dtype)
|
||||
|
||||
return sample
|
||||
|
||||
def set_timesteps(
|
||||
self,
|
||||
num_inference_steps: int = None,
|
||||
device: Union[str, torch.device] = None,
|
||||
original_inference_steps: Optional[int] = None,
|
||||
strength: int = 1.0,
|
||||
timesteps: Optional[list] = None,
|
||||
):
|
||||
"""
|
||||
Sets the discrete timesteps used for the diffusion chain (to be run before inference).
|
||||
|
||||
Args:
|
||||
num_inference_steps (`int`):
|
||||
The number of diffusion steps used when generating samples with a pre-trained model.
|
||||
device (`str` or `torch.device`, *optional*):
|
||||
The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
|
||||
original_inference_steps (`int`, *optional*):
|
||||
The original number of inference steps, which will be used to generate a linearly-spaced timestep
|
||||
schedule (which is different from the standard `diffusers` implementation). We will then take
|
||||
`num_inference_steps` timesteps from this schedule, evenly spaced in terms of indices, and use that as
|
||||
our final timestep schedule. If not set, this will default to the `original_inference_steps` attribute.
|
||||
"""
|
||||
|
||||
if num_inference_steps is not None and timesteps is not None:
|
||||
raise ValueError("Can only pass one of `num_inference_steps` or `custom_timesteps`.")
|
||||
|
||||
if timesteps is not None:
|
||||
for i in range(1, len(timesteps)):
|
||||
if timesteps[i] >= timesteps[i - 1]:
|
||||
raise ValueError("`custom_timesteps` must be in descending order.")
|
||||
|
||||
if timesteps[0] >= self.config.num_train_timesteps:
|
||||
raise ValueError(
|
||||
f"`timesteps` must start before `self.config.train_timesteps`:"
|
||||
f" {self.config.num_train_timesteps}."
|
||||
)
|
||||
|
||||
timesteps = np.array(timesteps, dtype=np.int64)
|
||||
else:
|
||||
if num_inference_steps > self.config.num_train_timesteps:
|
||||
raise ValueError(
|
||||
f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:"
|
||||
f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle"
|
||||
f" maximal {self.config.num_train_timesteps} timesteps."
|
||||
)
|
||||
|
||||
self.num_inference_steps = num_inference_steps
|
||||
original_steps = (
|
||||
original_inference_steps if original_inference_steps is not None else self.config.original_inference_steps
|
||||
)
|
||||
|
||||
if original_steps > self.config.num_train_timesteps:
|
||||
raise ValueError(
|
||||
f"`original_steps`: {original_steps} cannot be larger than `self.config.train_timesteps`:"
|
||||
f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle"
|
||||
f" maximal {self.config.num_train_timesteps} timesteps."
|
||||
)
|
||||
|
||||
if num_inference_steps > original_steps:
|
||||
raise ValueError(
|
||||
f"`num_inference_steps`: {num_inference_steps} cannot be larger than `original_inference_steps`:"
|
||||
f" {original_steps} because the final timestep schedule will be a subset of the"
|
||||
f" `original_inference_steps`-sized initial timestep schedule."
|
||||
)
|
||||
|
||||
# LCM Timesteps Setting
|
||||
# Currently, only linear spacing is supported.
|
||||
c = self.config.num_train_timesteps // original_steps
|
||||
# LCM Training Steps Schedule
|
||||
lcm_origin_timesteps = np.asarray(list(range(1, int(original_steps * strength) + 1))) * c - 1
|
||||
skipping_step = len(lcm_origin_timesteps) // num_inference_steps
|
||||
# LCM Inference Steps Schedule
|
||||
timesteps = lcm_origin_timesteps[::-skipping_step][:num_inference_steps]
|
||||
|
||||
self.timesteps = torch.from_numpy(timesteps.copy()).to(device=device, dtype=torch.long)
|
||||
|
||||
self._step_index = None
|
||||
|
||||
def get_scalings_for_boundary_condition_discrete(self, timestep):
|
||||
self.sigma_data = 0.5 # Default: 0.5
|
||||
scaled_timestep = timestep * self.config.timestep_scaling
|
||||
|
||||
c_skip = self.sigma_data**2 / (scaled_timestep**2 + self.sigma_data**2)
|
||||
c_out = scaled_timestep / (scaled_timestep**2 + self.sigma_data**2) ** 0.5
|
||||
return c_skip, c_out
|
||||
|
||||
def append_dims(self, x, target_dims):
|
||||
"""Appends dimensions to the end of a tensor until it has target_dims dimensions."""
|
||||
dims_to_append = target_dims - x.ndim
|
||||
if dims_to_append < 0:
|
||||
raise ValueError(f"input has {x.ndim} dims but target_dims is {target_dims}, which is less")
|
||||
return x[(...,) + (None,) * dims_to_append]
|
||||
|
||||
def extract_into_tensor(self, a, t, x_shape):
|
||||
b, *_ = t.shape
|
||||
out = a.gather(-1, t)
|
||||
return out.reshape(b, *((1,) * (len(x_shape) - 1)))
|
||||
|
||||
def step(
|
||||
self,
|
||||
model_output: torch.FloatTensor,
|
||||
timestep: torch.Tensor,
|
||||
sample: torch.FloatTensor,
|
||||
generator: Optional[torch.Generator] = None,
|
||||
return_dict: bool = True,
|
||||
) -> Union[LCMSingleStepSchedulerOutput, Tuple]:
|
||||
"""
|
||||
Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
|
||||
process from the learned model outputs (most often the predicted noise).
|
||||
|
||||
Args:
|
||||
model_output (`torch.FloatTensor`):
|
||||
The direct output from learned diffusion model.
|
||||
timestep (`float`):
|
||||
The current discrete timestep in the diffusion chain.
|
||||
sample (`torch.FloatTensor`):
|
||||
A current instance of a sample created by the diffusion process.
|
||||
generator (`torch.Generator`, *optional*):
|
||||
A random number generator.
|
||||
return_dict (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not to return a [`~schedulers.scheduling_lcm.LCMSchedulerOutput`] or `tuple`.
|
||||
Returns:
|
||||
[`~schedulers.scheduling_utils.LCMSchedulerOutput`] or `tuple`:
|
||||
If return_dict is `True`, [`~schedulers.scheduling_lcm.LCMSchedulerOutput`] is returned, otherwise a
|
||||
tuple is returned where the first element is the sample tensor.
|
||||
"""
|
||||
# 0. make sure everything is on the same device
|
||||
alphas_cumprod = self.alphas_cumprod.to(sample.device)
|
||||
|
||||
# 1. compute alphas, betas
|
||||
if timestep.ndim == 0:
|
||||
timestep = timestep.unsqueeze(0)
|
||||
alpha_prod_t = self.extract_into_tensor(alphas_cumprod, timestep, sample.shape)
|
||||
beta_prod_t = 1 - alpha_prod_t
|
||||
|
||||
# 2. Get scalings for boundary conditions
|
||||
c_skip, c_out = self.get_scalings_for_boundary_condition_discrete(timestep)
|
||||
c_skip, c_out = [self.append_dims(x, sample.ndim) for x in [c_skip, c_out]]
|
||||
|
||||
# 3. Compute the predicted original sample x_0 based on the model parameterization
|
||||
if self.config.prediction_type == "epsilon": # noise-prediction
|
||||
predicted_original_sample = (sample - torch.sqrt(beta_prod_t) * model_output) / torch.sqrt(alpha_prod_t)
|
||||
elif self.config.prediction_type == "sample": # x-prediction
|
||||
predicted_original_sample = model_output
|
||||
elif self.config.prediction_type == "v_prediction": # v-prediction
|
||||
predicted_original_sample = torch.sqrt(alpha_prod_t) * sample - torch.sqrt(beta_prod_t) * model_output
|
||||
else:
|
||||
raise ValueError(
|
||||
f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` or"
|
||||
" `v_prediction` for `LCMScheduler`."
|
||||
)
|
||||
|
||||
# 4. Clip or threshold "predicted x_0"
|
||||
if self.config.thresholding:
|
||||
predicted_original_sample = self._threshold_sample(predicted_original_sample)
|
||||
elif self.config.clip_sample:
|
||||
predicted_original_sample = predicted_original_sample.clamp(
|
||||
-self.config.clip_sample_range, self.config.clip_sample_range
|
||||
)
|
||||
|
||||
# 5. Denoise model output using boundary conditions
|
||||
denoised = c_out * predicted_original_sample + c_skip * sample
|
||||
|
||||
if not return_dict:
|
||||
return (denoised, )
|
||||
|
||||
return LCMSingleStepSchedulerOutput(denoised=denoised)
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise
|
||||
def add_noise(
|
||||
self,
|
||||
original_samples: torch.FloatTensor,
|
||||
noise: torch.FloatTensor,
|
||||
timesteps: torch.IntTensor,
|
||||
) -> torch.FloatTensor:
|
||||
# Make sure alphas_cumprod and timestep have same device and dtype as original_samples
|
||||
alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device, dtype=original_samples.dtype)
|
||||
timesteps = timesteps.to(original_samples.device)
|
||||
|
||||
sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5
|
||||
sqrt_alpha_prod = sqrt_alpha_prod.flatten()
|
||||
while len(sqrt_alpha_prod.shape) < len(original_samples.shape):
|
||||
sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1)
|
||||
|
||||
sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5
|
||||
sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten()
|
||||
while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape):
|
||||
sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1)
|
||||
|
||||
noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise
|
||||
return noisy_samples
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.get_velocity
|
||||
def get_velocity(
|
||||
self, sample: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor
|
||||
) -> torch.FloatTensor:
|
||||
# Make sure alphas_cumprod and timestep have same device and dtype as sample
|
||||
alphas_cumprod = self.alphas_cumprod.to(device=sample.device, dtype=sample.dtype)
|
||||
timesteps = timesteps.to(sample.device)
|
||||
|
||||
sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5
|
||||
sqrt_alpha_prod = sqrt_alpha_prod.flatten()
|
||||
while len(sqrt_alpha_prod.shape) < len(sample.shape):
|
||||
sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1)
|
||||
|
||||
sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5
|
||||
sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten()
|
||||
while len(sqrt_one_minus_alpha_prod.shape) < len(sample.shape):
|
||||
sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1)
|
||||
|
||||
velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample
|
||||
return velocity
|
||||
|
||||
def __len__(self):
|
||||
return self.config.num_train_timesteps
|
||||
File diff suppressed because it is too large
Load Diff
+32
-7
@@ -35,8 +35,15 @@ ADAPTERS_SDXL = {
|
||||
'Plus Face ViT-H SDXL': { 'name': 'ip-adapter-plus-face_sdxl_vit-h.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'sdxl_models' },
|
||||
'Ostris Composition ViT-H SDXL': { 'name': 'ip_plus_composition_sdxl.safetensors', 'repo': 'ostris/ip-composition-adapter', 'subfolder': '' },
|
||||
}
|
||||
ADAPTERS = { **ADAPTERS_SD15, **ADAPTERS_SDXL }
|
||||
ADAPTERS_ALL = { **ADAPTERS_SD15, **ADAPTERS_SDXL }
|
||||
ADAPTERS_SD3 = {
|
||||
'InstantX Large': { 'name': 'ip-adapter.bin', 'repo': 'InstantX/SD3.5-Large-IP-Adapter' },
|
||||
}
|
||||
ADAPTERS_F1 = {
|
||||
'XLabs AI v1': { 'name': 'ip_adapter.safetensors', 'repo': 'XLabs-AI/flux-ip-adapter' },
|
||||
'XLabs AI v2': { 'name': 'ip_adapter.safetensors', 'repo': 'XLabs-AI/flux-ip-adapter-v2' },
|
||||
}
|
||||
ADAPTERS = { **ADAPTERS_SD15, **ADAPTERS_SDXL, **ADAPTERS_SD3, **ADAPTERS_F1 }
|
||||
ADAPTERS_ALL = { **ADAPTERS_SD15, **ADAPTERS_SDXL, **ADAPTERS_SD3, **ADAPTERS_F1 }
|
||||
|
||||
|
||||
def get_adapters():
|
||||
@@ -45,6 +52,10 @@ def get_adapters():
|
||||
ADAPTERS = ADAPTERS_SD15
|
||||
elif shared.sd_model_type == 'sdxl':
|
||||
ADAPTERS = ADAPTERS_SDXL
|
||||
elif shared.sd_model_type == 'sd3':
|
||||
ADAPTERS = ADAPTERS_SD3
|
||||
elif shared.sd_model_type == 'f1':
|
||||
ADAPTERS = ADAPTERS_F1
|
||||
else:
|
||||
ADAPTERS = ADAPTERS_NONE
|
||||
return list(ADAPTERS)
|
||||
@@ -55,7 +66,7 @@ def get_images(input_images):
|
||||
if input_images is None or len(input_images) == 0:
|
||||
shared.log.error('IP adapter: no init images')
|
||||
return None
|
||||
if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
|
||||
if shared.sd_model_type not in ['sd', 'sdxl', 'sd3', 'f1']:
|
||||
shared.log.error('IP adapter: base model not supported')
|
||||
return None
|
||||
if isinstance(input_images, str):
|
||||
@@ -147,7 +158,7 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt
|
||||
if hasattr(p, 'ip_adapter_images'):
|
||||
del p.ip_adapter_images
|
||||
return False
|
||||
if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
|
||||
if shared.sd_model_type not in ['sd', 'sdxl', 'sd3', 'f1']:
|
||||
shared.log.error(f'IP adapter: model={shared.sd_model_type} class={pipe.__class__.__name__} not supported')
|
||||
return False
|
||||
if hasattr(p, 'ip_adapter_scales'):
|
||||
@@ -172,6 +183,9 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt
|
||||
for i in range(len(adapter_masks)):
|
||||
adapter_masks[i] = mask_processor.preprocess(adapter_masks[i], height=p.height, width=p.width)
|
||||
adapter_masks = mask_processor.preprocess(adapter_masks, height=p.height, width=p.width)
|
||||
if adapter_images is None:
|
||||
shared.log.error('IP adapter: no image provided')
|
||||
return False
|
||||
if len(adapters) < len(adapter_images):
|
||||
adapter_images = adapter_images[:len(adapters)]
|
||||
if len(adapters) < len(adapter_masks):
|
||||
@@ -212,13 +226,24 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt
|
||||
clip_subfolder = 'models/image_encoder'
|
||||
else:
|
||||
clip_subfolder = 'sdxl_models/image_encoder'
|
||||
elif 'ViT-H' in adapter_name:
|
||||
if 'ViT-H' in adapter_name:
|
||||
clip_subfolder = 'models/image_encoder' # this is vit-h
|
||||
elif 'ViT-G' in adapter_name:
|
||||
clip_subfolder = 'sdxl_models/image_encoder' # this is vit-g
|
||||
else:
|
||||
shared.log.error(f'IP adapter: unknown model type: {adapter_name}')
|
||||
return False
|
||||
if shared.sd_model_type == 'sd':
|
||||
clip_subfolder = 'models/image_encoder'
|
||||
elif shared.sd_model_type == 'sdxl':
|
||||
clip_subfolder = 'sdxl_models/image_encoder'
|
||||
elif shared.sd_model_type == 'sd3':
|
||||
shared.log.error(f'IP adapter: adapter={adapter_name} type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__}: unsupported base model')
|
||||
return False
|
||||
elif shared.sd_model_type == 'f1':
|
||||
shared.log.error(f'IP adapter: adapter={adapter_name} type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__}: unsupported base model')
|
||||
return False
|
||||
else:
|
||||
shared.log.error(f'IP adapter: unknown model type: {adapter_name}')
|
||||
return False
|
||||
|
||||
# load feature extractor used by ip adapter
|
||||
if pipe.feature_extractor is None:
|
||||
|
||||
+2
-1
@@ -126,4 +126,5 @@ except ImportError:
|
||||
except ImportError:
|
||||
pass # shrug...
|
||||
|
||||
errors.log.info(f'System packages: {get_packages()}')
|
||||
errors.log.info(f'Torch: torch=={torch.__version__} torchvision=={torchvision.__version__}')
|
||||
errors.log.info(f'Packages: diffusers=={diffusers.__version__} transformers=={transformers.__version__} accelerate=={accelerate.__version__} gradio=={gradio.__version__}')
|
||||
|
||||
@@ -32,14 +32,14 @@ def load_bnb(msg='', silent=False):
|
||||
global bnb # pylint: disable=global-statement
|
||||
if bnb is not None:
|
||||
return bnb
|
||||
fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
|
||||
log.debug(f'Quantization: type=bitsandbytes fn={fn}') # pylint: disable=protected-access
|
||||
install('bitsandbytes', quiet=True)
|
||||
try:
|
||||
import bitsandbytes
|
||||
bnb = bitsandbytes
|
||||
diffusers.utils.import_utils._bitsandbytes_available = True # pylint: disable=protected-access
|
||||
diffusers.utils.import_utils._bitsandbytes_version = '0.43.3' # pylint: disable=protected-access
|
||||
fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
|
||||
log.debug(f'Quantization: type=bitsandbytes version={bnb.__version__} fn={fn}') # pylint: disable=protected-access
|
||||
return bnb
|
||||
except Exception as e:
|
||||
if len(msg) > 0:
|
||||
@@ -54,12 +54,12 @@ def load_quanto(msg='', silent=False):
|
||||
global quanto # pylint: disable=global-statement
|
||||
if quanto is not None:
|
||||
return quanto
|
||||
fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
|
||||
log.debug(f'Quantization: type=quanto fn={fn}') # pylint: disable=protected-access
|
||||
install('optimum-quanto', quiet=True)
|
||||
try:
|
||||
from optimum import quanto as optimum_quanto # pylint: disable=no-name-in-module
|
||||
quanto = optimum_quanto
|
||||
fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
|
||||
log.debug(f'Quantization: type=quanto version={quanto.__version__} fn={fn}') # pylint: disable=protected-access
|
||||
return quanto
|
||||
except Exception as e:
|
||||
if len(msg) > 0:
|
||||
|
||||
+24
-2
@@ -5,13 +5,35 @@ import safetensors.torch
|
||||
from modules import shared, devices, model_quant
|
||||
|
||||
|
||||
def remove_entries_after_depth(d, depth, current_depth=0):
|
||||
if current_depth >= depth:
|
||||
return None
|
||||
if isinstance(d, dict):
|
||||
return {k: remove_entries_after_depth(v, depth, current_depth + 1) for k, v in d.items() if remove_entries_after_depth(v, depth, current_depth + 1) is not None}
|
||||
return d
|
||||
|
||||
|
||||
def list_to_dict(flat_list):
|
||||
result_dict = {}
|
||||
try:
|
||||
for item in flat_list:
|
||||
keys = item.split('.')
|
||||
d = result_dict
|
||||
for key in keys[:-1]:
|
||||
d = d.setdefault(key, {})
|
||||
d[keys[-1]] = None
|
||||
except Exception:
|
||||
pass
|
||||
return result_dict
|
||||
|
||||
|
||||
def get_safetensor_keys(filename):
|
||||
keys = []
|
||||
try:
|
||||
with safetensors.torch.safe_open(filename, framework="pt", device="cpu") as f:
|
||||
keys = f.keys()
|
||||
except Exception as e:
|
||||
shared.log.error(f'Load dict: path="{filename}" {e}')
|
||||
except Exception:
|
||||
pass
|
||||
return keys
|
||||
|
||||
|
||||
|
||||
+18
-11
@@ -18,7 +18,7 @@ PREDEFINED = [ # <https://huggingface.co/vladmandic/yolo-detailers/tree/main>
|
||||
|
||||
|
||||
class YoloResult:
|
||||
def __init__(self, cls: int, label: str, score: float, box: list[int], mask: Image.Image = None, item: Image.Image = None, size: float = 0, width = 0, height = 0, args = {}):
|
||||
def __init__(self, cls: int, label: str, score: float, box: list[int], mask: Image.Image = None, item: Image.Image = None, width = 0, height = 0, args = {}):
|
||||
self.cls = cls
|
||||
self.label = label
|
||||
self.score = score
|
||||
@@ -29,6 +29,9 @@ class YoloResult:
|
||||
self.height = height
|
||||
self.args = args
|
||||
|
||||
def __str__(self):
|
||||
return f'cls={self.cls} label={self.label} score={self.score} box={self.box} mask={self.mask} item={self.item} size={self.width}x{self.height} args={self.args}'
|
||||
|
||||
|
||||
class YoloRestorer(Detailer):
|
||||
def __init__(self):
|
||||
@@ -76,11 +79,15 @@ class YoloRestorer(Detailer):
|
||||
offload: bool = shared.opts.detailer_unload,
|
||||
) -> list[YoloResult]:
|
||||
|
||||
if model is None or (isinstance(model, str) and len(model) == 0):
|
||||
model = 'yolo11m'
|
||||
result = []
|
||||
if isinstance(model, str):
|
||||
model = self.models.get(model, None)
|
||||
if model is None:
|
||||
cached = self.models.get(model, None)
|
||||
if cached is None:
|
||||
_, model = self.load(model)
|
||||
else:
|
||||
model = cached
|
||||
if model is None:
|
||||
return result
|
||||
args = {
|
||||
@@ -136,7 +143,8 @@ class YoloRestorer(Detailer):
|
||||
draw = ImageDraw.Draw(mask_image)
|
||||
draw.rectangle(box, fill="white", outline=None, width=0)
|
||||
cropped = image.crop(box)
|
||||
result.append(YoloResult(cls=cls, label=label, score=round(score, 2), box=box, mask=mask_image, item=cropped, width=w, height=h, args=args))
|
||||
res = YoloResult(cls=cls, label=label, score=round(score, 2), box=box, mask=mask_image, item=cropped, width=w, height=h, args=args)
|
||||
result.append(res)
|
||||
if len(result) >= shared.opts.detailer_max:
|
||||
break
|
||||
return result
|
||||
@@ -156,10 +164,10 @@ class YoloRestorer(Detailer):
|
||||
try:
|
||||
model_file = modelloader.load_file_from_url(url=model_url, model_dir=shared.opts.yolo_dir, file_name=file_name)
|
||||
if model_file is not None:
|
||||
from ultralytics import YOLO # pylint: disable=import-outside-toplevel
|
||||
model = YOLO(model_file)
|
||||
import ultralytics
|
||||
model = ultralytics.YOLO(model_file)
|
||||
classes = list(model.names.values())
|
||||
shared.log.info(f'Load: type=Detailer name="{model_name}" model="{model_file}" classes={classes}')
|
||||
shared.log.info(f'Load: type=Detailer name="{model_name}" model="{model_file}" ultralytics={ultralytics.__version__} classes={classes}')
|
||||
self.models[model_name] = model
|
||||
return model_name, model
|
||||
except Exception as e:
|
||||
@@ -194,7 +202,6 @@ class YoloRestorer(Detailer):
|
||||
shared.log.info(f'Detailer: model="{name}" no items detected')
|
||||
continue
|
||||
|
||||
pp = None
|
||||
shared.opts.data['mask_apply_overlay'] = True
|
||||
resolution = 512 if shared.sd_model_type in ['none', 'sd', 'lcm', 'unknown'] else 1024
|
||||
orig_prompt: str = orig_p.get('all_prompts', [''])[0]
|
||||
@@ -330,9 +337,9 @@ class YoloRestorer(Detailer):
|
||||
iou = gr.Slider(label="Max overlap", elem_id=f"{tab}_detailer_iou", value=shared.opts.detailer_iou, minimum=0, maximum=1.0, step=0.05)
|
||||
with gr.Row():
|
||||
min_size = shared.opts.detailer_min_size if shared.opts.detailer_min_size < 1 else 0.0
|
||||
min_size = gr.Slider(label="Min size", elem_id=f"{tab}_detailer_min_size", value=min_size, minimum=0.1, maximum=1.0, step=0.05)
|
||||
max_size = shared.opts.detailer_min_size if shared.opts.detailer_min_size < 1 and shared.opts.detailer_min_size > 0 else 1.0
|
||||
max_size = gr.Slider(label="Max size", elem_id=f"{tab}_detailer_max_size", value=max_size, minimum=0.1, maximum=1.0, step=0.05)
|
||||
min_size = gr.Slider(label="Min size", elem_id=f"{tab}_detailer_min_size", value=min_size, minimum=0.0, maximum=1.0, step=0.05)
|
||||
max_size = shared.opts.detailer_max_size if shared.opts.detailer_max_size < 1 and shared.opts.detailer_max_size > 0 else 1.0
|
||||
max_size = gr.Slider(label="Max size", elem_id=f"{tab}_detailer_max_size", value=max_size, minimum=0.0, maximum=1.0, step=0.05)
|
||||
detailers.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[])
|
||||
classes.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[])
|
||||
strength.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[])
|
||||
|
||||
@@ -34,14 +34,14 @@ images_tensor_to_samples = processing_helpers.images_tensor_to_samples
|
||||
|
||||
|
||||
class Processed:
|
||||
def __init__(self, p: StableDiffusionProcessing, images_list, seed=-1, info="", subseed=None, all_prompts=None, all_negative_prompts=None, all_seeds=None, all_subseeds=None, index_of_first_image=0, infotexts=None, comments=""):
|
||||
def __init__(self, p: StableDiffusionProcessing, images_list, seed=-1, info=None, subseed=None, all_prompts=None, all_negative_prompts=None, all_seeds=None, all_subseeds=None, index_of_first_image=0, infotexts=None, comments=""):
|
||||
self.images = images_list
|
||||
self.prompt = p.prompt or ''
|
||||
self.negative_prompt = p.negative_prompt or ''
|
||||
self.seed = seed if seed != -1 else p.seed
|
||||
self.subseed = subseed
|
||||
self.subseed_strength = p.subseed_strength
|
||||
self.info = info
|
||||
self.info = info or create_infotext(p)
|
||||
self.comments = comments or ''
|
||||
self.width = p.width if hasattr(p, 'width') else (self.images[0].width if len(self.images) > 0 else 0)
|
||||
self.height = p.height if hasattr(p, 'height') else (self.images[0].height if len(self.images) > 0 else 0)
|
||||
@@ -80,7 +80,7 @@ class Processed:
|
||||
self.all_negative_prompts = all_negative_prompts or p.all_negative_prompts or [self.negative_prompt]
|
||||
self.all_seeds = all_seeds or p.all_seeds or [self.seed]
|
||||
self.all_subseeds = all_subseeds or p.all_subseeds or [self.subseed]
|
||||
self.infotexts = infotexts or [info]
|
||||
self.infotexts = infotexts or [self.info]
|
||||
|
||||
def js(self):
|
||||
obj = {
|
||||
|
||||
+33
-26
@@ -9,6 +9,7 @@ import numpy as np
|
||||
from modules import shared, errors, sd_models, processing, processing_vae, processing_helpers, sd_hijack_hypertile, prompt_parser_diffusers, timer
|
||||
from modules.processing_callbacks import diffusers_callback_legacy, diffusers_callback, set_callbacks_p
|
||||
from modules.processing_helpers import resize_hires, fix_prompts, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, get_generator, set_latents, apply_circular # pylint: disable=unused-import
|
||||
from modules.api import helpers
|
||||
|
||||
|
||||
debug = shared.log.trace if os.environ.get('SD_DIFFUSERS_DEBUG', None) is not None else lambda *args, **kwargs: None
|
||||
@@ -18,7 +19,9 @@ def task_specific_kwargs(p, model):
|
||||
task_args = {}
|
||||
is_img2img_model = bool('Zero123' in shared.sd_model.__class__.__name__)
|
||||
if len(getattr(p, 'init_images', [])) > 0:
|
||||
p.init_images = [p.convert('RGB') for p in p.init_images]
|
||||
if isinstance(p.init_images[0], str):
|
||||
p.init_images = [helpers.decode_base64_to_image(i, quiet=True) for i in p.init_images]
|
||||
p.init_images = [i.convert('RGB') if i.mode != 'RGB' else i for i in p.init_images]
|
||||
if sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE or len(getattr(p, 'init_images', [])) == 0 and not is_img2img_model:
|
||||
p.ops.append('txt2img')
|
||||
if hasattr(p, 'width') and hasattr(p, 'height'):
|
||||
@@ -27,7 +30,7 @@ def task_specific_kwargs(p, model):
|
||||
'height': 8 * math.ceil(p.height / 8),
|
||||
}
|
||||
elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or is_img2img_model) and len(getattr(p, 'init_images', [])) > 0:
|
||||
if shared.sd_model_type == 'sdxl':
|
||||
if shared.sd_model_type == 'sdxl' and hasattr(model, 'register_to_config'):
|
||||
model.register_to_config(requires_aesthetics_score = False)
|
||||
p.ops.append('img2img')
|
||||
task_args = {
|
||||
@@ -55,7 +58,7 @@ def task_specific_kwargs(p, model):
|
||||
'strength': p.denoising_strength,
|
||||
}
|
||||
elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INPAINTING or is_img2img_model) and len(getattr(p, 'init_images', [])) > 0:
|
||||
if shared.sd_model_type == 'sdxl':
|
||||
if shared.sd_model_type == 'sdxl' and hasattr(model, 'register_to_config'):
|
||||
model.register_to_config(requires_aesthetics_score = False)
|
||||
if p.detailer:
|
||||
p.ops.append('detailer')
|
||||
@@ -100,62 +103,64 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2
|
||||
if hasattr(model, "set_progress_bar_config"):
|
||||
model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba')
|
||||
args = {}
|
||||
if hasattr(model, 'pipe'): # recurse
|
||||
if hasattr(model, 'pipe') and not hasattr(model, 'no_recurse'): # recurse
|
||||
model = model.pipe
|
||||
signature = inspect.signature(type(model).__call__, follow_wrapped=True)
|
||||
possible = list(signature.parameters)
|
||||
|
||||
debug(f'Diffusers pipeline possible: {possible}')
|
||||
prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2)
|
||||
parser = 'Fixed attention'
|
||||
steps = kwargs.get("num_inference_steps", None) or len(getattr(p, 'timesteps', ['1']))
|
||||
clip_skip = kwargs.pop("clip_skip", 1)
|
||||
|
||||
# prompt_parser_diffusers.fix_position_ids(model)
|
||||
if shared.opts.prompt_attention != 'Fixed attention' and 'Onnx' not in model.__class__.__name__ and (
|
||||
parser = 'fixed'
|
||||
if shared.opts.prompt_attention != 'fixed' and 'Onnx' not in model.__class__.__name__ and (
|
||||
'StableDiffusion' in model.__class__.__name__ or
|
||||
'StableCascade' in model.__class__.__name__ or
|
||||
'Flux' in model.__class__.__name__
|
||||
):
|
||||
try:
|
||||
prompt_parser_diffusers.encode_prompts(model, p, prompts, negative_prompts, steps=steps, clip_skip=clip_skip)
|
||||
prompt_parser_diffusers.embedder = prompt_parser_diffusers.PromptEmbedder(prompts, negative_prompts, steps, clip_skip, p)
|
||||
parser = shared.opts.prompt_attention
|
||||
except Exception as e:
|
||||
shared.log.error(f'Prompt parser encode: {e}')
|
||||
if os.environ.get('SD_PROMPT_DEBUG', None) is not None:
|
||||
errors.display(e, 'Prompt parser encode')
|
||||
timer.process.record('encode', reset=False)
|
||||
else:
|
||||
prompt_parser_diffusers.embedder = None
|
||||
|
||||
if 'prompt' in possible:
|
||||
if 'OmniGen' in model.__class__.__name__:
|
||||
prompts = [p.replace('|image|', '<|image_1|>') for p in prompts]
|
||||
if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and len(p.prompt_embeds) > 0 and p.prompt_embeds[0] is not None:
|
||||
args['prompt_embeds'] = p.prompt_embeds[0]
|
||||
if hasattr(model, 'text_encoder') and hasattr(model, 'tokenizer') and 'prompt_embeds' in possible and prompt_parser_diffusers.embedder is not None:
|
||||
args['prompt_embeds'] = prompt_parser_diffusers.embedder('prompt_embeds')
|
||||
if 'StableCascade' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0:
|
||||
args['prompt_embeds_pooled'] = p.positive_pooleds[0].unsqueeze(0)
|
||||
elif 'XL' in model.__class__.__name__ and len(getattr(p, 'positive_pooleds', [])) > 0:
|
||||
args['pooled_prompt_embeds'] = p.positive_pooleds[0]
|
||||
elif 'StableDiffusion3' in model.__class__.__name__ and len(getattr(p, 'positive_pooleds', [])) > 0:
|
||||
args['pooled_prompt_embeds'] = p.positive_pooleds[0]
|
||||
elif 'Flux' in model.__class__.__name__ and len(getattr(p, 'positive_pooleds', [])) > 0:
|
||||
args['pooled_prompt_embeds'] = p.positive_pooleds[0]
|
||||
args['prompt_embeds_pooled'] = prompt_parser_diffusers.embedder('positive_pooleds').unsqueeze(0)
|
||||
elif 'XL' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None:
|
||||
args['pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('positive_pooleds')
|
||||
elif 'StableDiffusion3' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None:
|
||||
args['pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('positive_pooleds')
|
||||
elif 'Flux' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None:
|
||||
args['pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('positive_pooleds')
|
||||
else:
|
||||
args['prompt'] = prompts
|
||||
if 'negative_prompt' in possible:
|
||||
if hasattr(model, 'text_encoder') and 'negative_prompt_embeds' in possible and len(p.negative_embeds) > 0 and p.negative_embeds[0] is not None:
|
||||
args['negative_prompt_embeds'] = p.negative_embeds[0]
|
||||
if 'StableCascade' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0:
|
||||
args['negative_prompt_embeds_pooled'] = p.negative_pooleds[0].unsqueeze(0)
|
||||
if 'XL' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0:
|
||||
args['negative_pooled_prompt_embeds'] = p.negative_pooleds[0]
|
||||
if 'StableDiffusion3' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0:
|
||||
args['negative_pooled_prompt_embeds'] = p.negative_pooleds[0]
|
||||
if hasattr(model, 'text_encoder') and hasattr(model, 'tokenizer') and 'negative_prompt_embeds' in possible and prompt_parser_diffusers.embedder is not None:
|
||||
args['negative_prompt_embeds'] = prompt_parser_diffusers.embedder('negative_prompt_embeds')
|
||||
if 'StableCascade' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None:
|
||||
args['negative_prompt_embeds_pooled'] = prompt_parser_diffusers.embedder('negative_pooleds').unsqueeze(0)
|
||||
if 'XL' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None:
|
||||
args['negative_pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('negative_pooleds')
|
||||
if 'StableDiffusion3' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None:
|
||||
args['negative_pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('negative_pooleds')
|
||||
else:
|
||||
if 'PixArtSigmaPipeline' in model.__class__.__name__: # pixart-sigma pipeline throws list-of-list for negative prompt
|
||||
args['negative_prompt'] = negative_prompts[0]
|
||||
else:
|
||||
args['negative_prompt'] = negative_prompts
|
||||
|
||||
if 'clip_skip' in possible and parser == 'Fixed attention':
|
||||
if 'clip_skip' in possible and parser == 'fixed':
|
||||
if clip_skip == 1:
|
||||
pass # clip_skip = None
|
||||
else:
|
||||
@@ -180,6 +185,8 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2
|
||||
if hasattr(model, 'scheduler') and hasattr(model.scheduler, 'noise_sampler_seed') and hasattr(model.scheduler, 'noise_sampler'):
|
||||
model.scheduler.noise_sampler = None # noise needs to be reset instead of using cached values
|
||||
model.scheduler.noise_sampler_seed = p.seeds # some schedulers have internal noise generator and do not use pipeline generator
|
||||
if 'seed' in possible:
|
||||
args['seed'] = p.seed
|
||||
if 'noise_sampler_seed' in possible:
|
||||
args['noise_sampler_seed'] = p.seeds
|
||||
if 'guidance_scale' in possible:
|
||||
|
||||
@@ -3,8 +3,7 @@ import os
|
||||
import time
|
||||
import torch
|
||||
import numpy as np
|
||||
from modules import shared, processing_correction, extra_networks, timer
|
||||
|
||||
from modules import shared, processing_correction, extra_networks, timer, prompt_parser_diffusers
|
||||
|
||||
p = None
|
||||
debug_callback = shared.log.trace if os.environ.get('SD_CALLBACK_DEBUG', None) is not None else lambda *args, **kwargs: None
|
||||
@@ -14,6 +13,19 @@ def set_callbacks_p(processing):
|
||||
global p # pylint: disable=global-statement
|
||||
p = processing
|
||||
|
||||
def prompt_callback(step, kwargs):
|
||||
if prompt_parser_diffusers.embedder is None or 'prompt_embeds' not in kwargs:
|
||||
return kwargs
|
||||
try:
|
||||
prompt_embeds = prompt_parser_diffusers.embedder('prompt_embeds', step + 1)
|
||||
negative_prompt_embeds = prompt_parser_diffusers.embedder('negative_prompt_embeds', step + 1)
|
||||
if p.cfg_scale > 1: # Perform guidance
|
||||
prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) # Combined embeds
|
||||
assert prompt_embeds.shape == kwargs['prompt_embeds'].shape, f"prompt_embed shape mismatch {kwargs['prompt_embeds'].shape} {prompt_embeds.shape}"
|
||||
kwargs['prompt_embeds'] = prompt_embeds
|
||||
except Exception as e:
|
||||
debug_callback(f"Callback: {e}")
|
||||
return kwargs
|
||||
|
||||
def diffusers_callback_legacy(step: int, timestep: int, latents: typing.Union[torch.FloatTensor, np.ndarray]):
|
||||
if p is None:
|
||||
@@ -33,7 +45,7 @@ def diffusers_callback_legacy(step: int, timestep: int, latents: typing.Union[to
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
def diffusers_callback(pipe, step: int, timestep: int, kwargs: dict):
|
||||
def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {}):
|
||||
t0 = time.time()
|
||||
if p is None:
|
||||
return kwargs
|
||||
@@ -49,7 +61,7 @@ def diffusers_callback(pipe, step: int, timestep: int, kwargs: dict):
|
||||
if shared.state.interrupted or shared.state.skipped:
|
||||
raise AssertionError('Interrupted...')
|
||||
time.sleep(0.1)
|
||||
if hasattr(p, "extra_network_data"):
|
||||
if hasattr(p, "stepwise_lora"):
|
||||
extra_networks.activate(p, p.extra_network_data, step=step)
|
||||
if latents is None:
|
||||
return kwargs
|
||||
@@ -67,14 +79,7 @@ def diffusers_callback(pipe, step: int, timestep: int, kwargs: dict):
|
||||
pipe.set_ip_adapter_scale(ip_adapter_scales)
|
||||
if step != getattr(pipe, 'num_timesteps', 0):
|
||||
kwargs = processing_correction.correction_callback(p, timestep, kwargs)
|
||||
if p.scheduled_prompt and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs:
|
||||
try:
|
||||
i = (step + 1) % len(p.prompt_embeds)
|
||||
kwargs["prompt_embeds"] = p.prompt_embeds[i][0:1].expand(kwargs["prompt_embeds"].shape)
|
||||
j = (step + 1) % len(p.negative_embeds)
|
||||
kwargs["negative_prompt_embeds"] = p.negative_embeds[j][0:1].expand(kwargs["negative_prompt_embeds"].shape)
|
||||
except Exception as e:
|
||||
shared.log.debug(f"Callback: {e}")
|
||||
kwargs = prompt_callback(step, kwargs) # monkey patch for diffusers callback issues
|
||||
if step == int(getattr(pipe, 'num_timesteps', 100) * p.cfg_end) and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs:
|
||||
if "PAG" in shared.sd_model.__class__.__name__:
|
||||
pipe._guidance_scale = 1.001 if pipe._guidance_scale > 1 else pipe._guidance_scale # pylint: disable=protected-access
|
||||
|
||||
+206
-237
@@ -17,53 +17,44 @@ debug = shared.log.trace if os.environ.get('SD_PROCESS_DEBUG', None) is not None
|
||||
|
||||
@dataclass(repr=False)
|
||||
class StableDiffusionProcessing:
|
||||
"""
|
||||
The first set of paramaters: sd_models -> do_not_reload_embeddings represent the minimum required to create a StableDiffusionProcessing
|
||||
"""
|
||||
def __init__(self,
|
||||
sd_model=None,
|
||||
outpath_samples=None,
|
||||
outpath_grids=None,
|
||||
sd_model=None, # pylint: disable=unused-argument # local instance of sd_model
|
||||
# base params
|
||||
prompt: str = "",
|
||||
styles: List[str] = None,
|
||||
negative_prompt: str = "",
|
||||
seed: int = -1,
|
||||
subseed: int = -1,
|
||||
subseed_strength: float = 0,
|
||||
seed_resize_from_h: int = -1,
|
||||
seed_resize_from_w: int = -1,
|
||||
seed_enable_extras: bool = True,
|
||||
sampler_name: str = None,
|
||||
hr_sampler_name: str = None,
|
||||
batch_size: int = 1,
|
||||
n_iter: int = 1,
|
||||
steps: int = 50,
|
||||
cfg_scale: float = 7.0,
|
||||
image_cfg_scale: float = None,
|
||||
clip_skip: int = 1,
|
||||
width: int = 512,
|
||||
height: int = 512,
|
||||
full_quality: bool = True,
|
||||
detailer: bool = False,
|
||||
restore_faces: bool = False,
|
||||
tiling: bool = False,
|
||||
hidiffusion: bool = False,
|
||||
do_not_save_samples: bool = False,
|
||||
do_not_save_grid: bool = False,
|
||||
extra_generation_params: Dict[Any, Any] = None,
|
||||
overlay_images: Any = None,
|
||||
negative_prompt: str = None,
|
||||
# samplers
|
||||
sampler_index: int = None, # pylint: disable=unused-argument # used only to set sampler_name
|
||||
sampler_name: str = None,
|
||||
hr_sampler_name: str = None,
|
||||
eta: float = None,
|
||||
do_not_reload_embeddings: bool = False,
|
||||
denoising_strength: float = 0,
|
||||
# guidance
|
||||
cfg_scale: float = 7.0,
|
||||
cfg_end: float = 1,
|
||||
diffusers_guidance_rescale: float = 0.7,
|
||||
pag_scale: float = 0.0,
|
||||
pag_adaptive: float = 0.5,
|
||||
cfg_end: float = 1,
|
||||
resize_mode: int = 0,
|
||||
resize_name: str = 'None',
|
||||
resize_context: str = 'None',
|
||||
scale_by: float = 0,
|
||||
selected_scale_tab: int = 0,
|
||||
# styles
|
||||
styles: List[str] = [],
|
||||
# vae
|
||||
tiling: bool = False,
|
||||
full_quality: bool = True,
|
||||
# other
|
||||
hidiffusion: bool = False,
|
||||
do_not_reload_embeddings: bool = False,
|
||||
detailer: bool = False,
|
||||
restore_faces: bool = False,
|
||||
# hdr corrections
|
||||
hdr_mode: int = 0,
|
||||
hdr_brightness: float = 0,
|
||||
hdr_color: float = 0,
|
||||
@@ -76,92 +67,196 @@ class StableDiffusionProcessing:
|
||||
hdr_max_boundry: float = 1.0,
|
||||
hdr_color_picker: str = None,
|
||||
hdr_tint_ratio: float = 0,
|
||||
override_settings: Dict[str, Any] = None,
|
||||
# img2img
|
||||
init_images: list = None,
|
||||
resize_mode: int = 0,
|
||||
resize_name: str = 'None',
|
||||
resize_context: str = 'None',
|
||||
denoising_strength: float = 0.3,
|
||||
image_cfg_scale: float = None,
|
||||
initial_noise_multiplier: float = None, # pylint: disable=unused-argument # a1111 compatibility
|
||||
scale_by: float = 1,
|
||||
selected_scale_tab: int = 0, # pylint: disable=unused-argument # a1111 compatibility
|
||||
# inpaint
|
||||
mask: Any = None,
|
||||
latent_mask: Any = None,
|
||||
mask_for_overlay: Any = None,
|
||||
mask_blur: int = 4,
|
||||
paste_to: Any = None,
|
||||
inpainting_fill: int = 0,
|
||||
inpaint_full_res: bool = False,
|
||||
inpaint_full_res_padding: int = 0,
|
||||
inpainting_mask_invert: int = 0,
|
||||
overlay_images: Any = None,
|
||||
# refiner
|
||||
enable_hr: bool = False,
|
||||
firstphase_width: int = 0,
|
||||
firstphase_height: int = 0,
|
||||
hr_scale: float = 2.0,
|
||||
hr_force: bool = False,
|
||||
hr_resize_mode: int = 0,
|
||||
hr_resize_context: str = 'None',
|
||||
hr_upscaler: str = None,
|
||||
hr_second_pass_steps: int = 0,
|
||||
hr_resize_x: int = 0,
|
||||
hr_resize_y: int = 0,
|
||||
hr_denoising_strength: float = 0.0,
|
||||
refiner_steps: int = 5,
|
||||
refiner_start: float = 0,
|
||||
refiner_prompt: str = '',
|
||||
refiner_negative: str = '',
|
||||
hr_refiner_start: float = 0,
|
||||
# save options
|
||||
outpath_samples=None,
|
||||
outpath_grids=None,
|
||||
do_not_save_samples: bool = False,
|
||||
do_not_save_grid: bool = False,
|
||||
# scripts
|
||||
script_args: list = [],
|
||||
# overrides
|
||||
override_settings: Dict[str, Any] = {},
|
||||
override_settings_restore_afterwards: bool = True,
|
||||
sampler_index: int = None,
|
||||
script_args: list = None
|
||||
): # pylint: disable=unused-argument
|
||||
# metadata
|
||||
extra_generation_params: Dict[Any, Any] = {},
|
||||
):
|
||||
|
||||
# extra args set by processing loop
|
||||
self.task_args = {}
|
||||
|
||||
# state items
|
||||
self.state: str = ''
|
||||
self.ops = []
|
||||
self.skip = []
|
||||
self.outpath_samples: str = outpath_samples
|
||||
self.outpath_grids: str = outpath_grids
|
||||
self.prompt: str = prompt
|
||||
self.prompt_for_display: str = None
|
||||
self.negative_prompt: str = (negative_prompt or "")
|
||||
self.styles: list = styles or []
|
||||
self.seed: int = seed
|
||||
self.subseed: int = subseed
|
||||
self.subseed_strength: float = subseed_strength
|
||||
self.seed_resize_from_h: int = seed_resize_from_h
|
||||
self.seed_resize_from_w: int = seed_resize_from_w
|
||||
self.sampler_name: str = sampler_name
|
||||
self.hr_sampler_name: str = hr_sampler_name if hr_sampler_name != 'Same as primary' else sampler_name
|
||||
self.batch_size: int = batch_size
|
||||
self.n_iter: int = n_iter
|
||||
self.steps: int = steps
|
||||
self.hr_second_pass_steps = 0
|
||||
self.cfg_scale: float = cfg_scale
|
||||
self.scale_by: float = scale_by
|
||||
self.color_corrections = []
|
||||
self.is_control = False
|
||||
self.is_hr_pass = False
|
||||
self.is_refiner_pass = False
|
||||
self.is_api = False
|
||||
self.scheduled_prompt = False
|
||||
self.prompt_embeds = []
|
||||
self.positive_pooleds = []
|
||||
self.negative_embeds = []
|
||||
self.negative_pooleds = []
|
||||
self.disable_extra_networks = False
|
||||
self.iteration = 0
|
||||
|
||||
# initializers
|
||||
self.prompt = prompt
|
||||
self.seed = seed
|
||||
self.subseed = subseed
|
||||
self.subseed_strength = subseed_strength
|
||||
self.seed_resize_from_h = seed_resize_from_h
|
||||
self.seed_resize_from_w = seed_resize_from_w
|
||||
self.batch_size = batch_size
|
||||
self.n_iter = n_iter
|
||||
self.steps = steps
|
||||
self.clip_skip = clip_skip
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.negative_prompt = negative_prompt
|
||||
self.styles = styles
|
||||
self.tiling = tiling
|
||||
self.full_quality = full_quality
|
||||
self.hidiffusion = hidiffusion
|
||||
self.do_not_reload_embeddings = do_not_reload_embeddings
|
||||
self.detailer = detailer
|
||||
self.restore_faces = restore_faces
|
||||
self.init_images = init_images
|
||||
self.resize_mode = resize_mode
|
||||
self.resize_name = resize_name
|
||||
self.resize_context = resize_context
|
||||
self.denoising_strength = denoising_strength
|
||||
self.image_cfg_scale = image_cfg_scale
|
||||
self.scale_by = scale_by
|
||||
self.mask = mask
|
||||
self.image_mask = mask # TODO duplciate mask params
|
||||
self.latent_mask = latent_mask
|
||||
self.mask_blur = mask_blur
|
||||
self.inpainting_fill = inpainting_fill
|
||||
self.inpaint_full_res_padding = inpaint_full_res_padding
|
||||
self.inpainting_mask_invert = inpainting_mask_invert
|
||||
self.overlay_images = overlay_images
|
||||
self.enable_hr = enable_hr
|
||||
self.firstphase_width = firstphase_width
|
||||
self.firstphase_height = firstphase_height
|
||||
self.hr_scale = hr_scale
|
||||
self.hr_force = hr_force
|
||||
self.hr_resize_mode = hr_resize_mode
|
||||
self.hr_resize_context = hr_resize_context
|
||||
self.hr_upscaler = hr_upscaler
|
||||
self.hr_second_pass_steps = hr_second_pass_steps
|
||||
self.hr_resize_x = hr_resize_x
|
||||
self.hr_resize_y = hr_resize_y
|
||||
self.hr_upscale_to_x = hr_resize_x
|
||||
self.hr_upscale_to_y = hr_resize_y
|
||||
self.hr_denoising_strength = hr_denoising_strength
|
||||
self.refiner_steps = refiner_steps
|
||||
self.refiner_start = refiner_start
|
||||
self.refiner_prompt = refiner_prompt
|
||||
self.refiner_negative = refiner_negative
|
||||
self.hr_refiner_start = hr_refiner_start
|
||||
self.outpath_samples = outpath_samples
|
||||
self.outpath_grids = outpath_grids
|
||||
self.do_not_save_samples = do_not_save_samples
|
||||
self.do_not_save_grid = do_not_save_grid
|
||||
self.override_settings_restore_afterwards = override_settings_restore_afterwards
|
||||
self.extra_generation_params = extra_generation_params
|
||||
self.eta = eta
|
||||
self.cfg_scale = cfg_scale
|
||||
self.cfg_end = cfg_end
|
||||
self.diffusers_guidance_rescale = diffusers_guidance_rescale
|
||||
self.pag_scale = pag_scale
|
||||
self.pag_adaptive = pag_adaptive
|
||||
self.cfg_end = cfg_end
|
||||
self.width: int = width
|
||||
self.height: int = height
|
||||
self.full_quality: bool = full_quality
|
||||
self.detailer: bool = detailer
|
||||
self.restore_faces: bool = restore_faces
|
||||
self.tiling: bool = tiling
|
||||
self.hidiffusion: bool = hidiffusion
|
||||
self.do_not_save_samples: bool = do_not_save_samples
|
||||
self.do_not_save_grid: bool = do_not_save_grid
|
||||
self.extra_generation_params: dict = extra_generation_params or {}
|
||||
self.overlay_images = overlay_images
|
||||
self.eta = eta
|
||||
self.do_not_reload_embeddings = do_not_reload_embeddings
|
||||
self.paste_to = None
|
||||
self.color_corrections = None
|
||||
self.denoising_strength: float = denoising_strength
|
||||
self.selected_scale_tab = selected_scale_tab
|
||||
self.mask_for_overlay = mask_for_overlay
|
||||
self.paste_to = paste_to
|
||||
self.init_latent = None
|
||||
|
||||
# special handled items
|
||||
if firstphase_width != 0 or firstphase_height != 0:
|
||||
self.hr_upscale_to_x = self.width
|
||||
self.hr_upscale_to_y = self.height
|
||||
self.width = firstphase_width
|
||||
self.height = firstphase_height
|
||||
self.sampler_name = sampler_name or processing_helpers.get_sampler_name(sampler_index, img=True)
|
||||
self.hr_sampler_name: str = hr_sampler_name if hr_sampler_name != 'Same as primary' else self.sampler_name
|
||||
self.override_settings = {k: v for k, v in (override_settings or {}).items() if k not in shared.restricted_opts}
|
||||
self.override_settings_restore_afterwards = override_settings_restore_afterwards
|
||||
self.is_using_inpainting_conditioning = False # a111 compatibility
|
||||
self.disable_extra_networks = False
|
||||
# self.scripts = scripts.ScriptRunner() # set via property
|
||||
# self.script_args = script_args or [] # set via property
|
||||
self.per_script_args = {}
|
||||
self.inpaint_full_res = inpaint_full_res if isinstance(inpaint_full_res, bool) else self.inpaint_full_res
|
||||
self.inpaint_full_res = inpaint_full_res != 0 if isinstance(inpaint_full_res, int) else self.inpaint_full_res
|
||||
|
||||
# null items initialized later
|
||||
self.all_prompts = None
|
||||
self.all_negative_prompts = None
|
||||
self.all_seeds = None
|
||||
self.all_subseeds = None
|
||||
self.clip_skip = clip_skip
|
||||
|
||||
# a1111 compatibility items
|
||||
shared.opts.data['clip_skip'] = int(self.clip_skip) # for compatibility with a1111 sd_hijack_clip
|
||||
self.iteration = 0
|
||||
self.is_control = False
|
||||
self.is_hr_pass = False
|
||||
self.is_refiner_pass = False
|
||||
self.hr_force = False
|
||||
self.enable_hr = None
|
||||
self.hr_scale = None
|
||||
self.hr_upscaler = None
|
||||
self.hr_resize_mode = 0
|
||||
self.hr_resize_context = 'None'
|
||||
self.hr_resize_x = 0
|
||||
self.hr_resize_y = 0
|
||||
self.hr_upscale_to_x = 0
|
||||
self.hr_upscale_to_y = 0
|
||||
self.seed_enable_extras: bool = True
|
||||
self.is_using_inpainting_conditioning = False # a111 compatibility
|
||||
self.batch_index = 0
|
||||
self.refiner_switch_at = 0
|
||||
self.hr_prompt = ''
|
||||
self.all_hr_prompts = []
|
||||
self.hr_negative_prompt = ''
|
||||
self.all_hr_negative_prompts = []
|
||||
self.truncate_x = 0
|
||||
self.truncate_y = 0
|
||||
self.applied_old_hires_behavior_to = None
|
||||
self.refiner_steps = 5
|
||||
self.refiner_start = 0
|
||||
self.refiner_prompt = ''
|
||||
self.refiner_negative = ''
|
||||
self.ops = []
|
||||
self.resize_mode: int = resize_mode
|
||||
self.resize_name: str = resize_name
|
||||
self.resize_context: str = resize_context
|
||||
self.comments = {}
|
||||
self.sampler = None
|
||||
self.nmask = None
|
||||
self.initial_noise_multiplier = initial_noise_multiplier or shared.opts.initial_noise_multiplier
|
||||
self.image_conditioning = None
|
||||
self.prompt_for_display: str = None
|
||||
|
||||
# scripts
|
||||
self.scripts_value: scripts.ScriptRunner = field(default=None, init=False)
|
||||
self.script_args_value: list = field(default=None, init=False)
|
||||
self.scripts_setup_complete: bool = field(default=False, init=False)
|
||||
self.script_args = script_args
|
||||
self.per_script_args = {}
|
||||
|
||||
# settings to processing
|
||||
self.ddim_discretize = shared.opts.ddim_discretize
|
||||
self.s_min_uncond = shared.opts.s_min_uncond
|
||||
self.s_churn = shared.opts.s_churn
|
||||
@@ -171,18 +266,7 @@ class StableDiffusionProcessing:
|
||||
self.s_tmin = shared.opts.s_tmin
|
||||
self.s_tmax = float('inf') # not representable as a standard ui option
|
||||
self.task_args = {}
|
||||
# a1111 compatibility items
|
||||
self.batch_index = 0
|
||||
self.refiner_switch_at = 0
|
||||
self.hr_prompt = ''
|
||||
self.all_hr_prompts = []
|
||||
self.hr_negative_prompt = ''
|
||||
self.all_hr_negative_prompts = []
|
||||
self.comments = {}
|
||||
self.is_api = False
|
||||
self.scripts_value: scripts.ScriptRunner = field(default=None, init=False)
|
||||
self.script_args_value: list = field(default=None, init=False)
|
||||
self.scripts_setup_complete: bool = field(default=False, init=False)
|
||||
|
||||
# ip adapter
|
||||
self.ip_adapter_names = []
|
||||
self.ip_adapter_scales = [0.0]
|
||||
@@ -190,6 +274,7 @@ class StableDiffusionProcessing:
|
||||
self.ip_adapter_starts = [0.0]
|
||||
self.ip_adapter_ends = [1.0]
|
||||
self.ip_adapter_crops = []
|
||||
|
||||
# hdr
|
||||
self.hdr_mode=hdr_mode
|
||||
self.hdr_brightness=hdr_brightness
|
||||
@@ -203,7 +288,10 @@ class StableDiffusionProcessing:
|
||||
self.hdr_max_boundry=hdr_max_boundry
|
||||
self.hdr_color_picker=hdr_color_picker
|
||||
self.hdr_tint_ratio=hdr_tint_ratio
|
||||
|
||||
# globals
|
||||
self.embedder = None
|
||||
self.override = None
|
||||
self.scheduled_prompt: bool = False
|
||||
self.prompt_embeds = []
|
||||
self.positive_pooleds = []
|
||||
@@ -252,57 +340,9 @@ class StableDiffusionProcessing:
|
||||
|
||||
|
||||
class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
|
||||
|
||||
def __init__(self,
|
||||
enable_hr: bool = False,
|
||||
denoising_strength: float = 0.75,
|
||||
firstphase_width: int = 0,
|
||||
firstphase_height: int = 0,
|
||||
hr_scale: float = 2.0,
|
||||
hr_force: bool = False,
|
||||
hr_resize_mode: int = 0,
|
||||
hr_resize_context: str = 'None',
|
||||
hr_upscaler: str = None,
|
||||
hr_second_pass_steps: int = 0,
|
||||
hr_resize_x: int = 0,
|
||||
hr_resize_y: int = 0,
|
||||
refiner_steps: int = 5,
|
||||
refiner_start: float = 0,
|
||||
refiner_prompt: str = '',
|
||||
refiner_negative: str = '',
|
||||
**kwargs
|
||||
):
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access
|
||||
super().__init__(**kwargs)
|
||||
self.reprocess = {}
|
||||
self.enable_hr = enable_hr
|
||||
self.denoising_strength = denoising_strength
|
||||
self.hr_scale = hr_scale
|
||||
self.hr_upscaler = hr_upscaler
|
||||
self.hr_resize_mode = hr_resize_mode
|
||||
self.hr_resize_context = hr_resize_context
|
||||
self.hr_force = hr_force
|
||||
self.hr_second_pass_steps = hr_second_pass_steps
|
||||
self.hr_resize_x = hr_resize_x
|
||||
self.hr_resize_y = hr_resize_y
|
||||
self.hr_upscale_to_x = hr_resize_x
|
||||
self.hr_upscale_to_y = hr_resize_y
|
||||
if firstphase_width != 0 or firstphase_height != 0:
|
||||
self.hr_upscale_to_x = self.width
|
||||
self.hr_upscale_to_y = self.height
|
||||
self.width = firstphase_width
|
||||
self.height = firstphase_height
|
||||
self.truncate_x = 0
|
||||
self.truncate_y = 0
|
||||
self.applied_old_hires_behavior_to = None
|
||||
self.refiner_steps = refiner_steps
|
||||
self.refiner_start = refiner_start
|
||||
self.refiner_prompt = refiner_prompt
|
||||
self.refiner_negative = refiner_negative
|
||||
self.sampler = None
|
||||
self.scripts = None
|
||||
self.script_args = []
|
||||
|
||||
|
||||
def init(self, all_prompts=None, all_seeds=None, all_subseeds=None):
|
||||
if shared.native:
|
||||
@@ -360,41 +400,9 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
|
||||
|
||||
|
||||
class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
|
||||
|
||||
def __init__(self, init_images: list = None, resize_mode: int = 0, resize_name: str = 'None', resize_context: str = 'None', denoising_strength: float = 0.3, image_cfg_scale: float = None, mask: Any = None, mask_blur: int = 4, inpainting_fill: int = 0, inpaint_full_res: bool = False, inpaint_full_res_padding: int = 0, inpainting_mask_invert: int = 0, initial_noise_multiplier: float = None, scale_by: float = 1, refiner_steps: int = 5, refiner_start: float = 0, refiner_prompt: str = '', refiner_negative: str = '', **kwargs):
|
||||
def __init__(self, **kwargs):
|
||||
debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access
|
||||
super().__init__(**kwargs)
|
||||
self.init_images = init_images
|
||||
self.resize_mode: int = resize_mode
|
||||
self.resize_name: str = resize_name
|
||||
self.resize_context: str = resize_context
|
||||
self.denoising_strength: float = denoising_strength
|
||||
self.hr_denoising_strength: float = denoising_strength
|
||||
self.image_cfg_scale: float = image_cfg_scale
|
||||
self.init_latent = None
|
||||
self.image_mask = mask
|
||||
self.latent_mask = None
|
||||
self.mask_for_overlay = None
|
||||
self.mask_blur_x = mask_blur # a1111 compatibility item
|
||||
self.mask_blur_y = mask_blur # a1111 compatibility item
|
||||
self.mask_blur = mask_blur
|
||||
self.inpainting_fill = inpainting_fill
|
||||
self.inpaint_full_res = inpaint_full_res
|
||||
self.inpaint_full_res_padding = inpaint_full_res_padding
|
||||
self.inpainting_mask_invert = inpainting_mask_invert
|
||||
self.initial_noise_multiplier = shared.opts.initial_noise_multiplier if initial_noise_multiplier is None else initial_noise_multiplier
|
||||
self.mask = None
|
||||
self.nmask = None
|
||||
self.image_conditioning = None
|
||||
self.refiner_steps = refiner_steps
|
||||
self.refiner_start = refiner_start
|
||||
self.refiner_prompt = refiner_prompt
|
||||
self.refiner_negative = refiner_negative
|
||||
self.enable_hr = None
|
||||
self.is_batch = False
|
||||
self.scale_by = scale_by
|
||||
self.sampler = None
|
||||
self.scripts = None
|
||||
self.script_args = []
|
||||
|
||||
def init(self, all_prompts=None, all_seeds=None, all_subseeds=None):
|
||||
if hasattr(self, 'init_images') and self.init_images is not None and len(self.init_images) > 0:
|
||||
@@ -485,7 +493,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
|
||||
image = images.resize_image(self.resize_mode, image, self.width, self.height, upscaler_name=self.resize_name, context=self.resize_context)
|
||||
self.width = image.width
|
||||
self.height = image.height
|
||||
if self.image_mask is not None and shared.opts.mask_apply_overlay:
|
||||
if self.image_mask is not None and shared.opts.mask_apply_overlay and not hasattr(self, 'xyz'):
|
||||
image_masked = Image.new('RGBa', (image.width, image.height))
|
||||
image_to_paste = image.convert("RGBA").convert("RGBa")
|
||||
image_to_mask = ImageOps.invert(self.mask_for_overlay.convert('L')) if self.mask_for_overlay is not None else None
|
||||
@@ -544,47 +552,8 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
|
||||
|
||||
class StableDiffusionProcessingControl(StableDiffusionProcessingImg2Img):
|
||||
def __init__(self, **kwargs):
|
||||
debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access
|
||||
super().__init__(**kwargs)
|
||||
self.strength = None
|
||||
self.adapter_conditioning_scale = None
|
||||
self.adapter_conditioning_factor = None
|
||||
self.guess_mode = None
|
||||
self.controlnet_conditioning_scale = None
|
||||
self.control_guidance_start = None
|
||||
self.control_guidance_end = None
|
||||
self.control_mode = None
|
||||
self.reference_attn = None
|
||||
self.reference_adain = None
|
||||
self.attention_auto_machine_weight = None
|
||||
self.gn_auto_machine_weight = None
|
||||
self.style_fidelity = None
|
||||
self.ref_image = None
|
||||
self.image = None
|
||||
self.query_weight = None
|
||||
self.adain_weight = None
|
||||
self.adapter_conditioning_factor = 1.0
|
||||
self.attention = 'Attention'
|
||||
self.fidelity = 0.5
|
||||
self.mask_image = None
|
||||
self.override = None
|
||||
self.resize_mode_before = None
|
||||
self.resize_name_before = None
|
||||
self.width_before = None
|
||||
self.height_before = None
|
||||
self.scale_by_before = None
|
||||
self.selected_scale_tab_before = None
|
||||
self.resize_mode_after = None
|
||||
self.resize_name_after = None
|
||||
self.width_after = None
|
||||
self.height_after = None
|
||||
self.scale_by_after = None
|
||||
self.selected_scale_tab_after = None
|
||||
self.resize_mode_mask = None
|
||||
self.resize_name_mask = None
|
||||
self.width_mask = None
|
||||
self.height_mask = None
|
||||
self.scale_by_mask = None
|
||||
self.selected_scale_tab_mask = None
|
||||
|
||||
def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): # abstract
|
||||
pass
|
||||
|
||||
@@ -71,7 +71,7 @@ def process_base(p: processing.StableDiffusionProcessing):
|
||||
guidance_rescale=p.diffusers_guidance_rescale,
|
||||
denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None,
|
||||
denoising_end=p.refiner_start if use_refiner_start else 1 if use_denoise_start else None,
|
||||
output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np',
|
||||
output_type='latent',
|
||||
clip_skip=p.clip_skip,
|
||||
desc='Base',
|
||||
)
|
||||
@@ -101,7 +101,8 @@ def process_base(p: processing.StableDiffusionProcessing):
|
||||
output = SimpleNamespace(**output)
|
||||
if isinstance(output, list):
|
||||
output = SimpleNamespace(images=output)
|
||||
shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops)
|
||||
if hasattr(output, 'images'):
|
||||
shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops)
|
||||
timer.process.record('pipeline')
|
||||
hidiffusion.unapply()
|
||||
sd_models_compile.openvino_post_compile(op="base") # only executes on compiled vino models
|
||||
@@ -151,13 +152,14 @@ def process_hires(p: processing.StableDiffusionProcessing, output):
|
||||
p.is_hr_pass = True
|
||||
if hasattr(p, 'init_hr'):
|
||||
p.init_hr(p.hr_scale, p.hr_upscaler, force=p.hr_force)
|
||||
else: # fake hires for img2img
|
||||
p.hr_scale = p.scale_by
|
||||
p.hr_upscaler = p.resize_name
|
||||
p.hr_resize_mode = p.resize_mode
|
||||
p.hr_resize_context = p.resize_context
|
||||
p.hr_upscale_to_x = p.width
|
||||
p.hr_upscale_to_y = p.height
|
||||
else:
|
||||
if not p.is_hr_pass: # fake hires for img2img if not actual hr pass
|
||||
p.hr_scale = p.scale_by
|
||||
p.hr_upscaler = p.resize_name
|
||||
p.hr_resize_mode = p.resize_mode
|
||||
p.hr_resize_context = p.resize_context
|
||||
p.hr_upscale_to_x = p.width * p.hr_scale if p.hr_resize_x == 0 else p.hr_resize_x
|
||||
p.hr_upscale_to_y = p.height * p.hr_scale if p.hr_resize_y == 0 else p.hr_resize_y
|
||||
prev_job = shared.state.job
|
||||
|
||||
# hires runs on original pipeline
|
||||
@@ -175,7 +177,8 @@ def process_hires(p: processing.StableDiffusionProcessing, output):
|
||||
sd_hijack_hypertile.hypertile_set(p, hr=True)
|
||||
|
||||
latent_upscale = shared.latent_upscale_modes.get(p.hr_upscaler, None)
|
||||
if (latent_upscale is not None or p.hr_force) and getattr(p, 'hr_denoising_strength', p.denoising_strength) > 0:
|
||||
strength = p.hr_denoising_strength if p.hr_denoising_strength > 0 else p.denoising_strength
|
||||
if (latent_upscale is not None or p.hr_force) and strength > 0:
|
||||
p.ops.append('hires')
|
||||
sd_models_compile.openvino_recompile_model(p, hires=True, refiner=False)
|
||||
if shared.sd_model.__class__.__name__ == "OnnxRawPipeline":
|
||||
@@ -183,8 +186,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output):
|
||||
p.hr_force = True
|
||||
|
||||
# hires
|
||||
p.denoising_strength = getattr(p, 'hr_denoising_strength', p.denoising_strength)
|
||||
if p.hr_force and p.denoising_strength == 0:
|
||||
if p.hr_force and strength == 0:
|
||||
shared.log.warning('HiRes skip: denoising=0')
|
||||
p.hr_force = False
|
||||
if p.hr_force:
|
||||
@@ -202,9 +204,9 @@ def process_hires(p: processing.StableDiffusionProcessing, output):
|
||||
sd_models.move_model(shared.sd_model.unet, devices.device)
|
||||
if hasattr(shared.sd_model, 'transformer'):
|
||||
sd_models.move_model(shared.sd_model.transformer, devices.device)
|
||||
orig_denoise = p.denoising_strength
|
||||
p.denoising_strength = getattr(p, 'hr_denoising_strength', p.denoising_strength)
|
||||
update_sampler(p, shared.sd_model, second_pass=True)
|
||||
orig_denoise = p.denoising_strength
|
||||
p.denoising_strength = strength
|
||||
hires_args = set_pipeline_args(
|
||||
p=p,
|
||||
model=shared.sd_model,
|
||||
@@ -216,10 +218,10 @@ def process_hires(p: processing.StableDiffusionProcessing, output):
|
||||
eta=shared.opts.scheduler_eta,
|
||||
guidance_scale=p.image_cfg_scale if p.image_cfg_scale is not None else p.cfg_scale,
|
||||
guidance_rescale=p.diffusers_guidance_rescale,
|
||||
output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np',
|
||||
output_type='latent',
|
||||
clip_skip=p.clip_skip,
|
||||
image=output.images,
|
||||
strength=p.denoising_strength,
|
||||
strength=strength,
|
||||
desc='Hires',
|
||||
)
|
||||
shared.state.job = 'HiRes'
|
||||
@@ -277,7 +279,7 @@ def process_refine(p: processing.StableDiffusionProcessing, output):
|
||||
for i in range(len(output.images)):
|
||||
image = output.images[i]
|
||||
noise_level = round(350 * p.denoising_strength)
|
||||
output_type='latent' if hasattr(shared.sd_refiner, 'vae') else 'np'
|
||||
output_type='latent'
|
||||
if 'Upscale' in shared.sd_refiner.__class__.__name__ or 'Flux' in shared.sd_refiner.__class__.__name__:
|
||||
image = processing_vae.vae_decode(latents=image, model=shared.sd_model, full_quality=p.full_quality, output_type='pil', width=p.width, height=p.height)
|
||||
p.extra_generation_params['Noise level'] = noise_level
|
||||
@@ -345,7 +347,11 @@ def process_decode(p: processing.StableDiffusionProcessing, output):
|
||||
if not hasattr(output, 'images') and hasattr(output, 'frames'):
|
||||
shared.log.debug(f'Generated: frames={len(output.frames[0])}')
|
||||
output.images = output.frames[0]
|
||||
if hasattr(shared.sd_model, "vae") and output.images is not None and len(output.images) > 0:
|
||||
model = shared.sd_model if not is_refiner_enabled(p) else shared.sd_refiner
|
||||
if not hasattr(model, 'vae'):
|
||||
if hasattr(model, 'pipe') and hasattr(model.pipe, 'vae'):
|
||||
model = model.pipe
|
||||
if hasattr(model, "vae") and output.images is not None and len(output.images) > 0:
|
||||
if p.hr_resize_mode > 0 and (p.hr_upscaler != 'None' or p.hr_resize_mode == 5):
|
||||
width = max(getattr(p, 'width', 0), getattr(p, 'hr_upscale_to_x', 0))
|
||||
height = max(getattr(p, 'height', 0), getattr(p, 'hr_upscale_to_y', 0))
|
||||
@@ -354,7 +360,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output):
|
||||
height = getattr(p, 'height', 0)
|
||||
results = processing_vae.vae_decode(
|
||||
latents = output.images,
|
||||
model = shared.sd_model if not is_refiner_enabled(p) else shared.sd_refiner,
|
||||
model = model,
|
||||
full_quality = p.full_quality,
|
||||
width = width,
|
||||
height = height,
|
||||
|
||||
@@ -47,16 +47,23 @@ def apply_overlay(image: Image, paste_loc, index, overlays):
|
||||
return image
|
||||
debug(f'Apply overlay: image={image} loc={paste_loc} index={index} overlays={overlays}')
|
||||
overlay = overlays[index]
|
||||
if paste_loc is not None:
|
||||
x, y, w, h = paste_loc
|
||||
if image.width != w or image.height != h or x != 0 or y != 0:
|
||||
base_image = Image.new('RGBA', (overlay.width, overlay.height))
|
||||
image = images.resize_image(2, image, w, h)
|
||||
base_image.paste(image, (x, y))
|
||||
image = base_image
|
||||
image = image.convert('RGBA')
|
||||
image.alpha_composite(overlay)
|
||||
image = image.convert('RGB')
|
||||
if not isinstance(image, Image.Image) or not isinstance(overlay, Image.Image):
|
||||
return image
|
||||
try:
|
||||
if paste_loc is not None and (isinstance(paste_loc, tuple) or isinstance(paste_loc, list)):
|
||||
x, y, w, h = paste_loc
|
||||
if x is None or y is None or w is None or h is None:
|
||||
return image
|
||||
if image.width != w or image.height != h or x != 0 or y != 0:
|
||||
base_image = Image.new('RGBA', (overlay.width, overlay.height))
|
||||
image = images.resize_image(2, image, w, h)
|
||||
base_image.paste(image, (x, y))
|
||||
image = base_image
|
||||
image = image.convert('RGBA')
|
||||
image.alpha_composite(overlay)
|
||||
image = image.convert('RGB')
|
||||
except Exception as e:
|
||||
shared.log.error(f'Apply overlay: {e}')
|
||||
return image
|
||||
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ from modules import shared, sd_samplers_common, sd_vae, generation_parameters_co
|
||||
from modules.processing_class import StableDiffusionProcessing
|
||||
|
||||
|
||||
debug = shared.log.trace if os.environ.get('SD_PROCESS_DEBUG', None) is not None else lambda *args, **kwargs: None
|
||||
if not shared.native:
|
||||
from modules import sd_hijack
|
||||
else:
|
||||
@@ -39,30 +40,34 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
|
||||
ops.reverse()
|
||||
args = {
|
||||
# basic
|
||||
"Size": f"{p.width}x{p.height}" if hasattr(p, 'width') and hasattr(p, 'height') else None,
|
||||
"Sampler": p.sampler_name if p.sampler_name != 'Default' else None,
|
||||
"Steps": p.steps,
|
||||
"Seed": all_seeds[index],
|
||||
"Sampler": p.sampler_name if p.sampler_name != 'Default' else None,
|
||||
"Seed resize from": None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}",
|
||||
"CFG scale": p.cfg_scale if p.cfg_scale > 1.0 else None,
|
||||
"CFG end": p.cfg_end if p.cfg_end < 1.0 else None,
|
||||
"Size": f"{p.width}x{p.height}" if hasattr(p, 'width') and hasattr(p, 'height') else None,
|
||||
"Clip skip": p.clip_skip if p.clip_skip > 1 else None,
|
||||
"Batch": f'{p.n_iter}x{p.batch_size}' if p.n_iter > 1 or p.batch_size > 1 else None,
|
||||
"Parser": shared.opts.prompt_attention.split()[0],
|
||||
"Model": None if (not shared.opts.add_model_name_to_info) or (not shared.sd_model.sd_checkpoint_info.model_name) else shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', ''),
|
||||
"Model hash": getattr(p, 'sd_model_hash', None if (not shared.opts.add_model_hash_to_info) or (not shared.sd_model.sd_model_hash) else shared.sd_model.sd_model_hash),
|
||||
"VAE": (None if not shared.opts.add_model_name_to_info or sd_vae.loaded_vae_file is None else os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0]) if p.full_quality else 'TAESD',
|
||||
"Seed resize from": None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}",
|
||||
"Clip skip": p.clip_skip if p.clip_skip > 1 else None,
|
||||
"Prompt2": p.refiner_prompt if len(p.refiner_prompt) > 0 else None,
|
||||
"Negative2": p.refiner_negative if len(p.refiner_negative) > 0 else None,
|
||||
"Styles": "; ".join(p.styles) if p.styles is not None and len(p.styles) > 0 else None,
|
||||
"Tiling": p.tiling if p.tiling else None,
|
||||
# sdnext
|
||||
"Backend": 'Diffusers' if shared.native else 'Original',
|
||||
"App": 'SD.Next',
|
||||
"Version": git_commit,
|
||||
"Backend": 'Diffusers' if shared.native else 'Original',
|
||||
"Pipeline": 'LDM',
|
||||
"Parser": shared.opts.prompt_attention.split()[0],
|
||||
"Comment": comment,
|
||||
"Operations": '; '.join(ops).replace('"', '') if len(p.ops) > 0 else 'none',
|
||||
}
|
||||
if shared.opts.add_model_name_to_info and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None:
|
||||
args["Model"] = shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', '')
|
||||
if shared.opts.add_model_hash_to_info and getattr(shared.sd_model, 'sd_model_hash', None) is not None:
|
||||
args["Model hash"] = shared.sd_model.sd_model_hash
|
||||
# native
|
||||
if grid is None and (p.n_iter > 1 or p.batch_size > 1) and index >= 0:
|
||||
args['Index'] = f'{p.iteration + 1}x{index + 1}'
|
||||
@@ -165,7 +170,9 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
|
||||
if isinstance(v, str):
|
||||
if len(v) == 0 or v == '0x0':
|
||||
del args[k]
|
||||
debug(f'Infotext: args={args}')
|
||||
params_text = ", ".join([k if k == v else f'{k}: {generation_parameters_copypaste.quote(v)}' for k, v in args.items()])
|
||||
negative_prompt_text = f"\nNegative prompt: {all_negative_prompts[index]}" if all_negative_prompts[index] else ""
|
||||
infotext = f"{all_prompts[index]}{negative_prompt_text}\n{params_text}".strip()
|
||||
debug(f'Infotext: "{infotext}"')
|
||||
return infotext
|
||||
|
||||
@@ -35,7 +35,9 @@ def create_latents(image, p, dtype=None, device=None):
|
||||
|
||||
def full_vae_decode(latents, model):
|
||||
t0 = time.time()
|
||||
if not hasattr(model, 'vae'):
|
||||
if not hasattr(model, 'vae') and hasattr(model, 'pipe'):
|
||||
model = model.pipe
|
||||
if model is None or not hasattr(model, 'vae'):
|
||||
shared.log.error('VAE not found in model')
|
||||
return []
|
||||
if debug:
|
||||
@@ -147,6 +149,9 @@ def taesd_vae_encode(image):
|
||||
|
||||
def vae_decode(latents, model, output_type='np', full_quality=True, width=None, height=None):
|
||||
t0 = time.time()
|
||||
model = model or shared.sd_model
|
||||
if not hasattr(model, 'vae') and hasattr(model, 'pipe'):
|
||||
model = model.pipe
|
||||
if latents is None or not torch.is_tensor(latents): # already decoded
|
||||
return latents
|
||||
prev_job = shared.state.job
|
||||
@@ -169,8 +174,8 @@ def vae_decode(latents, model, output_type='np', full_quality=True, width=None,
|
||||
|
||||
if latents.shape[-1] <= 4: # not a latent, likely an image
|
||||
decoded = latents.float().cpu().numpy()
|
||||
elif full_quality and hasattr(shared.sd_model, "vae"):
|
||||
decoded = full_vae_decode(latents=latents, model=shared.sd_model)
|
||||
elif full_quality and hasattr(model, "vae"):
|
||||
decoded = full_vae_decode(latents=latents, model=model)
|
||||
else:
|
||||
decoded = taesd_vae_decode(latents=latents)
|
||||
|
||||
@@ -195,6 +200,8 @@ def vae_decode(latents, model, output_type='np', full_quality=True, width=None,
|
||||
def vae_encode(image, model, full_quality=True): # pylint: disable=unused-variable
|
||||
if shared.state.interrupted or shared.state.skipped:
|
||||
return []
|
||||
if not hasattr(model, 'vae') and hasattr(model, 'pipe'):
|
||||
model = model.pipe
|
||||
if not hasattr(model, 'vae'):
|
||||
shared.log.error('VAE not found in model')
|
||||
return []
|
||||
|
||||
@@ -73,7 +73,6 @@ def progressapi(req: ProgressRequest):
|
||||
elapsed = time.time() - shared.state.time_start if shared.state.time_start is not None else 0
|
||||
predicted = elapsed / progress if progress > 0 else None
|
||||
eta = predicted - elapsed if predicted is not None else None
|
||||
# shared.log.debug(f'Progress: step={step_x}:{step_y} batch={batch_x}:{batch_y} current={current} total={total} progress={progress} elapsed={elapsed} eta={eta}')
|
||||
id_live_preview = req.id_live_preview
|
||||
live_preview = None
|
||||
shared.state.set_current_image()
|
||||
|
||||
@@ -308,11 +308,11 @@ def parse_prompt_attention(text):
|
||||
res = []
|
||||
round_brackets = []
|
||||
square_brackets = []
|
||||
if opts.prompt_attention == 'Fixed attention':
|
||||
if opts.prompt_attention == 'fixed':
|
||||
res = [[text, 1.0]]
|
||||
debug(f'Prompt: parser="{opts.prompt_attention}" {res}')
|
||||
return res
|
||||
elif opts.prompt_attention == 'Compel parser':
|
||||
elif opts.prompt_attention == 'compel':
|
||||
conjunction = Compel.parse_prompt_string(text)
|
||||
if conjunction is None or conjunction.prompts is None or conjunction.prompts is None or len(conjunction.prompts[0].children) == 0:
|
||||
return [["", 1.0]]
|
||||
@@ -321,7 +321,7 @@ def parse_prompt_attention(text):
|
||||
res.append([frag.text, frag.weight])
|
||||
debug(f'Prompt: parser="{opts.prompt_attention}" {res}')
|
||||
return res
|
||||
elif opts.prompt_attention == 'A1111 parser':
|
||||
elif opts.prompt_attention == 'a1111':
|
||||
re_attention = re_attention_v1
|
||||
whitespace = ''
|
||||
else:
|
||||
@@ -360,7 +360,7 @@ def parse_prompt_attention(text):
|
||||
for i, part in enumerate(parts):
|
||||
if i > 0:
|
||||
res.append(["BREAK", -1])
|
||||
if opts.prompt_attention == 'Full parser':
|
||||
if opts.prompt_attention == 'native':
|
||||
part = re_clean.sub("", part)
|
||||
part = re_whitespace.sub(" ", part).strip()
|
||||
if len(part) == 0:
|
||||
@@ -392,15 +392,15 @@ if __name__ == "__main__":
|
||||
log.info(f'Schedules: {all_schedules}')
|
||||
for schedule in all_schedules:
|
||||
log.info(f'Schedule: {schedule[0]}')
|
||||
opts.data['prompt_attention'] = 'Fixed attention'
|
||||
opts.data['prompt_attention'] = 'fixed'
|
||||
output_list = parse_prompt_attention(schedule[1])
|
||||
log.info(f' Fixed: {output_list}')
|
||||
opts.data['prompt_attention'] = 'Compel parser'
|
||||
opts.data['prompt_attention'] = 'compel'
|
||||
output_list = parse_prompt_attention(schedule[1])
|
||||
log.info(f' Compel: {output_list}')
|
||||
opts.data['prompt_attention'] = 'A1111 parser'
|
||||
opts.data['prompt_attention'] = 'a1111'
|
||||
output_list = parse_prompt_attention(schedule[1])
|
||||
log.info(f' A1111: {output_list}')
|
||||
opts.data['prompt_attention'] = 'Full parser'
|
||||
opts.data['prompt_attention'] = 'native'
|
||||
log.info = parse_prompt_attention(schedule[1])
|
||||
log.info(f' Full: {output_list}')
|
||||
|
||||
+211
-145
@@ -2,6 +2,7 @@ import os
|
||||
import math
|
||||
import time
|
||||
import typing
|
||||
from collections import OrderedDict
|
||||
import torch
|
||||
from compel.embeddings_provider import BaseTextualInversionManager, EmbeddingsProvider
|
||||
from transformers import PreTrainedTokenizer
|
||||
@@ -14,7 +15,182 @@ debug('Trace: PROMPT')
|
||||
orig_encode_token_ids_to_embeddings = EmbeddingsProvider._encode_token_ids_to_embeddings # pylint: disable=protected-access
|
||||
token_dict = None # used by helper get_tokens
|
||||
token_type = None # used by helper get_tokens
|
||||
cache = {}
|
||||
cache = OrderedDict()
|
||||
embedder = None
|
||||
|
||||
|
||||
def prompt_compatible(pipe = None):
|
||||
pipe = pipe or shared.sd_model
|
||||
if (
|
||||
'StableDiffusion' not in pipe.__class__.__name__ and
|
||||
'DemoFusion' not in pipe.__class__.__name__ and
|
||||
'StableCascade' not in pipe.__class__.__name__ and
|
||||
'Flux' not in pipe.__class__.__name__
|
||||
):
|
||||
shared.log.warning(f"Prompt parser not supported: {pipe.__class__.__name__}")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def prepare_model(pipe = None):
|
||||
pipe = pipe or shared.sd_model
|
||||
if not hasattr(pipe, "text_encoder") and hasattr(shared.sd_model, "pipe"):
|
||||
pipe = pipe.pipe
|
||||
if not hasattr(pipe, "text_encoder"):
|
||||
return None
|
||||
if shared.opts.diffusers_offload_mode == "balanced":
|
||||
pipe = sd_models.apply_balanced_offload(pipe)
|
||||
elif hasattr(pipe, "maybe_free_model_hooks"):
|
||||
pipe.maybe_free_model_hooks()
|
||||
devices.torch_gc()
|
||||
return pipe
|
||||
|
||||
|
||||
class PromptEmbedder:
|
||||
def __init__(self, prompts, negative_prompts, steps, clip_skip, p):
|
||||
t0 = time.time()
|
||||
self.prompts = prompts
|
||||
self.negative_prompts = negative_prompts
|
||||
self.batchsize = len(self.prompts)
|
||||
self.attention = None
|
||||
self.allsame = self.compare_prompts() # collapses batched prompts to single prompt if possible
|
||||
self.steps = steps
|
||||
self.clip_skip = clip_skip
|
||||
# All embeds are nested lists, outer list batch length, inner schedule length
|
||||
self.prompt_embeds = [[]] * self.batchsize
|
||||
self.positive_pooleds = [[]] * self.batchsize
|
||||
self.negative_prompt_embeds = [[]] * self.batchsize
|
||||
self.negative_pooleds = [[]] * self.batchsize
|
||||
self.positive_schedule = None
|
||||
self.negative_schedule = None
|
||||
self.scheduled_prompt = False
|
||||
earlyout = self.checkcache(p)
|
||||
if earlyout:
|
||||
return
|
||||
pipe = prepare_model(p.sd_model)
|
||||
if pipe is None:
|
||||
shared.log.error("Prompt encode: cannot find text encoder in model")
|
||||
return
|
||||
# per prompt in batch
|
||||
for batchidx, (prompt, negative_prompt) in enumerate(zip(self.prompts, self.negative_prompts)):
|
||||
self.prepare_schedule(prompt, negative_prompt)
|
||||
if self.scheduled_prompt:
|
||||
self.scheduled_encode(pipe, batchidx)
|
||||
else:
|
||||
self.encode(pipe, prompt, negative_prompt, batchidx)
|
||||
self.checkcache(p)
|
||||
debug(f"Prompt encode: time={(time.time() - t0):.3f}")
|
||||
|
||||
def checkcache(self, p):
|
||||
if shared.opts.sd_textencoder_cache_size == 0:
|
||||
return False
|
||||
if self.attention != shared.opts.prompt_attention:
|
||||
debug(f"Prompt change: parser={shared.opts.prompt_attention}")
|
||||
cache.clear()
|
||||
return False
|
||||
|
||||
def flatten(xss):
|
||||
return [x for xs in xss for x in xs]
|
||||
|
||||
# unpack EN data in case of TE LoRA
|
||||
en_data = p.extra_network_data
|
||||
en_data = [idx.items for item in en_data.values() for idx in item]
|
||||
effective_batch = 1 if self.allsame else self.batchsize
|
||||
key = str([self.prompts, self.negative_prompts, effective_batch, self.clip_skip, self.steps, en_data])
|
||||
item = cache.get(key)
|
||||
if not item:
|
||||
if not any(flatten(emb) for emb in [self.prompt_embeds,
|
||||
self.negative_prompt_embeds,
|
||||
self.positive_pooleds,
|
||||
self.negative_pooleds]):
|
||||
return False
|
||||
else:
|
||||
cache[key] = {'prompt_embeds': self.prompt_embeds,
|
||||
'negative_prompt_embeds': self.negative_prompt_embeds,
|
||||
'positive_pooleds': self.positive_pooleds,
|
||||
'negative_pooleds': self.negative_pooleds,
|
||||
}
|
||||
debug(f"Prompt cache: add={key}")
|
||||
while len(cache) > int(shared.opts.sd_textencoder_cache_size):
|
||||
cache.popitem(last=False)
|
||||
if item:
|
||||
self.__dict__.update(cache[key])
|
||||
cache.move_to_end(key)
|
||||
if self.allsame and len(self.prompt_embeds) < self.batchsize:
|
||||
self.prompt_embeds = [self.prompt_embeds[0]] * self.batchsize
|
||||
self.positive_pooleds = [self.positive_pooleds[0]] * self.batchsize
|
||||
self.negative_prompt_embeds = [self.negative_prompt_embeds[0]] * self.batchsize
|
||||
self.negative_pooleds = [self.negative_pooleds[0]] * self.batchsize
|
||||
debug(f"Prompt cache: get={key}")
|
||||
return True
|
||||
|
||||
def compare_prompts(self):
|
||||
same = (self.prompts == [self.prompts[0]] * len(self.prompts) and self.negative_prompts == [self.negative_prompts[0]] * len(self.negative_prompts))
|
||||
if same:
|
||||
self.prompts = [self.prompts[0]]
|
||||
self.negative_prompts = [self.negative_prompts[0]]
|
||||
return same
|
||||
|
||||
def prepare_schedule(self, prompt, negative_prompt):
|
||||
self.positive_schedule, scheduled = get_prompt_schedule(prompt, self.steps)
|
||||
self.negative_schedule, neg_scheduled = get_prompt_schedule(negative_prompt, self.steps)
|
||||
self.scheduled_prompt = scheduled or neg_scheduled
|
||||
debug(f"Prompt schedule: positive={self.positive_schedule} negative={self.negative_schedule} scheduled={scheduled}")
|
||||
|
||||
def scheduled_encode(self, pipe, batchidx):
|
||||
prompt_dict = {} # index cache
|
||||
for i in range(max(len(self.positive_schedule), len(self.negative_schedule))):
|
||||
positive_prompt = self.positive_schedule[i % len(self.positive_schedule)]
|
||||
negative_prompt = self.negative_schedule[i % len(self.negative_schedule)]
|
||||
# skip repeated scheduled subprompts
|
||||
idx = prompt_dict.get(positive_prompt+negative_prompt)
|
||||
if idx is not None:
|
||||
self.extend_embeds(batchidx, idx)
|
||||
continue
|
||||
self.encode(pipe, positive_prompt, negative_prompt, batchidx)
|
||||
prompt_dict[positive_prompt+negative_prompt] = i
|
||||
|
||||
def extend_embeds(self, batchidx, idx): # Extends scheduled prompt via index
|
||||
if len(self.prompt_embeds[batchidx]) > 0:
|
||||
self.prompt_embeds[batchidx].append(self.prompt_embeds[batchidx][idx])
|
||||
if len(self.negative_prompt_embeds[batchidx]) > 0:
|
||||
self.negative_prompt_embeds[batchidx].append(self.negative_prompt_embeds[batchidx][idx])
|
||||
if len(self.positive_pooleds[batchidx]) > 0:
|
||||
self.positive_pooleds[batchidx].append(self.positive_pooleds[batchidx][idx])
|
||||
if len(self.negative_pooleds[batchidx]) > 0:
|
||||
self.negative_pooleds[batchidx].append(self.negative_pooleds[batchidx][idx])
|
||||
|
||||
def encode(self, pipe, positive_prompt, negative_prompt, batchidx):
|
||||
self.attention = shared.opts.prompt_attention
|
||||
if self.attention == "xhinker" or 'Flux' in pipe.__class__.__name__:
|
||||
prompt_embed, positive_pooled, negative_embed, negative_pooled = get_xhinker_text_embeddings(pipe, positive_prompt, negative_prompt, self.clip_skip)
|
||||
else:
|
||||
prompt_embed, positive_pooled, negative_embed, negative_pooled = get_weighted_text_embeddings(pipe, positive_prompt, negative_prompt, self.clip_skip)
|
||||
if prompt_embed is not None:
|
||||
self.prompt_embeds[batchidx].append(prompt_embed)
|
||||
if negative_embed is not None:
|
||||
self.negative_prompt_embeds[batchidx].append(negative_embed)
|
||||
if positive_pooled is not None:
|
||||
self.positive_pooleds[batchidx].append(positive_pooled)
|
||||
if negative_pooled is not None:
|
||||
self.negative_pooleds[batchidx].append(negative_pooled)
|
||||
|
||||
if debug_enabled:
|
||||
get_tokens(pipe, 'positive', positive_prompt)
|
||||
get_tokens(pipe, 'negative', negative_prompt)
|
||||
pipe = prepare_model()
|
||||
|
||||
def __call__(self, key, step=0):
|
||||
batch = getattr(self, key)
|
||||
res = []
|
||||
for i in range(self.batchsize):
|
||||
if len(batch[i]) == 0: # if asking for a null key, ie pooled on SD1.5
|
||||
return None
|
||||
try:
|
||||
res.append(batch[i][step])
|
||||
except IndexError:
|
||||
res.append(batch[i][0]) # if not scheduled, return default
|
||||
return torch.cat(res)
|
||||
|
||||
|
||||
def compel_hijack(self, token_ids: torch.Tensor, attention_mask: typing.Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
@@ -59,9 +235,9 @@ def insert_parser_highjack(pipename):
|
||||
debug("Load Standard Parser hijack")
|
||||
|
||||
|
||||
|
||||
insert_parser_highjack("Initialize")
|
||||
|
||||
|
||||
# from https://github.com/damian0815/compel/blob/main/src/compel/diffusers_textual_inversion_manager.py
|
||||
class DiffusersTextualInversionManager(BaseTextualInversionManager):
|
||||
def __init__(self, pipe, tokenizer):
|
||||
@@ -108,12 +284,6 @@ class DiffusersTextualInversionManager(BaseTextualInversionManager):
|
||||
|
||||
def get_prompt_schedule(prompt, steps):
|
||||
t0 = time.time()
|
||||
if shared.native:
|
||||
# TODO prompt scheduling
|
||||
# prompt schedule returns array of prompts which would require that each prompt is fed to the model per-step
|
||||
# prompt scheduling should instead interpolate between each prompt in schedule
|
||||
# this temporarily disables prompt scheduling
|
||||
return [prompt], False
|
||||
temp = []
|
||||
schedule = prompt_parser.get_learned_conditioning_prompt_schedules([prompt], steps)[0]
|
||||
if all(x == schedule[0] for x in schedule):
|
||||
@@ -126,25 +296,25 @@ def get_prompt_schedule(prompt, steps):
|
||||
return temp, len(schedule) > 1
|
||||
|
||||
|
||||
def get_tokens(msg, prompt):
|
||||
def get_tokens(pipe, msg, prompt):
|
||||
global token_dict, token_type # pylint: disable=global-statement
|
||||
if not shared.native:
|
||||
return
|
||||
if shared.sd_loaded and hasattr(shared.sd_model, 'tokenizer') and shared.sd_model.tokenizer is not None:
|
||||
return 0
|
||||
if shared.sd_loaded and hasattr(pipe, 'tokenizer') and pipe.tokenizer is not None:
|
||||
if token_dict is None or token_type != shared.sd_model_type:
|
||||
token_type = shared.sd_model_type
|
||||
fn = shared.sd_model.tokenizer.name_or_path
|
||||
fn = pipe.tokenizer.name_or_path
|
||||
if fn.endswith('tokenizer'):
|
||||
fn = os.path.join(shared.sd_model.tokenizer.name_or_path, 'vocab.json')
|
||||
fn = os.path.join(pipe.tokenizer.name_or_path, 'vocab.json')
|
||||
else:
|
||||
fn = os.path.join(shared.sd_model.tokenizer.name_or_path, 'tokenizer', 'vocab.json')
|
||||
fn = os.path.join(pipe.tokenizer.name_or_path, 'tokenizer', 'vocab.json')
|
||||
token_dict = shared.readfile(fn, silent=True)
|
||||
for k, v in shared.sd_model.tokenizer.added_tokens_decoder.items():
|
||||
for k, v in pipe.tokenizer.added_tokens_decoder.items():
|
||||
token_dict[str(v)] = k
|
||||
shared.log.debug(f'Tokenizer: words={len(token_dict)} file="{fn}"')
|
||||
has_bos_token = shared.sd_model.tokenizer.bos_token_id is not None
|
||||
has_eos_token = shared.sd_model.tokenizer.eos_token_id is not None
|
||||
ids = shared.sd_model.tokenizer(prompt)
|
||||
has_bos_token = pipe.tokenizer.bos_token_id is not None
|
||||
has_eos_token = pipe.tokenizer.eos_token_id is not None
|
||||
ids = pipe.tokenizer(prompt)
|
||||
ids = getattr(ids, 'input_ids', [])
|
||||
tokens = []
|
||||
for i in ids:
|
||||
@@ -155,118 +325,7 @@ def get_tokens(msg, prompt):
|
||||
tokens.append(f'UNK_{i}')
|
||||
token_count = len(ids) - int(has_bos_token) - int(has_eos_token)
|
||||
debug(f'Prompt tokenizer: type={msg} tokens={token_count} {tokens}')
|
||||
|
||||
|
||||
def encode_prompts(pipe, p, prompts: list, negative_prompts: list, steps: int, clip_skip: typing.Optional[int] = None):
|
||||
params_match = prompts == cache.get('prompts', None) and negative_prompts == cache.get('negative_prompts', None) and clip_skip == cache.get('clip_skip', None) and steps == cache.get('steps', None)
|
||||
if (
|
||||
'StableDiffusion' not in pipe.__class__.__name__ and
|
||||
'DemoFusion' not in pipe.__class__.__name__ and
|
||||
'StableCascade' not in pipe.__class__.__name__ and
|
||||
'Flux' not in pipe.__class__.__name__
|
||||
):
|
||||
shared.log.warning(f"Prompt parser not supported: {pipe.__class__.__name__}")
|
||||
return
|
||||
elif shared.opts.sd_textencoder_cache and cache.get('model_type', None) == shared.sd_model_type and params_match:
|
||||
p.prompt_embeds = cache.get('prompt_embeds', None)
|
||||
p.positive_pooleds = cache.get('positive_pooleds', None)
|
||||
p.negative_embeds = cache.get('negative_embeds', None)
|
||||
p.negative_pooleds = cache.get('negative_pooleds', None)
|
||||
p.scheduled_prompt = cache.get('scheduled_prompt', None)
|
||||
debug("Prompt encode: cached")
|
||||
return
|
||||
else:
|
||||
t0 = time.time()
|
||||
if shared.opts.diffusers_offload_mode == "balanced":
|
||||
pipe = sd_models.apply_balanced_offload(pipe)
|
||||
elif hasattr(pipe, "maybe_free_model_hooks"):
|
||||
pipe.maybe_free_model_hooks()
|
||||
devices.torch_gc()
|
||||
|
||||
prompt_embeds, positive_pooleds, negative_embeds, negative_pooleds = [], [], [], []
|
||||
last_prompt, last_negative = None, None
|
||||
for prompt, negative in zip(prompts, negative_prompts):
|
||||
prompt_embed, positive_pooled, negative_embed, negative_pooled = None, None, None, None
|
||||
if last_prompt == prompt and last_negative == negative:
|
||||
prompt_embeds.append(prompt_embeds[-1])
|
||||
negative_embeds.append(negative_embeds[-1])
|
||||
if len(positive_pooleds) > 0:
|
||||
positive_pooleds.append(positive_pooleds[-1])
|
||||
if len(negative_pooleds) > 0:
|
||||
negative_pooleds.append(negative_pooleds[-1])
|
||||
continue
|
||||
positive_schedule, scheduled = get_prompt_schedule(prompt, steps)
|
||||
negative_schedule, neg_scheduled = get_prompt_schedule(negative, steps)
|
||||
p.scheduled_prompt = scheduled or neg_scheduled
|
||||
p.prompt_embeds = []
|
||||
p.positive_pooleds = []
|
||||
p.negative_embeds = []
|
||||
p.negative_pooleds = []
|
||||
|
||||
for i in range(max(len(positive_schedule), len(negative_schedule))):
|
||||
positive_prompt = positive_schedule[i % len(positive_schedule)]
|
||||
negative_prompt = negative_schedule[i % len(negative_schedule)]
|
||||
if shared.opts.prompt_attention == "xhinker parser" or 'Flux' in pipe.__class__.__name__:
|
||||
prompt_embed, positive_pooled, negative_embed, negative_pooled = get_xhinker_text_embeddings(pipe, positive_prompt, negative_prompt, clip_skip)
|
||||
else:
|
||||
prompt_embed, positive_pooled, negative_embed, negative_pooled = get_weighted_text_embeddings(pipe, positive_prompt, negative_prompt, clip_skip)
|
||||
if prompt_embed is not None:
|
||||
prompt_embeds.append(prompt_embed)
|
||||
if negative_embed is not None:
|
||||
negative_embeds.append(negative_embed)
|
||||
if positive_pooled is not None:
|
||||
positive_pooleds.append(positive_pooled)
|
||||
if negative_pooled is not None:
|
||||
negative_pooleds.append(negative_pooled)
|
||||
last_prompt, last_negative = prompt, negative
|
||||
# TODO prompt scheduling
|
||||
# interpolation should happen here and then we can re-enable prompt scheduling
|
||||
# ive tried simple torch.mean and its not good-enough
|
||||
|
||||
def fix_length(embeds):
|
||||
max_len = max([e.shape[1] for e in embeds if e is not None])
|
||||
for i, e in enumerate(embeds):
|
||||
if e is not None and e.shape[1] < max_len:
|
||||
expanded = torch.zeros((e.shape[0], max_len, e.shape[2]), device=e.device, dtype=e.dtype)
|
||||
expanded[:, :e.shape[1], :] = e
|
||||
embeds[i] = expanded
|
||||
return torch.cat(embeds, dim=0).to(devices.device, dtype=devices.dtype)
|
||||
|
||||
if len(prompt_embeds) > 0:
|
||||
p.prompt_embeds.append(fix_length(prompt_embeds))
|
||||
if len(negative_embeds) > 0:
|
||||
p.negative_embeds.append(fix_length(negative_embeds))
|
||||
if len(positive_pooleds) > 0:
|
||||
p.positive_pooleds.append(fix_length(positive_pooleds))
|
||||
if len(negative_pooleds) > 0:
|
||||
p.negative_pooleds.append(fix_length(negative_pooleds))
|
||||
|
||||
if shared.opts.sd_textencoder_cache and p.batch_size == 1:
|
||||
cache.update({
|
||||
'prompt_embeds': p.prompt_embeds,
|
||||
'negative_embeds': p.negative_embeds,
|
||||
'positive_pooleds': p.positive_pooleds,
|
||||
'negative_pooleds': p.negative_pooleds,
|
||||
'scheduled_prompt': p.scheduled_prompt,
|
||||
'prompts': prompts,
|
||||
'negative_prompts': negative_prompts,
|
||||
'clip_skip': clip_skip,
|
||||
'steps': steps,
|
||||
'model_type': shared.sd_model_type
|
||||
})
|
||||
else:
|
||||
cache.clear()
|
||||
if debug_enabled:
|
||||
get_tokens('positive', prompts[0])
|
||||
get_tokens('negative', negative_prompts[0])
|
||||
if shared.opts.diffusers_offload_mode == "balanced":
|
||||
pipe = sd_models.apply_balanced_offload(pipe)
|
||||
elif hasattr(pipe, "maybe_free_model_hooks"):
|
||||
# text encoder will stay in the vram and cause oom, send everything back to cpu before continuing
|
||||
pipe.maybe_free_model_hooks()
|
||||
debug(f"Prompt encode: time={(time.time() - t0):.3f}")
|
||||
devices.torch_gc()
|
||||
return
|
||||
return token_count
|
||||
|
||||
|
||||
def normalize_prompt(pairs: list):
|
||||
@@ -286,14 +345,20 @@ def normalize_prompt(pairs: list):
|
||||
return pairs
|
||||
|
||||
|
||||
def get_prompts_with_weights(prompt: str):
|
||||
def get_prompts_with_weights(pipe, prompt: str):
|
||||
t0 = time.time()
|
||||
manager = DiffusersTextualInversionManager(shared.sd_model, shared.sd_model.tokenizer or shared.sd_model.tokenizer_2)
|
||||
prompt = manager.maybe_convert_prompt(prompt, shared.sd_model.tokenizer or shared.sd_model.tokenizer_2)
|
||||
manager = DiffusersTextualInversionManager(pipe, pipe.tokenizer or pipe.tokenizer_2)
|
||||
prompt = manager.maybe_convert_prompt(prompt, pipe.tokenizer or pipe.tokenizer_2)
|
||||
texts_and_weights = prompt_parser.parse_prompt_attention(prompt)
|
||||
if shared.opts.prompt_mean_norm:
|
||||
texts_and_weights = normalize_prompt(texts_and_weights)
|
||||
texts, text_weights = zip(*texts_and_weights)
|
||||
if debug_enabled:
|
||||
all_tokens = 0
|
||||
for text in texts:
|
||||
tokens = get_tokens(pipe, 'section', text)
|
||||
all_tokens += tokens
|
||||
debug(f'Prompt tokenizer: parser={shared.opts.prompt_attention} tokens={all_tokens}')
|
||||
debug(f'Prompt: weights={texts_and_weights} time={(time.time() - t0):.3f}')
|
||||
return texts, text_weights
|
||||
|
||||
@@ -354,7 +419,8 @@ def pad_to_same_length(pipe, embeds, empty_embedding_providers=None):
|
||||
embeds[i] = embed
|
||||
return embeds
|
||||
|
||||
def split_prompts(prompt, SD3 = False):
|
||||
|
||||
def split_prompts(pipe, prompt, SD3 = False):
|
||||
if prompt.find("TE2:") != -1:
|
||||
prompt, prompt2 = prompt.split("TE2:")
|
||||
else:
|
||||
@@ -372,7 +438,7 @@ def split_prompts(prompt, SD3 = False):
|
||||
prompt3 = " " if prompt3.strip() == "" else prompt3.strip()
|
||||
|
||||
if SD3 and prompt3 != " ":
|
||||
ps, _ws = get_prompts_with_weights(prompt3)
|
||||
ps, _ws = get_prompts_with_weights(pipe, prompt3)
|
||||
prompt3 = " ".join(ps)
|
||||
return prompt, prompt2, prompt3
|
||||
|
||||
@@ -380,15 +446,15 @@ def split_prompts(prompt, SD3 = False):
|
||||
def get_weighted_text_embeddings(pipe, prompt: str = "", neg_prompt: str = "", clip_skip: int = None):
|
||||
device = devices.device
|
||||
SD3 = hasattr(pipe, 'text_encoder_3')
|
||||
prompt, prompt_2, prompt_3 = split_prompts(prompt, SD3)
|
||||
neg_prompt, neg_prompt_2, neg_prompt_3 = split_prompts(neg_prompt, SD3)
|
||||
prompt, prompt_2, prompt_3 = split_prompts(pipe, prompt, SD3)
|
||||
neg_prompt, neg_prompt_2, neg_prompt_3 = split_prompts(pipe, neg_prompt, SD3)
|
||||
|
||||
if prompt != prompt_2:
|
||||
ps = [get_prompts_with_weights(p) for p in [prompt, prompt_2]]
|
||||
ns = [get_prompts_with_weights(p) for p in [neg_prompt, neg_prompt_2]]
|
||||
ps = [get_prompts_with_weights(pipe, p) for p in [prompt, prompt_2]]
|
||||
ns = [get_prompts_with_weights(pipe, p) for p in [neg_prompt, neg_prompt_2]]
|
||||
else:
|
||||
ps = 2 * [get_prompts_with_weights(prompt)]
|
||||
ns = 2 * [get_prompts_with_weights(neg_prompt)]
|
||||
ps = 2 * [get_prompts_with_weights(pipe, prompt)]
|
||||
ns = 2 * [get_prompts_with_weights(pipe, neg_prompt)]
|
||||
|
||||
positives, positive_weights = zip(*ps)
|
||||
negatives, negative_weights = zip(*ns)
|
||||
@@ -434,7 +500,7 @@ def get_weighted_text_embeddings(pipe, prompt: str = "", neg_prompt: str = "", c
|
||||
# negative prompt has no keywords
|
||||
embed, ntokens = embedding_providers[i].get_embeddings_for_weighted_prompt_fragments(text_batch=[negatives[i]], fragment_weights_batch=[negative_weights[i]], device=device, should_return_tokens=True)
|
||||
negative_prompt_embeds.append(embed)
|
||||
debug(f'Prompt: unpadded shape={prompt_embeds[0].shape} TE{i+1} ptokens={torch.count_nonzero(ptokens)} ntokens={torch.count_nonzero(ntokens)} time={(time.time() - t0):.3f}')
|
||||
debug(f'Prompt: unpadded={prompt_embeds[0].shape} TE{i+1} ptokens={torch.count_nonzero(ptokens)} ntokens={torch.count_nonzero(ntokens)} time={(time.time() - t0):.3f}')
|
||||
if SD3:
|
||||
t0 = time.time()
|
||||
pooled_prompt_embeds.append(embedding_providers[0].get_pooled_embeddings(texts=positives[0] if len(positives[0]) == 1 else [" ".join(positives[0])], device=device))
|
||||
@@ -443,7 +509,7 @@ def get_weighted_text_embeddings(pipe, prompt: str = "", neg_prompt: str = "", c
|
||||
negative_pooled_prompt_embeds.append(embedding_providers[1].get_pooled_embeddings(texts=negatives[-1] if len(negatives[-1]) == 1 else [" ".join(negatives[-1])], device=device))
|
||||
pooled_prompt_embeds = torch.cat(pooled_prompt_embeds, dim=-1)
|
||||
negative_pooled_prompt_embeds = torch.cat(negative_pooled_prompt_embeds, dim=-1)
|
||||
debug(f'Prompt: pooled shape={pooled_prompt_embeds[0].shape} time={(time.time() - t0):.3f}')
|
||||
debug(f'Prompt: pooled={pooled_prompt_embeds[0].shape} time={(time.time() - t0):.3f}')
|
||||
elif prompt_embeds[-1].shape[-1] > 768:
|
||||
t0 = time.time()
|
||||
if shared.opts.diffusers_pooled == "weighted":
|
||||
@@ -503,8 +569,8 @@ def get_weighted_text_embeddings(pipe, prompt: str = "", neg_prompt: str = "", c
|
||||
|
||||
def get_xhinker_text_embeddings(pipe, prompt: str = "", neg_prompt: str = "", clip_skip: int = None):
|
||||
is_sd3 = hasattr(pipe, 'text_encoder_3')
|
||||
prompt, prompt_2, _prompt_3 = split_prompts(prompt, is_sd3)
|
||||
neg_prompt, neg_prompt_2, _neg_prompt_3 = split_prompts(neg_prompt, is_sd3)
|
||||
prompt, prompt_2, _prompt_3 = split_prompts(pipe, prompt, is_sd3)
|
||||
neg_prompt, neg_prompt_2, _neg_prompt_3 = split_prompts(pipe, neg_prompt, is_sd3)
|
||||
try:
|
||||
prompt = pipe.maybe_convert_prompt(prompt, pipe.tokenizer)
|
||||
neg_prompt = pipe.maybe_convert_prompt(neg_prompt, pipe.tokenizer)
|
||||
|
||||
@@ -1305,7 +1305,7 @@ def get_weighted_text_embeddings_sd3(
|
||||
# ---------------------- get neg t5 embeddings -------------------------
|
||||
neg_prompt_tokens_3 = torch.tensor([neg_prompt_tokens_3], dtype=torch.long)
|
||||
|
||||
t5_neg_prompt_embeds = pipe.text_encoder_3(neg_prompt_tokens_3.to(pipe.pipe.text_encoder_3.device))[0].squeeze(0)
|
||||
t5_neg_prompt_embeds = pipe.text_encoder_3(neg_prompt_tokens_3.to(pipe.text_encoder_3.device))[0].squeeze(0)
|
||||
t5_neg_prompt_embeds = t5_neg_prompt_embeds.to(device=pipe.text_encoder_3.device)
|
||||
|
||||
# add weight to neg t5 embeddings
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
"""
|
||||
Credit and original implementation: <https://github.com/ToTheBeginning/PuLID>
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
sys.path.append(os.path.dirname(__file__))
|
||||
from pulid_sdxl import StableDiffusionXLPuLIDPipeline, StableDiffusionXLPuLIDPipelineImage, StableDiffusionXLPuLIDPipelineInpaint
|
||||
from pulid_utils import resize_numpy_image_long as resize
|
||||
import attention_processor as attention
|
||||
import pulid_sampling as sampling
|
||||
@@ -0,0 +1,418 @@
|
||||
# modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
NUM_ZERO = 0
|
||||
ORTHO = False
|
||||
ORTHO_v2 = False
|
||||
|
||||
|
||||
class AttnProcessor(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
attn,
|
||||
hidden_states,
|
||||
encoder_hidden_states=None,
|
||||
attention_mask=None,
|
||||
temb=None,
|
||||
id_embedding=None,
|
||||
id_scale=1.0,
|
||||
):
|
||||
residual = hidden_states
|
||||
|
||||
if attn.spatial_norm is not None:
|
||||
hidden_states = attn.spatial_norm(hidden_states, temb)
|
||||
|
||||
input_ndim = hidden_states.ndim
|
||||
|
||||
if input_ndim == 4:
|
||||
batch_size, channel, height, width = hidden_states.shape
|
||||
hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
|
||||
|
||||
batch_size, sequence_length, _ = (
|
||||
hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
|
||||
)
|
||||
attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
|
||||
|
||||
if attn.group_norm is not None:
|
||||
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
|
||||
|
||||
query = attn.to_q(hidden_states)
|
||||
|
||||
if encoder_hidden_states is None:
|
||||
encoder_hidden_states = hidden_states
|
||||
elif attn.norm_cross:
|
||||
encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
|
||||
|
||||
key = attn.to_k(encoder_hidden_states)
|
||||
value = attn.to_v(encoder_hidden_states)
|
||||
|
||||
query = attn.head_to_batch_dim(query)
|
||||
key = attn.head_to_batch_dim(key)
|
||||
value = attn.head_to_batch_dim(value)
|
||||
|
||||
attention_probs = attn.get_attention_scores(query, key, attention_mask)
|
||||
hidden_states = torch.bmm(attention_probs, value)
|
||||
hidden_states = attn.batch_to_head_dim(hidden_states)
|
||||
|
||||
# linear proj
|
||||
hidden_states = attn.to_out[0](hidden_states)
|
||||
# dropout
|
||||
hidden_states = attn.to_out[1](hidden_states)
|
||||
|
||||
if input_ndim == 4:
|
||||
hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
|
||||
|
||||
if attn.residual_connection:
|
||||
hidden_states = hidden_states + residual
|
||||
|
||||
hidden_states = hidden_states / attn.rescale_output_factor
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
class IDAttnProcessor(nn.Module):
|
||||
r"""
|
||||
Attention processor for ID-Adapater.
|
||||
Args:
|
||||
hidden_size (`int`):
|
||||
The hidden size of the attention layer.
|
||||
cross_attention_dim (`int`):
|
||||
The number of channels in the `encoder_hidden_states`.
|
||||
scale (`float`, defaults to 1.0):
|
||||
the weight scale of image prompt.
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size, cross_attention_dim=None):
|
||||
super().__init__()
|
||||
self.id_to_k = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False)
|
||||
self.id_to_v = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
attn,
|
||||
hidden_states,
|
||||
encoder_hidden_states=None,
|
||||
attention_mask=None,
|
||||
temb=None,
|
||||
id_embedding=None,
|
||||
id_scale=1.0,
|
||||
):
|
||||
residual = hidden_states
|
||||
|
||||
if attn.spatial_norm is not None:
|
||||
hidden_states = attn.spatial_norm(hidden_states, temb)
|
||||
|
||||
input_ndim = hidden_states.ndim
|
||||
|
||||
if input_ndim == 4:
|
||||
batch_size, channel, height, width = hidden_states.shape
|
||||
hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
|
||||
|
||||
batch_size, sequence_length, _ = (
|
||||
hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
|
||||
)
|
||||
attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
|
||||
|
||||
if attn.group_norm is not None:
|
||||
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
|
||||
|
||||
query = attn.to_q(hidden_states)
|
||||
|
||||
if encoder_hidden_states is None:
|
||||
encoder_hidden_states = hidden_states
|
||||
elif attn.norm_cross:
|
||||
encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
|
||||
|
||||
key = attn.to_k(encoder_hidden_states)
|
||||
value = attn.to_v(encoder_hidden_states)
|
||||
|
||||
query = attn.head_to_batch_dim(query)
|
||||
key = attn.head_to_batch_dim(key)
|
||||
value = attn.head_to_batch_dim(value)
|
||||
|
||||
attention_probs = attn.get_attention_scores(query, key, attention_mask)
|
||||
hidden_states = torch.bmm(attention_probs, value)
|
||||
hidden_states = attn.batch_to_head_dim(hidden_states)
|
||||
|
||||
# for id-adapter
|
||||
if id_embedding is not None:
|
||||
if NUM_ZERO == 0:
|
||||
id_key = self.id_to_k(id_embedding)
|
||||
id_value = self.id_to_v(id_embedding)
|
||||
else:
|
||||
zero_tensor = torch.zeros(
|
||||
(id_embedding.size(0), NUM_ZERO, id_embedding.size(-1)),
|
||||
dtype=id_embedding.dtype,
|
||||
device=id_embedding.device,
|
||||
)
|
||||
id_key = self.id_to_k(torch.cat((id_embedding, zero_tensor), dim=1))
|
||||
id_value = self.id_to_v(torch.cat((id_embedding, zero_tensor), dim=1))
|
||||
|
||||
id_key = attn.head_to_batch_dim(id_key).to(query.dtype)
|
||||
id_value = attn.head_to_batch_dim(id_value).to(query.dtype)
|
||||
|
||||
id_attention_probs = attn.get_attention_scores(query, id_key, None)
|
||||
id_hidden_states = torch.bmm(id_attention_probs, id_value)
|
||||
id_hidden_states = attn.batch_to_head_dim(id_hidden_states)
|
||||
|
||||
if not ORTHO:
|
||||
hidden_states = hidden_states + id_scale * id_hidden_states
|
||||
else:
|
||||
projection = (
|
||||
torch.sum((hidden_states * id_hidden_states), dim=-2, keepdim=True)
|
||||
/ torch.sum((hidden_states * hidden_states), dim=-2, keepdim=True)
|
||||
* hidden_states
|
||||
)
|
||||
orthogonal = id_hidden_states - projection
|
||||
hidden_states = hidden_states + id_scale * orthogonal
|
||||
|
||||
# linear proj
|
||||
hidden_states = attn.to_out[0](hidden_states)
|
||||
# dropout
|
||||
hidden_states = attn.to_out[1](hidden_states)
|
||||
|
||||
if input_ndim == 4:
|
||||
hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
|
||||
|
||||
if attn.residual_connection:
|
||||
hidden_states = hidden_states + residual
|
||||
|
||||
hidden_states = hidden_states / attn.rescale_output_factor
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
class AttnProcessor2_0(nn.Module):
|
||||
r"""
|
||||
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
if not hasattr(F, "scaled_dot_product_attention"):
|
||||
raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
attn,
|
||||
hidden_states,
|
||||
encoder_hidden_states=None,
|
||||
attention_mask=None,
|
||||
temb=None,
|
||||
id_embedding=None,
|
||||
id_scale=1.0,
|
||||
):
|
||||
residual = hidden_states
|
||||
|
||||
if attn.spatial_norm is not None:
|
||||
hidden_states = attn.spatial_norm(hidden_states, temb)
|
||||
|
||||
input_ndim = hidden_states.ndim
|
||||
|
||||
if input_ndim == 4:
|
||||
batch_size, channel, height, width = hidden_states.shape
|
||||
hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
|
||||
|
||||
batch_size, sequence_length, _ = (
|
||||
hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
|
||||
)
|
||||
|
||||
if attention_mask is not None:
|
||||
attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
|
||||
# scaled_dot_product_attention expects attention_mask shape to be
|
||||
# (batch, heads, source_length, target_length)
|
||||
attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])
|
||||
|
||||
if attn.group_norm is not None:
|
||||
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
|
||||
|
||||
query = attn.to_q(hidden_states)
|
||||
|
||||
if encoder_hidden_states is None:
|
||||
encoder_hidden_states = hidden_states
|
||||
elif attn.norm_cross:
|
||||
encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
|
||||
|
||||
key = attn.to_k(encoder_hidden_states)
|
||||
value = attn.to_v(encoder_hidden_states)
|
||||
|
||||
inner_dim = key.shape[-1]
|
||||
head_dim = inner_dim // attn.heads
|
||||
|
||||
query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
|
||||
key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
|
||||
# the output of sdp = (batch, num_heads, seq_len, head_dim)
|
||||
hidden_states = F.scaled_dot_product_attention(
|
||||
query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
|
||||
)
|
||||
|
||||
hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
|
||||
hidden_states = hidden_states.to(query.dtype)
|
||||
|
||||
# linear proj
|
||||
hidden_states = attn.to_out[0](hidden_states)
|
||||
# dropout
|
||||
hidden_states = attn.to_out[1](hidden_states)
|
||||
|
||||
if input_ndim == 4:
|
||||
hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
|
||||
|
||||
if attn.residual_connection:
|
||||
hidden_states = hidden_states + residual
|
||||
|
||||
hidden_states = hidden_states / attn.rescale_output_factor
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
class IDAttnProcessor2_0(torch.nn.Module):
|
||||
r"""
|
||||
Attention processor for ID-Adapater for PyTorch 2.0.
|
||||
Args:
|
||||
hidden_size (`int`):
|
||||
The hidden size of the attention layer.
|
||||
cross_attention_dim (`int`):
|
||||
The number of channels in the `encoder_hidden_states`.
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size, cross_attention_dim=None):
|
||||
super().__init__()
|
||||
if not hasattr(F, "scaled_dot_product_attention"):
|
||||
raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
|
||||
|
||||
self.id_to_k = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False)
|
||||
self.id_to_v = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
attn,
|
||||
hidden_states,
|
||||
encoder_hidden_states=None,
|
||||
attention_mask=None,
|
||||
temb=None,
|
||||
id_embedding=None,
|
||||
id_scale=1.0,
|
||||
):
|
||||
residual = hidden_states
|
||||
|
||||
if attn.spatial_norm is not None:
|
||||
hidden_states = attn.spatial_norm(hidden_states, temb)
|
||||
|
||||
input_ndim = hidden_states.ndim
|
||||
|
||||
if input_ndim == 4:
|
||||
batch_size, channel, height, width = hidden_states.shape
|
||||
hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
|
||||
|
||||
batch_size, sequence_length, _ = (
|
||||
hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
|
||||
)
|
||||
|
||||
if attention_mask is not None:
|
||||
attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
|
||||
# scaled_dot_product_attention expects attention_mask shape to be
|
||||
# (batch, heads, source_length, target_length)
|
||||
attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])
|
||||
|
||||
if attn.group_norm is not None:
|
||||
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
|
||||
|
||||
query = attn.to_q(hidden_states)
|
||||
|
||||
if encoder_hidden_states is None:
|
||||
encoder_hidden_states = hidden_states
|
||||
elif attn.norm_cross:
|
||||
encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
|
||||
|
||||
key = attn.to_k(encoder_hidden_states)
|
||||
value = attn.to_v(encoder_hidden_states)
|
||||
|
||||
inner_dim = key.shape[-1]
|
||||
head_dim = inner_dim // attn.heads
|
||||
|
||||
query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
|
||||
key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
|
||||
# the output of sdp = (batch, num_heads, seq_len, head_dim)
|
||||
hidden_states = F.scaled_dot_product_attention(query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False)
|
||||
hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
|
||||
hidden_states = hidden_states.to(query.dtype)
|
||||
|
||||
# for id embedding
|
||||
if id_embedding is not None:
|
||||
if NUM_ZERO == 0:
|
||||
id_key = self.id_to_k(id_embedding).to(query.dtype)
|
||||
id_value = self.id_to_v(id_embedding).to(query.dtype)
|
||||
else:
|
||||
zero_tensor = torch.zeros(
|
||||
(id_embedding.size(0), NUM_ZERO, id_embedding.size(-1)),
|
||||
dtype=id_embedding.dtype,
|
||||
device=id_embedding.device,
|
||||
)
|
||||
id_cat = torch.cat((id_embedding, zero_tensor), dim=1)
|
||||
id_key = self.id_to_k(id_cat).to(query.dtype)
|
||||
id_value = self.id_to_v(id_cat).to(query.dtype)
|
||||
|
||||
id_key = id_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
id_value = id_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
|
||||
# the output of sdp = (batch, num_heads, seq_len, head_dim)
|
||||
id_hidden_states = F.scaled_dot_product_attention(query, id_key, id_value, attn_mask=None, dropout_p=0.0, is_causal=False)
|
||||
id_hidden_states = id_hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
|
||||
id_hidden_states = id_hidden_states.to(query.dtype)
|
||||
|
||||
if not ORTHO and not ORTHO_v2:
|
||||
hidden_states = hidden_states + id_scale * id_hidden_states
|
||||
elif ORTHO_v2:
|
||||
orig_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.to(torch.float32)
|
||||
id_hidden_states = id_hidden_states.to(torch.float32)
|
||||
attn_map = query @ id_key.transpose(-2, -1)
|
||||
attn_mean = attn_map.softmax(dim=-1).mean(dim=1)
|
||||
attn_mean = attn_mean[:, :, :5].sum(dim=-1, keepdim=True)
|
||||
projection = (
|
||||
torch.sum((hidden_states * id_hidden_states), dim=-2, keepdim=True)
|
||||
/ torch.sum((hidden_states * hidden_states), dim=-2, keepdim=True)
|
||||
* hidden_states
|
||||
)
|
||||
orthogonal = id_hidden_states + (attn_mean - 1) * projection
|
||||
hidden_states = hidden_states + id_scale * orthogonal
|
||||
hidden_states = hidden_states.to(orig_dtype)
|
||||
else:
|
||||
orig_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.to(torch.float32)
|
||||
id_hidden_states = id_hidden_states.to(torch.float32)
|
||||
projection = (
|
||||
torch.sum((hidden_states * id_hidden_states), dim=-2, keepdim=True)
|
||||
/ torch.sum((hidden_states * hidden_states), dim=-2, keepdim=True)
|
||||
* hidden_states
|
||||
)
|
||||
orthogonal = id_hidden_states - projection
|
||||
hidden_states = hidden_states + id_scale * orthogonal
|
||||
hidden_states = hidden_states.to(orig_dtype)
|
||||
|
||||
# linear proj
|
||||
hidden_states = attn.to_out[0](hidden_states)
|
||||
# dropout
|
||||
hidden_states = attn.to_out[1](hidden_states)
|
||||
|
||||
if input_ndim == 4:
|
||||
hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
|
||||
|
||||
if attn.residual_connection:
|
||||
hidden_states = hidden_states + residual
|
||||
|
||||
hidden_states = hidden_states / attn.rescale_output_factor
|
||||
|
||||
return hidden_states
|
||||
@@ -0,0 +1,250 @@
|
||||
import math
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
# FFN
|
||||
def FeedForward(dim, mult=4):
|
||||
inner_dim = int(dim * mult)
|
||||
return nn.Sequential(
|
||||
nn.LayerNorm(dim),
|
||||
nn.Linear(dim, inner_dim, bias=False),
|
||||
nn.GELU(),
|
||||
nn.Linear(inner_dim, dim, bias=False),
|
||||
)
|
||||
|
||||
|
||||
def reshape_tensor(x, heads):
|
||||
bs, length, _width = x.shape
|
||||
# (bs, length, width) --> (bs, length, n_heads, dim_per_head)
|
||||
x = x.view(bs, length, heads, -1)
|
||||
# (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
|
||||
x = x.transpose(1, 2)
|
||||
# (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
|
||||
x = x.reshape(bs, heads, length, -1)
|
||||
return x
|
||||
|
||||
|
||||
class PerceiverAttentionCA(nn.Module):
|
||||
def __init__(self, *, dim=3072, dim_head=128, heads=16, kv_dim=2048):
|
||||
super().__init__()
|
||||
self.scale = dim_head ** -0.5
|
||||
self.dim_head = dim_head
|
||||
self.heads = heads
|
||||
inner_dim = dim_head * heads
|
||||
self.norm1 = nn.LayerNorm(dim if kv_dim is None else kv_dim)
|
||||
self.norm2 = nn.LayerNorm(dim)
|
||||
self.to_q = nn.Linear(dim, inner_dim, bias=False)
|
||||
self.to_kv = nn.Linear(dim if kv_dim is None else kv_dim, inner_dim * 2, bias=False)
|
||||
self.to_out = nn.Linear(inner_dim, dim, bias=False)
|
||||
|
||||
def forward(self, x, latents):
|
||||
"""
|
||||
Args:
|
||||
x (torch.Tensor): image features
|
||||
shape (b, n1, D)
|
||||
latent (torch.Tensor): latent features
|
||||
shape (b, n2, D)
|
||||
"""
|
||||
x = self.norm1(x)
|
||||
latents = self.norm2(latents)
|
||||
b, seq_len, _ = latents.shape
|
||||
q = self.to_q(latents)
|
||||
k, v = self.to_kv(x).chunk(2, dim=-1)
|
||||
q = reshape_tensor(q, self.heads)
|
||||
k = reshape_tensor(k, self.heads)
|
||||
v = reshape_tensor(v, self.heads)
|
||||
|
||||
# attention
|
||||
scale = 1 / math.sqrt(math.sqrt(self.dim_head))
|
||||
weight = (q * scale) @ (k * scale).transpose(-2, -1) # More stable with f16 than dividing afterwards
|
||||
weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype)
|
||||
out = weight @ v
|
||||
out = out.permute(0, 2, 1, 3).reshape(b, seq_len, -1)
|
||||
|
||||
return self.to_out(out)
|
||||
|
||||
|
||||
class PerceiverAttention(nn.Module):
|
||||
def __init__(self, *, dim, dim_head=64, heads=8, kv_dim=None):
|
||||
super().__init__()
|
||||
self.scale = dim_head ** -0.5
|
||||
self.dim_head = dim_head
|
||||
self.heads = heads
|
||||
inner_dim = dim_head * heads
|
||||
self.norm1 = nn.LayerNorm(dim if kv_dim is None else kv_dim)
|
||||
self.norm2 = nn.LayerNorm(dim)
|
||||
self.to_q = nn.Linear(dim, inner_dim, bias=False)
|
||||
self.to_kv = nn.Linear(dim if kv_dim is None else kv_dim, inner_dim * 2, bias=False)
|
||||
self.to_out = nn.Linear(inner_dim, dim, bias=False)
|
||||
|
||||
def forward(self, x, latents):
|
||||
"""
|
||||
Args:
|
||||
x (torch.Tensor): image features
|
||||
shape (b, n1, D)
|
||||
latent (torch.Tensor): latent features
|
||||
shape (b, n2, D)
|
||||
"""
|
||||
x = self.norm1(x)
|
||||
latents = self.norm2(latents)
|
||||
b, seq_len, _ = latents.shape
|
||||
q = self.to_q(latents)
|
||||
kv_input = torch.cat((x, latents), dim=-2)
|
||||
k, v = self.to_kv(kv_input).chunk(2, dim=-1)
|
||||
q = reshape_tensor(q, self.heads)
|
||||
k = reshape_tensor(k, self.heads)
|
||||
v = reshape_tensor(v, self.heads)
|
||||
|
||||
# attention
|
||||
scale = 1 / math.sqrt(math.sqrt(self.dim_head))
|
||||
weight = (q * scale) @ (k * scale).transpose(-2, -1) # More stable with f16 than dividing afterwards
|
||||
weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype)
|
||||
out = weight @ v
|
||||
out = out.permute(0, 2, 1, 3).reshape(b, seq_len, -1)
|
||||
|
||||
return self.to_out(out)
|
||||
|
||||
|
||||
class IDFormer(nn.Module):
|
||||
"""
|
||||
- perceiver resampler like arch (compared with previous MLP-like arch)
|
||||
- we concat id embedding (generated by arcface) and query tokens as latents
|
||||
- latents will attend each other and interact with vit features through cross-attention
|
||||
- vit features are multi-scaled and inserted into IDFormer in order, currently, each scale corresponds to two
|
||||
IDFormer layers
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
dim=1024,
|
||||
depth=10,
|
||||
dim_head=64,
|
||||
heads=16,
|
||||
num_id_token=5,
|
||||
num_queries=32,
|
||||
output_dim=2048,
|
||||
ff_mult=4,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.num_id_token = num_id_token
|
||||
self.dim = dim
|
||||
self.num_queries = num_queries
|
||||
assert depth % 5 == 0
|
||||
self.depth = depth // 5
|
||||
scale = dim ** -0.5
|
||||
self.latents = nn.Parameter(torch.randn(1, num_queries, dim) * scale)
|
||||
self.proj_out = nn.Parameter(scale * torch.randn(dim, output_dim))
|
||||
|
||||
self.layers = nn.ModuleList([])
|
||||
for _ in range(depth):
|
||||
self.layers.append(
|
||||
nn.ModuleList(
|
||||
[
|
||||
PerceiverAttention(dim=dim, dim_head=dim_head, heads=heads),
|
||||
FeedForward(dim=dim, mult=ff_mult),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
for i in range(5):
|
||||
setattr(
|
||||
self,
|
||||
f'mapping_{i}',
|
||||
nn.Sequential(
|
||||
nn.Linear(1024, 1024),
|
||||
nn.LayerNorm(1024),
|
||||
nn.LeakyReLU(),
|
||||
nn.Linear(1024, 1024),
|
||||
nn.LayerNorm(1024),
|
||||
nn.LeakyReLU(),
|
||||
nn.Linear(1024, dim),
|
||||
),
|
||||
)
|
||||
|
||||
self.id_embedding_mapping = nn.Sequential(
|
||||
nn.Linear(1280, 1024),
|
||||
nn.LayerNorm(1024),
|
||||
nn.LeakyReLU(),
|
||||
nn.Linear(1024, 1024),
|
||||
nn.LayerNorm(1024),
|
||||
nn.LeakyReLU(),
|
||||
nn.Linear(1024, dim * num_id_token),
|
||||
)
|
||||
|
||||
def forward(self, x, y):
|
||||
latents = self.latents.repeat(x.size(0), 1, 1)
|
||||
num_duotu = x.shape[1] if x.ndim == 3 else 1
|
||||
x = self.id_embedding_mapping(x)
|
||||
x = x.reshape(-1, self.num_id_token * num_duotu, self.dim)
|
||||
latents = torch.cat((latents, x), dim=1)
|
||||
for i in range(5):
|
||||
vit_feature = getattr(self, f'mapping_{i}')(y[i])
|
||||
ctx_feature = torch.cat((x, vit_feature), dim=1)
|
||||
for attn, ff in self.layers[i * self.depth: (i + 1) * self.depth]:
|
||||
latents = attn(ctx_feature, latents) + latents
|
||||
latents = ff(latents) + latents
|
||||
latents = latents[:, :self.num_queries]
|
||||
latents = latents @ self.proj_out
|
||||
return latents
|
||||
|
||||
|
||||
class IDEncoder(nn.Module):
|
||||
def __init__(self, width=1280, context_dim=2048, num_token=5):
|
||||
super().__init__()
|
||||
self.num_token = num_token
|
||||
self.context_dim = context_dim
|
||||
h1 = min((context_dim * num_token) // 4, 1024)
|
||||
h2 = min((context_dim * num_token) // 2, 1024)
|
||||
self.body = nn.Sequential(
|
||||
nn.Linear(width, h1),
|
||||
nn.LayerNorm(h1),
|
||||
nn.LeakyReLU(),
|
||||
nn.Linear(h1, h2),
|
||||
nn.LayerNorm(h2),
|
||||
nn.LeakyReLU(),
|
||||
nn.Linear(h2, context_dim * num_token),
|
||||
)
|
||||
|
||||
for i in range(5):
|
||||
setattr(
|
||||
self,
|
||||
f'mapping_{i}',
|
||||
nn.Sequential(
|
||||
nn.Linear(1024, 1024),
|
||||
nn.LayerNorm(1024),
|
||||
nn.LeakyReLU(),
|
||||
nn.Linear(1024, 1024),
|
||||
nn.LayerNorm(1024),
|
||||
nn.LeakyReLU(),
|
||||
nn.Linear(1024, context_dim),
|
||||
),
|
||||
)
|
||||
setattr(
|
||||
self,
|
||||
f'mapping_patch_{i}',
|
||||
nn.Sequential(
|
||||
nn.Linear(1024, 1024),
|
||||
nn.LayerNorm(1024),
|
||||
nn.LeakyReLU(),
|
||||
nn.Linear(1024, 1024),
|
||||
nn.LayerNorm(1024),
|
||||
nn.LeakyReLU(),
|
||||
nn.Linear(1024, context_dim),
|
||||
),
|
||||
)
|
||||
|
||||
def forward(self, x, y):
|
||||
# x shape [N, C]
|
||||
x = self.body(x)
|
||||
x = x.reshape(-1, self.num_token, self.context_dim)
|
||||
|
||||
hidden_states = ()
|
||||
for i, emb in enumerate(y):
|
||||
hidden_state = getattr(self, f'mapping_{i}')(emb[:, :1]) + getattr(self, f'mapping_patch_{i}')(
|
||||
emb[:, 1:]
|
||||
).mean(dim=1, keepdim=True)
|
||||
hidden_states += (hidden_state,)
|
||||
hidden_states = torch.cat(hidden_states, dim=1)
|
||||
|
||||
return torch.cat([x, hidden_states], dim=1)
|
||||
@@ -0,0 +1,11 @@
|
||||
from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD
|
||||
from .factory import create_model, create_model_and_transforms, create_model_from_pretrained, get_tokenizer, create_transforms
|
||||
from .factory import list_models, add_model_config, get_model_config, load_checkpoint
|
||||
from .loss import ClipLoss
|
||||
from .model import CLIP, CustomCLIP, CLIPTextCfg, CLIPVisionCfg,\
|
||||
convert_weights_to_lp, convert_weights_to_fp16, trace_model, get_cast_dtype
|
||||
from .openai import load_openai_model, list_openai_models
|
||||
from .pretrained import list_pretrained, list_pretrained_models_by_tag, list_pretrained_tags_by_model,\
|
||||
get_pretrained_url, download_pretrained_from_url, is_pretrained_cfg, get_pretrained_cfg, download_pretrained
|
||||
from .tokenizer import SimpleTokenizer, tokenize
|
||||
from .transform import image_transform
|
||||
Binary file not shown.
@@ -0,0 +1,2 @@
|
||||
OPENAI_DATASET_MEAN = (0.48145466, 0.4578275, 0.40821073)
|
||||
OPENAI_DATASET_STD = (0.26862954, 0.26130258, 0.27577711)
|
||||
@@ -0,0 +1,548 @@
|
||||
# --------------------------------------------------------
|
||||
# Adapted from https://github.com/microsoft/unilm/tree/master/beit
|
||||
# --------------------------------------------------------
|
||||
import math
|
||||
import os
|
||||
from functools import partial
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
try:
|
||||
from timm.models.layers import drop_path, to_2tuple, trunc_normal_
|
||||
except:
|
||||
from timm.layers import drop_path, to_2tuple, trunc_normal_
|
||||
|
||||
from .transformer import PatchDropout
|
||||
from .rope import VisionRotaryEmbedding, VisionRotaryEmbeddingFast
|
||||
|
||||
if os.getenv('ENV_TYPE') == 'deepspeed':
|
||||
try:
|
||||
from deepspeed.runtime.activation_checkpointing.checkpointing import checkpoint
|
||||
except:
|
||||
from torch.utils.checkpoint import checkpoint
|
||||
else:
|
||||
from torch.utils.checkpoint import checkpoint
|
||||
|
||||
try:
|
||||
import xformers
|
||||
import xformers.ops as xops
|
||||
XFORMERS_IS_AVAILBLE = True
|
||||
except:
|
||||
XFORMERS_IS_AVAILBLE = False
|
||||
|
||||
class DropPath(nn.Module):
|
||||
"""Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
|
||||
"""
|
||||
def __init__(self, drop_prob=None):
|
||||
super(DropPath, self).__init__()
|
||||
self.drop_prob = drop_prob
|
||||
|
||||
def forward(self, x):
|
||||
return drop_path(x, self.drop_prob, self.training)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return 'p={}'.format(self.drop_prob)
|
||||
|
||||
|
||||
class Mlp(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_features,
|
||||
hidden_features=None,
|
||||
out_features=None,
|
||||
act_layer=nn.GELU,
|
||||
norm_layer=nn.LayerNorm,
|
||||
drop=0.,
|
||||
subln=False,
|
||||
|
||||
):
|
||||
super().__init__()
|
||||
out_features = out_features or in_features
|
||||
hidden_features = hidden_features or in_features
|
||||
self.fc1 = nn.Linear(in_features, hidden_features)
|
||||
self.act = act_layer()
|
||||
|
||||
self.ffn_ln = norm_layer(hidden_features) if subln else nn.Identity()
|
||||
|
||||
self.fc2 = nn.Linear(hidden_features, out_features)
|
||||
self.drop = nn.Dropout(drop)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.fc1(x)
|
||||
x = self.act(x)
|
||||
# x = self.drop(x)
|
||||
# commit this for the orignal BERT implement
|
||||
x = self.ffn_ln(x)
|
||||
|
||||
x = self.fc2(x)
|
||||
x = self.drop(x)
|
||||
return x
|
||||
|
||||
class SwiGLU(nn.Module):
|
||||
def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.SiLU, drop=0.,
|
||||
norm_layer=nn.LayerNorm, subln=False):
|
||||
super().__init__()
|
||||
out_features = out_features or in_features
|
||||
hidden_features = hidden_features or in_features
|
||||
|
||||
self.w1 = nn.Linear(in_features, hidden_features)
|
||||
self.w2 = nn.Linear(in_features, hidden_features)
|
||||
|
||||
self.act = act_layer()
|
||||
self.ffn_ln = norm_layer(hidden_features) if subln else nn.Identity()
|
||||
self.w3 = nn.Linear(hidden_features, out_features)
|
||||
|
||||
self.drop = nn.Dropout(drop)
|
||||
|
||||
def forward(self, x):
|
||||
x1 = self.w1(x)
|
||||
x2 = self.w2(x)
|
||||
hidden = self.act(x1) * x2
|
||||
x = self.ffn_ln(hidden)
|
||||
x = self.w3(x)
|
||||
x = self.drop(x)
|
||||
return x
|
||||
|
||||
class Attention(nn.Module):
|
||||
def __init__(
|
||||
self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0.,
|
||||
proj_drop=0., window_size=None, attn_head_dim=None, xattn=False, rope=None, subln=False, norm_layer=nn.LayerNorm):
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
head_dim = dim // num_heads
|
||||
if attn_head_dim is not None:
|
||||
head_dim = attn_head_dim
|
||||
all_head_dim = head_dim * self.num_heads
|
||||
self.scale = qk_scale or head_dim ** -0.5
|
||||
|
||||
self.subln = subln
|
||||
if self.subln:
|
||||
self.q_proj = nn.Linear(dim, all_head_dim, bias=False)
|
||||
self.k_proj = nn.Linear(dim, all_head_dim, bias=False)
|
||||
self.v_proj = nn.Linear(dim, all_head_dim, bias=False)
|
||||
else:
|
||||
self.qkv = nn.Linear(dim, all_head_dim * 3, bias=False)
|
||||
|
||||
if qkv_bias:
|
||||
self.q_bias = nn.Parameter(torch.zeros(all_head_dim))
|
||||
self.v_bias = nn.Parameter(torch.zeros(all_head_dim))
|
||||
else:
|
||||
self.q_bias = None
|
||||
self.v_bias = None
|
||||
|
||||
if window_size:
|
||||
self.window_size = window_size
|
||||
self.num_relative_distance = (2 * window_size[0] - 1) * (2 * window_size[1] - 1) + 3
|
||||
self.relative_position_bias_table = nn.Parameter(
|
||||
torch.zeros(self.num_relative_distance, num_heads)) # 2*Wh-1 * 2*Ww-1, nH
|
||||
# cls to token & token 2 cls & cls to cls
|
||||
|
||||
# get pair-wise relative position index for each token inside the window
|
||||
coords_h = torch.arange(window_size[0])
|
||||
coords_w = torch.arange(window_size[1])
|
||||
coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww
|
||||
coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww
|
||||
relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww
|
||||
relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2
|
||||
relative_coords[:, :, 0] += window_size[0] - 1 # shift to start from 0
|
||||
relative_coords[:, :, 1] += window_size[1] - 1
|
||||
relative_coords[:, :, 0] *= 2 * window_size[1] - 1
|
||||
relative_position_index = \
|
||||
torch.zeros(size=(window_size[0] * window_size[1] + 1, ) * 2, dtype=relative_coords.dtype)
|
||||
relative_position_index[1:, 1:] = relative_coords.sum(-1) # Wh*Ww, Wh*Ww
|
||||
relative_position_index[0, 0:] = self.num_relative_distance - 3
|
||||
relative_position_index[0:, 0] = self.num_relative_distance - 2
|
||||
relative_position_index[0, 0] = self.num_relative_distance - 1
|
||||
|
||||
self.register_buffer("relative_position_index", relative_position_index)
|
||||
else:
|
||||
self.window_size = None
|
||||
self.relative_position_bias_table = None
|
||||
self.relative_position_index = None
|
||||
|
||||
self.attn_drop = nn.Dropout(attn_drop)
|
||||
self.inner_attn_ln = norm_layer(all_head_dim) if subln else nn.Identity()
|
||||
# self.proj = nn.Linear(all_head_dim, all_head_dim)
|
||||
self.proj = nn.Linear(all_head_dim, dim)
|
||||
self.proj_drop = nn.Dropout(proj_drop)
|
||||
self.xattn = xattn
|
||||
self.xattn_drop = attn_drop
|
||||
|
||||
self.rope = rope
|
||||
|
||||
def forward(self, x, rel_pos_bias=None, attn_mask=None):
|
||||
B, N, C = x.shape
|
||||
if self.subln:
|
||||
q = F.linear(input=x, weight=self.q_proj.weight, bias=self.q_bias)
|
||||
k = F.linear(input=x, weight=self.k_proj.weight, bias=None)
|
||||
v = F.linear(input=x, weight=self.v_proj.weight, bias=self.v_bias)
|
||||
|
||||
q = q.reshape(B, N, self.num_heads, -1).permute(0, 2, 1, 3) # B, num_heads, N, C
|
||||
k = k.reshape(B, N, self.num_heads, -1).permute(0, 2, 1, 3)
|
||||
v = v.reshape(B, N, self.num_heads, -1).permute(0, 2, 1, 3)
|
||||
else:
|
||||
|
||||
qkv_bias = None
|
||||
if self.q_bias is not None:
|
||||
qkv_bias = torch.cat((self.q_bias, torch.zeros_like(self.v_bias, requires_grad=False), self.v_bias))
|
||||
|
||||
qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias)
|
||||
qkv = qkv.reshape(B, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) # 3, B, num_heads, N, C
|
||||
q, k, v = qkv[0], qkv[1], qkv[2]
|
||||
|
||||
if self.rope:
|
||||
# slightly fast impl
|
||||
q_t = q[:, :, 1:, :]
|
||||
ro_q_t = self.rope(q_t)
|
||||
q = torch.cat((q[:, :, :1, :], ro_q_t), -2).type_as(v)
|
||||
|
||||
k_t = k[:, :, 1:, :]
|
||||
ro_k_t = self.rope(k_t)
|
||||
k = torch.cat((k[:, :, :1, :], ro_k_t), -2).type_as(v)
|
||||
|
||||
if self.xattn:
|
||||
q = q.permute(0, 2, 1, 3) # B, num_heads, N, C -> B, N, num_heads, C
|
||||
k = k.permute(0, 2, 1, 3)
|
||||
v = v.permute(0, 2, 1, 3)
|
||||
|
||||
x = xops.memory_efficient_attention(
|
||||
q, k, v,
|
||||
p=self.xattn_drop,
|
||||
scale=self.scale,
|
||||
)
|
||||
x = x.reshape(B, N, -1)
|
||||
x = self.inner_attn_ln(x)
|
||||
x = self.proj(x)
|
||||
x = self.proj_drop(x)
|
||||
else:
|
||||
q = q * self.scale
|
||||
attn = (q @ k.transpose(-2, -1))
|
||||
|
||||
if self.relative_position_bias_table is not None:
|
||||
relative_position_bias = \
|
||||
self.relative_position_bias_table[self.relative_position_index.view(-1)].view(
|
||||
self.window_size[0] * self.window_size[1] + 1,
|
||||
self.window_size[0] * self.window_size[1] + 1, -1) # Wh*Ww,Wh*Ww,nH
|
||||
relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww
|
||||
attn = attn + relative_position_bias.unsqueeze(0).type_as(attn)
|
||||
|
||||
if rel_pos_bias is not None:
|
||||
attn = attn + rel_pos_bias.type_as(attn)
|
||||
|
||||
if attn_mask is not None:
|
||||
attn_mask = attn_mask.bool()
|
||||
attn = attn.masked_fill(~attn_mask[:, None, None, :], float("-inf"))
|
||||
|
||||
attn = attn.softmax(dim=-1)
|
||||
attn = self.attn_drop(attn)
|
||||
|
||||
x = (attn @ v).transpose(1, 2).reshape(B, N, -1)
|
||||
x = self.inner_attn_ln(x)
|
||||
x = self.proj(x)
|
||||
x = self.proj_drop(x)
|
||||
return x
|
||||
|
||||
|
||||
class Block(nn.Module):
|
||||
|
||||
def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
|
||||
drop_path=0., init_values=None, act_layer=nn.GELU, norm_layer=nn.LayerNorm,
|
||||
window_size=None, attn_head_dim=None, xattn=False, rope=None, postnorm=False,
|
||||
subln=False, naiveswiglu=False):
|
||||
super().__init__()
|
||||
self.norm1 = norm_layer(dim)
|
||||
self.attn = Attention(
|
||||
dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale,
|
||||
attn_drop=attn_drop, proj_drop=drop, window_size=window_size, attn_head_dim=attn_head_dim,
|
||||
xattn=xattn, rope=rope, subln=subln, norm_layer=norm_layer)
|
||||
# NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
|
||||
self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
|
||||
self.norm2 = norm_layer(dim)
|
||||
mlp_hidden_dim = int(dim * mlp_ratio)
|
||||
|
||||
if naiveswiglu:
|
||||
self.mlp = SwiGLU(
|
||||
in_features=dim,
|
||||
hidden_features=mlp_hidden_dim,
|
||||
subln=subln,
|
||||
norm_layer=norm_layer,
|
||||
)
|
||||
else:
|
||||
self.mlp = Mlp(
|
||||
in_features=dim,
|
||||
hidden_features=mlp_hidden_dim,
|
||||
act_layer=act_layer,
|
||||
subln=subln,
|
||||
drop=drop
|
||||
)
|
||||
|
||||
if init_values is not None and init_values > 0:
|
||||
self.gamma_1 = nn.Parameter(init_values * torch.ones((dim)),requires_grad=True)
|
||||
self.gamma_2 = nn.Parameter(init_values * torch.ones((dim)),requires_grad=True)
|
||||
else:
|
||||
self.gamma_1, self.gamma_2 = None, None
|
||||
|
||||
self.postnorm = postnorm
|
||||
|
||||
def forward(self, x, rel_pos_bias=None, attn_mask=None):
|
||||
if self.gamma_1 is None:
|
||||
if self.postnorm:
|
||||
x = x + self.drop_path(self.norm1(self.attn(x, rel_pos_bias=rel_pos_bias, attn_mask=attn_mask)))
|
||||
x = x + self.drop_path(self.norm2(self.mlp(x)))
|
||||
else:
|
||||
x = x + self.drop_path(self.attn(self.norm1(x), rel_pos_bias=rel_pos_bias, attn_mask=attn_mask))
|
||||
x = x + self.drop_path(self.mlp(self.norm2(x)))
|
||||
else:
|
||||
if self.postnorm:
|
||||
x = x + self.drop_path(self.gamma_1 * self.norm1(self.attn(x, rel_pos_bias=rel_pos_bias, attn_mask=attn_mask)))
|
||||
x = x + self.drop_path(self.gamma_2 * self.norm2(self.mlp(x)))
|
||||
else:
|
||||
x = x + self.drop_path(self.gamma_1 * self.attn(self.norm1(x), rel_pos_bias=rel_pos_bias, attn_mask=attn_mask))
|
||||
x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x)))
|
||||
return x
|
||||
|
||||
|
||||
class PatchEmbed(nn.Module):
|
||||
""" Image to Patch Embedding
|
||||
"""
|
||||
def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):
|
||||
super().__init__()
|
||||
img_size = to_2tuple(img_size)
|
||||
patch_size = to_2tuple(patch_size)
|
||||
num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0])
|
||||
self.patch_shape = (img_size[0] // patch_size[0], img_size[1] // patch_size[1])
|
||||
self.img_size = img_size
|
||||
self.patch_size = patch_size
|
||||
self.num_patches = num_patches
|
||||
|
||||
self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
|
||||
|
||||
def forward(self, x, **kwargs):
|
||||
B, C, H, W = x.shape
|
||||
# FIXME look at relaxing size constraints
|
||||
assert H == self.img_size[0] and W == self.img_size[1], \
|
||||
f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})."
|
||||
x = self.proj(x).flatten(2).transpose(1, 2)
|
||||
return x
|
||||
|
||||
|
||||
class RelativePositionBias(nn.Module):
|
||||
|
||||
def __init__(self, window_size, num_heads):
|
||||
super().__init__()
|
||||
self.window_size = window_size
|
||||
self.num_relative_distance = (2 * window_size[0] - 1) * (2 * window_size[1] - 1) + 3
|
||||
self.relative_position_bias_table = nn.Parameter(
|
||||
torch.zeros(self.num_relative_distance, num_heads)) # 2*Wh-1 * 2*Ww-1, nH
|
||||
# cls to token & token 2 cls & cls to cls
|
||||
|
||||
# get pair-wise relative position index for each token inside the window
|
||||
coords_h = torch.arange(window_size[0])
|
||||
coords_w = torch.arange(window_size[1])
|
||||
coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww
|
||||
coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww
|
||||
relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww
|
||||
relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2
|
||||
relative_coords[:, :, 0] += window_size[0] - 1 # shift to start from 0
|
||||
relative_coords[:, :, 1] += window_size[1] - 1
|
||||
relative_coords[:, :, 0] *= 2 * window_size[1] - 1
|
||||
relative_position_index = \
|
||||
torch.zeros(size=(window_size[0] * window_size[1] + 1,) * 2, dtype=relative_coords.dtype)
|
||||
relative_position_index[1:, 1:] = relative_coords.sum(-1) # Wh*Ww, Wh*Ww
|
||||
relative_position_index[0, 0:] = self.num_relative_distance - 3
|
||||
relative_position_index[0:, 0] = self.num_relative_distance - 2
|
||||
relative_position_index[0, 0] = self.num_relative_distance - 1
|
||||
|
||||
self.register_buffer("relative_position_index", relative_position_index)
|
||||
|
||||
def forward(self):
|
||||
relative_position_bias = \
|
||||
self.relative_position_bias_table[self.relative_position_index.view(-1)].view(
|
||||
self.window_size[0] * self.window_size[1] + 1,
|
||||
self.window_size[0] * self.window_size[1] + 1, -1) # Wh*Ww,Wh*Ww,nH
|
||||
return relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww
|
||||
|
||||
|
||||
class EVAVisionTransformer(nn.Module):
|
||||
""" Vision Transformer with support for patch or hybrid CNN input stage
|
||||
"""
|
||||
def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
|
||||
num_heads=12, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0.,
|
||||
drop_path_rate=0., norm_layer=nn.LayerNorm, init_values=None, patch_dropout=0.,
|
||||
use_abs_pos_emb=True, use_rel_pos_bias=False, use_shared_rel_pos_bias=False, rope=False,
|
||||
use_mean_pooling=True, init_scale=0.001, grad_checkpointing=False, xattn=False, postnorm=False,
|
||||
pt_hw_seq_len=16, intp_freq=False, naiveswiglu=False, subln=False):
|
||||
super().__init__()
|
||||
|
||||
if not XFORMERS_IS_AVAILBLE:
|
||||
xattn = False
|
||||
|
||||
self.image_size = img_size
|
||||
self.num_classes = num_classes
|
||||
self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models
|
||||
|
||||
self.patch_embed = PatchEmbed(
|
||||
img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
|
||||
num_patches = self.patch_embed.num_patches
|
||||
|
||||
self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
|
||||
# self.mask_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
|
||||
if use_abs_pos_emb:
|
||||
self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
|
||||
else:
|
||||
self.pos_embed = None
|
||||
self.pos_drop = nn.Dropout(p=drop_rate)
|
||||
|
||||
if use_shared_rel_pos_bias:
|
||||
self.rel_pos_bias = RelativePositionBias(window_size=self.patch_embed.patch_shape, num_heads=num_heads)
|
||||
else:
|
||||
self.rel_pos_bias = None
|
||||
|
||||
if rope:
|
||||
half_head_dim = embed_dim // num_heads // 2
|
||||
hw_seq_len = img_size // patch_size
|
||||
self.rope = VisionRotaryEmbeddingFast(
|
||||
dim=half_head_dim,
|
||||
pt_seq_len=pt_hw_seq_len,
|
||||
ft_seq_len=hw_seq_len if intp_freq else None,
|
||||
# patch_dropout=patch_dropout
|
||||
)
|
||||
else:
|
||||
self.rope = None
|
||||
|
||||
self.naiveswiglu = naiveswiglu
|
||||
|
||||
dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule
|
||||
self.use_rel_pos_bias = use_rel_pos_bias
|
||||
self.blocks = nn.ModuleList([
|
||||
Block(
|
||||
dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,
|
||||
drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer,
|
||||
init_values=init_values, window_size=self.patch_embed.patch_shape if use_rel_pos_bias else None,
|
||||
xattn=xattn, rope=self.rope, postnorm=postnorm, subln=subln, naiveswiglu=naiveswiglu)
|
||||
for i in range(depth)])
|
||||
self.norm = nn.Identity() if use_mean_pooling else norm_layer(embed_dim)
|
||||
self.fc_norm = norm_layer(embed_dim) if use_mean_pooling else None
|
||||
self.head = nn.Linear(embed_dim, num_classes) if num_classes > 0 else nn.Identity()
|
||||
|
||||
if self.pos_embed is not None:
|
||||
trunc_normal_(self.pos_embed, std=.02)
|
||||
|
||||
trunc_normal_(self.cls_token, std=.02)
|
||||
# trunc_normal_(self.mask_token, std=.02)
|
||||
|
||||
self.apply(self._init_weights)
|
||||
self.fix_init_weight()
|
||||
|
||||
if isinstance(self.head, nn.Linear):
|
||||
trunc_normal_(self.head.weight, std=.02)
|
||||
self.head.weight.data.mul_(init_scale)
|
||||
self.head.bias.data.mul_(init_scale)
|
||||
|
||||
# setting a patch_dropout of 0. would mean it is disabled and this function would be the identity fn
|
||||
self.patch_dropout = PatchDropout(patch_dropout) if patch_dropout > 0. else nn.Identity()
|
||||
|
||||
self.grad_checkpointing = grad_checkpointing
|
||||
|
||||
def fix_init_weight(self):
|
||||
def rescale(param, layer_id):
|
||||
param.div_(math.sqrt(2.0 * layer_id))
|
||||
|
||||
for layer_id, layer in enumerate(self.blocks):
|
||||
rescale(layer.attn.proj.weight.data, layer_id + 1)
|
||||
if self.naiveswiglu:
|
||||
rescale(layer.mlp.w3.weight.data, layer_id + 1)
|
||||
else:
|
||||
rescale(layer.mlp.fc2.weight.data, layer_id + 1)
|
||||
|
||||
def get_cast_dtype(self) -> torch.dtype:
|
||||
return self.blocks[0].mlp.fc2.weight.dtype
|
||||
|
||||
def _init_weights(self, m):
|
||||
if isinstance(m, nn.Linear):
|
||||
trunc_normal_(m.weight, std=.02)
|
||||
if m.bias is not None:
|
||||
nn.init.constant_(m.bias, 0)
|
||||
elif isinstance(m, nn.LayerNorm):
|
||||
nn.init.constant_(m.bias, 0)
|
||||
nn.init.constant_(m.weight, 1.0)
|
||||
|
||||
def get_num_layers(self):
|
||||
return len(self.blocks)
|
||||
|
||||
def lock(self, unlocked_groups=0, freeze_bn_stats=False):
|
||||
assert unlocked_groups == 0, 'partial locking not currently supported for this model'
|
||||
for param in self.parameters():
|
||||
param.requires_grad = False
|
||||
|
||||
@torch.jit.ignore
|
||||
def set_grad_checkpointing(self, enable=True):
|
||||
self.grad_checkpointing = enable
|
||||
|
||||
@torch.jit.ignore
|
||||
def no_weight_decay(self):
|
||||
return {'pos_embed', 'cls_token'}
|
||||
|
||||
def get_classifier(self):
|
||||
return self.head
|
||||
|
||||
def reset_classifier(self, num_classes, global_pool=''):
|
||||
self.num_classes = num_classes
|
||||
self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity()
|
||||
|
||||
def forward_features(self, x, return_all_features=False, return_hidden=False, shuffle=False):
|
||||
|
||||
x = self.patch_embed(x)
|
||||
batch_size, seq_len, _ = x.size()
|
||||
|
||||
if shuffle:
|
||||
idx = torch.randperm(x.shape[1]) + 1
|
||||
zero = torch.LongTensor([0, ])
|
||||
idx = torch.cat([zero, idx])
|
||||
pos_embed = self.pos_embed[:, idx]
|
||||
|
||||
cls_tokens = self.cls_token.expand(batch_size, -1, -1) # stole cls_tokens impl from Phil Wang, thanks
|
||||
x = torch.cat((cls_tokens, x), dim=1)
|
||||
if shuffle:
|
||||
x = x + pos_embed
|
||||
elif self.pos_embed is not None:
|
||||
x = x + self.pos_embed
|
||||
x = self.pos_drop(x)
|
||||
|
||||
# a patch_dropout of 0. would mean it is disabled and this function would do nothing but return what was passed in
|
||||
if os.getenv('RoPE') == '1':
|
||||
if self.training and not isinstance(self.patch_dropout, nn.Identity):
|
||||
x, patch_indices_keep = self.patch_dropout(x)
|
||||
self.rope.forward = partial(self.rope.forward, patch_indices_keep=patch_indices_keep)
|
||||
else:
|
||||
self.rope.forward = partial(self.rope.forward, patch_indices_keep=None)
|
||||
x = self.patch_dropout(x)
|
||||
else:
|
||||
x = self.patch_dropout(x)
|
||||
|
||||
rel_pos_bias = self.rel_pos_bias() if self.rel_pos_bias is not None else None
|
||||
hidden_states = []
|
||||
for idx, blk in enumerate(self.blocks):
|
||||
if (0 < idx <= 20) and (idx % 4 == 0) and return_hidden:
|
||||
hidden_states.append(x)
|
||||
if self.grad_checkpointing:
|
||||
x = checkpoint(blk, x, (rel_pos_bias,))
|
||||
else:
|
||||
x = blk(x, rel_pos_bias=rel_pos_bias)
|
||||
|
||||
if not return_all_features:
|
||||
x = self.norm(x)
|
||||
if self.fc_norm is not None:
|
||||
return self.fc_norm(x.mean(1)), hidden_states
|
||||
else:
|
||||
return x[:, 0], hidden_states
|
||||
return x
|
||||
|
||||
def forward(self, x, return_all_features=False, return_hidden=False, shuffle=False):
|
||||
if return_all_features:
|
||||
return self.forward_features(x, return_all_features, return_hidden, shuffle)
|
||||
x, hidden_states = self.forward_features(x, return_all_features, return_hidden, shuffle)
|
||||
x = self.head(x)
|
||||
if return_hidden:
|
||||
return x, hidden_states
|
||||
return x
|
||||
@@ -0,0 +1,517 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple, Union, Dict, Any
|
||||
import torch
|
||||
|
||||
from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD
|
||||
from .model import CLIP, CustomCLIP, convert_weights_to_lp, convert_to_custom_text_state_dict,\
|
||||
get_cast_dtype
|
||||
from .openai import load_openai_model
|
||||
from .pretrained import is_pretrained_cfg, get_pretrained_cfg, download_pretrained, list_pretrained_tags_by_model
|
||||
from .transform import image_transform
|
||||
from .tokenizer import HFTokenizer, tokenize
|
||||
from .utils import resize_clip_pos_embed, resize_evaclip_pos_embed, resize_visual_pos_embed, resize_eva_pos_embed
|
||||
|
||||
|
||||
_MODEL_CONFIG_PATHS = [Path(__file__).parent / f"model_configs/"]
|
||||
_MODEL_CONFIGS = {} # directory (model_name: config) of model architecture configs
|
||||
|
||||
|
||||
def _natural_key(string_):
|
||||
return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_.lower())]
|
||||
|
||||
|
||||
def _rescan_model_configs():
|
||||
global _MODEL_CONFIGS
|
||||
|
||||
config_ext = ('.json',)
|
||||
config_files = []
|
||||
for config_path in _MODEL_CONFIG_PATHS:
|
||||
if config_path.is_file() and config_path.suffix in config_ext:
|
||||
config_files.append(config_path)
|
||||
elif config_path.is_dir():
|
||||
for ext in config_ext:
|
||||
config_files.extend(config_path.glob(f'*{ext}'))
|
||||
|
||||
for cf in config_files:
|
||||
with open(cf, "r", encoding="utf8") as f:
|
||||
model_cfg = json.load(f)
|
||||
if all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg')):
|
||||
_MODEL_CONFIGS[cf.stem] = model_cfg
|
||||
|
||||
_MODEL_CONFIGS = dict(sorted(_MODEL_CONFIGS.items(), key=lambda x: _natural_key(x[0])))
|
||||
|
||||
|
||||
_rescan_model_configs() # initial populate of model config registry
|
||||
|
||||
|
||||
def list_models():
|
||||
""" enumerate available model architectures based on config files """
|
||||
return list(_MODEL_CONFIGS.keys())
|
||||
|
||||
|
||||
def add_model_config(path):
|
||||
""" add model config path or file and update registry """
|
||||
if not isinstance(path, Path):
|
||||
path = Path(path)
|
||||
_MODEL_CONFIG_PATHS.append(path)
|
||||
_rescan_model_configs()
|
||||
|
||||
|
||||
def get_model_config(model_name):
|
||||
if model_name in _MODEL_CONFIGS:
|
||||
return deepcopy(_MODEL_CONFIGS[model_name])
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def get_tokenizer(model_name):
|
||||
config = get_model_config(model_name)
|
||||
tokenizer = HFTokenizer(config['text_cfg']['hf_tokenizer_name']) if 'hf_tokenizer_name' in config['text_cfg'] else tokenize
|
||||
return tokenizer
|
||||
|
||||
|
||||
# loading openai CLIP weights when is_openai=True for training
|
||||
def load_state_dict(checkpoint_path: str, map_location: str='cpu', model_key: str='model|module|state_dict', is_openai: bool=False, skip_list: list=[]):
|
||||
if is_openai:
|
||||
model = torch.jit.load(checkpoint_path, map_location="cpu").eval()
|
||||
state_dict = model.state_dict()
|
||||
for key in ["input_resolution", "context_length", "vocab_size"]:
|
||||
state_dict.pop(key, None)
|
||||
else:
|
||||
checkpoint = torch.load(checkpoint_path, map_location=map_location)
|
||||
for mk in model_key.split('|'):
|
||||
if isinstance(checkpoint, dict) and mk in checkpoint:
|
||||
state_dict = checkpoint[mk]
|
||||
break
|
||||
else:
|
||||
state_dict = checkpoint
|
||||
if next(iter(state_dict.items()))[0].startswith('module'):
|
||||
state_dict = {k[7:]: v for k, v in state_dict.items()}
|
||||
|
||||
for k in skip_list:
|
||||
if k in list(state_dict.keys()):
|
||||
logging.info(f"Removing key {k} from pretrained checkpoint")
|
||||
del state_dict[k]
|
||||
|
||||
if os.getenv('RoPE') == '1':
|
||||
for k in list(state_dict.keys()):
|
||||
if 'freqs_cos' in k or 'freqs_sin' in k:
|
||||
del state_dict[k]
|
||||
return state_dict
|
||||
|
||||
|
||||
|
||||
def load_checkpoint(model, checkpoint_path, model_key="model|module|state_dict", strict=True):
|
||||
state_dict = load_state_dict(checkpoint_path, model_key=model_key, is_openai=False)
|
||||
# detect old format and make compatible with new format
|
||||
if 'positional_embedding' in state_dict and not hasattr(model, 'positional_embedding'):
|
||||
state_dict = convert_to_custom_text_state_dict(state_dict)
|
||||
if 'text.logit_scale' in state_dict and hasattr(model, 'logit_scale'):
|
||||
state_dict['logit_scale'] = state_dict['text.logit_scale']
|
||||
del state_dict['text.logit_scale']
|
||||
|
||||
# resize_clip_pos_embed for CLIP and open CLIP
|
||||
if 'visual.positional_embedding' in state_dict:
|
||||
resize_clip_pos_embed(state_dict, model)
|
||||
# specified to eva_vit_model
|
||||
elif 'visual.pos_embed' in state_dict:
|
||||
resize_evaclip_pos_embed(state_dict, model)
|
||||
|
||||
# resize_clip_pos_embed(state_dict, model)
|
||||
incompatible_keys = model.load_state_dict(state_dict, strict=strict)
|
||||
logging.info(f"incompatible_keys.missing_keys: {incompatible_keys.missing_keys}")
|
||||
return incompatible_keys
|
||||
|
||||
def load_clip_visual_state_dict(checkpoint_path: str, map_location: str='cpu', is_openai: bool=False, skip_list:list=[]):
|
||||
state_dict = load_state_dict(checkpoint_path, map_location=map_location, is_openai=is_openai, skip_list=skip_list)
|
||||
|
||||
for k in list(state_dict.keys()):
|
||||
if not k.startswith('visual.'):
|
||||
del state_dict[k]
|
||||
for k in list(state_dict.keys()):
|
||||
if k.startswith('visual.'):
|
||||
new_k = k[7:]
|
||||
state_dict[new_k] = state_dict[k]
|
||||
del state_dict[k]
|
||||
return state_dict
|
||||
|
||||
def load_clip_text_state_dict(checkpoint_path: str, map_location: str='cpu', is_openai: bool=False, skip_list:list=[]):
|
||||
state_dict = load_state_dict(checkpoint_path, map_location=map_location, is_openai=is_openai, skip_list=skip_list)
|
||||
|
||||
for k in list(state_dict.keys()):
|
||||
if k.startswith('visual.'):
|
||||
del state_dict[k]
|
||||
return state_dict
|
||||
|
||||
def get_pretrained_tag(pretrained_model):
|
||||
pretrained_model = pretrained_model.lower()
|
||||
if "laion" in pretrained_model or "open_clip" in pretrained_model:
|
||||
return "open_clip"
|
||||
elif "openai" in pretrained_model:
|
||||
return "clip"
|
||||
elif "eva" in pretrained_model and "clip" in pretrained_model:
|
||||
return "eva_clip"
|
||||
else:
|
||||
return "other"
|
||||
|
||||
def load_pretrained_checkpoint(
|
||||
model,
|
||||
visual_checkpoint_path,
|
||||
text_checkpoint_path,
|
||||
strict=True,
|
||||
visual_model=None,
|
||||
text_model=None,
|
||||
model_key="model|module|state_dict",
|
||||
skip_list=[]):
|
||||
visual_tag = get_pretrained_tag(visual_model)
|
||||
text_tag = get_pretrained_tag(text_model)
|
||||
|
||||
logging.info(f"num of model state_dict keys: {len(model.state_dict().keys())}")
|
||||
visual_incompatible_keys, text_incompatible_keys = None, None
|
||||
if visual_checkpoint_path:
|
||||
if visual_tag == "eva_clip" or visual_tag == "open_clip":
|
||||
visual_state_dict = load_clip_visual_state_dict(visual_checkpoint_path, is_openai=False, skip_list=skip_list)
|
||||
elif visual_tag == "clip":
|
||||
visual_state_dict = load_clip_visual_state_dict(visual_checkpoint_path, is_openai=True, skip_list=skip_list)
|
||||
else:
|
||||
visual_state_dict = load_state_dict(visual_checkpoint_path, model_key=model_key, is_openai=False, skip_list=skip_list)
|
||||
|
||||
# resize_clip_pos_embed for CLIP and open CLIP
|
||||
if 'positional_embedding' in visual_state_dict:
|
||||
resize_visual_pos_embed(visual_state_dict, model)
|
||||
# specified to EVA model
|
||||
elif 'pos_embed' in visual_state_dict:
|
||||
resize_eva_pos_embed(visual_state_dict, model)
|
||||
|
||||
visual_incompatible_keys = model.visual.load_state_dict(visual_state_dict, strict=strict)
|
||||
logging.info(f"num of loaded visual_state_dict keys: {len(visual_state_dict.keys())}")
|
||||
logging.info(f"visual_incompatible_keys.missing_keys: {visual_incompatible_keys.missing_keys}")
|
||||
|
||||
if text_checkpoint_path:
|
||||
if text_tag == "eva_clip" or text_tag == "open_clip":
|
||||
text_state_dict = load_clip_text_state_dict(text_checkpoint_path, is_openai=False, skip_list=skip_list)
|
||||
elif text_tag == "clip":
|
||||
text_state_dict = load_clip_text_state_dict(text_checkpoint_path, is_openai=True, skip_list=skip_list)
|
||||
else:
|
||||
text_state_dict = load_state_dict(visual_checkpoint_path, model_key=model_key, is_openai=False, skip_list=skip_list)
|
||||
|
||||
text_incompatible_keys = model.text.load_state_dict(text_state_dict, strict=strict)
|
||||
|
||||
logging.info(f"num of loaded text_state_dict keys: {len(text_state_dict.keys())}")
|
||||
logging.info(f"text_incompatible_keys.missing_keys: {text_incompatible_keys.missing_keys}")
|
||||
|
||||
return visual_incompatible_keys, text_incompatible_keys
|
||||
|
||||
def create_model(
|
||||
model_name: str,
|
||||
pretrained: Optional[str] = None,
|
||||
precision: str = 'fp32',
|
||||
device: Union[str, torch.device] = 'cpu',
|
||||
jit: bool = False,
|
||||
force_quick_gelu: bool = False,
|
||||
force_custom_clip: bool = False,
|
||||
force_patch_dropout: Optional[float] = None,
|
||||
pretrained_image: str = '',
|
||||
pretrained_text: str = '',
|
||||
pretrained_hf: bool = True,
|
||||
pretrained_visual_model: str = None,
|
||||
pretrained_text_model: str = None,
|
||||
cache_dir: Optional[str] = None,
|
||||
skip_list: list = [],
|
||||
):
|
||||
model_name = model_name.replace('/', '-') # for callers using old naming with / in ViT names
|
||||
if isinstance(device, str):
|
||||
device = torch.device(device)
|
||||
|
||||
if pretrained and pretrained.lower() == 'openai':
|
||||
logging.info(f'Loading pretrained {model_name} from OpenAI.')
|
||||
model = load_openai_model(
|
||||
model_name,
|
||||
precision=precision,
|
||||
device=device,
|
||||
jit=jit,
|
||||
cache_dir=cache_dir,
|
||||
)
|
||||
else:
|
||||
model_cfg = get_model_config(model_name)
|
||||
if model_cfg is not None:
|
||||
logging.info(f'Loaded {model_name} model config.')
|
||||
else:
|
||||
logging.error(f'Model config for {model_name} not found; available models {list_models()}.')
|
||||
raise RuntimeError(f'Model config for {model_name} not found.')
|
||||
|
||||
if 'rope' in model_cfg.get('vision_cfg', {}):
|
||||
if model_cfg['vision_cfg']['rope']:
|
||||
os.environ['RoPE'] = "1"
|
||||
else:
|
||||
os.environ['RoPE'] = "0"
|
||||
|
||||
if force_quick_gelu:
|
||||
# override for use of QuickGELU on non-OpenAI transformer models
|
||||
model_cfg["quick_gelu"] = True
|
||||
|
||||
if force_patch_dropout is not None:
|
||||
# override the default patch dropout value
|
||||
model_cfg['vision_cfg']["patch_dropout"] = force_patch_dropout
|
||||
|
||||
cast_dtype = get_cast_dtype(precision)
|
||||
custom_clip = model_cfg.pop('custom_text', False) or force_custom_clip or ('hf_model_name' in model_cfg['text_cfg'])
|
||||
|
||||
|
||||
if custom_clip:
|
||||
if 'hf_model_name' in model_cfg.get('text_cfg', {}):
|
||||
model_cfg['text_cfg']['hf_model_pretrained'] = pretrained_hf
|
||||
model = CustomCLIP(**model_cfg, cast_dtype=cast_dtype)
|
||||
else:
|
||||
model = CLIP(**model_cfg, cast_dtype=cast_dtype)
|
||||
|
||||
pretrained_cfg = {}
|
||||
if pretrained:
|
||||
checkpoint_path = ''
|
||||
pretrained_cfg = get_pretrained_cfg(model_name, pretrained)
|
||||
if pretrained_cfg:
|
||||
checkpoint_path = download_pretrained(pretrained_cfg, cache_dir=cache_dir)
|
||||
elif os.path.exists(pretrained):
|
||||
checkpoint_path = pretrained
|
||||
|
||||
if checkpoint_path:
|
||||
logging.info(f'Loading pretrained {model_name} weights ({pretrained}).')
|
||||
load_checkpoint(model,
|
||||
checkpoint_path,
|
||||
model_key="model|module|state_dict",
|
||||
strict=False
|
||||
)
|
||||
else:
|
||||
error_str = (
|
||||
f'Pretrained weights ({pretrained}) not found for model {model_name}.'
|
||||
f'Available pretrained tags ({list_pretrained_tags_by_model(model_name)}.')
|
||||
logging.warning(error_str)
|
||||
raise RuntimeError(error_str)
|
||||
else:
|
||||
visual_checkpoint_path = ''
|
||||
text_checkpoint_path = ''
|
||||
|
||||
if pretrained_image:
|
||||
pretrained_visual_model = pretrained_visual_model.replace('/', '-') # for callers using old naming with / in ViT names
|
||||
pretrained_image_cfg = get_pretrained_cfg(pretrained_visual_model, pretrained_image)
|
||||
if 'timm_model_name' in model_cfg.get('vision_cfg', {}):
|
||||
# pretrained weight loading for timm models set via vision_cfg
|
||||
model_cfg['vision_cfg']['timm_model_pretrained'] = True
|
||||
elif pretrained_image_cfg:
|
||||
visual_checkpoint_path = download_pretrained(pretrained_image_cfg, cache_dir=cache_dir)
|
||||
elif os.path.exists(pretrained_image):
|
||||
visual_checkpoint_path = pretrained_image
|
||||
else:
|
||||
logging.warning(f'Pretrained weights ({visual_checkpoint_path}) not found for model {model_name}.visual.')
|
||||
raise RuntimeError(f'Pretrained weights ({visual_checkpoint_path}) not found for model {model_name}.visual.')
|
||||
|
||||
if pretrained_text:
|
||||
pretrained_text_model = pretrained_text_model.replace('/', '-') # for callers using old naming with / in ViT names
|
||||
pretrained_text_cfg = get_pretrained_cfg(pretrained_text_model, pretrained_text)
|
||||
if pretrained_image_cfg:
|
||||
text_checkpoint_path = download_pretrained(pretrained_text_cfg, cache_dir=cache_dir)
|
||||
elif os.path.exists(pretrained_text):
|
||||
text_checkpoint_path = pretrained_text
|
||||
else:
|
||||
logging.warning(f'Pretrained weights ({text_checkpoint_path}) not found for model {model_name}.text.')
|
||||
raise RuntimeError(f'Pretrained weights ({text_checkpoint_path}) not found for model {model_name}.text.')
|
||||
|
||||
if visual_checkpoint_path:
|
||||
logging.info(f'Loading pretrained {model_name}.visual weights ({visual_checkpoint_path}).')
|
||||
if text_checkpoint_path:
|
||||
logging.info(f'Loading pretrained {model_name}.text weights ({text_checkpoint_path}).')
|
||||
|
||||
if visual_checkpoint_path or text_checkpoint_path:
|
||||
load_pretrained_checkpoint(
|
||||
model,
|
||||
visual_checkpoint_path,
|
||||
text_checkpoint_path,
|
||||
strict=False,
|
||||
visual_model=pretrained_visual_model,
|
||||
text_model=pretrained_text_model,
|
||||
model_key="model|module|state_dict",
|
||||
skip_list=skip_list
|
||||
)
|
||||
|
||||
if "fp16" in precision or "bf16" in precision:
|
||||
logging.info(f'convert precision to {precision}')
|
||||
model = model.to(torch.bfloat16) if 'bf16' in precision else model.to(torch.float16)
|
||||
|
||||
model.to(device=device)
|
||||
|
||||
# set image / mean metadata from pretrained_cfg if available, or use default
|
||||
model.visual.image_mean = pretrained_cfg.get('mean', None) or OPENAI_DATASET_MEAN
|
||||
model.visual.image_std = pretrained_cfg.get('std', None) or OPENAI_DATASET_STD
|
||||
|
||||
if jit:
|
||||
model = torch.jit.script(model)
|
||||
|
||||
return model
|
||||
|
||||
|
||||
def create_model_and_transforms(
|
||||
model_name: str,
|
||||
pretrained: Optional[str] = None,
|
||||
precision: str = 'fp32',
|
||||
device: Union[str, torch.device] = 'cpu',
|
||||
jit: bool = False,
|
||||
force_quick_gelu: bool = False,
|
||||
force_custom_clip: bool = False,
|
||||
force_patch_dropout: Optional[float] = None,
|
||||
pretrained_image: str = '',
|
||||
pretrained_text: str = '',
|
||||
pretrained_hf: bool = True,
|
||||
pretrained_visual_model: str = None,
|
||||
pretrained_text_model: str = None,
|
||||
image_mean: Optional[Tuple[float, ...]] = None,
|
||||
image_std: Optional[Tuple[float, ...]] = None,
|
||||
cache_dir: Optional[str] = None,
|
||||
skip_list: list = [],
|
||||
):
|
||||
model = create_model(
|
||||
model_name,
|
||||
pretrained,
|
||||
precision=precision,
|
||||
device=device,
|
||||
jit=jit,
|
||||
force_quick_gelu=force_quick_gelu,
|
||||
force_custom_clip=force_custom_clip,
|
||||
force_patch_dropout=force_patch_dropout,
|
||||
pretrained_image=pretrained_image,
|
||||
pretrained_text=pretrained_text,
|
||||
pretrained_hf=pretrained_hf,
|
||||
pretrained_visual_model=pretrained_visual_model,
|
||||
pretrained_text_model=pretrained_text_model,
|
||||
cache_dir=cache_dir,
|
||||
skip_list=skip_list,
|
||||
)
|
||||
|
||||
image_mean = image_mean or getattr(model.visual, 'image_mean', None)
|
||||
image_std = image_std or getattr(model.visual, 'image_std', None)
|
||||
preprocess_train = image_transform(
|
||||
model.visual.image_size,
|
||||
is_train=True,
|
||||
mean=image_mean,
|
||||
std=image_std
|
||||
)
|
||||
preprocess_val = image_transform(
|
||||
model.visual.image_size,
|
||||
is_train=False,
|
||||
mean=image_mean,
|
||||
std=image_std
|
||||
)
|
||||
|
||||
return model, preprocess_train, preprocess_val
|
||||
|
||||
|
||||
def create_transforms(
|
||||
model_name: str,
|
||||
pretrained: Optional[str] = None,
|
||||
precision: str = 'fp32',
|
||||
device: Union[str, torch.device] = 'cpu',
|
||||
jit: bool = False,
|
||||
force_quick_gelu: bool = False,
|
||||
force_custom_clip: bool = False,
|
||||
force_patch_dropout: Optional[float] = None,
|
||||
pretrained_image: str = '',
|
||||
pretrained_text: str = '',
|
||||
pretrained_hf: bool = True,
|
||||
pretrained_visual_model: str = None,
|
||||
pretrained_text_model: str = None,
|
||||
image_mean: Optional[Tuple[float, ...]] = None,
|
||||
image_std: Optional[Tuple[float, ...]] = None,
|
||||
cache_dir: Optional[str] = None,
|
||||
skip_list: list = [],
|
||||
):
|
||||
model = create_model(
|
||||
model_name,
|
||||
pretrained,
|
||||
precision=precision,
|
||||
device=device,
|
||||
jit=jit,
|
||||
force_quick_gelu=force_quick_gelu,
|
||||
force_custom_clip=force_custom_clip,
|
||||
force_patch_dropout=force_patch_dropout,
|
||||
pretrained_image=pretrained_image,
|
||||
pretrained_text=pretrained_text,
|
||||
pretrained_hf=pretrained_hf,
|
||||
pretrained_visual_model=pretrained_visual_model,
|
||||
pretrained_text_model=pretrained_text_model,
|
||||
cache_dir=cache_dir,
|
||||
skip_list=skip_list,
|
||||
)
|
||||
|
||||
|
||||
image_mean = image_mean or getattr(model.visual, 'image_mean', None)
|
||||
image_std = image_std or getattr(model.visual, 'image_std', None)
|
||||
preprocess_train = image_transform(
|
||||
model.visual.image_size,
|
||||
is_train=True,
|
||||
mean=image_mean,
|
||||
std=image_std
|
||||
)
|
||||
preprocess_val = image_transform(
|
||||
model.visual.image_size,
|
||||
is_train=False,
|
||||
mean=image_mean,
|
||||
std=image_std
|
||||
)
|
||||
del model
|
||||
|
||||
return preprocess_train, preprocess_val
|
||||
|
||||
def create_model_from_pretrained(
|
||||
model_name: str,
|
||||
pretrained: str,
|
||||
precision: str = 'fp32',
|
||||
device: Union[str, torch.device] = 'cpu',
|
||||
jit: bool = False,
|
||||
force_quick_gelu: bool = False,
|
||||
force_custom_clip: bool = False,
|
||||
force_patch_dropout: Optional[float] = None,
|
||||
return_transform: bool = True,
|
||||
image_mean: Optional[Tuple[float, ...]] = None,
|
||||
image_std: Optional[Tuple[float, ...]] = None,
|
||||
cache_dir: Optional[str] = None,
|
||||
is_frozen: bool = False,
|
||||
):
|
||||
if not is_pretrained_cfg(model_name, pretrained) and not os.path.exists(pretrained):
|
||||
raise RuntimeError(
|
||||
f'{pretrained} is not a valid pretrained cfg or checkpoint for {model_name}.'
|
||||
f' Use open_clip.list_pretrained() to find one.')
|
||||
|
||||
model = create_model(
|
||||
model_name,
|
||||
pretrained,
|
||||
precision=precision,
|
||||
device=device,
|
||||
jit=jit,
|
||||
force_quick_gelu=force_quick_gelu,
|
||||
force_custom_clip=force_custom_clip,
|
||||
force_patch_dropout=force_patch_dropout,
|
||||
cache_dir=cache_dir,
|
||||
)
|
||||
|
||||
if is_frozen:
|
||||
for param in model.parameters():
|
||||
param.requires_grad = False
|
||||
|
||||
if not return_transform:
|
||||
return model
|
||||
|
||||
image_mean = image_mean or getattr(model.visual, 'image_mean', None)
|
||||
image_std = image_std or getattr(model.visual, 'image_std', None)
|
||||
preprocess = image_transform(
|
||||
model.visual.image_size,
|
||||
is_train=False,
|
||||
mean=image_mean,
|
||||
std=image_std
|
||||
)
|
||||
|
||||
return model, preprocess
|
||||
@@ -0,0 +1,57 @@
|
||||
# HF architecture dict:
|
||||
arch_dict = {
|
||||
# https://huggingface.co/docs/transformers/model_doc/roberta#roberta
|
||||
"roberta": {
|
||||
"config_names": {
|
||||
"context_length": "max_position_embeddings",
|
||||
"vocab_size": "vocab_size",
|
||||
"width": "hidden_size",
|
||||
"heads": "num_attention_heads",
|
||||
"layers": "num_hidden_layers",
|
||||
"layer_attr": "layer",
|
||||
"token_embeddings_attr": "embeddings"
|
||||
},
|
||||
"pooler": "mean_pooler",
|
||||
},
|
||||
# https://huggingface.co/docs/transformers/model_doc/xlm-roberta#transformers.XLMRobertaConfig
|
||||
"xlm-roberta": {
|
||||
"config_names": {
|
||||
"context_length": "max_position_embeddings",
|
||||
"vocab_size": "vocab_size",
|
||||
"width": "hidden_size",
|
||||
"heads": "num_attention_heads",
|
||||
"layers": "num_hidden_layers",
|
||||
"layer_attr": "layer",
|
||||
"token_embeddings_attr": "embeddings"
|
||||
},
|
||||
"pooler": "mean_pooler",
|
||||
},
|
||||
# https://huggingface.co/docs/transformers/model_doc/mt5#mt5
|
||||
"mt5": {
|
||||
"config_names": {
|
||||
# unlimited seqlen
|
||||
# https://github.com/google-research/text-to-text-transfer-transformer/issues/273
|
||||
# https://github.com/huggingface/transformers/blob/v4.24.0/src/transformers/models/t5/modeling_t5.py#L374
|
||||
"context_length": "",
|
||||
"vocab_size": "vocab_size",
|
||||
"width": "d_model",
|
||||
"heads": "num_heads",
|
||||
"layers": "num_layers",
|
||||
"layer_attr": "block",
|
||||
"token_embeddings_attr": "embed_tokens"
|
||||
},
|
||||
"pooler": "mean_pooler",
|
||||
},
|
||||
"bert": {
|
||||
"config_names": {
|
||||
"context_length": "max_position_embeddings",
|
||||
"vocab_size": "vocab_size",
|
||||
"width": "hidden_size",
|
||||
"heads": "num_attention_heads",
|
||||
"layers": "num_hidden_layers",
|
||||
"layer_attr": "layer",
|
||||
"token_embeddings_attr": "embeddings"
|
||||
},
|
||||
"pooler": "mean_pooler",
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
""" huggingface model adapter
|
||||
|
||||
Wraps HuggingFace transformers (https://github.com/huggingface/transformers) models for use as a text tower in CLIP model.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.nn import functional as F
|
||||
from torch import TensorType
|
||||
try:
|
||||
import transformers
|
||||
from transformers import AutoModel, AutoModelForMaskedLM, AutoTokenizer, AutoConfig, PretrainedConfig
|
||||
from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, \
|
||||
BaseModelOutputWithPoolingAndCrossAttentions
|
||||
except ImportError as e:
|
||||
transformers = None
|
||||
|
||||
|
||||
class BaseModelOutput:
|
||||
pass
|
||||
|
||||
|
||||
class PretrainedConfig:
|
||||
pass
|
||||
|
||||
from .hf_configs import arch_dict
|
||||
|
||||
# utils
|
||||
def _camel2snake(s):
|
||||
return re.sub(r'(?<!^)(?=[A-Z])', '_', s).lower()
|
||||
|
||||
# TODO: ?last - for gpt-like models
|
||||
_POOLERS = {}
|
||||
|
||||
def register_pooler(cls):
|
||||
"""Decorator registering pooler class"""
|
||||
_POOLERS[_camel2snake(cls.__name__)] = cls
|
||||
return cls
|
||||
|
||||
|
||||
@register_pooler
|
||||
class MeanPooler(nn.Module):
|
||||
"""Mean pooling"""
|
||||
def forward(self, x:BaseModelOutput, attention_mask:TensorType):
|
||||
masked_output = x.last_hidden_state * attention_mask.unsqueeze(-1)
|
||||
return masked_output.sum(dim=1) / attention_mask.sum(-1, keepdim=True)
|
||||
|
||||
@register_pooler
|
||||
class MaxPooler(nn.Module):
|
||||
"""Max pooling"""
|
||||
def forward(self, x:BaseModelOutput, attention_mask:TensorType):
|
||||
masked_output = x.last_hidden_state.masked_fill(attention_mask.unsqueeze(-1), -torch.inf)
|
||||
return masked_output.max(1).values
|
||||
|
||||
@register_pooler
|
||||
class ClsPooler(nn.Module):
|
||||
"""CLS token pooling"""
|
||||
def __init__(self, use_pooler_output=True):
|
||||
super().__init__()
|
||||
self.cls_token_position = 0
|
||||
self.use_pooler_output = use_pooler_output
|
||||
|
||||
def forward(self, x:BaseModelOutput, attention_mask:TensorType):
|
||||
|
||||
if (self.use_pooler_output and
|
||||
isinstance(x, (BaseModelOutputWithPooling, BaseModelOutputWithPoolingAndCrossAttentions)) and
|
||||
(x.pooler_output is not None)
|
||||
):
|
||||
return x.pooler_output
|
||||
|
||||
return x.last_hidden_state[:, self.cls_token_position, :]
|
||||
|
||||
class HFTextEncoder(nn.Module):
|
||||
"""HuggingFace model adapter"""
|
||||
def __init__(
|
||||
self,
|
||||
model_name_or_path: str,
|
||||
output_dim: int,
|
||||
tokenizer_name: str = None,
|
||||
config: PretrainedConfig = None,
|
||||
pooler_type: str = None,
|
||||
proj: str = None,
|
||||
pretrained: bool = True,
|
||||
masked_language_modeling: bool = False):
|
||||
super().__init__()
|
||||
|
||||
self.output_dim = output_dim
|
||||
|
||||
# TODO: find better way to get this information
|
||||
uses_transformer_pooler = (pooler_type == "cls_pooler")
|
||||
|
||||
if transformers is None:
|
||||
raise RuntimeError("Please `pip install transformers` to use pre-trained HuggingFace models")
|
||||
if config is None:
|
||||
self.config = AutoConfig.from_pretrained(model_name_or_path)
|
||||
if masked_language_modeling:
|
||||
create_func, model_args = (AutoModelForMaskedLM.from_pretrained, model_name_or_path) if pretrained else (
|
||||
AutoModelForMaskedLM.from_config, self.config)
|
||||
else:
|
||||
create_func, model_args = (AutoModel.from_pretrained, model_name_or_path) if pretrained else (
|
||||
AutoModel.from_config, self.config)
|
||||
# TODO: do all model configs have this attribute? PretrainedConfig does so yes??
|
||||
if hasattr(self.config, "is_encoder_decoder") and self.config.is_encoder_decoder:
|
||||
self.transformer = create_func(model_args)
|
||||
self.transformer = self.transformer.encoder
|
||||
else:
|
||||
self.transformer = create_func(model_args, add_pooling_layer=uses_transformer_pooler)
|
||||
else:
|
||||
self.config = config
|
||||
if masked_language_modeling:
|
||||
self.transformer = AutoModelForMaskedLM.from_config(config)
|
||||
else:
|
||||
self.transformer = AutoModel.from_config(config)
|
||||
|
||||
if pooler_type is None: # get default arch pooler
|
||||
self.pooler = _POOLERS[(arch_dict[self.config.model_type]["pooler"])]()
|
||||
else:
|
||||
self.pooler = _POOLERS[pooler_type]()
|
||||
|
||||
d_model = getattr(self.config, arch_dict[self.config.model_type]["config_names"]["width"])
|
||||
if (d_model == output_dim) and (proj is None): # do we always need a proj?
|
||||
self.proj = nn.Identity()
|
||||
elif proj == 'linear':
|
||||
self.proj = nn.Linear(d_model, output_dim, bias=False)
|
||||
elif proj == 'mlp':
|
||||
hidden_size = (d_model + output_dim) // 2
|
||||
self.proj = nn.Sequential(
|
||||
nn.Linear(d_model, hidden_size, bias=False),
|
||||
nn.GELU(),
|
||||
nn.Linear(hidden_size, output_dim, bias=False),
|
||||
)
|
||||
|
||||
# self.itm_proj = nn.Linear(d_model, 2, bias=False)
|
||||
# self.mlm_proj = nn.Linear(d_model, self.config.vocab_size), bias=False)
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
|
||||
|
||||
# def forward_itm(self, x:TensorType, image_embeds:TensorType) -> TensorType:
|
||||
# image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(x.device)
|
||||
# attn_mask = (x != self.config.pad_token_id).long()
|
||||
# out = self.transformer(
|
||||
# input_ids=x,
|
||||
# attention_mask=attn_mask,
|
||||
# encoder_hidden_states = image_embeds,
|
||||
# encoder_attention_mask = image_atts,
|
||||
# )
|
||||
# pooled_out = self.pooler(out, attn_mask)
|
||||
|
||||
# return self.itm_proj(pooled_out)
|
||||
|
||||
def mask(self, input_ids, vocab_size, device, targets=None, masked_indices=None, probability_matrix=None):
|
||||
if masked_indices is None:
|
||||
masked_indices = torch.bernoulli(probability_matrix).bool()
|
||||
|
||||
masked_indices[input_ids == self.tokenizer.pad_token_id] = False
|
||||
masked_indices[input_ids == self.tokenizer.cls_token_id] = False
|
||||
|
||||
if targets is not None:
|
||||
targets[~masked_indices] = -100 # We only compute loss on masked tokens
|
||||
|
||||
# 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])
|
||||
indices_replaced = torch.bernoulli(torch.full(input_ids.shape, 0.8)).bool() & masked_indices
|
||||
input_ids[indices_replaced] = self.tokenizer.mask_token_id
|
||||
|
||||
# 10% of the time, we replace masked input tokens with random word
|
||||
indices_random = torch.bernoulli(torch.full(input_ids.shape, 0.5)).bool() & masked_indices & ~indices_replaced
|
||||
random_words = torch.randint(vocab_size, input_ids.shape, dtype=torch.long).to(device)
|
||||
input_ids[indices_random] = random_words[indices_random]
|
||||
# The rest of the time (10% of the time) we keep the masked input tokens unchanged
|
||||
|
||||
if targets is not None:
|
||||
return input_ids, targets
|
||||
else:
|
||||
return input_ids
|
||||
|
||||
def forward_mlm(self, input_ids, image_embeds, mlm_probability=0.25):
|
||||
labels = input_ids.clone()
|
||||
attn_mask = (input_ids != self.config.pad_token_id).long()
|
||||
image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(input_ids.device)
|
||||
vocab_size = getattr(self.config, arch_dict[self.config.model_type]["config_names"]["vocab_size"])
|
||||
probability_matrix = torch.full(labels.shape, mlm_probability)
|
||||
input_ids, labels = self.mask(input_ids, vocab_size, input_ids.device, targets=labels,
|
||||
probability_matrix = probability_matrix)
|
||||
mlm_output = self.transformer(input_ids,
|
||||
attention_mask = attn_mask,
|
||||
encoder_hidden_states = image_embeds,
|
||||
encoder_attention_mask = image_atts,
|
||||
return_dict = True,
|
||||
labels = labels,
|
||||
)
|
||||
return mlm_output.loss
|
||||
# mlm_output = self.transformer(input_ids,
|
||||
# attention_mask = attn_mask,
|
||||
# encoder_hidden_states = image_embeds,
|
||||
# encoder_attention_mask = image_atts,
|
||||
# return_dict = True,
|
||||
# ).last_hidden_state
|
||||
# logits = self.mlm_proj(mlm_output)
|
||||
|
||||
# # logits = logits[:, :-1, :].contiguous().view(-1, vocab_size)
|
||||
# logits = logits[:, 1:, :].contiguous().view(-1, vocab_size)
|
||||
# labels = labels[:, 1:].contiguous().view(-1)
|
||||
|
||||
# mlm_loss = F.cross_entropy(
|
||||
# logits,
|
||||
# labels,
|
||||
# # label_smoothing=0.1,
|
||||
# )
|
||||
# return mlm_loss
|
||||
|
||||
|
||||
def forward(self, x:TensorType) -> TensorType:
|
||||
attn_mask = (x != self.config.pad_token_id).long()
|
||||
out = self.transformer(input_ids=x, attention_mask=attn_mask)
|
||||
pooled_out = self.pooler(out, attn_mask)
|
||||
|
||||
return self.proj(pooled_out)
|
||||
|
||||
def lock(self, unlocked_layers:int=0, freeze_layer_norm:bool=True):
|
||||
if not unlocked_layers: # full freezing
|
||||
for n, p in self.transformer.named_parameters():
|
||||
p.requires_grad = (not freeze_layer_norm) if "LayerNorm" in n.split(".") else False
|
||||
return
|
||||
|
||||
encoder = self.transformer.encoder if hasattr(self.transformer, 'encoder') else self.transformer
|
||||
layer_list = getattr(encoder, arch_dict[self.config.model_type]["config_names"]["layer_attr"])
|
||||
print(f"Unlocking {unlocked_layers}/{len(layer_list) + 1} layers of hf model")
|
||||
embeddings = getattr(
|
||||
self.transformer, arch_dict[self.config.model_type]["config_names"]["token_embeddings_attr"])
|
||||
modules = [embeddings, *layer_list][:-unlocked_layers]
|
||||
# freeze layers
|
||||
for module in modules:
|
||||
for n, p in module.named_parameters():
|
||||
p.requires_grad = (not freeze_layer_norm) if "LayerNorm" in n.split(".") else False
|
||||
|
||||
|
||||
@torch.jit.ignore
|
||||
def set_grad_checkpointing(self, enable=True):
|
||||
self.transformer.gradient_checkpointing_enable()
|
||||
|
||||
def get_num_layers(self):
|
||||
encoder = self.transformer.encoder if hasattr(self.transformer, 'encoder') else self.transformer
|
||||
layer_list = getattr(encoder, arch_dict[self.config.model_type]["config_names"]["layer_attr"])
|
||||
return len(layer_list)
|
||||
|
||||
def init_parameters(self):
|
||||
pass
|
||||
@@ -0,0 +1,138 @@
|
||||
import math
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.nn import functional as F
|
||||
|
||||
try:
|
||||
import torch.distributed.nn
|
||||
from torch import distributed as dist
|
||||
has_distributed = True
|
||||
except ImportError:
|
||||
has_distributed = False
|
||||
|
||||
try:
|
||||
import horovod.torch as hvd
|
||||
except ImportError:
|
||||
hvd = None
|
||||
|
||||
from timm.loss import LabelSmoothingCrossEntropy
|
||||
|
||||
|
||||
def gather_features(
|
||||
image_features,
|
||||
text_features,
|
||||
local_loss=False,
|
||||
gather_with_grad=False,
|
||||
rank=0,
|
||||
world_size=1,
|
||||
use_horovod=False
|
||||
):
|
||||
assert has_distributed, 'torch.distributed did not import correctly, please use a PyTorch version with support.'
|
||||
if use_horovod:
|
||||
assert hvd is not None, 'Please install horovod'
|
||||
if gather_with_grad:
|
||||
all_image_features = hvd.allgather(image_features)
|
||||
all_text_features = hvd.allgather(text_features)
|
||||
else:
|
||||
with torch.no_grad():
|
||||
all_image_features = hvd.allgather(image_features)
|
||||
all_text_features = hvd.allgather(text_features)
|
||||
if not local_loss:
|
||||
# ensure grads for local rank when all_* features don't have a gradient
|
||||
gathered_image_features = list(all_image_features.chunk(world_size, dim=0))
|
||||
gathered_text_features = list(all_text_features.chunk(world_size, dim=0))
|
||||
gathered_image_features[rank] = image_features
|
||||
gathered_text_features[rank] = text_features
|
||||
all_image_features = torch.cat(gathered_image_features, dim=0)
|
||||
all_text_features = torch.cat(gathered_text_features, dim=0)
|
||||
else:
|
||||
# We gather tensors from all gpus
|
||||
if gather_with_grad:
|
||||
all_image_features = torch.cat(torch.distributed.nn.all_gather(image_features), dim=0)
|
||||
all_text_features = torch.cat(torch.distributed.nn.all_gather(text_features), dim=0)
|
||||
# all_image_features = torch.cat(torch.distributed.nn.all_gather(image_features, async_op=True), dim=0)
|
||||
# all_text_features = torch.cat(torch.distributed.nn.all_gather(text_features, async_op=True), dim=0)
|
||||
else:
|
||||
gathered_image_features = [torch.zeros_like(image_features) for _ in range(world_size)]
|
||||
gathered_text_features = [torch.zeros_like(text_features) for _ in range(world_size)]
|
||||
dist.all_gather(gathered_image_features, image_features)
|
||||
dist.all_gather(gathered_text_features, text_features)
|
||||
if not local_loss:
|
||||
# ensure grads for local rank when all_* features don't have a gradient
|
||||
gathered_image_features[rank] = image_features
|
||||
gathered_text_features[rank] = text_features
|
||||
all_image_features = torch.cat(gathered_image_features, dim=0)
|
||||
all_text_features = torch.cat(gathered_text_features, dim=0)
|
||||
|
||||
return all_image_features, all_text_features
|
||||
|
||||
|
||||
class ClipLoss(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
local_loss=False,
|
||||
gather_with_grad=False,
|
||||
cache_labels=False,
|
||||
rank=0,
|
||||
world_size=1,
|
||||
use_horovod=False,
|
||||
smoothing=0.,
|
||||
):
|
||||
super().__init__()
|
||||
self.local_loss = local_loss
|
||||
self.gather_with_grad = gather_with_grad
|
||||
self.cache_labels = cache_labels
|
||||
self.rank = rank
|
||||
self.world_size = world_size
|
||||
self.use_horovod = use_horovod
|
||||
self.label_smoothing_cross_entropy = LabelSmoothingCrossEntropy(smoothing=smoothing) if smoothing > 0 else None
|
||||
|
||||
# cache state
|
||||
self.prev_num_logits = 0
|
||||
self.labels = {}
|
||||
|
||||
def forward(self, image_features, text_features, logit_scale=1.):
|
||||
device = image_features.device
|
||||
if self.world_size > 1:
|
||||
all_image_features, all_text_features = gather_features(
|
||||
image_features, text_features,
|
||||
self.local_loss, self.gather_with_grad, self.rank, self.world_size, self.use_horovod)
|
||||
|
||||
if self.local_loss:
|
||||
logits_per_image = logit_scale * image_features @ all_text_features.T
|
||||
logits_per_text = logit_scale * text_features @ all_image_features.T
|
||||
else:
|
||||
logits_per_image = logit_scale * all_image_features @ all_text_features.T
|
||||
logits_per_text = logits_per_image.T
|
||||
else:
|
||||
logits_per_image = logit_scale * image_features @ text_features.T
|
||||
logits_per_text = logit_scale * text_features @ image_features.T
|
||||
# calculated ground-truth and cache if enabled
|
||||
num_logits = logits_per_image.shape[0]
|
||||
if self.prev_num_logits != num_logits or device not in self.labels:
|
||||
labels = torch.arange(num_logits, device=device, dtype=torch.long)
|
||||
if self.world_size > 1 and self.local_loss:
|
||||
labels = labels + num_logits * self.rank
|
||||
if self.cache_labels:
|
||||
self.labels[device] = labels
|
||||
self.prev_num_logits = num_logits
|
||||
else:
|
||||
labels = self.labels[device]
|
||||
|
||||
if self.label_smoothing_cross_entropy:
|
||||
total_loss = (
|
||||
self.label_smoothing_cross_entropy(logits_per_image, labels) +
|
||||
self.label_smoothing_cross_entropy(logits_per_text, labels)
|
||||
) / 2
|
||||
else:
|
||||
total_loss = (
|
||||
F.cross_entropy(logits_per_image, labels) +
|
||||
F.cross_entropy(logits_per_text, labels)
|
||||
) / 2
|
||||
|
||||
acc = None
|
||||
i2t_acc = (logits_per_image.argmax(-1) == labels).sum() / len(logits_per_image)
|
||||
t2i_acc = (logits_per_text.argmax(-1) == labels).sum() / len(logits_per_text)
|
||||
acc = {"i2t": i2t_acc, "t2i": t2i_acc}
|
||||
return total_loss, acc
|
||||
@@ -0,0 +1,432 @@
|
||||
""" CLIP Model
|
||||
|
||||
Adapted from https://github.com/openai/CLIP. Originally MIT License, Copyright (c) 2021 OpenAI.
|
||||
"""
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Tuple, Union
|
||||
from functools import partial
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
try:
|
||||
from .hf_model import HFTextEncoder
|
||||
except:
|
||||
HFTextEncoder = None
|
||||
from .modified_resnet import ModifiedResNet
|
||||
from .timm_model import TimmModel
|
||||
from .eva_vit_model import EVAVisionTransformer
|
||||
from .transformer import LayerNorm, QuickGELU, Attention, VisionTransformer, TextTransformer
|
||||
|
||||
try:
|
||||
from apex.normalization import FusedLayerNorm
|
||||
except:
|
||||
FusedLayerNorm = LayerNorm
|
||||
|
||||
@dataclass
|
||||
class CLIPVisionCfg:
|
||||
layers: Union[Tuple[int, int, int, int], int] = 12
|
||||
width: int = 768
|
||||
head_width: int = 64
|
||||
mlp_ratio: float = 4.0
|
||||
patch_size: int = 16
|
||||
image_size: Union[Tuple[int, int], int] = 224
|
||||
ls_init_value: Optional[float] = None # layer scale initial value
|
||||
patch_dropout: float = 0. # what fraction of patches to dropout during training (0 would mean disabled and no patches dropped) - 0.5 to 0.75 recommended in the paper for optimal results
|
||||
global_average_pool: bool = False # whether to global average pool the last embedding layer, instead of using CLS token (https://arxiv.org/abs/2205.01580)
|
||||
drop_path_rate: Optional[float] = None # drop path rate
|
||||
timm_model_name: str = None # a valid model name overrides layers, width, patch_size
|
||||
timm_model_pretrained: bool = False # use (imagenet) pretrained weights for named model
|
||||
timm_pool: str = 'avg' # feature pooling for timm model ('abs_attn', 'rot_attn', 'avg', '')
|
||||
timm_proj: str = 'linear' # linear projection for timm model output ('linear', 'mlp', '')
|
||||
timm_proj_bias: bool = False # enable bias final projection
|
||||
eva_model_name: str = None # a valid eva model name overrides layers, width, patch_size
|
||||
qkv_bias: bool = True
|
||||
fusedLN: bool = False
|
||||
xattn: bool = False
|
||||
postnorm: bool = False
|
||||
rope: bool = False
|
||||
pt_hw_seq_len: int = 16 # 224/14
|
||||
intp_freq: bool = False
|
||||
naiveswiglu: bool = False
|
||||
subln: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class CLIPTextCfg:
|
||||
context_length: int = 77
|
||||
vocab_size: int = 49408
|
||||
width: int = 512
|
||||
heads: int = 8
|
||||
layers: int = 12
|
||||
ls_init_value: Optional[float] = None # layer scale initial value
|
||||
hf_model_name: str = None
|
||||
hf_tokenizer_name: str = None
|
||||
hf_model_pretrained: bool = True
|
||||
proj: str = 'mlp'
|
||||
pooler_type: str = 'mean_pooler'
|
||||
masked_language_modeling: bool = False
|
||||
fusedLN: bool = False
|
||||
xattn: bool = False
|
||||
attn_mask: bool = True
|
||||
|
||||
def get_cast_dtype(precision: str):
|
||||
cast_dtype = None
|
||||
if precision == 'bf16':
|
||||
cast_dtype = torch.bfloat16
|
||||
elif precision == 'fp16':
|
||||
cast_dtype = torch.float16
|
||||
return cast_dtype
|
||||
|
||||
|
||||
def _build_vision_tower(
|
||||
embed_dim: int,
|
||||
vision_cfg: CLIPVisionCfg,
|
||||
quick_gelu: bool = False,
|
||||
cast_dtype: Optional[torch.dtype] = None
|
||||
):
|
||||
if isinstance(vision_cfg, dict):
|
||||
vision_cfg = CLIPVisionCfg(**vision_cfg)
|
||||
|
||||
# OpenAI models are pretrained w/ QuickGELU but native nn.GELU is both faster and more
|
||||
# memory efficient in recent PyTorch releases (>= 1.10).
|
||||
# NOTE: timm models always use native GELU regardless of quick_gelu flag.
|
||||
act_layer = QuickGELU if quick_gelu else nn.GELU
|
||||
|
||||
if vision_cfg.eva_model_name:
|
||||
vision_heads = vision_cfg.width // vision_cfg.head_width
|
||||
norm_layer = LayerNorm
|
||||
|
||||
visual = EVAVisionTransformer(
|
||||
img_size=vision_cfg.image_size,
|
||||
patch_size=vision_cfg.patch_size,
|
||||
num_classes=embed_dim,
|
||||
use_mean_pooling=vision_cfg.global_average_pool, #False
|
||||
init_values=vision_cfg.ls_init_value,
|
||||
patch_dropout=vision_cfg.patch_dropout,
|
||||
embed_dim=vision_cfg.width,
|
||||
depth=vision_cfg.layers,
|
||||
num_heads=vision_heads,
|
||||
mlp_ratio=vision_cfg.mlp_ratio,
|
||||
qkv_bias=vision_cfg.qkv_bias,
|
||||
drop_path_rate=vision_cfg.drop_path_rate,
|
||||
norm_layer= partial(FusedLayerNorm, eps=1e-6) if vision_cfg.fusedLN else partial(norm_layer, eps=1e-6),
|
||||
xattn=vision_cfg.xattn,
|
||||
rope=vision_cfg.rope,
|
||||
postnorm=vision_cfg.postnorm,
|
||||
pt_hw_seq_len= vision_cfg.pt_hw_seq_len, # 224/14
|
||||
intp_freq= vision_cfg.intp_freq,
|
||||
naiveswiglu= vision_cfg.naiveswiglu,
|
||||
subln= vision_cfg.subln
|
||||
)
|
||||
elif vision_cfg.timm_model_name:
|
||||
visual = TimmModel(
|
||||
vision_cfg.timm_model_name,
|
||||
pretrained=vision_cfg.timm_model_pretrained,
|
||||
pool=vision_cfg.timm_pool,
|
||||
proj=vision_cfg.timm_proj,
|
||||
proj_bias=vision_cfg.timm_proj_bias,
|
||||
embed_dim=embed_dim,
|
||||
image_size=vision_cfg.image_size
|
||||
)
|
||||
act_layer = nn.GELU # so that text transformer doesn't use QuickGELU w/ timm models
|
||||
elif isinstance(vision_cfg.layers, (tuple, list)):
|
||||
vision_heads = vision_cfg.width * 32 // vision_cfg.head_width
|
||||
visual = ModifiedResNet(
|
||||
layers=vision_cfg.layers,
|
||||
output_dim=embed_dim,
|
||||
heads=vision_heads,
|
||||
image_size=vision_cfg.image_size,
|
||||
width=vision_cfg.width
|
||||
)
|
||||
else:
|
||||
vision_heads = vision_cfg.width // vision_cfg.head_width
|
||||
norm_layer = LayerNormFp32 if cast_dtype in (torch.float16, torch.bfloat16) else LayerNorm
|
||||
visual = VisionTransformer(
|
||||
image_size=vision_cfg.image_size,
|
||||
patch_size=vision_cfg.patch_size,
|
||||
width=vision_cfg.width,
|
||||
layers=vision_cfg.layers,
|
||||
heads=vision_heads,
|
||||
mlp_ratio=vision_cfg.mlp_ratio,
|
||||
ls_init_value=vision_cfg.ls_init_value,
|
||||
patch_dropout=vision_cfg.patch_dropout,
|
||||
global_average_pool=vision_cfg.global_average_pool,
|
||||
output_dim=embed_dim,
|
||||
act_layer=act_layer,
|
||||
norm_layer=norm_layer,
|
||||
)
|
||||
|
||||
return visual
|
||||
|
||||
|
||||
def _build_text_tower(
|
||||
embed_dim: int,
|
||||
text_cfg: CLIPTextCfg,
|
||||
quick_gelu: bool = False,
|
||||
cast_dtype: Optional[torch.dtype] = None,
|
||||
):
|
||||
if isinstance(text_cfg, dict):
|
||||
text_cfg = CLIPTextCfg(**text_cfg)
|
||||
|
||||
if text_cfg.hf_model_name:
|
||||
text = HFTextEncoder(
|
||||
text_cfg.hf_model_name,
|
||||
output_dim=embed_dim,
|
||||
tokenizer_name=text_cfg.hf_tokenizer_name,
|
||||
proj=text_cfg.proj,
|
||||
pooler_type=text_cfg.pooler_type,
|
||||
masked_language_modeling=text_cfg.masked_language_modeling
|
||||
)
|
||||
else:
|
||||
act_layer = QuickGELU if quick_gelu else nn.GELU
|
||||
norm_layer = LayerNorm
|
||||
|
||||
text = TextTransformer(
|
||||
context_length=text_cfg.context_length,
|
||||
vocab_size=text_cfg.vocab_size,
|
||||
width=text_cfg.width,
|
||||
heads=text_cfg.heads,
|
||||
layers=text_cfg.layers,
|
||||
ls_init_value=text_cfg.ls_init_value,
|
||||
output_dim=embed_dim,
|
||||
act_layer=act_layer,
|
||||
norm_layer= FusedLayerNorm if text_cfg.fusedLN else norm_layer,
|
||||
xattn=text_cfg.xattn,
|
||||
attn_mask=text_cfg.attn_mask,
|
||||
)
|
||||
return text
|
||||
|
||||
class CLIP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
embed_dim: int,
|
||||
vision_cfg: CLIPVisionCfg,
|
||||
text_cfg: CLIPTextCfg,
|
||||
quick_gelu: bool = False,
|
||||
cast_dtype: Optional[torch.dtype] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.visual = _build_vision_tower(embed_dim, vision_cfg, quick_gelu, cast_dtype)
|
||||
|
||||
text = _build_text_tower(embed_dim, text_cfg, quick_gelu, cast_dtype)
|
||||
self.transformer = text.transformer
|
||||
self.vocab_size = text.vocab_size
|
||||
self.token_embedding = text.token_embedding
|
||||
self.positional_embedding = text.positional_embedding
|
||||
self.ln_final = text.ln_final
|
||||
self.text_projection = text.text_projection
|
||||
self.register_buffer('attn_mask', text.attn_mask, persistent=False)
|
||||
|
||||
self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
|
||||
|
||||
def lock_image_tower(self, unlocked_groups=0, freeze_bn_stats=False):
|
||||
# lock image tower as per LiT - https://arxiv.org/abs/2111.07991
|
||||
self.visual.lock(unlocked_groups=unlocked_groups, freeze_bn_stats=freeze_bn_stats)
|
||||
|
||||
@torch.jit.ignore
|
||||
def set_grad_checkpointing(self, enable=True):
|
||||
self.visual.set_grad_checkpointing(enable)
|
||||
self.transformer.grad_checkpointing = enable
|
||||
|
||||
@torch.jit.ignore
|
||||
def no_weight_decay(self):
|
||||
return {'logit_scale'}
|
||||
|
||||
def encode_image(self, image, normalize: bool = False):
|
||||
features = self.visual(image)
|
||||
return F.normalize(features, dim=-1) if normalize else features
|
||||
|
||||
def encode_text(self, text, normalize: bool = False):
|
||||
cast_dtype = self.transformer.get_cast_dtype()
|
||||
|
||||
x = self.token_embedding(text).to(cast_dtype) # [batch_size, n_ctx, d_model]
|
||||
|
||||
x = x + self.positional_embedding.to(cast_dtype)
|
||||
x = x.permute(1, 0, 2) # NLD -> LND
|
||||
x = self.transformer(x, attn_mask=self.attn_mask)
|
||||
x = x.permute(1, 0, 2) # LND -> NLD
|
||||
x = self.ln_final(x) # [batch_size, n_ctx, transformer.width]
|
||||
# take features from the eot embedding (eot_token is the highest number in each sequence)
|
||||
x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection
|
||||
return F.normalize(x, dim=-1) if normalize else x
|
||||
|
||||
def forward(self, image, text):
|
||||
image_features = self.encode_image(image, normalize=True)
|
||||
text_features = self.encode_text(text, normalize=True)
|
||||
return image_features, text_features, self.logit_scale.exp()
|
||||
|
||||
|
||||
class CustomCLIP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
embed_dim: int,
|
||||
vision_cfg: CLIPVisionCfg,
|
||||
text_cfg: CLIPTextCfg,
|
||||
quick_gelu: bool = False,
|
||||
cast_dtype: Optional[torch.dtype] = None,
|
||||
itm_task: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.visual = _build_vision_tower(embed_dim, vision_cfg, quick_gelu, cast_dtype)
|
||||
self.text = _build_text_tower(embed_dim, text_cfg, quick_gelu, cast_dtype)
|
||||
self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
|
||||
|
||||
def lock_image_tower(self, unlocked_groups=0, freeze_bn_stats=False):
|
||||
# lock image tower as per LiT - https://arxiv.org/abs/2111.07991
|
||||
self.visual.lock(unlocked_groups=unlocked_groups, freeze_bn_stats=freeze_bn_stats)
|
||||
|
||||
def lock_text_tower(self, unlocked_layers:int=0, freeze_layer_norm:bool=True):
|
||||
self.text.lock(unlocked_layers, freeze_layer_norm)
|
||||
|
||||
@torch.jit.ignore
|
||||
def set_grad_checkpointing(self, enable=True):
|
||||
self.visual.set_grad_checkpointing(enable)
|
||||
self.text.set_grad_checkpointing(enable)
|
||||
|
||||
@torch.jit.ignore
|
||||
def no_weight_decay(self):
|
||||
return {'logit_scale'}
|
||||
|
||||
def encode_image(self, image, normalize: bool = False):
|
||||
features = self.visual(image)
|
||||
return F.normalize(features, dim=-1) if normalize else features
|
||||
|
||||
def encode_text(self, text, normalize: bool = False):
|
||||
features = self.text(text)
|
||||
return F.normalize(features, dim=-1) if normalize else features
|
||||
|
||||
def forward(self, image, text):
|
||||
image_features = self.encode_image(image, normalize=True)
|
||||
text_features = self.encode_text(text, normalize=True)
|
||||
return image_features, text_features, self.logit_scale.exp()
|
||||
|
||||
|
||||
def convert_weights_to_lp(model: nn.Module, dtype=torch.float16):
|
||||
"""Convert applicable model parameters to low-precision (bf16 or fp16)"""
|
||||
|
||||
def _convert_weights(l):
|
||||
|
||||
if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)):
|
||||
l.weight.data = l.weight.data.to(dtype)
|
||||
if l.bias is not None:
|
||||
l.bias.data = l.bias.data.to(dtype)
|
||||
|
||||
if isinstance(l, (nn.MultiheadAttention, Attention)):
|
||||
for attr in [*[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], "in_proj_bias", "bias_k", "bias_v"]:
|
||||
tensor = getattr(l, attr, None)
|
||||
if tensor is not None:
|
||||
tensor.data = tensor.data.to(dtype)
|
||||
|
||||
if isinstance(l, nn.Parameter):
|
||||
l.data = l.data.to(dtype)
|
||||
|
||||
for name in ["text_projection", "proj"]:
|
||||
if hasattr(l, name) and isinstance(l, nn.Parameter):
|
||||
attr = getattr(l, name, None)
|
||||
if attr is not None:
|
||||
attr.data = attr.data.to(dtype)
|
||||
|
||||
model.apply(_convert_weights)
|
||||
|
||||
|
||||
convert_weights_to_fp16 = convert_weights_to_lp # backwards compat
|
||||
|
||||
|
||||
# used to maintain checkpoint compatibility
|
||||
def convert_to_custom_text_state_dict(state_dict: dict):
|
||||
if 'text_projection' in state_dict:
|
||||
# old format state_dict, move text tower -> .text
|
||||
new_state_dict = {}
|
||||
for k, v in state_dict.items():
|
||||
if any(k.startswith(p) for p in (
|
||||
'text_projection',
|
||||
'positional_embedding',
|
||||
'token_embedding',
|
||||
'transformer',
|
||||
'ln_final',
|
||||
'logit_scale'
|
||||
)):
|
||||
k = 'text.' + k
|
||||
new_state_dict[k] = v
|
||||
return new_state_dict
|
||||
return state_dict
|
||||
|
||||
|
||||
def build_model_from_openai_state_dict(
|
||||
state_dict: dict,
|
||||
quick_gelu=True,
|
||||
cast_dtype=torch.float16,
|
||||
):
|
||||
vit = "visual.proj" in state_dict
|
||||
|
||||
if vit:
|
||||
vision_width = state_dict["visual.conv1.weight"].shape[0]
|
||||
vision_layers = len(
|
||||
[k for k in state_dict.keys() if k.startswith("visual.") and k.endswith(".attn.in_proj_weight")])
|
||||
vision_patch_size = state_dict["visual.conv1.weight"].shape[-1]
|
||||
grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5)
|
||||
image_size = vision_patch_size * grid_size
|
||||
else:
|
||||
counts: list = [
|
||||
len(set(k.split(".")[2] for k in state_dict if k.startswith(f"visual.layer{b}"))) for b in [1, 2, 3, 4]]
|
||||
vision_layers = tuple(counts)
|
||||
vision_width = state_dict["visual.layer1.0.conv1.weight"].shape[0]
|
||||
output_width = round((state_dict["visual.attnpool.positional_embedding"].shape[0] - 1) ** 0.5)
|
||||
vision_patch_size = None
|
||||
assert output_width ** 2 + 1 == state_dict["visual.attnpool.positional_embedding"].shape[0]
|
||||
image_size = output_width * 32
|
||||
|
||||
embed_dim = state_dict["text_projection"].shape[1]
|
||||
context_length = state_dict["positional_embedding"].shape[0]
|
||||
vocab_size = state_dict["token_embedding.weight"].shape[0]
|
||||
transformer_width = state_dict["ln_final.weight"].shape[0]
|
||||
transformer_heads = transformer_width // 64
|
||||
transformer_layers = len(set(k.split(".")[2] for k in state_dict if k.startswith(f"transformer.resblocks")))
|
||||
|
||||
vision_cfg = CLIPVisionCfg(
|
||||
layers=vision_layers,
|
||||
width=vision_width,
|
||||
patch_size=vision_patch_size,
|
||||
image_size=image_size,
|
||||
)
|
||||
text_cfg = CLIPTextCfg(
|
||||
context_length=context_length,
|
||||
vocab_size=vocab_size,
|
||||
width=transformer_width,
|
||||
heads=transformer_heads,
|
||||
layers=transformer_layers
|
||||
)
|
||||
model = CLIP(
|
||||
embed_dim,
|
||||
vision_cfg=vision_cfg,
|
||||
text_cfg=text_cfg,
|
||||
quick_gelu=quick_gelu, # OpenAI models were trained with QuickGELU
|
||||
cast_dtype=cast_dtype,
|
||||
)
|
||||
|
||||
for key in ["input_resolution", "context_length", "vocab_size"]:
|
||||
state_dict.pop(key, None)
|
||||
|
||||
convert_weights_to_fp16(model) # OpenAI state dicts are partially converted to float16
|
||||
model.load_state_dict(state_dict)
|
||||
return model.eval()
|
||||
|
||||
|
||||
def trace_model(model, batch_size=256, device=torch.device('cpu')):
|
||||
model.eval()
|
||||
image_size = model.visual.image_size
|
||||
example_images = torch.ones((batch_size, 3, image_size, image_size), device=device)
|
||||
example_text = torch.zeros((batch_size, model.context_length), dtype=torch.int, device=device)
|
||||
model = torch.jit.trace_module(
|
||||
model,
|
||||
inputs=dict(
|
||||
forward=(example_images, example_text),
|
||||
encode_text=(example_text,),
|
||||
encode_image=(example_images,)
|
||||
))
|
||||
model.visual.image_size = image_size
|
||||
return model
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"embed_dim": 512,
|
||||
"vision_cfg": {
|
||||
"image_size": 224,
|
||||
"layers": 12,
|
||||
"width": 768,
|
||||
"patch_size": 16,
|
||||
"eva_model_name": "eva-clip-b-16",
|
||||
"ls_init_value": 0.1,
|
||||
"drop_path_rate": 0.0
|
||||
},
|
||||
"text_cfg": {
|
||||
"context_length": 77,
|
||||
"vocab_size": 49408,
|
||||
"width": 512,
|
||||
"heads": 8,
|
||||
"layers": 12
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"embed_dim": 1024,
|
||||
"vision_cfg": {
|
||||
"image_size": 224,
|
||||
"layers": 40,
|
||||
"width": 1408,
|
||||
"head_width": 88,
|
||||
"mlp_ratio": 4.3637,
|
||||
"patch_size": 14,
|
||||
"eva_model_name": "eva-clip-g-14-x",
|
||||
"drop_path_rate": 0,
|
||||
"xattn": true,
|
||||
"fusedLN": true
|
||||
},
|
||||
"text_cfg": {
|
||||
"context_length": 77,
|
||||
"vocab_size": 49408,
|
||||
"width": 1024,
|
||||
"heads": 16,
|
||||
"layers": 24,
|
||||
"xattn": false,
|
||||
"fusedLN": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"embed_dim": 1024,
|
||||
"vision_cfg": {
|
||||
"image_size": 224,
|
||||
"layers": 40,
|
||||
"width": 1408,
|
||||
"head_width": 88,
|
||||
"mlp_ratio": 4.3637,
|
||||
"patch_size": 14,
|
||||
"eva_model_name": "eva-clip-g-14-x",
|
||||
"drop_path_rate": 0.4,
|
||||
"xattn": true,
|
||||
"fusedLN": true
|
||||
},
|
||||
"text_cfg": {
|
||||
"context_length": 77,
|
||||
"vocab_size": 49408,
|
||||
"width": 768,
|
||||
"heads": 12,
|
||||
"layers": 12,
|
||||
"xattn": false,
|
||||
"fusedLN": true
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user