From 152f45925ef36d21eb3ca584d7bb04128b1e3fb4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 6 Nov 2023 10:12:33 -0500 Subject: [PATCH] add pixart model --- CHANGELOG.md | 24 ++++++++++++++++-------- README.md | 13 +++++++------ html/reference.json | 9 ++++++--- javascript/ui.js | 1 - modules/ui.py | 1 - 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cec598fbf..84d833279 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,8 @@ ## Update for 2023-11-01 -Another pretty big release, this time with focus on -new models, new backends and optimizations - plus quite a few fixes +Another pretty big release, this time with focus on new models (3 new model types), new backends (olive) and optimizations +Plus quite a few fixes Also, [Wiki](https://github.com/vladmandic/automatic/wiki) has been updated with new content, so check it out! Some highlights: [OpenVINO](https://github.com/vladmandic/automatic/wiki/OpenVINO), [IntelArc](https://github.com/vladmandic/automatic/wiki/Intel-ARC), [DirectML](https://github.com/vladmandic/automatic/wiki/DirectML), [ONNX/Olive>](https://github.com/vladmandic/automatic/wiki/ONNX-Runtime) @@ -13,19 +13,24 @@ Some highlights: [OpenVINO](https://github.com/vladmandic/automatic/wiki/OpenVIN *Extra networks -> Reference* for easier select & auto-download Models can still be downloaded manually, this is just a convenience feature & a showcase for supported models - new model type: [Segmind SSD-1B](https://huggingface.co/segmind/SSD-1B) - its a *distilled* model, this time 50% smaller and faster version of SD-XL! + its a *distilled* model trained at 1024px, this time 50% smaller and faster version of SD-XL! (and quality does not suffer, its just more optimized) test shows batch-size:4 with 1k images at full quality used less than 6.5GB of VRAM and for further optimization, you can use built-in **TAESD** decoder, which results in batch-size:16 with 1k images using 7.9GB of VRAM - download using built-in **Huggingface** downloader: `segmind/SSD-1B` + select from extra networks -> reference or download using built-in **Huggingface** downloader: `segmind/SSD-1B` + - new model type: [Pixart-α XL 2](https://github.com/PixArt-alpha/PixArt-alpha) + in medium/512px and large/1024px variations + comparable in quality to SD 1.5 and SD-XL, but with better text encoder and highly optimized training pipeline + so finetunes can be done in as little as 10% compared to SD/SD-XL (note that due to much larger text encodeder, it is a large model) + select from extra networks -> reference or download using built-in **Huggingface** downloader: `PixArt-alpha/PixArt-XL-2-1024-MS` - new model type: [LCM: Latent Consistency Models](https://github.com/openai/consistency_models) - near-instant generate in a as little as 3 steps! + trained at 512px, but with near-instant generate in a as little as 3 steps! combined with OpenVINO, generate on CPU takes less than 5-10 seconds: and absolute beast when combined with **HyperTile** and **TAESD** decoder resulting in **28 FPS** (on RTX4090 for batch 16x16 at 512px) note: set sampler to **Default** before loading model as LCM comes with its own *LCMScheduler* sampler - download using built-in **Huggingface** downloader: `SimianLuo/LCM_Dreamshaper_v7` + select from extra networks -> reference or download using built-in **Huggingface** downloader: `SimianLuo/LCM_Dreamshaper_v7` - support for **Custom pipelines**, thanks @disty0 download using built-in **Huggingface** downloader think of them as plugins for diffusers not unlike original extensions that modify behavior of `ldm` backend @@ -54,15 +59,17 @@ Some highlights: [OpenVINO](https://github.com/vladmandic/automatic/wiki/OpenVIN - **Interrogator** module has been removed from `extensions-builtin` and fully implemented (and improved) natively - **UI** - - UI tweaks for default themes + - UI tweaks for default themes - UI switch core font in default theme to **noto-sans** previously default font was simply *system-ui*, but it lead to too much variations between browsers and platforms + - UI tweaks for mobile devices, thanks @iDeNoh - updated **Context menu** - right-click on any button (e.g. generate button) + right-click on any button in action menu (e.g. generate button) - **Extra networks** - sort by name, size, date, etc. - switch between *gallery* and *list* views - add tags from user metadata (in addition to tags in model metadata) for **lora** + - added **Reference** models for diffusers backend - faster enumeration of all networks on server startup - **Packages** - updated `diffusers` to 0.22.0, `transformers` to 4.34.1 @@ -85,6 +92,7 @@ Some highlights: [OpenVINO](https://github.com/vladmandic/automatic/wiki/OpenVIN - fix image name template using model name - fix image name sequence - fix model path using relative path + - fix safari/webkit layour, thanks @eadnams22 - fix `torch-rocm` and `tensorflow-rocm` version detection, thanks @xangelix - fix **chainner** upscalers color clipping - fix for base+refiner workflow in diffusers mode: number of steps, diffuser pipe mode diff --git a/README.md b/README.md index e56eb9d23..295d7a213 100644 --- a/README.md +++ b/README.md @@ -53,15 +53,16 @@ All Individual features are not listed here, instead check [ChangeLog](CHANGELOG Additional models will be added as they become available and there is public interest in them -- [Stable Diffusion](https://github.com/Stability-AI/stablediffusion/) 1.x and 2.x *(all variants)* -- [Stable Diffusion XL](https://github.com/Stability-AI/generative-models) +- [RunwayML Stable Diffusion](https://github.com/Stability-AI/stablediffusion/) 1.x and 2.x *(all variants)* +- [StabilityAI Stable Diffusion XL](https://github.com/Stability-AI/generative-models) - [Segmind SSD-1B](https://huggingface.co/segmind/SSD-1B) - [LCM: Latent Consistency Models](https://github.com/openai/consistency_models) - [Kandinsky](https://github.com/ai-forever/Kandinsky-2) 2.1 and 2.2 -- [Wuerstchen](https://huggingface.co/blog/wuertschen) -- [UniDiffusion](https://github.com/thu-ml/unidiffuser) -- [DeepFloyd IF](https://github.com/deep-floyd/IF) -- [SD-Distilled](https://huggingface.co/blog/sd_distillation) *(all variants)* +- [Pixart-α XL 2](https://github.com/PixArt-alpha/PixArt-alpha) Medium and Large +- [Warp Wuerstchen](https://huggingface.co/blog/wuertschen) +- [Tsinghua UniDiffusion](https://github.com/thu-ml/unidiffuser) +- [DeepFloyd IF](https://github.com/deep-floyd/IF) Medium and Large +- [Segmind SD Distilled](https://huggingface.co/blog/sd_distillation) *(all variants)* ## Platform support diff --git a/html/reference.json b/html/reference.json index 4a1ad9d34..1dbf2665c 100644 --- a/html/reference.json +++ b/html/reference.json @@ -17,6 +17,12 @@ "LCM Dreamshaper 7": { "path": "SimianLuo/LCM_Dreamshaper_v7" }, + "Pixart-α XL 2 Medium": { + "path": "PixArt-alpha/PixArt-XL-2-512x512" + }, + "Pixart-α XL 2 Large": { + "path": "PixArt-alpha/PixArt-XL-2-1024-MS" + }, "Warp Wuerstchen": { "path": "warp-ai/wuerstchen" }, @@ -33,8 +39,5 @@ "path": "thu-ml/unidiffuser-v1", "desc": "UniDiffuser is a unified diffusion framework to fit all distributions relevant to a set of multi-modal data in one transformer. UniDiffuser is able to perform image, text, text-to-image, image-to-text, and image-text pair generation by setting proper timesteps without additional overhead.\nSpecifically, UniDiffuser employs a variation of transformer, called U-ViT, which parameterizes the joint noise prediction network. Other components perform as encoders and decoders of different modalities, including a pretrained image autoencoder from Stable Diffusion, a pretrained image ViT-B/32 CLIP encoder, a pretrained text ViT-L CLIP encoder, and a GPT-2 text decoder finetuned by ourselves.", "preview": "unidiffuser-v1.jpg" - }, - "Sudo-AI Zero123": { - "path": "sudo-ai/zero123plus-v1.1" } } diff --git a/javascript/ui.js b/javascript/ui.js index 8581fa3db..d15ac24de 100644 --- a/javascript/ui.js +++ b/javascript/ui.js @@ -352,7 +352,6 @@ function selectVAE(name) { } function selectReference(name) { - console.log('HERE', name); desiredCheckpointName = name; gradioApp().getElementById('change_reference').click(); } diff --git a/modules/ui.py b/modules/ui.py index f18fc1456..262507000 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -1199,7 +1199,6 @@ def create_ui(startup_timer = None): loaded = modelloader.load_reference(model) if loaded: return model if loaded else opts.sd_model_checkpoint - print('HERE', model, loaded) return loaded button_set_reference = gr.Button('Change reference', elem_id='change_reference', visible=False)