From d7daefb8acb087494a1ec549197091dad615ae2f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 10 Aug 2025 21:37:11 -0400 Subject: [PATCH] update cogview links, fix auraflow and hidream loaders Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 4 ++-- TODO.md | 1 + cli/test-all-models.py | 16 +++++++++++++--- html/reference.json | 4 ++-- modules/shared.py | 2 +- pipelines/generic.py | 12 ++++++------ pipelines/model_auraflow.py | 9 +++++---- pipelines/model_hidream.py | 3 ++- wiki | 2 +- 9 files changed, 33 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9259d02ee..3ea02ae4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,7 +139,7 @@ Feature highlights include: - [ModernUI](https://github.com/user-attachments/assets/6f156154-0b0a-4be2-94f0-979e9f679501) has quite some redesign which should make it more user friendly and easier to navigate plus several new UI themes If you're still using **StandardUI**, give [ModernUI](https://vladmandic.github.io/sdnext-docs/Themes/) a try! - New models such as [WanAI 2.2](https://wan.video/) in 5B and A14B variants for both *text-to-video* and *image-to-video* workflows as well as *text-to-image* workflow! - and also [FreePix F-Lite](https://huggingface.co/Freepik/F-Lite), [Bria 3.2](https://huggingface.co/briaai/BRIA-3.2) and [bigASP 2.5](https://civitai.com/models/1789765?modelVersionId=2025412) + and also [FreePik F-Lite](https://huggingface.co/Freepik/F-Lite), [Bria 3.2](https://huggingface.co/briaai/BRIA-3.2) and [bigASP 2.5](https://civitai.com/models/1789765?modelVersionId=2025412) - Redesigned [Video](https://vladmandic.github.io/sdnext-docs/Video) interface with support for general video models plus optimized [FramePack](https://vladmandic.github.io/sdnext-docs/FramePack) and [LTXVideo](https://vladmandic.github.io/sdnext-docs/LTX) support - Fully integrated nudity detection and optional censorship with [NudeNet](https://vladmandic.github.io/sdnext-docs/NudeNet) - New background replacement and relightning methods using **Latent Bridge Matching** and new **PixelArt** processing filter @@ -184,7 +184,7 @@ For details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/master can also load unet/transformer fine-tunes in safetensors format using UNET loader simply select in *networks -> models -> reference* *note* 1.3B model is a bit too small for good results and 14B is very large at 78GB even without second-stage so aggressive quantization and offloading are recommended - - [FreePix F-Lite](https://huggingface.co/Freepik/F-Lite) in *7B, 10B and Texture* variants + - [FreePik F-Lite](https://huggingface.co/Freepik/F-Lite) in *7B, 10B and Texture* variants F-Lite is a 7B/10B model trained exclusively on copyright-safe and SFW content, trained on internal dataset comprising approximately 80 million copyright-safe images available via *networks -> models -> reference* - [Bria 3.2](https://huggingface.co/briaai/BRIA-3.2) diff --git a/TODO.md b/TODO.md index da9883bd8..6116f4b45 100644 --- a/TODO.md +++ b/TODO.md @@ -4,6 +4,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Future Candidates +- Unified `CLIPTextModelWithProjection` loader - [Modular pipelines and guiders](https://github.com/huggingface/diffusers/issues/11915) - Refactor: Sampler options - Feature: Diffusers [group offloading](https://github.com/vladmandic/sdnext/issues/4049) diff --git a/cli/test-all-models.py b/cli/test-all-models.py index 7115bc8f2..17559e430 100755 --- a/cli/test-all-models.py +++ b/cli/test-all-models.py @@ -1,4 +1,9 @@ #!/usr/bin/env python +""" +fal/AuraFlow-v0.3: sdnq unusupported + +""" + import io import os import time @@ -27,8 +32,8 @@ models = [ "stabilityai/stable-diffusion-3.5-medium", "stabilityai/stable-diffusion-3.5-large", "fal/AuraFlow-v0.3", - "THUDM/CogView3-Plus-3B", - "THUDM/CogView4-6B", + "zai-org/CogView4-6B", + "zai-org/CogView3-Plus-3B", "nvidia/Cosmos-Predict2-2B-Text2Image", "nvidia/Cosmos-Predict2-14B-Text2Image", "Qwen/Qwen-Image", @@ -70,8 +75,11 @@ styles_tbd = [ 'Fixed Kneeling on Bed', 'Fixed Girl in Sin City', 'Fixed Girl in a city', + 'Fixed Girl in Lace', 'Fixed Lady in Tokyo', 'Fixed MadMax selfie', + 'Fixed Party Yacht', + 'Fixed Yoga Girls', 'Fixed SDNext Neon', ] @@ -116,8 +124,10 @@ def generate(): # pylint: disable=redefined-outer-name b64 = data['images'][0].split(',',1)[0] image = Image.open(io.BytesIO(base64.b64decode(b64))) info = data['info'] - log.info(f' image: size={image.size} time={t1-t0:.2f} info="{len(info)}" fn="{fn}"') + log.info(f' image: size={image.width}x{image.height} time={t1-t0:.2f} info={len(info)}') image.save(fn) + else: + log.error(f' model: error="{model}" style="{style}" no image') except Exception as e: log.error(f' model: error="{model}" style="{style}" exception="{e}"') diff --git a/html/reference.json b/html/reference.json index 63cdbbdda..21d42a81f 100644 --- a/html/reference.json +++ b/html/reference.json @@ -520,13 +520,13 @@ }, "CogView 4": { - "path": "THUDM/CogView4-6B", + "path": "zai-org/CogView4-6B", "desc": "An innovative cascaded framework that enhances the performance of text-to-image diffusion. CogView is the first model implementing relay diffusion in the realm of text-to-image generation, executing the task by first creating low-resolution images and subsequently applying relay-based super-resolution.", "preview": "THUDM--CogView4-6B.jpg", "skip": true }, "CogView 3 Plus": { - "path": "THUDM/CogView3-Plus-3B", + "path": "zai-org/CogView3-Plus-3B", "desc": "An innovative cascaded framework that enhances the performance of text-to-image diffusion. CogView is the first model implementing relay diffusion in the realm of text-to-image generation, executing the task by first creating low-resolution images and subsequently applying relay-based super-resolution.", "preview": "THUDM--CogView3-Plus-3B.jpg", "skip": true diff --git a/modules/shared.py b/modules/shared.py index 55fc32010..25ca83b92 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -248,7 +248,7 @@ options_templates.update(options_section(('text_encoder', "Text Encoder"), { "diffusers_zeros_prompt_pad": OptionInfo(False, "Use zeros for prompt padding", gr.Checkbox), "te_hijack": OptionInfo(True, "Offload after prompt encode", gr.Checkbox), "te_optional_sep": OptionInfo("

Optional

", "", gr.HTML), - "te_shared_t5": OptionInfo(False, "T5: Use shared instance of text encoder"), + "te_shared_t5": OptionInfo(True, "T5: Use shared instance of text encoder"), "te_pooled_embeds": OptionInfo(False, "SDXL: Use weighted pooled embeds"), "te_complex_human_instruction": OptionInfo(True, "Sana: Use complex human instructions"), "te_use_mask": OptionInfo(True, "Lumina: Use mask in transformers"), diff --git a/pipelines/generic.py b/pipelines/generic.py index 08fe00275..7b9039413 100644 --- a/pipelines/generic.py +++ b/pipelines/generic.py @@ -8,8 +8,8 @@ from modules import shared, devices, sd_models, model_quant debug = shared.log.trace if os.environ.get('SD_LOAD_DEBUG', None) is not None else lambda *args, **kwargs: None -def load_transformer(repo_id, cls_name, load_config={}, subfolder="transformer"): - load_args, quant_args = model_quant.get_dit_args(load_config, module='Model', device_map=True) +def load_transformer(repo_id, cls_name, load_config={}, subfolder="transformer", allow_quant=True): + load_args, quant_args = model_quant.get_dit_args(load_config, module='Model', device_map=True, allow_quant=allow_quant) quant_type = model_quant.get_quant_type(quant_args) local_file = None @@ -56,8 +56,8 @@ def load_transformer(repo_id, cls_name, load_config={}, subfolder="transformer") return transformer -def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder"): - load_args, quant_args = model_quant.get_dit_args(load_config, module='TE', device_map=True) +def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder", allow_quant=True, allow_shared=True): + load_args, quant_args = model_quant.get_dit_args(load_config, module='TE', device_map=True, allow_quant=allow_quant) quant_type = model_quant.get_quant_type(quant_args) text_encoder = None @@ -92,7 +92,7 @@ def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder ) text_encoder = model_quant.do_post_load_quant(text_encoder, allow=quant_type is not None) # use shared t5 if possible - elif cls_name == transformers.T5EncoderModel: + elif cls_name == transformers.T5EncoderModel and allow_shared: with open(os.path.join('configs', 'flux', 'text_encoder_2', 'config.json'), encoding='utf8') as f: load_args['config'] = transformers.T5Config(**json.load(f)) if model_quant.check_nunchaku('TE'): @@ -114,7 +114,7 @@ def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder **load_args, **quant_args, ) - + # load from repo if text_encoder is None: shared.log.debug(f'Load model: text_encoder="{repo_id}" cls={cls_name.__name__} quant="{quant_type}" shared={shared.opts.te_shared_t5}') diff --git a/pipelines/model_auraflow.py b/pipelines/model_auraflow.py index c6f2ade77..06cc978ce 100644 --- a/pipelines/model_auraflow.py +++ b/pipelines/model_auraflow.py @@ -1,6 +1,6 @@ import transformers import diffusers -from modules import shared, sd_models, devices, model_quant +from modules import shared, sd_models, devices, model_quant, sd_hijack_te from pipelines import generic @@ -8,21 +8,22 @@ def load_auraflow(checkpoint_info, diffusers_load_config={}): repo_id = sd_models.path_to_repo(checkpoint_info) sd_models.hf_auth_check(checkpoint_info) - load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False) + load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config) shared.log.debug(f'Load model: type=AuraFlow repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') transformer = generic.load_transformer(repo_id, cls_name=diffusers.AuraFlowTransformer2DModel, load_config=diffusers_load_config) - text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.T5EncoderModel, load_config=diffusers_load_config) + text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.UMT5EncoderModel, load_config=diffusers_load_config, allow_quant=False) # auraflow uses EleutherAI/pile-t5-xl pipe = diffusers.AuraFlowPipeline.from_pretrained( repo_id, transformer=transformer, - text_encoder=text_encoder, + # text_encoder=text_encoder, cache_dir=shared.opts.diffusers_dir, **load_args, ) del text_encoder del transformer + sd_hijack_te.init_hijack(pipe) devices.torch_gc(force=True, reason='load') return pipe diff --git a/pipelines/model_hidream.py b/pipelines/model_hidream.py index a5c18d3bc..6d1a716b8 100644 --- a/pipelines/model_hidream.py +++ b/pipelines/model_hidream.py @@ -2,6 +2,7 @@ import os import transformers import diffusers from modules import shared, devices, sd_models, model_quant, sd_hijack_te +from pipelines import generic def load_llama(repo_id, diffusers_load_config={}): @@ -37,7 +38,7 @@ def load_hidream(checkpoint_info, diffusers_load_config={}): transformer = generic.load_transformer(repo_id, cls_name=diffusers.HiDreamImageTransformer2DModel, load_config=diffusers_load_config, subfolder="transformer") text_encoder_3 = generic.load_text_encoder(repo_id, cls_name=transformers.T5EncoderModel, load_config=diffusers_load_config, subfolder="text_encoder_3") - text_encoder_4, tokenizer_4 = load_text_encoders(repo_id, diffusers_load_config) + text_encoder_4, tokenizer_4 = load_llama(repo_id, diffusers_load_config) if shared.opts.teacache_enabled: from modules import teacache diff --git a/wiki b/wiki index 96e3932bf..0b658ebad 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 96e3932bffee9117951956074956e943f254702b +Subproject commit 0b658ebad7ad81ebb9cd131abf409878d9c8744f