mirror of
https://github.com/vladmandic/automatic
synced 2026-09-18 16:54:33 +02:00
update cogview links, fix auraflow and hidream loaders
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
+2
-2
@@ -139,7 +139,7 @@ Feature highlights include:
|
||||
- [ModernUI](https://github.com/user-attachments/assets/6f156154-0b0a-4be2-94f0-979e9f679501) has quite some redesign which should make it more user friendly and easier to navigate plus several new UI themes
|
||||
If you're still using **StandardUI**, give [ModernUI](https://vladmandic.github.io/sdnext-docs/Themes/) a try!
|
||||
- New models such as [WanAI 2.2](https://wan.video/) in 5B and A14B variants for both *text-to-video* and *image-to-video* workflows as well as *text-to-image* workflow!
|
||||
and also [FreePix F-Lite](https://huggingface.co/Freepik/F-Lite), [Bria 3.2](https://huggingface.co/briaai/BRIA-3.2) and [bigASP 2.5](https://civitai.com/models/1789765?modelVersionId=2025412)
|
||||
and also [FreePik F-Lite](https://huggingface.co/Freepik/F-Lite), [Bria 3.2](https://huggingface.co/briaai/BRIA-3.2) and [bigASP 2.5](https://civitai.com/models/1789765?modelVersionId=2025412)
|
||||
- Redesigned [Video](https://vladmandic.github.io/sdnext-docs/Video) interface with support for general video models plus optimized [FramePack](https://vladmandic.github.io/sdnext-docs/FramePack) and [LTXVideo](https://vladmandic.github.io/sdnext-docs/LTX) support
|
||||
- Fully integrated nudity detection and optional censorship with [NudeNet](https://vladmandic.github.io/sdnext-docs/NudeNet)
|
||||
- New background replacement and relightning methods using **Latent Bridge Matching** and new **PixelArt** processing filter
|
||||
@@ -184,7 +184,7 @@ For details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/master
|
||||
can also load unet/transformer fine-tunes in safetensors format using UNET loader
|
||||
simply select in *networks -> models -> reference*
|
||||
*note* 1.3B model is a bit too small for good results and 14B is very large at 78GB even without second-stage so aggressive quantization and offloading are recommended
|
||||
- [FreePix F-Lite](https://huggingface.co/Freepik/F-Lite) in *7B, 10B and Texture* variants
|
||||
- [FreePik F-Lite](https://huggingface.co/Freepik/F-Lite) in *7B, 10B and Texture* variants
|
||||
F-Lite is a 7B/10B model trained exclusively on copyright-safe and SFW content, trained on internal dataset comprising approximately 80 million copyright-safe images
|
||||
available via *networks -> models -> reference*
|
||||
- [Bria 3.2](https://huggingface.co/briaai/BRIA-3.2)
|
||||
|
||||
@@ -4,6 +4,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
|
||||
|
||||
## Future Candidates
|
||||
|
||||
- Unified `CLIPTextModelWithProjection` loader
|
||||
- [Modular pipelines and guiders](https://github.com/huggingface/diffusers/issues/11915)
|
||||
- Refactor: Sampler options
|
||||
- Feature: Diffusers [group offloading](https://github.com/vladmandic/sdnext/issues/4049)
|
||||
|
||||
+13
-3
@@ -1,4 +1,9 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
fal/AuraFlow-v0.3: sdnq unusupported
|
||||
|
||||
"""
|
||||
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
@@ -27,8 +32,8 @@ models = [
|
||||
"stabilityai/stable-diffusion-3.5-medium",
|
||||
"stabilityai/stable-diffusion-3.5-large",
|
||||
"fal/AuraFlow-v0.3",
|
||||
"THUDM/CogView3-Plus-3B",
|
||||
"THUDM/CogView4-6B",
|
||||
"zai-org/CogView4-6B",
|
||||
"zai-org/CogView3-Plus-3B",
|
||||
"nvidia/Cosmos-Predict2-2B-Text2Image",
|
||||
"nvidia/Cosmos-Predict2-14B-Text2Image",
|
||||
"Qwen/Qwen-Image",
|
||||
@@ -70,8 +75,11 @@ styles_tbd = [
|
||||
'Fixed Kneeling on Bed',
|
||||
'Fixed Girl in Sin City',
|
||||
'Fixed Girl in a city',
|
||||
'Fixed Girl in Lace',
|
||||
'Fixed Lady in Tokyo',
|
||||
'Fixed MadMax selfie',
|
||||
'Fixed Party Yacht',
|
||||
'Fixed Yoga Girls',
|
||||
'Fixed SDNext Neon',
|
||||
]
|
||||
|
||||
@@ -116,8 +124,10 @@ def generate(): # pylint: disable=redefined-outer-name
|
||||
b64 = data['images'][0].split(',',1)[0]
|
||||
image = Image.open(io.BytesIO(base64.b64decode(b64)))
|
||||
info = data['info']
|
||||
log.info(f' image: size={image.size} time={t1-t0:.2f} info="{len(info)}" fn="{fn}"')
|
||||
log.info(f' image: size={image.width}x{image.height} time={t1-t0:.2f} info={len(info)}')
|
||||
image.save(fn)
|
||||
else:
|
||||
log.error(f' model: error="{model}" style="{style}" no image')
|
||||
except Exception as e:
|
||||
log.error(f' model: error="{model}" style="{style}" exception="{e}"')
|
||||
|
||||
|
||||
+2
-2
@@ -520,13 +520,13 @@
|
||||
},
|
||||
|
||||
"CogView 4": {
|
||||
"path": "THUDM/CogView4-6B",
|
||||
"path": "zai-org/CogView4-6B",
|
||||
"desc": "An innovative cascaded framework that enhances the performance of text-to-image diffusion. CogView is the first model implementing relay diffusion in the realm of text-to-image generation, executing the task by first creating low-resolution images and subsequently applying relay-based super-resolution.",
|
||||
"preview": "THUDM--CogView4-6B.jpg",
|
||||
"skip": true
|
||||
},
|
||||
"CogView 3 Plus": {
|
||||
"path": "THUDM/CogView3-Plus-3B",
|
||||
"path": "zai-org/CogView3-Plus-3B",
|
||||
"desc": "An innovative cascaded framework that enhances the performance of text-to-image diffusion. CogView is the first model implementing relay diffusion in the realm of text-to-image generation, executing the task by first creating low-resolution images and subsequently applying relay-based super-resolution.",
|
||||
"preview": "THUDM--CogView3-Plus-3B.jpg",
|
||||
"skip": true
|
||||
|
||||
+1
-1
@@ -248,7 +248,7 @@ options_templates.update(options_section(('text_encoder', "Text Encoder"), {
|
||||
"diffusers_zeros_prompt_pad": OptionInfo(False, "Use zeros for prompt padding", gr.Checkbox),
|
||||
"te_hijack": OptionInfo(True, "Offload after prompt encode", gr.Checkbox),
|
||||
"te_optional_sep": OptionInfo("<h2>Optional</h2>", "", gr.HTML),
|
||||
"te_shared_t5": OptionInfo(False, "T5: Use shared instance of text encoder"),
|
||||
"te_shared_t5": OptionInfo(True, "T5: Use shared instance of text encoder"),
|
||||
"te_pooled_embeds": OptionInfo(False, "SDXL: Use weighted pooled embeds"),
|
||||
"te_complex_human_instruction": OptionInfo(True, "Sana: Use complex human instructions"),
|
||||
"te_use_mask": OptionInfo(True, "Lumina: Use mask in transformers"),
|
||||
|
||||
@@ -8,8 +8,8 @@ from modules import shared, devices, sd_models, model_quant
|
||||
debug = shared.log.trace if os.environ.get('SD_LOAD_DEBUG', None) is not None else lambda *args, **kwargs: None
|
||||
|
||||
|
||||
def load_transformer(repo_id, cls_name, load_config={}, subfolder="transformer"):
|
||||
load_args, quant_args = model_quant.get_dit_args(load_config, module='Model', device_map=True)
|
||||
def load_transformer(repo_id, cls_name, load_config={}, subfolder="transformer", allow_quant=True):
|
||||
load_args, quant_args = model_quant.get_dit_args(load_config, module='Model', device_map=True, allow_quant=allow_quant)
|
||||
quant_type = model_quant.get_quant_type(quant_args)
|
||||
|
||||
local_file = None
|
||||
@@ -56,8 +56,8 @@ def load_transformer(repo_id, cls_name, load_config={}, subfolder="transformer")
|
||||
return transformer
|
||||
|
||||
|
||||
def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder"):
|
||||
load_args, quant_args = model_quant.get_dit_args(load_config, module='TE', device_map=True)
|
||||
def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder", allow_quant=True, allow_shared=True):
|
||||
load_args, quant_args = model_quant.get_dit_args(load_config, module='TE', device_map=True, allow_quant=allow_quant)
|
||||
quant_type = model_quant.get_quant_type(quant_args)
|
||||
text_encoder = None
|
||||
|
||||
@@ -92,7 +92,7 @@ def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder
|
||||
)
|
||||
text_encoder = model_quant.do_post_load_quant(text_encoder, allow=quant_type is not None)
|
||||
# use shared t5 if possible
|
||||
elif cls_name == transformers.T5EncoderModel:
|
||||
elif cls_name == transformers.T5EncoderModel and allow_shared:
|
||||
with open(os.path.join('configs', 'flux', 'text_encoder_2', 'config.json'), encoding='utf8') as f:
|
||||
load_args['config'] = transformers.T5Config(**json.load(f))
|
||||
if model_quant.check_nunchaku('TE'):
|
||||
@@ -114,7 +114,7 @@ def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder
|
||||
**load_args,
|
||||
**quant_args,
|
||||
)
|
||||
|
||||
|
||||
# load from repo
|
||||
if text_encoder is None:
|
||||
shared.log.debug(f'Load model: text_encoder="{repo_id}" cls={cls_name.__name__} quant="{quant_type}" shared={shared.opts.te_shared_t5}')
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import transformers
|
||||
import diffusers
|
||||
from modules import shared, sd_models, devices, model_quant
|
||||
from modules import shared, sd_models, devices, model_quant, sd_hijack_te
|
||||
from pipelines import generic
|
||||
|
||||
|
||||
@@ -8,21 +8,22 @@ def load_auraflow(checkpoint_info, diffusers_load_config={}):
|
||||
repo_id = sd_models.path_to_repo(checkpoint_info)
|
||||
sd_models.hf_auth_check(checkpoint_info)
|
||||
|
||||
load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False)
|
||||
load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config)
|
||||
shared.log.debug(f'Load model: type=AuraFlow repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')
|
||||
|
||||
transformer = generic.load_transformer(repo_id, cls_name=diffusers.AuraFlowTransformer2DModel, load_config=diffusers_load_config)
|
||||
text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.T5EncoderModel, load_config=diffusers_load_config)
|
||||
text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.UMT5EncoderModel, load_config=diffusers_load_config, allow_quant=False) # auraflow uses EleutherAI/pile-t5-xl
|
||||
|
||||
pipe = diffusers.AuraFlowPipeline.from_pretrained(
|
||||
repo_id,
|
||||
transformer=transformer,
|
||||
text_encoder=text_encoder,
|
||||
# text_encoder=text_encoder,
|
||||
cache_dir=shared.opts.diffusers_dir,
|
||||
**load_args,
|
||||
)
|
||||
|
||||
del text_encoder
|
||||
del transformer
|
||||
sd_hijack_te.init_hijack(pipe)
|
||||
devices.torch_gc(force=True, reason='load')
|
||||
return pipe
|
||||
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
import transformers
|
||||
import diffusers
|
||||
from modules import shared, devices, sd_models, model_quant, sd_hijack_te
|
||||
from pipelines import generic
|
||||
|
||||
|
||||
def load_llama(repo_id, diffusers_load_config={}):
|
||||
@@ -37,7 +38,7 @@ def load_hidream(checkpoint_info, diffusers_load_config={}):
|
||||
|
||||
transformer = generic.load_transformer(repo_id, cls_name=diffusers.HiDreamImageTransformer2DModel, load_config=diffusers_load_config, subfolder="transformer")
|
||||
text_encoder_3 = generic.load_text_encoder(repo_id, cls_name=transformers.T5EncoderModel, load_config=diffusers_load_config, subfolder="text_encoder_3")
|
||||
text_encoder_4, tokenizer_4 = load_text_encoders(repo_id, diffusers_load_config)
|
||||
text_encoder_4, tokenizer_4 = load_llama(repo_id, diffusers_load_config)
|
||||
|
||||
if shared.opts.teacache_enabled:
|
||||
from modules import teacache
|
||||
|
||||
+1
-1
Submodule wiki updated: 96e3932bff...0b658ebad7
Reference in New Issue
Block a user