Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 17:24:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(pipelines): wire native_spec for arches without from_single_file

Browse Source 
CogView3+, CogView4, GLM-Image, HunyuanDiT, Joy, Kandinsky3 UNet,
PixArt, PRX. These classes lack from_single_file entirely; the UNET
dropdown fell through to cls.from_pretrained(local_file), which
decodes the .safetensors header as UTF-8 and crashes "is not a valid
JSON file".

TransformerSpec(cls=...) module-level in each model_<arch>.py.
Kandinsky3 also needs subfolder='unet' since it loads a UNet not a
transformer.
This commit is contained in:
CalamitousFelicitousness
2026-05-25 05:47:56 +01:00
parent e7d2e28d3d
commit e302518d73
7 changed files with 35 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipelines/model_cogview.py
+7 -2
View File
@@ -3,6 +3,11 @@ import diffusers
from modules import shared, devices, sd_models, model_quant, sd_hijack_te
from modules.logger import log
from pipelines import generic
from pipelines.native_transformer import TransformerSpec
COGVIEW3_SPEC = TransformerSpec(cls=diffusers.CogView3PlusTransformer2DModel)
COGVIEW4_SPEC = TransformerSpec(cls=diffusers.CogView4Transformer2DModel)
def load_cogview3(checkpoint_info, diffusers_load_config=None):
@@ -14,7 +19,7 @@ def load_cogview3(checkpoint_info, diffusers_load_config=None):
load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config)
log.debug(f'Load model: type=CogView3 repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')
transformer = generic.load_transformer(repo_id, cls_name=diffusers.CogView3PlusTransformer2DModel, load_config=diffusers_load_config, subfolder="transformer")
transformer = generic.load_transformer(repo_id, cls_name=diffusers.CogView3PlusTransformer2DModel, load_config=diffusers_load_config, subfolder="transformer", native_spec=COGVIEW3_SPEC)
text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.T5EncoderModel, load_config=diffusers_load_config, subfolder="text_encoder")
pipe = diffusers.CogView3PlusPipeline.from_pretrained(
@@ -40,7 +45,7 @@ def load_cogview4(checkpoint_info, diffusers_load_config=None):
load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config)
log.debug(f'Load model: type=CogView4 repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')
transformer = generic.load_transformer(repo_id, cls_name=diffusers.CogView4Transformer2DModel, load_config=diffusers_load_config, subfolder="transformer")
transformer = generic.load_transformer(repo_id, cls_name=diffusers.CogView4Transformer2DModel, load_config=diffusers_load_config, subfolder="transformer", native_spec=COGVIEW4_SPEC)
text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.GlmModel, load_config=diffusers_load_config, subfolder="text_encoder", allow_quant=True)
pipe = diffusers.CogView4Pipeline.from_pretrained(
pipelines/model_glm.py
+6 -1
View File
@@ -5,6 +5,10 @@ import diffusers
from modules import shared, devices, sd_models, model_quant, sd_hijack_te
from modules.logger import log, console
from pipelines import generic
from pipelines.native_transformer import TransformerSpec
GLM_IMAGE_SPEC = TransformerSpec(cls=diffusers.GlmImageTransformer2DModel)
class GLMTokenProgressProcessor(transformers.LogitsProcessor):
@@ -98,7 +102,8 @@ def load_glm_image(checkpoint_info, diffusers_load_config=None):
transformer = generic.load_transformer(
repo_id,
cls_name=diffusers.GlmImageTransformer2DModel,
load_config=diffusers_load_config
load_config=diffusers_load_config,
native_spec=GLM_IMAGE_SPEC,
)
# Load text encoder (ByT5 for glyph) - cannot use shared T5 as GLM-Image requires specific ByT5 encoder (1472 hidden size)
pipelines/model_hunyuandit.py
+5 -1
View File
@@ -3,6 +3,10 @@ import diffusers
from modules import shared, sd_models, devices, model_quant
from modules.logger import log
from pipelines import generic
from pipelines.native_transformer import TransformerSpec
HUNYUANDIT_SPEC = TransformerSpec(cls=diffusers.HunyuanDiT2DModel)
def load_hunyuandit(checkpoint_info, diffusers_load_config=None):
@@ -19,7 +23,7 @@ def load_hunyuandit(checkpoint_info, diffusers_load_config=None):
load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config)
log.debug(f'Load model: type=HunyuanDiT repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')
transformer = generic.load_transformer(repo_id, cls_name=diffusers.HunyuanDiT2DModel, load_config=diffusers_load_config)
transformer = generic.load_transformer(repo_id, cls_name=diffusers.HunyuanDiT2DModel, load_config=diffusers_load_config, native_spec=HUNYUANDIT_SPEC)
repo_te = 'Tencent-Hunyuan/HunyuanDiT-v1.2-Diffusers' if 'HunyuanDiT-v1' in repo_id else repo_id
text_encoder_2 = generic.load_text_encoder(repo_te, cls_name=transformers.T5EncoderModel, load_config=diffusers_load_config, subfolder="text_encoder_2", allow_shared=False) # this is not normal t5
pipelines/model_joy.py
+5
View File
@@ -3,6 +3,10 @@ import transformers
from modules import shared, devices, sd_models, model_quant, sd_hijack_te, sd_hijack_vae
from modules.logger import log
from pipelines import generic
from pipelines.native_transformer import TransformerSpec
JOY_SPEC = TransformerSpec(cls=diffusers.JoyImageEditTransformer3DModel)
def load_joy(checkpoint_info, diffusers_load_config=None):
@@ -18,6 +22,7 @@ def load_joy(checkpoint_info, diffusers_load_config=None):
repo_id,
cls_name=diffusers.JoyImageEditTransformer3DModel,
load_config=diffusers_load_config,
native_spec=JOY_SPEC,
)
text_encoder = generic.load_text_encoder(
repo_id,
pipelines/model_kandinsky.py
+2 -1
View File
@@ -6,6 +6,7 @@ from pipelines import generic
from pipelines.native_transformer import TransformerSpec
KANDINSKY3_UNET_SPEC = TransformerSpec(cls=diffusers.Kandinsky3UNet, subfolder='unet')
KANDINSKY5_SPEC = TransformerSpec(cls=diffusers.Kandinsky5Transformer3DModel)
@@ -54,7 +55,7 @@ def load_kandinsky3(checkpoint_info, diffusers_load_config=None):
load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config)
log.debug(f'Load model: type=Kandinsky30 repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')
unet = generic.load_transformer(repo_id, cls_name=diffusers.Kandinsky3UNet, load_config=diffusers_load_config, subfolder="unet", variant="fp16")
unet = generic.load_transformer(repo_id, cls_name=diffusers.Kandinsky3UNet, load_config=diffusers_load_config, subfolder="unet", variant="fp16", native_spec=KANDINSKY3_UNET_SPEC)
text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.T5EncoderModel, load_config=diffusers_load_config, subfolder="text_encoder", variant="fp16", allow_shared=False)
pipe = diffusers.Kandinsky3Pipeline.from_pretrained(
pipelines/model_pixart.py
+5 -1
View File
@@ -4,6 +4,10 @@ from huggingface_hub import file_exists
from modules import shared, devices, sd_models, model_quant, sd_hijack_te
from modules.logger import log
from pipelines import generic
from pipelines.native_transformer import TransformerSpec
PIXART_SPEC = TransformerSpec(cls=diffusers.PixArtTransformer2DModel)
def load_pixart(checkpoint_info, diffusers_load_config=None):
@@ -24,7 +28,7 @@ def load_pixart(checkpoint_info, diffusers_load_config=None):
load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False)
log.debug(f'Load model: type=PixArtSigma repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')
transformer = generic.load_transformer(repo_id, cls_name=diffusers.PixArtTransformer2DModel, load_config=diffusers_load_config)
transformer = generic.load_transformer(repo_id, cls_name=diffusers.PixArtTransformer2DModel, load_config=diffusers_load_config, native_spec=PIXART_SPEC)
text_encoder = generic.load_text_encoder(repo_id_tenc, cls_name=transformers.T5EncoderModel, load_config=diffusers_load_config)
pipe = diffusers.PixArtSigmaPipeline.from_pretrained(
pipelines/model_prx.py
+5 -1
View File
@@ -2,6 +2,10 @@ import diffusers
from modules import shared, devices, sd_models, model_quant, sd_hijack_te
from modules.logger import log
from pipelines import generic
from pipelines.native_transformer import TransformerSpec
PRX_SPEC = TransformerSpec(cls=diffusers.PRXTransformer2DModel)
def load_prx(checkpoint_info, diffusers_load_config=None):
@@ -14,7 +18,7 @@ def load_prx(checkpoint_info, diffusers_load_config=None):
log.debug(f'Load model: type=PRX repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')
from transformers.models.t5gemma.modeling_t5gemma import T5GemmaEncoder
transformer = generic.load_transformer(repo_id, cls_name=diffusers.PRXTransformer2DModel, load_config=diffusers_load_config)
transformer = generic.load_transformer(repo_id, cls_name=diffusers.PRXTransformer2DModel, load_config=diffusers_load_config, native_spec=PRX_SPEC)
text_encoder = generic.load_text_encoder(repo_id, cls_name=T5GemmaEncoder, load_config=diffusers_load_config)
pipe = diffusers.PRXPipeline.from_pretrained(