diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a61a55e9..43175fdc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ Plus continued work on modernization of codebase: UI is now fully TypeScript bas oh, that 12B encoder is MoE with 3.6B activated plus its prequantized using `mxfp4` *note* Lens comes with its own prompt-refiner, enable in settings -> model options (disabled by default) *note* original Lens implements only text-2-image, SD.Next adds image-2-image and inpaint workflows as well + - [Ideogram 4](https://huggingface.co/ideogram-ai/ideogram-4) open-weight 9.3B flow-matching single-stream DiT + Qwen3-VL-8B text encoder (shared and deduped) and Flux2 VAE, with dual-transformer asymmetric CFG + converted to a bf16-Diffusers repo with SDNQ-at-load + *note* requires structured JSON-caption prompts, a plain-text prompt returns the model's built-in safety placeholder - **Features** - **SDNQ** new quantization algorithm: *Hadamard Rotations* much higher quality than base SDNQ, but runs slightly slower diff --git a/data/reference.json b/data/reference.json index b28af9c3d..ebef80479 100644 --- a/data/reference.json +++ b/data/reference.json @@ -184,6 +184,14 @@ "size": 20.3, "date": "2025 November" }, + "Ideogram 4": { + "path": "CalamitousFelicitousness/Ideogram-4-bf16-Diffusers", + "desc": "Ideogram 4 is Ideogram's first open-weight text-to-image model: a 9.3B flow-matching single-stream DiT that uses a Qwen3-VL vision-language model as its text encoder, with strong in-image text rendering. Requires structured JSON-caption prompts; a plain-text prompt returns a built-in safety placeholder. Non-commercial license.", + "skip": true, + "extras": "sampler: Default, cfg_scale: 7.0, steps: 20, width: 1024, height: 1024", + "size": 50.0, + "date": "2026 June" + }, "Baidu ERNIE-Image": { "path": "baidu/ERNIE-Image", "preview": "baidu--ERNIE-Image.jpg", diff --git a/installer.py b/installer.py index 51ea09b65..cfcdcfce4 100644 --- a/installer.py +++ b/installer.py @@ -533,7 +533,7 @@ def check_diffusers(): t_start = time.time() if args.skip_all: return - target_commit = "ed0711878cfed79aba2e4cf0712b2fcd8bead577" # diffusers commit hash == 0.39.0.dev0 == 06-02-2026 + target_commit = "9b0818cf87413b4b9ca2501bf49406eed6d881af" # diffusers commit hash == 0.39.0.dev0 == 06-03-2026 (adds Ideogram 4) # if args.use_rocm or args.use_zluda or args.use_directml: # sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now pkg = package_spec('diffusers') diff --git a/modules/modeldata.py b/modules/modeldata.py index 8ca37098f..75162eecd 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -55,6 +55,8 @@ def get_model_type(pipe): model_type = 'f1' elif "ZImage" in name or "Z-Image" in name: model_type = 'zimage' + elif "Ideogram4" in name: + model_type = 'ideogram4' elif "LuminaDiMOO" in name: model_type = 'luminadimoo' elif "Lumina2" in name: diff --git a/modules/processing.py b/modules/processing.py index 56d6da3f1..d73ea8973 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -278,7 +278,7 @@ def process_init(p: StableDiffusionProcessing): else: p.all_subseeds = [int(subseed) + x for x in range(len(p.all_prompts))] if reset_prompts: - if not hasattr(p, 'keep_prompts'): + if not hasattr(p, 'keep_prompts') and not getattr(shared.sd_model, 'keep_prompts', False): p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts(p.all_prompts, p.all_negative_prompts, p.styles, p.all_seeds, p=p) p.prompts = p.all_prompts[(p.iteration * p.batch_size):((p.iteration+1) * p.batch_size)] p.negative_prompts = p.all_negative_prompts[(p.iteration * p.batch_size):((p.iteration+1) * p.batch_size)] diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index c14576675..0df281c47 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -173,6 +173,26 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict | No current_noise_pred = current_noise_pred.view(b, h_patches, w_patches, channels, 2, 2) current_noise_pred = current_noise_pred.permute(0, 3, 1, 4, 2, 5).reshape(b, channels, h_patches * 2, w_patches * 2) shared.state.current_noise_pred = current_noise_pred + elif 'Ideogram4' in pipe.__class__.__name__: # packed normalized [B, seq, 128] -> Flux.2 latent space for TAE FLUX.2 + latents = kwargs['latents'] + if latents.ndim == 3: + b, seq_len, packed_ch = latents.shape + vae_scale = getattr(pipe, 'vae_scale_factor', 8) + patch = getattr(pipe, 'patch_size', 2) + grid_h = getattr(p, 'height', 1024) // (vae_scale * patch) + grid_w = getattr(p, 'width', 1024) // (vae_scale * patch) + if grid_h * grid_w != seq_len: # fallback to square assumption + grid_h = grid_w = int(seq_len ** 0.5) + bn = pipe.vae.bn + mean = bn.running_mean.view(1, 1, -1).to(device=latents.device, dtype=torch.float32) + std = torch.sqrt(bn.running_var + pipe.vae.config.batch_norm_eps).view(1, 1, -1).to(device=latents.device, dtype=torch.float32) + z = latents.float() * std + mean + ae_ch = packed_ch // (patch * patch) + z = z.view(b, grid_h, grid_w, patch, patch, ae_ch).permute(0, 5, 1, 3, 2, 4).reshape(b, ae_ch, grid_h * patch, grid_w * patch) + shared.state.current_latent = z + else: + shared.state.current_latent = latents + shared.state.current_noise_pred = current_noise_pred else: shared.state.current_latent = kwargs['latents'] shared.state.current_noise_pred = current_noise_pred diff --git a/modules/sd_detect.py b/modules/sd_detect.py index 84d793876..f87d7ae25 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -151,6 +151,8 @@ def guess_by_name(fn, current_guess): new_guess = 'NucleusImage' elif 'z-image' in fn.lower() or 'z_image' in fn.lower(): new_guess = 'ZImage' + elif 'ideogram' in fn.lower(): + new_guess = 'Ideogram4' elif 'longcat-image' in fn.lower(): new_guess = 'LongCat' elif 'ovis-image' in fn.lower(): diff --git a/modules/sd_models.py b/modules/sd_models.py index 935e07361..5c5444104 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -543,6 +543,10 @@ def load_diffuser_force(detected_model_type: str, checkpoint_info: CheckpointInf from pipelines.model_z_image import load_z_image sd_model = load_z_image(checkpoint_info, diffusers_load_config) allow_post_quant = False + elif model_type in ['Ideogram4']: + from pipelines.model_ideogram4 import load_ideogram4 + sd_model = load_ideogram4(checkpoint_info, diffusers_load_config) + allow_post_quant = False elif model_type in ['LongCat']: from pipelines.model_longcat import load_longcat sd_model = load_longcat(checkpoint_info, diffusers_load_config) diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 151e5a174..7bbae7eda 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -11,7 +11,7 @@ from modules.image import convert SamplerData = namedtuple('SamplerData', ['name', 'constructor', 'aliases', 'options']) approximation_indexes = { "Simple": 0, "Approximate": 1, "TAESD": 2, "Full VAE": 3 } -flow_models = ['f1', 'f2', 'sd3', 'lumina', 'auraflow', 'sana', 'zimage', 'lumina2', 'cogview4', 'h1', 'cosmos', 'anima', 'chroma', 'omnigen', 'omnigen2', 'longcat'] +flow_models = ['f1', 'f2', 'sd3', 'lumina', 'auraflow', 'sana', 'zimage', 'lumina2', 'cogview4', 'h1', 'cosmos', 'anima', 'chroma', 'omnigen', 'omnigen2', 'longcat', 'ideogram4'] warned = False queue_lock = threading.Lock() diff --git a/modules/shared_items.py b/modules/shared_items.py index 4fd642458..613908ea7 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -69,6 +69,7 @@ pipelines = { 'FLEX': None, 'HiDreamO1': None, 'HunyuanImage3': None, + 'Ideogram4': None, 'Lens': None, 'LuminaDiMOO': None, 'Meissonic': None, diff --git a/modules/vae/sd_vae_taesd.py b/modules/vae/sd_vae_taesd.py index cbae08815..b661205e7 100644 --- a/modules/vae/sd_vae_taesd.py +++ b/modules/vae/sd_vae_taesd.py @@ -71,7 +71,7 @@ def get_model(model_cls, variant=None): elif model_cls in {'f1', 'h1', 'zimage', 'lumina2', 'chroma', 'longcat', 'omnigen2', 'flite', 'ovis', 'kandinsky5', 'glmimage', 'cogview3', 'cogview4', 'ultraflux'}: model_cls = 'f1' variant = 'TAE FLUX.1' - elif model_cls in {'f2', 'ernieimage', 'lens'}: + elif model_cls in {'f2', 'ernieimage', 'lens', 'ideogram4'}: model_cls = 'f2' variant = 'TAE FLUX.2' elif model_cls in {'sd3'}: diff --git a/pipelines/generic_shared.py b/pipelines/generic_shared.py index 588c51602..069427049 100644 --- a/pipelines/generic_shared.py +++ b/pipelines/generic_shared.py @@ -1,5 +1,6 @@ import os import transformers +from transformers.models.qwen3_vl import Qwen3VLModel shared_te_map = { @@ -95,4 +96,9 @@ shared_te_map = { 'target_repo': 'vladmandic/Anima-1.0-Base', 'target_subfolder': 'text_encoder', }, + + 'Qwen3-VL 8B Base': { + 'cls': Qwen3VLModel, + 'target_repo': 'Qwen/Qwen3-VL-8B-Instruct', + }, } diff --git a/pipelines/model_ideogram4.py b/pipelines/model_ideogram4.py new file mode 100644 index 000000000..d77cdb6a0 --- /dev/null +++ b/pipelines/model_ideogram4.py @@ -0,0 +1,105 @@ +import json +import diffusers +from transformers.models.qwen3_vl import Qwen3VLModel +from modules import shared, devices, sd_models, model_quant +from modules.logger import log +from pipelines import generic + + +def prompt_to_json(prompt): + """Normalize a JSON caption to the compact form Ideogram 4 trained on, or wrap plain text. + + Ideogram 4 expects a structured JSON caption serialized compactly. A valid JSON prompt is + re-serialized to that form; a plain-text prompt is wrapped in a minimal caption so it stays + in distribution instead of tripping the weight-baked "blocked by safety filter" placeholder. + """ + if isinstance(prompt, list): + return [prompt_to_json(p) for p in prompt] + if not isinstance(prompt, str) or len(prompt) == 0: + return prompt + try: + return json.dumps(json.loads(prompt), ensure_ascii=False, separators=(',', ':')) + except ValueError: + caption = {'high_level_description': prompt, 'compositional_deconstruction': {'background': prompt, 'elements': []}} + return json.dumps(caption, ensure_ascii=False, separators=(',', ':')) + + +class Ideogram4Pipeline(diffusers.Ideogram4Pipeline): + """SD.Next integration subclass for the diffusers-native Ideogram 4 pipeline. + + ``encode_prompt`` normalizes the prompt into the structured JSON the model expects, then + drives the Qwen3-VL tap. The tap calls ``language_model`` submodules directly, bypassing the + balanced-offload pre-forward hook, so the encoder is moved on-device for it and released after. + """ + + def encode_prompt(self, prompt, *args, **kwargs): + prompt = prompt_to_json(prompt) + self.text_encoder.to(self._execution_device) + try: + return super().encode_prompt(prompt, *args, **kwargs) + finally: + if shared.opts.diffusers_offload_mode != 'none': + self.text_encoder.to(devices.cpu) + + +def pin_transformers_if_fit(transformer, unconditional_transformer) -> bool: + """Keep both transformers resident under balanced offload when they fit the budget. + + Every denoise step runs both transformers, so balanced offload ping-pongs them across + PCIe each step. ``offload_never`` makes ``offload_allowed`` skip the per-step pre-sweep so + they stay resident, but only when both fit ``gpu_memory * max watermark`` (which leaves the + watermark headroom for activations); otherwise the normal offload path is kept. + """ + if shared.opts.diffusers_offload_mode != 'balanced' or shared.gpu_memory <= 0: + return False + if transformer is None or unconditional_transformer is None: + return False + size_gb = sum(p.numel() * p.element_size() for m in (transformer, unconditional_transformer) for p in m.parameters()) / (1024 ** 3) + budget_gb = shared.gpu_memory * shared.opts.diffusers_offload_max_gpu_memory + fits = size_gb <= budget_gb + if fits: + transformer.offload_never = True + unconditional_transformer.offload_never = True + log.info(f'Load model: type=Ideogram4 offload=balanced transformers={size_gb:.1f} budget={budget_gb:.1f} action={"pin-resident" if fits else "offload"}') + return fits + + +def load_ideogram4(checkpoint_info, diffusers_load_config=None): + if diffusers_load_config is None: + diffusers_load_config = {} + repo_id = sd_models.path_to_repo(checkpoint_info) + sd_models.hf_auth_check(checkpoint_info) + load_args, _ = model_quant.get_dit_args(diffusers_load_config, allow_quant=False) + log.debug(f'Load model: type=Ideogram4 repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') + + generic.set_pipeline('Ideogram4', Ideogram4Pipeline) + if repo_id is None or repo_id.lower() == 'none': + return None + + # Each transformer loads independently from its subfolder, so each gets its own SDNQ config. + cls = diffusers.Ideogram4Transformer2DModel + transformer = generic.load_transformer(repo_id, cls_name=cls, subfolder="transformer", load_config=diffusers_load_config) + unconditional_transformer = generic.load_transformer(repo_id, cls_name=cls, subfolder="unconditional_transformer", load_config=diffusers_load_config) + pin_transformers_if_fit(transformer, unconditional_transformer) + # shared_te_map redirects to the shared Qwen3-VL repo (deduped with VQA + prompt-enhance); + # the bundled text_encoder is the fallback when sharing is off. The vae, tokenizer, and + # scheduler load from the repo via from_pretrained. + text_encoder = generic.load_text_encoder(repo_id, cls_name=Qwen3VLModel, load_config=diffusers_load_config) + + pipe = Ideogram4Pipeline.from_pretrained( + repo_id, + cache_dir=shared.opts.diffusers_dir, + transformer=transformer, + unconditional_transformer=unconditional_transformer, + text_encoder=text_encoder, + **load_args, + ) + # The pipeline decodes internally; the CFG scale slider drives guidance_scale, which is + # mutually exclusive with the pipeline's default per-step guidance_schedule. + pipe.task_args = {'output_type': 'pil', 'guidance_schedule': None} # pylint: disable=attribute-defined-outside-init + # JSON captions must pass through verbatim; skip styles/wildcards that would strip the braces. + pipe.keep_prompts = True # pylint: disable=attribute-defined-outside-init + + del transformer, unconditional_transformer, text_encoder + devices.torch_gc(force=True, reason='load') + return pipe diff --git a/test/test-ideogram4-smoke.py b/test/test-ideogram4-smoke.py new file mode 100644 index 000000000..971671dff --- /dev/null +++ b/test/test-ideogram4-smoke.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +"""Standalone end-to-end smoke for diffusers-native Ideogram 4. + +Loads the split-projection bf16 diffusers folder and quantizes both transformers +with SDNQ at load (the same path SD.Next uses), loads the shared Qwen3-VL text +encoder, builds the diffusers ``Ideogram4Pipeline``, and generates an image. This +exercises the real pipeline: both transformers under SDNQ, the Qwen3-VL 13-layer +tap, the dual-branch asymmetric CFG loop, the logit-normal schedule, the vae.bn +latent denorm, and VAE decode. SDNQ int4 fits the two towers plus the encoder on a +24GB GPU. + +Usage: + python test/test-ideogram4-smoke.py --model /path/to/Ideogram-4-bf16-split --output out.png +""" + +import argparse +import os +import sys +import time + +parser = argparse.ArgumentParser() +parser.add_argument("--model", required=True, help="split-projection bf16 diffusers folder") +parser.add_argument("--output", default="ideogram4_smoke.png") +parser.add_argument("--prompt", default="a ginger cat wearing a tiny wizard hat reading a glowing spellbook, detailed digital illustration") +parser.add_argument("--height", type=int, default=1024) +parser.add_argument("--width", type=int, default=1024) +parser.add_argument("--steps", type=int, default=20) +parser.add_argument("--seed", type=int, default=0) +parser.add_argument("--weights-dtype", default="uint4", help="SDNQ weights dtype (uint4, int8, ...)") +parser.add_argument("--hf-cache", default=None, help="HF cache_dir for the shared Qwen3-VL encoder (default: HF default cache)") +args = parser.parse_args() + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) +os.chdir(REPO_ROOT) +os.environ["SD_INSTALL_QUIET"] = "1" + +# Our own args are already parsed; clear argv (and leave it cleared) so sdnext's +# shared.py / devices, which re-parse argv on import, don't see this script's flags. +sys.argv = [sys.argv[0]] + +import modules.cmd_args +import installer + +modules.cmd_args.parse_args() +installer.add_args(modules.cmd_args.parser) +modules.cmd_args.parsed, _ = modules.cmd_args.parser.parse_known_args([]) + +import torch +import diffusers +from transformers import AutoTokenizer +from transformers.models.qwen3_vl import Qwen3VLModel + +from modules import devices +from modules.sdnq import SDNQConfig + +TE_REPO = "Qwen/Qwen3-VL-8B-Instruct" + + +def main() -> int: + device = devices.device + cfg = SDNQConfig(weights_dtype=args.weights_dtype) + + print(f"loading transformer (sdnq {args.weights_dtype}) ...", flush=True) + transformer = diffusers.Ideogram4Transformer2DModel.from_pretrained(args.model, subfolder="transformer", quantization_config=cfg, torch_dtype=torch.bfloat16).to(device) + print("loading unconditional_transformer ...", flush=True) + uncond = diffusers.Ideogram4Transformer2DModel.from_pretrained(args.model, subfolder="unconditional_transformer", quantization_config=cfg, torch_dtype=torch.bfloat16).to(device) + print("loading text encoder Qwen3-VL ...", flush=True) + te_kwargs = {"cache_dir": args.hf_cache} if args.hf_cache else {} + text_encoder = Qwen3VLModel.from_pretrained(TE_REPO, quantization_config=SDNQConfig(weights_dtype=args.weights_dtype), torch_dtype=torch.bfloat16, **te_kwargs).to(device) + tokenizer = AutoTokenizer.from_pretrained(args.model, subfolder="tokenizer") + print("loading vae ...", flush=True) + vae = diffusers.AutoencoderKLFlux2.from_pretrained(args.model, subfolder="vae", torch_dtype=torch.bfloat16).to(device) + scheduler = diffusers.FlowMatchEulerDiscreteScheduler.from_pretrained(args.model, subfolder="scheduler") + + pipe = diffusers.Ideogram4Pipeline( + scheduler=scheduler, + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + transformer=transformer, + unconditional_transformer=uncond, + ) + + generator = torch.Generator(device=device).manual_seed(args.seed) + print(f"generating {args.width}x{args.height} steps={args.steps} ...", flush=True) + start = time.time() + out = pipe(prompt=args.prompt, num_inference_steps=args.steps, guidance_scale=7.0, guidance_schedule=None, width=args.width, height=args.height, generator=generator) + elapsed = time.time() - start + + image = out.images[0] + image.save(args.output) + if torch.cuda.is_available(): + print(f"peak VRAM: {torch.cuda.max_memory_allocated() / 1e9:.1f} GB", flush=True) + print(f"PASS: generated {args.output} in {elapsed:.1f}s, size={image.size}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main())