Merge pull request #4893 from vladmandic/feat/ideogram4

feat/Ideogram 4 support

CHANGELOG.md

@@ -25,6 +25,10 @@ Plus continued work on modernization of codebase: UI is now fully TypeScript bas
     oh, that 12B encoder is MoE with 3.6B activated plus its prequantized using `mxfp4`  
     *note* Lens comes with its own prompt-refiner, enable in settings -> model options (disabled by default)  
     *note* original Lens implements only text-2-image, SD.Next adds image-2-image and inpaint workflows as well  
+  - [Ideogram 4](https://huggingface.co/ideogram-ai/ideogram-4) open-weight 9.3B flow-matching single-stream DiT  
+    Qwen3-VL-8B text encoder (shared and deduped) and Flux2 VAE, with dual-transformer asymmetric CFG  
+    converted to a bf16-Diffusers repo with SDNQ-at-load  
+    *note* requires structured JSON-caption prompts, a plain-text prompt returns the model's built-in safety placeholder  
 - **Features**
   - **SDNQ** new quantization algorithm: *Hadamard Rotations*  
     much higher quality than base SDNQ, but runs slightly slower  

data/reference.json

@@ -184,6 +184,14 @@
     "size": 20.3,
     "date": "2025 November"
   },
+  "Ideogram 4": {
+    "path": "CalamitousFelicitousness/Ideogram-4-bf16-Diffusers",
+    "desc": "Ideogram 4 is Ideogram's first open-weight text-to-image model: a 9.3B flow-matching single-stream DiT that uses a Qwen3-VL vision-language model as its text encoder, with strong in-image text rendering. Requires structured JSON-caption prompts; a plain-text prompt returns a built-in safety placeholder. Non-commercial license.",
+    "skip": true,
+    "extras": "sampler: Default, cfg_scale: 7.0, steps: 20, width: 1024, height: 1024",
+    "size": 50.0,
+    "date": "2026 June"
+  },
   "Baidu ERNIE-Image": {
     "path": "baidu/ERNIE-Image",
     "preview": "baidu--ERNIE-Image.jpg",

installer.py

@@ -533,7 +533,7 @@ def check_diffusers():
     t_start = time.time()
     if args.skip_all:
         return
-    target_commit = "ed0711878cfed79aba2e4cf0712b2fcd8bead577" # diffusers commit hash == 0.39.0.dev0 == 06-02-2026
+    target_commit = "9b0818cf87413b4b9ca2501bf49406eed6d881af" # diffusers commit hash == 0.39.0.dev0 == 06-03-2026 (adds Ideogram 4)
     # if args.use_rocm or args.use_zluda or args.use_directml:
     #     sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now
     pkg = package_spec('diffusers')

modules/modeldata.py

@@ -55,6 +55,8 @@ def get_model_type(pipe):
         model_type = 'f1'
     elif "ZImage" in name or "Z-Image" in name:
         model_type = 'zimage'
+    elif "Ideogram4" in name:
+        model_type = 'ideogram4'
     elif "LuminaDiMOO" in name:
         model_type = 'luminadimoo'
     elif "Lumina2" in name:

modules/processing.py

@@ -278,7 +278,7 @@ def process_init(p: StableDiffusionProcessing):
         else:
             p.all_subseeds = [int(subseed) + x for x in range(len(p.all_prompts))]
     if reset_prompts:
-        if not hasattr(p, 'keep_prompts'):
+        if not hasattr(p, 'keep_prompts') and not getattr(shared.sd_model, 'keep_prompts', False):
             p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts(p.all_prompts, p.all_negative_prompts, p.styles, p.all_seeds, p=p)
             p.prompts = p.all_prompts[(p.iteration * p.batch_size):((p.iteration+1) * p.batch_size)]
             p.negative_prompts = p.all_negative_prompts[(p.iteration * p.batch_size):((p.iteration+1) * p.batch_size)]

modules/processing_callbacks.py

@@ -173,6 +173,26 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict | No
                 current_noise_pred = current_noise_pred.view(b, h_patches, w_patches, channels, 2, 2)
                 current_noise_pred = current_noise_pred.permute(0, 3, 1, 4, 2, 5).reshape(b, channels, h_patches * 2, w_patches * 2)
             shared.state.current_noise_pred = current_noise_pred
+        elif 'Ideogram4' in pipe.__class__.__name__:  # packed normalized [B, seq, 128] -> Flux.2 latent space for TAE FLUX.2
+            latents = kwargs['latents']
+            if latents.ndim == 3:
+                b, seq_len, packed_ch = latents.shape
+                vae_scale = getattr(pipe, 'vae_scale_factor', 8)
+                patch = getattr(pipe, 'patch_size', 2)
+                grid_h = getattr(p, 'height', 1024) // (vae_scale * patch)
+                grid_w = getattr(p, 'width', 1024) // (vae_scale * patch)
+                if grid_h * grid_w != seq_len:  # fallback to square assumption
+                    grid_h = grid_w = int(seq_len ** 0.5)
+                bn = pipe.vae.bn
+                mean = bn.running_mean.view(1, 1, -1).to(device=latents.device, dtype=torch.float32)
+                std = torch.sqrt(bn.running_var + pipe.vae.config.batch_norm_eps).view(1, 1, -1).to(device=latents.device, dtype=torch.float32)
+                z = latents.float() * std + mean
+                ae_ch = packed_ch // (patch * patch)
+                z = z.view(b, grid_h, grid_w, patch, patch, ae_ch).permute(0, 5, 1, 3, 2, 4).reshape(b, ae_ch, grid_h * patch, grid_w * patch)
+                shared.state.current_latent = z
+            else:
+                shared.state.current_latent = latents
+            shared.state.current_noise_pred = current_noise_pred
         else:
             shared.state.current_latent = kwargs['latents']
             shared.state.current_noise_pred = current_noise_pred

modules/sd_detect.py

@@ -151,6 +151,8 @@ def guess_by_name(fn, current_guess):
         new_guess = 'NucleusImage'
     elif 'z-image' in fn.lower() or 'z_image' in fn.lower():
         new_guess = 'ZImage'
+    elif 'ideogram' in fn.lower():
+        new_guess = 'Ideogram4'
     elif 'longcat-image' in fn.lower():
         new_guess = 'LongCat'
     elif 'ovis-image' in fn.lower():

modules/sd_models.py

@@ -543,6 +543,10 @@ def load_diffuser_force(detected_model_type: str, checkpoint_info: CheckpointInf
             from pipelines.model_z_image import load_z_image
             sd_model = load_z_image(checkpoint_info, diffusers_load_config)
             allow_post_quant = False
+        elif model_type in ['Ideogram4']:
+            from pipelines.model_ideogram4 import load_ideogram4
+            sd_model = load_ideogram4(checkpoint_info, diffusers_load_config)
+            allow_post_quant = False
         elif model_type in ['LongCat']:
             from pipelines.model_longcat import load_longcat
             sd_model = load_longcat(checkpoint_info, diffusers_load_config)

modules/sd_samplers_common.py

@@ -11,7 +11,7 @@ from modules.image import convert
 
 SamplerData = namedtuple('SamplerData', ['name', 'constructor', 'aliases', 'options'])
 approximation_indexes = { "Simple": 0, "Approximate": 1, "TAESD": 2, "Full VAE": 3 }
-flow_models = ['f1', 'f2', 'sd3', 'lumina', 'auraflow', 'sana', 'zimage', 'lumina2', 'cogview4', 'h1', 'cosmos', 'anima', 'chroma', 'omnigen', 'omnigen2', 'longcat']
+flow_models = ['f1', 'f2', 'sd3', 'lumina', 'auraflow', 'sana', 'zimage', 'lumina2', 'cogview4', 'h1', 'cosmos', 'anima', 'chroma', 'omnigen', 'omnigen2', 'longcat', 'ideogram4']
 warned = False
 queue_lock = threading.Lock()
 

modules/shared_items.py

@@ -69,6 +69,7 @@ pipelines = {
     'FLEX': None,
     'HiDreamO1': None,
     'HunyuanImage3': None,
+    'Ideogram4': None,
     'Lens': None,
     'LuminaDiMOO': None,
     'Meissonic': None,

modules/vae/sd_vae_taesd.py

@@ -71,7 +71,7 @@ def get_model(model_cls, variant=None):
     elif model_cls in {'f1', 'h1', 'zimage', 'lumina2', 'chroma', 'longcat', 'omnigen2', 'flite', 'ovis', 'kandinsky5', 'glmimage', 'cogview3', 'cogview4', 'ultraflux'}:
         model_cls = 'f1'
         variant = 'TAE FLUX.1'
-    elif model_cls in {'f2', 'ernieimage', 'lens'}:
+    elif model_cls in {'f2', 'ernieimage', 'lens', 'ideogram4'}:
         model_cls = 'f2'
         variant = 'TAE FLUX.2'
     elif model_cls in {'sd3'}:

pipelines/generic_shared.py

@@ -1,5 +1,6 @@
 import os
 import transformers
+from transformers.models.qwen3_vl import Qwen3VLModel
 
 
 shared_te_map = {
@@ -95,4 +96,9 @@ shared_te_map = {
         'target_repo': 'vladmandic/Anima-1.0-Base',
         'target_subfolder': 'text_encoder',
     },
+
+    'Qwen3-VL 8B Base': {
+        'cls': Qwen3VLModel,
+        'target_repo': 'Qwen/Qwen3-VL-8B-Instruct',
+    },
 }

pipelines/model_ideogram4.py

@@ -0,0 +1,105 @@
+import json
+import diffusers
+from transformers.models.qwen3_vl import Qwen3VLModel
+from modules import shared, devices, sd_models, model_quant
+from modules.logger import log
+from pipelines import generic
+
+
+def prompt_to_json(prompt):
+    """Normalize a JSON caption to the compact form Ideogram 4 trained on, or wrap plain text.
+
+    Ideogram 4 expects a structured JSON caption serialized compactly. A valid JSON prompt is
+    re-serialized to that form; a plain-text prompt is wrapped in a minimal caption so it stays
+    in distribution instead of tripping the weight-baked "blocked by safety filter" placeholder.
+    """
+    if isinstance(prompt, list):
+        return [prompt_to_json(p) for p in prompt]
+    if not isinstance(prompt, str) or len(prompt) == 0:
+        return prompt
+    try:
+        return json.dumps(json.loads(prompt), ensure_ascii=False, separators=(',', ':'))
+    except ValueError:
+        caption = {'high_level_description': prompt, 'compositional_deconstruction': {'background': prompt, 'elements': []}}
+        return json.dumps(caption, ensure_ascii=False, separators=(',', ':'))
+
+
+class Ideogram4Pipeline(diffusers.Ideogram4Pipeline):
+    """SD.Next integration subclass for the diffusers-native Ideogram 4 pipeline.
+
+    ``encode_prompt`` normalizes the prompt into the structured JSON the model expects, then
+    drives the Qwen3-VL tap. The tap calls ``language_model`` submodules directly, bypassing the
+    balanced-offload pre-forward hook, so the encoder is moved on-device for it and released after.
+    """
+
+    def encode_prompt(self, prompt, *args, **kwargs):
+        prompt = prompt_to_json(prompt)
+        self.text_encoder.to(self._execution_device)
+        try:
+            return super().encode_prompt(prompt, *args, **kwargs)
+        finally:
+            if shared.opts.diffusers_offload_mode != 'none':
+                self.text_encoder.to(devices.cpu)
+
+
+def pin_transformers_if_fit(transformer, unconditional_transformer) -> bool:
+    """Keep both transformers resident under balanced offload when they fit the budget.
+
+    Every denoise step runs both transformers, so balanced offload ping-pongs them across
+    PCIe each step. ``offload_never`` makes ``offload_allowed`` skip the per-step pre-sweep so
+    they stay resident, but only when both fit ``gpu_memory * max watermark`` (which leaves the
+    watermark headroom for activations); otherwise the normal offload path is kept.
+    """
+    if shared.opts.diffusers_offload_mode != 'balanced' or shared.gpu_memory <= 0:
+        return False
+    if transformer is None or unconditional_transformer is None:
+        return False
+    size_gb = sum(p.numel() * p.element_size() for m in (transformer, unconditional_transformer) for p in m.parameters()) / (1024 ** 3)
+    budget_gb = shared.gpu_memory * shared.opts.diffusers_offload_max_gpu_memory
+    fits = size_gb <= budget_gb
+    if fits:
+        transformer.offload_never = True
+        unconditional_transformer.offload_never = True
+    log.info(f'Load model: type=Ideogram4 offload=balanced transformers={size_gb:.1f} budget={budget_gb:.1f} action={"pin-resident" if fits else "offload"}')
+    return fits
+
+
+def load_ideogram4(checkpoint_info, diffusers_load_config=None):
+    if diffusers_load_config is None:
+        diffusers_load_config = {}
+    repo_id = sd_models.path_to_repo(checkpoint_info)
+    sd_models.hf_auth_check(checkpoint_info)
+    load_args, _ = model_quant.get_dit_args(diffusers_load_config, allow_quant=False)
+    log.debug(f'Load model: type=Ideogram4 repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')
+
+    generic.set_pipeline('Ideogram4', Ideogram4Pipeline)
+    if repo_id is None or repo_id.lower() == 'none':
+        return None
+
+    # Each transformer loads independently from its subfolder, so each gets its own SDNQ config.
+    cls = diffusers.Ideogram4Transformer2DModel
+    transformer = generic.load_transformer(repo_id, cls_name=cls, subfolder="transformer", load_config=diffusers_load_config)
+    unconditional_transformer = generic.load_transformer(repo_id, cls_name=cls, subfolder="unconditional_transformer", load_config=diffusers_load_config)
+    pin_transformers_if_fit(transformer, unconditional_transformer)
+    # shared_te_map redirects to the shared Qwen3-VL repo (deduped with VQA + prompt-enhance);
+    # the bundled text_encoder is the fallback when sharing is off. The vae, tokenizer, and
+    # scheduler load from the repo via from_pretrained.
+    text_encoder = generic.load_text_encoder(repo_id, cls_name=Qwen3VLModel, load_config=diffusers_load_config)
+
+    pipe = Ideogram4Pipeline.from_pretrained(
+        repo_id,
+        cache_dir=shared.opts.diffusers_dir,
+        transformer=transformer,
+        unconditional_transformer=unconditional_transformer,
+        text_encoder=text_encoder,
+        **load_args,
+    )
+    # The pipeline decodes internally; the CFG scale slider drives guidance_scale, which is
+    # mutually exclusive with the pipeline's default per-step guidance_schedule.
+    pipe.task_args = {'output_type': 'pil', 'guidance_schedule': None} # pylint: disable=attribute-defined-outside-init
+    # JSON captions must pass through verbatim; skip styles/wildcards that would strip the braces.
+    pipe.keep_prompts = True # pylint: disable=attribute-defined-outside-init
+
+    del transformer, unconditional_transformer, text_encoder
+    devices.torch_gc(force=True, reason='load')
+    return pipe

test/test-ideogram4-smoke.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+"""Standalone end-to-end smoke for diffusers-native Ideogram 4.
+
+Loads the split-projection bf16 diffusers folder and quantizes both transformers
+with SDNQ at load (the same path SD.Next uses), loads the shared Qwen3-VL text
+encoder, builds the diffusers ``Ideogram4Pipeline``, and generates an image. This
+exercises the real pipeline: both transformers under SDNQ, the Qwen3-VL 13-layer
+tap, the dual-branch asymmetric CFG loop, the logit-normal schedule, the vae.bn
+latent denorm, and VAE decode. SDNQ int4 fits the two towers plus the encoder on a
+24GB GPU.
+
+Usage:
+    python test/test-ideogram4-smoke.py --model /path/to/Ideogram-4-bf16-split --output out.png
+"""
+
+import argparse
+import os
+import sys
+import time
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--model", required=True, help="split-projection bf16 diffusers folder")
+parser.add_argument("--output", default="ideogram4_smoke.png")
+parser.add_argument("--prompt", default="a ginger cat wearing a tiny wizard hat reading a glowing spellbook, detailed digital illustration")
+parser.add_argument("--height", type=int, default=1024)
+parser.add_argument("--width", type=int, default=1024)
+parser.add_argument("--steps", type=int, default=20)
+parser.add_argument("--seed", type=int, default=0)
+parser.add_argument("--weights-dtype", default="uint4", help="SDNQ weights dtype (uint4, int8, ...)")
+parser.add_argument("--hf-cache", default=None, help="HF cache_dir for the shared Qwen3-VL encoder (default: HF default cache)")
+args = parser.parse_args()
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, REPO_ROOT)
+os.chdir(REPO_ROOT)
+os.environ["SD_INSTALL_QUIET"] = "1"
+
+# Our own args are already parsed; clear argv (and leave it cleared) so sdnext's
+# shared.py / devices, which re-parse argv on import, don't see this script's flags.
+sys.argv = [sys.argv[0]]
+
+import modules.cmd_args
+import installer
+
+modules.cmd_args.parse_args()
+installer.add_args(modules.cmd_args.parser)
+modules.cmd_args.parsed, _ = modules.cmd_args.parser.parse_known_args([])
+
+import torch
+import diffusers
+from transformers import AutoTokenizer
+from transformers.models.qwen3_vl import Qwen3VLModel
+
+from modules import devices
+from modules.sdnq import SDNQConfig
+
+TE_REPO = "Qwen/Qwen3-VL-8B-Instruct"
+
+
+def main() -> int:
+    device = devices.device
+    cfg = SDNQConfig(weights_dtype=args.weights_dtype)
+
+    print(f"loading transformer (sdnq {args.weights_dtype}) ...", flush=True)
+    transformer = diffusers.Ideogram4Transformer2DModel.from_pretrained(args.model, subfolder="transformer", quantization_config=cfg, torch_dtype=torch.bfloat16).to(device)
+    print("loading unconditional_transformer ...", flush=True)
+    uncond = diffusers.Ideogram4Transformer2DModel.from_pretrained(args.model, subfolder="unconditional_transformer", quantization_config=cfg, torch_dtype=torch.bfloat16).to(device)
+    print("loading text encoder Qwen3-VL ...", flush=True)
+    te_kwargs = {"cache_dir": args.hf_cache} if args.hf_cache else {}
+    text_encoder = Qwen3VLModel.from_pretrained(TE_REPO, quantization_config=SDNQConfig(weights_dtype=args.weights_dtype), torch_dtype=torch.bfloat16, **te_kwargs).to(device)
+    tokenizer = AutoTokenizer.from_pretrained(args.model, subfolder="tokenizer")
+    print("loading vae ...", flush=True)
+    vae = diffusers.AutoencoderKLFlux2.from_pretrained(args.model, subfolder="vae", torch_dtype=torch.bfloat16).to(device)
+    scheduler = diffusers.FlowMatchEulerDiscreteScheduler.from_pretrained(args.model, subfolder="scheduler")
+
+    pipe = diffusers.Ideogram4Pipeline(
+        scheduler=scheduler,
+        vae=vae,
+        text_encoder=text_encoder,
+        tokenizer=tokenizer,
+        transformer=transformer,
+        unconditional_transformer=uncond,
+    )
+
+    generator = torch.Generator(device=device).manual_seed(args.seed)
+    print(f"generating {args.width}x{args.height} steps={args.steps} ...", flush=True)
+    start = time.time()
+    out = pipe(prompt=args.prompt, num_inference_steps=args.steps, guidance_scale=7.0, guidance_schedule=None, width=args.width, height=args.height, generator=generator)
+    elapsed = time.time() - start
+
+    image = out.images[0]
+    image.save(args.output)
+    if torch.cuda.is_available():
+        print(f"peak VRAM: {torch.cuda.max_memory_allocated() / 1e9:.1f} GB", flush=True)
+    print(f"PASS: generated {args.output} in {elapsed:.1f}s, size={image.size}", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())