From 92e1cb9927a45932d8ef31dcf728bbc38bf89f57 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Fri, 5 Jun 2026 01:37:55 +0100 Subject: [PATCH] fix(ideogram4): pass JSON captions through unmangled The dynamic-prompt brace processor in apply_styles_to_prompts strips the {} and [] out of a JSON caption, leaving non-JSON that trips the model's weight-baked safety placeholder. Let a model opt out of style and wildcard processing via keep_prompts and set it for Ideogram4, then normalize the prompt in encode_prompt: valid JSON to the compact training form, plain text wrapped into a minimal caption so basic prompts still generate. --- modules/processing.py | 2 +- pipelines/model_ideogram4.py | 32 +++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index 56d6da3f1..d73ea8973 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -278,7 +278,7 @@ def process_init(p: StableDiffusionProcessing): else: p.all_subseeds = [int(subseed) + x for x in range(len(p.all_prompts))] if reset_prompts: - if not hasattr(p, 'keep_prompts'): + if not hasattr(p, 'keep_prompts') and not getattr(shared.sd_model, 'keep_prompts', False): p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts(p.all_prompts, p.all_negative_prompts, p.styles, p.all_seeds, p=p) p.prompts = p.all_prompts[(p.iteration * p.batch_size):((p.iteration+1) * p.batch_size)] p.negative_prompts = p.all_negative_prompts[(p.iteration * p.batch_size):((p.iteration+1) * p.batch_size)] diff --git a/pipelines/model_ideogram4.py b/pipelines/model_ideogram4.py index d6ebe0ccc..b7cae290f 100644 --- a/pipelines/model_ideogram4.py +++ b/pipelines/model_ideogram4.py @@ -1,3 +1,4 @@ +import json import diffusers from transformers import AutoTokenizer from transformers.models.qwen3_vl import Qwen3VLModel @@ -6,18 +7,37 @@ from modules.logger import log from pipelines import generic +def prompt_to_json(prompt): + """Normalize a JSON caption to the compact form Ideogram 4 trained on, or wrap plain text. + + Ideogram 4 expects a structured JSON caption serialized compactly. A valid JSON prompt is + re-serialized to that form; a plain-text prompt is wrapped in a minimal caption so it stays + in distribution instead of tripping the weight-baked "blocked by safety filter" placeholder. + """ + if isinstance(prompt, list): + return [prompt_to_json(p) for p in prompt] + if not isinstance(prompt, str) or len(prompt) == 0: + return prompt + try: + return json.dumps(json.loads(prompt), ensure_ascii=False, separators=(',', ':')) + except ValueError: + caption = {'high_level_description': prompt, 'compositional_deconstruction': {'background': prompt, 'elements': []}} + return json.dumps(caption, ensure_ascii=False, separators=(',', ':')) + + class Ideogram4Pipeline(diffusers.Ideogram4Pipeline): """SD.Next integration subclass for the diffusers-native Ideogram 4 pipeline. - ``encode_prompt`` drives the Qwen3-VL tap by calling ``language_model`` submodules - directly, which bypasses the balanced-offload pre-forward hook. Move the encoder - on-device for the tap and release it afterward so it does not pin VRAM. + ``encode_prompt`` normalizes the prompt into the structured JSON the model expects, then + drives the Qwen3-VL tap. The tap calls ``language_model`` submodules directly, bypassing the + balanced-offload pre-forward hook, so the encoder is moved on-device for it and released after. """ - def encode_prompt(self, *args, **kwargs): + def encode_prompt(self, prompt, *args, **kwargs): + prompt = prompt_to_json(prompt) self.text_encoder.to(self._execution_device) try: - return super().encode_prompt(*args, **kwargs) + return super().encode_prompt(prompt, *args, **kwargs) finally: if shared.opts.diffusers_offload_mode != 'none': self.text_encoder.to(devices.cpu) @@ -78,6 +98,8 @@ def load_ideogram4(checkpoint_info, diffusers_load_config=None): # The pipeline decodes internally; the CFG scale slider drives guidance_scale, which is # mutually exclusive with the pipeline's default per-step guidance_schedule. pipe.task_args = {'output_type': 'pil', 'guidance_schedule': None} # pylint: disable=attribute-defined-outside-init + # JSON captions must pass through verbatim; skip styles/wildcards that would strip the braces. + pipe.keep_prompts = True # pylint: disable=attribute-defined-outside-init del transformer, unconditional_transformer, text_encoder, vae devices.torch_gc(force=True, reason='load')