fix(ideogram4): pass JSON captions through unmangled

The dynamic-prompt brace processor in apply_styles_to_prompts strips the
{} and [] out of a JSON caption, leaving non-JSON that trips the model's
weight-baked safety placeholder. Let a model opt out of style and wildcard
processing via keep_prompts and set it for Ideogram4, then normalize the
prompt in encode_prompt: valid JSON to the compact training form, plain
text wrapped into a minimal caption so basic prompts still generate.
This commit is contained in:
CalamitousFelicitousness
2026-06-05 01:37:55 +01:00
parent 5c1a52ee4c
commit 92e1cb9927
2 changed files with 28 additions and 6 deletions
+1 -1
View File
@@ -278,7 +278,7 @@ def process_init(p: StableDiffusionProcessing):
else:
p.all_subseeds = [int(subseed) + x for x in range(len(p.all_prompts))]
if reset_prompts:
if not hasattr(p, 'keep_prompts'):
if not hasattr(p, 'keep_prompts') and not getattr(shared.sd_model, 'keep_prompts', False):
p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts(p.all_prompts, p.all_negative_prompts, p.styles, p.all_seeds, p=p)
p.prompts = p.all_prompts[(p.iteration * p.batch_size):((p.iteration+1) * p.batch_size)]
p.negative_prompts = p.all_negative_prompts[(p.iteration * p.batch_size):((p.iteration+1) * p.batch_size)]
+27 -5
View File
@@ -1,3 +1,4 @@
import json
import diffusers
from transformers import AutoTokenizer
from transformers.models.qwen3_vl import Qwen3VLModel
@@ -6,18 +7,37 @@ from modules.logger import log
from pipelines import generic
def prompt_to_json(prompt):
"""Normalize a JSON caption to the compact form Ideogram 4 trained on, or wrap plain text.
Ideogram 4 expects a structured JSON caption serialized compactly. A valid JSON prompt is
re-serialized to that form; a plain-text prompt is wrapped in a minimal caption so it stays
in distribution instead of tripping the weight-baked "blocked by safety filter" placeholder.
"""
if isinstance(prompt, list):
return [prompt_to_json(p) for p in prompt]
if not isinstance(prompt, str) or len(prompt) == 0:
return prompt
try:
return json.dumps(json.loads(prompt), ensure_ascii=False, separators=(',', ':'))
except ValueError:
caption = {'high_level_description': prompt, 'compositional_deconstruction': {'background': prompt, 'elements': []}}
return json.dumps(caption, ensure_ascii=False, separators=(',', ':'))
class Ideogram4Pipeline(diffusers.Ideogram4Pipeline):
"""SD.Next integration subclass for the diffusers-native Ideogram 4 pipeline.
``encode_prompt`` drives the Qwen3-VL tap by calling ``language_model`` submodules
directly, which bypasses the balanced-offload pre-forward hook. Move the encoder
on-device for the tap and release it afterward so it does not pin VRAM.
``encode_prompt`` normalizes the prompt into the structured JSON the model expects, then
drives the Qwen3-VL tap. The tap calls ``language_model`` submodules directly, bypassing the
balanced-offload pre-forward hook, so the encoder is moved on-device for it and released after.
"""
def encode_prompt(self, *args, **kwargs):
def encode_prompt(self, prompt, *args, **kwargs):
prompt = prompt_to_json(prompt)
self.text_encoder.to(self._execution_device)
try:
return super().encode_prompt(*args, **kwargs)
return super().encode_prompt(prompt, *args, **kwargs)
finally:
if shared.opts.diffusers_offload_mode != 'none':
self.text_encoder.to(devices.cpu)
@@ -78,6 +98,8 @@ def load_ideogram4(checkpoint_info, diffusers_load_config=None):
# The pipeline decodes internally; the CFG scale slider drives guidance_scale, which is
# mutually exclusive with the pipeline's default per-step guidance_schedule.
pipe.task_args = {'output_type': 'pil', 'guidance_schedule': None} # pylint: disable=attribute-defined-outside-init
# JSON captions must pass through verbatim; skip styles/wildcards that would strip the braces.
pipe.keep_prompts = True # pylint: disable=attribute-defined-outside-init
del transformer, unconditional_transformer, text_encoder, vae
devices.torch_gc(force=True, reason='load')