Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 01:04:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(ltx): patch connectors regression and group model dropdown

Browse Source 
- monkey-patch LTX2ConnectorTransformer1d.forward to restore pre-#13564
  padding logic when the upstream torch.flip pattern is detected; fixes
  word-order scrambling in audio dialogue tracks
- reorganize LTX model entries into version-group separators (2.3 v1.1,
  2.3 v1.0, 2.0, 0.9.x) with base/distilled subgroups; separators are
  selectable no-ops handled in run_ltx
This commit is contained in:
CalamitousFelicitousness
2026-05-21 03:01:51 +01:00
parent 32d245898c
commit 394d4e5b84
3 changed files with 156 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/ltx/ltx_diffusers_patch.py
+93
View File
@@ -0,0 +1,93 @@
"""Workaround for huggingface/diffusers#13564 connectors padding regression.
PR #13564 (merged 2026-05-08) refactored LTX2ConnectorTransformer1d's padding
logic from a loop-based gather-and-pad into a vectorized mask-then-flip. The
new code applies torch.flip(hidden_states, dims=[1]) after replacing padding
positions with learned registers, which reverses the order of valid prompt
tokens. Audio cross-attention is position-sensitive, so reversed token order
produces jumbled dialogue (right vocabulary, wrong word order). Visual quality
is mostly unaffected because spatial cross-attention is less position-sensitive.
This module restores the pre-#13564 forward at import time when the broken
pattern is detected. Safe to leave in place after upstream fixes the bug:
detection will skip the monkey-patch when the source no longer matches.
"""
import inspect
import torch
import torch.nn.functional as F
from modules.logger import log
_PATCH_APPLIED = False
_BROKEN_MARKER = 'torch.flip(hidden_states, dims=[1])'
def _patched_forward(
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor | None = None,
attn_mask_binarize_threshold: float = -9000.0,
):
batch_size, seq_len, _ = hidden_states.shape
if self.learnable_registers is not None:
if seq_len % self.num_learnable_registers != 0:
raise ValueError(
f"The `hidden_states` sequence length {hidden_states.shape[1]} should be divisible by the number"
f" of learnable registers {self.num_learnable_registers}"
)
num_register_repeats = seq_len // self.num_learnable_registers
registers = (
self.learnable_registers.unsqueeze(0).expand(num_register_repeats, -1, -1).reshape(seq_len, -1)
)
binary_attn_mask = (attention_mask >= attn_mask_binarize_threshold).int()
if binary_attn_mask.ndim == 4:
binary_attn_mask = binary_attn_mask.squeeze(1).squeeze(1)
hidden_states_non_padded = [hidden_states[i, binary_attn_mask[i].bool(), :] for i in range(batch_size)]
valid_seq_lens = [x.shape[0] for x in hidden_states_non_padded]
pad_lengths = [seq_len - vsl for vsl in valid_seq_lens]
padded_hidden_states = [
F.pad(x, pad=(0, 0, 0, p), value=0) for x, p in zip(hidden_states_non_padded, pad_lengths)
]
padded_hidden_states = torch.cat([x.unsqueeze(0) for x in padded_hidden_states], dim=0)
flipped_mask = torch.flip(binary_attn_mask, dims=[1]).unsqueeze(-1)
hidden_states = flipped_mask * padded_hidden_states + (1 - flipped_mask) * registers
attention_mask = torch.zeros_like(attention_mask)
rotary_emb = self.rope(batch_size, seq_len, device=hidden_states.device)
for block in self.transformer_blocks:
if torch.is_grad_enabled() and self.gradient_checkpointing:
hidden_states = self._gradient_checkpointing_func(block, hidden_states, attention_mask, rotary_emb) # pylint: disable=protected-access
else:
hidden_states = block(hidden_states, attention_mask=attention_mask, rotary_emb=rotary_emb)
hidden_states = self.norm_out(hidden_states)
return hidden_states, attention_mask
def apply_patch():
global _PATCH_APPLIED # pylint: disable=global-statement
if _PATCH_APPLIED:
return
try:
from diffusers.pipelines.ltx2.connectors import LTX2ConnectorTransformer1d
except ImportError:
_PATCH_APPLIED = True
return
try:
source = inspect.getsource(LTX2ConnectorTransformer1d.forward)
except (OSError, TypeError):
source = ''
if _BROKEN_MARKER in source:
LTX2ConnectorTransformer1d.forward = _patched_forward
log.info('LTX2: patched diffusers connectors padding to fix audio token order (upstream #13564 regression)')
_PATCH_APPLIED = True
modules/ltx/ltx_process.py
+6
View File
@@ -6,7 +6,10 @@ from PIL import Image
from modules import shared, errors, timer, memstats, progress, processing, sd_models, sd_samplers, devices, extra_networks, call_queue
from modules.logger import log
from modules.ltx import ltx_capabilities
from modules.ltx.ltx_diffusers_patch import apply_patch as apply_ltx_diffusers_patch
from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, load_upsample_2x, get_conditions, get_generator, get_prompts, ltx_scheduler_opts, vae_decode
apply_ltx_diffusers_patch()
from modules.processing_callbacks import diffusers_callback
from modules.video_models.video_vae import set_vae_params
from modules.video_models.video_save import save_video
@@ -154,6 +157,9 @@ def run_ltx(task_id,
if model is None or len(model) == 0 or model == 'None':
yield from abort('Video: no model selected', ok=True)
return
if model.startswith('─'):
yield from abort('Video: dropdown separator selected, pick an actual model below', ok=True)
return
check_av()
progress.add_task_to_queue(task_id)
modules/video_models/models_def.py
+57 -49
View File
@@ -129,6 +129,7 @@ try:
'LTX Video': [
Model(name='None'),
Model(name='─────── LTX-2.3 v1.1 ───────'),
Model(name='LTXVideo 2.3-1.1 22B T2V Distilled',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-Distilled-1.1',
@@ -141,6 +142,21 @@ try:
repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='LTXVideo 2.3-1.1 22B T2V Distilled SDNQ-4Bit',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-Distilled-1.1-sdnq-dynamic-int4',
repo_cls=getattr(diffusers, 'LTX2Pipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='LTXVideo 2.3-1.1 22B I2V Distilled SDNQ-4Bit',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-Distilled-1.1-sdnq-dynamic-int4',
repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='─────── LTX-2.3 v1.0 ───────'),
Model(name='─── Base ───'),
Model(name='LTXVideo 2.3 22B T2V',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3',
@@ -159,37 +175,6 @@ try:
repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='LTXVideo 2.3 22B T2V Distilled',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-Distilled',
repo_cls=getattr(diffusers, 'LTX2Pipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='LTXVideo 2.3 22B I2V Distilled',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-Distilled',
repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='LTXVideo 2.3 22B Condition Distilled',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-Distilled',
repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='LTXVideo 2.3-1.1 22B T2V Distilled SDNQ-4Bit',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-Distilled-1.1-sdnq-dynamic-int4',
repo_cls=getattr(diffusers, 'LTX2Pipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='LTXVideo 2.3-1.1 22B I2V Distilled SDNQ-4Bit',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-Distilled-1.1-sdnq-dynamic-int4',
repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='LTXVideo 2.3 22B T2V SDNQ-4Bit',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-sdnq-dynamic-int4',
@@ -208,6 +193,25 @@ try:
repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='─── Distilled ───'),
Model(name='LTXVideo 2.3 22B T2V Distilled',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-Distilled',
repo_cls=getattr(diffusers, 'LTX2Pipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='LTXVideo 2.3 22B I2V Distilled',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-Distilled',
repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='LTXVideo 2.3 22B Condition Distilled',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-Distilled',
repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None),
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='LTXVideo 2.3 22B T2V Distilled SDNQ-4Bit',
url='https://huggingface.co/Lightricks/LTX-2.3',
repo='OzzyGT/LTX-2.3-Distilled-sdnq-dynamic-int4',
@@ -227,6 +231,7 @@ try:
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='─────── LTX-2.0 ───────'),
Model(name='LTXVideo 2.0 19B T2V Dev',
url='https://huggingface.co/Lightricks/LTX-2',
repo='Lightricks/LTX-2',
@@ -252,12 +257,8 @@ try:
te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None),
dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)),
Model(name='LTXVideo 0.9.8 13B Distilled',
url='https://huggingface.co/Lightricks/LTX-Video-0.9.8-13B-distilled',
repo='Lightricks/LTX-Video-0.9.8-13B-distilled',
repo_cls=getattr(diffusers, 'LTXConditionPipeline', None),
te_cls=getattr(transformers, 'T5EncoderModel', None),
dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)),
Model(name='─────── LTX-0.9.x ───────'),
Model(name='─── Base ───'),
Model(name='LTXVideo 0.9.7 13B Dev',
url='https://huggingface.co/Lightricks/LTX-Video-0.9.7-dev',
repo='a-r-r-o-w/LTX-Video-0.9.7-diffusers',
@@ -276,18 +277,6 @@ try:
repo_cls=getattr(diffusers, 'LTXConditionPipeline', None),
te_cls=getattr(transformers, 'T5EncoderModel', None),
dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)),
Model(name='LTXVideo 0.9.6 2B T2V Distilled',
url='https://huggingface.co/Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25',
repo='Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25',
repo_cls=getattr(diffusers, 'LTXConditionPipeline', None),
te_cls=getattr(transformers, 'T5EncoderModel', None),
dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)),
Model(name='LTXVideo 0.9.6 2B I2V Distilled',
url='https://huggingface.co/Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25',
repo='Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25',
repo_cls=getattr(diffusers, 'LTXConditionPipeline', None),
te_cls=getattr(transformers, 'T5EncoderModel', None),
dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)),
Model(name='LTXVideo 0.9.5 T2V', # https://github.com/huggingface/diffusers/pull/10968
url='https://huggingface.co/Lightricks/LTX-Video-0.9.5',
repo='Lightricks/LTX-Video-0.9.5',
@@ -324,6 +313,25 @@ try:
repo_cls=getattr(diffusers, 'LTXImageToVideoPipeline', None),
te_cls=getattr(transformers, 'T5EncoderModel', None),
dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)),
Model(name='─── Distilled ───'),
Model(name='LTXVideo 0.9.8 13B Distilled',
url='https://huggingface.co/Lightricks/LTX-Video-0.9.8-13B-distilled',
repo='Lightricks/LTX-Video-0.9.8-13B-distilled',
repo_cls=getattr(diffusers, 'LTXConditionPipeline', None),
te_cls=getattr(transformers, 'T5EncoderModel', None),
dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)),
Model(name='LTXVideo 0.9.6 2B T2V Distilled',
url='https://huggingface.co/Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25',
repo='Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25',
repo_cls=getattr(diffusers, 'LTXConditionPipeline', None),
te_cls=getattr(transformers, 'T5EncoderModel', None),
dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)),
Model(name='LTXVideo 0.9.6 2B I2V Distilled',
url='https://huggingface.co/Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25',
repo='Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25',
repo_cls=getattr(diffusers, 'LTXConditionPipeline', None),
te_cls=getattr(transformers, 'T5EncoderModel', None),
dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)),
],
'WAN Video': [
Model(name='None'),