diff --git a/modules/ltx/ltx_diffusers_patch.py b/modules/ltx/ltx_diffusers_patch.py new file mode 100644 index 000000000..c1c84384a --- /dev/null +++ b/modules/ltx/ltx_diffusers_patch.py @@ -0,0 +1,93 @@ +"""Workaround for huggingface/diffusers#13564 connectors padding regression. + +PR #13564 (merged 2026-05-08) refactored LTX2ConnectorTransformer1d's padding +logic from a loop-based gather-and-pad into a vectorized mask-then-flip. The +new code applies torch.flip(hidden_states, dims=[1]) after replacing padding +positions with learned registers, which reverses the order of valid prompt +tokens. Audio cross-attention is position-sensitive, so reversed token order +produces jumbled dialogue (right vocabulary, wrong word order). Visual quality +is mostly unaffected because spatial cross-attention is less position-sensitive. + +This module restores the pre-#13564 forward at import time when the broken +pattern is detected. Safe to leave in place after upstream fixes the bug: +detection will skip the monkey-patch when the source no longer matches. +""" + +import inspect + +import torch +import torch.nn.functional as F + +from modules.logger import log + + +_PATCH_APPLIED = False +_BROKEN_MARKER = 'torch.flip(hidden_states, dims=[1])' + + +def _patched_forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + attn_mask_binarize_threshold: float = -9000.0, +): + batch_size, seq_len, _ = hidden_states.shape + + if self.learnable_registers is not None: + if seq_len % self.num_learnable_registers != 0: + raise ValueError( + f"The `hidden_states` sequence length {hidden_states.shape[1]} should be divisible by the number" + f" of learnable registers {self.num_learnable_registers}" + ) + + num_register_repeats = seq_len // self.num_learnable_registers + registers = ( + self.learnable_registers.unsqueeze(0).expand(num_register_repeats, -1, -1).reshape(seq_len, -1) + ) + + binary_attn_mask = (attention_mask >= attn_mask_binarize_threshold).int() + if binary_attn_mask.ndim == 4: + binary_attn_mask = binary_attn_mask.squeeze(1).squeeze(1) + + hidden_states_non_padded = [hidden_states[i, binary_attn_mask[i].bool(), :] for i in range(batch_size)] + valid_seq_lens = [x.shape[0] for x in hidden_states_non_padded] + pad_lengths = [seq_len - vsl for vsl in valid_seq_lens] + padded_hidden_states = [ + F.pad(x, pad=(0, 0, 0, p), value=0) for x, p in zip(hidden_states_non_padded, pad_lengths) + ] + padded_hidden_states = torch.cat([x.unsqueeze(0) for x in padded_hidden_states], dim=0) + + flipped_mask = torch.flip(binary_attn_mask, dims=[1]).unsqueeze(-1) + hidden_states = flipped_mask * padded_hidden_states + (1 - flipped_mask) * registers + + attention_mask = torch.zeros_like(attention_mask) + + rotary_emb = self.rope(batch_size, seq_len, device=hidden_states.device) + + for block in self.transformer_blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func(block, hidden_states, attention_mask, rotary_emb) # pylint: disable=protected-access + else: + hidden_states = block(hidden_states, attention_mask=attention_mask, rotary_emb=rotary_emb) + + hidden_states = self.norm_out(hidden_states) + return hidden_states, attention_mask + + +def apply_patch(): + global _PATCH_APPLIED # pylint: disable=global-statement + if _PATCH_APPLIED: + return + try: + from diffusers.pipelines.ltx2.connectors import LTX2ConnectorTransformer1d + except ImportError: + _PATCH_APPLIED = True + return + try: + source = inspect.getsource(LTX2ConnectorTransformer1d.forward) + except (OSError, TypeError): + source = '' + if _BROKEN_MARKER in source: + LTX2ConnectorTransformer1d.forward = _patched_forward + log.info('LTX2: patched diffusers connectors padding to fix audio token order (upstream #13564 regression)') + _PATCH_APPLIED = True diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 2398eb234..5539a570f 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -6,7 +6,10 @@ from PIL import Image from modules import shared, errors, timer, memstats, progress, processing, sd_models, sd_samplers, devices, extra_networks, call_queue from modules.logger import log from modules.ltx import ltx_capabilities +from modules.ltx.ltx_diffusers_patch import apply_patch as apply_ltx_diffusers_patch from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, load_upsample_2x, get_conditions, get_generator, get_prompts, ltx_scheduler_opts, vae_decode + +apply_ltx_diffusers_patch() from modules.processing_callbacks import diffusers_callback from modules.video_models.video_vae import set_vae_params from modules.video_models.video_save import save_video @@ -154,6 +157,9 @@ def run_ltx(task_id, if model is None or len(model) == 0 or model == 'None': yield from abort('Video: no model selected', ok=True) return + if model.startswith('─'): + yield from abort('Video: dropdown separator selected, pick an actual model below', ok=True) + return check_av() progress.add_task_to_queue(task_id) diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index 4362406fe..492ea7f43 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -129,6 +129,7 @@ try: 'LTX Video': [ Model(name='None'), + Model(name='─────── LTX-2.3 v1.1 ───────'), Model(name='LTXVideo 2.3-1.1 22B T2V Distilled', url='https://huggingface.co/Lightricks/LTX-2.3', repo='OzzyGT/LTX-2.3-Distilled-1.1', @@ -141,6 +142,21 @@ try: repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='LTXVideo 2.3-1.1 22B T2V Distilled SDNQ-4Bit', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3-Distilled-1.1-sdnq-dynamic-int4', + repo_cls=getattr(diffusers, 'LTX2Pipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='LTXVideo 2.3-1.1 22B I2V Distilled SDNQ-4Bit', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3-Distilled-1.1-sdnq-dynamic-int4', + repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + + Model(name='─────── LTX-2.3 v1.0 ───────'), + Model(name='─── Base ───'), Model(name='LTXVideo 2.3 22B T2V', url='https://huggingface.co/Lightricks/LTX-2.3', repo='OzzyGT/LTX-2.3', @@ -159,37 +175,6 @@ try: repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None), te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), - Model(name='LTXVideo 2.3 22B T2V Distilled', - url='https://huggingface.co/Lightricks/LTX-2.3', - repo='OzzyGT/LTX-2.3-Distilled', - repo_cls=getattr(diffusers, 'LTX2Pipeline', None), - te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), - dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), - Model(name='LTXVideo 2.3 22B I2V Distilled', - url='https://huggingface.co/Lightricks/LTX-2.3', - repo='OzzyGT/LTX-2.3-Distilled', - repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), - te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), - dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), - Model(name='LTXVideo 2.3 22B Condition Distilled', - url='https://huggingface.co/Lightricks/LTX-2.3', - repo='OzzyGT/LTX-2.3-Distilled', - repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None), - te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), - dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), - - Model(name='LTXVideo 2.3-1.1 22B T2V Distilled SDNQ-4Bit', - url='https://huggingface.co/Lightricks/LTX-2.3', - repo='OzzyGT/LTX-2.3-Distilled-1.1-sdnq-dynamic-int4', - repo_cls=getattr(diffusers, 'LTX2Pipeline', None), - te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), - dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), - Model(name='LTXVideo 2.3-1.1 22B I2V Distilled SDNQ-4Bit', - url='https://huggingface.co/Lightricks/LTX-2.3', - repo='OzzyGT/LTX-2.3-Distilled-1.1-sdnq-dynamic-int4', - repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), - te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), - dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), Model(name='LTXVideo 2.3 22B T2V SDNQ-4Bit', url='https://huggingface.co/Lightricks/LTX-2.3', repo='OzzyGT/LTX-2.3-sdnq-dynamic-int4', @@ -208,6 +193,25 @@ try: repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None), te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='─── Distilled ───'), + Model(name='LTXVideo 2.3 22B T2V Distilled', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3-Distilled', + repo_cls=getattr(diffusers, 'LTX2Pipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='LTXVideo 2.3 22B I2V Distilled', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3-Distilled', + repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='LTXVideo 2.3 22B Condition Distilled', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3-Distilled', + repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), Model(name='LTXVideo 2.3 22B T2V Distilled SDNQ-4Bit', url='https://huggingface.co/Lightricks/LTX-2.3', repo='OzzyGT/LTX-2.3-Distilled-sdnq-dynamic-int4', @@ -227,6 +231,7 @@ try: te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='─────── LTX-2.0 ───────'), Model(name='LTXVideo 2.0 19B T2V Dev', url='https://huggingface.co/Lightricks/LTX-2', repo='Lightricks/LTX-2', @@ -252,12 +257,8 @@ try: te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), - Model(name='LTXVideo 0.9.8 13B Distilled', - url='https://huggingface.co/Lightricks/LTX-Video-0.9.8-13B-distilled', - repo='Lightricks/LTX-Video-0.9.8-13B-distilled', - repo_cls=getattr(diffusers, 'LTXConditionPipeline', None), - te_cls=getattr(transformers, 'T5EncoderModel', None), - dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)), + Model(name='─────── LTX-0.9.x ───────'), + Model(name='─── Base ───'), Model(name='LTXVideo 0.9.7 13B Dev', url='https://huggingface.co/Lightricks/LTX-Video-0.9.7-dev', repo='a-r-r-o-w/LTX-Video-0.9.7-diffusers', @@ -276,18 +277,6 @@ try: repo_cls=getattr(diffusers, 'LTXConditionPipeline', None), te_cls=getattr(transformers, 'T5EncoderModel', None), dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)), - Model(name='LTXVideo 0.9.6 2B T2V Distilled', - url='https://huggingface.co/Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', - repo='Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', - repo_cls=getattr(diffusers, 'LTXConditionPipeline', None), - te_cls=getattr(transformers, 'T5EncoderModel', None), - dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)), - Model(name='LTXVideo 0.9.6 2B I2V Distilled', - url='https://huggingface.co/Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', - repo='Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', - repo_cls=getattr(diffusers, 'LTXConditionPipeline', None), - te_cls=getattr(transformers, 'T5EncoderModel', None), - dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)), Model(name='LTXVideo 0.9.5 T2V', # https://github.com/huggingface/diffusers/pull/10968 url='https://huggingface.co/Lightricks/LTX-Video-0.9.5', repo='Lightricks/LTX-Video-0.9.5', @@ -324,6 +313,25 @@ try: repo_cls=getattr(diffusers, 'LTXImageToVideoPipeline', None), te_cls=getattr(transformers, 'T5EncoderModel', None), dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)), + Model(name='─── Distilled ───'), + Model(name='LTXVideo 0.9.8 13B Distilled', + url='https://huggingface.co/Lightricks/LTX-Video-0.9.8-13B-distilled', + repo='Lightricks/LTX-Video-0.9.8-13B-distilled', + repo_cls=getattr(diffusers, 'LTXConditionPipeline', None), + te_cls=getattr(transformers, 'T5EncoderModel', None), + dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)), + Model(name='LTXVideo 0.9.6 2B T2V Distilled', + url='https://huggingface.co/Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', + repo='Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', + repo_cls=getattr(diffusers, 'LTXConditionPipeline', None), + te_cls=getattr(transformers, 'T5EncoderModel', None), + dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)), + Model(name='LTXVideo 0.9.6 2B I2V Distilled', + url='https://huggingface.co/Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', + repo='Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', + repo_cls=getattr(diffusers, 'LTXConditionPipeline', None), + te_cls=getattr(transformers, 'T5EncoderModel', None), + dit_cls=getattr(diffusers, 'LTXVideoTransformer3DModel', None)), ], 'WAN Video': [ Model(name='None'),