diff --git a/CHANGELOG.md b/CHANGELOG.md index 681cc1331..47e1f166b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ - [MiniMax H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) in *base* and *ref* variants MiniMax-H3 is an amazing, but absolutely massive at 32B text-encoder and 33B transformer video model for details, see [MiniMax wiki page](wiki/MiniMax) + - [LTX-2.5](https://huggingface.co/Lightricks/LTX-2.5) from Lightricks + 22B joint audio and video generation in *Distilled* and *Dev* variants, + each as *text-to-video*, *image-to-video* and *conditioning* rows + paired with a *Gemma 4* text encoder and a duration head: + tick *auto duration* and the clip length is predicted from the prompt instead of set by hand + *note*: LTX-2.5 is a [gated model](https://vladmandic.github.io/sdnext-docs/Gated/) + *note*: image conditioning is now re-compressed to match what the models were trained on, + at CRF 18 for LTX-2.5 and 33 for earlier LTX-2.x versions - **Detailer**: Pretty much *detailer.next* :) Detailer detection models were traditionally *YOLO* models, but now we can also use: - [Facebook-SAM3](https://huggingface.co/facebook/sam3) hybrid promptable concept segmentation and detection network @@ -43,6 +51,12 @@ - improve pipeline detection for non-cached models - cleanup alt offload codepaths - hf progress bars + - ltx: send the guidance stack and cross-timestep on every 2.x call path + - ltx: distilled variants no longer force dynamic shifting on, which remapped their sigma schedule + - ltx: sampler shift now reaches flow-match schedulers + - ltx: reload the latent upsampler when the model or its repo changes + - video: take the audio sample rate from the loaded vocoder + - video: keep the shared text encoder out of the registry rows ## Update for 2026-08-07 diff --git a/modules/ltx/ltx_capabilities.py b/modules/ltx/ltx_capabilities.py index 0d1909580..662d8c6d9 100644 --- a/modules/ltx/ltx_capabilities.py +++ b/modules/ltx/ltx_capabilities.py @@ -9,7 +9,7 @@ class LTXCaps: name: str repo_cls_name: str family: str # '0.9' or '2.x' - variant: str # '0.9', '2.0', '2.3' (finer-grained sub-variant) + variant: str # '0.9', '2.0', '2.3', '2.5' (finer-grained sub-variant) is_distilled: bool is_i2v: bool supports_input_media: bool @@ -41,6 +41,13 @@ class LTXCaps: # for refine; same-res refine oversaturates. Condition variants rebuild conditions per stage. supports_two_stage_refine: bool = False stage2_dev_lora_repo: Optional[str] = None + # 2.5 keeps the stage 2 LoRA in the model repo; 2.0 and 2.3 each have their own + stage2_dev_lora_weight: Optional[str] = None + # tied to the family VAE: the wrong one drifts per-channel latent statistics + upsample_repo: Optional[str] = None + # 2.5 ships the distilled scheduler config, so its Dev rows restore the terminal shift + scheduler_shift_terminal: Optional[float] = None + supports_auto_duration: bool = False CONDITION_CLASSES = {'LTXConditionPipeline', 'LTX2ConditionPipeline'} @@ -84,7 +91,12 @@ def get_caps(model_name: str) -> Optional[LTXCaps]: family = '2.x' if is_ltx2 else '0.9' # 2.x sub-variant detection: unknown 2.x mirrors fall through to '2.0' (conservative default). if is_ltx2: - variant = '2.3' if '2.3' in model_name else '2.0' + if '2.5' in model_name: + variant = '2.5' + elif '2.3' in model_name: + variant = '2.3' + else: + variant = '2.0' else: variant = '0.9' is_distilled = 'Distilled' in model_name @@ -106,11 +118,12 @@ def get_caps(model_name: str) -> Optional[LTXCaps]: supports_stg=is_ltx2, supports_audio=is_ltx2, supports_frame_rate_kwarg=is_ltx2, - use_cross_timestep=(variant == '2.3'), + use_cross_timestep=variant in ('2.3', '2.5'), default_cfg=3.0, default_steps=30 if is_ltx2 else 50, default_sampler_shift=-1.0, - default_dynamic_shift=is_ltx2, + # distilled ships use_dynamic_shifting=False and runs explicit sigmas, which shifting remaps + default_dynamic_shift=is_ltx2 and not is_distilled, default_width=768, default_height=512, default_frames=121 if is_ltx2 else 161, @@ -122,7 +135,10 @@ def get_caps(model_name: str) -> Optional[LTXCaps]: caps.default_steps = 8 if is_ltx2 and not is_distilled: - if variant == '2.3': + if variant == '2.5': + caps.stage2_dev_lora_repo = 'Lightricks/LTX-2.5-Diffusers' + caps.stage2_dev_lora_weight = 'ltx-2.5-22b-distilled-lora-450-bf16.safetensors' + elif variant == '2.3': caps.stage2_dev_lora_repo = 'CalamitousFelicitousness/LTX-2.3-distilled-lora-384-Diffusers' elif variant == '2.0': caps.stage2_dev_lora_repo = 'CalamitousFelicitousness/LTX-2.0-distilled-lora-384-Diffusers' @@ -130,16 +146,22 @@ def get_caps(model_name: str) -> Optional[LTXCaps]: caps.supports_two_stage_refine = is_ltx2 if is_ltx2: - if variant == '2.3': + if variant == '2.5': + caps.upsample_repo = 'Lightricks/LTX-2.5-Diffusers' + caps.stg_default_blocks = [28] + caps.supports_auto_duration = True + elif variant == '2.3': + caps.upsample_repo = 'CalamitousFelicitousness/LTX-2.3-Spatial-Upsampler-x2-1.1-Diffusers' caps.stg_default_blocks = [28] - elif variant == '2.0': - caps.stg_default_blocks = [29] else: - caps.stg_default_blocks = [28] + caps.upsample_repo = 'Lightricks/LTX-2' + caps.stg_default_blocks = [29] if not is_distilled: # canonical T2V composition from huggingface/diffusers#13217 caps.stg_default_scale = 1.0 caps.modality_default_scale = 3.0 caps.guidance_rescale_default = 0.7 + if variant == '2.5': + caps.scheduler_shift_terminal = 0.1 return caps diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 5342bf597..c684dcee2 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -12,16 +12,12 @@ from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsamp apply_ltx_diffusers_patch() from modules.processing_callbacks import diffusers_callback from modules.video_models.video_vae import set_vae_params -from modules.video_models.video_save import save_video +from modules.video_models.video_save import save_video, get_audio_rate from modules.video_models.video_utils import check_av debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None upsample_repo_id_09 = 'a-r-r-o-w/LTX-Video-0.9.7-Latent-Spatial-Upsampler-diffusers' -# Upsampler weights are tied to the family VAE; using the wrong one preserves structure -# but drifts per-channel latent statistics (decodes desaturated / crushed contrast). -upsample_repo_id_20 = 'Lightricks/LTX-2' -upsample_repo_id_23 = 'CalamitousFelicitousness/LTX-2.3-Spatial-Upsampler-x2-1.1-Diffusers' upsample_pipe = None upsample_pipe_2x = None @@ -93,7 +89,7 @@ def _latent_pass(caps, prompt_embeds, prompt_attention_mask, negative_prompt_emb 'negative_prompt_attention_mask': negative_prompt_attention_mask, 'width': get_bucket(width), 'height': get_bucket(height), - 'num_frames': get_frames(frames), + 'num_frames': get_frames(frames) if frames is not None else None, # None defers to the duration head 'num_inference_steps': steps, 'generator': get_generator(seed), 'callback_on_step_end': diffusers_callback, @@ -131,6 +127,7 @@ def run_ltx(task_id, width: int, height: int, frames: int, + auto_duration: bool, steps: int, sampler_index: int, guidance_scale: float, @@ -199,6 +196,10 @@ def run_ltx(task_id, yield from abort(f'Video: cls={shared.sd_model.__class__.__name__} selected model is not LTX', ok=True) return + auto_frames = bool(auto_duration) and caps.supports_auto_duration + if auto_duration and not auto_frames: + log.warning(f'LTX: model="{model}" auto duration unsupported, using frames={get_frames(frames)}') + # Lightricks TI2VidTwoStagesPipeline: Stage 1 at half-res, 2x upsample, Stage 2 refine at target. # Auto-couple when the user picks Refine but not Upsample. Both Dev and Distilled refine paths # expect upsampled latents; same-res refine on Distilled produces oversaturation. Condition @@ -328,14 +329,15 @@ def run_ltx(task_id, p.task_args.update(_canonical_ltx2_guidance(caps)) if caps.family == '2.x': p.task_args['use_cross_timestep'] = caps.use_cross_timestep + if auto_frames: + p.task_args['num_frames'] = None framewise = caps.family == '0.9' set_vae_params(p, framewise=framewise) # Scheduler + shared.opts mutation is wrapped in ltx_scheduler_opts so restore runs on - # every exit path (normal return, abort, interrupt, Stage 2 scheduler swap). See the - # helper's docstring for the five pieces of state it snapshots. - with ltx_scheduler_opts(shared.sd_model, dynamic_shift=dynamic_shift, sampler_shift=sampler_shift): + # every exit path (normal return, abort, interrupt, Stage 2 scheduler swap). + with ltx_scheduler_opts(shared.sd_model, dynamic_shift=dynamic_shift, sampler_shift=sampler_shift, shift_terminal=caps.scheduler_shift_terminal): if selected is not None: video_overrides.set_overrides(p, selected) @@ -382,7 +384,7 @@ def run_ltx(task_id, negative_prompt_attention_mask=negative_prompt_attention_mask, width=base_w, height=base_h, - frames=frames, + frames=None if auto_frames else frames, steps=steps, guidance_scale=p.cfg_scale, mp4_fps=mp4_fps, @@ -391,6 +393,11 @@ def run_ltx(task_id, seed=p.seed, image=p.task_args.get('image'), ) + if auto_frames and torch.is_tensor(latents): + # upsample and refine take the realized length; re-predicting would drift + frames = (latents.shape[-3] - 1) * getattr(shared.sd_model, 'vae_temporal_compression_ratio', 8) + 1 + p.frames = frames + log.debug(f'LTX: auto duration frames={frames}') else: processed = processing.process_images(p) if processed is None or processed.images is None or len(processed.images) == 0: @@ -451,8 +458,7 @@ def run_ltx(task_id, upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe, exclude=upsample_exclude, silent=True) else: global upsample_pipe_2x # pylint: disable=global-statement - upsample_repo = upsample_repo_id_23 if caps.variant == '2.3' else upsample_repo_id_20 - upsample_pipe_2x = load_upsample_2x(upsample_pipe_2x, upsample_repo) + upsample_pipe_2x = load_upsample_2x(upsample_pipe_2x, caps.upsample_repo, caps.variant) upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x, exclude=upsample_exclude, silent=True) # 2.x base pass returns denormalized latents; latents_normalized=False tells the # upsampler "already raw, do not denormalize again". @@ -534,12 +540,15 @@ def run_ltx(task_id, shift_terminal=None, ) if caps.supports_canonical_stage2: - log.debug(f'LTX: stage=2 distilled=LoRA repo={caps.stage2_dev_lora_repo}') + log.debug(f'LTX: stage=2 distilled=LoRA repo={caps.stage2_dev_lora_repo} weight={caps.stage2_dev_lora_weight}') offline_args = {'local_files_only': True} if shared.opts.offline_mode else {} + # 2.5 keeps the LoRA in the model repo, so the file has to be named + lora_args ={'weight_name': caps.stage2_dev_lora_weight} if caps.stage2_dev_lora_weight is not None else {} shared.sd_model.load_lora_weights( caps.stage2_dev_lora_repo, adapter_name=STAGE2_DEV_LORA_ADAPTER, cache_dir=shared.opts.hfcache_dir, + **lora_args, **offline_args, ) shared.sd_model.set_adapters([STAGE2_DEV_LORA_ADAPTER], [1.0]) @@ -628,10 +637,7 @@ def run_ltx(task_id, if not audio_enable: audio = None - try: - aac_sample_rate = shared.sd_model.vocoder.config.output_sampling_rate - except Exception: - aac_sample_rate = 24000 + aac_sample_rate = get_audio_rate(p) if mp4_interpolate > 0 and pixels is not None: p.video_interpolate = mp4_interpolate diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index 3d381933f..550b79686 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -29,6 +29,7 @@ def _model_change(model_name: str): gr.update(interactive=False), # decode_timestep gr.update(interactive=False), # image_cond_noise_scale gr.update(visible=False), # audio_accordion + gr.update(visible=False, value=False), # auto_duration ) # 2.x refine runs fixed canonical schedules; refine_strength only feeds 0.9.x LTXConditionPipeline. refine_strength_interactive = caps.family == '0.9' @@ -36,6 +37,7 @@ def _model_change(model_name: str): # Distilled T2V/I2V). auto_refine_upsample at ltx_process.py:179 couples the stages once Refine # is on. Condition variants are excluded by supports_two_stage_refine. refine_default = caps.supports_two_stage_refine + auto_duration_update =gr.update(visible=True) if caps.supports_auto_duration else gr.update(visible=False, value=False) return ( gr.update(visible=caps.supports_input_media), gr.update(visible=caps.supports_multi_condition), @@ -52,6 +54,7 @@ def _model_change(model_name: str): gr.update(interactive=caps.supports_decode_timestep), gr.update(interactive=caps.supports_image_cond_noise_scale), gr.update(visible=caps.supports_audio), + auto_duration_update, ) @@ -71,6 +74,8 @@ def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_i seed = gr.Number(label='LTX seed', value=-1, elem_id='ltx_seed', container=True) random_seed = ToolButton(ui_symbols.random, elem_id='ltx_seed_random') random_seed.click(fn=lambda: -1, show_progress='hidden', inputs=[], outputs=[seed]) + with gr.Row(): + auto_duration = gr.Checkbox(label='LTX auto duration', value=False, elem_id='ltx_auto_duration', visible=False) input_media_accordion = gr.Accordion(open=False, label="Input media", elem_id='ltx_input_media_accordion', visible=False) with input_media_accordion: ltx_init_image = gr.Image(label='Image', elem_id='ltx_init_image', type='pil', image_mode='RGB', width=256, height=256) @@ -145,6 +150,7 @@ def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_i decode_timestep, image_cond_noise_scale, audio_accordion, + auto_duration, ], ) @@ -155,7 +161,7 @@ def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_i video_inputs = [ model, prompt, negative, styles, - width, height, frames, + width, height, frames, auto_duration, steps, sampler_index, guidance_scale, sampler_shift, dynamic_shift, seed, diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index 107dafab1..12dcdc27d 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -149,6 +149,52 @@ try: 'LTX Video': [ Model(name='None'), + Model(name='─────── LTX-2.5 ───────'), + Model(name='─── Distilled ───'), + Model(name='LTXVideo 2.5 22B T2V Distilled', + url='https://huggingface.co/Lightricks/LTX-2.5', + repo='Lightricks/LTX-2.5-Diffusers', + repo_cls='LTX2Pipeline', + te_cls='Gemma4UnifiedForConditionalGeneration', + dit_cls='LTX2VideoTransformer3DModel'), + Model(name='LTXVideo 2.5 22B I2V Distilled', + url='https://huggingface.co/Lightricks/LTX-2.5', + repo='Lightricks/LTX-2.5-Diffusers', + repo_cls='LTX2ImageToVideoPipeline', + te_cls='Gemma4UnifiedForConditionalGeneration', + dit_cls='LTX2VideoTransformer3DModel'), + Model(name='LTXVideo 2.5 22B Condition Distilled', + url='https://huggingface.co/Lightricks/LTX-2.5', + repo='Lightricks/LTX-2.5-Diffusers', + repo_cls='LTX2ConditionPipeline', + te_cls='Gemma4UnifiedForConditionalGeneration', + dit_cls='LTX2VideoTransformer3DModel'), + Model(name='─── Dev ───'), # transformer_full is the guided model, transformer the distilled one + Model(name='LTXVideo 2.5 22B T2V Dev', + url='https://huggingface.co/Lightricks/LTX-2.5', + repo='Lightricks/LTX-2.5-Diffusers', + repo_cls='LTX2Pipeline', + te_cls='Gemma4UnifiedForConditionalGeneration', + dit_cls='LTX2VideoTransformer3DModel', + dit_folder='transformer_full', + dit_kwarg='transformer'), + Model(name='LTXVideo 2.5 22B I2V Dev', + url='https://huggingface.co/Lightricks/LTX-2.5', + repo='Lightricks/LTX-2.5-Diffusers', + repo_cls='LTX2ImageToVideoPipeline', + te_cls='Gemma4UnifiedForConditionalGeneration', + dit_cls='LTX2VideoTransformer3DModel', + dit_folder='transformer_full', + dit_kwarg='transformer'), + Model(name='LTXVideo 2.5 22B Condition Dev', + url='https://huggingface.co/Lightricks/LTX-2.5', + repo='Lightricks/LTX-2.5-Diffusers', + repo_cls='LTX2ConditionPipeline', + te_cls='Gemma4UnifiedForConditionalGeneration', + dit_cls='LTX2VideoTransformer3DModel', + dit_folder='transformer_full', + dit_kwarg='transformer'), + Model(name='─────── LTX-2.3 v1.1 ───────'), Model(name='LTXVideo 2.3-1.1 22B T2V Distilled', url='https://huggingface.co/Lightricks/LTX-2.3', diff --git a/modules/video_models/video_overrides.py b/modules/video_models/video_overrides.py index 69afe7fb4..1964bdc59 100644 --- a/modules/video_models/video_overrides.py +++ b/modules/video_models/video_overrides.py @@ -17,17 +17,24 @@ def load_override(selected: Model, **load_args): # LTX if 'LTXVideo 0.9.5 I2V' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLLTXVideo.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir, **load_args) - # OzzyGT LTX-2.3 mirrors pack connectors/ twice by design: sharded (*-00001-of-0000N + - # .index.json) and unsharded diffusion_pytorch_model.safetensors of the byte-identical - # weights. snapshot_download faithfully fetches both; diffusers' component loader picks - # sharded when the index is present. ignore_patterns skips the ~6.3 GB unsharded copy - # without reaching for a cleaner upstream mirror. + # OzzyGT LTX-2.3 mirrors and the LTX-2.5 repo pack connectors/ twice by design: sharded + # (*-00001-of-0000N + .index.json) and unsharded diffusion_pytorch_model.safetensors of the + # byte-identical weights. snapshot_download faithfully fetches both; diffusers' component + # loader picks sharded when the index is present. ignore_patterns skips the ~6.3 GB unsharded + # copy without reaching for a cleaner upstream mirror. ltx2_redundant_connector_repos = { 'OzzyGT/LTX-2.3', 'OzzyGT/LTX-2.3-sdnq-dynamic-int4', } - if selected.repo in ltx2_redundant_connector_repos: - kwargs['ignore_patterns'] = ['connectors/diffusion_pytorch_model.safetensors'] + ltx2_ignore = [] + if selected.repo in ltx2_redundant_connector_repos or 'LTXVideo 2.5' in selected.name: + ltx2_ignore.append('connectors/diffusion_pytorch_model.safetensors') + if 'LTXVideo 2.5' in selected.name: + # the pipeline fetch pulls every model-index folder except passed components: transformer_full + # is not one, the diffusion decoder is a separate pipeline, the LoRA is fetched on demand + ltx2_ignore += ['transformer_full/*', 'diffusion_decoder/*', 'ltx-2.5-22b-distilled-lora-450-bf16.safetensors'] + if ltx2_ignore: + kwargs['ignore_patterns'] = ltx2_ignore # LTX2TextConnectors weights are byte-identical across all 2.3 variants (verified by blob # hash). Pre-load from a canonical repo so per-variant fetches skip connectors/ entirely. # FP16 variants share OzzyGT/LTX-2.3; SDNQ variants share the pre-quantized mirror. diff --git a/ui/locale/locale_en.json b/ui/locale/locale_en.json index 532f1c81c..b216e8c41 100644 --- a/ui/locale/locale_en.json +++ b/ui/locale/locale_en.json @@ -832,10 +832,11 @@ {"id":"","label":"LTX model","localized":"","hint":"","ui":"video"}, {"id":"","label":"LTX frames number","localized":"","hint":"","ui":"video"}, {"id":"","label":"LTX frames skip","localized":"","hint":"","ui":"video"}, - {"id":"","label":"LTX enable upsampling","localized":"","hint":"","ui":"video"}, - {"id":"","label":"LTX upsample ratio","localized":"","hint":"","ui":"video"}, - {"id":"","label":"LTX enable refine","localized":"","hint":"","ui":"video"}, - {"id":"","label":"LTX refine strength","localized":"","hint":"","ui":"video"}, + {"id":"","label":"LTX auto duration","localized":"","hint":"Clip length is predicted from the prompt and the frames setting is ignored","ui":"video"}, + {"id":"","label":"LTX upscale","localized":"","hint":"","ui":"video"}, + {"id":"","label":"LTX scale","localized":"","hint":"","ui":"video"}, + {"id":"","label":"LTX refine","localized":"","hint":"","ui":"video"}, + {"id":"","label":"LTX strength","localized":"","hint":"","ui":"video"}, {"id":"","label":"LTX decode timestep","localized":"","hint":"","ui":"video"}, {"id":"","label":"LTX save audio","localized":"","hint":"LTX-2 audio-capable models always generate audio from the same prompt as video; this toggle controls whether the audio track is included in the saved video file","ui":"video"}, {"id":"","label":"Loop","localized":"","hint":"","ui":"extras"},