mirror of
https://github.com/vladmandic/automatic
synced 2026-09-19 17:24:32 +02:00
feat(ltx): add the ltx-2.5 model family
2.5 reuses the LTX-2 pipeline classes, so it is described through the capability table: Gemma 4 text encoder, cross-timestep conditioning, and the upsampler and stage 2 LoRA that now ship inside the model repo. The repo carries the distilled scheduler config, so Dev rows restore the terminal shift, and the Dev transformer sits in transformer_full. Distilled rows stop forcing dynamic shifting on, which remapped their sigma schedule. Auto duration hands the clip length to the duration head.
This commit is contained in:
CalamitousFelicitousness
@@ -9,7 +9,7 @@ class LTXCaps:
|
||||
name: str
|
||||
repo_cls_name: str
|
||||
family: str # '0.9' or '2.x'
|
||||
variant: str # '0.9', '2.0', '2.3' (finer-grained sub-variant)
|
||||
variant: str # '0.9', '2.0', '2.3', '2.5' (finer-grained sub-variant)
|
||||
is_distilled: bool
|
||||
is_i2v: bool
|
||||
supports_input_media: bool
|
||||
@@ -41,6 +41,13 @@ class LTXCaps:
|
||||
# for refine; same-res refine oversaturates. Condition variants rebuild conditions per stage.
|
||||
supports_two_stage_refine: bool = False
|
||||
stage2_dev_lora_repo: Optional[str] = None
|
||||
# 2.5 keeps the stage 2 LoRA in the model repo; 2.0 and 2.3 each have their own
|
||||
stage2_dev_lora_weight: Optional[str] = None
|
||||
# tied to the family VAE: the wrong one drifts per-channel latent statistics
|
||||
upsample_repo: Optional[str] = None
|
||||
# 2.5 ships the distilled scheduler config, so its Dev rows restore the terminal shift
|
||||
scheduler_shift_terminal: Optional[float] = None
|
||||
supports_auto_duration: bool = False
|
||||
|
||||
|
||||
CONDITION_CLASSES = {'LTXConditionPipeline', 'LTX2ConditionPipeline'}
|
||||
@@ -84,7 +91,12 @@ def get_caps(model_name: str) -> Optional[LTXCaps]:
|
||||
family = '2.x' if is_ltx2 else '0.9'
|
||||
# 2.x sub-variant detection: unknown 2.x mirrors fall through to '2.0' (conservative default).
|
||||
if is_ltx2:
|
||||
variant = '2.3' if '2.3' in model_name else '2.0'
|
||||
if '2.5' in model_name:
|
||||
variant = '2.5'
|
||||
elif '2.3' in model_name:
|
||||
variant = '2.3'
|
||||
else:
|
||||
variant = '2.0'
|
||||
else:
|
||||
variant = '0.9'
|
||||
is_distilled = 'Distilled' in model_name
|
||||
@@ -106,11 +118,12 @@ def get_caps(model_name: str) -> Optional[LTXCaps]:
|
||||
supports_stg=is_ltx2,
|
||||
supports_audio=is_ltx2,
|
||||
supports_frame_rate_kwarg=is_ltx2,
|
||||
use_cross_timestep=(variant == '2.3'),
|
||||
use_cross_timestep=variant in ('2.3', '2.5'),
|
||||
default_cfg=3.0,
|
||||
default_steps=30 if is_ltx2 else 50,
|
||||
default_sampler_shift=-1.0,
|
||||
default_dynamic_shift=is_ltx2,
|
||||
# distilled ships use_dynamic_shifting=False and runs explicit sigmas, which shifting remaps
|
||||
default_dynamic_shift=is_ltx2 and not is_distilled,
|
||||
default_width=768,
|
||||
default_height=512,
|
||||
default_frames=121 if is_ltx2 else 161,
|
||||
@@ -122,7 +135,10 @@ def get_caps(model_name: str) -> Optional[LTXCaps]:
|
||||
caps.default_steps = 8
|
||||
|
||||
if is_ltx2 and not is_distilled:
|
||||
if variant == '2.3':
|
||||
if variant == '2.5':
|
||||
caps.stage2_dev_lora_repo = 'Lightricks/LTX-2.5-Diffusers'
|
||||
caps.stage2_dev_lora_weight = 'ltx-2.5-22b-distilled-lora-450-bf16.safetensors'
|
||||
elif variant == '2.3':
|
||||
caps.stage2_dev_lora_repo = 'CalamitousFelicitousness/LTX-2.3-distilled-lora-384-Diffusers'
|
||||
elif variant == '2.0':
|
||||
caps.stage2_dev_lora_repo = 'CalamitousFelicitousness/LTX-2.0-distilled-lora-384-Diffusers'
|
||||
@@ -130,16 +146,22 @@ def get_caps(model_name: str) -> Optional[LTXCaps]:
|
||||
caps.supports_two_stage_refine = is_ltx2
|
||||
|
||||
if is_ltx2:
|
||||
if variant == '2.3':
|
||||
if variant == '2.5':
|
||||
caps.upsample_repo = 'Lightricks/LTX-2.5-Diffusers'
|
||||
caps.stg_default_blocks = [28]
|
||||
caps.supports_auto_duration = True
|
||||
elif variant == '2.3':
|
||||
caps.upsample_repo = 'CalamitousFelicitousness/LTX-2.3-Spatial-Upsampler-x2-1.1-Diffusers'
|
||||
caps.stg_default_blocks = [28]
|
||||
elif variant == '2.0':
|
||||
caps.stg_default_blocks = [29]
|
||||
else:
|
||||
caps.stg_default_blocks = [28]
|
||||
caps.upsample_repo = 'Lightricks/LTX-2'
|
||||
caps.stg_default_blocks = [29]
|
||||
if not is_distilled:
|
||||
# canonical T2V composition from huggingface/diffusers#13217
|
||||
caps.stg_default_scale = 1.0
|
||||
caps.modality_default_scale = 3.0
|
||||
caps.guidance_rescale_default = 0.7
|
||||
if variant == '2.5':
|
||||
caps.scheduler_shift_terminal = 0.1
|
||||
|
||||
return caps
|
||||
|
||||
+23
-17
@@ -12,16 +12,12 @@ from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsamp
|
||||
apply_ltx_diffusers_patch()
|
||||
from modules.processing_callbacks import diffusers_callback
|
||||
from modules.video_models.video_vae import set_vae_params
|
||||
from modules.video_models.video_save import save_video
|
||||
from modules.video_models.video_save import save_video, get_audio_rate
|
||||
from modules.video_models.video_utils import check_av
|
||||
|
||||
|
||||
debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
|
||||
upsample_repo_id_09 = 'a-r-r-o-w/LTX-Video-0.9.7-Latent-Spatial-Upsampler-diffusers'
|
||||
# Upsampler weights are tied to the family VAE; using the wrong one preserves structure
|
||||
# but drifts per-channel latent statistics (decodes desaturated / crushed contrast).
|
||||
upsample_repo_id_20 = 'Lightricks/LTX-2'
|
||||
upsample_repo_id_23 = 'CalamitousFelicitousness/LTX-2.3-Spatial-Upsampler-x2-1.1-Diffusers'
|
||||
upsample_pipe = None
|
||||
upsample_pipe_2x = None
|
||||
|
||||
@@ -93,7 +89,7 @@ def _latent_pass(caps, prompt_embeds, prompt_attention_mask, negative_prompt_emb
|
||||
'negative_prompt_attention_mask': negative_prompt_attention_mask,
|
||||
'width': get_bucket(width),
|
||||
'height': get_bucket(height),
|
||||
'num_frames': get_frames(frames),
|
||||
'num_frames': get_frames(frames) if frames is not None else None, # None defers to the duration head
|
||||
'num_inference_steps': steps,
|
||||
'generator': get_generator(seed),
|
||||
'callback_on_step_end': diffusers_callback,
|
||||
@@ -131,6 +127,7 @@ def run_ltx(task_id,
|
||||
width: int,
|
||||
height: int,
|
||||
frames: int,
|
||||
auto_duration: bool,
|
||||
steps: int,
|
||||
sampler_index: int,
|
||||
guidance_scale: float,
|
||||
@@ -199,6 +196,10 @@ def run_ltx(task_id,
|
||||
yield from abort(f'Video: cls={shared.sd_model.__class__.__name__} selected model is not LTX', ok=True)
|
||||
return
|
||||
|
||||
auto_frames = bool(auto_duration) and caps.supports_auto_duration
|
||||
if auto_duration and not auto_frames:
|
||||
log.warning(f'LTX: model="{model}" auto duration unsupported, using frames={get_frames(frames)}')
|
||||
|
||||
# Lightricks TI2VidTwoStagesPipeline: Stage 1 at half-res, 2x upsample, Stage 2 refine at target.
|
||||
# Auto-couple when the user picks Refine but not Upsample. Both Dev and Distilled refine paths
|
||||
# expect upsampled latents; same-res refine on Distilled produces oversaturation. Condition
|
||||
@@ -328,14 +329,15 @@ def run_ltx(task_id,
|
||||
p.task_args.update(_canonical_ltx2_guidance(caps))
|
||||
if caps.family == '2.x':
|
||||
p.task_args['use_cross_timestep'] = caps.use_cross_timestep
|
||||
if auto_frames:
|
||||
p.task_args['num_frames'] = None
|
||||
|
||||
framewise = caps.family == '0.9'
|
||||
set_vae_params(p, framewise=framewise)
|
||||
|
||||
# Scheduler + shared.opts mutation is wrapped in ltx_scheduler_opts so restore runs on
|
||||
# every exit path (normal return, abort, interrupt, Stage 2 scheduler swap). See the
|
||||
# helper's docstring for the five pieces of state it snapshots.
|
||||
with ltx_scheduler_opts(shared.sd_model, dynamic_shift=dynamic_shift, sampler_shift=sampler_shift):
|
||||
# every exit path (normal return, abort, interrupt, Stage 2 scheduler swap).
|
||||
with ltx_scheduler_opts(shared.sd_model, dynamic_shift=dynamic_shift, sampler_shift=sampler_shift, shift_terminal=caps.scheduler_shift_terminal):
|
||||
if selected is not None:
|
||||
video_overrides.set_overrides(p, selected)
|
||||
|
||||
@@ -382,7 +384,7 @@ def run_ltx(task_id,
|
||||
negative_prompt_attention_mask=negative_prompt_attention_mask,
|
||||
width=base_w,
|
||||
height=base_h,
|
||||
frames=frames,
|
||||
frames=None if auto_frames else frames,
|
||||
steps=steps,
|
||||
guidance_scale=p.cfg_scale,
|
||||
mp4_fps=mp4_fps,
|
||||
@@ -391,6 +393,11 @@ def run_ltx(task_id,
|
||||
seed=p.seed,
|
||||
image=p.task_args.get('image'),
|
||||
)
|
||||
if auto_frames and torch.is_tensor(latents):
|
||||
# upsample and refine take the realized length; re-predicting would drift
|
||||
frames = (latents.shape[-3] - 1) * getattr(shared.sd_model, 'vae_temporal_compression_ratio', 8) + 1
|
||||
p.frames = frames
|
||||
log.debug(f'LTX: auto duration frames={frames}')
|
||||
else:
|
||||
processed = processing.process_images(p)
|
||||
if processed is None or processed.images is None or len(processed.images) == 0:
|
||||
@@ -451,8 +458,7 @@ def run_ltx(task_id,
|
||||
upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe, exclude=upsample_exclude, silent=True)
|
||||
else:
|
||||
global upsample_pipe_2x # pylint: disable=global-statement
|
||||
upsample_repo = upsample_repo_id_23 if caps.variant == '2.3' else upsample_repo_id_20
|
||||
upsample_pipe_2x = load_upsample_2x(upsample_pipe_2x, upsample_repo)
|
||||
upsample_pipe_2x = load_upsample_2x(upsample_pipe_2x, caps.upsample_repo, caps.variant)
|
||||
upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x, exclude=upsample_exclude, silent=True)
|
||||
# 2.x base pass returns denormalized latents; latents_normalized=False tells the
|
||||
# upsampler "already raw, do not denormalize again".
|
||||
@@ -534,12 +540,15 @@ def run_ltx(task_id,
|
||||
shift_terminal=None,
|
||||
)
|
||||
if caps.supports_canonical_stage2:
|
||||
log.debug(f'LTX: stage=2 distilled=LoRA repo={caps.stage2_dev_lora_repo}')
|
||||
log.debug(f'LTX: stage=2 distilled=LoRA repo={caps.stage2_dev_lora_repo} weight={caps.stage2_dev_lora_weight}')
|
||||
offline_args = {'local_files_only': True} if shared.opts.offline_mode else {}
|
||||
# 2.5 keeps the LoRA in the model repo, so the file has to be named
|
||||
lora_args ={'weight_name': caps.stage2_dev_lora_weight} if caps.stage2_dev_lora_weight is not None else {}
|
||||
shared.sd_model.load_lora_weights(
|
||||
caps.stage2_dev_lora_repo,
|
||||
adapter_name=STAGE2_DEV_LORA_ADAPTER,
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
**lora_args,
|
||||
**offline_args,
|
||||
)
|
||||
shared.sd_model.set_adapters([STAGE2_DEV_LORA_ADAPTER], [1.0])
|
||||
@@ -628,10 +637,7 @@ def run_ltx(task_id,
|
||||
if not audio_enable:
|
||||
audio = None
|
||||
|
||||
try:
|
||||
aac_sample_rate = shared.sd_model.vocoder.config.output_sampling_rate
|
||||
except Exception:
|
||||
aac_sample_rate = 24000
|
||||
aac_sample_rate = get_audio_rate(p)
|
||||
|
||||
if mp4_interpolate > 0 and pixels is not None:
|
||||
p.video_interpolate = mp4_interpolate
|
||||
|
||||
@@ -29,6 +29,7 @@ def _model_change(model_name: str):
|
||||
gr.update(interactive=False), # decode_timestep
|
||||
gr.update(interactive=False), # image_cond_noise_scale
|
||||
gr.update(visible=False), # audio_accordion
|
||||
gr.update(visible=False, value=False), # auto_duration
|
||||
)
|
||||
# 2.x refine runs fixed canonical schedules; refine_strength only feeds 0.9.x LTXConditionPipeline.
|
||||
refine_strength_interactive = caps.family == '0.9'
|
||||
@@ -36,6 +37,7 @@ def _model_change(model_name: str):
|
||||
# Distilled T2V/I2V). auto_refine_upsample at ltx_process.py:179 couples the stages once Refine
|
||||
# is on. Condition variants are excluded by supports_two_stage_refine.
|
||||
refine_default = caps.supports_two_stage_refine
|
||||
auto_duration_update =gr.update(visible=True) if caps.supports_auto_duration else gr.update(visible=False, value=False)
|
||||
return (
|
||||
gr.update(visible=caps.supports_input_media),
|
||||
gr.update(visible=caps.supports_multi_condition),
|
||||
@@ -52,6 +54,7 @@ def _model_change(model_name: str):
|
||||
gr.update(interactive=caps.supports_decode_timestep),
|
||||
gr.update(interactive=caps.supports_image_cond_noise_scale),
|
||||
gr.update(visible=caps.supports_audio),
|
||||
auto_duration_update,
|
||||
)
|
||||
|
||||
|
||||
@@ -71,6 +74,8 @@ def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_i
|
||||
seed = gr.Number(label='LTX seed', value=-1, elem_id='ltx_seed', container=True)
|
||||
random_seed = ToolButton(ui_symbols.random, elem_id='ltx_seed_random')
|
||||
random_seed.click(fn=lambda: -1, show_progress='hidden', inputs=[], outputs=[seed])
|
||||
with gr.Row():
|
||||
auto_duration = gr.Checkbox(label='LTX auto duration', value=False, elem_id='ltx_auto_duration', visible=False)
|
||||
input_media_accordion = gr.Accordion(open=False, label="Input media", elem_id='ltx_input_media_accordion', visible=False)
|
||||
with input_media_accordion:
|
||||
ltx_init_image = gr.Image(label='Image', elem_id='ltx_init_image', type='pil', image_mode='RGB', width=256, height=256)
|
||||
@@ -145,6 +150,7 @@ def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_i
|
||||
decode_timestep,
|
||||
image_cond_noise_scale,
|
||||
audio_accordion,
|
||||
auto_duration,
|
||||
],
|
||||
)
|
||||
|
||||
@@ -155,7 +161,7 @@ def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_i
|
||||
video_inputs = [
|
||||
model,
|
||||
prompt, negative, styles,
|
||||
width, height, frames,
|
||||
width, height, frames, auto_duration,
|
||||
steps, sampler_index,
|
||||
guidance_scale, sampler_shift, dynamic_shift,
|
||||
seed,
|
||||
|
||||
@@ -149,6 +149,52 @@ try:
|
||||
'LTX Video': [
|
||||
Model(name='None'),
|
||||
|
||||
Model(name='─────── LTX-2.5 ───────'),
|
||||
Model(name='─── Distilled ───'),
|
||||
Model(name='LTXVideo 2.5 22B T2V Distilled',
|
||||
url='https://huggingface.co/Lightricks/LTX-2.5',
|
||||
repo='Lightricks/LTX-2.5-Diffusers',
|
||||
repo_cls='LTX2Pipeline',
|
||||
te_cls='Gemma4UnifiedForConditionalGeneration',
|
||||
dit_cls='LTX2VideoTransformer3DModel'),
|
||||
Model(name='LTXVideo 2.5 22B I2V Distilled',
|
||||
url='https://huggingface.co/Lightricks/LTX-2.5',
|
||||
repo='Lightricks/LTX-2.5-Diffusers',
|
||||
repo_cls='LTX2ImageToVideoPipeline',
|
||||
te_cls='Gemma4UnifiedForConditionalGeneration',
|
||||
dit_cls='LTX2VideoTransformer3DModel'),
|
||||
Model(name='LTXVideo 2.5 22B Condition Distilled',
|
||||
url='https://huggingface.co/Lightricks/LTX-2.5',
|
||||
repo='Lightricks/LTX-2.5-Diffusers',
|
||||
repo_cls='LTX2ConditionPipeline',
|
||||
te_cls='Gemma4UnifiedForConditionalGeneration',
|
||||
dit_cls='LTX2VideoTransformer3DModel'),
|
||||
Model(name='─── Dev ───'), # transformer_full is the guided model, transformer the distilled one
|
||||
Model(name='LTXVideo 2.5 22B T2V Dev',
|
||||
url='https://huggingface.co/Lightricks/LTX-2.5',
|
||||
repo='Lightricks/LTX-2.5-Diffusers',
|
||||
repo_cls='LTX2Pipeline',
|
||||
te_cls='Gemma4UnifiedForConditionalGeneration',
|
||||
dit_cls='LTX2VideoTransformer3DModel',
|
||||
dit_folder='transformer_full',
|
||||
dit_kwarg='transformer'),
|
||||
Model(name='LTXVideo 2.5 22B I2V Dev',
|
||||
url='https://huggingface.co/Lightricks/LTX-2.5',
|
||||
repo='Lightricks/LTX-2.5-Diffusers',
|
||||
repo_cls='LTX2ImageToVideoPipeline',
|
||||
te_cls='Gemma4UnifiedForConditionalGeneration',
|
||||
dit_cls='LTX2VideoTransformer3DModel',
|
||||
dit_folder='transformer_full',
|
||||
dit_kwarg='transformer'),
|
||||
Model(name='LTXVideo 2.5 22B Condition Dev',
|
||||
url='https://huggingface.co/Lightricks/LTX-2.5',
|
||||
repo='Lightricks/LTX-2.5-Diffusers',
|
||||
repo_cls='LTX2ConditionPipeline',
|
||||
te_cls='Gemma4UnifiedForConditionalGeneration',
|
||||
dit_cls='LTX2VideoTransformer3DModel',
|
||||
dit_folder='transformer_full',
|
||||
dit_kwarg='transformer'),
|
||||
|
||||
Model(name='─────── LTX-2.3 v1.1 ───────'),
|
||||
Model(name='LTXVideo 2.3-1.1 22B T2V Distilled',
|
||||
url='https://huggingface.co/Lightricks/LTX-2.3',
|
||||
|
||||
@@ -17,17 +17,24 @@ def load_override(selected: Model, **load_args):
|
||||
# LTX
|
||||
if 'LTXVideo 0.9.5 I2V' in selected.name:
|
||||
kwargs['vae'] = diffusers.AutoencoderKLLTXVideo.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir, **load_args)
|
||||
# OzzyGT LTX-2.3 mirrors pack connectors/ twice by design: sharded (*-00001-of-0000N +
|
||||
# .index.json) and unsharded diffusion_pytorch_model.safetensors of the byte-identical
|
||||
# weights. snapshot_download faithfully fetches both; diffusers' component loader picks
|
||||
# sharded when the index is present. ignore_patterns skips the ~6.3 GB unsharded copy
|
||||
# without reaching for a cleaner upstream mirror.
|
||||
# OzzyGT LTX-2.3 mirrors and the LTX-2.5 repo pack connectors/ twice by design: sharded
|
||||
# (*-00001-of-0000N + .index.json) and unsharded diffusion_pytorch_model.safetensors of the
|
||||
# byte-identical weights. snapshot_download faithfully fetches both; diffusers' component
|
||||
# loader picks sharded when the index is present. ignore_patterns skips the ~6.3 GB unsharded
|
||||
# copy without reaching for a cleaner upstream mirror.
|
||||
ltx2_redundant_connector_repos = {
|
||||
'OzzyGT/LTX-2.3',
|
||||
'OzzyGT/LTX-2.3-sdnq-dynamic-int4',
|
||||
}
|
||||
if selected.repo in ltx2_redundant_connector_repos:
|
||||
kwargs['ignore_patterns'] = ['connectors/diffusion_pytorch_model.safetensors']
|
||||
ltx2_ignore = []
|
||||
if selected.repo in ltx2_redundant_connector_repos or 'LTXVideo 2.5' in selected.name:
|
||||
ltx2_ignore.append('connectors/diffusion_pytorch_model.safetensors')
|
||||
if 'LTXVideo 2.5' in selected.name:
|
||||
# the pipeline fetch pulls every model-index folder except passed components: transformer_full
|
||||
# is not one, the diffusion decoder is a separate pipeline, the LoRA is fetched on demand
|
||||
ltx2_ignore += ['transformer_full/*', 'diffusion_decoder/*', 'ltx-2.5-22b-distilled-lora-450-bf16.safetensors']
|
||||
if ltx2_ignore:
|
||||
kwargs['ignore_patterns'] = ltx2_ignore
|
||||
# LTX2TextConnectors weights are byte-identical across all 2.3 variants (verified by blob
|
||||
# hash). Pre-load from a canonical repo so per-variant fetches skip connectors/ entirely.
|
||||
# FP16 variants share OzzyGT/LTX-2.3; SDNQ variants share the pre-quantized mirror.
|
||||
|
||||
Reference in New Issue
Block a user