From 2f00a5764d69157d86084cd98596556fc639d58b Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Thu, 16 Apr 2026 03:38:13 +0100 Subject: [PATCH 01/15] fix(video): friendlier error messages for missing inputs Replace "init image not set" / "last image not set" with actionable messages that tell the user what to do. Matches the phrasing used by the Caption tab for the same failure class. --- modules/video_models/video_run.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 51c781e6f..3454faf69 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -65,14 +65,14 @@ def generate(*args, **kwargs): log.warning('Video: op=T2V init image not supported') elif 'I2V' in model: if init_image is None: - return video_utils.queue_err('init image not set') + return video_utils.queue_err('No input image provided. Please upload or select an image.') p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') log.debug(f'Video: op=I2V init={init_image} resized={p.task_args["image"]}') elif 'FLF2V' in model: if init_image is None: - return video_utils.queue_err('init image not set') + return video_utils.queue_err('No input image provided. Please upload or select an image.') if last_image is None: - return video_utils.queue_err('last image not set') + return video_utils.queue_err('No last frame image provided. Please upload or select an image.') p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') p.task_args['last_image'] = images.resize_image(resize_mode=2, im=last_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') log.debug(f'Video: op=FLF2V init={init_image} last={last_image} resized={p.task_args["image"]}') @@ -82,7 +82,7 @@ def generate(*args, **kwargs): log.debug(f'Video: op=VACE reference={init_image} resized={p.task_args["reference_images"]}') elif 'Animate' in model: if init_image is None: - return video_utils.queue_err('init image not set') + return video_utils.queue_err('No input image provided. Please upload or select an image.') p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') p.task_args['mode'] = 'animate' p.task_args['pose_video'] = [] # input pose video to condition the generation on. must be a list of PIL images. From ca93f0b9b658ff20b105c7592dd04155b89a186b Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Thu, 16 Apr 2026 03:43:44 +0100 Subject: [PATCH 02/15] fix(video): correct pipeline classes for ltx 2.3 i2v and condition variants The 2.3 I2V variants (Dev, Distilled, SDNQ-4Bit, Distilled SDNQ-4Bit) were registered against LTX2Pipeline, whose __call__ does not accept an image kwarg; init images were silently dropped. Route them through LTX2ImageToVideoPipeline instead. Also register the four 2.3 Condition variants under LTX2ConditionPipeline so the multi-condition (image/video/gallery prefix) generation path is reachable on LTX 2.3. --- modules/video_models/models_def.py | 32 ++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index 229e06c50..4e53f0c9f 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -150,7 +150,13 @@ try: Model(name='LTXVideo 2.3 22B I2V', url='https://huggingface.co/Lightricks/LTX-2.3', repo='OzzyGT/LTX-2.3', - repo_cls=getattr(diffusers, 'LTX2Pipeline', None), + repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='LTXVideo 2.3 22B Condition', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3', + repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None), te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), Model(name='LTXVideo 2.3 22B T2V Distilled', @@ -162,7 +168,13 @@ try: Model(name='LTXVideo 2.3 22B I2V Distilled', url='https://huggingface.co/Lightricks/LTX-2.3', repo='OzzyGT/LTX-2.3-Distilled', - repo_cls=getattr(diffusers, 'LTX2Pipeline', None), + repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='LTXVideo 2.3 22B Condition Distilled', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3-Distilled', + repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None), te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), @@ -187,7 +199,13 @@ try: Model(name='LTXVideo 2.3 22B I2V SDNQ-4Bit', url='https://huggingface.co/Lightricks/LTX-2.3', repo='OzzyGT/LTX-2.3-sdnq-dynamic-int4', - repo_cls=getattr(diffusers, 'LTX2Pipeline', None), + repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='LTXVideo 2.3 22B Condition SDNQ-4Bit', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3-sdnq-dynamic-int4', + repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None), te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), Model(name='LTXVideo 2.3 22B T2V Distilled SDNQ-4Bit', @@ -199,7 +217,13 @@ try: Model(name='LTXVideo 2.3 22B I2V Distilled SDNQ-4Bit', url='https://huggingface.co/Lightricks/LTX-2.3', repo='OzzyGT/LTX-2.3-Distilled-sdnq-dynamic-int4', - repo_cls=getattr(diffusers, 'LTX2Pipeline', None), + repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='LTXVideo 2.3 22B Condition Distilled SDNQ-4Bit', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3-Distilled-sdnq-dynamic-int4', + repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None), te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), From acac6157b025b419dfbaf47b574eb1c564416326 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Thu, 16 Apr 2026 03:44:12 +0100 Subject: [PATCH 03/15] refactor(ltx): unify tab across 0.9.x and 2.x pipeline families Rework the LTX Video tab so one UI handles every registered variant (0.9.0 through 2.3, Dev/Distilled/SDNQ-4Bit, T2V/I2V/Condition). Per- variant behavior is driven from a single capability lookup rather than substring matching on model names scattered across the backend. - modules/ltx/ltx_capabilities.py: new module computing family, is_i2v, distilled, supports_input_media, supports_multi_condition, supports_image_cond_noise_scale, supports_decode_timestep, supports_stg, supports_audio, supports_frame_rate_kwarg, and the default CFG / steps / sampler_shift for a given model name by reading its registered repo_cls in models_def. - modules/ltx/ltx_ui.py: capability-gated UI. Selecting a model rewires accordion visibility, slider interactivity, and defaults via a single model.change handler. New controls: dedicated image input slot inside the LTX tab (replaces the disconnected shared init_image for I2V), condition strength slider, CFG / sampler shift / dynamic shift sliders that were previously unreachable. Input media accordion restructured so the image slot is always-visible while the video / gallery prefix tabs only appear on Condition pipelines. - modules/ltx/ltx_process.py: route the base pass through processing.process_images(p) so LTX inherits standard scheduler wiring, extra_networks activation, VAE handling, and error plumbing from StableDiffusionProcessingVideo. The multi-pass latent path (upsample / refine) stays on direct pipeline calls for latent re-entry. Refine noise control gets family-specific kwargs: denoise_strength for 0.9.x LTXConditionPipeline, noise_scale for all 2.x pipelines; the prior strength= injection crashed on 2.x and only affected conditioning intensity on 0.9.x. Add torch_gc between every stage boundary (base to upsample to refine to vae decode) so the CUDA allocator cache does not retain the prior pass's allocations across stages. Remove the TypeError fallback that silently passed raw latents to save_video when VAE decode returned None on OOM; those errors now surface cleanly. - modules/ltx/ltx_util.py: get_conditions grows a family parameter and builds LTX2VideoCondition (frames, index, strength) for 2.x or LTXVideoCondition (image, video, frame_index, strength) for 0.9.x. get_bucket floors to max(32, vae_spatial_compression_ratio) since LTX pipelines validate divisibility by 32 regardless of family. - modules/video_models/video_overrides.py: extend the I2V generator reset to cover LTX2ImageToVideoPipeline and both Condition classes. Keep the strength= kwarg injection gated to 0.9.x LTXConditionPipeline only; LTX2ConditionPipeline.__call__ does not accept it (per-condition strength lives on the LTX2VideoCondition dataclass instead). --- modules/ltx/ltx_capabilities.py | 108 +++++ modules/ltx/ltx_process.py | 498 ++++++++++++++++-------- modules/ltx/ltx_ui.py | 117 ++++-- modules/ltx/ltx_util.py | 75 ++-- modules/video_models/video_overrides.py | 3 +- 5 files changed, 584 insertions(+), 217 deletions(-) create mode 100644 modules/ltx/ltx_capabilities.py diff --git a/modules/ltx/ltx_capabilities.py b/modules/ltx/ltx_capabilities.py new file mode 100644 index 000000000..2dae4c654 --- /dev/null +++ b/modules/ltx/ltx_capabilities.py @@ -0,0 +1,108 @@ +from dataclasses import dataclass, field +from typing import Optional + +from modules.logger import log + + +@dataclass +class LTXCaps: + name: str + repo_cls_name: str + family: str # '0.9' or '2.x' + is_distilled: bool + is_i2v: bool + supports_input_media: bool # accordion visible for any pipeline that accepts image/video input + supports_multi_condition: bool # uses conditions=[LTX(2)VideoCondition(...)] kwarg; Condition classes only + supports_image_cond_noise_scale: bool + supports_decode_timestep: bool + supports_stg: bool + supports_audio: bool + supports_frame_rate_kwarg: bool + default_cfg: float + default_steps: int + default_sampler_shift: float + default_width: int + default_height: int + default_frames: int + default_frame_rate: int + stg_default_scale: float = 0.0 + stg_default_blocks: list = field(default_factory=list) + + +CONDITION_CLASSES = {'LTXConditionPipeline', 'LTX2ConditionPipeline'} +LTX2_CLASSES = {'LTX2Pipeline', 'LTX2ImageToVideoPipeline', 'LTX2ConditionPipeline'} +ALL_LTX_CLASSES = { + 'LTXPipeline', + 'LTXImageToVideoPipeline', + 'LTXConditionPipeline', + 'LTX2Pipeline', + 'LTX2ImageToVideoPipeline', + 'LTX2ConditionPipeline', +} + + +def _repo_cls_name(model_name: str) -> Optional[str]: + from modules.video_models.models_def import models + entries = models.get('LTX Video', []) + for m in entries: + if m.name == model_name: + if m.repo_cls is None: + return None + return m.repo_cls.__name__ + return None + + +def get_caps(model_name: str) -> Optional[LTXCaps]: + if not model_name or model_name == 'None': + return None + cls_name = _repo_cls_name(model_name) + if cls_name is None: + log.warning(f'LTX caps: model="{model_name}" has no repo_cls registered') + return None + if cls_name not in ALL_LTX_CLASSES: + log.warning(f'LTX caps: model="{model_name}" repo_cls="{cls_name}" is not an LTX pipeline') + return None + + is_ltx2 = cls_name in LTX2_CLASSES + family = '2.x' if is_ltx2 else '0.9' + is_distilled = 'Distilled' in model_name + is_i2v = 'I2V' in model_name or cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') + is_condition_cls = cls_name in CONDITION_CLASSES + supports_input_media = is_i2v or is_condition_cls + + caps = LTXCaps( + name=model_name, + repo_cls_name=cls_name, + family=family, + is_distilled=is_distilled, + is_i2v=is_i2v, + supports_input_media=supports_input_media, + supports_multi_condition=is_condition_cls, + supports_image_cond_noise_scale=(cls_name == 'LTXConditionPipeline'), + supports_decode_timestep=(family == '0.9'), + supports_stg=is_ltx2, + supports_audio=is_ltx2, + supports_frame_rate_kwarg=is_ltx2, + default_cfg=4.0 if is_ltx2 else 3.0, + default_steps=40 if is_ltx2 else 50, + default_sampler_shift=-1.0, + default_width=768, + default_height=512, + default_frames=121 if is_ltx2 else 161, + default_frame_rate=24 if is_ltx2 else 25, + ) + + if is_distilled: + caps.default_cfg = 1.0 + caps.default_steps = 8 + + if is_ltx2: + if '2.3' in model_name: + caps.stg_default_blocks = [28] + elif '2.0' in model_name: + caps.stg_default_blocks = [29] + else: + caps.stg_default_blocks = [28] + caps.stg_default_scale = 0.0 + + return caps diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 1b56500d8..605611374 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -1,61 +1,103 @@ import os +import copy import time +import numpy as np import torch from PIL import Image -from modules import shared, errors, timer, memstats, progress, processing, sd_models, sd_samplers, extra_networks, call_queue +from modules import shared, errors, timer, memstats, progress, processing, sd_models, sd_samplers, devices, extra_networks, call_queue from modules.logger import log +from modules.ltx import ltx_capabilities +from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, get_conditions, get_generator, get_prompts, vae_decode +from modules.processing_callbacks import diffusers_callback from modules.video_models.video_vae import set_vae_params from modules.video_models.video_save import save_video from modules.video_models.video_utils import check_av -from modules.processing_callbacks import diffusers_callback -from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, get_conditions, get_generator, get_prompts, vae_decode debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None -# engine, model = 'LTX Video', 'LTXVideo 0.9.7 13B' -upsample_repo_id = "a-r-r-o-w/LTX-Video-0.9.7-Latent-Spatial-Upsampler-diffusers" +upsample_repo_id_09 = 'a-r-r-o-w/LTX-Video-0.9.7-Latent-Spatial-Upsampler-diffusers' upsample_pipe = None +def _latent_pass(caps, prompt, negative, width, height, frames, steps, guidance_scale, mp4_fps, conditions, image_cond_noise_scale, seed, image=None): + base_args = { + 'prompt': prompt, + 'negative_prompt': negative, + 'width': get_bucket(width), + 'height': get_bucket(height), + 'num_frames': get_frames(frames), + 'num_inference_steps': steps, + 'generator': get_generator(seed), + 'callback_on_step_end': diffusers_callback, + 'output_type': 'latent', + } + if guidance_scale is not None and guidance_scale > 0: + base_args['guidance_scale'] = guidance_scale + if caps.supports_frame_rate_kwarg: + base_args['frame_rate'] = float(mp4_fps) + if caps.supports_image_cond_noise_scale and image_cond_noise_scale is not None: + base_args['image_cond_noise_scale'] = image_cond_noise_scale + if caps.supports_multi_condition and conditions: + base_args['conditions'] = conditions + if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and image is not None: + base_args['image'] = image + if caps.family == '2.x' and caps.is_distilled: + # distilled 2.x was trained with a fixed sigma schedule; override diffusers' linspace default + from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES + base_args['sigmas'] = list(DISTILLED_SIGMA_VALUES) + base_args.pop('num_inference_steps', None) + log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=latent_pass args_keys={list(base_args.keys())}') + result = shared.sd_model(**base_args) + latents = result.frames[0] if hasattr(result, 'frames') else None + audio = None + if hasattr(result, 'audio') and result.audio is not None: + audio = result.audio[0].float().cpu() + return latents, audio + + def run_ltx(task_id, _ui_state, - model:str, - prompt:str, - negative:str, - styles:list[str], - width:int, - height:int, - frames:int, - steps:int, - sampler_index:int, - seed:int, - upsample_enable:bool, - upsample_ratio:float, - refine_enable:bool, - refine_strength:float, + model: str, + prompt: str, + negative: str, + styles: list, + width: int, + height: int, + frames: int, + steps: int, + sampler_index: int, + guidance_scale: float, + sampler_shift: float, + dynamic_shift: bool, + seed: int, + upsample_enable: bool, + upsample_ratio: float, + refine_enable: bool, + refine_strength: float, condition_strength: float, + ltx_init_image, condition_image, condition_last, condition_files, condition_video, - condition_video_frames:int, - condition_video_skip:int, - decode_timestep:float, - image_cond_noise_scale:float, - mp4_fps:int, - mp4_interpolate:int, - mp4_codec:str, - mp4_ext:str, - mp4_opt:str, - mp4_video:bool, - mp4_frames:bool, - mp4_sf:bool, - audio_enable:bool, + condition_video_frames: int, + condition_video_skip: int, + decode_timestep: float, + image_cond_noise_scale: float, + mp4_fps: int, + mp4_interpolate: int, + mp4_codec: str, + mp4_ext: str, + mp4_opt: str, + mp4_video: bool, + mp4_frames: bool, + mp4_sf: bool, + audio_enable: bool, _overrides, ): - def abort(e, ok:bool=False, p=None): + def abort(e, ok: bool = False, p=None): if ok: log.info(e) else: @@ -67,189 +109,292 @@ def run_ltx(task_id, progress.finish_task(task_id) yield None, f'LTX Error: {str(e)}' - if model is None or len(model) == 0: + if model is None or len(model) == 0 or model == 'None': yield from abort('Video: no model selected', ok=True) return - # from diffusers import LTXConditionPipeline # pylint: disable=unused-import check_av() progress.add_task_to_queue(task_id) + with call_queue.get_lock(): progress.start_task(task_id) memstats.reset_stats() timer.process.reset() yield None, 'LTX: Loading...' + engine = 'LTX Video' load_model(engine, model) - debug(f'Video: cls={shared.sd_model.__class__.__name__} op=init model="{model}"') - if not shared.sd_model.__class__.__name__.startswith("LTX"): - yield from abort(f'Video: cls={shared.sd_model.__class__.__name__} selected model is not LTX model', ok=True) + caps = ltx_capabilities.get_caps(model) + if caps is None or not shared.sd_model.__class__.__name__.startswith('LTX'): + yield from abort(f'Video: cls={shared.sd_model.__class__.__name__} selected model is not LTX', ok=True) return videojob = shared.state.begin('Video', task_id=task_id) shared.state.job_count = 1 + from modules.video_models import models_def, video_overrides + selected = next((m for m in models_def.models.get(engine, []) if m.name == model), None) + + effective_init_image = ltx_init_image if ltx_init_image is not None else condition_image + + if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and effective_init_image is None: + yield from abort('No input image provided. Please upload or select an image.', ok=True) + return + + condition_images = [] + if effective_init_image is not None: + condition_images.append(effective_init_image) + if condition_last is not None: + condition_images.append(condition_last) + conditions = [] + if caps.supports_multi_condition: + conditions = get_conditions( + width, height, condition_strength, + condition_images, condition_files, condition_video, + condition_video_frames, condition_video_skip, + family=caps.family, + ) + + sampler_name = processing.get_sampler_name(sampler_index) + sd_samplers.create_sampler(sampler_name, shared.sd_model) + log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=init caps={caps.family} styles={styles} sampler={shared.sd_model.scheduler.__class__.__name__}') + + from modules.paths import resolve_output_path p = processing.StableDiffusionProcessingVideo( + sd_model=shared.sd_model, video_engine=engine, video_model=model, prompt=prompt, negative_prompt=negative, styles=styles, - width=width, - height=height, - frames=frames, - steps=steps, - sampler_index=sampler_index, - seed=seed, + seed=int(seed) if seed is not None else -1, + sampler_name=sampler_name, + sampler_shift=float(sampler_shift), + steps=int(steps), + width=get_bucket(width), + height=get_bucket(height), + frames=get_frames(frames), + cfg_scale=float(guidance_scale) if guidance_scale is not None and guidance_scale > 0 else caps.default_cfg, + denoising_strength=float(condition_strength) if condition_strength is not None else 1.0, + init_image=effective_init_image, + vae_type='Default', + vae_tile_frames=16, ) + p.scripts = None + p.script_args = None + p.do_not_save_grid = True + p.do_not_save_samples = not mp4_frames + p.outpath_samples = resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_video) p.ops.append('video') - condition_images = [] - if condition_image is not None: - condition_images.append(condition_image) - if condition_last is not None: - condition_images.append(condition_last) - conditions = get_conditions( - width, - height, - condition_strength, - condition_images, - condition_files, - condition_video, - condition_video_frames, - condition_video_skip, - ) + p.task_args['num_inference_steps'] = p.steps + p.task_args['width'] = p.width + p.task_args['height'] = p.height + # force pil: 'latent' output triggers frame collapse in process_samples + p.task_args['output_type'] = 'pil' + if caps.supports_frame_rate_kwarg: + p.task_args['frame_rate'] = float(mp4_fps) + if caps.supports_image_cond_noise_scale and image_cond_noise_scale is not None: + p.task_args['image_cond_noise_scale'] = image_cond_noise_scale + if caps.supports_decode_timestep and decode_timestep is not None: + p.task_args['decode_timestep'] = decode_timestep + if caps.supports_multi_condition and conditions: + p.task_args['conditions'] = conditions - prompt, negative, networks = get_prompts(prompt, negative, styles) - sampler_name = processing.get_sampler_name(sampler_index) - sd_samplers.create_sampler(sampler_name, shared.sd_model) - log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=init styles={styles} networks={networks} sampler={shared.sd_model.scheduler.__class__.__name__}') + if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and effective_init_image is not None: + from modules import images + p.task_args['image'] = images.resize_image(resize_mode=2, im=effective_init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') - extra_networks.activate(p, networks) - framewise = 'LTX2' not in shared.sd_model.__class__.__name__ + if caps.family == '2.x' and caps.is_distilled: + # distilled 2.x was trained with a fixed sigma schedule; override diffusers' linspace default + from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES + p.task_args['sigmas'] = list(DISTILLED_SIGMA_VALUES) + p.task_args.pop('num_inference_steps', None) + + framewise = caps.family == '0.9' set_vae_params(p, framewise=framewise) + orig_dynamic_shift = shared.opts.schedulers_dynamic_shift + orig_sampler_shift = shared.opts.schedulers_shift + shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift + shared.opts.data['schedulers_shift'] = sampler_shift + if hasattr(shared.sd_model, 'scheduler') and hasattr(shared.sd_model.scheduler, 'config') and hasattr(shared.sd_model.scheduler, 'register_to_config'): + if hasattr(shared.sd_model.scheduler.config, 'use_dynamic_shifting'): + shared.sd_model.scheduler.config.use_dynamic_shifting = dynamic_shift + shared.sd_model.scheduler.register_to_config(use_dynamic_shifting=dynamic_shift) + if hasattr(shared.sd_model.scheduler.config, 'flow_shift') and sampler_shift is not None and sampler_shift >= 0: + shared.sd_model.scheduler.config.flow_shift = sampler_shift + shared.sd_model.scheduler.register_to_config(flow_shift=sampler_shift) + shared.sd_model.default_scheduler = copy.deepcopy(shared.sd_model.scheduler) + + if selected is not None: + video_overrides.set_overrides(p, selected) + t0 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) t1 = time.time() - if 'LTX2' in shared.sd_model.__class__.__name__: - output_type = 'np' - else: - output_type = 'latent' - base_args = { - "prompt": prompt, - "negative_prompt": negative, - "width": get_bucket(width), - "height": get_bucket(height), - "num_frames": get_frames(frames), - "num_inference_steps": steps, - "generator": get_generator(seed), - "callback_on_step_end": diffusers_callback, - "output_type": output_type, - } - if 'LTX2' in shared.sd_model.__class__.__name__: - base_args["frame_rate"] = float(mp4_fps) - if 'Condition' in shared.sd_model.__class__.__name__: - base_args["image_cond_noise_scale"] = image_cond_noise_scale - if len(conditions) > 0: - base_args["conditions"] = conditions - log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=base {base_args}') - if debug: - log.trace(f'LTX args: {base_args}') - yield None, 'LTX: Generate in progress...' samplejob = shared.state.begin('Sample') + yield None, 'LTX: Generate in progress...' + + audio = None + pixels = None + frames_out = None + needs_latent_path = upsample_enable or refine_enable + try: - result = shared.sd_model(**base_args) - latents = result.frames[0] + if needs_latent_path: + prompt_final, negative_final, networks = get_prompts(prompt, negative, styles) + extra_networks.activate(p, networks) + latents, audio = _latent_pass( + caps=caps, + prompt=prompt_final, + negative=negative_final, + width=width, + height=height, + frames=frames, + steps=steps, + guidance_scale=p.cfg_scale, + mp4_fps=mp4_fps, + conditions=conditions, + image_cond_noise_scale=image_cond_noise_scale if caps.supports_image_cond_noise_scale else None, + seed=int(seed) if seed is not None else -1, + image=p.task_args.get('image'), + ) + else: + processed = processing.process_images(p) + if processed is None or processed.images is None or len(processed.images) == 0: + yield from abort('Video: process_images returned no frames', ok=True, p=p) + return + pixels = processed.images + if getattr(processed, 'audio', None) is not None: + audio = processed.audio + latents = None except AssertionError as e: yield from abort(e, ok=True, p=p) return except Exception as e: yield from abort(e, ok=False, p=p) return - if audio_enable and hasattr(result, 'audio') and result.audio is not None: - audio = result.audio[0].float().cpu() - else: - audio = None - try: - if debug: - log.trace(f'LTX result frames={latents.shape if latents is not None else None} audio={audio.shape if audio is not None else None}') - except Exception: - pass t2 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + devices.torch_gc(force=True, reason='ltx:base') t3 = time.time() timer.process.add('offload', t1 - t0) timer.process.add('base', t2 - t1) timer.process.add('offload', t3 - t2) shared.state.end(samplejob) - if upsample_enable: + if upsample_enable and latents is not None: t4 = time.time() upsamplejob = shared.state.begin('Upsample') - global upsample_pipe # pylint: disable=global-statement - upsample_pipe = load_upsample(upsample_pipe, upsample_repo_id) - upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) - upscale_args = { - "width": get_bucket(upsample_ratio * width), - "height": get_bucket(upsample_ratio * height), - "generator": get_generator(seed), - "output_type": output_type, - } - if latents.ndim == 4: - latents = latents.unsqueeze(0) # add batch dimension - log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=upsample latents={latents.shape} {upscale_args}') - yield None, 'LTX: Upsample in progress...' try: - upsampled_latents = upsample_pipe(latents=latents, **upscale_args).frames[0] + if caps.family == '0.9': + global upsample_pipe # pylint: disable=global-statement + upsample_pipe = load_upsample(upsample_pipe, upsample_repo_id_09) + upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) + up_args = { + 'width': get_bucket(upsample_ratio * width), + 'height': get_bucket(upsample_ratio * height), + 'generator': get_generator(int(seed) if seed is not None else -1), + 'output_type': 'latent', + } + if latents.ndim == 4: + latents = latents.unsqueeze(0) + log.debug(f'Video: op=upsample family=0.9 latents={latents.shape} {up_args}') + yield None, 'LTX: Upsample in progress...' + latents = upsample_pipe(latents=latents, **up_args).frames[0] + upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) + else: + from diffusers.pipelines.ltx2.pipeline_ltx2_latent_upsample import LTX2LatentUpsamplePipeline + log.info(f'Video load: cls={LTX2LatentUpsamplePipeline.__name__} family=2.x') + up_pipe = LTX2LatentUpsamplePipeline.from_pretrained( + 'Lightricks/LTX-2-Latent-Upsampler', + vae=shared.sd_model.vae, + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + ) + up_pipe = sd_models.apply_balanced_offload(up_pipe) + up_args = { + 'width': get_bucket(upsample_ratio * width), + 'height': get_bucket(upsample_ratio * height), + 'num_frames': get_frames(frames), + 'latents_normalized': True, + 'generator': get_generator(int(seed) if seed is not None else -1), + 'output_type': 'latent', + } + if latents.ndim == 4: + latents = latents.unsqueeze(0) + log.debug(f'Video: op=upsample family=2.x latents={latents.shape} {up_args}') + yield None, 'LTX: Upsample in progress...' + latents = up_pipe(latents=latents, **up_args).frames[0] + up_pipe = sd_models.apply_balanced_offload(up_pipe) except AssertionError as e: yield from abort(e, ok=True, p=p) return except Exception as e: yield from abort(e, ok=False, p=p) return - latents = upsampled_latents t5 = time.time() - upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) - t6 = time.time() timer.process.add('upsample', t5 - t4) - timer.process.add('offload', t6 - t5) shared.state.end(upsamplejob) - if refine_enable: + if refine_enable and latents is not None: t7 = time.time() refinejob = shared.state.begin('Refine') shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + devices.torch_gc(force=True, reason='ltx:refine') + # refine is the terminal stage when enabled: let the pipeline decode internally so the final vae pass runs + # inside the same offload/cudnn context as a normal generation, matching the Generic Video tab refine_args = { - "prompt": prompt, - "negative_prompt": negative, - "width": get_bucket(upsample_ratio * width), - "height": get_bucket(upsample_ratio * height), - "num_frames": get_frames(frames), - "denoise_strength": refine_strength, - "num_inference_steps": steps, - "image_cond_noise_scale": image_cond_noise_scale, - "generator": get_generator(seed), - "callback_on_step_end": diffusers_callback, - "output_type": output_type, + 'prompt': prompt_final, + 'negative_prompt': negative_final, + 'width': get_bucket((upsample_ratio if upsample_enable else 1.0) * width), + 'height': get_bucket((upsample_ratio if upsample_enable else 1.0) * height), + 'num_frames': get_frames(frames), + 'num_inference_steps': steps, + 'generator': get_generator(int(seed) if seed is not None else -1), + 'callback_on_step_end': diffusers_callback, + 'output_type': 'pil', } + if p.cfg_scale is not None and p.cfg_scale > 0: + refine_args['guidance_scale'] = p.cfg_scale + if caps.supports_frame_rate_kwarg: + refine_args['frame_rate'] = float(mp4_fps) + if caps.supports_image_cond_noise_scale and image_cond_noise_scale is not None: + refine_args['image_cond_noise_scale'] = image_cond_noise_scale + if caps.supports_multi_condition and conditions: + refine_args['conditions'] = conditions + if caps.family == '2.x': + if caps.is_distilled: + # distilled variants have a canonical Stage-2 refine schedule they were trained on; + # see diffusers.pipelines.ltx2.utils and Lightricks/LTX-2 ti2vid_two_stages pipeline + from diffusers.pipelines.ltx2.utils import STAGE_2_DISTILLED_SIGMA_VALUES + refine_args['sigmas'] = list(STAGE_2_DISTILLED_SIGMA_VALUES) + else: + # non-distilled: truncate the default linspace schedule to match user-controlled refine_strength + default_sigmas = np.linspace(1.0, 1.0 / steps, steps) + num_skip = max(steps - max(int(steps * refine_strength), 1), 0) + refine_args['sigmas'] = default_sigmas[num_skip:].tolist() + refine_args.pop('num_inference_steps', None) + elif caps.repo_cls_name == 'LTXConditionPipeline': + refine_args['denoise_strength'] = refine_strength if latents.ndim == 4: - latents = latents.unsqueeze(0) # add batch dimension - - log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=refine latents={latents.shape} {refine_args}') - if len(conditions) > 0: - refine_args["conditions"] = conditions + latents = latents.unsqueeze(0) + log.debug(f'Video: op=refine cls={caps.repo_cls_name} latents={latents.shape}') yield None, 'LTX: Refine in progress...' try: - refined_latents = shared.sd_model(latents=latents, **refine_args).frames[0] + result = shared.sd_model(latents=latents, **refine_args) + pixels = result.frames[0] if hasattr(result, 'frames') else None + if hasattr(result, 'audio') and result.audio is not None: + audio = result.audio[0].float().cpu() + latents = None except AssertionError as e: yield from abort(e, ok=True, p=p) return except Exception as e: yield from abort(e, ok=False, p=p) return - latents = refined_latents t8 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) t9 = time.time() @@ -257,26 +402,37 @@ def run_ltx(task_id, timer.process.add('offload', t9 - t8) shared.state.end(refinejob) - extra_networks.deactivate(p) + shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift + shared.opts.data['schedulers_shift'] = orig_sampler_shift - yield None, 'LTX: VAE decode in progress...' - try: - if torch.is_tensor(latents): - frames = vae_decode(latents, decode_timestep, seed) - else: - frames = latents - except TypeError: - frames = latents # likely because the latents are already decoded - except AssertionError as e: - yield from abort(e, ok=True, p=p) - return - except Exception as e: - yield from abort(e, ok=False, p=p) - return - t10 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - t11 = time.time() - timer.process.add('offload', t11 - t10) + if needs_latent_path: + extra_networks.deactivate(p) + + if needs_latent_path and latents is not None: + # only reached when upsample ran without refine; refine decodes through the pipeline and sets latents=None + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'], force=True) + devices.torch_gc(force=True, reason='ltx:vae') + yield None, 'LTX: VAE decode in progress...' + try: + if torch.is_tensor(latents): + # 0.9.x returns raw latents with output_type='latent'; 2.x pre-denormalizes them + frames_out = vae_decode(latents, decode_timestep if caps.supports_decode_timestep else 0.0, int(seed) if seed is not None else -1, denormalize=caps.family == '0.9') + else: + frames_out = latents + except AssertionError as e: + yield from abort(e, ok=True, p=p) + return + except Exception as e: + yield from abort(e, ok=False, p=p) + return + pixels = frames_out + t10 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + t11 = time.time() + timer.process.add('offload', t11 - t10) + + if not audio_enable: + audio = None try: aac_sample_rate = shared.sd_model.vocoder.config.output_sampling_rate @@ -285,7 +441,7 @@ def run_ltx(task_id, num_frames, video_file, _thumb = save_video( p=p, - pixels=frames, + pixels=pixels, audio=audio, mp4_fps=mp4_fps, mp4_codec=mp4_codec, @@ -300,22 +456,26 @@ def run_ltx(task_id, ) t_end = time.time() - if isinstance(frames, list) and isinstance(frames[0], Image.Image): - w, h = frames[0].size - elif frames.ndim == 5: - _n, _c, _t, h, w = frames.shape - elif frames.ndim == 4: - _n, h, w, _c = frames.shape + if isinstance(pixels, list) and len(pixels) > 0 and isinstance(pixels[0], Image.Image): + w, h = pixels[0].size + elif hasattr(pixels, 'ndim') and pixels.ndim == 5: + _n, _c, _t, h, w = pixels.shape + elif hasattr(pixels, 'ndim') and pixels.ndim == 4: + _n, h, w, _c = pixels.shape + elif hasattr(pixels, 'shape'): + h, w = pixels.shape[-2], pixels.shape[-1] else: - h, w = frames.shape[-2], frames.shape[-1] + w, h = p.width, p.height resolution = f'{w}x{h}' if num_frames > 0 else None summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ') memory = shared.mem_mon.summary() - fps = f'{num_frames/(t_end-t0):.2f}' - its = f'{(steps)/(t_end-t0):.2f}' + total_time = max(t_end - t0, 1e-6) + fps = f'{num_frames/total_time:.2f}' + its = f'{(steps)/total_time:.2f}' shared.state.end(videojob) progress.finish_task(task_id) + p.close() log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={fps} its={its} resolution={resolution} time={t_end-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') - yield video_file, f'LTX: Generation completed | File {video_file} | Frames {len(frames)} | Resolution {resolution} | f/s {fps} | it/s {its} '+ f"

{summary} {memory}

" + yield video_file, f'LTX: Generation completed | File {video_file} | Frames {num_frames} | Resolution {resolution} | f/s {fps} | it/s {its} ' + f"

{summary} {memory}

" diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index 846efcd0b..a19539a4c 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -3,13 +3,50 @@ import gradio as gr from modules import ui_sections from modules.logger import log from modules.video_models.models_def import models -from modules.ltx import ltx_process +from modules.ltx import ltx_process, ltx_capabilities debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None -def create_ui(prompt, negative, styles, overrides, init_image, init_strength, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, width, height, frames, seed): +def _model_change(model_name: str): + caps = ltx_capabilities.get_caps(model_name) + if caps is None: + return ( + gr.update(visible=False), # input_media_accordion + gr.update(visible=False), # multi_condition_group + gr.update(visible=False), # upsample_accordion + gr.update(visible=False), # refine_accordion + gr.update(value=False), # upsample_enable (reset) + gr.update(value=False), # refine_enable (reset) + gr.update(interactive=True), # refine_strength + gr.update(), # guidance_scale + gr.update(), # steps + gr.update(), # sampler_shift + gr.update(interactive=False), # decode_timestep + gr.update(interactive=False), # image_cond_noise_scale + gr.update(visible=False), # audio_accordion + ) + # distilled 2.x variants use a fixed canonical refine schedule; the strength slider is meaningless there + refine_strength_interactive = not (caps.family == '2.x' and caps.is_distilled) + return ( + gr.update(visible=caps.supports_input_media), + gr.update(visible=caps.supports_multi_condition), + gr.update(visible=True), + gr.update(visible=True), + gr.update(value=False), + gr.update(value=False), + gr.update(interactive=refine_strength_interactive), + gr.update(value=caps.default_cfg), + gr.update(value=caps.default_steps), + gr.update(value=caps.default_sampler_shift), + gr.update(interactive=caps.supports_decode_timestep), + gr.update(interactive=caps.supports_image_cond_noise_scale), + gr.update(visible=caps.supports_audio), + ) + + +def create_ui(prompt, negative, styles, overrides, init_image, _init_strength, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, width, height, frames, seed): with gr.Row(): with gr.Column(variant='compact', elem_id="ltx_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): @@ -17,39 +54,73 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la with gr.Row(): ltx_models = [m.name for m in models['LTX Video']] if 'LTX Video' in models else ['None'] model = gr.Dropdown(label='LTX model', choices=ltx_models, value=ltx_models[0], elem_id="ltx_model") - with gr.Accordion(open=False, label="Condition", elem_id='ltx_condition_accordion'): - with gr.Tabs(): - with gr.Tab('Video', id='ltx_condition_video_tab'): - condition_video = gr.Video(label='Video', type='filepath', elem_id="ltx_condition_video", width=256, height=256, source='upload') - with gr.Row(): - condition_video_frames = gr.Slider(label='LTX frames number', minimum=-1, maximum=1024, step=1, value=-1, elem_id="ltx_condition_video_frames") - condition_video_skip = gr.Slider(label='LTX frames skip', minimum=0, maximum=1024, step=1, value=0, elem_id="ltx_condition_video_sip") - with gr.Tab('Gallery', id='ltx_condition_batch_tab'): - condition_files = gr.Files(label="Image Batch", interactive=True, elem_id="ltx_condition_batch") - with gr.Accordion(open=False, label="Upsample", elem_id='ltx_upsample_accordion'): + input_media_accordion = gr.Accordion(open=False, label="Input media", elem_id='ltx_input_media_accordion', visible=False) + with input_media_accordion: + ltx_init_image = gr.Image(label='Image', elem_id='ltx_init_image', type='pil', image_mode='RGB', width=256, height=256) + ltx_condition_strength = gr.Slider(label='LTX input strength', minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id='ltx_condition_strength') + multi_condition_group = gr.Group(visible=False) + with multi_condition_group: + gr.Markdown('**Prefix conditioning**: supply a video or gallery to anchor the opening frames', elem_id='ltx_prefix_conditioning_label') + with gr.Tabs(): + with gr.Tab('Video prefix', id='ltx_condition_video_tab'): + condition_video = gr.Video(label='Video', type='filepath', elem_id="ltx_condition_video", width=256, height=256, source='upload') + with gr.Row(): + condition_video_frames = gr.Slider(label='LTX frames number', minimum=-1, maximum=1024, step=1, value=-1, elem_id="ltx_condition_video_frames") + condition_video_skip = gr.Slider(label='LTX frames skip', minimum=0, maximum=1024, step=1, value=0, elem_id="ltx_condition_video_sip") + with gr.Tab('Gallery prefix', id='ltx_condition_batch_tab'): + condition_files = gr.Files(label="Image Batch", interactive=True, elem_id="ltx_condition_batch") + upsample_accordion = gr.Accordion(open=False, label="Upsample", elem_id='ltx_upsample_accordion') + with upsample_accordion: with gr.Row(): upsample_enable = gr.Checkbox(label='LTX enable upsampling', value=False, elem_id="ltx_upsample_enable") - upsample_ratio = gr.Slider(label='LTX upsample ratio', minimum=1.0, maximum=4.0, step=0.1, value=2.0, elem_id="ltx_upsample_ratio", interactive=False) - with gr.Accordion(open=False, label="Refine", elem_id='ltx_refine_accordion'): + upsample_ratio = gr.Slider(label='LTX upsample ratio', minimum=1.0, maximum=4.0, step=0.1, value=2.0, elem_id="ltx_upsample_ratio") + refine_accordion = gr.Accordion(open=False, label="Refine", elem_id='ltx_refine_accordion') + with refine_accordion: with gr.Row(): refine_enable = gr.Checkbox(label='LTX enable refine', value=False, elem_id="ltx_refine_enable") refine_strength = gr.Slider(label='LTX refine strength', minimum=0.1, maximum=1.0, step=0.05, value=0.4, elem_id="ltx_refine_strength") - with gr.Accordion(open=False, label="Advanced", elem_id='ltx_parameters_accordion'): - steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "ltx", default_steps=50) + parameters_accordion = gr.Accordion(open=False, label="Advanced", elem_id='ltx_parameters_accordion') + with parameters_accordion: + steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "ltx", default_steps=40) with gr.Row(): - decode_timestep = gr.Slider(label='LTX decode timestep', minimum=0.01, maximum=1.0, step=0.01, value=0.05, elem_id="ltx_decode_timestep") - image_cond_noise_scale = gr.Slider(label='Noise scale', minimum=0.01, maximum=1.0, step=0.01, value=0.025, elem_id="ltx_image_cond_noise_scale") - with gr.Accordion(open=False, label="Audio", elem_id='ltx_audio_accordion'): + guidance_scale = gr.Slider(label='LTX guidance scale', minimum=0.0, maximum=14.0, step=0.1, value=4.0, elem_id="ltx_guidance_scale") + with gr.Row(): + sampler_shift = gr.Slider(label='LTX sampler shift', minimum=-1.0, maximum=20.0, step=0.1, value=-1.0, elem_id="ltx_sampler_shift") + dynamic_shift = gr.Checkbox(label='LTX dynamic shift', value=False, elem_id="ltx_dynamic_shift") + with gr.Row(): + decode_timestep = gr.Slider(label='LTX decode timestep', minimum=0.0, maximum=1.0, step=0.01, value=0.05, elem_id="ltx_decode_timestep") + image_cond_noise_scale = gr.Slider(label='LTX image cond noise scale', minimum=0.0, maximum=1.0, step=0.005, value=0.025, elem_id="ltx_image_cond_noise_scale") + audio_accordion = gr.Accordion(open=False, label="Audio", elem_id='ltx_audio_accordion', visible=False) + with audio_accordion: with gr.Row(): audio_enable = gr.Checkbox(label='LTX enable audio', value=False, elem_id="ltx_audio_enable") with gr.Column(elem_id='ltx-output-column', scale=2) as _column_output: with gr.Row(): video = gr.Video(label="Output", show_label=False, elem_id='ltx_output_video', elem_classes=['control-image'], height=512, autoplay=False) - # video = gr.Gallery(value=[], label="Output", show_label=False, elem_id='ltx_output_video', elem_classes=['control-image'], height=512) with gr.Row(): text = gr.HTML('', elem_id='ltx_generation_info', show_label=False) + model.change( + fn=_model_change, + inputs=[model], + outputs=[ + input_media_accordion, + multi_condition_group, + upsample_accordion, + refine_accordion, + upsample_enable, + refine_enable, + refine_strength, + guidance_scale, + steps, + sampler_shift, + decode_timestep, + image_cond_noise_scale, + audio_accordion, + ], + ) + task_id = gr.Textbox(visible=False, value='') ui_state = gr.Textbox(visible=False, value='') state_inputs = [task_id, ui_state] @@ -58,10 +129,12 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la model, prompt, negative, styles, width, height, frames, - steps, sampler_index, seed, + steps, sampler_index, + guidance_scale, sampler_shift, dynamic_shift, + seed, upsample_enable, upsample_ratio, refine_enable, refine_strength, - init_strength, init_image, last_image, condition_files, condition_video, condition_video_frames, condition_video_skip, + ltx_condition_strength, ltx_init_image, init_image, last_image, condition_files, condition_video, condition_video_frames, condition_video_skip, decode_timestep, image_cond_noise_scale, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, audio_enable, diff --git a/modules/ltx/ltx_util.py b/modules/ltx/ltx_util.py index e5655af78..e31641bf9 100644 --- a/modules/ltx/ltx_util.py +++ b/modules/ltx/ltx_util.py @@ -9,9 +9,12 @@ loaded_model: str = None def get_bucket(size: int): - if not hasattr(shared.sd_model, 'vae_temporal_compression_ratio'): - return int(size) - (int(size) % 32) - return int(size) - (int(size) % shared.sd_model.vae_temporal_compression_ratio) + # LTX pipelines validate width/height divisible by 32 across all families + ratio = getattr(shared.sd_model, 'vae_spatial_compression_ratio', None) + if not isinstance(ratio, int) or ratio < 32: + ratio = 32 + size = int(size) + return size - (size % ratio) def get_frames(frames: int): @@ -57,8 +60,30 @@ def load_upsample(upsample_pipe, upsample_repo_id): return upsample_pipe -def get_conditions(width, height, condition_strength, condition_images, condition_files, condition_video, condition_video_frames, condition_video_skip): +def _condition_cls(family: str): + if family == '2.x': + try: + from diffusers.pipelines.ltx2.pipeline_ltx2_condition import LTX2VideoCondition + return LTX2VideoCondition + except ImportError: + log.warning('LTX conditions: LTX2VideoCondition not available in installed diffusers') + return None from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition + return LTXVideoCondition + + +def make_condition(condition_cls, family: str, frames, strength: float, is_video: bool): + if family == '2.x': + return condition_cls(frames=frames, index=0, strength=strength) + if is_video: + return condition_cls(video=frames, frame_index=0, strength=strength) + return condition_cls(image=frames, frame_index=0, strength=strength) + + +def get_conditions(width, height, condition_strength, condition_images, condition_files, condition_video, condition_video_frames, condition_video_skip, family: str = '0.9'): + condition_cls = _condition_cls(family) + if condition_cls is None: + return [] conditions = [] if condition_images is not None: for condition_image in condition_images: @@ -67,32 +92,32 @@ def get_conditions(width, height, condition_strength, condition_images, conditio from modules.api.api import decode_base64_to_image condition_image = decode_base64_to_image(condition_image) condition_image = condition_image.convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) - conditions.append(LTXVideoCondition(image=condition_image, frame_index=0, strength=condition_strength)) - log.debug(f'Video condition: image={condition_image.size} strength={condition_strength}') + conditions.append(make_condition(condition_cls, family, condition_image, condition_strength, is_video=False)) + log.debug(f'Video condition: family={family} image={condition_image.size} strength={condition_strength}') except Exception as e: log.error(f'LTX condition image: {e}') if condition_files is not None: - condition_images = [] + batch_images = [] for fn in condition_files: try: if hasattr(fn, 'name'): condition_image = Image.open(fn.name).convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) else: condition_image = fn.convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) - condition_images.append(condition_image) + batch_images.append(condition_image) except Exception as e: log.error(f'LTX condition files: {e}') - if len(condition_images) > 0: - conditions.append(LTXVideoCondition(video=condition_images, frame_index=0, strength=condition_strength)) - log.debug(f'Video condition: files={len(condition_images)} size={condition_images[0].size} strength={condition_strength}') + if len(batch_images) > 0: + conditions.append(make_condition(condition_cls, family, batch_images, condition_strength, is_video=True)) + log.debug(f'Video condition: family={family} files={len(batch_images)} size={batch_images[0].size} strength={condition_strength}') if condition_video is not None: from modules.video_models.video_utils import get_video_frames try: condition_frames = get_video_frames(condition_video, num_frames=condition_video_frames, skip_frames=condition_video_skip) condition_frames = [f.convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) for f in condition_frames] if len(condition_frames) > 0: - conditions.append(LTXVideoCondition(video=condition_frames, frame_index=0, strength=condition_strength)) - log.debug(f'Video condition: frames={len(condition_frames)} size={condition_frames[0].size} strength={condition_strength}') + conditions.append(make_condition(condition_cls, family, condition_frames, condition_strength, is_video=True)) + log.debug(f'Video condition: family={family} frames={len(condition_frames)} size={condition_frames[0].size} strength={condition_strength}') except Exception as e: log.error(f'LTX condition video: {e}') return conditions @@ -114,16 +139,19 @@ def get_generator(seed): return torch.Generator().manual_seed(seed) -def vae_decode(latents, decode_timestep, seed): +def vae_decode(latents, decode_timestep, seed, denormalize: bool = True): t0 = time.time() - log.debug(f'Video: cls={shared.sd_model.vae.__class__.__name__} op=vae latents={latents.shape} timestep={decode_timestep}') + if latents.ndim == 4: + latents = latents.unsqueeze(0) + log.debug(f'Video: cls={shared.sd_model.vae.__class__.__name__} op=vae latents={latents.shape} timestep={decode_timestep} denormalize={denormalize}') from diffusers.utils.torch_utils import randn_tensor - latents = shared.sd_model._denormalize_latents( # pylint: disable=protected-access - latents, - shared.sd_model.vae.latents_mean, - shared.sd_model.vae.latents_std, - shared.sd_model.vae.config.scaling_factor - ) + if denormalize: + latents = shared.sd_model._denormalize_latents( # pylint: disable=protected-access + latents, + shared.sd_model.vae.latents_mean, + shared.sd_model.vae.latents_std, + shared.sd_model.vae.config.scaling_factor + ) latents = latents.to(device=devices.device, dtype=devices.dtype) if not shared.sd_model.vae.config.timestep_conditioning: timestep = None @@ -132,10 +160,7 @@ def vae_decode(latents, decode_timestep, seed): timestep = torch.tensor([decode_timestep], device=devices.device, dtype=latents.dtype) noise_scale = torch.tensor([decode_timestep], device=devices.device, dtype=devices.dtype)[:, None, None, None, None] latents = (1 - noise_scale) * latents + noise_scale * noise - frames = shared.sd_model.vae.decode(latents, timestep, return_dict=False)[0] # n, c, f, h, w - # frames = frames.squeeze(0) if frames.ndim == 5 else frames - # frames = frames.permute(1, 2, 3, 0) - # frames = shared.sd_model.video_processor.postprocess_video(frames, output_type='pil') + frames = shared.sd_model.vae.decode(latents, timestep, return_dict=False)[0] t1 = time.time() timer.process.add('vae', t1 - t0) return frames diff --git a/modules/video_models/video_overrides.py b/modules/video_models/video_overrides.py index 141b7be0c..d1f74e992 100644 --- a/modules/video_models/video_overrides.py +++ b/modules/video_models/video_overrides.py @@ -46,7 +46,8 @@ def set_overrides(p: processing.StableDiffusionProcessingVideo, selected: Model) if 'SkyReelsV2DiffusionForcing' in cls: p.task_args['overlap_history'] = 17 # LTX - if cls == 'LTXImageToVideoPipeline' or cls == 'LTXConditionPipeline': + ltx_i2v_classes = ('LTXImageToVideoPipeline', 'LTXConditionPipeline', 'LTX2ImageToVideoPipeline', 'LTX2ConditionPipeline') + if cls in ltx_i2v_classes: p.task_args['generator'] = None if cls == 'LTXConditionPipeline': p.task_args['strength'] = p.denoising_strength From 05abd99285001e63e0951cbba10693e349bf7f9d Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 19 Apr 2026 03:36:16 +0100 Subject: [PATCH 04/15] fix(video): invalidate ltx cache on pipe-class mismatch Move cache tracking from ltx_util into video_load where shared.sd_model lives, and invalidate the name-based hit when the cached class no longer matches the current pipeline (e.g. after Unload Models triggers an auto-reload of the default checkpoint). - Drop the duplicate module-level loaded_model cache in ltx_util - Add a pipe-class isinstance check around the cache hit in video_load --- modules/ltx/ltx_util.py | 16 ++++------------ modules/video_models/video_load.py | 6 ++++++ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/modules/ltx/ltx_util.py b/modules/ltx/ltx_util.py index e31641bf9..310a865fb 100644 --- a/modules/ltx/ltx_util.py +++ b/modules/ltx/ltx_util.py @@ -5,11 +5,8 @@ from modules import devices, shared, sd_models, timer, extra_networks from modules.logger import log -loaded_model: str = None - - def get_bucket(size: int): - # LTX pipelines validate width/height divisible by 32 across all families + # LTX pipes validate width/height divisible by 32 across all families. ratio = getattr(shared.sd_model, 'vae_spatial_compression_ratio', None) if not isinstance(ratio, int) or ratio < 32: ratio = 32 @@ -22,21 +19,16 @@ def get_frames(frames: int): def load_model(engine: str, model: str): - global loaded_model # pylint: disable=global-statement - if not shared.sd_loaded: - loaded_model = None - if loaded_model == model: - return - if model is None or model == '' or model=='None': - loaded_model = None + if model is None or model == '' or model == 'None': shared.sd_model = None return t0 = time.time() from modules.video_models import models_def, video_load selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] + # video_load owns the cache; pipe-class mismatch inside it invalidates the name-based hit + # when Unload Models (or any external swap) silently replaced shared.sd_model. log.info(f'Video load: engine="{engine}" selected="{model}" {selected}') video_load.load_model(selected) - loaded_model = model t1 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) t2 = time.time() diff --git a/modules/video_models/video_load.py b/modules/video_models/video_load.py index 373751b7e..c09197a0f 100644 --- a/modules/video_models/video_load.py +++ b/modules/video_models/video_load.py @@ -36,6 +36,12 @@ def load_model(selected: models_def.Model): global loaded_model # pylint: disable=global-statement if not shared.sd_loaded: loaded_model = None + elif loaded_model == selected.name and selected.repo_cls is not None and not isinstance(shared.sd_model, selected.repo_cls): + # shared.sd_model auto-reloads the default checkpoint when model_data.sd_model is None, + # which silently swaps the pipe class behind the name-based cache. Pipe-class mismatch + # is the reliable signal that the cached name no longer maps to the cached object. + log.warning(f'Video load: cached model="{selected.name}" pipe class swapped to {type(shared.sd_model).__name__}; forcing reload') + loaded_model = None if loaded_model == selected.name: return '' if shared.sd_loaded: From 5cf46d2f81ed28fa562dedefbbec37ea2ece3b75 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 19 Apr 2026 03:37:25 +0100 Subject: [PATCH 05/15] feat(ltx): canonical LTX-2.x Stage 2 recipe (LoRA + guidance + connectors) Implement the Lightricks two-stage recipe (diffusers PR #13217) for the LTX-2.x Dev family: Stage 1 at half-res with full four-way guidance, 2x latent upsample, Stage 2 with distilled LoRA + scheduler swap + identity guidance on STAGE_2_DISTILLED_SIGMA_VALUES. Extends to both LTX-2.0 and LTX-2.3 Dev via per-family distilled-LoRA repos carried on the caps; Distilled variants take the same flow minus the LoRA swap. Auto-couples Refine with a fixed 2x upsample on any Dev variant with a known LoRA when the user enables Refine without Upsample. - caps: is_ltx_2_3, use_cross_timestep, default_dynamic_shift, stage2_dev_lora_repo, supports_canonical_stage2, modality_default_scale, guidance_rescale_default; LTX-2.x defaults realigned to canonical cfg=3.0 / steps=30; per-variant STG block and four-way guidance wired for non-distilled 2.x - process: canonical Stage 1/Stage 2 helpers, scheduler + opts snapshot under try/finally, per-family upsampler repo, audio latents threaded from Stage 1 into Stage 2, use_cross_timestep gated per caps - overrides: skip the redundant unsharded LTX-2.3 connectors blob and share LTX2TextConnectors weights across 2.3 variants when te_shared_t5 - load: Gemma3 shared-TE path for LTX-2.3; gate use_dynamic_shifting=False override to 0.9.x only so LTX-2.x stays on its canonical token-count dynamic shift --- modules/ltx/ltx_capabilities.py | 36 +- modules/ltx/ltx_process.py | 676 +++++++++++++++--------- modules/video_models/video_load.py | 13 +- modules/video_models/video_overrides.py | 31 +- 4 files changed, 489 insertions(+), 267 deletions(-) diff --git a/modules/ltx/ltx_capabilities.py b/modules/ltx/ltx_capabilities.py index 2dae4c654..d345eaa2a 100644 --- a/modules/ltx/ltx_capabilities.py +++ b/modules/ltx/ltx_capabilities.py @@ -10,23 +10,34 @@ class LTXCaps: repo_cls_name: str family: str # '0.9' or '2.x' is_distilled: bool + is_ltx_2_3: bool is_i2v: bool - supports_input_media: bool # accordion visible for any pipeline that accepts image/video input - supports_multi_condition: bool # uses conditions=[LTX(2)VideoCondition(...)] kwarg; Condition classes only + supports_input_media: bool + supports_multi_condition: bool supports_image_cond_noise_scale: bool supports_decode_timestep: bool supports_stg: bool supports_audio: bool supports_frame_rate_kwarg: bool + # 2.3 transformer cross-attn reads the other modality's sigma; unset falls back to 2.0's + # independent-sigma path, which is a joint-distribution mismatch for 2.3 weights. + use_cross_timestep: bool default_cfg: float default_steps: int default_sampler_shift: float + default_dynamic_shift: bool default_width: int default_height: int default_frames: int default_frame_rate: int stg_default_scale: float = 0.0 stg_default_blocks: list = field(default_factory=list) + # Dev 2.x trained under cfg + stg + modality + rescale four-way composition; + # distilled bakes these into its sigma schedule and stays at pipeline identity. + modality_default_scale: float = 1.0 + guidance_rescale_default: float = 0.0 + supports_canonical_stage2: bool = False + stage2_dev_lora_repo: Optional[str] = None CONDITION_CLASSES = {'LTXConditionPipeline', 'LTX2ConditionPipeline'} @@ -69,12 +80,14 @@ def get_caps(model_name: str) -> Optional[LTXCaps]: is_i2v = 'I2V' in model_name or cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') is_condition_cls = cls_name in CONDITION_CLASSES supports_input_media = is_i2v or is_condition_cls + is_ltx_2_3 = is_ltx2 and '2.3' in model_name caps = LTXCaps( name=model_name, repo_cls_name=cls_name, family=family, is_distilled=is_distilled, + is_ltx_2_3=is_ltx_2_3, is_i2v=is_i2v, supports_input_media=supports_input_media, supports_multi_condition=is_condition_cls, @@ -83,9 +96,11 @@ def get_caps(model_name: str) -> Optional[LTXCaps]: supports_stg=is_ltx2, supports_audio=is_ltx2, supports_frame_rate_kwarg=is_ltx2, - default_cfg=4.0 if is_ltx2 else 3.0, - default_steps=40 if is_ltx2 else 50, + use_cross_timestep=is_ltx_2_3, + default_cfg=3.0, + default_steps=30 if is_ltx2 else 50, default_sampler_shift=-1.0, + default_dynamic_shift=is_ltx2, default_width=768, default_height=512, default_frames=121 if is_ltx2 else 161, @@ -96,6 +111,13 @@ def get_caps(model_name: str) -> Optional[LTXCaps]: caps.default_cfg = 1.0 caps.default_steps = 8 + if is_ltx2 and not is_distilled: + if is_ltx_2_3: + caps.stage2_dev_lora_repo = 'CalamitousFelicitousness/LTX-2.3-distilled-lora-384-Diffusers' + elif '2.0' in model_name: + caps.stage2_dev_lora_repo = 'CalamitousFelicitousness/LTX-2.0-distilled-lora-384-Diffusers' + caps.supports_canonical_stage2 = caps.stage2_dev_lora_repo is not None + if is_ltx2: if '2.3' in model_name: caps.stg_default_blocks = [28] @@ -103,6 +125,10 @@ def get_caps(model_name: str) -> Optional[LTXCaps]: caps.stg_default_blocks = [29] else: caps.stg_default_blocks = [28] - caps.stg_default_scale = 0.0 + if not is_distilled: + # canonical T2V composition from huggingface/diffusers#13217 + caps.stg_default_scale = 1.0 + caps.modality_default_scale = 3.0 + caps.guidance_rescale_default = 0.7 return caps diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 605611374..eeb14fab8 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -1,7 +1,5 @@ import os -import copy import time -import numpy as np import torch from PIL import Image @@ -17,8 +15,51 @@ from modules.video_models.video_utils import check_av debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None upsample_repo_id_09 = 'a-r-r-o-w/LTX-Video-0.9.7-Latent-Spatial-Upsampler-diffusers' +# Upsampler weights are tied to the family VAE; using the wrong one preserves structure +# but drifts per-channel latent statistics (decodes desaturated / crushed contrast). +upsample_repo_id_20 = 'Lightricks/LTX-2' +upsample_repo_id_23 = 'CalamitousFelicitousness/LTX-2.3-Spatial-Upsampler-x2-1.1-Diffusers' upsample_pipe = None +STAGE2_DEV_LORA_ADAPTER = 'ltx2_stage2_distilled' + + +def _canonical_ltx2_guidance(caps) -> dict: + # Four-way composition (cfg + stg + modality + rescale) from huggingface/diffusers#13217. + # Distilled bakes these into its sigma schedule; skip or we double-apply. + if caps.family != '2.x' or caps.is_distilled: + return {} + return { + 'stg_scale': caps.stg_default_scale, + 'modality_scale': caps.modality_default_scale, + 'guidance_rescale': caps.guidance_rescale_default, + 'spatio_temporal_guidance_blocks': list(caps.stg_default_blocks), + 'audio_guidance_scale': 7.0, + 'audio_stg_scale': 1.0, + 'audio_modality_scale': 3.0, + 'audio_guidance_rescale': 0.7, + } + + +def _canonical_stage2_dev_kwargs() -> dict: + # Stage 2 identity guidance from huggingface/diffusers#13217. The distilled LoRA makes Dev + # behave like Distilled, which was trained at identity; Stage 1's four-way composition on + # top double-dips and produces striping/flicker. + from diffusers.pipelines.ltx2.utils import STAGE_2_DISTILLED_SIGMA_VALUES + return { + 'sigmas': list(STAGE_2_DISTILLED_SIGMA_VALUES), + 'noise_scale': float(STAGE_2_DISTILLED_SIGMA_VALUES[0]), + 'guidance_scale': 1.0, + 'stg_scale': 0.0, + 'modality_scale': 1.0, + 'guidance_rescale': 0.0, + 'audio_guidance_scale': 1.0, + 'audio_stg_scale': 0.0, + 'audio_modality_scale': 1.0, + 'audio_guidance_rescale': 0.0, + 'spatio_temporal_guidance_blocks': None, + } + def _latent_pass(caps, prompt, negative, width, height, frames, steps, guidance_scale, mp4_fps, conditions, image_cond_noise_scale, seed, image=None): base_args = { @@ -43,17 +84,21 @@ def _latent_pass(caps, prompt, negative, width, height, frames, steps, guidance_ if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and image is not None: base_args['image'] = image if caps.family == '2.x' and caps.is_distilled: - # distilled 2.x was trained with a fixed sigma schedule; override diffusers' linspace default from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES base_args['sigmas'] = list(DISTILLED_SIGMA_VALUES) base_args.pop('num_inference_steps', None) + base_args.update(_canonical_ltx2_guidance(caps)) + if caps.use_cross_timestep: + base_args['use_cross_timestep'] = True log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=latent_pass args_keys={list(base_args.keys())}') result = shared.sd_model(**base_args) + # video latents strip the batch dim; audio latents keep it so LTX2Pipeline.prepare_audio_latents + # can rewrap them when re-entered as ndim==4 at Stage 2. latents = result.frames[0] if hasattr(result, 'frames') else None - audio = None + audio_latents = None if hasattr(result, 'audio') and result.audio is not None: - audio = result.audio[0].float().cpu() - return latents, audio + audio_latents = result.audio + return latents, audio_latents def run_ltx(task_id, @@ -128,6 +173,40 @@ def run_ltx(task_id, yield from abort(f'Video: cls={shared.sd_model.__class__.__name__} selected model is not LTX', ok=True) return + # Lightricks TI2VidTwoStagesPipeline: Stage 1 at half-res, 2x upsample, Stage 2 refine at target. + # Auto-couple when the user picks Refine but not Upsample. Condition variants still need per-stage + # conditioning rebuild, so keep them on the same-resolution path. + auto_refine_upsample = ( + refine_enable + and caps.supports_canonical_stage2 + and not upsample_enable + and not caps.supports_multi_condition + ) + effective_upsample_enable = upsample_enable or auto_refine_upsample + effective_upsample_ratio = upsample_ratio if upsample_enable else 2.0 + target_w = get_bucket(width) + target_h = get_bucket(height) + if auto_refine_upsample: + # Stage 1 at target/2 needs multiple-of-32; 2x upsample then forces final divisible by 64. + # Derive final from base, otherwise Stage 2 silently falls to base*2 != target. + base_w = get_bucket(target_w // 2) + base_h = get_bucket(target_h // 2) + final_w = base_w * 2 + final_h = base_h * 2 + if (final_w, final_h) != (target_w, target_h): + log.warning(f'LTX: two-stage refine needs resolution divisible by 64; adjusting {target_w}x{target_h} -> {final_w}x{final_h}') + elif effective_upsample_enable: + base_w = target_w + base_h = target_h + final_w = get_bucket(effective_upsample_ratio * target_w) + final_h = get_bucket(effective_upsample_ratio * target_h) + else: + base_w = target_w + base_h = target_h + final_w = target_w + final_h = target_h + log.debug(f'LTX: resolution planning target={target_w}x{target_h} base={base_w}x{base_h} final={final_w}x{final_h} auto_refine_upsample={auto_refine_upsample}') + videojob = shared.state.begin('Video', task_id=task_id) shared.state.job_count = 1 @@ -170,8 +249,8 @@ def run_ltx(task_id, sampler_name=sampler_name, sampler_shift=float(sampler_shift), steps=int(steps), - width=get_bucket(width), - height=get_bucket(height), + width=base_w, + height=base_h, frames=get_frames(frames), cfg_scale=float(guidance_scale) if guidance_scale is not None and guidance_scale > 0 else caps.default_cfg, denoising_strength=float(condition_strength) if condition_strength is not None else 1.0, @@ -205,277 +284,354 @@ def run_ltx(task_id, p.task_args['image'] = images.resize_image(resize_mode=2, im=effective_init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') if caps.family == '2.x' and caps.is_distilled: - # distilled 2.x was trained with a fixed sigma schedule; override diffusers' linspace default from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES p.task_args['sigmas'] = list(DISTILLED_SIGMA_VALUES) p.task_args.pop('num_inference_steps', None) + p.task_args.update(_canonical_ltx2_guidance(caps)) framewise = caps.family == '0.9' set_vae_params(p, framewise=framewise) + # Snapshot scheduler + shared.opts before mutation so the try/finally restores on every exit + # path (abort, interrupt, Stage 2 scheduler swap). Without this, run-specific sampler settings + # leak into shared.opts.data and across runs/tabs, and the default_scheduler snapshot from + # video_load.py:171 gets clobbered by a deepcopy of the mutated scheduler on every run. orig_dynamic_shift = shared.opts.schedulers_dynamic_shift orig_sampler_shift = shared.opts.schedulers_shift - shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift - shared.opts.data['schedulers_shift'] = sampler_shift - if hasattr(shared.sd_model, 'scheduler') and hasattr(shared.sd_model.scheduler, 'config') and hasattr(shared.sd_model.scheduler, 'register_to_config'): - if hasattr(shared.sd_model.scheduler.config, 'use_dynamic_shifting'): - shared.sd_model.scheduler.config.use_dynamic_shifting = dynamic_shift - shared.sd_model.scheduler.register_to_config(use_dynamic_shifting=dynamic_shift) - if hasattr(shared.sd_model.scheduler.config, 'flow_shift') and sampler_shift is not None and sampler_shift >= 0: - shared.sd_model.scheduler.config.flow_shift = sampler_shift - shared.sd_model.scheduler.register_to_config(flow_shift=sampler_shift) - shared.sd_model.default_scheduler = copy.deepcopy(shared.sd_model.scheduler) - - if selected is not None: - video_overrides.set_overrides(p, selected) - - t0 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - t1 = time.time() - - samplejob = shared.state.begin('Sample') - yield None, 'LTX: Generate in progress...' - - audio = None - pixels = None - frames_out = None - needs_latent_path = upsample_enable or refine_enable + orig_scheduler = shared.sd_model.scheduler + orig_default_scheduler = getattr(shared.sd_model, 'default_scheduler', None) + orig_use_dynamic_shifting = getattr(orig_scheduler.config, 'use_dynamic_shifting', None) if hasattr(orig_scheduler, 'config') else None + orig_flow_shift = getattr(orig_scheduler.config, 'flow_shift', None) if hasattr(orig_scheduler, 'config') else None try: - if needs_latent_path: - prompt_final, negative_final, networks = get_prompts(prompt, negative, styles) - extra_networks.activate(p, networks) - latents, audio = _latent_pass( - caps=caps, - prompt=prompt_final, - negative=negative_final, - width=width, - height=height, - frames=frames, - steps=steps, - guidance_scale=p.cfg_scale, - mp4_fps=mp4_fps, - conditions=conditions, - image_cond_noise_scale=image_cond_noise_scale if caps.supports_image_cond_noise_scale else None, - seed=int(seed) if seed is not None else -1, - image=p.task_args.get('image'), - ) - else: - processed = processing.process_images(p) - if processed is None or processed.images is None or len(processed.images) == 0: - yield from abort('Video: process_images returned no frames', ok=True, p=p) - return - pixels = processed.images - if getattr(processed, 'audio', None) is not None: - audio = processed.audio - latents = None - except AssertionError as e: - yield from abort(e, ok=True, p=p) - return - except Exception as e: - yield from abort(e, ok=False, p=p) - return + shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift + shared.opts.data['schedulers_shift'] = sampler_shift + if hasattr(shared.sd_model, 'scheduler') and hasattr(shared.sd_model.scheduler, 'config') and hasattr(shared.sd_model.scheduler, 'register_to_config'): + if hasattr(shared.sd_model.scheduler.config, 'use_dynamic_shifting'): + shared.sd_model.scheduler.config.use_dynamic_shifting = dynamic_shift + shared.sd_model.scheduler.register_to_config(use_dynamic_shifting=dynamic_shift) + if hasattr(shared.sd_model.scheduler.config, 'flow_shift') and sampler_shift is not None and sampler_shift >= 0: + shared.sd_model.scheduler.config.flow_shift = sampler_shift + shared.sd_model.scheduler.register_to_config(flow_shift=sampler_shift) + # Do NOT re-snapshot default_scheduler; that overwrites video_load.py:171's load-time + # snapshot with the run-mutated config, so reset_scheduler then carries the last run's choice. - t2 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - devices.torch_gc(force=True, reason='ltx:base') - t3 = time.time() - timer.process.add('offload', t1 - t0) - timer.process.add('base', t2 - t1) - timer.process.add('offload', t3 - t2) - shared.state.end(samplejob) + if selected is not None: + video_overrides.set_overrides(p, selected) - if upsample_enable and latents is not None: - t4 = time.time() - upsamplejob = shared.state.begin('Upsample') - try: - if caps.family == '0.9': - global upsample_pipe # pylint: disable=global-statement - upsample_pipe = load_upsample(upsample_pipe, upsample_repo_id_09) - upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) - up_args = { - 'width': get_bucket(upsample_ratio * width), - 'height': get_bucket(upsample_ratio * height), - 'generator': get_generator(int(seed) if seed is not None else -1), - 'output_type': 'latent', - } - if latents.ndim == 4: - latents = latents.unsqueeze(0) - log.debug(f'Video: op=upsample family=0.9 latents={latents.shape} {up_args}') - yield None, 'LTX: Upsample in progress...' - latents = upsample_pipe(latents=latents, **up_args).frames[0] - upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) - else: - from diffusers.pipelines.ltx2.pipeline_ltx2_latent_upsample import LTX2LatentUpsamplePipeline - log.info(f'Video load: cls={LTX2LatentUpsamplePipeline.__name__} family=2.x') - up_pipe = LTX2LatentUpsamplePipeline.from_pretrained( - 'Lightricks/LTX-2-Latent-Upsampler', - vae=shared.sd_model.vae, - cache_dir=shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - ) - up_pipe = sd_models.apply_balanced_offload(up_pipe) - up_args = { - 'width': get_bucket(upsample_ratio * width), - 'height': get_bucket(upsample_ratio * height), - 'num_frames': get_frames(frames), - 'latents_normalized': True, - 'generator': get_generator(int(seed) if seed is not None else -1), - 'output_type': 'latent', - } - if latents.ndim == 4: - latents = latents.unsqueeze(0) - log.debug(f'Video: op=upsample family=2.x latents={latents.shape} {up_args}') - yield None, 'LTX: Upsample in progress...' - latents = up_pipe(latents=latents, **up_args).frames[0] - up_pipe = sd_models.apply_balanced_offload(up_pipe) - except AssertionError as e: - yield from abort(e, ok=True, p=p) - return - except Exception as e: - yield from abort(e, ok=False, p=p) - return - t5 = time.time() - timer.process.add('upsample', t5 - t4) - shared.state.end(upsamplejob) - - if refine_enable and latents is not None: - t7 = time.time() - refinejob = shared.state.begin('Refine') + t0 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - devices.torch_gc(force=True, reason='ltx:refine') - # refine is the terminal stage when enabled: let the pipeline decode internally so the final vae pass runs - # inside the same offload/cudnn context as a normal generation, matching the Generic Video tab - refine_args = { - 'prompt': prompt_final, - 'negative_prompt': negative_final, - 'width': get_bucket((upsample_ratio if upsample_enable else 1.0) * width), - 'height': get_bucket((upsample_ratio if upsample_enable else 1.0) * height), - 'num_frames': get_frames(frames), - 'num_inference_steps': steps, - 'generator': get_generator(int(seed) if seed is not None else -1), - 'callback_on_step_end': diffusers_callback, - 'output_type': 'pil', - } - if p.cfg_scale is not None and p.cfg_scale > 0: - refine_args['guidance_scale'] = p.cfg_scale - if caps.supports_frame_rate_kwarg: - refine_args['frame_rate'] = float(mp4_fps) - if caps.supports_image_cond_noise_scale and image_cond_noise_scale is not None: - refine_args['image_cond_noise_scale'] = image_cond_noise_scale - if caps.supports_multi_condition and conditions: - refine_args['conditions'] = conditions - if caps.family == '2.x': - if caps.is_distilled: - # distilled variants have a canonical Stage-2 refine schedule they were trained on; - # see diffusers.pipelines.ltx2.utils and Lightricks/LTX-2 ti2vid_two_stages pipeline - from diffusers.pipelines.ltx2.utils import STAGE_2_DISTILLED_SIGMA_VALUES - refine_args['sigmas'] = list(STAGE_2_DISTILLED_SIGMA_VALUES) - else: - # non-distilled: truncate the default linspace schedule to match user-controlled refine_strength - default_sigmas = np.linspace(1.0, 1.0 / steps, steps) - num_skip = max(steps - max(int(steps * refine_strength), 1), 0) - refine_args['sigmas'] = default_sigmas[num_skip:].tolist() - refine_args.pop('num_inference_steps', None) - elif caps.repo_cls_name == 'LTXConditionPipeline': - refine_args['denoise_strength'] = refine_strength - if latents.ndim == 4: - latents = latents.unsqueeze(0) - log.debug(f'Video: op=refine cls={caps.repo_cls_name} latents={latents.shape}') - yield None, 'LTX: Refine in progress...' - try: - result = shared.sd_model(latents=latents, **refine_args) - pixels = result.frames[0] if hasattr(result, 'frames') else None - if hasattr(result, 'audio') and result.audio is not None: - audio = result.audio[0].float().cpu() - latents = None - except AssertionError as e: - yield from abort(e, ok=True, p=p) - return - except Exception as e: - yield from abort(e, ok=False, p=p) - return - t8 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - t9 = time.time() - timer.process.add('refine', t8 - t7) - timer.process.add('offload', t9 - t8) - shared.state.end(refinejob) + t1 = time.time() - shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift - shared.opts.data['schedulers_shift'] = orig_sampler_shift + samplejob = shared.state.begin('Sample') + yield None, 'LTX: Generate in progress...' - if needs_latent_path: - extra_networks.deactivate(p) - - if needs_latent_path and latents is not None: - # only reached when upsample ran without refine; refine decodes through the pipeline and sets latents=None - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'], force=True) - devices.torch_gc(force=True, reason='ltx:vae') - yield None, 'LTX: VAE decode in progress...' - try: - if torch.is_tensor(latents): - # 0.9.x returns raw latents with output_type='latent'; 2.x pre-denormalizes them - frames_out = vae_decode(latents, decode_timestep if caps.supports_decode_timestep else 0.0, int(seed) if seed is not None else -1, denormalize=caps.family == '0.9') - else: - frames_out = latents - except AssertionError as e: - yield from abort(e, ok=True, p=p) - return - except Exception as e: - yield from abort(e, ok=False, p=p) - return - pixels = frames_out - t10 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - t11 = time.time() - timer.process.add('offload', t11 - t10) - - if not audio_enable: audio = None + stage1_audio_latents = None + pixels = None + frames_out = None + needs_latent_path = upsample_enable or refine_enable - try: - aac_sample_rate = shared.sd_model.vocoder.config.output_sampling_rate - except Exception: - aac_sample_rate = 24000 + try: + if needs_latent_path: + prompt_final, negative_final, networks = get_prompts(prompt, negative, styles) + extra_networks.activate(p, networks) + latents, stage1_audio_latents = _latent_pass( + caps=caps, + prompt=prompt_final, + negative=negative_final, + width=base_w, + height=base_h, + frames=frames, + steps=steps, + guidance_scale=p.cfg_scale, + mp4_fps=mp4_fps, + conditions=conditions, + image_cond_noise_scale=image_cond_noise_scale if caps.supports_image_cond_noise_scale else None, + seed=int(seed) if seed is not None else -1, + image=p.task_args.get('image'), + ) + else: + processed = processing.process_images(p) + if processed is None or processed.images is None or len(processed.images) == 0: + yield from abort('Video: process_images returned no frames', ok=True, p=p) + return + pixels = processed.images + if getattr(processed, 'audio', None) is not None: + audio = processed.audio + latents = None + except AssertionError as e: + yield from abort(e, ok=True, p=p) + return + except Exception as e: + yield from abort(e, ok=False, p=p) + return - num_frames, video_file, _thumb = save_video( - p=p, - pixels=pixels, - audio=audio, - mp4_fps=mp4_fps, - mp4_codec=mp4_codec, - mp4_opt=mp4_opt, - mp4_ext=mp4_ext, - mp4_sf=mp4_sf, - mp4_video=mp4_video, - mp4_frames=mp4_frames, - mp4_interpolate=mp4_interpolate, - aac_sample_rate=aac_sample_rate, - metadata={}, - ) + t2 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + devices.torch_gc(force=True, reason='ltx:base') + t3 = time.time() + timer.process.add('offload', t1 - t0) + timer.process.add('base', t2 - t1) + timer.process.add('offload', t3 - t2) + shared.state.end(samplejob) - t_end = time.time() - if isinstance(pixels, list) and len(pixels) > 0 and isinstance(pixels[0], Image.Image): - w, h = pixels[0].size - elif hasattr(pixels, 'ndim') and pixels.ndim == 5: - _n, _c, _t, h, w = pixels.shape - elif hasattr(pixels, 'ndim') and pixels.ndim == 4: - _n, h, w, _c = pixels.shape - elif hasattr(pixels, 'shape'): - h, w = pixels.shape[-2], pixels.shape[-1] - else: - w, h = p.width, p.height - resolution = f'{w}x{h}' if num_frames > 0 else None - summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ') - memory = shared.mem_mon.summary() - total_time = max(t_end - t0, 1e-6) - fps = f'{num_frames/total_time:.2f}' - its = f'{(steps)/total_time:.2f}' + if effective_upsample_enable and latents is not None: + t4 = time.time() + upsamplejob = shared.state.begin('Upsample') + try: + if caps.family == '0.9': + global upsample_pipe # pylint: disable=global-statement + upsample_pipe = load_upsample(upsample_pipe, upsample_repo_id_09) + upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) + up_args = { + 'width': final_w, + 'height': final_h, + 'generator': get_generator(int(seed) if seed is not None else -1), + 'output_type': 'latent', + } + if latents.ndim == 4: + latents = latents.unsqueeze(0) + log.debug(f'Video: op=upsample family=0.9 latents={latents.shape} {up_args}') + yield None, 'LTX: Upsample in progress...' + latents = upsample_pipe(latents=latents, **up_args).frames[0] + upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) + else: + from diffusers.pipelines.ltx2.pipeline_ltx2_latent_upsample import LTX2LatentUpsamplePipeline + from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel + # Skip apply_balanced_offload on the upsampler; checkpoint_name differs from the main + # pipe so the shared OffloadHook (sd_offload.py:488) would rebuild and force a heavy + # re-init on the next refine. At ~2.3GB it fits on device; free after the pass. + upsample_repo = upsample_repo_id_23 if '2.3' in caps.name else upsample_repo_id_20 + log.info(f'Video load: cls={LTX2LatentUpsamplePipeline.__name__} family=2.x repo={upsample_repo} auto={auto_refine_upsample}') + latent_upsampler = LTX2LatentUpsamplerModel.from_pretrained( + upsample_repo, + subfolder='latent_upsampler', + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + ).to(devices.device) + up_pipe = LTX2LatentUpsamplePipeline(vae=shared.sd_model.vae, latent_upsampler=latent_upsampler) + # 2.x base pass returns denormalized latents; latents_normalized=False tells the + # upsampler "already raw, do not denormalize again". + up_args = { + 'width': final_w, + 'height': final_h, + 'num_frames': get_frames(frames), + 'latents_normalized': False, + 'generator': get_generator(int(seed) if seed is not None else -1), + 'output_type': 'latent', + } + if latents.ndim == 4: + latents = latents.unsqueeze(0) + log.debug(f'Video: op=upsample family=2.x latents={latents.shape} {up_args}') + yield None, 'LTX: Upsample in progress...' + latents = up_pipe(latents=latents, **up_args).frames[0] + latent_upsampler.to('cpu') + del up_pipe, latent_upsampler + devices.torch_gc(force=True, reason='ltx:upsample') + except AssertionError as e: + yield from abort(e, ok=True, p=p) + return + except Exception as e: + yield from abort(e, ok=False, p=p) + return + t5 = time.time() + timer.process.add('upsample', t5 - t4) + shared.state.end(upsamplejob) - shared.state.end(videojob) - progress.finish_task(task_id) - p.close() + if refine_enable and latents is not None: + t7 = time.time() + refinejob = shared.state.begin('Refine') + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + devices.torch_gc(force=True, reason='ltx:refine') + # Refine is terminal: let the pipe decode internally so the final VAE pass runs inside + # the same offload/cudnn context as a normal generation (matches Generic Video tab). + refine_args = { + 'prompt': prompt_final, + 'negative_prompt': negative_final, + 'width': final_w, + 'height': final_h, + 'num_frames': get_frames(frames), + 'num_inference_steps': steps, + 'generator': get_generator(int(seed) if seed is not None else -1), + 'callback_on_step_end': diffusers_callback, + 'output_type': 'pil', + } + if p.cfg_scale is not None and p.cfg_scale > 0: + refine_args['guidance_scale'] = p.cfg_scale + if caps.supports_frame_rate_kwarg: + refine_args['frame_rate'] = float(mp4_fps) + if caps.supports_image_cond_noise_scale and image_cond_noise_scale is not None: + refine_args['image_cond_noise_scale'] = image_cond_noise_scale + if caps.supports_multi_condition and conditions: + refine_args['conditions'] = conditions + # Thread Stage-1 I2V init image through Stage 2 so first-frame identity survives refine. + if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and p.task_args.get('image') is not None: + refine_args['image'] = p.task_args['image'] + # Thread Stage-1 audio latents into Stage 2 on 2.x. The video branch cross-attends + # audio every layer; letting prepare_audio_latents fall back to fresh noise biases + # the video branch off-distribution (desaturated output on distilled 2.x). + if caps.family == '2.x': + if stage1_audio_latents is not None: + refine_args['audio_latents'] = stage1_audio_latents.to(device=devices.device) + if caps.use_cross_timestep: + refine_args['use_cross_timestep'] = True - log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={fps} its={its} resolution={resolution} time={t_end-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') - yield video_file, f'LTX: Generation completed | File {video_file} | Frames {num_frames} | Resolution {resolution} | f/s {fps} | it/s {its} ' + f"

{summary} {memory}

" + saved_scheduler_stage2 = None + try: + if caps.supports_canonical_stage2: + # Dev 2.x Stage 2: swap scheduler, fuse distilled LoRA, 3 steps on the distilled + # sigma schedule at identity guidance (huggingface/diffusers#13217). + log.info(f'LTX: canonical Stage 2 via distilled LoRA repo={caps.stage2_dev_lora_repo}') + from diffusers import FlowMatchEulerDiscreteScheduler + offline_args = {'local_files_only': True} if shared.opts.offline_mode else {} + saved_scheduler_stage2 = shared.sd_model.scheduler + shared.sd_model.scheduler = FlowMatchEulerDiscreteScheduler.from_config( + saved_scheduler_stage2.config, + use_dynamic_shifting=False, + shift_terminal=None, + ) + shared.sd_model.load_lora_weights( + caps.stage2_dev_lora_repo, + adapter_name=STAGE2_DEV_LORA_ADAPTER, + cache_dir=shared.opts.hfcache_dir, + **offline_args, + ) + shared.sd_model.set_adapters([STAGE2_DEV_LORA_ADAPTER], [1.0]) + # Do NOT apply _canonical_ltx2_guidance on this path; its audio-branch kwargs + # would clobber the identity set. + refine_args.update(_canonical_stage2_dev_kwargs()) + refine_args.pop('num_inference_steps', None) + elif caps.family == '2.x': + # Distilled 2.x. Dev 2.x with a LoRA hit the branch above. + from diffusers.pipelines.ltx2.utils import STAGE_2_DISTILLED_SIGMA_VALUES + refine_args['sigmas'] = list(STAGE_2_DISTILLED_SIGMA_VALUES) + refine_args.pop('num_inference_steps', None) + # LTX2Pipeline/LTX2ImageToVideoPipeline default noise_scale=0.0 when not passed; + # sigma=0 user latents mismatched against sigmas[0] scheduler collapses output. + # LTX2ConditionPipeline auto-infers this; do the same explicitly for T2V/I2V. + refine_args['noise_scale'] = float(refine_args['sigmas'][0]) + refine_args.update(_canonical_ltx2_guidance(caps)) + elif caps.repo_cls_name == 'LTXConditionPipeline': + refine_args['denoise_strength'] = refine_strength + if latents.ndim == 4: + latents = latents.unsqueeze(0) + log.debug(f'Video: op=refine cls={caps.repo_cls_name} latents={latents.shape} canonical_stage2={caps.supports_canonical_stage2}') + yield None, 'LTX: Refine in progress...' + try: + result = shared.sd_model(latents=latents, **refine_args) + pixels = result.frames[0] if hasattr(result, 'frames') else None + if hasattr(result, 'audio') and result.audio is not None: + audio = result.audio[0].float().cpu() + latents = None + except AssertionError as e: + yield from abort(e, ok=True, p=p) + return + except Exception as e: + yield from abort(e, ok=False, p=p) + return + finally: + if saved_scheduler_stage2 is not None: + try: + from modules.lora.extra_networks_lora import unload_diffusers + unload_diffusers() + except Exception as e: + log.warning(f'LTX: canonical Stage 2 LoRA unload failed: {e}') + shared.sd_model.scheduler = saved_scheduler_stage2 + log.debug('LTX: canonical Stage 2 cleanup done (LoRA unloaded, scheduler restored)') + t8 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + t9 = time.time() + timer.process.add('refine', t8 - t7) + timer.process.add('offload', t9 - t8) + shared.state.end(refinejob) + + if needs_latent_path: + extra_networks.deactivate(p) + + if needs_latent_path and latents is not None: + # Only reached on upsample-without-refine; refine decodes through the pipe and nulls latents. + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'], force=True) + devices.torch_gc(force=True, reason='ltx:vae') + yield None, 'LTX: VAE decode in progress...' + try: + if torch.is_tensor(latents): + # 0.9.x returns raw latents with output_type='latent'; 2.x pre-denormalizes. + frames_out = vae_decode(latents, decode_timestep if caps.supports_decode_timestep else 0.0, int(seed) if seed is not None else -1, denormalize=caps.family == '0.9') + else: + frames_out = latents + except AssertionError as e: + yield from abort(e, ok=True, p=p) + return + except Exception as e: + yield from abort(e, ok=False, p=p) + return + pixels = frames_out + t10 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + t11 = time.time() + timer.process.add('offload', t11 - t10) + + if not audio_enable: + audio = None + + try: + aac_sample_rate = shared.sd_model.vocoder.config.output_sampling_rate + except Exception: + aac_sample_rate = 24000 + + num_frames, video_file, _thumb = save_video( + p=p, + pixels=pixels, + audio=audio, + mp4_fps=mp4_fps, + mp4_codec=mp4_codec, + mp4_opt=mp4_opt, + mp4_ext=mp4_ext, + mp4_sf=mp4_sf, + mp4_video=mp4_video, + mp4_frames=mp4_frames, + mp4_interpolate=mp4_interpolate, + aac_sample_rate=aac_sample_rate, + metadata={}, + ) + + t_end = time.time() + if isinstance(pixels, list) and len(pixels) > 0 and isinstance(pixels[0], Image.Image): + w, h = pixels[0].size + elif hasattr(pixels, 'ndim') and pixels.ndim == 5: + _n, _c, _t, h, w = pixels.shape + elif hasattr(pixels, 'ndim') and pixels.ndim == 4: + _n, h, w, _c = pixels.shape + elif hasattr(pixels, 'shape'): + h, w = pixels.shape[-2], pixels.shape[-1] + else: + w, h = p.width, p.height + resolution = f'{w}x{h}' if num_frames > 0 else None + summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ') + memory = shared.mem_mon.summary() + total_time = max(t_end - t0, 1e-6) + fps = f'{num_frames/total_time:.2f}' + its = f'{(steps)/total_time:.2f}' + + shared.state.end(videojob) + progress.finish_task(task_id) + p.close() + + log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={fps} its={its} resolution={resolution} time={t_end-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') + yield video_file, f'LTX: Generation completed | File {video_file} | Frames {num_frames} | Resolution {resolution} | f/s {fps} | it/s {its} ' + f"

{summary} {memory}

" + finally: + shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift + shared.opts.data['schedulers_shift'] = orig_sampler_shift + if shared.sd_model.scheduler is not orig_scheduler: + shared.sd_model.scheduler = orig_scheduler + if orig_default_scheduler is not None and shared.sd_model.default_scheduler is not orig_default_scheduler: + shared.sd_model.default_scheduler = orig_default_scheduler + if hasattr(shared.sd_model.scheduler, 'config') and hasattr(shared.sd_model.scheduler, 'register_to_config'): + if orig_use_dynamic_shifting is not None and hasattr(shared.sd_model.scheduler.config, 'use_dynamic_shifting'): + shared.sd_model.scheduler.config.use_dynamic_shifting = orig_use_dynamic_shifting + shared.sd_model.scheduler.register_to_config(use_dynamic_shifting=orig_use_dynamic_shifting) + if orig_flow_shift is not None and hasattr(shared.sd_model.scheduler.config, 'flow_shift'): + shared.sd_model.scheduler.config.flow_shift = orig_flow_shift + shared.sd_model.scheduler.register_to_config(flow_shift=orig_flow_shift) + log.debug(f'LTX: scheduler/opts restored dynamic_shift={orig_dynamic_shift} sampler_shift={orig_sampler_shift}') diff --git a/modules/video_models/video_load.py b/modules/video_models/video_load.py index c09197a0f..819706767 100644 --- a/modules/video_models/video_load.py +++ b/modules/video_models/video_load.py @@ -88,6 +88,13 @@ def load_model(selected: models_def.Model): selected.te = 'ai-forever/Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers' selected.te_folder = 'text_encoder' selected.te_revision = None + if selected.te_cls.__name__ == 'Gemma3ForConditionalGeneration' and shared.opts.te_shared_t5: + if 'SDNQ' in selected.name: + selected.te = 'OzzyGT/LTX-2.3-sdnq-dynamic-int4' + else: + selected.te = 'OzzyGT/LTX-2.3' + selected.te_folder = 'text_encoder' + selected.te_revision = None log.debug(f'Video load: module=te repo="{selected.te or selected.repo}" folder="{selected.te_folder}" cls={selected.te_cls.__name__} quant={model_quant.get_quant_type(quant_args)} loader={_loader("transformers")}') kwargs["text_encoder"] = selected.te_cls.from_pretrained( @@ -158,7 +165,11 @@ def load_model(selected: models_def.Model): return msg t1 = time.time() - if shared.sd_model.__class__.__name__.startswith("LTX"): + cls_name = shared.sd_model.__class__.__name__ + # LTX 0.9.x is plain linear; pin use_dynamic_shifting=False against upstream config drift. + # LTX-2.x canonical is token-count-based dynamic shift (base_shift=0.95, max_shift=2.05); + # disabling it there would take the model off-distribution. + if cls_name.startswith("LTX") and not cls_name.startswith("LTX2"): shared.sd_model.scheduler.config.use_dynamic_shifting = False shared.sd_model.default_scheduler = copy.deepcopy(shared.sd_model.scheduler) if hasattr(shared.sd_model, "scheduler") else None shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(selected.repo) diff --git a/modules/video_models/video_overrides.py b/modules/video_models/video_overrides.py index d1f74e992..34ee151d2 100644 --- a/modules/video_models/video_overrides.py +++ b/modules/video_models/video_overrides.py @@ -1,7 +1,7 @@ import os import torch import diffusers -from modules import shared, processing +from modules import shared, processing, devices from modules.logger import log from modules.video_models.models_def import Model @@ -17,6 +17,35 @@ def load_override(selected: Model, **load_args): # LTX if 'LTXVideo 0.9.5 I2V' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLLTXVideo.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir, **load_args) + # OzzyGT LTX-2.3 mirrors ship connectors twice: sharded safetensors + .index.json plus a + # redundant unsharded diffusion_pytorch_model.safetensors of the same weights. Diffusers + # fetches both but loads sharded; skip the ~6.3 GB duplicate. + ltx2_redundant_connector_repos = { + 'OzzyGT/LTX-2.3', + 'OzzyGT/LTX-2.3-sdnq-dynamic-int4', + } + if selected.repo in ltx2_redundant_connector_repos: + kwargs['ignore_patterns'] = ['connectors/diffusion_pytorch_model.safetensors'] + # LTX2TextConnectors weights are byte-identical across all 2.3 variants (verified by blob + # hash). Pre-load from a canonical repo so per-variant fetches skip connectors/ entirely. + # FP16 variants share OzzyGT/LTX-2.3; SDNQ variants share the pre-quantized mirror. + ltx2_connectors_cls = None + try: + from diffusers.pipelines.ltx2 import LTX2TextConnectors + ltx2_connectors_cls = LTX2TextConnectors + except ImportError as e: + log.warning(f'Video load: LTX2TextConnectors unavailable ({e}); dedup of LTX-2.3 connectors disabled') + if ('LTXVideo 2.3' in selected.name and shared.opts.te_shared_t5 and ltx2_connectors_cls is not None): + conn_repo = 'OzzyGT/LTX-2.3-sdnq-dynamic-int4' if 'SDNQ' in selected.name else 'OzzyGT/LTX-2.3' + log.debug(f'Video load: module=connectors repo="{conn_repo}" cls={ltx2_connectors_cls.__name__} shared={shared.opts.te_shared_t5}') + kwargs['connectors'] = ltx2_connectors_cls.from_pretrained( + conn_repo, + subfolder='connectors', + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + ignore_patterns=['connectors/diffusion_pytorch_model.safetensors'], + **load_args, + ) # WAN if 'WAN 2.1 14B' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLWan.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir, **load_args) From 318a5397dc1dde3a988a6aba2c7b09f828d8806f Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 19 Apr 2026 03:37:59 +0100 Subject: [PATCH 06/15] feat(ltx): dynamic_shift checkbox and refine_strength gate Wire the dynamic_shift checkbox into _model_change so its value resets to caps.default_dynamic_shift on model swap, and narrow the refine_strength interactive gate to the 0.9.x family. On 2.x refine_strength is unused: Distilled runs a pre-baked sigma schedule, Dev runs the canonical Stage 2 distilled-LoRA schedule, neither consumes the slider. --- modules/ltx/ltx_ui.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index a19539a4c..bd5f523e1 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -23,12 +23,13 @@ def _model_change(model_name: str): gr.update(), # guidance_scale gr.update(), # steps gr.update(), # sampler_shift + gr.update(), # dynamic_shift gr.update(interactive=False), # decode_timestep gr.update(interactive=False), # image_cond_noise_scale gr.update(visible=False), # audio_accordion ) - # distilled 2.x variants use a fixed canonical refine schedule; the strength slider is meaningless there - refine_strength_interactive = not (caps.family == '2.x' and caps.is_distilled) + # 2.x refine runs fixed canonical schedules; refine_strength only feeds 0.9.x LTXConditionPipeline. + refine_strength_interactive = caps.family == '0.9' return ( gr.update(visible=caps.supports_input_media), gr.update(visible=caps.supports_multi_condition), @@ -40,6 +41,7 @@ def _model_change(model_name: str): gr.update(value=caps.default_cfg), gr.update(value=caps.default_steps), gr.update(value=caps.default_sampler_shift), + gr.update(value=caps.default_dynamic_shift), gr.update(interactive=caps.supports_decode_timestep), gr.update(interactive=caps.supports_image_cond_noise_scale), gr.update(visible=caps.supports_audio), @@ -115,6 +117,7 @@ def create_ui(prompt, negative, styles, overrides, init_image, _init_strength, l guidance_scale, steps, sampler_shift, + dynamic_shift, decode_timestep, image_cond_noise_scale, audio_accordion, From 19b930a221e4ebd07a2f00fa1896062b66a51449 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 19 Apr 2026 03:42:51 +0100 Subject: [PATCH 07/15] feat(video): hide LTX engine from general video tab dropdown The LTX tab owns LTX generation; keep the general video engine dropdown from listing LTX so users have a single entry point per engine. --- modules/video_models/video_ui.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/video_models/video_ui.py b/modules/video_models/video_ui.py index 4614e306e..e14147a55 100644 --- a/modules/video_models/video_ui.py +++ b/modules/video_models/video_ui.py @@ -9,6 +9,14 @@ from modules.video_models import video_run debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None +# Engines surfaced on their own dedicated tab; hide from the general Video tab dropdown +# so users aren't offered two paths to the same models. +HIDDEN_ENGINES = {'LTX Video'} + + +def visible_engines(): + return [name for name in models_def.models if name not in HIDDEN_ENGINES] + def engine_change(engine): debug(f'Video change: engine="{engine}"') @@ -127,7 +135,7 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la with gr.Row(): generate = gr.Button('Generate', elem_id="video_generate_btn", variant='primary', visible=False) with gr.Row(): - engine = gr.Dropdown(label='Video engine', choices=list(models_def.models), value='None', elem_id="video_engine") + engine = gr.Dropdown(label='Video engine', choices=visible_engines(), value='None', elem_id="video_engine") model = gr.Dropdown(label='Video model', choices=[''], value='None', elem_id="video_model") btn_load = ToolButton(ui_symbols.loading, elem_id="video_model_load") with gr.Row(): From 44759f5f927aff1731cd1803e064fb737fee60c8 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 19 Apr 2026 04:38:47 +0100 Subject: [PATCH 08/15] refactor(video): per-engine self-contained tab layout Each engine tab (Generic, FramePack, LTX) now owns its own size, seed, frame count, and init imagery. The previously shared Size and Inputs subtabs are removed; persistent encoding settings move to a single Output subtab constructed first so engine tabs can consume its widgets. FramePack drops its local fps/interpolate sliders and consumes the shared Output widgets; LTX drops the redundant shared init_image parameter and keeps only its own ltx_init_image, with run_ltx's condition_image fallback removed in turn. --- modules/framepack/framepack_ui.py | 7 ++-- modules/ltx/ltx_process.py | 15 +++----- modules/ltx/ltx_ui.py | 16 ++++++-- modules/ui_video.py | 25 ++++++------ modules/video_models/video_ui.py | 63 ++++++++++++------------------- 5 files changed, 62 insertions(+), 64 deletions(-) diff --git a/modules/framepack/framepack_ui.py b/modules/framepack/framepack_ui.py index 394387df7..93b57863a 100644 --- a/modules/framepack/framepack_ui.py +++ b/modules/framepack/framepack_ui.py @@ -12,7 +12,7 @@ def change_sections(duration, mp4_fps, mp4_interpolate, latent_ws, variant): return gr.update(value=f'Target video: {num_frames} frames in {num_sections} sections'), gr.update(lines=max(2, 2*num_sections//3)) -def create_ui(prompt, negative, styles, _overrides, init_image, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf): +def create_ui(prompt, negative, styles, _overrides, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf): with gr.Row(): with gr.Column(variant='compact', elem_id="framepack_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): @@ -22,11 +22,12 @@ def create_ui(prompt, negative, styles, _overrides, init_image, last_image, mp4_ with gr.Row(): resolution = gr.Slider(label="FP resolution", minimum=240, maximum=1088, value=640, step=16) duration = gr.Slider(label="FP duration", minimum=1, maximum=120, value=4, step=0.1) - mp4_fps = gr.Slider(label="FP target FPS", minimum=1, maximum=60, value=24, step=1) - mp4_interpolate = gr.Slider(label="FP interpolation", minimum=0, maximum=10, value=0, step=1) with gr.Row(): section_html = gr.HTML(show_label=False, elem_id="framepack_section_html") with gr.Accordion(label="Inputs", open=False): + with gr.Row(): + init_image = gr.Image(label='FP init image', elem_id='framepack_init_image', type='pil', image_mode='RGB', width=256, height=256) + last_image = gr.Image(label='FP last image', elem_id='framepack_last_image', type='pil', image_mode='RGB', width=256, height=256) with gr.Row(): start_weight = gr.Slider(label="FP init strength", value=1.0, minimum=0.0, maximum=2.0, step=0.05, elem_id="framepack_start_weight") end_weight = gr.Slider(label="FP end strength", value=1.0, minimum=0.0, maximum=2.0, step=0.05, elem_id="framepack_end_weight") diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index eeb14fab8..9c64bafc8 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -122,7 +122,6 @@ def run_ltx(task_id, refine_strength: float, condition_strength: float, ltx_init_image, - condition_image, condition_last, condition_files, condition_video, @@ -213,15 +212,13 @@ def run_ltx(task_id, from modules.video_models import models_def, video_overrides selected = next((m for m in models_def.models.get(engine, []) if m.name == model), None) - effective_init_image = ltx_init_image if ltx_init_image is not None else condition_image - - if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and effective_init_image is None: + if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and ltx_init_image is None: yield from abort('No input image provided. Please upload or select an image.', ok=True) return condition_images = [] - if effective_init_image is not None: - condition_images.append(effective_init_image) + if ltx_init_image is not None: + condition_images.append(ltx_init_image) if condition_last is not None: condition_images.append(condition_last) conditions = [] @@ -254,7 +251,7 @@ def run_ltx(task_id, frames=get_frames(frames), cfg_scale=float(guidance_scale) if guidance_scale is not None and guidance_scale > 0 else caps.default_cfg, denoising_strength=float(condition_strength) if condition_strength is not None else 1.0, - init_image=effective_init_image, + init_image=ltx_init_image, vae_type='Default', vae_tile_frames=16, ) @@ -279,9 +276,9 @@ def run_ltx(task_id, if caps.supports_multi_condition and conditions: p.task_args['conditions'] = conditions - if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and effective_init_image is not None: + if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and ltx_init_image is not None: from modules import images - p.task_args['image'] = images.resize_image(resize_mode=2, im=effective_init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + p.task_args['image'] = images.resize_image(resize_mode=2, im=ltx_init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') if caps.family == '2.x' and caps.is_distilled: from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index bd5f523e1..de72efda1 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -1,6 +1,7 @@ import os import gradio as gr -from modules import ui_sections +from modules import ui_sections, ui_symbols +from modules.ui_components import ToolButton from modules.logger import log from modules.video_models.models_def import models from modules.ltx import ltx_process, ltx_capabilities @@ -48,7 +49,7 @@ def _model_change(model_name: str): ) -def create_ui(prompt, negative, styles, overrides, init_image, _init_strength, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, width, height, frames, seed): +def create_ui(prompt, negative, styles, overrides, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf): with gr.Row(): with gr.Column(variant='compact', elem_id="ltx_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): @@ -56,10 +57,19 @@ def create_ui(prompt, negative, styles, overrides, init_image, _init_strength, l with gr.Row(): ltx_models = [m.name for m in models['LTX Video']] if 'LTX Video' in models else ['None'] model = gr.Dropdown(label='LTX model', choices=ltx_models, value=ltx_models[0], elem_id="ltx_model") + with gr.Accordion(open=False, label='Size', elem_id='ltx_size_accordion'): + width, height = ui_sections.create_resolution_inputs('ltx', default_width=832, default_height=480) + with gr.Row(): + frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=121, elem_id='ltx_frames') + seed = gr.Number(label='Initial seed', value=-1, elem_id='ltx_seed', container=True) + random_seed = ToolButton(ui_symbols.random, elem_id='ltx_seed_random') + random_seed.click(fn=lambda: -1, show_progress='hidden', inputs=[], outputs=[seed]) input_media_accordion = gr.Accordion(open=False, label="Input media", elem_id='ltx_input_media_accordion', visible=False) with input_media_accordion: ltx_init_image = gr.Image(label='Image', elem_id='ltx_init_image', type='pil', image_mode='RGB', width=256, height=256) ltx_condition_strength = gr.Slider(label='LTX input strength', minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id='ltx_condition_strength') + with gr.Row(): + last_image = gr.Image(label='Last image', elem_id='ltx_last_image', type='pil', image_mode='RGB', width=256, height=256) multi_condition_group = gr.Group(visible=False) with multi_condition_group: gr.Markdown('**Prefix conditioning**: supply a video or gallery to anchor the opening frames', elem_id='ltx_prefix_conditioning_label') @@ -137,7 +147,7 @@ def create_ui(prompt, negative, styles, overrides, init_image, _init_strength, l seed, upsample_enable, upsample_ratio, refine_enable, refine_strength, - ltx_condition_strength, ltx_init_image, init_image, last_image, condition_files, condition_video, condition_video_frames, condition_video_skip, + ltx_condition_strength, ltx_init_image, last_image, condition_files, condition_video, condition_video_frames, condition_video_skip, decode_timestep, image_cond_noise_scale, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, audio_enable, diff --git a/modules/ui_video.py b/modules/ui_video.py index c6026fdbd..4d9362973 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -28,24 +28,27 @@ def create_ui(): with gr.Row(elem_id="video_interface", equal_height=False): with gr.Tabs(elem_classes=['video-tabs'], elem_id='video-tabs'): overrides = ui_common.create_override_inputs('video') - with gr.Tab('Size', id='video-size-tab') as _video_size_tab: - from modules.video_models import video_ui - width, height, frames, seed, reuse_seed = video_ui.create_ui_size() - with gr.Tab('Inputs', id='video-inputs-tab') as _video_inputs_tab: - from modules.video_models import video_ui - init_image, init_strength, last_image = video_ui.create_ui_inputs() - with gr.Tab('Video Output', id='video-outputs-tab') as _video_outputs_tab: + with gr.Tab('Output', id='video-outputs-tab') as _video_outputs_tab: from modules.video_models import video_ui mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf = video_ui.create_ui_outputs() - with gr.Tab('Models', id='video-core-tab') as video_core_tab: + with gr.Tab('Generic', id='video-core-tab') as video_core_tab: from modules.video_models import video_ui - engine, model, steps, sampler_index = video_ui.create_ui(prompt, negative, styles, overrides, init_image, init_strength, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, width, height, frames, seed, reuse_seed) + engine, model, steps, sampler_index, width, height, frames, seed = video_ui.create_ui( + prompt, negative, styles, overrides, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, + ) with gr.Tab('FramePack', id='framepack-tab') as framepack_tab: from modules.framepack import framepack_ui - framepack_ui.create_ui(prompt, negative, styles, overrides, init_image, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf) + framepack_ui.create_ui( + prompt, negative, styles, overrides, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, + ) with gr.Tab('LTX', id='ltx-tab') as ltx_tab: from modules.ltx import ltx_ui - ltx_ui.create_ui(prompt, negative, styles, overrides, init_image, init_strength, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, width, height, frames, seed) + ltx_ui.create_ui( + prompt, negative, styles, overrides, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, + ) paste_fields = [ (prompt, "Prompt"), # cannot add more fields as they are not defined yet diff --git a/modules/video_models/video_ui.py b/modules/video_models/video_ui.py index e14147a55..17121c7a3 100644 --- a/modules/video_models/video_ui.py +++ b/modules/video_models/video_ui.py @@ -86,17 +86,6 @@ def run_video(*args): return video_utils.queue_err(f'model not found: engine="{engine}" model="{model}"') -def create_ui_inputs(): - with gr.Row(): - with gr.Column(variant='compact', elem_id="video_inputs", elem_classes=['settings-column'], scale=1): - init_strength = gr.Slider(label='Init strength', minimum=0.0, maximum=1.0, step=0.01, value=0.8, elem_id="video_denoising_strength") - gr.HTML("
  Init image") - init_image = gr.Image(elem_id="video_image", show_label=False, type="pil", image_mode="RGB", width=256, height=256) - gr.HTML("
  Last image") - last_image = gr.Image(elem_id="video_last", show_label=False, type="pil", image_mode="RGB", width=256, height=256) - return init_image, init_strength, last_image - - def create_ui_outputs(): with gr.Row(): with gr.Column(variant='compact', elem_id="video_outputs", elem_classes=['settings-column'], scale=1): @@ -105,31 +94,17 @@ def create_ui_outputs(): mp4_interpolate = gr.Slider(label="Video interpolation", minimum=0, maximum=10, value=0, step=1) with gr.Row(): mp4_codec = gr.Dropdown(label="Video codec", choices=['none', 'libx264'], value='libx264', type='value') - ui_common.create_refresh_button(mp4_codec, video_utils.get_codecs, elem_id="framepack_mp4_codec_refresh") - mp4_ext = gr.Textbox(label="Video format", value='mp4', elem_id="framepack_mp4_ext") - mp4_opt = gr.Textbox(label="Video options", value='crf:16', elem_id="framepack_mp4_opt") + ui_common.create_refresh_button(mp4_codec, video_utils.get_codecs, elem_id="video_mp4_codec_refresh") + mp4_ext = gr.Textbox(label="Video format", value='mp4', elem_id="video_mp4_ext") + mp4_opt = gr.Textbox(label="Video options", value='crf:16', elem_id="video_mp4_opt") with gr.Row(): - mp4_video = gr.Checkbox(label='Video save video', value=True, elem_id="framepack_mp4_video") - mp4_frames = gr.Checkbox(label='Video save frames', value=False, elem_id="framepack_mp4_frames") - mp4_sf = gr.Checkbox(label='Video save safetensors', value=False, elem_id="framepack_mp4_sf") + mp4_video = gr.Checkbox(label='Video save video', value=True, elem_id="video_mp4_video") + mp4_frames = gr.Checkbox(label='Video save frames', value=False, elem_id="video_mp4_frames") + mp4_sf = gr.Checkbox(label='Video save safetensors', value=False, elem_id="video_mp4_sf") return mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf -def create_ui_size(): - with gr.Row(): - with gr.Column(variant='compact', elem_id="video_size", elem_classes=['settings-column'], scale=1): - with gr.Row(): - width, height = ui_sections.create_resolution_inputs('video', default_width=832, default_height=480) - with gr.Row(): - frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=17, elem_id="video_frames") - seed = gr.Number(label='Initial seed', value=-1, elem_id="video_seed", container=True) - random_seed = ToolButton(ui_symbols.random, elem_id="video_seed_random") - reuse_seed = ToolButton(ui_symbols.reuse, elem_id="video_seed_reuse") - random_seed.click(fn=lambda: -1, show_progress='hidden', inputs=[], outputs=[seed]) - return width, height, frames, seed, reuse_seed - - -def create_ui(prompt, negative, styles, overrides, init_image, init_strength, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, width, height, frames, seed, reuse_seed): +def create_ui(prompt, negative, styles, overrides, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf): with gr.Row(): with gr.Column(variant='compact', elem_id="video_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): @@ -138,7 +113,7 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la engine = gr.Dropdown(label='Video engine', choices=visible_engines(), value='None', elem_id="video_engine") model = gr.Dropdown(label='Video model', choices=[''], value='None', elem_id="video_model") btn_load = ToolButton(ui_symbols.loading, elem_id="video_model_load") - with gr.Row(): + with gr.Accordion(open=True, label="Model", elem_id='video_model_accordion'): url = gr.HTML(label='Model URL', elem_id='video_model_url', value='

') with gr.Accordion(open=False, label="Parameters", elem_id='video_parameters_accordion'): steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video", default_steps=50) @@ -148,6 +123,21 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la with gr.Row(): guidance_scale = gr.Slider(label='Guidance scale', minimum=-1.0, maximum=14.0, step=0.1, value=-1.0, elem_id="video_guidance_scale") guidance_true = gr.Slider(label='True guidance', minimum=-1.0, maximum=14.0, step=0.1, value=-1.0, elem_id="video_guidance_true") + with gr.Accordion(open=False, label="Size", elem_id='video_size_accordion'): + with gr.Row(): + width, height = ui_sections.create_resolution_inputs('video', default_width=832, default_height=480) + with gr.Row(): + frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=17, elem_id="video_frames") + seed = gr.Number(label='Initial seed', value=-1, elem_id="video_seed", container=True) + random_seed = ToolButton(ui_symbols.random, elem_id="video_seed_random") + reuse_seed = ToolButton(ui_symbols.reuse, elem_id="video_seed_reuse") + random_seed.click(fn=lambda: -1, show_progress='hidden', inputs=[], outputs=[seed]) + with gr.Accordion(open=False, label="Inputs", elem_id='video_inputs_accordion'): + init_strength = gr.Slider(label='Init strength', minimum=0.0, maximum=1.0, step=0.01, value=0.8, elem_id="video_denoising_strength") + gr.HTML("
  Init image") + init_image = gr.Image(elem_id="video_image", show_label=False, type="pil", image_mode="RGB", width=256, height=256) + gr.HTML("
  Last image") + last_image = gr.Image(elem_id="video_last", show_label=False, type="pil", image_mode="RGB", width=256, height=256) with gr.Accordion(open=False, label="Decode", elem_id='video_decode_accordion'): with gr.Row(): vae_type = gr.Dropdown(label='VAE decode', choices=['Default', 'Tiny', 'Remote', 'Upscale'], value='Default', elem_id="video_vae_type") @@ -163,18 +153,15 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la with gr.Tab('Frames', id='out-gallery'): gallery, gen_info, html_info, _html_info_formatted, html_log = ui_common.create_output_panel("video", prompt=prompt, preview=False, transfer=False, scale=2) - # connect reuse seed button ui_common.connect_reuse_seed(seed, reuse_seed, gen_info, is_subseed=False) - # handle engine and model change engine.change(fn=engine_change, inputs=[engine], outputs=[model]) model.change(fn=model_change, inputs=[engine, model], outputs=[url]) btn_load.click(fn=model_load, inputs=[engine, model], outputs=[html_log]) - # hidden fields + task_id = gr.Textbox(visible=False, value='') ui_state = gr.Textbox(visible=False, value='') state_inputs = [task_id, ui_state] - # generate args video_inputs = [ engine, model, prompt, negative, styles, @@ -206,4 +193,4 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la show_progress='hidden', ) generate.click(**video_dict) - return [engine, model, steps, sampler_index] + return engine, model, steps, sampler_index, width, height, frames, seed From ef00f93570ec62654c6dd707b767c55fe448341c Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 19 Apr 2026 04:52:15 +0100 Subject: [PATCH 09/15] fix(video): drop empty Model URL accordion on generic tab --- modules/video_models/video_ui.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/video_models/video_ui.py b/modules/video_models/video_ui.py index 17121c7a3..baab34bd1 100644 --- a/modules/video_models/video_ui.py +++ b/modules/video_models/video_ui.py @@ -113,8 +113,7 @@ def create_ui(prompt, negative, styles, overrides, mp4_fps, mp4_interpolate, mp4 engine = gr.Dropdown(label='Video engine', choices=visible_engines(), value='None', elem_id="video_engine") model = gr.Dropdown(label='Video model', choices=[''], value='None', elem_id="video_model") btn_load = ToolButton(ui_symbols.loading, elem_id="video_model_load") - with gr.Accordion(open=True, label="Model", elem_id='video_model_accordion'): - url = gr.HTML(label='Model URL', elem_id='video_model_url', value='

') + url = gr.HTML(label='Model URL', elem_id='video_model_url', value='

') with gr.Accordion(open=False, label="Parameters", elem_id='video_parameters_accordion'): steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video", default_steps=50) with gr.Row(): From 7e5d040c4c324bad2014b5b6c7e0658a6bcbd22e Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Mon, 20 Apr 2026 00:26:15 +0100 Subject: [PATCH 10/15] refactor(ltx): address PR #4783 review threads 1, 3-6 LTXCaps gains a `variant` field ('0.9', '2.0', '2.3') replacing `is_ltx_2_3`; variant-specific branches check `caps.variant == '2.3'` instead of grepping the model name. ltx_util.load_upsample_2x mirrors load_upsample's contract so the 2.x path owns a module-level cache and stops reloading ~2.3 GB every run. The cached pipe is stamped with a synthetic `CheckpointInfo('ltx-upsampler-2.x')` so it gets its own OffloadHook slot and can go through apply_balanced_offload without invalidating the main pipe's module map. The hardcoded `.to('cpu')` and post-pass torch_gc are gone; the second apply_balanced_offload handles spill. video_overrides comment on OzzyGT LTX-2.3 connectors states plainly that mirrors pack weights twice by design; ignore_patterns is the surgical workaround, not an hf_hub bug. Drive-by: load_upsample log line used `__class__.__name__` (always 'type'); switched to `__name__`. --- modules/ltx/ltx_capabilities.py | 20 ++++++++++------- modules/ltx/ltx_process.py | 29 ++++++++---------------- modules/ltx/ltx_util.py | 30 ++++++++++++++++++++++++- modules/video_models/video_overrides.py | 8 ++++--- 4 files changed, 55 insertions(+), 32 deletions(-) diff --git a/modules/ltx/ltx_capabilities.py b/modules/ltx/ltx_capabilities.py index d345eaa2a..499f8b57b 100644 --- a/modules/ltx/ltx_capabilities.py +++ b/modules/ltx/ltx_capabilities.py @@ -9,8 +9,8 @@ class LTXCaps: name: str repo_cls_name: str family: str # '0.9' or '2.x' + variant: str # '0.9', '2.0', '2.3' (finer-grained sub-variant) is_distilled: bool - is_ltx_2_3: bool is_i2v: bool supports_input_media: bool supports_multi_condition: bool @@ -76,18 +76,22 @@ def get_caps(model_name: str) -> Optional[LTXCaps]: is_ltx2 = cls_name in LTX2_CLASSES family = '2.x' if is_ltx2 else '0.9' + # 2.x sub-variant detection: unknown 2.x mirrors fall through to '2.0' (conservative default). + if is_ltx2: + variant = '2.3' if '2.3' in model_name else '2.0' + else: + variant = '0.9' is_distilled = 'Distilled' in model_name is_i2v = 'I2V' in model_name or cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') is_condition_cls = cls_name in CONDITION_CLASSES supports_input_media = is_i2v or is_condition_cls - is_ltx_2_3 = is_ltx2 and '2.3' in model_name caps = LTXCaps( name=model_name, repo_cls_name=cls_name, family=family, + variant=variant, is_distilled=is_distilled, - is_ltx_2_3=is_ltx_2_3, is_i2v=is_i2v, supports_input_media=supports_input_media, supports_multi_condition=is_condition_cls, @@ -96,7 +100,7 @@ def get_caps(model_name: str) -> Optional[LTXCaps]: supports_stg=is_ltx2, supports_audio=is_ltx2, supports_frame_rate_kwarg=is_ltx2, - use_cross_timestep=is_ltx_2_3, + use_cross_timestep=(variant == '2.3'), default_cfg=3.0, default_steps=30 if is_ltx2 else 50, default_sampler_shift=-1.0, @@ -112,16 +116,16 @@ def get_caps(model_name: str) -> Optional[LTXCaps]: caps.default_steps = 8 if is_ltx2 and not is_distilled: - if is_ltx_2_3: + if variant == '2.3': caps.stage2_dev_lora_repo = 'CalamitousFelicitousness/LTX-2.3-distilled-lora-384-Diffusers' - elif '2.0' in model_name: + elif variant == '2.0': caps.stage2_dev_lora_repo = 'CalamitousFelicitousness/LTX-2.0-distilled-lora-384-Diffusers' caps.supports_canonical_stage2 = caps.stage2_dev_lora_repo is not None if is_ltx2: - if '2.3' in model_name: + if variant == '2.3': caps.stg_default_blocks = [28] - elif '2.0' in model_name: + elif variant == '2.0': caps.stg_default_blocks = [29] else: caps.stg_default_blocks = [28] diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 9c64bafc8..18aab0d44 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -6,7 +6,7 @@ from PIL import Image from modules import shared, errors, timer, memstats, progress, processing, sd_models, sd_samplers, devices, extra_networks, call_queue from modules.logger import log from modules.ltx import ltx_capabilities -from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, get_conditions, get_generator, get_prompts, vae_decode +from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, load_upsample_2x, get_conditions, get_generator, get_prompts, vae_decode from modules.processing_callbacks import diffusers_callback from modules.video_models.video_vae import set_vae_params from modules.video_models.video_save import save_video @@ -20,6 +20,7 @@ upsample_repo_id_09 = 'a-r-r-o-w/LTX-Video-0.9.7-Latent-Spatial-Upsampler-diffus upsample_repo_id_20 = 'Lightricks/LTX-2' upsample_repo_id_23 = 'CalamitousFelicitousness/LTX-2.3-Spatial-Upsampler-x2-1.1-Diffusers' upsample_pipe = None +upsample_pipe_2x = None STAGE2_DEV_LORA_ADAPTER = 'ltx2_stage2_distilled' @@ -394,20 +395,10 @@ def run_ltx(task_id, latents = upsample_pipe(latents=latents, **up_args).frames[0] upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) else: - from diffusers.pipelines.ltx2.pipeline_ltx2_latent_upsample import LTX2LatentUpsamplePipeline - from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel - # Skip apply_balanced_offload on the upsampler; checkpoint_name differs from the main - # pipe so the shared OffloadHook (sd_offload.py:488) would rebuild and force a heavy - # re-init on the next refine. At ~2.3GB it fits on device; free after the pass. - upsample_repo = upsample_repo_id_23 if '2.3' in caps.name else upsample_repo_id_20 - log.info(f'Video load: cls={LTX2LatentUpsamplePipeline.__name__} family=2.x repo={upsample_repo} auto={auto_refine_upsample}') - latent_upsampler = LTX2LatentUpsamplerModel.from_pretrained( - upsample_repo, - subfolder='latent_upsampler', - cache_dir=shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - ).to(devices.device) - up_pipe = LTX2LatentUpsamplePipeline(vae=shared.sd_model.vae, latent_upsampler=latent_upsampler) + global upsample_pipe_2x # pylint: disable=global-statement + upsample_repo = upsample_repo_id_23 if caps.variant == '2.3' else upsample_repo_id_20 + upsample_pipe_2x = load_upsample_2x(upsample_pipe_2x, upsample_repo) + upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x) # 2.x base pass returns denormalized latents; latents_normalized=False tells the # upsampler "already raw, do not denormalize again". up_args = { @@ -420,12 +411,10 @@ def run_ltx(task_id, } if latents.ndim == 4: latents = latents.unsqueeze(0) - log.debug(f'Video: op=upsample family=2.x latents={latents.shape} {up_args}') + log.debug(f'Video: op=upsample family=2.x latents={latents.shape} auto={auto_refine_upsample} {up_args}') yield None, 'LTX: Upsample in progress...' - latents = up_pipe(latents=latents, **up_args).frames[0] - latent_upsampler.to('cpu') - del up_pipe, latent_upsampler - devices.torch_gc(force=True, reason='ltx:upsample') + latents = upsample_pipe_2x(latents=latents, **up_args).frames[0] + upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x) except AssertionError as e: yield from abort(e, ok=True, p=p) return diff --git a/modules/ltx/ltx_util.py b/modules/ltx/ltx_util.py index 310a865fb..b404879f7 100644 --- a/modules/ltx/ltx_util.py +++ b/modules/ltx/ltx_util.py @@ -40,7 +40,7 @@ def load_upsample(upsample_pipe, upsample_repo_id): if upsample_pipe is None: t0 = time.time() from diffusers.pipelines.ltx.pipeline_ltx_latent_upsample import LTXLatentUpsamplePipeline - log.info(f'Video load: cls={LTXLatentUpsamplePipeline.__class__.__name__} repo="{upsample_repo_id}"') + log.info(f'Video load: cls={LTXLatentUpsamplePipeline.__name__} repo="{upsample_repo_id}"') upsample_pipe = LTXLatentUpsamplePipeline.from_pretrained( upsample_repo_id, vae=shared.sd_model.vae, @@ -52,6 +52,34 @@ def load_upsample(upsample_pipe, upsample_repo_id): return upsample_pipe +def load_upsample_2x(upsample_pipe, upsample_repo_id): + # 2.x ships the upsampler as a bare nn.Module in a subfolder; no from_pretrained on the + # pipeline wrapper, so we load the model + construct the pipeline manually. + if upsample_pipe is None: + t0 = time.time() + from diffusers.pipelines.ltx2.pipeline_ltx2_latent_upsample import LTX2LatentUpsamplePipeline + from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel + from modules import sd_checkpoint + log.info(f'Video load: cls={LTX2LatentUpsamplePipeline.__name__} repo="{upsample_repo_id}"') + latent_upsampler = LTX2LatentUpsamplerModel.from_pretrained( + upsample_repo_id, + subfolder='latent_upsampler', + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + ).to(devices.device) + upsample_pipe = LTX2LatentUpsamplePipeline( + vae=shared.sd_model.vae, + latent_upsampler=latent_upsampler, + ) + # Synthetic checkpoint_info gives this pipe its own OffloadHook cache slot, so routing + # it through apply_balanced_offload does not invalidate the main pipe's module map + # (sd_offload.py:488 keys on sd_checkpoint_info.name). + upsample_pipe.sd_checkpoint_info = sd_checkpoint.CheckpointInfo('ltx-upsampler-2.x') + t1 = time.time() + timer.process.add('load', t1 - t0) + return upsample_pipe + + def _condition_cls(family: str): if family == '2.x': try: diff --git a/modules/video_models/video_overrides.py b/modules/video_models/video_overrides.py index 34ee151d2..582acc92e 100644 --- a/modules/video_models/video_overrides.py +++ b/modules/video_models/video_overrides.py @@ -17,9 +17,11 @@ def load_override(selected: Model, **load_args): # LTX if 'LTXVideo 0.9.5 I2V' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLLTXVideo.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir, **load_args) - # OzzyGT LTX-2.3 mirrors ship connectors twice: sharded safetensors + .index.json plus a - # redundant unsharded diffusion_pytorch_model.safetensors of the same weights. Diffusers - # fetches both but loads sharded; skip the ~6.3 GB duplicate. + # OzzyGT LTX-2.3 mirrors pack connectors/ twice by design: sharded (*-00001-of-0000N + + # .index.json) and unsharded diffusion_pytorch_model.safetensors of the byte-identical + # weights. snapshot_download faithfully fetches both; diffusers' component loader picks + # sharded when the index is present. ignore_patterns skips the ~6.3 GB unsharded copy + # without reaching for a cleaner upstream mirror. ltx2_redundant_connector_repos = { 'OzzyGT/LTX-2.3', 'OzzyGT/LTX-2.3-sdnq-dynamic-int4', From 7be6250b7a4b703cfd2d5ee7a5e85e837b18a3ef Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Mon, 20 Apr 2026 00:28:30 +0100 Subject: [PATCH 11/15] refactor(ltx): extract temp_scheduler_opts context manager Collapses the scheduler + shared.opts snapshot/mutate/restore block in run_ltx into a single `with temp_scheduler_opts(...)` statement. Same five pieces of state are snapshotted and restored as before; the mechanism is unchanged per PR #4783 thread 2 feedback, only readability. --- modules/ltx/ltx_process.py | 45 +++++-------------------------------- modules/ltx/ltx_util.py | 46 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 40 deletions(-) diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 18aab0d44..330efcc9f 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -6,7 +6,7 @@ from PIL import Image from modules import shared, errors, timer, memstats, progress, processing, sd_models, sd_samplers, devices, extra_networks, call_queue from modules.logger import log from modules.ltx import ltx_capabilities -from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, load_upsample_2x, get_conditions, get_generator, get_prompts, vae_decode +from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, load_upsample_2x, get_conditions, get_generator, get_prompts, temp_scheduler_opts, vae_decode from modules.processing_callbacks import diffusers_callback from modules.video_models.video_vae import set_vae_params from modules.video_models.video_save import save_video @@ -290,30 +290,10 @@ def run_ltx(task_id, framewise = caps.family == '0.9' set_vae_params(p, framewise=framewise) - # Snapshot scheduler + shared.opts before mutation so the try/finally restores on every exit - # path (abort, interrupt, Stage 2 scheduler swap). Without this, run-specific sampler settings - # leak into shared.opts.data and across runs/tabs, and the default_scheduler snapshot from - # video_load.py:171 gets clobbered by a deepcopy of the mutated scheduler on every run. - orig_dynamic_shift = shared.opts.schedulers_dynamic_shift - orig_sampler_shift = shared.opts.schedulers_shift - orig_scheduler = shared.sd_model.scheduler - orig_default_scheduler = getattr(shared.sd_model, 'default_scheduler', None) - orig_use_dynamic_shifting = getattr(orig_scheduler.config, 'use_dynamic_shifting', None) if hasattr(orig_scheduler, 'config') else None - orig_flow_shift = getattr(orig_scheduler.config, 'flow_shift', None) if hasattr(orig_scheduler, 'config') else None - - try: - shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift - shared.opts.data['schedulers_shift'] = sampler_shift - if hasattr(shared.sd_model, 'scheduler') and hasattr(shared.sd_model.scheduler, 'config') and hasattr(shared.sd_model.scheduler, 'register_to_config'): - if hasattr(shared.sd_model.scheduler.config, 'use_dynamic_shifting'): - shared.sd_model.scheduler.config.use_dynamic_shifting = dynamic_shift - shared.sd_model.scheduler.register_to_config(use_dynamic_shifting=dynamic_shift) - if hasattr(shared.sd_model.scheduler.config, 'flow_shift') and sampler_shift is not None and sampler_shift >= 0: - shared.sd_model.scheduler.config.flow_shift = sampler_shift - shared.sd_model.scheduler.register_to_config(flow_shift=sampler_shift) - # Do NOT re-snapshot default_scheduler; that overwrites video_load.py:171's load-time - # snapshot with the run-mutated config, so reset_scheduler then carries the last run's choice. - + # Scheduler + shared.opts mutation is wrapped in temp_scheduler_opts so restore runs on + # every exit path (normal return, abort, interrupt, Stage 2 scheduler swap). See the + # helper's docstring for the five pieces of state it snapshots. + with temp_scheduler_opts(shared.sd_model, dynamic_shift=dynamic_shift, sampler_shift=sampler_shift): if selected is not None: video_overrides.set_overrides(p, selected) @@ -606,18 +586,3 @@ def run_ltx(task_id, log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={fps} its={its} resolution={resolution} time={t_end-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') yield video_file, f'LTX: Generation completed | File {video_file} | Frames {num_frames} | Resolution {resolution} | f/s {fps} | it/s {its} ' + f"

{summary} {memory}

" - finally: - shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift - shared.opts.data['schedulers_shift'] = orig_sampler_shift - if shared.sd_model.scheduler is not orig_scheduler: - shared.sd_model.scheduler = orig_scheduler - if orig_default_scheduler is not None and shared.sd_model.default_scheduler is not orig_default_scheduler: - shared.sd_model.default_scheduler = orig_default_scheduler - if hasattr(shared.sd_model.scheduler, 'config') and hasattr(shared.sd_model.scheduler, 'register_to_config'): - if orig_use_dynamic_shifting is not None and hasattr(shared.sd_model.scheduler.config, 'use_dynamic_shifting'): - shared.sd_model.scheduler.config.use_dynamic_shifting = orig_use_dynamic_shifting - shared.sd_model.scheduler.register_to_config(use_dynamic_shifting=orig_use_dynamic_shifting) - if orig_flow_shift is not None and hasattr(shared.sd_model.scheduler.config, 'flow_shift'): - shared.sd_model.scheduler.config.flow_shift = orig_flow_shift - shared.sd_model.scheduler.register_to_config(flow_shift=orig_flow_shift) - log.debug(f'LTX: scheduler/opts restored dynamic_shift={orig_dynamic_shift} sampler_shift={orig_sampler_shift}') diff --git a/modules/ltx/ltx_util.py b/modules/ltx/ltx_util.py index b404879f7..099ad94bd 100644 --- a/modules/ltx/ltx_util.py +++ b/modules/ltx/ltx_util.py @@ -1,4 +1,5 @@ import time +from contextlib import contextmanager import torch from PIL import Image from modules import devices, shared, sd_models, timer, extra_networks @@ -80,6 +81,51 @@ def load_upsample_2x(upsample_pipe, upsample_repo_id): return upsample_pipe +@contextmanager +def temp_scheduler_opts(sd_model, *, dynamic_shift=None, sampler_shift=None): + # Run-scoped override of shared.opts scheduler settings and scheduler.config. Snapshots + # five pieces of state (shared.opts dynamic_shift + shift, scheduler object, default_scheduler + # snapshot, and scheduler.config use_dynamic_shifting + flow_shift) and restores every one on + # exit. Keeps run-specific sampler settings out of config.json and prevents default_scheduler + # from getting clobbered by a deepcopy of the mutated scheduler at video_load.py:171. The + # scheduler-object restore matters for Stage 2 refine, which swaps the scheduler entirely. + orig_dynamic_shift = shared.opts.schedulers_dynamic_shift + orig_sampler_shift = shared.opts.schedulers_shift + orig_scheduler = sd_model.scheduler + orig_default_scheduler = getattr(sd_model, 'default_scheduler', None) + orig_use_dynamic_shifting = getattr(orig_scheduler.config, 'use_dynamic_shifting', None) if hasattr(orig_scheduler, 'config') else None + orig_flow_shift = getattr(orig_scheduler.config, 'flow_shift', None) if hasattr(orig_scheduler, 'config') else None + + try: + if dynamic_shift is not None: + shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift + if sampler_shift is not None: + shared.opts.data['schedulers_shift'] = sampler_shift + if hasattr(sd_model, 'scheduler') and hasattr(sd_model.scheduler, 'config') and hasattr(sd_model.scheduler, 'register_to_config'): + if dynamic_shift is not None and hasattr(sd_model.scheduler.config, 'use_dynamic_shifting'): + sd_model.scheduler.config.use_dynamic_shifting = dynamic_shift + sd_model.scheduler.register_to_config(use_dynamic_shifting=dynamic_shift) + if sampler_shift is not None and sampler_shift >= 0 and hasattr(sd_model.scheduler.config, 'flow_shift'): + sd_model.scheduler.config.flow_shift = sampler_shift + sd_model.scheduler.register_to_config(flow_shift=sampler_shift) + yield + finally: + shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift + shared.opts.data['schedulers_shift'] = orig_sampler_shift + if sd_model.scheduler is not orig_scheduler: + sd_model.scheduler = orig_scheduler + if orig_default_scheduler is not None and sd_model.default_scheduler is not orig_default_scheduler: + sd_model.default_scheduler = orig_default_scheduler + if hasattr(sd_model.scheduler, 'config') and hasattr(sd_model.scheduler, 'register_to_config'): + if orig_use_dynamic_shifting is not None and hasattr(sd_model.scheduler.config, 'use_dynamic_shifting'): + sd_model.scheduler.config.use_dynamic_shifting = orig_use_dynamic_shifting + sd_model.scheduler.register_to_config(use_dynamic_shifting=orig_use_dynamic_shifting) + if orig_flow_shift is not None and hasattr(sd_model.scheduler.config, 'flow_shift'): + sd_model.scheduler.config.flow_shift = orig_flow_shift + sd_model.scheduler.register_to_config(flow_shift=orig_flow_shift) + log.debug(f'LTX: scheduler/opts restored dynamic_shift={orig_dynamic_shift} sampler_shift={orig_sampler_shift}') + + def _condition_cls(family: str): if family == '2.x': try: From f79e466183e9b1ee58777a644852eec6bf767d18 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Tue, 21 Apr 2026 02:13:09 +0100 Subject: [PATCH 12/15] refactor(ltx): rename temp_scheduler_opts to ltx_scheduler_opts Per PR #4783 review; namespace the helper with the module. --- modules/ltx/ltx_process.py | 6 +++--- modules/ltx/ltx_util.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 330efcc9f..33de05531 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -6,7 +6,7 @@ from PIL import Image from modules import shared, errors, timer, memstats, progress, processing, sd_models, sd_samplers, devices, extra_networks, call_queue from modules.logger import log from modules.ltx import ltx_capabilities -from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, load_upsample_2x, get_conditions, get_generator, get_prompts, temp_scheduler_opts, vae_decode +from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, load_upsample_2x, get_conditions, get_generator, get_prompts, ltx_scheduler_opts, vae_decode from modules.processing_callbacks import diffusers_callback from modules.video_models.video_vae import set_vae_params from modules.video_models.video_save import save_video @@ -290,10 +290,10 @@ def run_ltx(task_id, framewise = caps.family == '0.9' set_vae_params(p, framewise=framewise) - # Scheduler + shared.opts mutation is wrapped in temp_scheduler_opts so restore runs on + # Scheduler + shared.opts mutation is wrapped in ltx_scheduler_opts so restore runs on # every exit path (normal return, abort, interrupt, Stage 2 scheduler swap). See the # helper's docstring for the five pieces of state it snapshots. - with temp_scheduler_opts(shared.sd_model, dynamic_shift=dynamic_shift, sampler_shift=sampler_shift): + with ltx_scheduler_opts(shared.sd_model, dynamic_shift=dynamic_shift, sampler_shift=sampler_shift): if selected is not None: video_overrides.set_overrides(p, selected) diff --git a/modules/ltx/ltx_util.py b/modules/ltx/ltx_util.py index 099ad94bd..f1ef78391 100644 --- a/modules/ltx/ltx_util.py +++ b/modules/ltx/ltx_util.py @@ -82,7 +82,7 @@ def load_upsample_2x(upsample_pipe, upsample_repo_id): @contextmanager -def temp_scheduler_opts(sd_model, *, dynamic_shift=None, sampler_shift=None): +def ltx_scheduler_opts(sd_model, *, dynamic_shift=None, sampler_shift=None): # Run-scoped override of shared.opts scheduler settings and scheduler.config. Snapshots # five pieces of state (shared.opts dynamic_shift + shift, scheduler object, default_scheduler # snapshot, and scheduler.config use_dynamic_shifting + flow_shift) and restores every one on From b58f0ae282ef2ba2ce12e87404cb0331e4212f34 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Tue, 21 Apr 2026 02:13:58 +0100 Subject: [PATCH 13/15] fix(ltx): exclude shared VAE from upsample pipe balanced_offload The upsample pipes receive shared.sd_model.vae as a constructor formality; the forward pass is pure latent to latent. Main pipe already owns that VAE's accelerate hook lifecycle, so walking it again from the upsample pipe raises "Cannot copy out of meta tensor" when the main pipe has offloaded params to meta. Skip it in the walk. --- modules/ltx/ltx_process.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 33de05531..3fde6df45 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -358,10 +358,16 @@ def run_ltx(task_id, t4 = time.time() upsamplejob = shared.state.begin('Upsample') try: + # Shared-VAE exclude: both upsample pipes receive shared.sd_model.vae as a + # constructor formality (pure latent -> latent forward). The main pipe already + # owns the VAE's hook lifecycle, so walking it again here hits meta tensors + # from the prior offload pass. Excluding also shortens the walk to the one + # module that actually belongs to this pipe: latent_upsampler. + upsample_exclude = ['vae'] if caps.family == '0.9': global upsample_pipe # pylint: disable=global-statement upsample_pipe = load_upsample(upsample_pipe, upsample_repo_id_09) - upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) + upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe, exclude=upsample_exclude) up_args = { 'width': final_w, 'height': final_h, @@ -373,12 +379,12 @@ def run_ltx(task_id, log.debug(f'Video: op=upsample family=0.9 latents={latents.shape} {up_args}') yield None, 'LTX: Upsample in progress...' latents = upsample_pipe(latents=latents, **up_args).frames[0] - upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) + upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe, exclude=upsample_exclude) else: global upsample_pipe_2x # pylint: disable=global-statement upsample_repo = upsample_repo_id_23 if caps.variant == '2.3' else upsample_repo_id_20 upsample_pipe_2x = load_upsample_2x(upsample_pipe_2x, upsample_repo) - upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x) + upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x, exclude=upsample_exclude) # 2.x base pass returns denormalized latents; latents_normalized=False tells the # upsampler "already raw, do not denormalize again". up_args = { @@ -394,7 +400,7 @@ def run_ltx(task_id, log.debug(f'Video: op=upsample family=2.x latents={latents.shape} auto={auto_refine_upsample} {up_args}') yield None, 'LTX: Upsample in progress...' latents = upsample_pipe_2x(latents=latents, **up_args).frames[0] - upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x) + upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x, exclude=upsample_exclude) except AssertionError as e: yield from abort(e, ok=True, p=p) return From f72f89c99364a3ebf1cd21e6ef2ed374a5ca9413 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Tue, 21 Apr 2026 02:14:52 +0100 Subject: [PATCH 14/15] chore(ltx): silent=True on run-internal offload walks Modules already inventoried at load time; repeating the six-line dump at each upsample or refine boundary is redundant. silent=True suppresses the per-module DEBUG lines; op=init and Model class= INFO stay intact. --- modules/ltx/ltx_process.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 3fde6df45..54e5d562e 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -298,7 +298,7 @@ def run_ltx(task_id, video_overrides.set_overrides(p, selected) t0 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, silent=True) t1 = time.time() samplejob = shared.state.begin('Sample') @@ -346,7 +346,11 @@ def run_ltx(task_id, return t2 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + # silent=True everywhere in run_ltx: per-module stats were already dumped during the + # load-time balanced_offload pass. Upsample/refine boundaries force a rebuild because + # the global offload_hook_instance is keyed on checkpoint_name (sd_offload.py:488), + # but re-logging the same inventory adds noise without information. + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, silent=True) devices.torch_gc(force=True, reason='ltx:base') t3 = time.time() timer.process.add('offload', t1 - t0) @@ -367,7 +371,7 @@ def run_ltx(task_id, if caps.family == '0.9': global upsample_pipe # pylint: disable=global-statement upsample_pipe = load_upsample(upsample_pipe, upsample_repo_id_09) - upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe, exclude=upsample_exclude) + upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe, exclude=upsample_exclude, silent=True) up_args = { 'width': final_w, 'height': final_h, @@ -379,12 +383,12 @@ def run_ltx(task_id, log.debug(f'Video: op=upsample family=0.9 latents={latents.shape} {up_args}') yield None, 'LTX: Upsample in progress...' latents = upsample_pipe(latents=latents, **up_args).frames[0] - upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe, exclude=upsample_exclude) + upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe, exclude=upsample_exclude, silent=True) else: global upsample_pipe_2x # pylint: disable=global-statement upsample_repo = upsample_repo_id_23 if caps.variant == '2.3' else upsample_repo_id_20 upsample_pipe_2x = load_upsample_2x(upsample_pipe_2x, upsample_repo) - upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x, exclude=upsample_exclude) + upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x, exclude=upsample_exclude, silent=True) # 2.x base pass returns denormalized latents; latents_normalized=False tells the # upsampler "already raw, do not denormalize again". up_args = { @@ -400,7 +404,7 @@ def run_ltx(task_id, log.debug(f'Video: op=upsample family=2.x latents={latents.shape} auto={auto_refine_upsample} {up_args}') yield None, 'LTX: Upsample in progress...' latents = upsample_pipe_2x(latents=latents, **up_args).frames[0] - upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x, exclude=upsample_exclude) + upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x, exclude=upsample_exclude, silent=True) except AssertionError as e: yield from abort(e, ok=True, p=p) return @@ -414,7 +418,7 @@ def run_ltx(task_id, if refine_enable and latents is not None: t7 = time.time() refinejob = shared.state.begin('Refine') - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, silent=True) devices.torch_gc(force=True, reason='ltx:refine') # Refine is terminal: let the pipe decode internally so the final VAE pass runs inside # the same offload/cudnn context as a normal generation (matches Generic Video tab). @@ -512,7 +516,7 @@ def run_ltx(task_id, shared.sd_model.scheduler = saved_scheduler_stage2 log.debug('LTX: canonical Stage 2 cleanup done (LoRA unloaded, scheduler restored)') t8 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, silent=True) t9 = time.time() timer.process.add('refine', t8 - t7) timer.process.add('offload', t9 - t8) @@ -523,7 +527,7 @@ def run_ltx(task_id, if needs_latent_path and latents is not None: # Only reached on upsample-without-refine; refine decodes through the pipe and nulls latents. - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'], force=True) + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'], force=True, silent=True) devices.torch_gc(force=True, reason='ltx:vae') yield None, 'LTX: VAE decode in progress...' try: @@ -540,7 +544,7 @@ def run_ltx(task_id, return pixels = frames_out t10 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, silent=True) t11 = time.time() timer.process.add('offload', t11 - t10) From 67c77f7c4f1efcc3fc0b8bdada51d97473792cf1 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Tue, 21 Apr 2026 02:46:40 +0100 Subject: [PATCH 15/15] feat(ltx): default Refine on for Dev 2.x T2V/I2V Lightricks' production recipe for Dev is Stage 1 + 2x upsample + Stage 2 refine. Until now the UI reset Refine to False on every model change, so users picking Dev got a single-pass generation that does not match the recommended flow. Default Refine to on for variants that support the canonical Stage 2 recipe; multi-condition variants stay off. --- modules/ltx/ltx_ui.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index de72efda1..94bd857ef 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -31,13 +31,17 @@ def _model_change(model_name: str): ) # 2.x refine runs fixed canonical schedules; refine_strength only feeds 0.9.x LTXConditionPipeline. refine_strength_interactive = caps.family == '0.9' + # Default Refine on for Dev 2.x T2V/I2V: Lightricks' production recipe is Stage 1 + 2x upsample + # + Stage 2 refine (auto_refine_upsample at ltx_process.py:179 couples the stages once Refine is on). + # Multi-condition variants are excluded for the same reason auto_refine_upsample excludes them. + refine_default = caps.supports_canonical_stage2 and not caps.supports_multi_condition return ( gr.update(visible=caps.supports_input_media), gr.update(visible=caps.supports_multi_condition), gr.update(visible=True), gr.update(visible=True), gr.update(value=False), - gr.update(value=False), + gr.update(value=refine_default), gr.update(interactive=refine_strength_interactive), gr.update(value=caps.default_cfg), gr.update(value=caps.default_steps),