diff --git a/modules/framepack/framepack_ui.py b/modules/framepack/framepack_ui.py index 394387df7..93b57863a 100644 --- a/modules/framepack/framepack_ui.py +++ b/modules/framepack/framepack_ui.py @@ -12,7 +12,7 @@ def change_sections(duration, mp4_fps, mp4_interpolate, latent_ws, variant): return gr.update(value=f'Target video: {num_frames} frames in {num_sections} sections'), gr.update(lines=max(2, 2*num_sections//3)) -def create_ui(prompt, negative, styles, _overrides, init_image, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf): +def create_ui(prompt, negative, styles, _overrides, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf): with gr.Row(): with gr.Column(variant='compact', elem_id="framepack_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): @@ -22,11 +22,12 @@ def create_ui(prompt, negative, styles, _overrides, init_image, last_image, mp4_ with gr.Row(): resolution = gr.Slider(label="FP resolution", minimum=240, maximum=1088, value=640, step=16) duration = gr.Slider(label="FP duration", minimum=1, maximum=120, value=4, step=0.1) - mp4_fps = gr.Slider(label="FP target FPS", minimum=1, maximum=60, value=24, step=1) - mp4_interpolate = gr.Slider(label="FP interpolation", minimum=0, maximum=10, value=0, step=1) with gr.Row(): section_html = gr.HTML(show_label=False, elem_id="framepack_section_html") with gr.Accordion(label="Inputs", open=False): + with gr.Row(): + init_image = gr.Image(label='FP init image', elem_id='framepack_init_image', type='pil', image_mode='RGB', width=256, height=256) + last_image = gr.Image(label='FP last image', elem_id='framepack_last_image', type='pil', image_mode='RGB', width=256, height=256) with gr.Row(): start_weight = gr.Slider(label="FP init strength", value=1.0, minimum=0.0, maximum=2.0, step=0.05, elem_id="framepack_start_weight") end_weight = gr.Slider(label="FP end strength", value=1.0, minimum=0.0, maximum=2.0, step=0.05, elem_id="framepack_end_weight") diff --git a/modules/ltx/ltx_capabilities.py b/modules/ltx/ltx_capabilities.py new file mode 100644 index 000000000..499f8b57b --- /dev/null +++ b/modules/ltx/ltx_capabilities.py @@ -0,0 +1,138 @@ +from dataclasses import dataclass, field +from typing import Optional + +from modules.logger import log + + +@dataclass +class LTXCaps: + name: str + repo_cls_name: str + family: str # '0.9' or '2.x' + variant: str # '0.9', '2.0', '2.3' (finer-grained sub-variant) + is_distilled: bool + is_i2v: bool + supports_input_media: bool + supports_multi_condition: bool + supports_image_cond_noise_scale: bool + supports_decode_timestep: bool + supports_stg: bool + supports_audio: bool + supports_frame_rate_kwarg: bool + # 2.3 transformer cross-attn reads the other modality's sigma; unset falls back to 2.0's + # independent-sigma path, which is a joint-distribution mismatch for 2.3 weights. + use_cross_timestep: bool + default_cfg: float + default_steps: int + default_sampler_shift: float + default_dynamic_shift: bool + default_width: int + default_height: int + default_frames: int + default_frame_rate: int + stg_default_scale: float = 0.0 + stg_default_blocks: list = field(default_factory=list) + # Dev 2.x trained under cfg + stg + modality + rescale four-way composition; + # distilled bakes these into its sigma schedule and stays at pipeline identity. + modality_default_scale: float = 1.0 + guidance_rescale_default: float = 0.0 + supports_canonical_stage2: bool = False + stage2_dev_lora_repo: Optional[str] = None + + +CONDITION_CLASSES = {'LTXConditionPipeline', 'LTX2ConditionPipeline'} +LTX2_CLASSES = {'LTX2Pipeline', 'LTX2ImageToVideoPipeline', 'LTX2ConditionPipeline'} +ALL_LTX_CLASSES = { + 'LTXPipeline', + 'LTXImageToVideoPipeline', + 'LTXConditionPipeline', + 'LTX2Pipeline', + 'LTX2ImageToVideoPipeline', + 'LTX2ConditionPipeline', +} + + +def _repo_cls_name(model_name: str) -> Optional[str]: + from modules.video_models.models_def import models + entries = models.get('LTX Video', []) + for m in entries: + if m.name == model_name: + if m.repo_cls is None: + return None + return m.repo_cls.__name__ + return None + + +def get_caps(model_name: str) -> Optional[LTXCaps]: + if not model_name or model_name == 'None': + return None + cls_name = _repo_cls_name(model_name) + if cls_name is None: + log.warning(f'LTX caps: model="{model_name}" has no repo_cls registered') + return None + if cls_name not in ALL_LTX_CLASSES: + log.warning(f'LTX caps: model="{model_name}" repo_cls="{cls_name}" is not an LTX pipeline') + return None + + is_ltx2 = cls_name in LTX2_CLASSES + family = '2.x' if is_ltx2 else '0.9' + # 2.x sub-variant detection: unknown 2.x mirrors fall through to '2.0' (conservative default). + if is_ltx2: + variant = '2.3' if '2.3' in model_name else '2.0' + else: + variant = '0.9' + is_distilled = 'Distilled' in model_name + is_i2v = 'I2V' in model_name or cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') + is_condition_cls = cls_name in CONDITION_CLASSES + supports_input_media = is_i2v or is_condition_cls + + caps = LTXCaps( + name=model_name, + repo_cls_name=cls_name, + family=family, + variant=variant, + is_distilled=is_distilled, + is_i2v=is_i2v, + supports_input_media=supports_input_media, + supports_multi_condition=is_condition_cls, + supports_image_cond_noise_scale=(cls_name == 'LTXConditionPipeline'), + supports_decode_timestep=(family == '0.9'), + supports_stg=is_ltx2, + supports_audio=is_ltx2, + supports_frame_rate_kwarg=is_ltx2, + use_cross_timestep=(variant == '2.3'), + default_cfg=3.0, + default_steps=30 if is_ltx2 else 50, + default_sampler_shift=-1.0, + default_dynamic_shift=is_ltx2, + default_width=768, + default_height=512, + default_frames=121 if is_ltx2 else 161, + default_frame_rate=24 if is_ltx2 else 25, + ) + + if is_distilled: + caps.default_cfg = 1.0 + caps.default_steps = 8 + + if is_ltx2 and not is_distilled: + if variant == '2.3': + caps.stage2_dev_lora_repo = 'CalamitousFelicitousness/LTX-2.3-distilled-lora-384-Diffusers' + elif variant == '2.0': + caps.stage2_dev_lora_repo = 'CalamitousFelicitousness/LTX-2.0-distilled-lora-384-Diffusers' + caps.supports_canonical_stage2 = caps.stage2_dev_lora_repo is not None + + if is_ltx2: + if variant == '2.3': + caps.stg_default_blocks = [28] + elif variant == '2.0': + caps.stg_default_blocks = [29] + else: + caps.stg_default_blocks = [28] + if not is_distilled: + # canonical T2V composition from huggingface/diffusers#13217 + caps.stg_default_scale = 1.0 + caps.modality_default_scale = 3.0 + caps.guidance_rescale_default = 0.7 + + return caps diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 1b56500d8..54e5d562e 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -3,59 +3,146 @@ import time import torch from PIL import Image -from modules import shared, errors, timer, memstats, progress, processing, sd_models, sd_samplers, extra_networks, call_queue +from modules import shared, errors, timer, memstats, progress, processing, sd_models, sd_samplers, devices, extra_networks, call_queue from modules.logger import log +from modules.ltx import ltx_capabilities +from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, load_upsample_2x, get_conditions, get_generator, get_prompts, ltx_scheduler_opts, vae_decode +from modules.processing_callbacks import diffusers_callback from modules.video_models.video_vae import set_vae_params from modules.video_models.video_save import save_video from modules.video_models.video_utils import check_av -from modules.processing_callbacks import diffusers_callback -from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, get_conditions, get_generator, get_prompts, vae_decode debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None -# engine, model = 'LTX Video', 'LTXVideo 0.9.7 13B' -upsample_repo_id = "a-r-r-o-w/LTX-Video-0.9.7-Latent-Spatial-Upsampler-diffusers" +upsample_repo_id_09 = 'a-r-r-o-w/LTX-Video-0.9.7-Latent-Spatial-Upsampler-diffusers' +# Upsampler weights are tied to the family VAE; using the wrong one preserves structure +# but drifts per-channel latent statistics (decodes desaturated / crushed contrast). +upsample_repo_id_20 = 'Lightricks/LTX-2' +upsample_repo_id_23 = 'CalamitousFelicitousness/LTX-2.3-Spatial-Upsampler-x2-1.1-Diffusers' upsample_pipe = None +upsample_pipe_2x = None + +STAGE2_DEV_LORA_ADAPTER = 'ltx2_stage2_distilled' + + +def _canonical_ltx2_guidance(caps) -> dict: + # Four-way composition (cfg + stg + modality + rescale) from huggingface/diffusers#13217. + # Distilled bakes these into its sigma schedule; skip or we double-apply. + if caps.family != '2.x' or caps.is_distilled: + return {} + return { + 'stg_scale': caps.stg_default_scale, + 'modality_scale': caps.modality_default_scale, + 'guidance_rescale': caps.guidance_rescale_default, + 'spatio_temporal_guidance_blocks': list(caps.stg_default_blocks), + 'audio_guidance_scale': 7.0, + 'audio_stg_scale': 1.0, + 'audio_modality_scale': 3.0, + 'audio_guidance_rescale': 0.7, + } + + +def _canonical_stage2_dev_kwargs() -> dict: + # Stage 2 identity guidance from huggingface/diffusers#13217. The distilled LoRA makes Dev + # behave like Distilled, which was trained at identity; Stage 1's four-way composition on + # top double-dips and produces striping/flicker. + from diffusers.pipelines.ltx2.utils import STAGE_2_DISTILLED_SIGMA_VALUES + return { + 'sigmas': list(STAGE_2_DISTILLED_SIGMA_VALUES), + 'noise_scale': float(STAGE_2_DISTILLED_SIGMA_VALUES[0]), + 'guidance_scale': 1.0, + 'stg_scale': 0.0, + 'modality_scale': 1.0, + 'guidance_rescale': 0.0, + 'audio_guidance_scale': 1.0, + 'audio_stg_scale': 0.0, + 'audio_modality_scale': 1.0, + 'audio_guidance_rescale': 0.0, + 'spatio_temporal_guidance_blocks': None, + } + + +def _latent_pass(caps, prompt, negative, width, height, frames, steps, guidance_scale, mp4_fps, conditions, image_cond_noise_scale, seed, image=None): + base_args = { + 'prompt': prompt, + 'negative_prompt': negative, + 'width': get_bucket(width), + 'height': get_bucket(height), + 'num_frames': get_frames(frames), + 'num_inference_steps': steps, + 'generator': get_generator(seed), + 'callback_on_step_end': diffusers_callback, + 'output_type': 'latent', + } + if guidance_scale is not None and guidance_scale > 0: + base_args['guidance_scale'] = guidance_scale + if caps.supports_frame_rate_kwarg: + base_args['frame_rate'] = float(mp4_fps) + if caps.supports_image_cond_noise_scale and image_cond_noise_scale is not None: + base_args['image_cond_noise_scale'] = image_cond_noise_scale + if caps.supports_multi_condition and conditions: + base_args['conditions'] = conditions + if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and image is not None: + base_args['image'] = image + if caps.family == '2.x' and caps.is_distilled: + from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES + base_args['sigmas'] = list(DISTILLED_SIGMA_VALUES) + base_args.pop('num_inference_steps', None) + base_args.update(_canonical_ltx2_guidance(caps)) + if caps.use_cross_timestep: + base_args['use_cross_timestep'] = True + log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=latent_pass args_keys={list(base_args.keys())}') + result = shared.sd_model(**base_args) + # video latents strip the batch dim; audio latents keep it so LTX2Pipeline.prepare_audio_latents + # can rewrap them when re-entered as ndim==4 at Stage 2. + latents = result.frames[0] if hasattr(result, 'frames') else None + audio_latents = None + if hasattr(result, 'audio') and result.audio is not None: + audio_latents = result.audio + return latents, audio_latents def run_ltx(task_id, _ui_state, - model:str, - prompt:str, - negative:str, - styles:list[str], - width:int, - height:int, - frames:int, - steps:int, - sampler_index:int, - seed:int, - upsample_enable:bool, - upsample_ratio:float, - refine_enable:bool, - refine_strength:float, + model: str, + prompt: str, + negative: str, + styles: list, + width: int, + height: int, + frames: int, + steps: int, + sampler_index: int, + guidance_scale: float, + sampler_shift: float, + dynamic_shift: bool, + seed: int, + upsample_enable: bool, + upsample_ratio: float, + refine_enable: bool, + refine_strength: float, condition_strength: float, - condition_image, + ltx_init_image, condition_last, condition_files, condition_video, - condition_video_frames:int, - condition_video_skip:int, - decode_timestep:float, - image_cond_noise_scale:float, - mp4_fps:int, - mp4_interpolate:int, - mp4_codec:str, - mp4_ext:str, - mp4_opt:str, - mp4_video:bool, - mp4_frames:bool, - mp4_sf:bool, - audio_enable:bool, + condition_video_frames: int, + condition_video_skip: int, + decode_timestep: float, + image_cond_noise_scale: float, + mp4_fps: int, + mp4_interpolate: int, + mp4_codec: str, + mp4_ext: str, + mp4_opt: str, + mp4_video: bool, + mp4_frames: bool, + mp4_sf: bool, + audio_enable: bool, _overrides, ): - def abort(e, ok:bool=False, p=None): + def abort(e, ok: bool = False, p=None): if ok: log.info(e) else: @@ -67,255 +154,445 @@ def run_ltx(task_id, progress.finish_task(task_id) yield None, f'LTX Error: {str(e)}' - if model is None or len(model) == 0: + if model is None or len(model) == 0 or model == 'None': yield from abort('Video: no model selected', ok=True) return - # from diffusers import LTXConditionPipeline # pylint: disable=unused-import check_av() progress.add_task_to_queue(task_id) + with call_queue.get_lock(): progress.start_task(task_id) memstats.reset_stats() timer.process.reset() yield None, 'LTX: Loading...' + engine = 'LTX Video' load_model(engine, model) - debug(f'Video: cls={shared.sd_model.__class__.__name__} op=init model="{model}"') - if not shared.sd_model.__class__.__name__.startswith("LTX"): - yield from abort(f'Video: cls={shared.sd_model.__class__.__name__} selected model is not LTX model', ok=True) + caps = ltx_capabilities.get_caps(model) + if caps is None or not shared.sd_model.__class__.__name__.startswith('LTX'): + yield from abort(f'Video: cls={shared.sd_model.__class__.__name__} selected model is not LTX', ok=True) return + # Lightricks TI2VidTwoStagesPipeline: Stage 1 at half-res, 2x upsample, Stage 2 refine at target. + # Auto-couple when the user picks Refine but not Upsample. Condition variants still need per-stage + # conditioning rebuild, so keep them on the same-resolution path. + auto_refine_upsample = ( + refine_enable + and caps.supports_canonical_stage2 + and not upsample_enable + and not caps.supports_multi_condition + ) + effective_upsample_enable = upsample_enable or auto_refine_upsample + effective_upsample_ratio = upsample_ratio if upsample_enable else 2.0 + target_w = get_bucket(width) + target_h = get_bucket(height) + if auto_refine_upsample: + # Stage 1 at target/2 needs multiple-of-32; 2x upsample then forces final divisible by 64. + # Derive final from base, otherwise Stage 2 silently falls to base*2 != target. + base_w = get_bucket(target_w // 2) + base_h = get_bucket(target_h // 2) + final_w = base_w * 2 + final_h = base_h * 2 + if (final_w, final_h) != (target_w, target_h): + log.warning(f'LTX: two-stage refine needs resolution divisible by 64; adjusting {target_w}x{target_h} -> {final_w}x{final_h}') + elif effective_upsample_enable: + base_w = target_w + base_h = target_h + final_w = get_bucket(effective_upsample_ratio * target_w) + final_h = get_bucket(effective_upsample_ratio * target_h) + else: + base_w = target_w + base_h = target_h + final_w = target_w + final_h = target_h + log.debug(f'LTX: resolution planning target={target_w}x{target_h} base={base_w}x{base_h} final={final_w}x{final_h} auto_refine_upsample={auto_refine_upsample}') + videojob = shared.state.begin('Video', task_id=task_id) shared.state.job_count = 1 + from modules.video_models import models_def, video_overrides + selected = next((m for m in models_def.models.get(engine, []) if m.name == model), None) + + if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and ltx_init_image is None: + yield from abort('No input image provided. Please upload or select an image.', ok=True) + return + + condition_images = [] + if ltx_init_image is not None: + condition_images.append(ltx_init_image) + if condition_last is not None: + condition_images.append(condition_last) + conditions = [] + if caps.supports_multi_condition: + conditions = get_conditions( + width, height, condition_strength, + condition_images, condition_files, condition_video, + condition_video_frames, condition_video_skip, + family=caps.family, + ) + + sampler_name = processing.get_sampler_name(sampler_index) + sd_samplers.create_sampler(sampler_name, shared.sd_model) + log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=init caps={caps.family} styles={styles} sampler={shared.sd_model.scheduler.__class__.__name__}') + + from modules.paths import resolve_output_path p = processing.StableDiffusionProcessingVideo( + sd_model=shared.sd_model, video_engine=engine, video_model=model, prompt=prompt, negative_prompt=negative, styles=styles, - width=width, - height=height, - frames=frames, - steps=steps, - sampler_index=sampler_index, - seed=seed, + seed=int(seed) if seed is not None else -1, + sampler_name=sampler_name, + sampler_shift=float(sampler_shift), + steps=int(steps), + width=base_w, + height=base_h, + frames=get_frames(frames), + cfg_scale=float(guidance_scale) if guidance_scale is not None and guidance_scale > 0 else caps.default_cfg, + denoising_strength=float(condition_strength) if condition_strength is not None else 1.0, + init_image=ltx_init_image, + vae_type='Default', + vae_tile_frames=16, ) + p.scripts = None + p.script_args = None + p.do_not_save_grid = True + p.do_not_save_samples = not mp4_frames + p.outpath_samples = resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_video) p.ops.append('video') - condition_images = [] - if condition_image is not None: - condition_images.append(condition_image) - if condition_last is not None: - condition_images.append(condition_last) - conditions = get_conditions( - width, - height, - condition_strength, - condition_images, - condition_files, - condition_video, - condition_video_frames, - condition_video_skip, - ) + p.task_args['num_inference_steps'] = p.steps + p.task_args['width'] = p.width + p.task_args['height'] = p.height + # force pil: 'latent' output triggers frame collapse in process_samples + p.task_args['output_type'] = 'pil' + if caps.supports_frame_rate_kwarg: + p.task_args['frame_rate'] = float(mp4_fps) + if caps.supports_image_cond_noise_scale and image_cond_noise_scale is not None: + p.task_args['image_cond_noise_scale'] = image_cond_noise_scale + if caps.supports_decode_timestep and decode_timestep is not None: + p.task_args['decode_timestep'] = decode_timestep + if caps.supports_multi_condition and conditions: + p.task_args['conditions'] = conditions - prompt, negative, networks = get_prompts(prompt, negative, styles) - sampler_name = processing.get_sampler_name(sampler_index) - sd_samplers.create_sampler(sampler_name, shared.sd_model) - log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=init styles={styles} networks={networks} sampler={shared.sd_model.scheduler.__class__.__name__}') + if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and ltx_init_image is not None: + from modules import images + p.task_args['image'] = images.resize_image(resize_mode=2, im=ltx_init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') - extra_networks.activate(p, networks) - framewise = 'LTX2' not in shared.sd_model.__class__.__name__ + if caps.family == '2.x' and caps.is_distilled: + from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES + p.task_args['sigmas'] = list(DISTILLED_SIGMA_VALUES) + p.task_args.pop('num_inference_steps', None) + p.task_args.update(_canonical_ltx2_guidance(caps)) + + framewise = caps.family == '0.9' set_vae_params(p, framewise=framewise) - t0 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - t1 = time.time() - if 'LTX2' in shared.sd_model.__class__.__name__: - output_type = 'np' - else: - output_type = 'latent' - base_args = { - "prompt": prompt, - "negative_prompt": negative, - "width": get_bucket(width), - "height": get_bucket(height), - "num_frames": get_frames(frames), - "num_inference_steps": steps, - "generator": get_generator(seed), - "callback_on_step_end": diffusers_callback, - "output_type": output_type, - } - if 'LTX2' in shared.sd_model.__class__.__name__: - base_args["frame_rate"] = float(mp4_fps) - if 'Condition' in shared.sd_model.__class__.__name__: - base_args["image_cond_noise_scale"] = image_cond_noise_scale - if len(conditions) > 0: - base_args["conditions"] = conditions - log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=base {base_args}') + # Scheduler + shared.opts mutation is wrapped in ltx_scheduler_opts so restore runs on + # every exit path (normal return, abort, interrupt, Stage 2 scheduler swap). See the + # helper's docstring for the five pieces of state it snapshots. + with ltx_scheduler_opts(shared.sd_model, dynamic_shift=dynamic_shift, sampler_shift=sampler_shift): + if selected is not None: + video_overrides.set_overrides(p, selected) + + t0 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, silent=True) + t1 = time.time() + + samplejob = shared.state.begin('Sample') + yield None, 'LTX: Generate in progress...' - if debug: - log.trace(f'LTX args: {base_args}') - yield None, 'LTX: Generate in progress...' - samplejob = shared.state.begin('Sample') - try: - result = shared.sd_model(**base_args) - latents = result.frames[0] - except AssertionError as e: - yield from abort(e, ok=True, p=p) - return - except Exception as e: - yield from abort(e, ok=False, p=p) - return - if audio_enable and hasattr(result, 'audio') and result.audio is not None: - audio = result.audio[0].float().cpu() - else: audio = None - try: - if debug: - log.trace(f'LTX result frames={latents.shape if latents is not None else None} audio={audio.shape if audio is not None else None}') - except Exception: - pass + stage1_audio_latents = None + pixels = None + frames_out = None + needs_latent_path = upsample_enable or refine_enable - t2 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - t3 = time.time() - timer.process.add('offload', t1 - t0) - timer.process.add('base', t2 - t1) - timer.process.add('offload', t3 - t2) - shared.state.end(samplejob) - - if upsample_enable: - t4 = time.time() - upsamplejob = shared.state.begin('Upsample') - global upsample_pipe # pylint: disable=global-statement - upsample_pipe = load_upsample(upsample_pipe, upsample_repo_id) - upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) - upscale_args = { - "width": get_bucket(upsample_ratio * width), - "height": get_bucket(upsample_ratio * height), - "generator": get_generator(seed), - "output_type": output_type, - } - if latents.ndim == 4: - latents = latents.unsqueeze(0) # add batch dimension - log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=upsample latents={latents.shape} {upscale_args}') - yield None, 'LTX: Upsample in progress...' try: - upsampled_latents = upsample_pipe(latents=latents, **upscale_args).frames[0] + if needs_latent_path: + prompt_final, negative_final, networks = get_prompts(prompt, negative, styles) + extra_networks.activate(p, networks) + latents, stage1_audio_latents = _latent_pass( + caps=caps, + prompt=prompt_final, + negative=negative_final, + width=base_w, + height=base_h, + frames=frames, + steps=steps, + guidance_scale=p.cfg_scale, + mp4_fps=mp4_fps, + conditions=conditions, + image_cond_noise_scale=image_cond_noise_scale if caps.supports_image_cond_noise_scale else None, + seed=int(seed) if seed is not None else -1, + image=p.task_args.get('image'), + ) + else: + processed = processing.process_images(p) + if processed is None or processed.images is None or len(processed.images) == 0: + yield from abort('Video: process_images returned no frames', ok=True, p=p) + return + pixels = processed.images + if getattr(processed, 'audio', None) is not None: + audio = processed.audio + latents = None except AssertionError as e: yield from abort(e, ok=True, p=p) return except Exception as e: yield from abort(e, ok=False, p=p) return - latents = upsampled_latents - t5 = time.time() - upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) - t6 = time.time() - timer.process.add('upsample', t5 - t4) - timer.process.add('offload', t6 - t5) - shared.state.end(upsamplejob) - if refine_enable: - t7 = time.time() - refinejob = shared.state.begin('Refine') - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - refine_args = { - "prompt": prompt, - "negative_prompt": negative, - "width": get_bucket(upsample_ratio * width), - "height": get_bucket(upsample_ratio * height), - "num_frames": get_frames(frames), - "denoise_strength": refine_strength, - "num_inference_steps": steps, - "image_cond_noise_scale": image_cond_noise_scale, - "generator": get_generator(seed), - "callback_on_step_end": diffusers_callback, - "output_type": output_type, - } - if latents.ndim == 4: - latents = latents.unsqueeze(0) # add batch dimension + t2 = time.time() + # silent=True everywhere in run_ltx: per-module stats were already dumped during the + # load-time balanced_offload pass. Upsample/refine boundaries force a rebuild because + # the global offload_hook_instance is keyed on checkpoint_name (sd_offload.py:488), + # but re-logging the same inventory adds noise without information. + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, silent=True) + devices.torch_gc(force=True, reason='ltx:base') + t3 = time.time() + timer.process.add('offload', t1 - t0) + timer.process.add('base', t2 - t1) + timer.process.add('offload', t3 - t2) + shared.state.end(samplejob) + + if effective_upsample_enable and latents is not None: + t4 = time.time() + upsamplejob = shared.state.begin('Upsample') + try: + # Shared-VAE exclude: both upsample pipes receive shared.sd_model.vae as a + # constructor formality (pure latent -> latent forward). The main pipe already + # owns the VAE's hook lifecycle, so walking it again here hits meta tensors + # from the prior offload pass. Excluding also shortens the walk to the one + # module that actually belongs to this pipe: latent_upsampler. + upsample_exclude = ['vae'] + if caps.family == '0.9': + global upsample_pipe # pylint: disable=global-statement + upsample_pipe = load_upsample(upsample_pipe, upsample_repo_id_09) + upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe, exclude=upsample_exclude, silent=True) + up_args = { + 'width': final_w, + 'height': final_h, + 'generator': get_generator(int(seed) if seed is not None else -1), + 'output_type': 'latent', + } + if latents.ndim == 4: + latents = latents.unsqueeze(0) + log.debug(f'Video: op=upsample family=0.9 latents={latents.shape} {up_args}') + yield None, 'LTX: Upsample in progress...' + latents = upsample_pipe(latents=latents, **up_args).frames[0] + upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe, exclude=upsample_exclude, silent=True) + else: + global upsample_pipe_2x # pylint: disable=global-statement + upsample_repo = upsample_repo_id_23 if caps.variant == '2.3' else upsample_repo_id_20 + upsample_pipe_2x = load_upsample_2x(upsample_pipe_2x, upsample_repo) + upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x, exclude=upsample_exclude, silent=True) + # 2.x base pass returns denormalized latents; latents_normalized=False tells the + # upsampler "already raw, do not denormalize again". + up_args = { + 'width': final_w, + 'height': final_h, + 'num_frames': get_frames(frames), + 'latents_normalized': False, + 'generator': get_generator(int(seed) if seed is not None else -1), + 'output_type': 'latent', + } + if latents.ndim == 4: + latents = latents.unsqueeze(0) + log.debug(f'Video: op=upsample family=2.x latents={latents.shape} auto={auto_refine_upsample} {up_args}') + yield None, 'LTX: Upsample in progress...' + latents = upsample_pipe_2x(latents=latents, **up_args).frames[0] + upsample_pipe_2x = sd_models.apply_balanced_offload(upsample_pipe_2x, exclude=upsample_exclude, silent=True) + except AssertionError as e: + yield from abort(e, ok=True, p=p) + return + except Exception as e: + yield from abort(e, ok=False, p=p) + return + t5 = time.time() + timer.process.add('upsample', t5 - t4) + shared.state.end(upsamplejob) + + if refine_enable and latents is not None: + t7 = time.time() + refinejob = shared.state.begin('Refine') + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, silent=True) + devices.torch_gc(force=True, reason='ltx:refine') + # Refine is terminal: let the pipe decode internally so the final VAE pass runs inside + # the same offload/cudnn context as a normal generation (matches Generic Video tab). + refine_args = { + 'prompt': prompt_final, + 'negative_prompt': negative_final, + 'width': final_w, + 'height': final_h, + 'num_frames': get_frames(frames), + 'num_inference_steps': steps, + 'generator': get_generator(int(seed) if seed is not None else -1), + 'callback_on_step_end': diffusers_callback, + 'output_type': 'pil', + } + if p.cfg_scale is not None and p.cfg_scale > 0: + refine_args['guidance_scale'] = p.cfg_scale + if caps.supports_frame_rate_kwarg: + refine_args['frame_rate'] = float(mp4_fps) + if caps.supports_image_cond_noise_scale and image_cond_noise_scale is not None: + refine_args['image_cond_noise_scale'] = image_cond_noise_scale + if caps.supports_multi_condition and conditions: + refine_args['conditions'] = conditions + # Thread Stage-1 I2V init image through Stage 2 so first-frame identity survives refine. + if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and p.task_args.get('image') is not None: + refine_args['image'] = p.task_args['image'] + # Thread Stage-1 audio latents into Stage 2 on 2.x. The video branch cross-attends + # audio every layer; letting prepare_audio_latents fall back to fresh noise biases + # the video branch off-distribution (desaturated output on distilled 2.x). + if caps.family == '2.x': + if stage1_audio_latents is not None: + refine_args['audio_latents'] = stage1_audio_latents.to(device=devices.device) + if caps.use_cross_timestep: + refine_args['use_cross_timestep'] = True + + saved_scheduler_stage2 = None + try: + if caps.supports_canonical_stage2: + # Dev 2.x Stage 2: swap scheduler, fuse distilled LoRA, 3 steps on the distilled + # sigma schedule at identity guidance (huggingface/diffusers#13217). + log.info(f'LTX: canonical Stage 2 via distilled LoRA repo={caps.stage2_dev_lora_repo}') + from diffusers import FlowMatchEulerDiscreteScheduler + offline_args = {'local_files_only': True} if shared.opts.offline_mode else {} + saved_scheduler_stage2 = shared.sd_model.scheduler + shared.sd_model.scheduler = FlowMatchEulerDiscreteScheduler.from_config( + saved_scheduler_stage2.config, + use_dynamic_shifting=False, + shift_terminal=None, + ) + shared.sd_model.load_lora_weights( + caps.stage2_dev_lora_repo, + adapter_name=STAGE2_DEV_LORA_ADAPTER, + cache_dir=shared.opts.hfcache_dir, + **offline_args, + ) + shared.sd_model.set_adapters([STAGE2_DEV_LORA_ADAPTER], [1.0]) + # Do NOT apply _canonical_ltx2_guidance on this path; its audio-branch kwargs + # would clobber the identity set. + refine_args.update(_canonical_stage2_dev_kwargs()) + refine_args.pop('num_inference_steps', None) + elif caps.family == '2.x': + # Distilled 2.x. Dev 2.x with a LoRA hit the branch above. + from diffusers.pipelines.ltx2.utils import STAGE_2_DISTILLED_SIGMA_VALUES + refine_args['sigmas'] = list(STAGE_2_DISTILLED_SIGMA_VALUES) + refine_args.pop('num_inference_steps', None) + # LTX2Pipeline/LTX2ImageToVideoPipeline default noise_scale=0.0 when not passed; + # sigma=0 user latents mismatched against sigmas[0] scheduler collapses output. + # LTX2ConditionPipeline auto-infers this; do the same explicitly for T2V/I2V. + refine_args['noise_scale'] = float(refine_args['sigmas'][0]) + refine_args.update(_canonical_ltx2_guidance(caps)) + elif caps.repo_cls_name == 'LTXConditionPipeline': + refine_args['denoise_strength'] = refine_strength + if latents.ndim == 4: + latents = latents.unsqueeze(0) + log.debug(f'Video: op=refine cls={caps.repo_cls_name} latents={latents.shape} canonical_stage2={caps.supports_canonical_stage2}') + yield None, 'LTX: Refine in progress...' + try: + result = shared.sd_model(latents=latents, **refine_args) + pixels = result.frames[0] if hasattr(result, 'frames') else None + if hasattr(result, 'audio') and result.audio is not None: + audio = result.audio[0].float().cpu() + latents = None + except AssertionError as e: + yield from abort(e, ok=True, p=p) + return + except Exception as e: + yield from abort(e, ok=False, p=p) + return + finally: + if saved_scheduler_stage2 is not None: + try: + from modules.lora.extra_networks_lora import unload_diffusers + unload_diffusers() + except Exception as e: + log.warning(f'LTX: canonical Stage 2 LoRA unload failed: {e}') + shared.sd_model.scheduler = saved_scheduler_stage2 + log.debug('LTX: canonical Stage 2 cleanup done (LoRA unloaded, scheduler restored)') + t8 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, silent=True) + t9 = time.time() + timer.process.add('refine', t8 - t7) + timer.process.add('offload', t9 - t8) + shared.state.end(refinejob) + + if needs_latent_path: + extra_networks.deactivate(p) + + if needs_latent_path and latents is not None: + # Only reached on upsample-without-refine; refine decodes through the pipe and nulls latents. + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'], force=True, silent=True) + devices.torch_gc(force=True, reason='ltx:vae') + yield None, 'LTX: VAE decode in progress...' + try: + if torch.is_tensor(latents): + # 0.9.x returns raw latents with output_type='latent'; 2.x pre-denormalizes. + frames_out = vae_decode(latents, decode_timestep if caps.supports_decode_timestep else 0.0, int(seed) if seed is not None else -1, denormalize=caps.family == '0.9') + else: + frames_out = latents + except AssertionError as e: + yield from abort(e, ok=True, p=p) + return + except Exception as e: + yield from abort(e, ok=False, p=p) + return + pixels = frames_out + t10 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, silent=True) + t11 = time.time() + timer.process.add('offload', t11 - t10) + + if not audio_enable: + audio = None - log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=refine latents={latents.shape} {refine_args}') - if len(conditions) > 0: - refine_args["conditions"] = conditions - yield None, 'LTX: Refine in progress...' try: - refined_latents = shared.sd_model(latents=latents, **refine_args).frames[0] - except AssertionError as e: - yield from abort(e, ok=True, p=p) - return - except Exception as e: - yield from abort(e, ok=False, p=p) - return - latents = refined_latents - t8 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - t9 = time.time() - timer.process.add('refine', t8 - t7) - timer.process.add('offload', t9 - t8) - shared.state.end(refinejob) + aac_sample_rate = shared.sd_model.vocoder.config.output_sampling_rate + except Exception: + aac_sample_rate = 24000 - extra_networks.deactivate(p) + num_frames, video_file, _thumb = save_video( + p=p, + pixels=pixels, + audio=audio, + mp4_fps=mp4_fps, + mp4_codec=mp4_codec, + mp4_opt=mp4_opt, + mp4_ext=mp4_ext, + mp4_sf=mp4_sf, + mp4_video=mp4_video, + mp4_frames=mp4_frames, + mp4_interpolate=mp4_interpolate, + aac_sample_rate=aac_sample_rate, + metadata={}, + ) - yield None, 'LTX: VAE decode in progress...' - try: - if torch.is_tensor(latents): - frames = vae_decode(latents, decode_timestep, seed) + t_end = time.time() + if isinstance(pixels, list) and len(pixels) > 0 and isinstance(pixels[0], Image.Image): + w, h = pixels[0].size + elif hasattr(pixels, 'ndim') and pixels.ndim == 5: + _n, _c, _t, h, w = pixels.shape + elif hasattr(pixels, 'ndim') and pixels.ndim == 4: + _n, h, w, _c = pixels.shape + elif hasattr(pixels, 'shape'): + h, w = pixels.shape[-2], pixels.shape[-1] else: - frames = latents - except TypeError: - frames = latents # likely because the latents are already decoded - except AssertionError as e: - yield from abort(e, ok=True, p=p) - return - except Exception as e: - yield from abort(e, ok=False, p=p) - return - t10 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - t11 = time.time() - timer.process.add('offload', t11 - t10) + w, h = p.width, p.height + resolution = f'{w}x{h}' if num_frames > 0 else None + summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ') + memory = shared.mem_mon.summary() + total_time = max(t_end - t0, 1e-6) + fps = f'{num_frames/total_time:.2f}' + its = f'{(steps)/total_time:.2f}' - try: - aac_sample_rate = shared.sd_model.vocoder.config.output_sampling_rate - except Exception: - aac_sample_rate = 24000 + shared.state.end(videojob) + progress.finish_task(task_id) + p.close() - num_frames, video_file, _thumb = save_video( - p=p, - pixels=frames, - audio=audio, - mp4_fps=mp4_fps, - mp4_codec=mp4_codec, - mp4_opt=mp4_opt, - mp4_ext=mp4_ext, - mp4_sf=mp4_sf, - mp4_video=mp4_video, - mp4_frames=mp4_frames, - mp4_interpolate=mp4_interpolate, - aac_sample_rate=aac_sample_rate, - metadata={}, - ) - - t_end = time.time() - if isinstance(frames, list) and isinstance(frames[0], Image.Image): - w, h = frames[0].size - elif frames.ndim == 5: - _n, _c, _t, h, w = frames.shape - elif frames.ndim == 4: - _n, h, w, _c = frames.shape - else: - h, w = frames.shape[-2], frames.shape[-1] - resolution = f'{w}x{h}' if num_frames > 0 else None - summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ') - memory = shared.mem_mon.summary() - fps = f'{num_frames/(t_end-t0):.2f}' - its = f'{(steps)/(t_end-t0):.2f}' - - shared.state.end(videojob) - progress.finish_task(task_id) - - log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={fps} its={its} resolution={resolution} time={t_end-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') - yield video_file, f'LTX: Generation completed | File {video_file} | Frames {len(frames)} | Resolution {resolution} | f/s {fps} | it/s {its} '+ f"

{summary} {memory}

" + log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={fps} its={its} resolution={resolution} time={t_end-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') + yield video_file, f'LTX: Generation completed | File {video_file} | Frames {num_frames} | Resolution {resolution} | f/s {fps} | it/s {its} ' + f"

{summary} {memory}

" diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index 846efcd0b..94bd857ef 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -1,15 +1,59 @@ import os import gradio as gr -from modules import ui_sections +from modules import ui_sections, ui_symbols +from modules.ui_components import ToolButton from modules.logger import log from modules.video_models.models_def import models -from modules.ltx import ltx_process +from modules.ltx import ltx_process, ltx_capabilities debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None -def create_ui(prompt, negative, styles, overrides, init_image, init_strength, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, width, height, frames, seed): +def _model_change(model_name: str): + caps = ltx_capabilities.get_caps(model_name) + if caps is None: + return ( + gr.update(visible=False), # input_media_accordion + gr.update(visible=False), # multi_condition_group + gr.update(visible=False), # upsample_accordion + gr.update(visible=False), # refine_accordion + gr.update(value=False), # upsample_enable (reset) + gr.update(value=False), # refine_enable (reset) + gr.update(interactive=True), # refine_strength + gr.update(), # guidance_scale + gr.update(), # steps + gr.update(), # sampler_shift + gr.update(), # dynamic_shift + gr.update(interactive=False), # decode_timestep + gr.update(interactive=False), # image_cond_noise_scale + gr.update(visible=False), # audio_accordion + ) + # 2.x refine runs fixed canonical schedules; refine_strength only feeds 0.9.x LTXConditionPipeline. + refine_strength_interactive = caps.family == '0.9' + # Default Refine on for Dev 2.x T2V/I2V: Lightricks' production recipe is Stage 1 + 2x upsample + # + Stage 2 refine (auto_refine_upsample at ltx_process.py:179 couples the stages once Refine is on). + # Multi-condition variants are excluded for the same reason auto_refine_upsample excludes them. + refine_default = caps.supports_canonical_stage2 and not caps.supports_multi_condition + return ( + gr.update(visible=caps.supports_input_media), + gr.update(visible=caps.supports_multi_condition), + gr.update(visible=True), + gr.update(visible=True), + gr.update(value=False), + gr.update(value=refine_default), + gr.update(interactive=refine_strength_interactive), + gr.update(value=caps.default_cfg), + gr.update(value=caps.default_steps), + gr.update(value=caps.default_sampler_shift), + gr.update(value=caps.default_dynamic_shift), + gr.update(interactive=caps.supports_decode_timestep), + gr.update(interactive=caps.supports_image_cond_noise_scale), + gr.update(visible=caps.supports_audio), + ) + + +def create_ui(prompt, negative, styles, overrides, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf): with gr.Row(): with gr.Column(variant='compact', elem_id="ltx_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): @@ -17,39 +61,83 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la with gr.Row(): ltx_models = [m.name for m in models['LTX Video']] if 'LTX Video' in models else ['None'] model = gr.Dropdown(label='LTX model', choices=ltx_models, value=ltx_models[0], elem_id="ltx_model") - with gr.Accordion(open=False, label="Condition", elem_id='ltx_condition_accordion'): - with gr.Tabs(): - with gr.Tab('Video', id='ltx_condition_video_tab'): - condition_video = gr.Video(label='Video', type='filepath', elem_id="ltx_condition_video", width=256, height=256, source='upload') - with gr.Row(): - condition_video_frames = gr.Slider(label='LTX frames number', minimum=-1, maximum=1024, step=1, value=-1, elem_id="ltx_condition_video_frames") - condition_video_skip = gr.Slider(label='LTX frames skip', minimum=0, maximum=1024, step=1, value=0, elem_id="ltx_condition_video_sip") - with gr.Tab('Gallery', id='ltx_condition_batch_tab'): - condition_files = gr.Files(label="Image Batch", interactive=True, elem_id="ltx_condition_batch") - with gr.Accordion(open=False, label="Upsample", elem_id='ltx_upsample_accordion'): + with gr.Accordion(open=False, label='Size', elem_id='ltx_size_accordion'): + width, height = ui_sections.create_resolution_inputs('ltx', default_width=832, default_height=480) + with gr.Row(): + frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=121, elem_id='ltx_frames') + seed = gr.Number(label='Initial seed', value=-1, elem_id='ltx_seed', container=True) + random_seed = ToolButton(ui_symbols.random, elem_id='ltx_seed_random') + random_seed.click(fn=lambda: -1, show_progress='hidden', inputs=[], outputs=[seed]) + input_media_accordion = gr.Accordion(open=False, label="Input media", elem_id='ltx_input_media_accordion', visible=False) + with input_media_accordion: + ltx_init_image = gr.Image(label='Image', elem_id='ltx_init_image', type='pil', image_mode='RGB', width=256, height=256) + ltx_condition_strength = gr.Slider(label='LTX input strength', minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id='ltx_condition_strength') + with gr.Row(): + last_image = gr.Image(label='Last image', elem_id='ltx_last_image', type='pil', image_mode='RGB', width=256, height=256) + multi_condition_group = gr.Group(visible=False) + with multi_condition_group: + gr.Markdown('**Prefix conditioning**: supply a video or gallery to anchor the opening frames', elem_id='ltx_prefix_conditioning_label') + with gr.Tabs(): + with gr.Tab('Video prefix', id='ltx_condition_video_tab'): + condition_video = gr.Video(label='Video', type='filepath', elem_id="ltx_condition_video", width=256, height=256, source='upload') + with gr.Row(): + condition_video_frames = gr.Slider(label='LTX frames number', minimum=-1, maximum=1024, step=1, value=-1, elem_id="ltx_condition_video_frames") + condition_video_skip = gr.Slider(label='LTX frames skip', minimum=0, maximum=1024, step=1, value=0, elem_id="ltx_condition_video_sip") + with gr.Tab('Gallery prefix', id='ltx_condition_batch_tab'): + condition_files = gr.Files(label="Image Batch", interactive=True, elem_id="ltx_condition_batch") + upsample_accordion = gr.Accordion(open=False, label="Upsample", elem_id='ltx_upsample_accordion') + with upsample_accordion: with gr.Row(): upsample_enable = gr.Checkbox(label='LTX enable upsampling', value=False, elem_id="ltx_upsample_enable") - upsample_ratio = gr.Slider(label='LTX upsample ratio', minimum=1.0, maximum=4.0, step=0.1, value=2.0, elem_id="ltx_upsample_ratio", interactive=False) - with gr.Accordion(open=False, label="Refine", elem_id='ltx_refine_accordion'): + upsample_ratio = gr.Slider(label='LTX upsample ratio', minimum=1.0, maximum=4.0, step=0.1, value=2.0, elem_id="ltx_upsample_ratio") + refine_accordion = gr.Accordion(open=False, label="Refine", elem_id='ltx_refine_accordion') + with refine_accordion: with gr.Row(): refine_enable = gr.Checkbox(label='LTX enable refine', value=False, elem_id="ltx_refine_enable") refine_strength = gr.Slider(label='LTX refine strength', minimum=0.1, maximum=1.0, step=0.05, value=0.4, elem_id="ltx_refine_strength") - with gr.Accordion(open=False, label="Advanced", elem_id='ltx_parameters_accordion'): - steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "ltx", default_steps=50) + parameters_accordion = gr.Accordion(open=False, label="Advanced", elem_id='ltx_parameters_accordion') + with parameters_accordion: + steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "ltx", default_steps=40) with gr.Row(): - decode_timestep = gr.Slider(label='LTX decode timestep', minimum=0.01, maximum=1.0, step=0.01, value=0.05, elem_id="ltx_decode_timestep") - image_cond_noise_scale = gr.Slider(label='Noise scale', minimum=0.01, maximum=1.0, step=0.01, value=0.025, elem_id="ltx_image_cond_noise_scale") - with gr.Accordion(open=False, label="Audio", elem_id='ltx_audio_accordion'): + guidance_scale = gr.Slider(label='LTX guidance scale', minimum=0.0, maximum=14.0, step=0.1, value=4.0, elem_id="ltx_guidance_scale") + with gr.Row(): + sampler_shift = gr.Slider(label='LTX sampler shift', minimum=-1.0, maximum=20.0, step=0.1, value=-1.0, elem_id="ltx_sampler_shift") + dynamic_shift = gr.Checkbox(label='LTX dynamic shift', value=False, elem_id="ltx_dynamic_shift") + with gr.Row(): + decode_timestep = gr.Slider(label='LTX decode timestep', minimum=0.0, maximum=1.0, step=0.01, value=0.05, elem_id="ltx_decode_timestep") + image_cond_noise_scale = gr.Slider(label='LTX image cond noise scale', minimum=0.0, maximum=1.0, step=0.005, value=0.025, elem_id="ltx_image_cond_noise_scale") + audio_accordion = gr.Accordion(open=False, label="Audio", elem_id='ltx_audio_accordion', visible=False) + with audio_accordion: with gr.Row(): audio_enable = gr.Checkbox(label='LTX enable audio', value=False, elem_id="ltx_audio_enable") with gr.Column(elem_id='ltx-output-column', scale=2) as _column_output: with gr.Row(): video = gr.Video(label="Output", show_label=False, elem_id='ltx_output_video', elem_classes=['control-image'], height=512, autoplay=False) - # video = gr.Gallery(value=[], label="Output", show_label=False, elem_id='ltx_output_video', elem_classes=['control-image'], height=512) with gr.Row(): text = gr.HTML('', elem_id='ltx_generation_info', show_label=False) + model.change( + fn=_model_change, + inputs=[model], + outputs=[ + input_media_accordion, + multi_condition_group, + upsample_accordion, + refine_accordion, + upsample_enable, + refine_enable, + refine_strength, + guidance_scale, + steps, + sampler_shift, + dynamic_shift, + decode_timestep, + image_cond_noise_scale, + audio_accordion, + ], + ) + task_id = gr.Textbox(visible=False, value='') ui_state = gr.Textbox(visible=False, value='') state_inputs = [task_id, ui_state] @@ -58,10 +146,12 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la model, prompt, negative, styles, width, height, frames, - steps, sampler_index, seed, + steps, sampler_index, + guidance_scale, sampler_shift, dynamic_shift, + seed, upsample_enable, upsample_ratio, refine_enable, refine_strength, - init_strength, init_image, last_image, condition_files, condition_video, condition_video_frames, condition_video_skip, + ltx_condition_strength, ltx_init_image, last_image, condition_files, condition_video, condition_video_frames, condition_video_skip, decode_timestep, image_cond_noise_scale, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, audio_enable, diff --git a/modules/ltx/ltx_util.py b/modules/ltx/ltx_util.py index e5655af78..f1ef78391 100644 --- a/modules/ltx/ltx_util.py +++ b/modules/ltx/ltx_util.py @@ -1,17 +1,18 @@ import time +from contextlib import contextmanager import torch from PIL import Image from modules import devices, shared, sd_models, timer, extra_networks from modules.logger import log -loaded_model: str = None - - def get_bucket(size: int): - if not hasattr(shared.sd_model, 'vae_temporal_compression_ratio'): - return int(size) - (int(size) % 32) - return int(size) - (int(size) % shared.sd_model.vae_temporal_compression_ratio) + # LTX pipes validate width/height divisible by 32 across all families. + ratio = getattr(shared.sd_model, 'vae_spatial_compression_ratio', None) + if not isinstance(ratio, int) or ratio < 32: + ratio = 32 + size = int(size) + return size - (size % ratio) def get_frames(frames: int): @@ -19,21 +20,16 @@ def get_frames(frames: int): def load_model(engine: str, model: str): - global loaded_model # pylint: disable=global-statement - if not shared.sd_loaded: - loaded_model = None - if loaded_model == model: - return - if model is None or model == '' or model=='None': - loaded_model = None + if model is None or model == '' or model == 'None': shared.sd_model = None return t0 = time.time() from modules.video_models import models_def, video_load selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] + # video_load owns the cache; pipe-class mismatch inside it invalidates the name-based hit + # when Unload Models (or any external swap) silently replaced shared.sd_model. log.info(f'Video load: engine="{engine}" selected="{model}" {selected}') video_load.load_model(selected) - loaded_model = model t1 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) t2 = time.time() @@ -45,7 +41,7 @@ def load_upsample(upsample_pipe, upsample_repo_id): if upsample_pipe is None: t0 = time.time() from diffusers.pipelines.ltx.pipeline_ltx_latent_upsample import LTXLatentUpsamplePipeline - log.info(f'Video load: cls={LTXLatentUpsamplePipeline.__class__.__name__} repo="{upsample_repo_id}"') + log.info(f'Video load: cls={LTXLatentUpsamplePipeline.__name__} repo="{upsample_repo_id}"') upsample_pipe = LTXLatentUpsamplePipeline.from_pretrained( upsample_repo_id, vae=shared.sd_model.vae, @@ -57,8 +53,103 @@ def load_upsample(upsample_pipe, upsample_repo_id): return upsample_pipe -def get_conditions(width, height, condition_strength, condition_images, condition_files, condition_video, condition_video_frames, condition_video_skip): +def load_upsample_2x(upsample_pipe, upsample_repo_id): + # 2.x ships the upsampler as a bare nn.Module in a subfolder; no from_pretrained on the + # pipeline wrapper, so we load the model + construct the pipeline manually. + if upsample_pipe is None: + t0 = time.time() + from diffusers.pipelines.ltx2.pipeline_ltx2_latent_upsample import LTX2LatentUpsamplePipeline + from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel + from modules import sd_checkpoint + log.info(f'Video load: cls={LTX2LatentUpsamplePipeline.__name__} repo="{upsample_repo_id}"') + latent_upsampler = LTX2LatentUpsamplerModel.from_pretrained( + upsample_repo_id, + subfolder='latent_upsampler', + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + ).to(devices.device) + upsample_pipe = LTX2LatentUpsamplePipeline( + vae=shared.sd_model.vae, + latent_upsampler=latent_upsampler, + ) + # Synthetic checkpoint_info gives this pipe its own OffloadHook cache slot, so routing + # it through apply_balanced_offload does not invalidate the main pipe's module map + # (sd_offload.py:488 keys on sd_checkpoint_info.name). + upsample_pipe.sd_checkpoint_info = sd_checkpoint.CheckpointInfo('ltx-upsampler-2.x') + t1 = time.time() + timer.process.add('load', t1 - t0) + return upsample_pipe + + +@contextmanager +def ltx_scheduler_opts(sd_model, *, dynamic_shift=None, sampler_shift=None): + # Run-scoped override of shared.opts scheduler settings and scheduler.config. Snapshots + # five pieces of state (shared.opts dynamic_shift + shift, scheduler object, default_scheduler + # snapshot, and scheduler.config use_dynamic_shifting + flow_shift) and restores every one on + # exit. Keeps run-specific sampler settings out of config.json and prevents default_scheduler + # from getting clobbered by a deepcopy of the mutated scheduler at video_load.py:171. The + # scheduler-object restore matters for Stage 2 refine, which swaps the scheduler entirely. + orig_dynamic_shift = shared.opts.schedulers_dynamic_shift + orig_sampler_shift = shared.opts.schedulers_shift + orig_scheduler = sd_model.scheduler + orig_default_scheduler = getattr(sd_model, 'default_scheduler', None) + orig_use_dynamic_shifting = getattr(orig_scheduler.config, 'use_dynamic_shifting', None) if hasattr(orig_scheduler, 'config') else None + orig_flow_shift = getattr(orig_scheduler.config, 'flow_shift', None) if hasattr(orig_scheduler, 'config') else None + + try: + if dynamic_shift is not None: + shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift + if sampler_shift is not None: + shared.opts.data['schedulers_shift'] = sampler_shift + if hasattr(sd_model, 'scheduler') and hasattr(sd_model.scheduler, 'config') and hasattr(sd_model.scheduler, 'register_to_config'): + if dynamic_shift is not None and hasattr(sd_model.scheduler.config, 'use_dynamic_shifting'): + sd_model.scheduler.config.use_dynamic_shifting = dynamic_shift + sd_model.scheduler.register_to_config(use_dynamic_shifting=dynamic_shift) + if sampler_shift is not None and sampler_shift >= 0 and hasattr(sd_model.scheduler.config, 'flow_shift'): + sd_model.scheduler.config.flow_shift = sampler_shift + sd_model.scheduler.register_to_config(flow_shift=sampler_shift) + yield + finally: + shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift + shared.opts.data['schedulers_shift'] = orig_sampler_shift + if sd_model.scheduler is not orig_scheduler: + sd_model.scheduler = orig_scheduler + if orig_default_scheduler is not None and sd_model.default_scheduler is not orig_default_scheduler: + sd_model.default_scheduler = orig_default_scheduler + if hasattr(sd_model.scheduler, 'config') and hasattr(sd_model.scheduler, 'register_to_config'): + if orig_use_dynamic_shifting is not None and hasattr(sd_model.scheduler.config, 'use_dynamic_shifting'): + sd_model.scheduler.config.use_dynamic_shifting = orig_use_dynamic_shifting + sd_model.scheduler.register_to_config(use_dynamic_shifting=orig_use_dynamic_shifting) + if orig_flow_shift is not None and hasattr(sd_model.scheduler.config, 'flow_shift'): + sd_model.scheduler.config.flow_shift = orig_flow_shift + sd_model.scheduler.register_to_config(flow_shift=orig_flow_shift) + log.debug(f'LTX: scheduler/opts restored dynamic_shift={orig_dynamic_shift} sampler_shift={orig_sampler_shift}') + + +def _condition_cls(family: str): + if family == '2.x': + try: + from diffusers.pipelines.ltx2.pipeline_ltx2_condition import LTX2VideoCondition + return LTX2VideoCondition + except ImportError: + log.warning('LTX conditions: LTX2VideoCondition not available in installed diffusers') + return None from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition + return LTXVideoCondition + + +def make_condition(condition_cls, family: str, frames, strength: float, is_video: bool): + if family == '2.x': + return condition_cls(frames=frames, index=0, strength=strength) + if is_video: + return condition_cls(video=frames, frame_index=0, strength=strength) + return condition_cls(image=frames, frame_index=0, strength=strength) + + +def get_conditions(width, height, condition_strength, condition_images, condition_files, condition_video, condition_video_frames, condition_video_skip, family: str = '0.9'): + condition_cls = _condition_cls(family) + if condition_cls is None: + return [] conditions = [] if condition_images is not None: for condition_image in condition_images: @@ -67,32 +158,32 @@ def get_conditions(width, height, condition_strength, condition_images, conditio from modules.api.api import decode_base64_to_image condition_image = decode_base64_to_image(condition_image) condition_image = condition_image.convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) - conditions.append(LTXVideoCondition(image=condition_image, frame_index=0, strength=condition_strength)) - log.debug(f'Video condition: image={condition_image.size} strength={condition_strength}') + conditions.append(make_condition(condition_cls, family, condition_image, condition_strength, is_video=False)) + log.debug(f'Video condition: family={family} image={condition_image.size} strength={condition_strength}') except Exception as e: log.error(f'LTX condition image: {e}') if condition_files is not None: - condition_images = [] + batch_images = [] for fn in condition_files: try: if hasattr(fn, 'name'): condition_image = Image.open(fn.name).convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) else: condition_image = fn.convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) - condition_images.append(condition_image) + batch_images.append(condition_image) except Exception as e: log.error(f'LTX condition files: {e}') - if len(condition_images) > 0: - conditions.append(LTXVideoCondition(video=condition_images, frame_index=0, strength=condition_strength)) - log.debug(f'Video condition: files={len(condition_images)} size={condition_images[0].size} strength={condition_strength}') + if len(batch_images) > 0: + conditions.append(make_condition(condition_cls, family, batch_images, condition_strength, is_video=True)) + log.debug(f'Video condition: family={family} files={len(batch_images)} size={batch_images[0].size} strength={condition_strength}') if condition_video is not None: from modules.video_models.video_utils import get_video_frames try: condition_frames = get_video_frames(condition_video, num_frames=condition_video_frames, skip_frames=condition_video_skip) condition_frames = [f.convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) for f in condition_frames] if len(condition_frames) > 0: - conditions.append(LTXVideoCondition(video=condition_frames, frame_index=0, strength=condition_strength)) - log.debug(f'Video condition: frames={len(condition_frames)} size={condition_frames[0].size} strength={condition_strength}') + conditions.append(make_condition(condition_cls, family, condition_frames, condition_strength, is_video=True)) + log.debug(f'Video condition: family={family} frames={len(condition_frames)} size={condition_frames[0].size} strength={condition_strength}') except Exception as e: log.error(f'LTX condition video: {e}') return conditions @@ -114,16 +205,19 @@ def get_generator(seed): return torch.Generator().manual_seed(seed) -def vae_decode(latents, decode_timestep, seed): +def vae_decode(latents, decode_timestep, seed, denormalize: bool = True): t0 = time.time() - log.debug(f'Video: cls={shared.sd_model.vae.__class__.__name__} op=vae latents={latents.shape} timestep={decode_timestep}') + if latents.ndim == 4: + latents = latents.unsqueeze(0) + log.debug(f'Video: cls={shared.sd_model.vae.__class__.__name__} op=vae latents={latents.shape} timestep={decode_timestep} denormalize={denormalize}') from diffusers.utils.torch_utils import randn_tensor - latents = shared.sd_model._denormalize_latents( # pylint: disable=protected-access - latents, - shared.sd_model.vae.latents_mean, - shared.sd_model.vae.latents_std, - shared.sd_model.vae.config.scaling_factor - ) + if denormalize: + latents = shared.sd_model._denormalize_latents( # pylint: disable=protected-access + latents, + shared.sd_model.vae.latents_mean, + shared.sd_model.vae.latents_std, + shared.sd_model.vae.config.scaling_factor + ) latents = latents.to(device=devices.device, dtype=devices.dtype) if not shared.sd_model.vae.config.timestep_conditioning: timestep = None @@ -132,10 +226,7 @@ def vae_decode(latents, decode_timestep, seed): timestep = torch.tensor([decode_timestep], device=devices.device, dtype=latents.dtype) noise_scale = torch.tensor([decode_timestep], device=devices.device, dtype=devices.dtype)[:, None, None, None, None] latents = (1 - noise_scale) * latents + noise_scale * noise - frames = shared.sd_model.vae.decode(latents, timestep, return_dict=False)[0] # n, c, f, h, w - # frames = frames.squeeze(0) if frames.ndim == 5 else frames - # frames = frames.permute(1, 2, 3, 0) - # frames = shared.sd_model.video_processor.postprocess_video(frames, output_type='pil') + frames = shared.sd_model.vae.decode(latents, timestep, return_dict=False)[0] t1 = time.time() timer.process.add('vae', t1 - t0) return frames diff --git a/modules/ui_video.py b/modules/ui_video.py index c6026fdbd..4d9362973 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -28,24 +28,27 @@ def create_ui(): with gr.Row(elem_id="video_interface", equal_height=False): with gr.Tabs(elem_classes=['video-tabs'], elem_id='video-tabs'): overrides = ui_common.create_override_inputs('video') - with gr.Tab('Size', id='video-size-tab') as _video_size_tab: - from modules.video_models import video_ui - width, height, frames, seed, reuse_seed = video_ui.create_ui_size() - with gr.Tab('Inputs', id='video-inputs-tab') as _video_inputs_tab: - from modules.video_models import video_ui - init_image, init_strength, last_image = video_ui.create_ui_inputs() - with gr.Tab('Video Output', id='video-outputs-tab') as _video_outputs_tab: + with gr.Tab('Output', id='video-outputs-tab') as _video_outputs_tab: from modules.video_models import video_ui mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf = video_ui.create_ui_outputs() - with gr.Tab('Models', id='video-core-tab') as video_core_tab: + with gr.Tab('Generic', id='video-core-tab') as video_core_tab: from modules.video_models import video_ui - engine, model, steps, sampler_index = video_ui.create_ui(prompt, negative, styles, overrides, init_image, init_strength, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, width, height, frames, seed, reuse_seed) + engine, model, steps, sampler_index, width, height, frames, seed = video_ui.create_ui( + prompt, negative, styles, overrides, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, + ) with gr.Tab('FramePack', id='framepack-tab') as framepack_tab: from modules.framepack import framepack_ui - framepack_ui.create_ui(prompt, negative, styles, overrides, init_image, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf) + framepack_ui.create_ui( + prompt, negative, styles, overrides, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, + ) with gr.Tab('LTX', id='ltx-tab') as ltx_tab: from modules.ltx import ltx_ui - ltx_ui.create_ui(prompt, negative, styles, overrides, init_image, init_strength, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, width, height, frames, seed) + ltx_ui.create_ui( + prompt, negative, styles, overrides, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, + ) paste_fields = [ (prompt, "Prompt"), # cannot add more fields as they are not defined yet diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index 229e06c50..4e53f0c9f 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -150,7 +150,13 @@ try: Model(name='LTXVideo 2.3 22B I2V', url='https://huggingface.co/Lightricks/LTX-2.3', repo='OzzyGT/LTX-2.3', - repo_cls=getattr(diffusers, 'LTX2Pipeline', None), + repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='LTXVideo 2.3 22B Condition', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3', + repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None), te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), Model(name='LTXVideo 2.3 22B T2V Distilled', @@ -162,7 +168,13 @@ try: Model(name='LTXVideo 2.3 22B I2V Distilled', url='https://huggingface.co/Lightricks/LTX-2.3', repo='OzzyGT/LTX-2.3-Distilled', - repo_cls=getattr(diffusers, 'LTX2Pipeline', None), + repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='LTXVideo 2.3 22B Condition Distilled', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3-Distilled', + repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None), te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), @@ -187,7 +199,13 @@ try: Model(name='LTXVideo 2.3 22B I2V SDNQ-4Bit', url='https://huggingface.co/Lightricks/LTX-2.3', repo='OzzyGT/LTX-2.3-sdnq-dynamic-int4', - repo_cls=getattr(diffusers, 'LTX2Pipeline', None), + repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='LTXVideo 2.3 22B Condition SDNQ-4Bit', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3-sdnq-dynamic-int4', + repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None), te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), Model(name='LTXVideo 2.3 22B T2V Distilled SDNQ-4Bit', @@ -199,7 +217,13 @@ try: Model(name='LTXVideo 2.3 22B I2V Distilled SDNQ-4Bit', url='https://huggingface.co/Lightricks/LTX-2.3', repo='OzzyGT/LTX-2.3-Distilled-sdnq-dynamic-int4', - repo_cls=getattr(diffusers, 'LTX2Pipeline', None), + repo_cls=getattr(diffusers, 'LTX2ImageToVideoPipeline', None), + te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), + dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), + Model(name='LTXVideo 2.3 22B Condition Distilled SDNQ-4Bit', + url='https://huggingface.co/Lightricks/LTX-2.3', + repo='OzzyGT/LTX-2.3-Distilled-sdnq-dynamic-int4', + repo_cls=getattr(diffusers, 'LTX2ConditionPipeline', None), te_cls=getattr(transformers, 'Gemma3ForConditionalGeneration', None), dit_cls=getattr(diffusers, 'LTX2VideoTransformer3DModel', None)), diff --git a/modules/video_models/video_load.py b/modules/video_models/video_load.py index 373751b7e..819706767 100644 --- a/modules/video_models/video_load.py +++ b/modules/video_models/video_load.py @@ -36,6 +36,12 @@ def load_model(selected: models_def.Model): global loaded_model # pylint: disable=global-statement if not shared.sd_loaded: loaded_model = None + elif loaded_model == selected.name and selected.repo_cls is not None and not isinstance(shared.sd_model, selected.repo_cls): + # shared.sd_model auto-reloads the default checkpoint when model_data.sd_model is None, + # which silently swaps the pipe class behind the name-based cache. Pipe-class mismatch + # is the reliable signal that the cached name no longer maps to the cached object. + log.warning(f'Video load: cached model="{selected.name}" pipe class swapped to {type(shared.sd_model).__name__}; forcing reload') + loaded_model = None if loaded_model == selected.name: return '' if shared.sd_loaded: @@ -82,6 +88,13 @@ def load_model(selected: models_def.Model): selected.te = 'ai-forever/Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers' selected.te_folder = 'text_encoder' selected.te_revision = None + if selected.te_cls.__name__ == 'Gemma3ForConditionalGeneration' and shared.opts.te_shared_t5: + if 'SDNQ' in selected.name: + selected.te = 'OzzyGT/LTX-2.3-sdnq-dynamic-int4' + else: + selected.te = 'OzzyGT/LTX-2.3' + selected.te_folder = 'text_encoder' + selected.te_revision = None log.debug(f'Video load: module=te repo="{selected.te or selected.repo}" folder="{selected.te_folder}" cls={selected.te_cls.__name__} quant={model_quant.get_quant_type(quant_args)} loader={_loader("transformers")}') kwargs["text_encoder"] = selected.te_cls.from_pretrained( @@ -152,7 +165,11 @@ def load_model(selected: models_def.Model): return msg t1 = time.time() - if shared.sd_model.__class__.__name__.startswith("LTX"): + cls_name = shared.sd_model.__class__.__name__ + # LTX 0.9.x is plain linear; pin use_dynamic_shifting=False against upstream config drift. + # LTX-2.x canonical is token-count-based dynamic shift (base_shift=0.95, max_shift=2.05); + # disabling it there would take the model off-distribution. + if cls_name.startswith("LTX") and not cls_name.startswith("LTX2"): shared.sd_model.scheduler.config.use_dynamic_shifting = False shared.sd_model.default_scheduler = copy.deepcopy(shared.sd_model.scheduler) if hasattr(shared.sd_model, "scheduler") else None shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(selected.repo) diff --git a/modules/video_models/video_overrides.py b/modules/video_models/video_overrides.py index 141b7be0c..582acc92e 100644 --- a/modules/video_models/video_overrides.py +++ b/modules/video_models/video_overrides.py @@ -1,7 +1,7 @@ import os import torch import diffusers -from modules import shared, processing +from modules import shared, processing, devices from modules.logger import log from modules.video_models.models_def import Model @@ -17,6 +17,37 @@ def load_override(selected: Model, **load_args): # LTX if 'LTXVideo 0.9.5 I2V' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLLTXVideo.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir, **load_args) + # OzzyGT LTX-2.3 mirrors pack connectors/ twice by design: sharded (*-00001-of-0000N + + # .index.json) and unsharded diffusion_pytorch_model.safetensors of the byte-identical + # weights. snapshot_download faithfully fetches both; diffusers' component loader picks + # sharded when the index is present. ignore_patterns skips the ~6.3 GB unsharded copy + # without reaching for a cleaner upstream mirror. + ltx2_redundant_connector_repos = { + 'OzzyGT/LTX-2.3', + 'OzzyGT/LTX-2.3-sdnq-dynamic-int4', + } + if selected.repo in ltx2_redundant_connector_repos: + kwargs['ignore_patterns'] = ['connectors/diffusion_pytorch_model.safetensors'] + # LTX2TextConnectors weights are byte-identical across all 2.3 variants (verified by blob + # hash). Pre-load from a canonical repo so per-variant fetches skip connectors/ entirely. + # FP16 variants share OzzyGT/LTX-2.3; SDNQ variants share the pre-quantized mirror. + ltx2_connectors_cls = None + try: + from diffusers.pipelines.ltx2 import LTX2TextConnectors + ltx2_connectors_cls = LTX2TextConnectors + except ImportError as e: + log.warning(f'Video load: LTX2TextConnectors unavailable ({e}); dedup of LTX-2.3 connectors disabled') + if ('LTXVideo 2.3' in selected.name and shared.opts.te_shared_t5 and ltx2_connectors_cls is not None): + conn_repo = 'OzzyGT/LTX-2.3-sdnq-dynamic-int4' if 'SDNQ' in selected.name else 'OzzyGT/LTX-2.3' + log.debug(f'Video load: module=connectors repo="{conn_repo}" cls={ltx2_connectors_cls.__name__} shared={shared.opts.te_shared_t5}') + kwargs['connectors'] = ltx2_connectors_cls.from_pretrained( + conn_repo, + subfolder='connectors', + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + ignore_patterns=['connectors/diffusion_pytorch_model.safetensors'], + **load_args, + ) # WAN if 'WAN 2.1 14B' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLWan.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir, **load_args) @@ -46,7 +77,8 @@ def set_overrides(p: processing.StableDiffusionProcessingVideo, selected: Model) if 'SkyReelsV2DiffusionForcing' in cls: p.task_args['overlap_history'] = 17 # LTX - if cls == 'LTXImageToVideoPipeline' or cls == 'LTXConditionPipeline': + ltx_i2v_classes = ('LTXImageToVideoPipeline', 'LTXConditionPipeline', 'LTX2ImageToVideoPipeline', 'LTX2ConditionPipeline') + if cls in ltx_i2v_classes: p.task_args['generator'] = None if cls == 'LTXConditionPipeline': p.task_args['strength'] = p.denoising_strength diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 51c781e6f..3454faf69 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -65,14 +65,14 @@ def generate(*args, **kwargs): log.warning('Video: op=T2V init image not supported') elif 'I2V' in model: if init_image is None: - return video_utils.queue_err('init image not set') + return video_utils.queue_err('No input image provided. Please upload or select an image.') p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') log.debug(f'Video: op=I2V init={init_image} resized={p.task_args["image"]}') elif 'FLF2V' in model: if init_image is None: - return video_utils.queue_err('init image not set') + return video_utils.queue_err('No input image provided. Please upload or select an image.') if last_image is None: - return video_utils.queue_err('last image not set') + return video_utils.queue_err('No last frame image provided. Please upload or select an image.') p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') p.task_args['last_image'] = images.resize_image(resize_mode=2, im=last_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') log.debug(f'Video: op=FLF2V init={init_image} last={last_image} resized={p.task_args["image"]}') @@ -82,7 +82,7 @@ def generate(*args, **kwargs): log.debug(f'Video: op=VACE reference={init_image} resized={p.task_args["reference_images"]}') elif 'Animate' in model: if init_image is None: - return video_utils.queue_err('init image not set') + return video_utils.queue_err('No input image provided. Please upload or select an image.') p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') p.task_args['mode'] = 'animate' p.task_args['pose_video'] = [] # input pose video to condition the generation on. must be a list of PIL images. diff --git a/modules/video_models/video_ui.py b/modules/video_models/video_ui.py index 4614e306e..baab34bd1 100644 --- a/modules/video_models/video_ui.py +++ b/modules/video_models/video_ui.py @@ -9,6 +9,14 @@ from modules.video_models import video_run debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None +# Engines surfaced on their own dedicated tab; hide from the general Video tab dropdown +# so users aren't offered two paths to the same models. +HIDDEN_ENGINES = {'LTX Video'} + + +def visible_engines(): + return [name for name in models_def.models if name not in HIDDEN_ENGINES] + def engine_change(engine): debug(f'Video change: engine="{engine}"') @@ -78,17 +86,6 @@ def run_video(*args): return video_utils.queue_err(f'model not found: engine="{engine}" model="{model}"') -def create_ui_inputs(): - with gr.Row(): - with gr.Column(variant='compact', elem_id="video_inputs", elem_classes=['settings-column'], scale=1): - init_strength = gr.Slider(label='Init strength', minimum=0.0, maximum=1.0, step=0.01, value=0.8, elem_id="video_denoising_strength") - gr.HTML("
  Init image") - init_image = gr.Image(elem_id="video_image", show_label=False, type="pil", image_mode="RGB", width=256, height=256) - gr.HTML("
  Last image") - last_image = gr.Image(elem_id="video_last", show_label=False, type="pil", image_mode="RGB", width=256, height=256) - return init_image, init_strength, last_image - - def create_ui_outputs(): with gr.Row(): with gr.Column(variant='compact', elem_id="video_outputs", elem_classes=['settings-column'], scale=1): @@ -97,41 +94,26 @@ def create_ui_outputs(): mp4_interpolate = gr.Slider(label="Video interpolation", minimum=0, maximum=10, value=0, step=1) with gr.Row(): mp4_codec = gr.Dropdown(label="Video codec", choices=['none', 'libx264'], value='libx264', type='value') - ui_common.create_refresh_button(mp4_codec, video_utils.get_codecs, elem_id="framepack_mp4_codec_refresh") - mp4_ext = gr.Textbox(label="Video format", value='mp4', elem_id="framepack_mp4_ext") - mp4_opt = gr.Textbox(label="Video options", value='crf:16', elem_id="framepack_mp4_opt") + ui_common.create_refresh_button(mp4_codec, video_utils.get_codecs, elem_id="video_mp4_codec_refresh") + mp4_ext = gr.Textbox(label="Video format", value='mp4', elem_id="video_mp4_ext") + mp4_opt = gr.Textbox(label="Video options", value='crf:16', elem_id="video_mp4_opt") with gr.Row(): - mp4_video = gr.Checkbox(label='Video save video', value=True, elem_id="framepack_mp4_video") - mp4_frames = gr.Checkbox(label='Video save frames', value=False, elem_id="framepack_mp4_frames") - mp4_sf = gr.Checkbox(label='Video save safetensors', value=False, elem_id="framepack_mp4_sf") + mp4_video = gr.Checkbox(label='Video save video', value=True, elem_id="video_mp4_video") + mp4_frames = gr.Checkbox(label='Video save frames', value=False, elem_id="video_mp4_frames") + mp4_sf = gr.Checkbox(label='Video save safetensors', value=False, elem_id="video_mp4_sf") return mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf -def create_ui_size(): - with gr.Row(): - with gr.Column(variant='compact', elem_id="video_size", elem_classes=['settings-column'], scale=1): - with gr.Row(): - width, height = ui_sections.create_resolution_inputs('video', default_width=832, default_height=480) - with gr.Row(): - frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=17, elem_id="video_frames") - seed = gr.Number(label='Initial seed', value=-1, elem_id="video_seed", container=True) - random_seed = ToolButton(ui_symbols.random, elem_id="video_seed_random") - reuse_seed = ToolButton(ui_symbols.reuse, elem_id="video_seed_reuse") - random_seed.click(fn=lambda: -1, show_progress='hidden', inputs=[], outputs=[seed]) - return width, height, frames, seed, reuse_seed - - -def create_ui(prompt, negative, styles, overrides, init_image, init_strength, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, width, height, frames, seed, reuse_seed): +def create_ui(prompt, negative, styles, overrides, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf): with gr.Row(): with gr.Column(variant='compact', elem_id="video_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): generate = gr.Button('Generate', elem_id="video_generate_btn", variant='primary', visible=False) with gr.Row(): - engine = gr.Dropdown(label='Video engine', choices=list(models_def.models), value='None', elem_id="video_engine") + engine = gr.Dropdown(label='Video engine', choices=visible_engines(), value='None', elem_id="video_engine") model = gr.Dropdown(label='Video model', choices=[''], value='None', elem_id="video_model") btn_load = ToolButton(ui_symbols.loading, elem_id="video_model_load") - with gr.Row(): - url = gr.HTML(label='Model URL', elem_id='video_model_url', value='

') + url = gr.HTML(label='Model URL', elem_id='video_model_url', value='

') with gr.Accordion(open=False, label="Parameters", elem_id='video_parameters_accordion'): steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video", default_steps=50) with gr.Row(): @@ -140,6 +122,21 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la with gr.Row(): guidance_scale = gr.Slider(label='Guidance scale', minimum=-1.0, maximum=14.0, step=0.1, value=-1.0, elem_id="video_guidance_scale") guidance_true = gr.Slider(label='True guidance', minimum=-1.0, maximum=14.0, step=0.1, value=-1.0, elem_id="video_guidance_true") + with gr.Accordion(open=False, label="Size", elem_id='video_size_accordion'): + with gr.Row(): + width, height = ui_sections.create_resolution_inputs('video', default_width=832, default_height=480) + with gr.Row(): + frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=17, elem_id="video_frames") + seed = gr.Number(label='Initial seed', value=-1, elem_id="video_seed", container=True) + random_seed = ToolButton(ui_symbols.random, elem_id="video_seed_random") + reuse_seed = ToolButton(ui_symbols.reuse, elem_id="video_seed_reuse") + random_seed.click(fn=lambda: -1, show_progress='hidden', inputs=[], outputs=[seed]) + with gr.Accordion(open=False, label="Inputs", elem_id='video_inputs_accordion'): + init_strength = gr.Slider(label='Init strength', minimum=0.0, maximum=1.0, step=0.01, value=0.8, elem_id="video_denoising_strength") + gr.HTML("
  Init image") + init_image = gr.Image(elem_id="video_image", show_label=False, type="pil", image_mode="RGB", width=256, height=256) + gr.HTML("
  Last image") + last_image = gr.Image(elem_id="video_last", show_label=False, type="pil", image_mode="RGB", width=256, height=256) with gr.Accordion(open=False, label="Decode", elem_id='video_decode_accordion'): with gr.Row(): vae_type = gr.Dropdown(label='VAE decode', choices=['Default', 'Tiny', 'Remote', 'Upscale'], value='Default', elem_id="video_vae_type") @@ -155,18 +152,15 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la with gr.Tab('Frames', id='out-gallery'): gallery, gen_info, html_info, _html_info_formatted, html_log = ui_common.create_output_panel("video", prompt=prompt, preview=False, transfer=False, scale=2) - # connect reuse seed button ui_common.connect_reuse_seed(seed, reuse_seed, gen_info, is_subseed=False) - # handle engine and model change engine.change(fn=engine_change, inputs=[engine], outputs=[model]) model.change(fn=model_change, inputs=[engine, model], outputs=[url]) btn_load.click(fn=model_load, inputs=[engine, model], outputs=[html_log]) - # hidden fields + task_id = gr.Textbox(visible=False, value='') ui_state = gr.Textbox(visible=False, value='') state_inputs = [task_id, ui_state] - # generate args video_inputs = [ engine, model, prompt, negative, styles, @@ -198,4 +192,4 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la show_progress='hidden', ) generate.click(**video_dict) - return [engine, model, steps, sampler_index] + return engine, model, steps, sampler_index, width, height, frames, seed