Merge pull request #4970 from vladmandic/feat/video-models

Feat/video models

modules/ltx/ltx_process.py

@@ -234,8 +234,6 @@ def run_ltx(task_id,
         condition_images = []
         if ltx_init_image is not None:
             condition_images.append(ltx_init_image)
-        if condition_last is not None:
-            condition_images.append(condition_last)
         conditions = []
         conditions_stage2 = []
         if caps.supports_multi_condition:
@@ -246,14 +244,14 @@ def run_ltx(task_id,
                 base_w, base_h, condition_strength,
                 condition_images, condition_files, condition_video,
                 condition_video_frames, condition_video_skip,
-                family=caps.family,
+                family=caps.family, num_frames=get_frames(frames), condition_last=condition_last,
             )
             if (final_w, final_h) != (base_w, base_h):
                 conditions_stage2 = get_conditions(
                     final_w, final_h, condition_strength,
                     condition_images, condition_files, condition_video,
                     condition_video_frames, condition_video_skip,
-                    family=caps.family,
+                    family=caps.family, num_frames=get_frames(frames), condition_last=condition_last,
                 )
             else:
                 conditions_stage2 = conditions

modules/ltx/ltx_ui.py

@@ -16,6 +16,7 @@ def _model_change(model_name: str):
         return (
             gr.update(visible=False),   # input_media_accordion
             gr.update(visible=False),   # multi_condition_group
+            gr.update(visible=False),   # last_image
             gr.update(visible=False),   # upsample_accordion
             gr.update(visible=False),   # refine_accordion
             gr.update(value=False),     # upsample_enable (reset)
@@ -38,6 +39,7 @@ def _model_change(model_name: str):
     return (
         gr.update(visible=caps.supports_input_media),
         gr.update(visible=caps.supports_multi_condition),
+        gr.update(visible=caps.supports_multi_condition),  # last_image
         gr.update(visible=True),
         gr.update(visible=True),
         gr.update(value=False),
@@ -73,7 +75,7 @@ def create_ui(prompt, negative, styles, overrides, mp4_fps, mp4_interpolate, mp4
                 ltx_init_image = gr.Image(label='Image', elem_id='ltx_init_image', type='pil', image_mode='RGB', width=256, height=256)
                 ltx_condition_strength = gr.Slider(label='LTX input strength', minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id='ltx_condition_strength')
                 with gr.Row():
-                    last_image = gr.Image(label='Last image', elem_id='ltx_last_image', type='pil', image_mode='RGB', width=256, height=256)
+                    last_image = gr.Image(label='Last image', elem_id='ltx_last_image', type='pil', image_mode='RGB', width=256, height=256, visible=False)
                 multi_condition_group = gr.Group(visible=False)
                 with multi_condition_group:
                     gr.Markdown('**Prefix conditioning**: supply a video or gallery to anchor the opening frames', elem_id='ltx_prefix_conditioning_label')
@@ -123,6 +125,7 @@ def create_ui(prompt, negative, styles, overrides, mp4_fps, mp4_interpolate, mp4
         outputs=[
             input_media_accordion,
             multi_condition_group,
+            last_image,
             upsample_accordion,
             refine_accordion,
             upsample_enable,

modules/ltx/ltx_util.py

@@ -138,15 +138,15 @@ def _condition_cls(family: str):
     return LTXVideoCondition
 
 
-def make_condition(condition_cls, family: str, frames, strength: float, is_video: bool):
+def make_condition(condition_cls, family: str, frames, strength: float, is_video: bool, index: int = 0):
     if family == '2.x':
-        return condition_cls(frames=frames, index=0, strength=strength)
+        return condition_cls(frames=frames, index=index, strength=strength)
     if is_video:
-        return condition_cls(video=frames, frame_index=0, strength=strength)
-    return condition_cls(image=frames, frame_index=0, strength=strength)
+        return condition_cls(video=frames, frame_index=index, strength=strength)
+    return condition_cls(image=frames, frame_index=index, strength=strength)
 
 
-def get_conditions(width, height, condition_strength, condition_images, condition_files, condition_video, condition_video_frames, condition_video_skip, family: str = '0.9'):
+def get_conditions(width, height, condition_strength, condition_images, condition_files, condition_video, condition_video_frames, condition_video_skip, family: str = '0.9', num_frames=None, condition_last=None):
     condition_cls = _condition_cls(family)
     if condition_cls is None:
         return []
@@ -186,6 +186,18 @@ def get_conditions(width, height, condition_strength, condition_images, conditio
                 log.debug(f'Video condition: family={family} frames={len(condition_frames)} size={condition_frames[0].size} strength={condition_strength}')
         except Exception as e:
             log.error(f'LTX condition video: {e}')
+    if condition_last is not None:
+        try:
+            if isinstance(condition_last, str):
+                from modules.api.api import decode_base64_to_image
+                condition_last = decode_base64_to_image(condition_last)
+            condition_last = condition_last.convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS)
+            # 2.x reads index as a latent index and accepts -1 for the final frame; 0.9 uses a pixel index.
+            last_index = -1 if family == '2.x' else max((num_frames or 1) - 1, 0)
+            conditions.append(make_condition(condition_cls, family, condition_last, condition_strength, is_video=False, index=last_index))
+            log.debug(f'Video condition: family={family} last={condition_last.size} index={last_index} strength={condition_strength}')
+        except Exception as e:
+            log.error(f'LTX condition last image: {e}')
     return conditions
 
 

modules/processing_args.py

@@ -211,6 +211,15 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:l
     if hasattr(model, 'pipe') and not hasattr(model, 'no_recurse'): # recurse
         model = model.pipe
         has_vae = has_vae or hasattr(model, 'vae')
+    # Wan 2.2 MoE: apply the high/low-noise expert boundary at generation time so it is tunable without
+    # a reload. Both experts resident (transformer + transformer_2) is the combined stage; single-expert
+    # stages keep their load-time boundary. -1 means use the value the checkpoint shipped with.
+    if getattr(model, 'transformer', None) is not None and getattr(model, 'transformer_2', None) is not None and getattr(getattr(model, 'config', None), 'boundary_ratio', None) is not None and hasattr(model, 'register_to_config'):
+        if not hasattr(model, 'wan_boundary_default'):
+            model.wan_boundary_default = model.config.boundary_ratio
+        boundary_target = shared.opts.model_wan_boundary if shared.opts.model_wan_boundary >= 0 else model.wan_boundary_default
+        if boundary_target is not None and model.config.boundary_ratio != boundary_target:
+            model.register_to_config(boundary_ratio=boundary_target)
     if hasattr(model, "set_progress_bar_config"):
         if disable_pbar:
             model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba', disable=disable_pbar)

modules/ui_definitions.py

@@ -126,7 +126,7 @@ def create_settings(cmd_opts):
         "model_h1_llama_repo": OptionInfo("Default", "LLama repo", gr.Textbox),
         "model_wan_sep": OptionInfo("<h2>WanAI</h2>", "", gr.HTML),
         "model_wan_stage": OptionInfo("low noise", "Processing stage", gr.Radio, {"choices": ['high noise', 'low noise', 'combined'] }),
-        "model_wan_boundary": OptionInfo(0.85, "Stage boundary ratio", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.05 }),
+        "model_wan_boundary": OptionInfo(-1, "Stage boundary ratio", gr.Slider, {"minimum": -1, "maximum": 1.0, "step": 0.05 }),
         "model_chrono_sep": OptionInfo("<h2>ChronoEdit</h2>", "", gr.HTML),
         "model_chrono_temporal_steps": OptionInfo(0, "Temporal steps", gr.Slider, {"minimum": 0, "maximum": 50, "step": 1 }),
         "model_qwen_layer_sep": OptionInfo("<h2>Qwen layered</h2>", "", gr.HTML),

modules/video_models/video_overrides.py

@@ -52,9 +52,9 @@ def load_override(selected: Model, **load_args):
     if 'WAN 2.1 14B' in selected.name:
         kwargs['vae'] = diffusers.AutoencoderKLWan.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir, **load_args)
     if ('A14B' in selected.name) or ('14B VACE' in selected.name):
-        if shared.opts.model_wan_stage == 'combined':
-            kwargs['boundary_ratio'] = shared.opts.model_wan_boundary
-        elif shared.opts.model_wan_stage == 'high noise':
+        # combined keeps both experts loaded and tunes boundary_ratio at runtime (set_pipeline_args), so
+        # it is not set here; only the single-expert stages need load time because they drop a transformer.
+        if shared.opts.model_wan_stage == 'high noise':
             kwargs['transformer_2'] = None
             kwargs['boundary_ratio'] = 0.0
         elif shared.opts.model_wan_stage == 'low noise':

modules/video_models/video_run.py

@@ -67,7 +67,13 @@ def generate(*args, **kwargs):
         if init_image is None:
             return video_utils.queue_err('No input image provided. Please upload or select an image.')
         p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
-        log.debug(f'Video: op=I2V init={init_image} resized={p.task_args["image"]}')
+        if last_image is not None and video_utils.supports_last_frame(shared.sd_model):
+            p.task_args['last_image'] = images.resize_image(resize_mode=2, im=last_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
+            log.debug(f'Video: op=FLF2V init={init_image} last={last_image} resized={p.task_args["image"]}')
+        elif last_image is not None:
+            log.warning(f'Video: op=I2V model="{model}" last frame not supported, ignoring')
+        else:
+            log.debug(f'Video: op=I2V init={init_image} resized={p.task_args["image"]}')
     elif 'FLF2V' in model:
         if init_image is None:
             return video_utils.queue_err('No input image provided. Please upload or select an image.')

modules/video_models/video_utils.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import time
+import inspect
 from PIL import Image
 from installer import install
 from modules import shared, sd_models, timer, errors, devices
@@ -19,6 +20,18 @@ def get_url(url):
     return f'<a href="{url}" target="_blank" rel="noopener noreferrer" class="video-model-link">{url}</a><br><br>' if url else '<br><br>'
 
 
+def supports_last_frame(model):
+    # last-frame (FLF2V) conditioning needs a pipeline whose __call__ accepts `last_image`.
+    # wan 2.2 5b accepts the arg but masks timesteps from the first frame only, so it drops the last frame.
+    try:
+        params = list(inspect.signature(type(model).__call__, follow_wrapped=True).parameters)
+    except (ValueError, TypeError):
+        return False
+    if 'last_image' not in params:
+        return False
+    return not getattr(getattr(model, 'config', None), 'expand_timesteps', False)
+
+
 def check_av():
     install('av')
     try:

pipelines/model_wanai.py

@@ -43,7 +43,8 @@ def load_wan(checkpoint_info, diffusers_load_config=None):
         elif shared.opts.model_wan_stage == 'combined' or shared.opts.model_wan_stage == 'both':
             transformer = generic.load_transformer(repo_id, cls_name=transformer_cls, load_config=diffusers_load_config, subfolder='transformer')
             transformer_2 = generic.load_transformer(repo_id, cls_name=transformer_cls, load_config=diffusers_load_config, subfolder='transformer_2')
-            boundary_ratio = shared.opts.model_wan_boundary
+            # load with the checkpoint's boundary; the slider override is applied at runtime in set_pipeline_args
+            boundary_ratio = None
         else:
             log.error(f'Load model: type=WanAI stage="{shared.opts.model_wan_stage}" unsupported')
             return None
@@ -71,15 +72,16 @@ def load_wan(checkpoint_info, diffusers_load_config=None):
         diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["wanai"] = diffusers.WanPipeline
         diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["wanai"] = WanImagePipeline
     log.debug(f'Load model: type=WanAI model="{checkpoint_info.name}" repo="{repo_id}" cls={pipe_cls.__name__} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args} stage="{shared.opts.model_wan_stage}" boundary={boundary_ratio}')
-    pipe = pipe_cls.from_pretrained(
-        repo_id,
-        transformer=transformer,
-        transformer_2=transformer_2,
-        text_encoder=text_encoder,
-        boundary_ratio=boundary_ratio,
-        cache_dir=shared.opts.diffusers_dir,
+    wan_args = {
+        'transformer': transformer,
+        'transformer_2': transformer_2,
+        'text_encoder': text_encoder,
+        'cache_dir': shared.opts.diffusers_dir,
         **load_args,
-    )
+    }
+    if boundary_ratio is not None: # omit so from_pretrained keeps the checkpoint's shipped boundary_ratio
+        wan_args['boundary_ratio'] = boundary_ratio
+    pipe = pipe_cls.from_pretrained(repo_id, **wan_args)
     pipe.task_args = {
         'num_frames': 1,
         'output_type': 'np',

ui/locale/locale_en.json

@@ -1375,7 +1375,7 @@
     {"id":"","label":"Search Docs","localized":"","hint":"","ui":"system_tab_docs"},
     {"id":"","label":"Search GitHub Wiki Pages","localized":"","hint":"","ui":"system_tab_wiki"},
     {"id":"","label":"Search Changelog","localized":"","hint":"","ui":"system_tab_changelog"},
-    {"id":"","label":"Stage boundary ratio","localized":"","hint":"","ui":"settings_model_options"},
+    {"id":"","label":"Stage boundary ratio","localized":"","hint":"Timestep fraction at which the Wan 2.2 A14B mixture-of-experts hands off from the high-noise expert (coarse layout and motion) to the low-noise expert (detail and refinement). Lower values keep the high-noise expert running longer; values that are too low leave the result under-refined.<br><br>-1 uses the boundary the checkpoint shipped with and is recommended; 0 to 1 set it explicitly. Affects the combined stage only.<br>Default -1.","ui":"settings_model_options"},
     {"id":"","label":"sequential","localized":"","hint":"","ui":"settings_offload"},
     {"id":"","label":"SVD rank size","localized":"","hint":"Rank of the low-rank correction added by <b><i>Use SVD quantization</i></b>. Higher ranks recover more accuracy but add parameters and compute.<br><br>Applies only when <b><i>Use SVD quantization</i></b> is enabled.<br><br>Default is <b>32</b>.","reload":"model","ui":"settings_quantization"},
     {"id":"","label":"SVD steps","localized":"","hint":"Number of iterations used to estimate the low-rank correction for <b><i>Use SVD quantization</i></b>. More steps refine the estimate at the cost of longer quantization.<br><br>Applies only when <b><i>Use SVD quantization</i></b> is enabled.<br><br>Default is <b>8</b>.","reload":"model","ui":"settings_quantization"},