diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index d108a8fc1..e7bb3ce2a 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -234,8 +234,6 @@ def run_ltx(task_id, condition_images = [] if ltx_init_image is not None: condition_images.append(ltx_init_image) - if condition_last is not None: - condition_images.append(condition_last) conditions = [] conditions_stage2 = [] if caps.supports_multi_condition: @@ -246,14 +244,14 @@ def run_ltx(task_id, base_w, base_h, condition_strength, condition_images, condition_files, condition_video, condition_video_frames, condition_video_skip, - family=caps.family, + family=caps.family, num_frames=get_frames(frames), condition_last=condition_last, ) if (final_w, final_h) != (base_w, base_h): conditions_stage2 = get_conditions( final_w, final_h, condition_strength, condition_images, condition_files, condition_video, condition_video_frames, condition_video_skip, - family=caps.family, + family=caps.family, num_frames=get_frames(frames), condition_last=condition_last, ) else: conditions_stage2 = conditions diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index e5b2f1582..4990a5796 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -16,6 +16,7 @@ def _model_change(model_name: str): return ( gr.update(visible=False), # input_media_accordion gr.update(visible=False), # multi_condition_group + gr.update(visible=False), # last_image gr.update(visible=False), # upsample_accordion gr.update(visible=False), # refine_accordion gr.update(value=False), # upsample_enable (reset) @@ -38,6 +39,7 @@ def _model_change(model_name: str): return ( gr.update(visible=caps.supports_input_media), gr.update(visible=caps.supports_multi_condition), + gr.update(visible=caps.supports_multi_condition), # last_image gr.update(visible=True), gr.update(visible=True), gr.update(value=False), @@ -73,7 +75,7 @@ def create_ui(prompt, negative, styles, overrides, mp4_fps, mp4_interpolate, mp4 ltx_init_image = gr.Image(label='Image', elem_id='ltx_init_image', type='pil', image_mode='RGB', width=256, height=256) ltx_condition_strength = gr.Slider(label='LTX input strength', minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id='ltx_condition_strength') with gr.Row(): - last_image = gr.Image(label='Last image', elem_id='ltx_last_image', type='pil', image_mode='RGB', width=256, height=256) + last_image = gr.Image(label='Last image', elem_id='ltx_last_image', type='pil', image_mode='RGB', width=256, height=256, visible=False) multi_condition_group = gr.Group(visible=False) with multi_condition_group: gr.Markdown('**Prefix conditioning**: supply a video or gallery to anchor the opening frames', elem_id='ltx_prefix_conditioning_label') @@ -123,6 +125,7 @@ def create_ui(prompt, negative, styles, overrides, mp4_fps, mp4_interpolate, mp4 outputs=[ input_media_accordion, multi_condition_group, + last_image, upsample_accordion, refine_accordion, upsample_enable, diff --git a/modules/ltx/ltx_util.py b/modules/ltx/ltx_util.py index f1ef78391..8aa0b12ec 100644 --- a/modules/ltx/ltx_util.py +++ b/modules/ltx/ltx_util.py @@ -138,15 +138,15 @@ def _condition_cls(family: str): return LTXVideoCondition -def make_condition(condition_cls, family: str, frames, strength: float, is_video: bool): +def make_condition(condition_cls, family: str, frames, strength: float, is_video: bool, index: int = 0): if family == '2.x': - return condition_cls(frames=frames, index=0, strength=strength) + return condition_cls(frames=frames, index=index, strength=strength) if is_video: - return condition_cls(video=frames, frame_index=0, strength=strength) - return condition_cls(image=frames, frame_index=0, strength=strength) + return condition_cls(video=frames, frame_index=index, strength=strength) + return condition_cls(image=frames, frame_index=index, strength=strength) -def get_conditions(width, height, condition_strength, condition_images, condition_files, condition_video, condition_video_frames, condition_video_skip, family: str = '0.9'): +def get_conditions(width, height, condition_strength, condition_images, condition_files, condition_video, condition_video_frames, condition_video_skip, family: str = '0.9', num_frames=None, condition_last=None): condition_cls = _condition_cls(family) if condition_cls is None: return [] @@ -186,6 +186,18 @@ def get_conditions(width, height, condition_strength, condition_images, conditio log.debug(f'Video condition: family={family} frames={len(condition_frames)} size={condition_frames[0].size} strength={condition_strength}') except Exception as e: log.error(f'LTX condition video: {e}') + if condition_last is not None: + try: + if isinstance(condition_last, str): + from modules.api.api import decode_base64_to_image + condition_last = decode_base64_to_image(condition_last) + condition_last = condition_last.convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) + # 2.x reads index as a latent index and accepts -1 for the final frame; 0.9 uses a pixel index. + last_index = -1 if family == '2.x' else max((num_frames or 1) - 1, 0) + conditions.append(make_condition(condition_cls, family, condition_last, condition_strength, is_video=False, index=last_index)) + log.debug(f'Video condition: family={family} last={condition_last.size} index={last_index} strength={condition_strength}') + except Exception as e: + log.error(f'LTX condition last image: {e}') return conditions diff --git a/modules/processing_args.py b/modules/processing_args.py index befb24d54..36872e2e5 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -211,6 +211,15 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:l if hasattr(model, 'pipe') and not hasattr(model, 'no_recurse'): # recurse model = model.pipe has_vae = has_vae or hasattr(model, 'vae') + # Wan 2.2 MoE: apply the high/low-noise expert boundary at generation time so it is tunable without + # a reload. Both experts resident (transformer + transformer_2) is the combined stage; single-expert + # stages keep their load-time boundary. -1 means use the value the checkpoint shipped with. + if getattr(model, 'transformer', None) is not None and getattr(model, 'transformer_2', None) is not None and getattr(getattr(model, 'config', None), 'boundary_ratio', None) is not None and hasattr(model, 'register_to_config'): + if not hasattr(model, 'wan_boundary_default'): + model.wan_boundary_default = model.config.boundary_ratio + boundary_target = shared.opts.model_wan_boundary if shared.opts.model_wan_boundary >= 0 else model.wan_boundary_default + if boundary_target is not None and model.config.boundary_ratio != boundary_target: + model.register_to_config(boundary_ratio=boundary_target) if hasattr(model, "set_progress_bar_config"): if disable_pbar: model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba', disable=disable_pbar) diff --git a/modules/ui_definitions.py b/modules/ui_definitions.py index dc9406034..1eccfccb7 100644 --- a/modules/ui_definitions.py +++ b/modules/ui_definitions.py @@ -126,7 +126,7 @@ def create_settings(cmd_opts): "model_h1_llama_repo": OptionInfo("Default", "LLama repo", gr.Textbox), "model_wan_sep": OptionInfo("

WanAI

", "", gr.HTML), "model_wan_stage": OptionInfo("low noise", "Processing stage", gr.Radio, {"choices": ['high noise', 'low noise', 'combined'] }), - "model_wan_boundary": OptionInfo(0.85, "Stage boundary ratio", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.05 }), + "model_wan_boundary": OptionInfo(-1, "Stage boundary ratio", gr.Slider, {"minimum": -1, "maximum": 1.0, "step": 0.05 }), "model_chrono_sep": OptionInfo("

ChronoEdit

", "", gr.HTML), "model_chrono_temporal_steps": OptionInfo(0, "Temporal steps", gr.Slider, {"minimum": 0, "maximum": 50, "step": 1 }), "model_qwen_layer_sep": OptionInfo("

Qwen layered

", "", gr.HTML), diff --git a/modules/video_models/video_overrides.py b/modules/video_models/video_overrides.py index 028e544ba..6d918b617 100644 --- a/modules/video_models/video_overrides.py +++ b/modules/video_models/video_overrides.py @@ -52,9 +52,9 @@ def load_override(selected: Model, **load_args): if 'WAN 2.1 14B' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLWan.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir, **load_args) if ('A14B' in selected.name) or ('14B VACE' in selected.name): - if shared.opts.model_wan_stage == 'combined': - kwargs['boundary_ratio'] = shared.opts.model_wan_boundary - elif shared.opts.model_wan_stage == 'high noise': + # combined keeps both experts loaded and tunes boundary_ratio at runtime (set_pipeline_args), so + # it is not set here; only the single-expert stages need load time because they drop a transformer. + if shared.opts.model_wan_stage == 'high noise': kwargs['transformer_2'] = None kwargs['boundary_ratio'] = 0.0 elif shared.opts.model_wan_stage == 'low noise': diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 569220a45..93a666480 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -67,7 +67,13 @@ def generate(*args, **kwargs): if init_image is None: return video_utils.queue_err('No input image provided. Please upload or select an image.') p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') - log.debug(f'Video: op=I2V init={init_image} resized={p.task_args["image"]}') + if last_image is not None and video_utils.supports_last_frame(shared.sd_model): + p.task_args['last_image'] = images.resize_image(resize_mode=2, im=last_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + log.debug(f'Video: op=FLF2V init={init_image} last={last_image} resized={p.task_args["image"]}') + elif last_image is not None: + log.warning(f'Video: op=I2V model="{model}" last frame not supported, ignoring') + else: + log.debug(f'Video: op=I2V init={init_image} resized={p.task_args["image"]}') elif 'FLF2V' in model: if init_image is None: return video_utils.queue_err('No input image provided. Please upload or select an image.') diff --git a/modules/video_models/video_utils.py b/modules/video_models/video_utils.py index 18ae1249d..090b850e8 100644 --- a/modules/video_models/video_utils.py +++ b/modules/video_models/video_utils.py @@ -1,6 +1,7 @@ import os import sys import time +import inspect from PIL import Image from installer import install from modules import shared, sd_models, timer, errors, devices @@ -19,6 +20,18 @@ def get_url(url): return f'{url}

' if url else '

' +def supports_last_frame(model): + # last-frame (FLF2V) conditioning needs a pipeline whose __call__ accepts `last_image`. + # wan 2.2 5b accepts the arg but masks timesteps from the first frame only, so it drops the last frame. + try: + params = list(inspect.signature(type(model).__call__, follow_wrapped=True).parameters) + except (ValueError, TypeError): + return False + if 'last_image' not in params: + return False + return not getattr(getattr(model, 'config', None), 'expand_timesteps', False) + + def check_av(): install('av') try: diff --git a/pipelines/model_wanai.py b/pipelines/model_wanai.py index 0884c059e..2a7a421ea 100644 --- a/pipelines/model_wanai.py +++ b/pipelines/model_wanai.py @@ -43,7 +43,8 @@ def load_wan(checkpoint_info, diffusers_load_config=None): elif shared.opts.model_wan_stage == 'combined' or shared.opts.model_wan_stage == 'both': transformer = generic.load_transformer(repo_id, cls_name=transformer_cls, load_config=diffusers_load_config, subfolder='transformer') transformer_2 = generic.load_transformer(repo_id, cls_name=transformer_cls, load_config=diffusers_load_config, subfolder='transformer_2') - boundary_ratio = shared.opts.model_wan_boundary + # load with the checkpoint's boundary; the slider override is applied at runtime in set_pipeline_args + boundary_ratio = None else: log.error(f'Load model: type=WanAI stage="{shared.opts.model_wan_stage}" unsupported') return None @@ -71,15 +72,16 @@ def load_wan(checkpoint_info, diffusers_load_config=None): diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["wanai"] = diffusers.WanPipeline diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["wanai"] = WanImagePipeline log.debug(f'Load model: type=WanAI model="{checkpoint_info.name}" repo="{repo_id}" cls={pipe_cls.__name__} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args} stage="{shared.opts.model_wan_stage}" boundary={boundary_ratio}') - pipe = pipe_cls.from_pretrained( - repo_id, - transformer=transformer, - transformer_2=transformer_2, - text_encoder=text_encoder, - boundary_ratio=boundary_ratio, - cache_dir=shared.opts.diffusers_dir, + wan_args = { + 'transformer': transformer, + 'transformer_2': transformer_2, + 'text_encoder': text_encoder, + 'cache_dir': shared.opts.diffusers_dir, **load_args, - ) + } + if boundary_ratio is not None: # omit so from_pretrained keeps the checkpoint's shipped boundary_ratio + wan_args['boundary_ratio'] = boundary_ratio + pipe = pipe_cls.from_pretrained(repo_id, **wan_args) pipe.task_args = { 'num_frames': 1, 'output_type': 'np', diff --git a/ui/locale/locale_en.json b/ui/locale/locale_en.json index 88fd21dad..8ec96e8d4 100644 --- a/ui/locale/locale_en.json +++ b/ui/locale/locale_en.json @@ -1375,7 +1375,7 @@ {"id":"","label":"Search Docs","localized":"","hint":"","ui":"system_tab_docs"}, {"id":"","label":"Search GitHub Wiki Pages","localized":"","hint":"","ui":"system_tab_wiki"}, {"id":"","label":"Search Changelog","localized":"","hint":"","ui":"system_tab_changelog"}, - {"id":"","label":"Stage boundary ratio","localized":"","hint":"","ui":"settings_model_options"}, + {"id":"","label":"Stage boundary ratio","localized":"","hint":"Timestep fraction at which the Wan 2.2 A14B mixture-of-experts hands off from the high-noise expert (coarse layout and motion) to the low-noise expert (detail and refinement). Lower values keep the high-noise expert running longer; values that are too low leave the result under-refined.

-1 uses the boundary the checkpoint shipped with and is recommended; 0 to 1 set it explicitly. Affects the combined stage only.
Default -1.","ui":"settings_model_options"}, {"id":"","label":"sequential","localized":"","hint":"","ui":"settings_offload"}, {"id":"","label":"SVD rank size","localized":"","hint":"Rank of the low-rank correction added by Use SVD quantization. Higher ranks recover more accuracy but add parameters and compute.

Applies only when Use SVD quantization is enabled.

Default is 32.","reload":"model","ui":"settings_quantization"}, {"id":"","label":"SVD steps","localized":"","hint":"Number of iterations used to estimate the low-rank correction for Use SVD quantization. More steps refine the estimate at the cost of longer quantization.

Applies only when Use SVD quantization is enabled.

Default is 8.","reload":"model","ui":"settings_quantization"},