From 5347b5701d35d9e59c08bc8b409580c6b09945b0 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 18 Apr 2025 17:42:36 -0400 Subject: [PATCH] add wan21-flv2v-720p Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 ++ modules/ui_video.py | 6 ++++-- modules/video_models/models_def.py | 6 ++++++ modules/video_models/video_overrides.py | 2 +- modules/video_models/video_run.py | 12 ++++++++++-- 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12d233b00..d1e009550 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ see [Nunchaku Wiki](https://github.com/vladmandic/sdnext/wiki/Nunchaku) for installation guide and list of supported models & features - [LTXVideo 0.9.6](https://github.com/Lightricks/LTX-Video?tab=readme-ov-file) T2V and I2V in both **Standard** and **Distilled** variants + - [WAN 2.1 FLF2V 14B 720P](https://huggingface.co/Wan-AI/Wan2.1-FLF2V-14B-720P) + new first-to-last image video model from WAN-AI - [CFG-Zero](https://github.com/WeichenFan/CFG-Zero-star) new guidance method optimized for flow-matching models implemented for **FLUX.1, HiDream-I1, SD3.x, CogView4, HunyuanVideo, WanAI** enable and configure in *settings -> pipeline modifiers -> cfg zero* diff --git a/modules/ui_video.py b/modules/ui_video.py index b53810d4b..2fd23778c 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -114,9 +114,11 @@ def create_ui(): vae_type = gr.Dropdown(label='VAE decode', choices=['Default', 'Tiny', 'Remote'], value='Default', elem_id="video_vae_type") vae_tile_frames = gr.Slider(label='Tile frames', minimum=1, maximum=64, step=1, value=16, elem_id="video_vae_tile_frames") with gr.Accordion(open=False, label="Init image", elem_id='video_init_accordion'): + init_strength = gr.Slider(label='Init strength', minimum=0.0, maximum=1.0, step=0.01, value=0.5, elem_id="video_denoising_strength") gr.HTML("
  Init image") init_image = gr.Image(elem_id="video_image", show_label=False, type="pil", image_mode="RGB", height=512) - init_strength = gr.Slider(label='Init strength', minimum=0.0, maximum=1.0, step=0.01, value=0.5, elem_id="video_denoising_strength") + gr.HTML("
  Last image") + last_image = gr.Image(elem_id="video_last", show_label=False, type="pil", image_mode="RGB", height=512) with gr.Accordion(open=True, label="Output", elem_id='video_output_accordion'): with gr.Row(): save_frames = gr.Checkbox(label='Save image frames', value=False, elem_id="video_save_frames") @@ -167,7 +169,7 @@ def create_ui(): sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, - init_image, init_strength, + init_image, init_strength, last_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index 0d471cd60..f3c69c756 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -158,6 +158,12 @@ models = { repo_cls=diffusers.WanImageToVideoPipeline, te_cls=transformers.T5EncoderModel, dit_cls=diffusers.WanTransformer3DModel), + Model(name='WAN 2.1 14B FLF2V 720p', + url='https://huggingface.co/Wan-AI/Wan2.1-FLF2V-14B-720P', + repo='Wan-AI/Wan2.1-FLF2V-14B-720P-diffusers', + repo_cls=diffusers.WanImageToVideoPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.WanTransformer3DModel), ], 'Mochi Video': [ Model(name='None'), diff --git a/modules/video_models/video_overrides.py b/modules/video_models/video_overrides.py index 4168a1441..655256e95 100644 --- a/modules/video_models/video_overrides.py +++ b/modules/video_models/video_overrides.py @@ -14,7 +14,7 @@ def load_override(selected: Model): kwargs['vae'] = diffusers.AutoencoderKLAllegro.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir) if selected.name == 'LTXVideo 0.9.5 I2V': kwargs['vae'] = diffusers.AutoencoderKLLTXVideo.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir) - if selected.name == 'WAN 2.1 14B I2V 480p' or selected.name == 'WAN 2.1 14B I2V 720p': + if 'WAN 2.1 14B' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLWan.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir) debug(f'Video overrides: model="{selected.name}" kwargs={list(kwargs)}') return kwargs diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index b0248e914..6a3e13ebb 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -8,7 +8,7 @@ debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None e def generate(*args, **kwargs): - task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, init_strength, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args + task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, init_strength, last_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args if engine is None or model is None or engine == 'None' or model == 'None': return video_utils.queue_err('model not selected') found = [model.name for model in models_def.models.get(engine, [])] @@ -57,9 +57,17 @@ def generate(*args, **kwargs): return video_utils.queue_err('init image not set') p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') shared.log.debug(f'Video: op=I2V init={init_image} resized={p.task_args["image"]}') + elif 'FLF2V' in model: + if init_image is None: + return video_utils.queue_err('init image not set') + if last_image is None: + return video_utils.queue_err('last image not set') + p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + p.task_args['last_image'] = images.resize_image(resize_mode=2, im=last_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + shared.log.debug(f'Video: op=FLF2V init={init_image} last={last_image} resized={p.task_args["image"]}') elif 'T2V' in model: if init_image is not None: - shared.log.debug('Video: op=T2V init image not supported') + shared.log.warning('Video: op=T2V init image not supported') # cleanup memory shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)