From b2432db88e71d05cdfa6644a9bb7e233b11bb9ae Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 21 Mar 2025 09:39:00 -0400 Subject: [PATCH] fix wan and add latte Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 ++ installer.py | 2 +- modules/ui_video.py | 44 ++++++++++++++++-------------- modules/video_models/models_def.py | 9 ++++++ modules/video_models/run_wan.py | 4 +-- modules/video_models/video_load.py | 3 +- wiki | 2 +- 7 files changed, 42 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d099df92..6bc01e323 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ - Video: HunyuanVideo-I2V incompatible with latest transformers - Video: LTXVideo-095 support for conditioned input - Video: LTXVideo-095 support for offloading + - Video: FasterCache: https://github.com/huggingface/diffusers/pull/10163 + - Video: PyramidAttention: https://github.com/huggingface/diffusers/pull/9562 ### Highlights for 2025-03-20 @@ -31,6 +33,7 @@ Plus support for CogView-4, new CLiP models, improvements to remote VAE, additio - [CogVideoX](https://huggingface.co/THUDM/CogVideoX-5b): *2B, 5B* | *T2V, I2V* - [Allegro](https://huggingface.co/rhymes-ai/Allegro): *T2V* - [Mochi1](https://huggingface.co/genmo/mochi-1-preview): *T2V* + - [Latte1](https://huggingface.co/maxin-cn/Latte-1): *T2V - decoding: - **Default**: use vae from model - **Tiny VAE**: support for *Hunyuan, WAN, Mochi* diff --git a/installer.py b/installer.py index 0411ef496..cacb0a62d 100644 --- a/installer.py +++ b/installer.py @@ -538,7 +538,7 @@ def check_diffusers(): t_start = time.time() if args.skip_all or args.skip_git: return - sha = '56f740051dae2d410677292a5c9e5b66e60f87dc' # diffusers commit hash + sha = '844221ae4e20a8939ee052f75874e284f75d4c5c' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/ui_video.py b/modules/ui_video.py index 81f68c50a..20edfc89e 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -26,7 +26,6 @@ def model_change(engine, model): sd_models.unload_model_weights() msg = 'Video model unloaded' return [msg, - gr.update(visible='I2V' in selected.name) if selected else gr.update(visible=False), video_utils.get_url(selected.url if selected else None), ] @@ -84,24 +83,29 @@ def create_ui(): seed = gr.Number(label='Initial seed', value=-1, elem_id="video_seed", container=True) random_seed = ToolButton(ui_symbols.random, elem_id="video_random_seed", label='Random seed') reuse_seed = ToolButton(ui_symbols.reuse, elem_id="video_reuse_seed", label='Reuse seed') - steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video") - with gr.Row(): - sampler_shift = gr.Slider(label='Sampler shift', minimum=0.0, maximum=20.0, step=0.1, value=7.0, elem_id="video_scheduler_shift") - dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift", interactive=False) # TODO video: dynamic shift - with gr.Row(): - guidance_scale = gr.Slider(label='Guidance scale', minimum=0.0, maximum=14.0, step=0.1, value=6.0, elem_id="video_guidance_scale") - guidance_true = gr.Slider(label='True guidance', minimum=0.0, maximum=14.0, step=0.1, value=1.0, elem_id="video_guidance_true") - with gr.Row(): - vae_type = gr.Dropdown(label='VAE decode', choices=['Default', 'Tiny', 'Remote'], value='Default', elem_id="video_vae_type") - vae_tile_frames = gr.Slider(label='Tile frames', minimum=1, maximum=64, step=1, value=16, elem_id="video_vae_tile_frames") - with gr.Row(): - with gr.Group(visible=False, elem_id='video_init_image') as image_group: - gr.HTML("
  Init image") - init_image = gr.Image(elem_id="video_image", show_label=False, type="pil", image_mode="RGB", height=512) - with gr.Row(): - save_frames = gr.Checkbox(label='Save image frames', value=False, elem_id="video_save_frames") - with gr.Row(): - video_type, video_duration, video_loop, video_pad, video_interpolate = ui_sections.create_video_inputs(tab='video') + with gr.Accordion(open=True, label="Parameters", elem_id='video_parameters_accordion'): + steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video") + with gr.Row(): + sampler_shift = gr.Slider(label='Sampler shift', minimum=0.0, maximum=20.0, step=0.1, value=7.0, elem_id="video_scheduler_shift") + dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift", interactive=False) # TODO video: dynamic shift + with gr.Row(): + guidance_scale = gr.Slider(label='Guidance scale', minimum=0.0, maximum=14.0, step=0.1, value=6.0, elem_id="video_guidance_scale") + guidance_true = gr.Slider(label='True guidance', minimum=0.0, maximum=14.0, step=0.1, value=1.0, elem_id="video_guidance_true") + with gr.Accordion(open=True, label="Decode", elem_id='video_decode_accordion'): + with gr.Row(): + vae_type = gr.Dropdown(label='VAE decode', choices=['Default', 'Tiny', 'Remote'], value='Default', elem_id="video_vae_type") + vae_tile_frames = gr.Slider(label='Tile frames', minimum=1, maximum=64, step=1, value=16, elem_id="video_vae_tile_frames") + with gr.Accordion(open=False, label="Init image", elem_id='video_init_accordion'): + gr.HTML("
  Init image") + init_image = gr.Image(elem_id="video_image", show_label=False, type="pil", image_mode="RGB", height=512) + with gr.Accordion(open=False, label="Accelerate", elem_id='video_accelerate_accordion'): + faster_cache = gr.Checkbox(label='FasterCache', value=False, elem_id="video_faster_cache") + pyramid_attention = gr.Checkbox(label='PyramidAttention', value=False, elem_id="video_pyramid_attention") + with gr.Accordion(open=True, label="Output", elem_id='video_output_accordion'): + with gr.Row(): + save_frames = gr.Checkbox(label='Save image frames', value=False, elem_id="video_save_frames") + with gr.Row(): + video_type, video_duration, video_loop, video_pad, video_interpolate = ui_sections.create_video_inputs(tab='video') override_settings = ui_common.create_override_inputs('video') # output panel with gallery and video tabs @@ -117,7 +121,7 @@ def create_ui(): random_seed.click(fn=lambda: -1, show_progress=False, inputs=[], outputs=[seed]) # handle engine and model change engine.change(fn=engine_change, inputs=[engine], outputs=[model]) - model.change(fn=model_change, inputs=[engine, model], outputs=[html_log, image_group, url]) + model.change(fn=model_change, inputs=[engine, model], outputs=[html_log, url]) # setup extra networks ui_extra_networks.setup_ui(extra_networks_ui, gallery) diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index 5ff58fd82..4395eb817 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -139,6 +139,15 @@ models = { te_cls=transformers.T5EncoderModel, dit_cls=diffusers.MochiTransformer3DModel), ], + 'Latte Video': [ + Model(name='None'), + Model(name='Latte 1 T2V', + url='https://huggingface.co/maxin-cn/Latte-1', + repo='maxin-cn/Latte-1', + repo_cls=diffusers.LattePipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LatteTransformer3DModel), + ], 'Allegro Video': [ Model(name='None'), Model(name='Allegro T2V', diff --git a/modules/video_models/run_wan.py b/modules/video_models/run_wan.py index 7868aa020..fd50c1fc5 100644 --- a/modules/video_models/run_wan.py +++ b/modules/video_models/run_wan.py @@ -13,9 +13,9 @@ def generate(*args, **kwargs): return video_utils.queue_err('model not selected') found = [model.name for model in models_def.models.get(engine, [])] selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None - if not shared.sd_loaded or 'WAN' not in shared.sd_model.__class__.__name__: + if not shared.sd_loaded or 'Wan' not in shared.sd_model.__class__.__name__: video_load.load_model(selected) - if not shared.sd_loaded or 'WAN' not in shared.sd_model.__class__.__name__: + if not shared.sd_loaded or 'Wan' not in shared.sd_model.__class__.__name__: return video_utils.queue_err('model not loaded') debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') diff --git a/modules/video_models/video_load.py b/modules/video_models/video_load.py index 55016cfc9..f380a8d88 100644 --- a/modules/video_models/video_load.py +++ b/modules/video_models/video_load.py @@ -73,7 +73,8 @@ def load_model(selected: models_def.Model): if selected.te_hijack: shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt shared.sd_model.encode_prompt = video_utils.hijack_encode_prompt - shared.sd_model.vae.enable_slicing() + if hasattr(shared.sd_model.vae, 'enable_slicing'): + shared.sd_model.vae.enable_slicing() loaded_model = selected.name msg = f'Video load: cls={shared.sd_model.__class__.__name__} model="{selected.name}" time={t1-t0:.2f}' shared.log.info(msg) diff --git a/wiki b/wiki index d50882dcb..3f46b4f74 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit d50882dcb83a1441591b0d491efd143e12c1930a +Subproject commit 3f46b4f742e439dee1d012c9e5e2ddf2a6b79aa6