video tab major update

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log for SD.Next
 
-## Update for 2025-03-19
+## Update for 2025-03-21
 
 ### ToDo/Limitations
 
@@ -12,15 +12,19 @@
   - Video: HunyuanVideo-I2V incompatible with latest transformers <https://github.com/huggingface/diffusers/issues/11118>  
   - Video: LTXVideo-095 support for conditioned input  
   - Video: LTXVideo-095 support for offloading  
-  - Video: FasterCache: https://github.com/huggingface/diffusers/pull/10163
-  - Video: PyramidAttention: https://github.com/huggingface/diffusers/pull/9562
+  - Video: FasterCache and PyramidAttentionBroadcast granular config  
+  - Video: FasterCache and PyramidAttentionBroadcast for LTX and WAN <https://github.com/huggingface/diffusers/issues/11134>  
+  - Video: HunyuanVideo-I2V-16ch <https://github.com/huggingface/diffusers/pull/11066>
+  - Video: CogVideo-15 support
 
-### Highlights for 2025-03-20
+### Highlights for 2025-03-21
 
-Brand new Video processing module with support for all latest models: **WAN21, Hunyuan, LTX, Cog, Allegro, Mochi1** and more!  
-Plus support for CogView-4, new CLiP models, improvements to remote VAE, additional docs/guides.
+Brand new Video processing module with support for all latest models: **WAN21, Hunyuan, LTX, Cog, Allegro, Mochi1, Latte1**  
+And combined with *on-the-fly quantization*, support for *Local/Tiny/Remote* VAE, acceleration modules such as *FasterCache or PAB* and more!  
 
-### Details for 2025-03-20
+Plus support for CogView-4, new CLiP models, improvements to remote VAE, additional docs/guides  
+
+### Details for 2025-03-21
 
 - **Video tab**
   - initial release so consider this as alpha version    
@@ -39,6 +43,9 @@ Plus support for CogView-4, new CLiP models, improvements to remote VAE, additio
     - **Tiny VAE**: support for *Hunyuan, WAN, Mochi*  
     - **Remote VAE**: support for *Hunyuan*  
   - **LoRA**: support for *Hunyuan, LTX, WAN, Mochi, Cog*  
+  - acceleration:  
+    - [FasterCache](https://huggingface.co/papers/2410.19355): support for *Hunyuan, Mochi, Latte, Allegro, Cog*  
+    - [PyramidAttentionBroadcast](https://huggingface.co/papers/2408.12588): support for *Hunyuan, Mochi, Latte, Allegro, Cog*  
   - additional key points:  
     - all models are auto-downloaded upon first use  
       uses *system paths -> huggingface* folder  

modules/processing_args.py

@@ -26,7 +26,7 @@ def task_specific_kwargs(p, model):
             p.init_images = [helpers.decode_base64_to_image(i, quiet=True) for i in p.init_images]
         if isinstance(p.init_images[0], Image.Image):
             p.init_images = [i.convert('RGB') if i.mode != 'RGB' else i for i in p.init_images if i is not None]
-    if (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE or len(getattr(p, 'init_images', [])) == 0) and not is_img2img_model:
+    if (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE or len(getattr(p, 'init_images', [])) == 0) and not is_img2img_model and 'video' not in p.ops:
         p.ops.append('txt2img')
         if hasattr(p, 'width') and hasattr(p, 'height'):
             task_args = {
@@ -238,13 +238,13 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t
     if hasattr(model, 'scheduler') and hasattr(model.scheduler, 'noise_sampler_seed') and hasattr(model.scheduler, 'noise_sampler'):
         model.scheduler.noise_sampler = None # noise needs to be reset instead of using cached values
         model.scheduler.noise_sampler_seed = p.seeds # some schedulers have internal noise generator and do not use pipeline generator
-    if 'seed' in possible:
+    if 'seed' in possible and p.seed is not None:
         args['seed'] = p.seed
-    if 'noise_sampler_seed' in possible:
+    if 'noise_sampler_seed' in possible and p.seeds is not None:
         args['noise_sampler_seed'] = p.seeds
-    if 'guidance_scale' in possible:
+    if 'guidance_scale' in possible and p.cfg_scale is not None and p.cfg_scale > 0:
         args['guidance_scale'] = p.cfg_scale
-    if 'img_guidance_scale' in possible and hasattr(p, 'image_cfg_scale'):
+    if 'img_guidance_scale' in possible and hasattr(p, 'image_cfg_scale') and p.image_cfg_scale is not None and p.image_cfg_scale > 0:
         args['img_guidance_scale'] = p.image_cfg_scale
     if 'generator' in possible:
         generator = get_generator(p)
@@ -304,9 +304,10 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t
     # handle remaining args
     for arg in kwargs:
         if arg in possible: # add kwargs
+            if type(kwargs[arg]) == float or type(kwargs[arg]) == int:
+                if kwargs[arg] <= -1: # skip -1 as default value
+                    continue
             args[arg] = kwargs[arg]
-        else:
-            pass
 
     task_kwargs = task_specific_kwargs(p, model)
     for arg in task_kwargs:

modules/processing_class.py

@@ -236,9 +236,13 @@ class StableDiffusionProcessing:
             self.height = firstphase_height
         self.sampler_name = sampler_name or processing_helpers.get_sampler_name(sampler_index, img=True)
         self.hr_sampler_name: str = hr_sampler_name if hr_sampler_name != 'Same as primary' else self.sampler_name
-        self.override_settings = {k: v for k, v in (override_settings or {}).items() if k not in shared.restricted_opts}
         self.inpaint_full_res = inpaint_full_res if isinstance(inpaint_full_res, bool) else self.inpaint_full_res
         self.inpaint_full_res = inpaint_full_res != 0 if isinstance(inpaint_full_res, int) else self.inpaint_full_res
+        try:
+            self.override_settings = {k: v for k, v in (override_settings or {}).items() if k not in shared.restricted_opts}
+        except Exception as e:
+            shared.log.error(f'Override: {override_settings} {e}')
+            self.override_settings = {}
 
         # null items initialized later
         self.prompts = None

modules/processing_info.py

@@ -186,6 +186,9 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
     for k, v in args.copy().items():
         if v is None:
             del args[k]
+        if type(v) is float or type(v) is int:
+            if v <= -1:
+                del args[k]
         if isinstance(v, str):
             if len(v) == 0 or v == '0x0':
                 del args[k]

modules/shared.py

@@ -779,7 +779,7 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"),
     'schedulers_beta_end': OptionInfo(0, "Beta end", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.00001, "visible": native}),
     'schedulers_timesteps_range': OptionInfo(1000, "Timesteps range", gr.Slider, {"minimum": 250, "maximum": 4000, "step": 1, "visible": native}),
     'schedulers_shift': OptionInfo(3, "Sampler shift", gr.Slider, {"minimum": 0.1, "maximum": 10, "step": 0.1, "visible": False}),
-    'schedulers_dynamic_shift': OptionInfo(True, "Sampler dynamic shift", gr.Checkbox, {"visible": False}),
+    'schedulers_dynamic_shift': OptionInfo(False, "Sampler dynamic shift", gr.Checkbox, {"visible": False}),
 
     # managed from ui.py for backend original k-diffusion
     "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching", gr.Checkbox, {"visible": not native}),

modules/ui_symbols.py

@@ -20,6 +20,7 @@ reuse = '♻️'
 info = 'ℹ' # noqa
 reset = '🔄'
 upload = '⬆️'
+loading = '↺'
 reuse = '⬅️'
 search = '🔍'
 preview = '🖼️'

modules/ui_video.py

@@ -1,59 +1,69 @@
+import os
 import gradio as gr
 from modules import shared, sd_models, timer, images, ui_common, ui_sections, ui_symbols, call_queue, generation_parameters_copypaste
 from modules.ui_components import ToolButton
-from modules.video_models import models_def, video_utils, video_load
+from modules.video_models import models_def, video_utils
+
+
+debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
 
 
 def engine_change(engine):
+    debug(f'Video change: engine="{engine}"')
     found = [model.name for model in models_def.models.get(engine, [])]
     return gr.update(choices=found, value=found[0] if len(found) > 0 else None)
 
 
 def model_change(engine, model):
+    debug(f'Video change: engine="{engine}" model="{model}"')
     found = [model.name for model in models_def.models.get(engine, [])]
     selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
-    yield ['Video model loading',
-            gr.update(visible='I2V' in selected.name) if selected else gr.update(visible=False),
-            video_utils.get_url(selected.url if selected else None),
-           ]
+    return video_utils.get_url(selected.url if selected else None)
+
+
+def model_load(engine, model):
+    debug(f'Video load: engine="{engine}" model="{model}"')
+    found = [model.name for model in models_def.models.get(engine, [])]
+    selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
+    yield f'Video model loading: {selected.name}'
     if selected:
         if 'None' in selected.name:
             sd_models.unload_model_weights()
             msg = 'Video model unloaded'
         else:
+            from modules.video_models import video_load
             msg = video_load.load_model(selected)
     else:
         sd_models.unload_model_weights()
         msg = 'Video model unloaded'
-    return [msg,
-            video_utils.get_url(selected.url if selected else None),
-           ]
+    yield msg
+    return msg
 
 
 def run_video(*args):
     engine, model = args[2], args[3]
+    debug(f'Video run: engine="{engine}" model="{model}"')
     found = [model.name for model in models_def.models.get(engine, [])]
     selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
+    if not selected or engine is None or model is None or engine == 'None' or model == 'None':
+        return video_utils.queue_err('model not selected')
+    debug(f'Video run: {str(selected)}')
+    from modules.video_models import video_run
     if selected and 'Hunyuan' in selected.name:
-        from modules.video_models import run_hunyuan
-        return run_hunyuan.generate(*args)
+        return video_run.generate('Hunyuan', *args)
     elif selected and 'LTX' in selected.name:
-        from modules.video_models import run_ltx
-        return run_ltx.generate(*args)
+        return video_run.generate('LTX', *args)
     elif selected and 'Mochi' in selected.name:
-        from modules.video_models import run_mochi
-        return run_mochi.generate(*args)
+        return video_run.generate('Mochi', *args)
     elif selected and 'Cog' in selected.name:
-        from modules.video_models import run_cog
-        return run_cog.generate(*args)
+        return video_run.generate('Cog', *args)
     elif selected and 'Allegro' in selected.name:
-        from modules.video_models import run_allegro
-        return run_allegro.generate(*args)
+        return video_run.generate('Allegro', *args)
     elif selected and 'WAN' in selected.name:
-        from modules.video_models import run_wan
-        return run_wan.generate(*args)
-    shared.log.error(f'Video model not found: args={args}')
-    return [], None, '', '', f'Video model not found: engine={engine} model={model}'
+        return video_run.generate('Wan', *args)
+    elif selected and 'Latte' in selected.name:
+        return video_run.generate('Latte', *args)
+    return video_utils.queue_err(f'model not found: engine="{engine}" model="{model}"')
 
 
 def create_ui():
@@ -74,23 +84,25 @@ def create_ui():
                 with gr.Row():
                     engine = gr.Dropdown(label='Engine', choices=list(models_def.models), value='None', elem_id="video_engine")
                     model = gr.Dropdown(label='Model', choices=[''], value=None, elem_id="video_model")
+                    btn_load = ToolButton(ui_symbols.loading, elem_id="video_model_load", label='Load model')
                 with gr.Row():
-                    url = gr.HTML(label='Model URL', elem_id='video_model_url', value='')
-                with gr.Row():
-                    width, height = ui_sections.create_resolution_inputs('video', default_width=720, default_height=480)
-                with gr.Row():
-                    frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=15, elem_id="video_frames")
-                    seed = gr.Number(label='Initial seed', value=-1, elem_id="video_seed", container=True)
-                    random_seed = ToolButton(ui_symbols.random, elem_id="video_random_seed", label='Random seed')
-                    reuse_seed = ToolButton(ui_symbols.reuse, elem_id="video_reuse_seed", label='Reuse seed')
+                    url = gr.HTML(label='Model URL', elem_id='video_model_url', value='<br><br>')
+                with gr.Accordion(open=True, label="Size", elem_id='video_size_accordion'):
+                    with gr.Row():
+                        width, height = ui_sections.create_resolution_inputs('video', default_width=720, default_height=480)
+                    with gr.Row():
+                        frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=15, elem_id="video_frames")
+                        seed = gr.Number(label='Initial seed', value=-1, elem_id="video_seed", container=True)
+                        random_seed = ToolButton(ui_symbols.random, elem_id="video_random_seed", label='Random seed')
+                        reuse_seed = ToolButton(ui_symbols.reuse, elem_id="video_reuse_seed", label='Reuse seed')
                 with gr.Accordion(open=True, label="Parameters", elem_id='video_parameters_accordion'):
                     steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video")
                     with gr.Row():
-                        sampler_shift = gr.Slider(label='Sampler shift', minimum=0.0, maximum=20.0, step=0.1, value=7.0, elem_id="video_scheduler_shift")
-                        dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift", interactive=False) # TODO video: dynamic shift
+                        sampler_shift = gr.Slider(label='Sampler shift', minimum=-1.0, maximum=20.0, step=0.1, value=-1.0, elem_id="video_scheduler_shift")
+                        dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift")
                     with gr.Row():
-                        guidance_scale = gr.Slider(label='Guidance scale', minimum=0.0, maximum=14.0, step=0.1, value=6.0, elem_id="video_guidance_scale")
-                        guidance_true = gr.Slider(label='True guidance', minimum=0.0, maximum=14.0, step=0.1, value=1.0, elem_id="video_guidance_true")
+                        guidance_scale = gr.Slider(label='Guidance scale', minimum=-1.0, maximum=14.0, step=0.1, value=-1.0, elem_id="video_guidance_scale")
+                        guidance_true = gr.Slider(label='True guidance', minimum=-1.0, maximum=14.0, step=0.1, value=-1.0, elem_id="video_guidance_true")
                 with gr.Accordion(open=True, label="Decode", elem_id='video_decode_accordion'):
                     with gr.Row():
                         vae_type = gr.Dropdown(label='VAE decode', choices=['Default', 'Tiny', 'Remote'], value='Default', elem_id="video_vae_type")
@@ -121,7 +133,8 @@ def create_ui():
             random_seed.click(fn=lambda: -1, show_progress=False, inputs=[], outputs=[seed])
             # handle engine and model change
             engine.change(fn=engine_change, inputs=[engine], outputs=[model])
-            model.change(fn=model_change, inputs=[engine, model], outputs=[html_log, url])
+            model.change(fn=model_change, inputs=[engine, model], outputs=[url])
+            btn_load.click(fn=model_load, inputs=[engine, model], outputs=[html_log])
             # setup extra networks
             ui_extra_networks.setup_ui(extra_networks_ui, gallery)
 
@@ -154,6 +167,7 @@ def create_ui():
             vae_type, vae_tile_frames,
             save_frames,
             video_type, video_duration, video_loop, video_pad, video_interpolate,
+            faster_cache, pyramid_attention,
             override_settings,
         ]
         # generate function

modules/video_models/models_def.py

@@ -3,6 +3,32 @@ import diffusers
 import transformers
 
 
+"""
+Hunyuan Video T2V: pass/pass/pass
+Hunyuan Video I2V: pass/pass/pass, transformers incompatibility
+SkyReels Hunyuan T2V:
+SkyReels Hunyuan I2V:
+Fast Hunyuan T2V:
+LTXVideo 0.9.5 T2V:
+LTXVideo 0.9.5 I2V:
+LTXVideo 0.9.1 T2V:
+LTXVideo 0.9.1 I2V:
+LTXVideo 0.9.0 T2V:
+LTXVideo 0.9.0 I2V:
+WAN 2.1 1.3B T2V: pass/pass/pass
+WAN 2.1 14B T2V: pass/fail/fail, error loading shard
+WAN 2.1 14B I2V 480p:
+WAN 2.1 14B I2V 720p:
+Mochi 1 T2V: pass/pass/pass
+Latte 1 T2V: pass/fail/fail, float vs bfloat during generate
+Allegro T2V: pass/pass/fail, output is pure gray
+CogVideoX 1.0 2B T2V: pass/pass/pass
+CogVideoX 1.0 5B T2V:
+CogVideoX 1.0 5B I2V:
+CogVideoX 1.5 5B T2V: pass/pass/fail, output is pure black
+CogVideoX 1.5 5B I2V: pass/pass/pass
+"""
+
 @dataclass
 class Model():
     name: str
@@ -19,6 +45,9 @@ class Model():
     vae_hijack: bool = True
     vae_remote: bool = False
 
+    def __str__(self):
+        return f'name="{self.name}" url="{self.url}" repo="{self.repo}" repo_cls="{self.repo_cls}" dit="{self.dit}" dit_cls="{self.dit_cls}" dit_folder="{self.dit_folder}" te="{self.te}" te_cls="{self.te_cls}" te_folder="{self.te_folder}" te_hijack={self.te_hijack} vae_hijack={self.vae_hijack} vae_remote={self.vae_remote}'
+
 
 models = {
     'None': [],
@@ -167,7 +196,7 @@ models = {
               dit_cls=diffusers.CogVideoXTransformer3DModel),
         Model(name='CogVideoX 1.0 5B T2V',
               url='https://huggingface.co/THUDM/CogVideoX-5b',
-              repo='THUDM/THUDM/CogVideoX-5b',
+              repo='THUDM/CogVideoX-5b',
               repo_cls=diffusers.CogVideoXPipeline,
               te_cls=transformers.T5EncoderModel,
               dit_cls=diffusers.CogVideoXTransformer3DModel),

modules/video_models/run_allegro.py

@@ -1,91 +0,0 @@
-import os
-import time
-from modules import shared, errors, sd_models, processing, devices, images, ui_common
-from modules.video_models import models_def, video_utils, video_load, video_vae
-
-
-debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
-
-
-def generate(*args, **kwargs):
-    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
-    if engine is None or model is None or engine == 'None' or model == 'None':
-        return video_utils.queue_err('model not selected')
-    found = [model.name for model in models_def.models.get(engine, [])]
-    selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
-    if not shared.sd_loaded or 'Allegro' not in shared.sd_model.__class__.__name__:
-        video_load.load_model(selected)
-    if not shared.sd_loaded or 'Allegro' not in shared.sd_model.__class__.__name__:
-        return video_utils.queue_err('model not loaded')
-    debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}')
-
-    p = processing.StableDiffusionProcessingVideo(
-        sd_model=shared.sd_model,
-        prompt=prompt,
-        negative_prompt=negative,
-        styles=styles,
-        seed=int(seed),
-        sampler_name = processing.get_sampler_name(sampler_index),
-        sampler_shift=float(sampler_shift),
-        steps=int(steps),
-        width=8 * int(width // 8),
-        height=8 * int(height // 8),
-        frames=int(frames),
-        init_image=init_image,
-        cfg_scale=float(guidance_scale),
-        diffusers_guidance_rescale=float(guidance_true),
-        vae_type=vae_type,
-        vae_tile_frames=int(vae_tile_frames),
-        override_settings=override_settings,
-    )
-    p.scripts = None
-    p.script_args = None
-    p.state = ui_state
-    p.do_not_save_grid = True
-    p.do_not_save_samples = not save_frames
-    if 'I2V' in model:
-        if init_image is None:
-            return video_utils.queue_err('init image not set')
-        p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
-
-    # cleanup memory
-    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
-    devices.torch_gc(force=True)
-
-    # set args
-    processing.fix_seed(p)
-    video_vae.set_vae_params(p)
-    video_utils.set_prompt(p)
-    p.task_args['output_type'] = 'pil'
-    p.ops.append('video')
-    orig_dynamic_shift = shared.opts.schedulers_dynamic_shift
-    orig_sampler_shift = shared.opts.schedulers_shift
-    shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift
-    shared.opts.data['schedulers_shift'] = sampler_shift
-    debug(f'Video: task_args={p.task_args}')
-
-    # run processing
-    shared.state.disable_preview = True
-    shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}')
-    err = None
-    t0 = time.time()
-    try:
-        processed = processing.process_images(p)
-    except Exception as e:
-        err = str(e)
-        errors.display(e, 'video')
-    t1 = time.time()
-    shared.state.disable_preview = False
-    shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift
-    shared.opts.data['schedulers_shift'] = orig_sampler_shift
-    p.close()
-
-    # done
-    if err:
-        return video_utils.queue_err(err)
-    if processed is None or len(processed.images) == 0:
-        return video_utils.queue_err('processing failed')
-    shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}')
-    video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
-    generation_info_js = processed.js() if processed is not None else ''
-    return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)

modules/video_models/run_cog.py

@@ -1,91 +0,0 @@
-import os
-import time
-from modules import shared, errors, sd_models, processing, devices, images, ui_common
-from modules.video_models import models_def, video_utils, video_load, video_vae
-
-
-debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
-
-
-def generate(*args, **kwargs):
-    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
-    if engine is None or model is None or engine == 'None' or model == 'None':
-        return video_utils.queue_err('model not selected')
-    found = [model.name for model in models_def.models.get(engine, [])]
-    selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
-    if not shared.sd_loaded or 'Cog' not in shared.sd_model.__class__.__name__:
-        video_load.load_model(selected)
-    if not shared.sd_loaded or 'Cog' not in shared.sd_model.__class__.__name__:
-        return video_utils.queue_err('model not loaded')
-    debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}')
-
-    p = processing.StableDiffusionProcessingVideo(
-        sd_model=shared.sd_model,
-        prompt=prompt,
-        negative_prompt=negative,
-        styles=styles,
-        seed=int(seed),
-        sampler_name = processing.get_sampler_name(sampler_index),
-        sampler_shift=float(sampler_shift),
-        steps=int(steps),
-        width=8 * int(width // 8),
-        height=8 * int(height // 8),
-        frames=int(frames),
-        init_image=init_image,
-        cfg_scale=float(guidance_scale),
-        diffusers_guidance_rescale=float(guidance_true),
-        vae_type=vae_type,
-        vae_tile_frames=int(vae_tile_frames),
-        override_settings=override_settings,
-    )
-    p.scripts = None
-    p.script_args = None
-    p.state = ui_state
-    p.do_not_save_grid = True
-    p.do_not_save_samples = not save_frames
-    if 'I2V' in model:
-        if init_image is None:
-            return video_utils.queue_err('init image not set')
-        p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
-
-    # cleanup memory
-    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
-    devices.torch_gc(force=True)
-
-    # set args
-    processing.fix_seed(p)
-    video_vae.set_vae_params(p)
-    video_utils.set_prompt(p)
-    p.task_args['output_type'] = 'pil'
-    p.ops.append('video')
-    orig_dynamic_shift = shared.opts.schedulers_dynamic_shift
-    orig_sampler_shift = shared.opts.schedulers_shift
-    shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift
-    shared.opts.data['schedulers_shift'] = sampler_shift
-    debug(f'Video: task_args={p.task_args}')
-
-    # run processing
-    shared.state.disable_preview = True
-    shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}')
-    err = None
-    t0 = time.time()
-    try:
-        processed = processing.process_images(p)
-    except Exception as e:
-        err = str(e)
-        errors.display(e, 'video')
-    t1 = time.time()
-    shared.state.disable_preview = False
-    shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift
-    shared.opts.data['schedulers_shift'] = orig_sampler_shift
-    p.close()
-
-    # done
-    if err:
-        return video_utils.queue_err(err)
-    if processed is None or len(processed.images) == 0:
-        return video_utils.queue_err('processing failed')
-    shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}')
-    video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
-    generation_info_js = processed.js() if processed is not None else ''
-    return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)

modules/video_models/run_hunyuan.py

@@ -1,94 +0,0 @@
-import os
-import time
-from modules import shared, errors, sd_models, processing, devices, images, ui_common
-from modules.video_models import models_def, video_utils, video_load, video_vae
-
-
-debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
-
-
-def generate(*args, **kwargs):
-    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
-    if engine is None or model is None or engine == 'None' or model == 'None':
-        return video_utils.queue_err('model not selected')
-    found = [model.name for model in models_def.models.get(engine, [])]
-    selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
-    if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__:
-        video_load.load_model(selected)
-    if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__:
-        return video_utils.queue_err('model not loaded')
-    debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}')
-
-    p = processing.StableDiffusionProcessingVideo(
-        sd_model=shared.sd_model,
-        prompt=prompt,
-        negative_prompt=negative,
-        styles=styles,
-        seed=int(seed),
-        sampler_name = processing.get_sampler_name(sampler_index),
-        sampler_shift=float(sampler_shift),
-        steps=int(steps),
-        width=16 * int(width // 16),
-        height=16 * int(height // 16),
-        frames=int(frames),
-        init_image=init_image,
-        cfg_scale=float(guidance_scale),
-        diffusers_guidance_rescale=float(guidance_true),
-        vae_type=vae_type,
-        vae_tile_frames=int(vae_tile_frames),
-        override_settings=override_settings,
-    )
-    if p.vae_type == 'Remote' and not selected.vae_remote:
-        shared.log.warning(f'Video: model={selected.name} remote vae not supported')
-        p.vae_type = 'Default'
-    p.scripts = None
-    p.script_args = None
-    p.state = ui_state
-    p.do_not_save_grid = True
-    p.do_not_save_samples = not save_frames
-    if 'I2V' in model:
-        if init_image is None:
-            return video_utils.queue_err('init image not set')
-        p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
-
-    # cleanup memory
-    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
-    devices.torch_gc(force=True)
-
-    # set args
-    processing.fix_seed(p)
-    video_vae.set_vae_params(p)
-    video_utils.set_prompt(p)
-    p.task_args['output_type'] = 'latent' if (p.vae_type == 'Remote') else 'pil'
-    p.ops.append('video')
-    orig_dynamic_shift = shared.opts.schedulers_dynamic_shift
-    orig_sampler_shift = shared.opts.schedulers_shift
-    shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift
-    shared.opts.data['schedulers_shift'] = sampler_shift
-    debug(f'Video: task_args={p.task_args}')
-
-    # run processing
-    shared.state.disable_preview = True
-    shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}')
-    err = None
-    t0 = time.time()
-    try:
-        processed = processing.process_images(p)
-    except Exception as e:
-        err = str(e)
-        errors.display(e, 'video')
-    t1 = time.time()
-    shared.state.disable_preview = False
-    shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift
-    shared.opts.data['schedulers_shift'] = orig_sampler_shift
-    p.close()
-
-    # done
-    if err:
-        return video_utils.queue_err(err)
-    if processed is None or len(processed.images) == 0:
-        return video_utils.queue_err('processing failed')
-    shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}')
-    video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
-    generation_info_js = processed.js() if processed is not None else ''
-    return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)

modules/video_models/run_ltx.py

@@ -1,91 +0,0 @@
-import os
-import time
-from modules import shared, errors, sd_models, processing, devices, images, ui_common
-from modules.video_models import models_def, video_utils, video_load, video_vae
-
-
-debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
-
-
-def generate(*args, **kwargs):
-    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
-    if engine is None or model is None or engine == 'None' or model == 'None':
-        return video_utils.queue_err('model not selected')
-    found = [model.name for model in models_def.models.get(engine, [])]
-    selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
-    if not shared.sd_loaded or 'LTX' not in shared.sd_model.__class__.__name__:
-        video_load.load_model(selected)
-    if not shared.sd_loaded or 'LTX' not in shared.sd_model.__class__.__name__:
-        return video_utils.queue_err('model not loaded')
-    debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}')
-
-    p = processing.StableDiffusionProcessingVideo(
-        sd_model=shared.sd_model,
-        prompt=prompt,
-        negative_prompt=negative,
-        styles=styles,
-        seed=int(seed),
-        sampler_name = processing.get_sampler_name(sampler_index),
-        sampler_shift=float(sampler_shift),
-        steps=int(steps),
-        width=32 * int(width // 32),
-        height=32 * int(height // 32),
-        frames=int(frames),
-        init_image=init_image,
-        cfg_scale=float(guidance_scale),
-        diffusers_guidance_rescale=float(guidance_true),
-        vae_type=vae_type,
-        vae_tile_frames=int(vae_tile_frames),
-        override_settings=override_settings,
-    )
-    p.scripts = None
-    p.script_args = None
-    p.state = ui_state
-    p.do_not_save_grid = True
-    p.do_not_save_samples = not save_frames
-    if 'I2V' in model:
-        if init_image is None:
-            return video_utils.queue_err('init image not set')
-        p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
-
-    # cleanup memory
-    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
-    devices.torch_gc(force=True)
-
-    # set args
-    processing.fix_seed(p)
-    video_vae.set_vae_params(p)
-    video_utils.set_prompt(p)
-    p.task_args['output_type'] = 'pil'
-    p.ops.append('video')
-    orig_dynamic_shift = shared.opts.schedulers_dynamic_shift
-    orig_sampler_shift = shared.opts.schedulers_shift
-    shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift
-    shared.opts.data['schedulers_shift'] = sampler_shift
-    debug(f'Video: task_args={p.task_args}')
-
-    # run processing
-    shared.state.disable_preview = True
-    shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}')
-    err = None
-    t0 = time.time()
-    try:
-        processed = processing.process_images(p)
-    except Exception as e:
-        err = str(e)
-        errors.display(e, 'video')
-    t1 = time.time()
-    shared.state.disable_preview = False
-    shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift
-    shared.opts.data['schedulers_shift'] = orig_sampler_shift
-    p.close()
-
-    # done
-    if err:
-        return video_utils.queue_err(err)
-    if processed is None or len(processed.images) == 0:
-        return video_utils.queue_err('processing failed')
-    shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}')
-    video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
-    generation_info_js = processed.js() if processed is not None else ''
-    return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)

modules/video_models/run_mochi.py

@@ -1,91 +0,0 @@
-import os
-import time
-from modules import shared, errors, sd_models, processing, devices, images, ui_common
-from modules.video_models import models_def, video_utils, video_load, video_vae
-
-
-debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
-
-
-def generate(*args, **kwargs):
-    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
-    if engine is None or model is None or engine == 'None' or model == 'None':
-        return video_utils.queue_err('model not selected')
-    found = [model.name for model in models_def.models.get(engine, [])]
-    selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
-    if not shared.sd_loaded or 'Mochi' not in shared.sd_model.__class__.__name__:
-        video_load.load_model(selected)
-    if not shared.sd_loaded or 'Mochi' not in shared.sd_model.__class__.__name__:
-        return video_utils.queue_err('model not loaded')
-    debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}')
-
-    p = processing.StableDiffusionProcessingVideo(
-        sd_model=shared.sd_model,
-        prompt=prompt,
-        negative_prompt=negative,
-        styles=styles,
-        seed=int(seed),
-        sampler_name = processing.get_sampler_name(sampler_index),
-        sampler_shift=float(sampler_shift),
-        steps=int(steps),
-        width=8 * int(width // 8),
-        height=8 * int(height // 8),
-        frames=int(frames),
-        init_image=init_image,
-        cfg_scale=float(guidance_scale),
-        diffusers_guidance_rescale=float(guidance_true),
-        vae_type=vae_type,
-        vae_tile_frames=int(vae_tile_frames),
-        override_settings=override_settings,
-    )
-    p.scripts = None
-    p.script_args = None
-    p.state = ui_state
-    p.do_not_save_grid = True
-    p.do_not_save_samples = not save_frames
-    if 'I2V' in model:
-        if init_image is None:
-            return video_utils.queue_err('init image not set')
-        p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
-
-    # cleanup memory
-    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
-    devices.torch_gc(force=True)
-
-    # set args
-    processing.fix_seed(p)
-    video_vae.set_vae_params(p)
-    video_utils.set_prompt(p)
-    p.task_args['output_type'] = 'pil'
-    p.ops.append('video')
-    orig_dynamic_shift = shared.opts.schedulers_dynamic_shift
-    orig_sampler_shift = shared.opts.schedulers_shift
-    shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift
-    shared.opts.data['schedulers_shift'] = sampler_shift
-    debug(f'Video: task_args={p.task_args}')
-
-    # run processing
-    shared.state.disable_preview = True
-    shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}')
-    err = None
-    t0 = time.time()
-    try:
-        processed = processing.process_images(p)
-    except Exception as e:
-        err = str(e)
-        errors.display(e, 'video')
-    t1 = time.time()
-    shared.state.disable_preview = False
-    shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift
-    shared.opts.data['schedulers_shift'] = orig_sampler_shift
-    p.close()
-
-    # done
-    if err:
-        return video_utils.queue_err(err)
-    if processed is None or len(processed.images) == 0:
-        return video_utils.queue_err('processing failed')
-    shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}')
-    video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
-    generation_info_js = processed.js() if processed is not None else ''
-    return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)

modules/video_models/video_cache.py

@@ -0,0 +1,45 @@
+import os
+import diffusers
+from modules import shared, errors
+
+
+debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
+
+
+def set_cache(faster_cache=False, pyramid_attention_broadcast=False):
+    if not shared.sd_loaded or not hasattr(shared.sd_model, 'transformer'):
+        return
+    if not hasattr(shared.sd_model.transformer, 'enable_cache'):
+        debug(f'Video cache: cls={shared.sd_model.transformer.__class__.__name__} not supported')
+        return
+    try:
+        if faster_cache: # https://github.com/huggingface/diffusers/pull/10163
+            config = diffusers.FasterCacheConfig(
+                spatial_attention_block_skip_range=2,
+                spatial_attention_timestep_skip_range=(-1, 681),
+                current_timestep_callback=lambda: shared.sd_model.current_timestep,
+                attention_weight_callback=lambda _: 0.3,
+                unconditional_batch_skip_range=5,
+                unconditional_batch_timestep_skip_range=(-1, 781),
+                tensor_format="BFCHW",
+            )
+            shared.sd_model.transformer.disable_cache()
+            shared.sd_model.transformer.enable_cache(config)
+            shared.log.debug(f'Video cache: type={config.__class__.__name__}')
+            debug(f'Video cache: {vars(config)}')
+        elif pyramid_attention_broadcast: # https://github.com/huggingface/diffusers/pull/9562
+            config = diffusers.PyramidAttentionBroadcastConfig(
+                spatial_attention_block_skip_range=2,
+                spatial_attention_timestep_skip_range=(100, 800),
+                current_timestep_callback=lambda: shared.sd_model.current_timestep,
+            )
+            shared.sd_model.transformer.disable_cache()
+            shared.sd_model.transformer.enable_cache(config)
+            shared.log.debug(f'Video cache: type={config.__class__.__name__}')
+            debug(f'Video cache: {vars(config)}')
+        else:
+            debug('Video cache: not enabled')
+            shared.sd_model.transformer.disable_cache()
+    except Exception as e:
+        shared.log.error(f'Video cache: error={e}')
+        errors.display(e, 'video cache')

modules/video_models/video_run.py

@@ -1,21 +1,21 @@
 import os
 import time
 from modules import shared, errors, sd_models, processing, devices, images, ui_common
-from modules.video_models import models_def, video_utils, video_load, video_vae
+from modules.video_models import models_def, video_utils, video_load, video_vae, video_cache
 
 
 debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
 
 
-def generate(*args, **kwargs):
-    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
+def generate(keyword, *args, **kwargs):
+    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, faster_cache, pyramid_attention, override_settings = args
     if engine is None or model is None or engine == 'None' or model == 'None':
         return video_utils.queue_err('model not selected')
     found = [model.name for model in models_def.models.get(engine, [])]
     selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
-    if not shared.sd_loaded or 'Wan' not in shared.sd_model.__class__.__name__:
+    if not shared.sd_loaded or keyword not in shared.sd_model.__class__.__name__:
         video_load.load_model(selected)
-    if not shared.sd_loaded or 'Wan' not in shared.sd_model.__class__.__name__:
+    if not shared.sd_loaded or keyword not in shared.sd_model.__class__.__name__:
         return video_utils.queue_err('model not loaded')
     debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}')
 
@@ -58,6 +58,7 @@ def generate(*args, **kwargs):
     # set args
     processing.fix_seed(p)
     video_vae.set_vae_params(p)
+    video_cache.set_cache(faster_cache=faster_cache, pyramid_attention_broadcast=pyramid_attention)
     video_utils.set_prompt(p)
     p.task_args['output_type'] = 'latent' if (p.vae_type == 'Remote') else 'pil'
     p.ops.append('video')

modules/video_models/video_utils.py

@@ -18,7 +18,7 @@ def get_quant(args):
 
 
 def get_url(url):
-    return f'&nbsp <a href="{url}" target="_blank" rel="noopener noreferrer" style="color: var(--button-primary-background-fill); font-weight: normal">{url}</a><br>' if url else ''
+    return f'&nbsp <a href="{url}" target="_blank" rel="noopener noreferrer" style="color: var(--button-primary-background-fill); font-weight: normal">{url}</a><br><br>' if url else '<br><br>'
 
 
 def set_prompt(p):