video unified component loader

Signed-off-by: Vladimir Mandic <mandic00@live.com>
2026-09-19 17:24:32 +02:00 · 2025-03-19 18:14:23 -04:00
parent 9bf6838962
commit 79a5391e47
11 changed files with 531 additions and 47 deletions
@@ -6,6 +6,7 @@
  - Gemma3 requires `transformers==git+https://github.com/huggingface/transformers@v4.49.0-Gemma-3`
  - Remote VAE encode for SD15 and Flux.1: <https://github.com/huggingface/diffusers/issues/11069>
  - HunyuanVideo-I2V: <https://github.com/huggingface/diffusers/issues/11118>
+  - LTXVideo condition input

 ### Highlights for 2025-03-17

@@ -538,7 +538,7 @@ def check_diffusers():
    t_start = time.time()
    if args.skip_all or args.skip_git:
        return
-    sha = '82188cef0487837b8c70fc3f36ea63c05c85f341' # diffusers commit hash
+    sha = '56f740051dae2d410677292a5c9e5b66e60f87dc' # diffusers commit hash
    pkg = pkg_resources.working_set.by_key.get('diffusers', None)
    minor = int(pkg.version.split('.')[1] if pkg is not None else 0)
    cur = opts.get('diffusers_version', '') if minor > 0 else ''
@@ -1,7 +1,7 @@
 import gradio as gr
 from modules import shared, sd_models, timer, images, ui_common, ui_sections, ui_symbols, call_queue, generation_parameters_copypaste
 from modules.ui_components import ToolButton
-from modules.video_models import models_def, video_utils, hunyuan, ltx
+from modules.video_models import models_def, video_utils


 def engine_change(engine):
@@ -12,6 +12,10 @@ def engine_change(engine):
 def model_change(engine, model):
    found = [model.name for model in models_def.models.get(engine, [])]
    selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
+    yield ['Video model loading',
+            gr.update(visible='I2V' in selected.name) if selected else gr.update(visible=False),
+            video_utils.get_url(selected.url if selected else None),
+           ]
    if selected:
        if 'None' in selected.name:
            sd_models.unload_model_weights()
@@ -21,7 +25,10 @@ def model_change(engine, model):
    else:
        sd_models.unload_model_weights()
        msg = 'Video model unloaded'
-    return [msg, gr.update(visible='I2V' in selected.name) if selected else gr.update(visible=False)]
+    return [msg,
+            gr.update(visible='I2V' in selected.name) if selected else gr.update(visible=False),
+            video_utils.get_url(selected.url if selected else None),
+           ]


 def run_video(*args):
@@ -29,10 +36,20 @@ def run_video(*args):
    found = [model.name for model in models_def.models.get(engine, [])]
    selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
    if selected and 'Hunyuan' in selected.name:
-        return hunyuan.generate(*args)
+        from modules.video_models import run_hunyuan
+        return run_hunyuan.generate(*args)
    elif selected and 'LTX' in selected.name:
-        pass
-        # return ltx.generate(*args)
+        from modules.video_models import run_ltx
+        return run_ltx.generate(*args)
+    elif selected and 'Mochi' in selected.name:
+        from modules.video_models import run_mochi
+        return run_mochi.generate(*args)
+    elif selected and 'Cog' in selected.name:
+        from modules.video_models import run_cog
+        return run_cog.generate(*args)
+    elif selected and 'Allegro' in selected.name:
+        from modules.video_models import run_allegro
+        return run_allegro.generate(*args)
    shared.log.error(f'Video model not found: args={args}')
    return [], None, '', '', f'Video model not found: engine={engine} model={model}'

@@ -55,6 +72,8 @@ def create_ui():
                with gr.Row():
                    engine = gr.Dropdown(label='Engine', choices=list(models_def.models), value='None', elem_id="video_engine")
                    model = gr.Dropdown(label='Model', choices=[''], value=None, elem_id="video_model")
+                with gr.Row():
+                    url = gr.HTML(label='Model URL', elem_id='video_model_url', value='')
                with gr.Row():
                    width, height = ui_sections.create_resolution_inputs('video', default_width=720, default_height=480)
                with gr.Row():
@@ -65,7 +84,7 @@ def create_ui():
                steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video")
                with gr.Row():
                    sampler_shift = gr.Slider(label='Sampler shift', minimum=0.0, maximum=20.0, step=0.1, value=7.0, elem_id="video_scheduler_shift")
-                    dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift")
+                    dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift", interactive=False)
                with gr.Row():
                    guidance_scale = gr.Slider(label='Guidance scale', minimum=0.0, maximum=14.0, step=0.1, value=6.0, elem_id="video_guidance_scale")
                    guidance_true = gr.Slider(label='True guidance', minimum=0.0, maximum=14.0, step=0.1, value=1.0, elem_id="video_guidance_true")
@@ -95,7 +114,7 @@ def create_ui():
            random_seed.click(fn=lambda: -1, show_progress=False, inputs=[], outputs=[seed])
            # handle engine and model change
            engine.change(fn=engine_change, inputs=[engine], outputs=[model])
-            model.change(fn=model_change, inputs=[engine, model], outputs=[html_log, image_group])
+            model.change(fn=model_change, inputs=[engine, model], outputs=[html_log, image_group, url])
            # setup extra networks
            ui_extra_networks.setup_ui(extra_networks_ui, gallery)

@@ -6,6 +6,7 @@ import transformers
@dataclass
 class Model():
    name: str
+    url: str = ''
    repo: str = None
    repo_cls: classmethod = None
    dit: str = None
@@ -23,16 +24,19 @@ models = {
    'Hunyuan Video': [
        Model(name='None'),
        Model(name='Hunyuan Video T2V',
+              url='https://huggingface.co/tencent/HunyuanVideo',
              repo='hunyuanvideo-community/HunyuanVideo',
              repo_cls=diffusers.HunyuanVideoPipeline,
              te_cls=transformers.LlamaModel,
              dit_cls=diffusers.HunyuanVideoTransformer3DModel),
        Model(name='Hunyuan Video I2V', # https://github.com/huggingface/diffusers/pull/10983
+              url='https://huggingface.co/tencent/HunyuanVideo-I2V',
              repo='hunyuanvideo-community/HunyuanVideo-I2V',
              repo_cls=diffusers.HunyuanVideoImageToVideoPipeline,
              te_cls=transformers.LlavaForConditionalGeneration,
              dit_cls=diffusers.HunyuanVideoTransformer3DModel),
        Model(name='SkyReels Hunyuan T2V', # https://github.com/huggingface/diffusers/pull/10837
+              url='https://huggingface.co/Skywork/SkyReels-V1-Hunyuan-T2V',
              repo='hunyuanvideo-community/HunyuanVideo',
              repo_cls=diffusers.HunyuanVideoPipeline,
              te_cls=transformers.LlamaModel,
@@ -40,28 +44,108 @@ models = {
              dit_folder=None,
              dit_cls=diffusers.HunyuanVideoTransformer3DModel),
        Model(name='SkyReels Hunyuan I2V', # https://github.com/huggingface/diffusers/pull/10837
+              url='https://huggingface.co/Skywork/SkyReels-V1-Hunyuan-I2V',
              repo='hunyuanvideo-community/HunyuanVideo',
              te_cls=transformers.LlamaModel,
              dit='Skywork/SkyReels-V1-Hunyuan-I2V',
              dit_folder=None,
              dit_cls=diffusers.HunyuanVideoTransformer3DModel),
        Model(name='Fast Hunyuan T2V', # https://github.com/hao-ai-lab/FastVideo/blob/8a77cf22c9b9e7f931f42bc4b35d21fd91d24e45/fastvideo/models/hunyuan/inference.py#L213
+              url='https://huggingface.co/FastVideo/FastHunyuan',
              repo='hunyuanvideo-community/HunyuanVideo',
              repo_cls=diffusers.HunyuanVideoPipeline,
              te_cls=transformers.LlamaModel,
              dit='FastVideo/FastHunyuan-diffusers',
              dit_cls=diffusers.HunyuanVideoTransformer3DModel),
    ],
+    'LTX Video': [
+        Model(name='None'),
+        Model(name='LTXVideo 0.9.5 T2V', # https://github.com/huggingface/diffusers/pull/10968
+              url='https://huggingface.co/Lightricks/LTX-Video-0.9.5',
+              repo='YiYiXu/ltx-95',
+              repo_cls=diffusers.LTXPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.LTXVideoTransformer3DModel),
+        Model(name='LTXVideo 0.9.5 I2V',
+              url='https://huggingface.co/Lightricks/LTX-Video-0.9.5',
+              repo='YiYiXu/ltx-95',
+              repo_cls=diffusers.LTXConditionPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.LTXVideoTransformer3DModel),
+        Model(name='LTXVideo 0.9.1 T2V',
+              url='https://huggingface.co/a-r-r-o-w/LTX-Video-0.9.1-diffusers',
+              repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers',
+              repo_cls=diffusers.LTXPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.LTXVideoTransformer3DModel),
+        Model(name='LTXVideo 0.9.1 I2V',
+              url='https://huggingface.co/a-r-r-o-w/LTX-Video-0.9.1-diffusers',
+              repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers',
+              repo_cls=diffusers.LTXImageToVideoPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.LTXVideoTransformer3DModel),
+        Model(name='LTXVideo 0.9.0 T2V',
+              url='https://huggingface.co/a-r-r-o-w/LTX-Video-diffusers',
+              repo='a-r-r-o-w/LTX-Video-diffusers',
+              repo_cls=diffusers.LTXPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.LTXVideoTransformer3DModel),
+        Model(name='LTXVideo 0.9.0 I2V',
+              url='https://huggingface.co/a-r-r-o-w/LTX-Video-diffusers',
+              repo='a-r-r-o-w/LTX-Video-diffusers',
+              repo_cls=diffusers.LTXImageToVideoPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.LTXVideoTransformer3DModel),
+    ],
+    'Mochi Video': [
+        Model(name='None'),
+        Model(name='Mochi 1 T2V',
+              url='https://huggingface.co/genmo/mochi-1-preview',
+              repo='genmo/mochi-1-preview',
+              repo_cls=diffusers.MochiPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.MochiTransformer3DModel),
+    ],
+    'Allegro Video': [
+        Model(name='None'),
+        Model(name='Allegro T2V',
+              url='https://huggingface.co/rhymes-ai/Allegro',
+              repo='rhymes-ai/Allegro',
+              repo_cls=diffusers.AllegroPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.AllegroTransformer3DModel),
+    ],
+    'Cog Video': [
+        Model(name='None'),
+        Model(name='CogVideoX 1.0 2B T2V',
+              url='https://huggingface.co/THUDM/CogVideoX-2b',
+              repo='THUDM/CogVideoX-2b',
+              repo_cls=diffusers.CogVideoXPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.CogVideoXTransformer3DModel),
+        Model(name='CogVideoX 1.0 5B T2V',
+              url='https://huggingface.co/THUDM/CogVideoX-5b',
+              repo='THUDM/THUDM/CogVideoX-5b',
+              repo_cls=diffusers.CogVideoXPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.CogVideoXTransformer3DModel),
+        Model(name='CogVideoX 1.0 5B I2V',
+              url='https://huggingface.co/THUDM/CogVideoX-5b-I2V',
+              repo='THUDM/CogVideoX-5b-I2V',
+              repo_cls=diffusers.CogVideoXImageToVideoPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.CogVideoXTransformer3DModel),
+        Model(name='CogVideoX 1.5 5B T2V',
+              url='https://huggingface.co/THUDM/THUDM/CogVideoX1.5-5B',
+              repo='THUDM/CogVideoX1.5-5B',
+              repo_cls=diffusers.CogVideoXPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.CogVideoXTransformer3DModel),
+        Model(name='CogVideoX 1.5 5B I2V',
+              url='https://huggingface.co/THUDM/CogVideoX1.5-5B-I2V',
+              repo='THUDM/CogVideoX1.5-5B-I2V',
+              repo_cls=diffusers.CogVideoXImageToVideoPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.CogVideoXTransformer3DModel),
+    ],
 }
-
-"""
-'LTX Video': [
-    Model(name='None'),
-    Model(name='LTXVideo 0.9.0 T2V', repo='a-r-r-o-w/LTX-Video-diffusers', subfolder='transformer'),
-    Model(name='LTXVideo 0.9.1 T2V', repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', subfolder='transformer'),
-    Model(name='LTXVideo 0.9.5 T2V', repo='Lightricks/LTX-Video-0.9.5'), # https://github.com/huggingface/diffusers/pull/10968
-    Model(name='LTXVideo 0.9.0 I2V', repo='a-r-r-o-w/LTX-Video-diffusers', subfolder='transformer'),
-    Model(name='LTXVideo 0.9.1 I2V', repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', subfolder='transformer'),
-    Model(name='LTXVideo 0.9.5 I2V', repo='Lightricks/LTX-Video-0.9.5', subfolder='transformer'),
-],"
-"""
@@ -0,0 +1,91 @@
+import os
+import time
+from modules import shared, errors, sd_models, processing, devices, images, ui_common
+from modules.video_models import models_def, video_utils
+
+
+debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
+
+
+def generate(*args, **kwargs):
+    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
+    if engine is None or model is None or engine == 'None' or model == 'None':
+        return video_utils.queue_err('model not selected')
+    found = [model.name for model in models_def.models.get(engine, [])]
+    selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
+    if not shared.sd_loaded or 'Allegro' not in shared.sd_model.__class__.__name__:
+        video_utils.load_model(selected)
+    if not shared.sd_loaded or 'Allegro' not in shared.sd_model.__class__.__name__:
+        return video_utils.queue_err('model not loaded')
+    debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}')
+
+    p = processing.StableDiffusionProcessingVideo(
+        sd_model=shared.sd_model,
+        prompt=prompt,
+        negative_prompt=negative,
+        styles=styles,
+        seed=int(seed),
+        sampler_name = processing.get_sampler_name(sampler_index),
+        sampler_shift=float(sampler_shift),
+        steps=int(steps),
+        width=8 * int(width // 8),
+        height=8 * int(height // 8),
+        frames=int(frames),
+        init_image=init_image,
+        cfg_scale=float(guidance_scale),
+        diffusers_guidance_rescale=float(guidance_true),
+        vae_type=vae_type,
+        vae_tile_frames=int(vae_tile_frames),
+        override_settings=override_settings,
+    )
+    p.scripts = None
+    p.script_args = None
+    p.state = ui_state
+    p.do_not_save_grid = True
+    p.do_not_save_samples = not save_frames
+    if 'I2V' in model:
+        if init_image is None:
+            return video_utils.queue_err('init image not set')
+        p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
+
+    # cleanup memory
+    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
+    devices.torch_gc(force=True)
+
+    # set args
+    processing.fix_seed(p)
+    video_utils.set_vae_params(p.frames, vae_tile_frames)
+    video_utils.set_prompt(p)
+    p.task_args['output_type'] = 'pil'
+    p.ops.append('video')
+    orig_dynamic_shift = shared.opts.schedulers_dynamic_shift
+    orig_sampler_shift = shared.opts.schedulers_shift
+    shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift # todo video sampler dynamic shift
+    shared.opts.data['schedulers_shift'] = sampler_shift
+    debug(f'Video: task_args={p.task_args}')
+
+    # run processing
+    shared.state.disable_preview = True
+    shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}')
+    err = None
+    t0 = time.time()
+    try:
+        processed = processing.process_images(p)
+    except Exception as e:
+        err = str(e)
+        errors.display(e, 'video')
+    t1 = time.time()
+    shared.state.disable_preview = False
+    shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift
+    shared.opts.data['schedulers_shift'] = orig_sampler_shift
+    p.close()
+
+    # done
+    if err:
+        return video_utils.queue_err(err)
+    if processed is None or len(processed.images) == 0:
+        return video_utils.queue_err('processing failed')
+    shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}')
+    video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
+    generation_info_js = processed.js() if processed is not None else ''
+    return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)
@@ -0,0 +1,91 @@
+import os
+import time
+from modules import shared, errors, sd_models, processing, devices, images, ui_common
+from modules.video_models import models_def, video_utils
+
+
+debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
+
+
+def generate(*args, **kwargs):
+    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
+    if engine is None or model is None or engine == 'None' or model == 'None':
+        return video_utils.queue_err('model not selected')
+    found = [model.name for model in models_def.models.get(engine, [])]
+    selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
+    if not shared.sd_loaded or 'Cog' not in shared.sd_model.__class__.__name__:
+        video_utils.load_model(selected)
+    if not shared.sd_loaded or 'Cog' not in shared.sd_model.__class__.__name__:
+        return video_utils.queue_err('model not loaded')
+    debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}')
+
+    p = processing.StableDiffusionProcessingVideo(
+        sd_model=shared.sd_model,
+        prompt=prompt,
+        negative_prompt=negative,
+        styles=styles,
+        seed=int(seed),
+        sampler_name = processing.get_sampler_name(sampler_index),
+        sampler_shift=float(sampler_shift),
+        steps=int(steps),
+        width=8 * int(width // 8),
+        height=8 * int(height // 8),
+        frames=int(frames),
+        init_image=init_image,
+        cfg_scale=float(guidance_scale),
+        diffusers_guidance_rescale=float(guidance_true),
+        vae_type=vae_type,
+        vae_tile_frames=int(vae_tile_frames),
+        override_settings=override_settings,
+    )
+    p.scripts = None
+    p.script_args = None
+    p.state = ui_state
+    p.do_not_save_grid = True
+    p.do_not_save_samples = not save_frames
+    if 'I2V' in model:
+        if init_image is None:
+            return video_utils.queue_err('init image not set')
+        p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
+
+    # cleanup memory
+    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
+    devices.torch_gc(force=True)
+
+    # set args
+    processing.fix_seed(p)
+    video_utils.set_vae_params(p.frames, vae_tile_frames)
+    video_utils.set_prompt(p)
+    p.task_args['output_type'] = 'pil'
+    p.ops.append('video')
+    orig_dynamic_shift = shared.opts.schedulers_dynamic_shift
+    orig_sampler_shift = shared.opts.schedulers_shift
+    shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift # todo video sampler dynamic shift
+    shared.opts.data['schedulers_shift'] = sampler_shift
+    debug(f'Video: task_args={p.task_args}')
+
+    # run processing
+    shared.state.disable_preview = True
+    shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}')
+    err = None
+    t0 = time.time()
+    try:
+        processed = processing.process_images(p)
+    except Exception as e:
+        err = str(e)
+        errors.display(e, 'video')
+    t1 = time.time()
+    shared.state.disable_preview = False
+    shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift
+    shared.opts.data['schedulers_shift'] = orig_sampler_shift
+    p.close()
+
+    # done
+    if err:
+        return video_utils.queue_err(err)
+    if processed is None or len(processed.images) == 0:
+        return video_utils.queue_err('processing failed')
+    shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}')
+    video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
+    generation_info_js = processed.js() if processed is not None else ''
+    return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)
@@ -11,9 +11,9 @@ def generate(*args, **kwargs):
    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
    if engine is None or model is None or engine == 'None' or model == 'None':
        return video_utils.queue_err('model not selected')
+    found = [model.name for model in models_def.models.get(engine, [])]
+    selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
    if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__:
-        found = [model.name for model in models_def.models.get(engine, [])]
-        selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
        video_utils.load_model(selected)
    if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__:
        return video_utils.queue_err('model not loaded')
@@ -21,6 +21,8 @@ def generate(*args, **kwargs):

    p = processing.StableDiffusionProcessingVideo(
        sd_model=shared.sd_model,
+        prompt=prompt,
+        negative_prompt=negative,
        styles=styles,
        seed=int(seed),
        sampler_name = processing.get_sampler_name(sampler_index),
@@ -50,29 +52,16 @@ def generate(*args, **kwargs):
    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
    devices.torch_gc(force=True)

-    # handle sampler and seed
+    # set args
+    processing.fix_seed(p)
+    video_utils.set_vae_params(p.frames, vae_tile_frames)
+    video_utils.set_prompt(p)
+    p.task_args['output_type'] = 'pil'
+    p.ops.append('video')
    orig_dynamic_shift = shared.opts.schedulers_dynamic_shift
    orig_sampler_shift = shared.opts.schedulers_shift
    shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift # todo video sampler dynamic shift
    shared.opts.data['schedulers_shift'] = sampler_shift
-
-    # handle vae
-    if vae_tile_frames > p.frames:
-        shared.sd_model.vae.tile_sample_min_num_frames = vae_tile_frames
-        shared.sd_model.vae.use_framewise_decoding = True
-        shared.sd_model.vae.enable_tiling()
-    else:
-        shared.sd_model.vae.use_framewise_decoding = False
-        shared.sd_model.vae.disable_tiling()
-
-    # set args
-    processing.fix_seed(p)
-    p.prompt = shared.prompt_styles.apply_styles_to_prompt(prompt, p.styles)
-    p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(negative, p.styles)
-    p.task_args['prompt'] = p.prompt
-    p.task_args['negative_prompt'] = p.negative_prompt
-    p.task_args['output_type'] = 'pil'
-    p.ops.append('video')
    debug(f'Video: task_args={p.task_args}')

    # run processing
@@ -90,15 +79,13 @@ def generate(*args, **kwargs):
    shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift
    shared.opts.data['schedulers_shift'] = orig_sampler_shift
    p.close()
+
+    # done
    if err:
        return video_utils.queue_err(err)
    if processed is None or len(processed.images) == 0:
        return video_utils.queue_err('processing failed')
    shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}')
-    if video_type != 'None':
-        video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
-    else:
-        video_file = None
-
+    video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
    generation_info_js = processed.js() if processed is not None else ''
    return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)
@@ -0,0 +1,91 @@
+import os
+import time
+from modules import shared, errors, sd_models, processing, devices, images, ui_common
+from modules.video_models import models_def, video_utils
+
+
+debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
+
+
+def generate(*args, **kwargs):
+    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
+    if engine is None or model is None or engine == 'None' or model == 'None':
+        return video_utils.queue_err('model not selected')
+    found = [model.name for model in models_def.models.get(engine, [])]
+    selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
+    if not shared.sd_loaded or 'LTX' not in shared.sd_model.__class__.__name__:
+        video_utils.load_model(selected)
+    if not shared.sd_loaded or 'LTX' not in shared.sd_model.__class__.__name__:
+        return video_utils.queue_err('model not loaded')
+    debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}')
+
+    p = processing.StableDiffusionProcessingVideo(
+        sd_model=shared.sd_model,
+        prompt=prompt,
+        negative_prompt=negative,
+        styles=styles,
+        seed=int(seed),
+        sampler_name = processing.get_sampler_name(sampler_index),
+        sampler_shift=float(sampler_shift),
+        steps=int(steps),
+        width=32 * int(width // 32),
+        height=32 * int(height // 32),
+        frames=int(frames),
+        init_image=init_image,
+        cfg_scale=float(guidance_scale),
+        diffusers_guidance_rescale=float(guidance_true),
+        vae_type=vae_type,
+        vae_tile_frames=int(vae_tile_frames),
+        override_settings=override_settings,
+    )
+    p.scripts = None
+    p.script_args = None
+    p.state = ui_state
+    p.do_not_save_grid = True
+    p.do_not_save_samples = not save_frames
+    if 'I2V' in model:
+        if init_image is None:
+            return video_utils.queue_err('init image not set')
+        p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
+
+    # cleanup memory
+    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
+    devices.torch_gc(force=True)
+
+    # set args
+    processing.fix_seed(p)
+    video_utils.set_vae_params(p.frames, vae_tile_frames)
+    video_utils.set_prompt(p)
+    p.task_args['output_type'] = 'pil'
+    p.ops.append('video')
+    orig_dynamic_shift = shared.opts.schedulers_dynamic_shift
+    orig_sampler_shift = shared.opts.schedulers_shift
+    shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift # todo video sampler dynamic shift
+    shared.opts.data['schedulers_shift'] = sampler_shift
+    debug(f'Video: task_args={p.task_args}')
+
+    # run processing
+    shared.state.disable_preview = True
+    shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}')
+    err = None
+    t0 = time.time()
+    try:
+        processed = processing.process_images(p)
+    except Exception as e:
+        err = str(e)
+        errors.display(e, 'video')
+    t1 = time.time()
+    shared.state.disable_preview = False
+    shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift
+    shared.opts.data['schedulers_shift'] = orig_sampler_shift
+    p.close()
+
+    # done
+    if err:
+        return video_utils.queue_err(err)
+    if processed is None or len(processed.images) == 0:
+        return video_utils.queue_err('processing failed')
+    shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}')
+    video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
+    generation_info_js = processed.js() if processed is not None else ''
+    return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)
@@ -0,0 +1,91 @@
+import os
+import time
+from modules import shared, errors, sd_models, processing, devices, images, ui_common
+from modules.video_models import models_def, video_utils
+
+
+debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
+
+
+def generate(*args, **kwargs):
+    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
+    if engine is None or model is None or engine == 'None' or model == 'None':
+        return video_utils.queue_err('model not selected')
+    found = [model.name for model in models_def.models.get(engine, [])]
+    selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
+    if not shared.sd_loaded or 'Mochi' not in shared.sd_model.__class__.__name__:
+        video_utils.load_model(selected)
+    if not shared.sd_loaded or 'Mochi' not in shared.sd_model.__class__.__name__:
+        return video_utils.queue_err('model not loaded')
+    debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}')
+
+    p = processing.StableDiffusionProcessingVideo(
+        sd_model=shared.sd_model,
+        prompt=prompt,
+        negative_prompt=negative,
+        styles=styles,
+        seed=int(seed),
+        sampler_name = processing.get_sampler_name(sampler_index),
+        sampler_shift=float(sampler_shift),
+        steps=int(steps),
+        width=8 * int(width // 8),
+        height=8 * int(height // 8),
+        frames=int(frames),
+        init_image=init_image,
+        cfg_scale=float(guidance_scale),
+        diffusers_guidance_rescale=float(guidance_true),
+        vae_type=vae_type,
+        vae_tile_frames=int(vae_tile_frames),
+        override_settings=override_settings,
+    )
+    p.scripts = None
+    p.script_args = None
+    p.state = ui_state
+    p.do_not_save_grid = True
+    p.do_not_save_samples = not save_frames
+    if 'I2V' in model:
+        if init_image is None:
+            return video_utils.queue_err('init image not set')
+        p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
+
+    # cleanup memory
+    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
+    devices.torch_gc(force=True)
+
+    # set args
+    processing.fix_seed(p)
+    video_utils.set_vae_params(p.frames, vae_tile_frames)
+    video_utils.set_prompt(p)
+    p.task_args['output_type'] = 'pil'
+    p.ops.append('video')
+    orig_dynamic_shift = shared.opts.schedulers_dynamic_shift
+    orig_sampler_shift = shared.opts.schedulers_shift
+    shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift # todo video sampler dynamic shift
+    shared.opts.data['schedulers_shift'] = sampler_shift
+    debug(f'Video: task_args={p.task_args}')
+
+    # run processing
+    shared.state.disable_preview = True
+    shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}')
+    err = None
+    t0 = time.time()
+    try:
+        processed = processing.process_images(p)
+    except Exception as e:
+        err = str(e)
+        errors.display(e, 'video')
+    t1 = time.time()
+    shared.state.disable_preview = False
+    shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift
+    shared.opts.data['schedulers_shift'] = orig_sampler_shift
+    p.close()
+
+    # done
+    if err:
+        return video_utils.queue_err(err)
+    if processed is None or len(processed.images) == 0:
+        return video_utils.queue_err('processing failed')
+    shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}')
+    video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
+    generation_info_js = processed.js() if processed is not None else ''
+    return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)
@@ -1,6 +1,6 @@
 import os
 import time
-from modules import shared, timer, sd_models, sd_checkpoint, model_quant, devices
+from modules import shared, errors, timer, sd_models, sd_checkpoint, model_quant, devices
 from modules.video_models import models_def


@@ -18,6 +18,32 @@ def get_quant(args):
    return None


+def get_url(url):
+    return f'&nbsp <a href="{url}" target="_blank" rel="noopener noreferrer" style="color: var(--button-primary-background-fill); font-weight: normal">{url}</a><br>' if url else ''
+
+
+def set_prompt(p):
+    p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles)
+    p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles)
+    p.task_args['prompt'] = p.prompt
+    p.task_args['negative_prompt'] = p.negative_prompt
+
+
+def set_vae_params(frames, tile_frames):
+    if tile_frames > frames:
+        if hasattr(shared.sd_model.vae, 'tile_sample_min_num_frames'):
+            shared.sd_model.vae.tile_sample_min_num_frames = tile_frames
+        if hasattr(shared.sd_model.vae, 'use_framewise_decoding'):
+            shared.sd_model.vae.use_framewise_decoding = True
+        if hasattr(shared.sd_model.vae, 'enable_tiling'):
+            shared.sd_model.vae.enable_tiling()
+    else:
+        if hasattr(shared.sd_model.vae, 'use_framewise_decoding'):
+            shared.sd_model.vae.use_framewise_decoding = False
+        if hasattr(shared.sd_model.vae, 'disable_tiling'):
+            shared.sd_model.vae.disable_tiling()
+
+
 def hijack_vae_decode(*args, **kwargs):
    t0 = time.time()
    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'])
@@ -62,6 +88,7 @@ def load_model(selected: models_def.Model):
        )
    except Exception as e:
        shared.log.error(f'video load: module=te cls={selected.te_cls.__name__} {e}')
+        errors.display(e, 'video')
        text_encoder = None

    # transformer
@@ -77,6 +104,7 @@ def load_model(selected: models_def.Model):
        )
    except Exception as e:
        shared.log.error(f'video load: module=transformer cls={selected.dit_cls.__name__} {e}')
+        errors.display(e, 'video')
        transformer = None

    # model
@@ -91,6 +119,7 @@ def load_model(selected: models_def.Model):
        )
    except Exception as e:
        shared.log.error(f'video load: module=pipe repo="{selected.repo}" cls={selected.repo_cls.__name__} {e}')
+        errors.display(e, 'video')

    t1 = time.time()
    sd_models.set_diffuser_options(shared.sd_model)