update video tab

Signed-off-by: Vladimir Mandic <mandic00@live.com>

TODO.md

@@ -4,6 +4,14 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
 
 ## Current
 
+- Video tab:
+  - remote vae
+  - tiny vae
+  - lora
+  - accelerators: teacache, pab, fastercache, paraattention, perflow
+  - modernui tab
+- Detailer daemon: https://github.com/muerrilla/sd-webui-detail-daemon/blob/main/scripts/detail_daemon.py
+
 ## Future Candidates
 
 - Redesign postprocessing  

modules/onnx_impl/execution_providers.py

@@ -107,10 +107,14 @@ def install_execution_provider(ep: ExecutionProvider):
     elif ep == ExecutionProvider.OpenVINO:
         packages.append("openvino")
         packages.append("onnxruntime-openvino")
+    log.info(f'ONNX install: {packages}')
     for package in packages:
         res += install(package)
     res += '</pre><br>'
     res += 'Server restart required'
     log.info("Server restart required")
-    importlib.reload(ort)
+    try:
+        importlib.reload(ort)
+    except Exception:
+        pass
     return res

modules/processing_diffusers.py

@@ -71,8 +71,10 @@ def process_base(p: processing.StableDiffusionProcessing):
         eta=shared.opts.scheduler_eta,
         guidance_scale=p.cfg_scale,
         guidance_rescale=p.diffusers_guidance_rescale,
+        true_cfg_scale=p.diffusers_guidance_rescale,
         denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None,
         denoising_end=p.refiner_start if use_refiner_start else 1 if use_denoise_start else None,
+        num_frames=getattr(p, 'frames', None),
         output_type='latent',
         clip_skip=p.clip_skip,
         desc='Base',
@@ -364,7 +366,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output):
             else:
                 width = getattr(p, 'width', 0)
                 height = getattr(p, 'height', 0)
-            frames = p.task_args.get('num_frames', None)
+            frames = p.task_args.get('num_frames', None) or getattr(p, 'frames', None)
             if isinstance(output.images, list):
                 results = []
                 for i in range(len(output.images)):

modules/ui_caption.py

@@ -45,9 +45,9 @@ def create_ui():
                         vlm_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model', elem_id='vlm_model')
                     with gr.Accordion(label='Advanced options', open=False, visible=True):
                         with gr.Row():
-                            vlm_max_tokens = gr.Slider(label='Max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1, elem_id='vlm_max_tokens')
-                            vlm_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1, elem_id='vlm_num_beams')
-                            vlm_temperature = gr.Slider(label='Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01, elem_id='vlm_temperature')
+                            vlm_max_tokens = gr.Slider(label='VLM max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1, elem_id='vlm_max_tokens')
+                            vlm_num_beams = gr.Slider(label='VLM num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1, elem_id='vlm_num_beams')
+                            vlm_temperature = gr.Slider(label='VLM Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01, elem_id='vlm_temperature')
                         with gr.Row():
                             vlm_top_k = gr.Slider(label='Top-K', value=shared.opts.interrogate_vlm_top_k, minimum=0, maximum=99, step=1, elem_id='vlm_top_k')
                             vlm_top_p = gr.Slider(label='Top-P', value=shared.opts.interrogate_vlm_top_p, minimum=0.0, maximum=1.0, step=0.01, elem_id='vlm_top_p')
@@ -90,7 +90,7 @@ def create_ui():
                             clip_max_flavors = gr.Slider(label='Max flavors', value=shared.opts.interrogate_clip_max_flavors, minimum=1, maximum=64, step=1, elem_id='clip_max_flavors')
                             clip_flavor_count = gr.Slider(label='Intermediates', value=shared.opts.interrogate_clip_flavor_count, minimum=256, maximum=4096, step=8, elem_id='clip_flavor_intermediate_count')
                         with gr.Row():
-                            clip_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_clip_num_beams, minimum=1, maximum=16, step=1, elem_id='clip_num_beams')
+                            clip_num_beams = gr.Slider(label='CLiP num beams', value=shared.opts.interrogate_clip_num_beams, minimum=1, maximum=16, step=1, elem_id='clip_num_beams')
                         clip_min_length.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
                         clip_max_length.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
                         clip_chunk_size.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])

modules/ui_video.py

@@ -1,30 +1,23 @@
-# TODO hunyuanvideo: prompt_template, lora
-# TODO hunyuanvideo: teacache, pab, fastercache, paraattention, perflow
-# TODO modernui video tab
-
 import gradio as gr
 from modules import shared, sd_models, timer, images, ui_common, ui_sections, ui_symbols, call_queue, generation_parameters_copypaste
 from modules.ui_components import ToolButton
-from modules.video_models import hunyuan
+from modules.video_models import models_def, video_utils, hunyuan, ltx
 
 
 def engine_change(engine):
-    found = [model.name for model in hunyuan.models.get(engine, [])]
+    found = [model.name for model in models_def.models.get(engine, [])]
     return gr.update(choices=found, value=found[0] if len(found) > 0 else None)
 
 
 def model_change(engine, model):
-    found = [model.name for model in hunyuan.models.get(engine, [])]
-    selected = [m for m in hunyuan.models[engine] if m.name == model][0] if len(found) > 0 else None
+    found = [model.name for model in models_def.models.get(engine, [])]
+    selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
     if selected:
         if 'None' in selected.name:
             sd_models.unload_model_weights()
             msg = 'Video model unloaded'
-        elif 'Hunyuan' in selected.name:
-            msg = hunyuan.load(selected)
-        elif model != 'None':
-            msg = f'Video model not found: engine={engine} model={model}'
-            shared.log.error(msg)
+        else:
+            msg = video_utils.load_model(selected)
     else:
         sd_models.unload_model_weights()
         msg = 'Video model unloaded'
@@ -33,10 +26,13 @@ def model_change(engine, model):
 
 def run_video(*args):
     engine, model = args[2], args[3]
-    found = [model.name for model in hunyuan.models.get(engine, [])]
-    selected = [m for m in hunyuan.models[engine] if m.name == model][0] if len(found) > 0 else None
+    found = [model.name for model in models_def.models.get(engine, [])]
+    selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
     if selected and 'Hunyuan' in selected.name:
         return hunyuan.generate(*args)
+    elif selected and 'LTX' in selected.name:
+        pass
+        # return ltx.generate(*args)
     shared.log.error(f'Video model not found: args={args}')
     return [], None, '', '', f'Video model not found: engine={engine} model={model}'
 
@@ -57,7 +53,7 @@ def create_ui():
             with gr.Column(variant='compact', elem_id="video_settings", scale=1):
 
                 with gr.Row():
-                    engine = gr.Dropdown(label='Engine', choices=list(hunyuan.models), value='None', elem_id="video_engine")
+                    engine = gr.Dropdown(label='Engine', choices=list(models_def.models), value='None', elem_id="video_engine")
                     model = gr.Dropdown(label='Model', choices=[''], value=None, elem_id="video_model")
                 with gr.Row():
                     width, height = ui_sections.create_resolution_inputs('video', default_width=720, default_height=480)
@@ -69,6 +65,7 @@ def create_ui():
                 steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video")
                 with gr.Row():
                     sampler_shift = gr.Slider(label='Sampler shift', minimum=0.0, maximum=20.0, step=0.1, value=7.0, elem_id="video_scheduler_shift")
+                    dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift")
                 with gr.Row():
                     guidance_scale = gr.Slider(label='Guidance scale', minimum=0.0, maximum=14.0, step=0.1, value=6.0, elem_id="video_guidance_scale")
                     guidance_true = gr.Slider(label='True guidance', minimum=0.0, maximum=14.0, step=0.1, value=1.0, elem_id="video_guidance_true")
@@ -86,7 +83,7 @@ def create_ui():
                 override_settings = ui_common.create_override_inputs('video')
 
             # output panel with gallery and video tabs
-            with gr.Column(elem_id='video-output-column', scale=3) as _column_output:
+            with gr.Column(elem_id='video-output-column', scale=2) as _column_output:
                 with gr.Tabs(elem_classes=['video-output-tabs'], elem_id='video-output-tabs'):
                     with gr.Tab('Frames', id='out-gallery'):
                         gallery, gen_info, html_info, _html_info_formatted, html_log = ui_common.create_output_panel("video", prompt=prompt, preview=False, transfer=False, scale=2)
@@ -124,7 +121,7 @@ def create_ui():
             width, height,
             frames,
             steps, sampler_index,
-            sampler_shift,
+            sampler_shift, dynamic_shift,
             seed,
             guidance_scale, guidance_true,
             init_image,

modules/video_models/hunyuan.py

@@ -1,187 +1,22 @@
-from dataclasses import dataclass
 import os
 import time
-import torch
-import transformers
-import diffusers
-from modules import shared, errors, sd_models, sd_checkpoint, sd_samplers, processing, model_quant, devices, images, timer, ui_common
+from modules import shared, errors, sd_models, processing, devices, images, ui_common
+from modules.video_models import models_def, video_utils
 
 
-@dataclass
-class Model():
-    name: str
-    repo: str
-    dit: str
-    subfolder: str
-
-models = {
-    'None': [],
-    'Hunyuan Video': [
-        Model(name='None', repo=None, dit=None, subfolder=None),
-        Model(name='Hunyuan Video T2V', repo='hunyuanvideo-community/HunyuanVideo', dit='hunyuanvideo-community/HunyuanVideo', subfolder='transformer'),
-        Model(name='Hunyuan Video I2V', repo='hunyuanvideo-community/HunyuanVideo-I2V', dit='hunyuanvideo-community/HunyuanVideo-I2V', subfolder='transformer'), # https://github.com/huggingface/diffusers/pull/10983
-        Model(name='SkyReels Hunyuan T2V', repo='hunyuanvideo-community/HunyuanVideo', dit='Skywork/SkyReels-V1-Hunyuan-T2V', subfolder=None), # https://github.com/huggingface/diffusers/pull/10837
-        Model(name='SkyReels Hunyuan I2V', repo='hunyuanvideo-community/HunyuanVideo', dit='Skywork/SkyReels-V1-Hunyuan-I2V', subfolder=None),
-        Model(name='Fast Hunyuan T2V', repo='hunyuanvideo-community/HunyuanVideo', dit='FastVideo/FastHunyuan-diffusers', subfolder='transformer'), # https://github.com/hao-ai-lab/FastVideo/blob/8a77cf22c9b9e7f931f42bc4b35d21fd91d24e45/fastvideo/models/hunyuan/inference.py#L213
-    ]
-}
 debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
-loaded_model = None
-prompt_template = {
-    "template": (
-        "<|start_header_id|>system<|end_header_id|>"
-        "\nDescribe the video by detailing the following aspects: \n"
-        "1. The main content and theme of the video.\n"
-        "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects.\n"
-        "3. Actions, events, behaviors, temporal relationships, and physical movement changes of the objects.\n"
-        "4. Background environment, light, style and atmosphere.\n"
-        "5. Camera angles, movements, and transitions used in the video.\n"
-        "<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>"
-    ),
-    "crop_start": 95,
-}
-
-
-def hijack_decode(*args, **kwargs):
-    t0 = time.time()
-    vae: diffusers.AutoencoderKLHunyuanVideo = shared.sd_model.vae
-    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'])
-    res = shared.sd_model.vae.orig_decode(*args, **kwargs)
-    t1 = time.time()
-    timer.process.add('vae', t1-t0)
-    debug(f'Video: vae={vae.__class__.__name__} tile={vae.tile_sample_min_width}:{vae.tile_sample_min_height}:{vae.tile_sample_min_num_frames} stride={vae.tile_sample_stride_width}:{vae.tile_sample_stride_height}:{vae.tile_sample_stride_num_frames} time={t1-t0:.2f}')
-    return res
-
-
-def hijack_encode_prompt(*args, **kwargs):
-    t0 = time.time()
-    res = shared.sd_model.orig_encode_prompt(*args, **kwargs)
-    t1 = time.time()
-    timer.process.add('te', t1-t0)
-    debug(f'Video: te={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}')
-    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
-    return res
-
-
-def get_quant(args):
-    if args is not None and "quantization_config" in args:
-        return args['quantization_config'].__class__.__name__
-    return None
-
-
-def load(selected):
-    if selected is None:
-        return
-    global loaded_model # pylint: disable=global-statement
-    if loaded_model == selected.name:
-        return
-    sd_models.unload_model_weights()
-    t0 = time.time()
-
-    quant_args = model_quant.create_config(module='Model')
-    cls = diffusers.HunyuanVideoTransformer3DModel
-    try:
-        debug(f'Video load: module=transofrmer repo="{selected.dit}" subfolder="{selected.subfolder}" cls={cls.__name__} quant={get_quant(quant_args)}')
-        transformer = cls.from_pretrained(
-            pretrained_model_name_or_path=selected.dit,
-            subfolder=selected.subfolder,
-            torch_dtype=devices.dtype,
-            cache_dir=shared.opts.hfcache_dir,
-            **quant_args
-        )
-    except Exception as e:
-        shared.log.error(f'video load: module=transformer repo="{selected.dit}" subfolder="{selected.subfolder}" cls={cls.__name__} {e}')
-
-    quant_args = model_quant.create_config(module='Text Encoder')
-    if 'I2V' in selected.repo:
-        cls = transformers.LlavaForConditionalGeneration
-    else:
-        cls = transformers.LlamaModel
-    try:
-        debug(f'Video load: module=te repo="{selected.repo}" cls={cls.__name__} quant={get_quant(quant_args)}')
-        text_encoder = cls.from_pretrained(
-            pretrained_model_name_or_path=selected.repo,
-            subfolder="text_encoder",
-            cache_dir=shared.opts.hfcache_dir,
-            torch_dtype=devices.dtype,
-            # torch_dtype='auto', # special case as text and vision nested models have different dtypes
-            # attn_implementation="flash_attention_2", # testing different attention types
-            **quant_args
-        )
-    except Exception as e:
-        shared.log.error(f'video load: module=te repo="{selected.repo}" cls={cls.__name__} {e}')
-
-    cls = transformers.CLIPTextModel
-    try:
-        debug(f'Video load: module=clip repo="{selected.repo}" cls={cls.__name__} quant=None')
-        text_encoder_2 = transformers.CLIPTextModel.from_pretrained(
-            pretrained_model_name_or_path=selected.repo,
-            subfolder="text_encoder_2",
-            cache_dir=shared.opts.hfcache_dir,
-            torch_dtype=devices.dtype,
-        )
-    except Exception as e:
-        shared.log.error(f'video load: module=clip repo="{selected.repo}" cls={cls.__name__} {e}')
-
-    cls = diffusers.AutoencoderKLHunyuanVideo
-    try:
-        debug(f'Video load: module=vae repo="{selected.repo}" cls={cls.__name__} quant=None')
-        vae = diffusers.AutoencoderKLHunyuanVideo.from_pretrained(
-            pretrained_model_name_or_path=selected.repo,
-            subfolder="vae",
-            cache_dir=shared.opts.hfcache_dir,
-            torch_dtype=devices.dtype,
-        )
-    except Exception as e:
-        shared.log.error(f'video load: module=vae repo="{selected.repo}" cls={cls.__name__} {e}')
-
-    if selected.name == 'Hunyuan Video I2V':
-        cls = diffusers.HunyuanVideoImageToVideoPipeline
-    elif selected.name == 'SkyReels Hunyuan I2V':
-        cls = diffusers.HunyuanSkyreelsImageToVideoPipeline
-    else:
-        cls = diffusers.HunyuanVideoPipeline
-    try:
-        debug(f'Video load: module=pipe repo="{selected.repo}" cls={cls.__name__} quant=None')
-        shared.sd_model = cls.from_pretrained(
-            pretrained_model_name_or_path=selected.repo,
-            transformer=transformer,
-            text_encoder=text_encoder,
-            text_encoder_2=text_encoder_2,
-            vae=vae,
-            cache_dir=shared.opts.hfcache_dir,
-            torch_dtype=devices.dtype,
-        )
-    except Exception as e:
-        shared.log.error(f'video load: module=pipe repo="{selected.repo}" cls={cls.__name__} {e}')
-
-    t1 = time.time()
-    sd_models.set_diffuser_options(shared.sd_model)
-    shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(selected.repo)
-    shared.sd_model.sd_model_hash = None
-    shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode
-    shared.sd_model.vae.decode = hijack_decode
-    shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt
-    shared.sd_model.encode_prompt = hijack_encode_prompt
-    shared.sd_model.vae.enable_slicing()
-    loaded_model = selected.name
-    msg = f'Video load: cls={shared.sd_model.__class__.__name__} model="{selected.name}" time={t1-t0:.2f}'
-    shared.log.info(msg)
-    return msg
 
 
 def generate(*args, **kwargs):
-    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
+    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
     if engine is None or model is None or engine == 'None' or model == 'None':
-        shared.log.error('Video: model not selected')
-        return [], None, '', '', 'Video model not selected'
+        return video_utils.queue_err('model not selected')
     if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__:
-        found = [model.name for model in models.get(engine, [])]
-        selected = [m for m in models[engine] if m.name == model][0] if len(found) > 0 else None
-        load(selected)
+        found = [model.name for model in models_def.models.get(engine, [])]
+        selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
+        video_utils.load_model(selected)
     if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__:
-        shared.log.error('Video: model not loaded')
-        return [], None, '', '', 'Video model not loaded'
+        return video_utils.queue_err('model not loaded')
     debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}')
 
     p = processing.StableDiffusionProcessingVideo(
@@ -202,27 +37,24 @@ def generate(*args, **kwargs):
         override_settings=override_settings,
     )
     p.scripts = None
-    p.script_args = args
+    p.script_args = None
     p.state = ui_state
     p.do_not_save_grid = True
     p.do_not_save_samples = not save_frames
     if 'I2V' in model:
         if init_image is None:
-            shared.log.error('Video: init image not set')
-            return [], None, '', '', 'Error: init image not set'
-        p.task_args['image'] = init_image
-        # from PIL import Image
-        # p.task_args['image'] = init_image.resize((336, 336), Image.Resampling.LANCZOS)
+            return video_utils.queue_err('init image not set')
+        p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
 
+    # cleanup memory
     shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
     devices.torch_gc(force=True)
 
     # handle sampler and seed
-    if p.sampler_name != 'Default':
-        shared.sd_model.scheduler = sd_samplers.create_sampler(p.sampler_name, shared.sd_model)
-        p.sampler_name = 'Default' # avoid double creation
-    if hasattr(shared.sd_model.scheduler, '_shift') and sampler_shift > 0:
-        shared.sd_model.scheduler._shift = sampler_shift # pylint: disable=protected-access
+    orig_dynamic_shift = shared.opts.schedulers_dynamic_shift
+    orig_sampler_shift = shared.opts.schedulers_shift
+    shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift # todo video sampler dynamic shift
+    shared.opts.data['schedulers_shift'] = sampler_shift
 
     # handle vae
     if vae_tile_frames > p.frames:
@@ -237,39 +69,32 @@ def generate(*args, **kwargs):
     processing.fix_seed(p)
     p.prompt = shared.prompt_styles.apply_styles_to_prompt(prompt, p.styles)
     p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(negative, p.styles)
-    p.task_args['width'] = p.width
-    p.task_args['height'] = p.height
-    p.task_args['num_inference_steps'] = p.steps
-    p.task_args['num_frames'] = p.frames
-    p.task_args['generator'] = torch.manual_seed(p.seed)
-    p.task_args['guidance_scale'] = p.cfg_scale
-    p.task_args['true_cfg_scale'] = p.diffusers_guidance_rescale
-    # p.task_args['prompt_template'] = prompt_template # t2v and i2v have different templates
-    p.task_args['output_type'] = 'pil'
     p.task_args['prompt'] = p.prompt
     p.task_args['negative_prompt'] = p.negative_prompt
+    p.task_args['output_type'] = 'pil'
     p.ops.append('video')
     debug(f'Video: task_args={p.task_args}')
 
     # run processing
     shared.state.disable_preview = True
     shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}')
+    err = None
     t0 = time.time()
     try:
         processed = processing.process_images(p)
     except Exception as e:
-        shared.log.error(f'Video: exception={e}')
+        err = str(e)
         errors.display(e, 'video')
-        processed = None
-        shared.state.disable_preview = False
-        return [], None, '', '', str(e)
     t1 = time.time()
     shared.state.disable_preview = False
-
+    shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift
+    shared.opts.data['schedulers_shift'] = orig_sampler_shift
     p.close()
+    if err:
+        return video_utils.queue_err(err)
     if processed is None or len(processed.images) == 0:
-        return [], None, '', '', 'Video: processing failed'
-    shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}')
+        return video_utils.queue_err('processing failed')
+    shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}')
     if video_type != 'None':
         video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
     else:

modules/video_models/models_def.py

@@ -0,0 +1,67 @@
+from dataclasses import dataclass
+import diffusers
+import transformers
+
+
+@dataclass
+class Model():
+    name: str
+    repo: str = None
+    repo_cls: classmethod = None
+    dit: str = None
+    dit_cls: classmethod = None
+    dit_folder: str = 'transformer'
+    te: str = None
+    te_cls: classmethod = None
+    te_folder: str = 'text_encoder'
+    te_hijack: bool = True
+    vae_hijack: bool = True
+
+
+models = {
+    'None': [],
+    'Hunyuan Video': [
+        Model(name='None'),
+        Model(name='Hunyuan Video T2V',
+              repo='hunyuanvideo-community/HunyuanVideo',
+              repo_cls=diffusers.HunyuanVideoPipeline,
+              te_cls=transformers.LlamaModel,
+              dit_cls=diffusers.HunyuanVideoTransformer3DModel),
+        Model(name='Hunyuan Video I2V', # https://github.com/huggingface/diffusers/pull/10983
+              repo='hunyuanvideo-community/HunyuanVideo-I2V',
+              repo_cls=diffusers.HunyuanVideoImageToVideoPipeline,
+              te_cls=transformers.LlavaForConditionalGeneration,
+              dit_cls=diffusers.HunyuanVideoTransformer3DModel),
+        Model(name='SkyReels Hunyuan T2V', # https://github.com/huggingface/diffusers/pull/10837
+              repo='hunyuanvideo-community/HunyuanVideo',
+              repo_cls=diffusers.HunyuanVideoPipeline,
+              te_cls=transformers.LlamaModel,
+              dit='Skywork/SkyReels-V1-Hunyuan-T2V',
+              dit_folder=None,
+              dit_cls=diffusers.HunyuanVideoTransformer3DModel),
+        Model(name='SkyReels Hunyuan I2V', # https://github.com/huggingface/diffusers/pull/10837
+              repo='hunyuanvideo-community/HunyuanVideo',
+              te_cls=transformers.LlamaModel,
+              dit='Skywork/SkyReels-V1-Hunyuan-I2V',
+              dit_folder=None,
+              dit_cls=diffusers.HunyuanVideoTransformer3DModel),
+        Model(name='Fast Hunyuan T2V', # https://github.com/hao-ai-lab/FastVideo/blob/8a77cf22c9b9e7f931f42bc4b35d21fd91d24e45/fastvideo/models/hunyuan/inference.py#L213
+              repo='hunyuanvideo-community/HunyuanVideo',
+              repo_cls=diffusers.HunyuanVideoPipeline,
+              te_cls=transformers.LlamaModel,
+              dit='FastVideo/FastHunyuan-diffusers',
+              dit_cls=diffusers.HunyuanVideoTransformer3DModel),
+    ],
+}
+
+"""
+'LTX Video': [
+    Model(name='None'),
+    Model(name='LTXVideo 0.9.0 T2V', repo='a-r-r-o-w/LTX-Video-diffusers', subfolder='transformer'),
+    Model(name='LTXVideo 0.9.1 T2V', repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', subfolder='transformer'),
+    Model(name='LTXVideo 0.9.5 T2V', repo='Lightricks/LTX-Video-0.9.5'), # https://github.com/huggingface/diffusers/pull/10968
+    Model(name='LTXVideo 0.9.0 I2V', repo='a-r-r-o-w/LTX-Video-diffusers', subfolder='transformer'),
+    Model(name='LTXVideo 0.9.1 I2V', repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', subfolder='transformer'),
+    Model(name='LTXVideo 0.9.5 I2V', repo='Lightricks/LTX-Video-0.9.5', subfolder='transformer'),
+],"
+"""

modules/video_models/video_utils.py

@@ -0,0 +1,109 @@
+import os
+import time
+from modules import shared, timer, sd_models, sd_checkpoint, model_quant, devices
+from modules.video_models import models_def
+
+
+debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
+
+
+def queue_err(msg):
+    shared.log.error(f'Video: {msg}')
+    return [], None, '', '', f'Error: {msg}'
+
+
+def get_quant(args):
+    if args is not None and "quantization_config" in args:
+        return args['quantization_config'].__class__.__name__
+    return None
+
+
+def hijack_vae_decode(*args, **kwargs):
+    t0 = time.time()
+    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'])
+    res = shared.sd_model.vae.orig_decode(*args, **kwargs)
+    t1 = time.time()
+    timer.process.add('vae', t1-t0)
+    debug(f'Video decode: vae={shared.sd_model.vae.__class__.__name__} time={t1-t0:.2f}')
+    return res
+
+
+def hijack_encode_prompt(*args, **kwargs):
+    t0 = time.time()
+    res = shared.sd_model.orig_encode_prompt(*args, **kwargs)
+    t1 = time.time()
+    timer.process.add('te', t1-t0)
+    debug(f'Video encode: te={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}')
+    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
+    return res
+
+loaded_model = None
+
+
+def load_model(selected: models_def.Model):
+    if selected is None:
+        return
+    global loaded_model # pylint: disable=global-statement
+    if loaded_model == selected.name:
+        return
+    sd_models.unload_model_weights()
+    t0 = time.time()
+
+    # text encoder
+    try:
+        quant_args = model_quant.create_config(module='Text Encoder')
+        debug(f'Video load: module=te repo="{selected.te or selected.repo}" folder="{selected.te_folder}" cls={selected.te_cls.__name__} quant={get_quant(quant_args)}')
+        text_encoder = selected.te_cls.from_pretrained(
+            pretrained_model_name_or_path=selected.te or selected.repo,
+            subfolder=selected.te_folder,
+            cache_dir=shared.opts.hfcache_dir,
+            torch_dtype=devices.dtype,
+            **quant_args
+        )
+    except Exception as e:
+        shared.log.error(f'video load: module=te cls={selected.te_cls.__name__} {e}')
+        text_encoder = None
+
+    # transformer
+    try:
+        quant_args = model_quant.create_config(module='Model')
+        debug(f'Video load: module=transformer repo="{selected.dit or selected.repo}" folder="{selected.dit_folder}" cls={selected.dit_cls.__name__} quant={get_quant(quant_args)}')
+        transformer = selected.dit_cls.from_pretrained(
+            pretrained_model_name_or_path=selected.dit or selected.repo,
+            subfolder=selected.dit_folder,
+            torch_dtype=devices.dtype,
+            cache_dir=shared.opts.hfcache_dir,
+            **quant_args
+        )
+    except Exception as e:
+        shared.log.error(f'video load: module=transformer cls={selected.dit_cls.__name__} {e}')
+        transformer = None
+
+    # model
+    try:
+        debug(f'Video load: module=pipe repo="{selected.repo}" cls={selected.repo_cls.__name__}')
+        shared.sd_model = selected.repo_cls.from_pretrained(
+            pretrained_model_name_or_path=selected.repo,
+            transformer=transformer,
+            text_encoder=text_encoder,
+            cache_dir=shared.opts.hfcache_dir,
+            torch_dtype=devices.dtype,
+        )
+    except Exception as e:
+        shared.log.error(f'video load: module=pipe repo="{selected.repo}" cls={selected.repo_cls.__name__} {e}')
+
+    t1 = time.time()
+    sd_models.set_diffuser_options(shared.sd_model)
+    shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(selected.repo)
+    shared.sd_model.sd_model_hash = None
+    if selected.vae_hijack:
+        shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode
+        shared.sd_model.vae.decode = hijack_vae_decode
+    if selected.te_hijack:
+        shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt
+        shared.sd_model.encode_prompt = hijack_encode_prompt
+    shared.sd_model.vae.enable_slicing()
+    loaded_model = selected.name
+    msg = f'Video load: cls={shared.sd_model.__class__.__name__} model="{selected.name}" time={t1-t0:.2f}'
+    shared.log.info(msg)
+    return msg

scripts/ltxvideo.py

@@ -11,6 +11,7 @@ from modules.teacache.teacache_ltx import teacache_forward
 repos = {
     '0.9.0': 'a-r-r-o-w/LTX-Video-diffusers',
     '0.9.1': 'a-r-r-o-w/LTX-Video-0.9.1-diffusers',
+    '0.9.5': 'Lightricks/LTX-Video-0.9.5',
     'custom': None,
 }
 
@@ -31,7 +32,6 @@ def load_quants(kwargs, repo_id):
 
 def hijack_decode(*args, **kwargs):
     t0 = time.time()
-    # vae: diffusers.AutoencoderKLHunyuanVideo = shared.sd_model.vae
     shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'])
     res = shared.sd_model.vae.orig_decode(*args, **kwargs)
     t1 = time.time()