video tab first prototype

Signed-off-by: Vladimir Mandic <mandic00@live.com>

cli/api-grid.py

@@ -45,12 +45,15 @@ options = Options()
 
 
 def post():
-    req = requests.post(f'{server.url}{server.api}',
-                        json=vars(options),
-                        timeout=300,
-                        verify=False,
-                        auth=requests.auth.HTTPBasicAuth(server.user, server.password) if (server.user is not None) and (server.password is not None) else None)
-    return { 'error': req.status_code, 'reason': req.reason, 'url': req.url } if req.status_code != 200 else req.json()
+    try:
+        req = requests.post(f'{server.url}{server.api}',
+                            json=vars(options),
+                            timeout=300,
+                            verify=False,
+                            auth=requests.auth.HTTPBasicAuth(server.user, server.password) if (server.user is not None) and (server.password is not None) else None)
+        return { 'error': req.status_code, 'reason': req.reason, 'url': req.url } if req.status_code != 200 else req.json()
+    except Exception as e:
+        return { 'error': 0, 'reason': str(e), 'url': server.url }
 
 
 def generate(ts: float, x: int, y: int): # pylint: disable=redefined-outer-name
@@ -105,8 +108,15 @@ def grid(x_file: str, y_file: str):
             setattr(options, param[0].strip(), param[1].strip())
 
     log.info(server)
-    x = open(x_file, encoding='utf8').read().splitlines() if x_file is not None else []
-    y = open(y_file, encoding='utf8').read().splitlines() if y_file is not None else []
+    os.makedirs(server.folder, exist_ok=True)
+    try:
+        x = open(x_file, encoding='utf8').read().splitlines() if x_file is not None else []
+        y = open(y_file, encoding='utf8').read().splitlines() if y_file is not None else []
+    except Exception as e:
+        log.error(f'read file: x={x_file} y={y_file} {e}')
+        return
+    x = [line for line in x if ':' in line]
+    y = [line for line in y if ':' in line]
     t0 = time.time()
     log.info(f'grid: x={len(x)} y={len(y)} prefix={round(t0)}')
     vertical = []

javascript/black-teal-reimagined.css

@@ -816,6 +816,8 @@ svg.feather.feather-image,
 #txt2img_extra_search,
 #img2img_description,
 #img2img_extra_search,
+#video_description,
+#video_extra_search,
 #control_description,
 #control_extra_search {
   margin-top: 50px;

javascript/extraNetworks.js

@@ -9,6 +9,7 @@ const getENActiveTab = () => {
   if (gradioApp().getElementById('tab_txt2img').style.display === 'block') tabName = 'txt2img';
   else if (gradioApp().getElementById('tab_img2img').style.display === 'block') tabName = 'img2img';
   else if (gradioApp().getElementById('tab_control').style.display === 'block') tabName = 'control';
+  else if (gradioApp().getElementById('tab_video').style.display === 'block') tabName = 'video';
   // log('getENActiveTab', tabName);
   return tabName;
 };
@@ -491,7 +492,7 @@ function setupExtraNetworksForTab(tabname) {
 }
 
 async function showNetworks() {
-  for (const tabname of ['txt2img', 'img2img', 'control']) {
+  for (const tabname of ['txt2img', 'img2img', 'control', 'video']) {
     if (window.opts.extra_networks_show) gradioApp().getElementById(`${tabname}_extra_networks_btn`).click();
   }
   log('showNetworks');
@@ -501,6 +502,7 @@ async function setupExtraNetworks() {
   setupExtraNetworksForTab('txt2img');
   setupExtraNetworksForTab('img2img');
   setupExtraNetworksForTab('control');
+  setupExtraNetworksForTab('video');
 
   function registerPrompt(tabname, id) {
     const textarea = gradioApp().querySelector(`#${id} > label > textarea`);
@@ -515,6 +517,8 @@ async function setupExtraNetworks() {
   registerPrompt('img2img', 'img2img_neg_prompt');
   registerPrompt('control', 'control_prompt');
   registerPrompt('control', 'control_neg_prompt');
+  registerPrompt('video', 'video_prompt');
+  registerPrompt('video', 'video_neg_prompt');
   log('initNetworks', window.opts.extra_networks_card_size);
   document.documentElement.style.setProperty('--card-size', `${window.opts.extra_networks_card_size}px`);
 }

javascript/progressBar.js

@@ -14,12 +14,14 @@ function checkPaused(state) {
   lastState.paused = state ? !state : !lastState.paused;
   const t_el = document.getElementById('txt2img_pause');
   const i_el = document.getElementById('img2img_pause');
+  const v_el = document.getElementById('video_pause');
   if (t_el) t_el.innerText = lastState.paused ? 'Resume' : 'Pause';
   if (i_el) i_el.innerText = lastState.paused ? 'Resume' : 'Pause';
+  if (v_el) v_el.innerText = lastState.paused ? 'Resume' : 'Pause';
 }
 
 function setProgress(res) {
-  const elements = ['txt2img_generate', 'img2img_generate', 'extras_generate', 'control_generate'];
+  const elements = ['txt2img_generate', 'img2img_generate', 'extras_generate', 'control_generate', 'video_generate'];
   const progress = res?.progress || 0;
   const job = res?.job || '';
   let perc = '';

javascript/sdnext.css

@@ -113,6 +113,7 @@ button.custom-button { border-radius: var(--button-large-radius); padding: var(-
   min-width: unset; display: block !important; }
 #txt2img_prompt, #txt2img_neg_prompt, #img2img_prompt, #img2img_neg_prompt, #control_prompt, #control_neg_prompt, #video_prompt, #video_neg_prompt { display: contents; }
 #txt2img_actions_column, #img2img_actions_column, #control_actions, #video_actions { flex-flow: wrap; justify-content: space-between; }
+#txt2img_seed, #img2img_seed, #control_seed, #video_seed { min-width: 90px !important }
 
 .interrogate { position: absolute; right: 2.8em; top: 0.2em; max-width: fit-content; background: none !important; z-index: 50; font-size: 1.5em !important; }
 .interrogate:hover { background: var(--button-primary-background-fill-hover) !important; }
@@ -399,7 +400,8 @@ div:has(>#tab-gallery-folders) { flex-grow: 0 !important; background-color: var(
     #img2img_actions_column { display: flex; min-width: fit-content !important; flex-direction: row;justify-content: space-evenly; align-items: center;}
     #txt2img_generate_box, #img2img_generate_box, #txt2img_enqueue_wrapper,#img2img_enqueue_wrapper {display: flex;flex-direction: column;height: 4em !important;align-items: stretch;justify-content: space-evenly;}
     #img2img_interface, #img2img_results, #img2img_footer p { text-wrap: wrap; min-width: 100% !important; max-width: 100% !important;} /* maintain single column for from image operations on larger mobile devices */
-    #txt2img_sampler, #txt2img_batch, #txt2img_seed_group, #txt2img_advanced, #txt2img_second_pass, #img2img_sampling_group, #img2img_resize_group, #img2img_batch_group, #img2img_seed_group, #img2img_denoise_group, #img2img_advanced_group { width: 100% !important; } /* fix from text/image UI elements to prevent them from moving around within the UI */
+    #txt2img_sampler, #txt2img_batch, #txt2img_seed_group, #txt2img_advanced, #txt2img_second_pass, #img2img_sampling_group, #img2img_resize_group, #img2img_batch_group, #img2img_seed_group, #img2img_denoise_group, #img2img_advanced_group { width: 100% !important; } /* fix from text/image UI 
+    elements to prevent them from moving around within the UI */
     #img2img_resize_group .gradio-radio>div { display: flex; flex-direction: column; width: unset !important; }
     #inpaint_controls div { display:flex;flex-direction: row;}
     #inpaint_controls .gradio-radio>div { display: flex; flex-direction: column !important; }

javascript/ui.js

@@ -155,6 +155,11 @@ function switch_to_control(...args) {
   return Array.from(arguments);
 }
 
+function switch_to_video(...args) {
+  switchToTab('Video');
+  return Array.from(arguments);
+}
+
 function switch_to_caption(...args) {
   switchToTab('Caption');
   return Array.from(arguments);

modules/lora/network.py

@@ -17,6 +17,7 @@ class SdVersion(enum.Enum):
     SDXL = 4
     SC = 5
     F1 = 6
+    HV = 7
 
 
 class NetworkOnDisk:
@@ -56,6 +57,8 @@ class NetworkOnDisk:
             return 'sd3'
         if base.startswith("flux"):
             return 'f1'
+        if base.startswith("hunyuan_video"):
+            return 'hv'
 
         if arch.startswith("stable-diffusion-v1"):
             return 'sd1'
@@ -65,6 +68,8 @@ class NetworkOnDisk:
             return 'sc'
         if arch.startswith("flux"):
             return 'f1'
+        if arch.startswith("hunyuan-video"):
+            return 'hv'
 
         if "v1-5" in str(self.metadata.get('ss_sd_model_name', "")):
             return 'sd1'

modules/modeldata.py

@@ -45,7 +45,7 @@ def get_model_type(pipe):
         model_type = 'cogvideox'
     elif "Sana" in name:
         model_type = 'sana'
-    elif 'HunyuanVideoPipeline' in name:
+    elif 'HunyuanVideoPipeline' in name or 'HunyuanSkyreels' in name:
         model_type = 'hunyuanvideo'
     else:
         model_type = name

modules/modelloader.py

@@ -64,6 +64,9 @@ def download_civit_meta(model_path: str, model_id):
 def download_civit_preview(model_path: str, preview_url: str):
     ext = os.path.splitext(preview_url)[1]
     preview_file = os.path.splitext(model_path)[0] + ext
+    if preview_file.endswith('.mp4'):
+        shared.log.warning(f'CivitAI download: url="{preview_url}" skip video')
+        return ''
     if os.path.exists(preview_file):
         return ''
     res = f'CivitAI download: url={preview_url} file="{preview_file}"'

modules/processing.py

@@ -6,7 +6,7 @@ import numpy as np
 from PIL import Image, ImageOps
 from modules import shared, devices, errors, images, scripts, memstats, lowvram, script_callbacks, extra_networks, detailer, sd_models, sd_checkpoint, sd_vae, processing_helpers, timer, face_restoration, token_merge
 from modules.sd_hijack_hypertile import context_hypertile_vae, context_hypertile_unet
-from modules.processing_class import StableDiffusionProcessing, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, StableDiffusionProcessingControl # pylint: disable=unused-import
+from modules.processing_class import StableDiffusionProcessing, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, StableDiffusionProcessingControl, StableDiffusionProcessingVideo # pylint: disable=unused-import
 from modules.processing_info import create_infotext
 from modules.modeldata import model_data
 from modules import pag

modules/processing_class.py

@@ -360,6 +360,15 @@ class StableDiffusionProcessing:
         self.scripts = None
 
 
+class StableDiffusionProcessingVideo(StableDiffusionProcessing):
+    def __init__(self, **kwargs):
+        self.prompt_template: str = None
+        self.frames: int = 1
+        self.scheduler_shift: float = 0.0
+        self.vae_tile_frames: int = 0
+        debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access
+        super().__init__(**kwargs)
+
 class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
     def __init__(self, **kwargs):
         debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access
@@ -592,9 +601,6 @@ class StableDiffusionProcessingControl(StableDiffusionProcessingImg2Img):
             self.hr_upscale_to_x, self.hr_upscale_to_y = 8 * int(self.width * scale / 8), 8 * int(self.height * scale / 8)
         else:
             self.hr_upscale_to_x, self.hr_upscale_to_y = self.hr_resize_x, self.hr_resize_y
-        # hypertile_set(self, hr=True)
-        # shared.state.job_count = 2 * self.n_iter
-        # shared.log.debug(f'Control refine: upscaler="{self.hr_upscaler}" scale={scale} fixed={not use_scale} size={self.hr_upscale_to_x}x{self.hr_upscale_to_y}')
 
 
 def switch_class(p: StableDiffusionProcessing, new_class: type, dct: dict = None):

modules/processing_info.py

@@ -50,7 +50,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
         "Size": f"{p.width}x{p.height}" if hasattr(p, 'width') and hasattr(p, 'height') else None,
         "Sampler": p.sampler_name if p.sampler_name != 'Default' else None,
         "Seed": all_seeds[index],
-        "Seed resize from": None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}",
+        "Seed resize from": None if p.seed_resize_from_w <= 0 or p.seed_resize_from_h <= 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}",
         "CFG scale": p.cfg_scale if p.cfg_scale > 1.0 else None,
         "CFG rescale": p.diffusers_guidance_rescale if p.diffusers_guidance_rescale > 0 else None,
         "CFG end": p.cfg_end if p.cfg_end < 1.0 else None,

modules/ui.py

@@ -147,7 +147,7 @@ def create_ui(startup_timer = None):
             timer.startup.record("ui-control")
 
     with gr.Blocks(analytics_enabled=False) as video_interface:
-        if shared.native and shared.cmd_opts.experimental:
+        if shared.native:
             from modules import ui_video
             ui_video.create_ui()
             timer.startup.record("ui-video")

modules/ui_sections.py

@@ -121,18 +121,18 @@ def create_batch_inputs(tab, accordion=True):
     return batch_count, batch_size
 
 
-def create_seed_inputs(tab, reuse_visible=True):
-    with gr.Accordion(open=False, label="Seed", elem_id=f"{tab}_seed_group", elem_classes=["small-accordion"]):
+def create_seed_inputs(tab, reuse_visible=True, accordion=True, subseed_visible=True, seed_resize_visible=False):
+    with gr.Accordion(open=False, label="Seed", elem_id=f"{tab}_seed_group", elem_classes=["small-accordion"]) if accordion else gr.Group():
         with gr.Row(elem_id=f"{tab}_seed_row", variant="compact"):
             seed = gr.Number(label='Initial seed', value=-1, elem_id=f"{tab}_seed", container=True)
             random_seed = ToolButton(ui_symbols.random, elem_id=f"{tab}_random_seed", label='Random seed')
             reuse_seed = ToolButton(ui_symbols.reuse, elem_id=f"{tab}_reuse_seed", label='Reuse seed', visible=reuse_visible)
-        with gr.Row(elem_id=f"{tab}_subseed_row", variant="compact", visible=True):
+        with gr.Row(elem_id=f"{tab}_subseed_row", variant="compact", visible=subseed_visible):
             subseed = gr.Number(label='Variation', value=-1, elem_id=f"{tab}_subseed", container=True)
             random_subseed = ToolButton(ui_symbols.random, elem_id=f"{tab}_random_subseed")
             reuse_subseed = ToolButton(ui_symbols.reuse, elem_id=f"{tab}_reuse_subseed", visible=reuse_visible)
             subseed_strength = gr.Slider(label='Variation strength', value=0.0, minimum=0, maximum=1, step=0.01, elem_id=f"{tab}_subseed_strength")
-        with gr.Row(visible=False):
+        with gr.Row(visible=seed_resize_visible):
             seed_resize_from_w = gr.Slider(minimum=0, maximum=4096, step=8, label="Resize seed from width", value=0, elem_id=f"{tab}_seed_resize_from_w")
             seed_resize_from_h = gr.Slider(minimum=0, maximum=4096, step=8, label="Resize seed from height", value=0, elem_id=f"{tab}_seed_resize_from_h")
         random_seed.click(fn=lambda: -1, show_progress=False, inputs=[], outputs=[seed])
@@ -150,7 +150,7 @@ def create_video_inputs(tab:str):
         ]
     with gr.Column():
         video_codecs = ['None', 'GIF', 'PNG', 'MP4/MP4V', 'MP4/AVC1', 'MP4/JVT3', 'MKV/H264', 'AVI/DIVX', 'AVI/RGBA', 'MJPEG/MJPG', 'MPG/MPG1', 'AVR/AVR1']
-        video_type = gr.Dropdown(label='Video type', choices=video_codecs, value='None', elem_id=f"{tab}_video_type")
+        video_type = gr.Dropdown(label='Save video', choices=video_codecs, value='None', elem_id=f"{tab}_video_type")
     with gr.Column():
         video_duration = gr.Slider(label='Duration', minimum=0.25, maximum=300, step=0.25, value=2, visible=False, elem_id=f"{tab}_video_duration")
         video_loop = gr.Checkbox(label='Loop', value=True, visible=False, elem_id=f"{tab}_video_loop")

modules/ui_video.py

@@ -1,40 +1,21 @@
-# TODO hunyuanvideo: seed, scheduler, scheduler_shift, guidance_scale=1.0, true_cfg_scale=6.0, num_inference_steps=30, prompt_template, vae, offloading
+# TODO hunyuanvideo: prompt_template, lora
+# TODO hunyuanvideo: teacache, pab, fastercache, paraattention, perflow
 # TODO modernui video tab
 
-from dataclasses import dataclass
 import gradio as gr
-from modules import shared, images, ui_common, ui_sections, sd_models, call_queue, generation_parameters_copypaste
+from modules import shared, sd_models, timer, images, ui_common, ui_sections, ui_symbols, call_queue, generation_parameters_copypaste
+from modules.ui_components import ToolButton
 from modules.video_models import hunyuan
 
 
-@dataclass
-class Model():
-    name: str
-    repo: str
-    dit: str
-
-
-MODELS = {
-    'None': [],
-    'Hunyuan Video': [
-        Model('None', None, None),
-        Model('Hunyuan Video T2V', 'hunyuanvideo-community/HunyuanVideo', None),
-        Model('Hunyuan Video I2V', 'hunyuanvideo-community/HunyuanVideo', 'hunyuanvideo-community/HunyuanVideo-I2V'), # https://github.com/huggingface/diffusers/pull/10983
-        Model('SkyReels Hunyuan T2V', 'hunyuanvideo-community/HunyuanVideo', 'Skywork/SkyReels-V1-Hunyuan-T2V'), # https://github.com/huggingface/diffusers/pull/10837
-        Model('SkyReels Hunyuan I2V', 'hunyuanvideo-community/HunyuanVideo', 'Skywork/SkyReels-V1-Hunyuan-I2V'),
-        Model('Fast Hunyuan T2V', 'hunyuanvideo-community/HunyuanVideo', 'hunyuan-video-t2v-720p/transformers/mp_rank_00_model_states.pt'), # https://github.com/hao-ai-lab/FastVideo/blob/8a77cf22c9b9e7f931f42bc4b35d21fd91d24e45/fastvideo/models/hunyuan/inference.py#L213
-    ]
-}
-
-
 def engine_change(engine):
-    models = [model.name for model in MODELS.get(engine, [])]
-    return gr.update(choices=models, value=models[0] if len(models) > 0 else None)
+    found = [model.name for model in hunyuan.models.get(engine, [])]
+    return gr.update(choices=found, value=found[0] if len(found) > 0 else None)
 
 
 def model_change(engine, model):
-    models = [model.name for model in MODELS.get(engine, [])]
-    selected = [m for m in MODELS[engine] if m.name == model][0] if len(models) > 0 else None
+    found = [model.name for model in hunyuan.models.get(engine, [])]
+    selected = [m for m in hunyuan.models[engine] if m.name == model][0] if len(found) > 0 else None
     if selected:
         if 'None' in selected.name:
             sd_models.unload_model_weights()
@@ -52,8 +33,8 @@ def model_change(engine, model):
 
 def run_video(*args):
     engine, model = args[2], args[3]
-    models = [model.name for model in MODELS.get(engine, [])]
-    selected = [m for m in MODELS[engine] if m.name == model][0] if len(models) > 0 else None
+    found = [model.name for model in hunyuan.models.get(engine, [])]
+    selected = [m for m in hunyuan.models[engine] if m.name == model][0] if len(found) > 0 else None
     if selected and 'Hunyuan' in selected.name:
         return hunyuan.generate(*args)
     shared.log.error(f'Video model not found: args={args}')
@@ -61,38 +42,60 @@ def run_video(*args):
 
 
 def create_ui():
-    shared.log.debug('UI initialize: txt2img')
+    shared.log.debug('UI initialize: video')
     with gr.Blocks(analytics_enabled=False) as _video_interface:
-        prompt, styles, _negative, generate, _reprocess, paste, _networks, _token_counter, _token_button, _token_counter_negative, _token_button_negative = ui_sections.create_toprow(is_img2img=False, id_part="video", negative_visible=False, reprocess_visible=False)
+        prompt, styles, negative, generate, _reprocess, paste, networks_button, _token_counter, _token_button, _token_counter_negative, _token_button_negative = ui_sections.create_toprow(is_img2img=False, id_part="video", negative_visible=True, reprocess_visible=False)
         prompt_image = gr.File(label="", elem_id="video_prompt_image", file_count="single", type="binary", visible=False)
         prompt_image.change(fn=images.image_data, inputs=[prompt_image], outputs=[prompt, prompt_image])
 
+        with gr.Row(variant='compact', elem_id="video_extra_networks", elem_classes=["extra_networks_root"], visible=False) as extra_networks_ui:
+            from modules import ui_extra_networks
+            extra_networks_ui = ui_extra_networks.create_ui(extra_networks_ui, networks_button, 'video', skip_indexing=shared.opts.extra_network_skip_indexing)
+            timer.startup.record('ui-networks')
+
         with gr.Row(elem_id="video_interface", equal_height=False):
             with gr.Column(variant='compact', elem_id="video_settings", scale=1):
 
                 with gr.Row():
-                    engine = gr.Dropdown(label='Engine', choices=list(MODELS), value='None')
-                    model = gr.Dropdown(label='Model', choices=[''], value=None)
+                    engine = gr.Dropdown(label='Engine', choices=list(hunyuan.models), value='None', elem_id="video_engine")
+                    model = gr.Dropdown(label='Model', choices=[''], value=None, elem_id="video_model")
                 with gr.Row():
                     width, height = ui_sections.create_resolution_inputs('video', default_width=720, default_height=480)
                 with gr.Row():
-                    frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=15)
+                    frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=15, elem_id="video_frames")
+                    seed = gr.Number(label='Initial seed', value=-1, elem_id="video_seed", container=True)
+                    random_seed = ToolButton(ui_symbols.random, elem_id="video_random_seed", label='Random seed')
+                    reuse_seed = ToolButton(ui_symbols.reuse, elem_id="video_reuse_seed", label='Reuse seed')
+                steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video")
                 with gr.Row():
-                    with gr.Group(visible=False) as image_group:
+                    sampler_shift = gr.Slider(label='Sampler shift', minimum=0.0, maximum=20.0, step=0.1, value=7.0, elem_id="video_scheduler_shift")
+                with gr.Row():
+                    guidance_scale = gr.Slider(label='Guidance scale', minimum=0.0, maximum=14.0, step=0.1, value=6.0, elem_id="video_guidance_scale")
+                    guidance_true = gr.Slider(label='True guidance', minimum=0.0, maximum=14.0, step=0.1, value=1.0, elem_id="video_guidance_true")
+                with gr.Row():
+                    vae_type = gr.Dropdown(label='VAE decode', choices=['Default', 'Tiny', 'Remote'], value='Default', elem_id="video_vae_type")
+                    vae_tile_frames = gr.Slider(label='Tile frames', minimum=1, maximum=64, step=1, value=16, elem_id="video_vae_tile_frames")
+                with gr.Row():
+                    with gr.Group(visible=False, elem_id='video_init_image') as image_group:
                         gr.HTML("<br>&nbsp Init image")
-                        image = gr.Image(elem_id="video_image", show_label=False, source="upload", interactive=True, type="pil", tool="select", image_mode="RGB", height=512)
+                        init_image = gr.Image(elem_id="video_image", show_label=False, source="upload", interactive=True, type="pil", tool="select", image_mode="RGB", height=512)
                 with gr.Row():
-                    save_frames = gr.Checkbox(label='Save image frames', value=False)
+                    save_frames = gr.Checkbox(label='Save image frames', value=False, elem_id="video_save_frames")
                 with gr.Row():
-                    cc, duration, loop, pad, interpolate = ui_sections.create_video_inputs(tab='video')
+                    video_type, video_duration, video_loop, video_pad, video_interpolate = ui_sections.create_video_inputs(tab='video')
                 override_settings = ui_common.create_override_inputs('video')
 
             # output panel with gallery
             gallery, gen_info, html_info, _html_info_formatted, html_log = ui_common.create_output_panel("video", prompt=prompt, preview=False, transfer=False, scale=2)
+            # connect reuse seed button
+            ui_common.connect_reuse_seed(seed, reuse_seed, gen_info, is_subseed=False)
+            random_seed.click(fn=lambda: -1, show_progress=False, inputs=[], outputs=[seed])
+            # handle engine and model change
+            engine.change(fn=engine_change, inputs=[engine], outputs=[model])
+            model.change(fn=model_change, inputs=[engine, model], outputs=[html_log, image_group])
+            # setup extra networks
+            ui_extra_networks.setup_ui(extra_networks_ui, gallery)
 
-        # handle engine and model change
-        engine.change(fn=engine_change, inputs=[engine], outputs=[model])
-        model.change(fn=model_change, inputs=[engine, model], outputs=[html_log, image_group])
         # handle restore fields
         paste_fields = [
             (prompt, "Prompt"),
@@ -101,7 +104,7 @@ def create_ui():
             (height, "Size-2"),
             (frames, "Frames"),
         ]
-        generation_parameters_copypaste.add_paste_fields("txt2img", None, paste_fields, override_settings)
+        generation_parameters_copypaste.add_paste_fields("video", None, paste_fields, override_settings)
         bindings = generation_parameters_copypaste.ParamBinding(paste_button=paste, tabname="video", source_text_component=prompt, source_image_component=None)
         generation_parameters_copypaste.register_paste_params_button(bindings)
         # hidden fields
@@ -111,12 +114,18 @@ def create_ui():
         video_args = [
             task_id, ui_state,
             engine, model,
-            prompt, styles,
+            prompt, negative, styles,
             width, height,
             frames,
-            image,
+            steps, sampler_index,
+            sampler_shift,
+            seed,
+            guidance_scale, guidance_true,
+            init_image,
+            vae_type, vae_tile_frames,
             save_frames,
-            cc, duration, loop, pad, interpolate,
+            video_type, video_duration, video_loop, video_pad, video_interpolate,
+            override_settings,
         ]
         # generate function
         video_dict = dict(

modules/video_models/hunyuan.py

@@ -1,13 +1,260 @@
-from modules import shared
+from dataclasses import dataclass
+import os
+import time
+import torch
+import transformers
+import diffusers
+from modules import shared, sd_models, sd_checkpoint, sd_samplers, processing, model_quant, devices, images, timer, ui_common
+
+
+@dataclass
+class Model():
+    name: str
+    repo: str
+    dit: str
+    subfolder: str
+
+models = {
+    'None': [],
+    'Hunyuan Video': [
+        Model(name='None', repo=None, dit=None, subfolder=None),
+        Model(name='Hunyuan Video T2V', repo='hunyuanvideo-community/HunyuanVideo', dit='hunyuanvideo-community/HunyuanVideo', subfolder='transformer'),
+        Model(name='Hunyuan Video I2V', repo='hunyuanvideo-community/HunyuanVideo-I2V', dit='hunyuanvideo-community/HunyuanVideo-I2V', subfolder='transformer'), # https://github.com/huggingface/diffusers/pull/10983
+        Model(name='SkyReels Hunyuan T2V', repo='hunyuanvideo-community/HunyuanVideo', dit='Skywork/SkyReels-V1-Hunyuan-T2V', subfolder=None), # https://github.com/huggingface/diffusers/pull/10837
+        Model(name='SkyReels Hunyuan I2V', repo='hunyuanvideo-community/HunyuanVideo', dit='Skywork/SkyReels-V1-Hunyuan-I2V', subfolder=None),
+        Model(name='Fast Hunyuan T2V', repo='hunyuanvideo-community/HunyuanVideo', dit='FastVideo/FastHunyuan-diffusers', subfolder='transformer'), # https://github.com/hao-ai-lab/FastVideo/blob/8a77cf22c9b9e7f931f42bc4b35d21fd91d24e45/fastvideo/models/hunyuan/inference.py#L213
+    ]
+}
+debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
+loaded_model = None
+prompt_template = {
+    "template": (
+        "<|start_header_id|>system<|end_header_id|>"
+        "\nDescribe the video by detailing the following aspects: \n"
+        "1. The main content and theme of the video.\n"
+        "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects.\n"
+        "3. Actions, events, behaviors, temporal relationships, and physical movement changes of the objects.\n"
+        "4. Background environment, light, style and atmosphere.\n"
+        "5. Camera angles, movements, and transitions used in the video.\n"
+        "<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>"
+    ),
+    "crop_start": 95,
+}
+
+
+def hijack_decode(*args, **kwargs):
+    t0 = time.time()
+    vae: diffusers.AutoencoderKLHunyuanVideo = shared.sd_model.vae
+    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'])
+    res = shared.sd_model.vae.orig_decode(*args, **kwargs)
+    t1 = time.time()
+    timer.process.add('vae', t1-t0)
+    shared.log.debug(f'Video: vae={vae.__class__.__name__} tile={vae.tile_sample_min_width}:{vae.tile_sample_min_height}:{vae.tile_sample_min_num_frames} stride={vae.tile_sample_stride_width}:{vae.tile_sample_stride_height}:{vae.tile_sample_stride_num_frames} time={t1-t0:.2f}')
+    return res
+
+
+def hijack_encode_prompt(*args, **kwargs):
+    t0 = time.time()
+    res = shared.sd_model.orig_encode_prompt(*args, **kwargs)
+    t1 = time.time()
+    timer.process.add('te', t1-t0)
+    shared.log.debug(f'Video: te={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}')
+    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
+    return res
 
 
 def load(selected):
-    msg = f'Video load: model="{selected.name}" repo="{selected.repo}" dit="{selected.dit}"'
+    if selected is None:
+        return
+    global loaded_model # pylint: disable=global-statement
+    if loaded_model == selected.name:
+        return
+    sd_models.unload_model_weights()
+    t0 = time.time()
+
+    quant_args = model_quant.create_config(module='Model')
+    cls = diffusers.HunyuanVideoTransformer3DModel
+    try:
+        debug(f'Video load: module=transofrmer repo="{selected.dit}" subfolder="{selected.subfolder}" cls={cls.__name__} quant={quant_args is not None}')
+        transformer = cls.from_pretrained(
+            pretrained_model_name_or_path=selected.dit,
+            subfolder=selected.subfolder,
+            torch_dtype=devices.dtype,
+            cache_dir=shared.opts.hfcache_dir,
+            **quant_args
+        )
+    except Exception as e:
+        shared.log.error(f'video load: module=transformer repo="{selected.dit}" subfolder="{selected.subfolder}" cls={cls.__name__} {e}')
+
+    quant_args = model_quant.create_config(module='Text Encoder')
+    if 'I2V' in selected.repo:
+        cls = transformers.LlavaForConditionalGeneration
+    else:
+        cls = transformers.LlamaModel
+    try:
+        debug(f'Video load: module=te repo="{selected.repo}" cls={cls.__name__} quant={quant_args is not None}')
+        text_encoder = cls.from_pretrained(
+            pretrained_model_name_or_path=selected.repo,
+            subfolder="text_encoder",
+            cache_dir=shared.opts.hfcache_dir,
+            torch_dtype=devices.dtype,
+            **quant_args
+        )
+    except Exception as e:
+        shared.log.error(f'video load: module=te repo="{selected.repo}" cls={cls.__name__} {e}')
+
+    cls = transformers.CLIPTextModel
+    try:
+        debug(f'Video load: module=clip repo="{selected.repo}" cls={cls.__name__} quant=False')
+        text_encoder_2 = transformers.CLIPTextModel.from_pretrained(
+            pretrained_model_name_or_path=selected.repo,
+            subfolder="text_encoder_2",
+            cache_dir=shared.opts.hfcache_dir,
+            torch_dtype=devices.dtype,
+        )
+    except Exception as e:
+        shared.log.error(f'video load: module=clip repo="{selected.repo}" cls={cls.__name__} {e}')
+
+    cls = diffusers.AutoencoderKLHunyuanVideo
+    try:
+        debug(f'Video load: module=vae repo="{selected.repo}" cls={cls.__name__} quant=False')
+        vae = diffusers.AutoencoderKLHunyuanVideo.from_pretrained(
+            pretrained_model_name_or_path=selected.repo,
+            subfolder="vae",
+            cache_dir=shared.opts.hfcache_dir,
+            torch_dtype=devices.dtype,
+        )
+    except Exception as e:
+        shared.log.error(f'video load: module=vae repo="{selected.repo}" cls={cls.__name__} {e}')
+
+    if selected.name == 'Hunyuan Video I2V':
+        cls = diffusers.HunyuanVideoImageToVideoPipeline
+    elif selected.name == 'SkyReels Hunyuan I2V':
+        cls = diffusers.HunyuanSkyreelsImageToVideoPipeline
+    else:
+        cls = diffusers.HunyuanVideoPipeline
+    try:
+        debug(f'Video load: module=pipe repo="{selected.repo}" cls={cls.__name__} quant=False')
+        shared.sd_model = cls.from_pretrained(
+            pretrained_model_name_or_path=selected.repo,
+            transformer=transformer,
+            text_encoder=text_encoder,
+            text_encoder_2=text_encoder_2,
+            vae=vae,
+            cache_dir=shared.opts.hfcache_dir,
+            torch_dtype=devices.dtype,
+        )
+    except Exception as e:
+        shared.log.error(f'video load: module=pipe repo="{selected.repo}" cls={cls.__name__} {e}')
+
+    t1 = time.time()
+    sd_models.set_diffuser_options(shared.sd_model)
+    shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(selected.repo)
+    shared.sd_model.sd_model_hash = None
+    shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode
+    shared.sd_model.vae.decode = hijack_decode
+    shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt
+    shared.sd_model.encode_prompt = hijack_encode_prompt
+    shared.sd_model.vae.enable_slicing()
+    loaded_model = selected.name
+    msg = f'Video load: cls={shared.sd_model.__class__.__name__} model="{selected.name}" time={t1-t0:.2f}'
     shared.log.info(msg)
     return msg
 
 
 def generate(*args, **kwargs):
-    # TODO hunyuanvideo: check if loaded
-    shared.log.debug(f'Video generate: args={args} kwargs={kwargs}')
-    return [], '', '', 'TBD'
+    task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
+    if engine is None or model is None or engine == 'None' or model == 'None':
+        shared.log.error('Video: model not selected')
+        return [], '', '', 'Video model not selected'
+    if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__:
+        found = [model.name for model in models.get(engine, [])]
+        selected = [m for m in models[engine] if m.name == model][0] if len(found) > 0 else None
+        load(selected)
+    if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__:
+        shared.log.error('Video: model not loaded')
+        return [], '', '', 'Video model not loaded'
+    debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}')
+
+    p = processing.StableDiffusionProcessingVideo(
+        sd_model=shared.sd_model,
+        styles=styles,
+        seed=int(seed),
+        sampler_name = processing.get_sampler_name(sampler_index),
+        sampler_shift=float(sampler_shift),
+        steps=int(steps),
+        width=16 * int(width // 16),
+        height=16 * int(height // 16),
+        frames=int(frames),
+        init_image=init_image,
+        cfg_scale=float(guidance_scale),
+        diffusers_guidance_rescale=float(guidance_true),
+        vae_type=vae_type,
+        vae_tile_frames=int(vae_tile_frames),
+        override_settings=override_settings,
+    )
+    p.scripts = None
+    p.script_args = args
+    p.state = ui_state
+    p.do_not_save_grid = True
+    p.do_not_save_samples = not save_frames
+    if 'I2V' in model:
+        if init_image is None:
+            shared.log.error('Video: init image not set')
+            return [], '', '', 'Error: init image not set'
+        p.task_args['image'] = init_image
+
+    shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
+    devices.torch_gc(force=True)
+
+    # handle sampler and seed
+    if p.sampler_name != 'Default':
+        shared.sd_model.scheduler = sd_samplers.create_sampler(p.sampler_name, shared.sd_model)
+        p.sampler_name = 'Default' # avoid double creation
+    if hasattr(shared.sd_model.scheduler, '_shift') and sampler_shift > 0:
+        shared.sd_model.scheduler._shift = sampler_shift # pylint: disable=protected-access
+
+    # handle vae
+    if vae_tile_frames > p.frames:
+        shared.sd_model.vae.tile_sample_min_num_frames = vae_tile_frames
+        shared.sd_model.vae.use_framewise_decoding = True
+        shared.sd_model.vae.enable_tiling()
+    else:
+        shared.sd_model.vae.use_framewise_decoding = False
+        shared.sd_model.vae.disable_tiling()
+
+    # set args
+    processing.fix_seed(p)
+    p.prompt = shared.prompt_styles.apply_styles_to_prompt(prompt, p.styles)
+    p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(negative, p.styles)
+    p.task_args['width'] = p.width
+    p.task_args['height'] = p.height
+    p.task_args['num_inference_steps'] = p.steps
+    p.task_args['num_frames'] = p.frames
+    p.task_args['generator'] = torch.manual_seed(p.seed)
+    p.task_args['guidance_scale'] = p.cfg_scale
+    p.task_args['true_cfg_scale'] = p.diffusers_guidance_rescale
+    p.task_args['prompt_template'] = prompt_template
+    p.task_args['output_type'] = 'pil'
+    p.task_args['prompt'] = p.prompt
+    p.task_args['negative_prompt'] = p.negative_prompt
+    p.ops.append('video')
+    debug(f'Video: task_args={p.task_args}')
+
+    # run processing
+    shared.state.disable_preview = True
+    shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}')
+    t0 = time.time()
+    processed = processing.process_images(p)
+    t1 = time.time()
+    shared.state.disable_preview = False
+
+    p.close()
+    if processed is None or len(processed.images) == 0:
+        return [], '', '', 'Error: processing failed'
+    shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}')
+    if video_type != 'None':
+        images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
+
+    generation_info_js = processed.js() if processed is not None else ''
+    return processed.images, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)