diff --git a/cli/api-grid.py b/cli/api-grid.py index bd45ea066..21eb685a4 100755 --- a/cli/api-grid.py +++ b/cli/api-grid.py @@ -45,12 +45,15 @@ options = Options() def post(): - req = requests.post(f'{server.url}{server.api}', - json=vars(options), - timeout=300, - verify=False, - auth=requests.auth.HTTPBasicAuth(server.user, server.password) if (server.user is not None) and (server.password is not None) else None) - return { 'error': req.status_code, 'reason': req.reason, 'url': req.url } if req.status_code != 200 else req.json() + try: + req = requests.post(f'{server.url}{server.api}', + json=vars(options), + timeout=300, + verify=False, + auth=requests.auth.HTTPBasicAuth(server.user, server.password) if (server.user is not None) and (server.password is not None) else None) + return { 'error': req.status_code, 'reason': req.reason, 'url': req.url } if req.status_code != 200 else req.json() + except Exception as e: + return { 'error': 0, 'reason': str(e), 'url': server.url } def generate(ts: float, x: int, y: int): # pylint: disable=redefined-outer-name @@ -105,8 +108,15 @@ def grid(x_file: str, y_file: str): setattr(options, param[0].strip(), param[1].strip()) log.info(server) - x = open(x_file, encoding='utf8').read().splitlines() if x_file is not None else [] - y = open(y_file, encoding='utf8').read().splitlines() if y_file is not None else [] + os.makedirs(server.folder, exist_ok=True) + try: + x = open(x_file, encoding='utf8').read().splitlines() if x_file is not None else [] + y = open(y_file, encoding='utf8').read().splitlines() if y_file is not None else [] + except Exception as e: + log.error(f'read file: x={x_file} y={y_file} {e}') + return + x = [line for line in x if ':' in line] + y = [line for line in y if ':' in line] t0 = time.time() log.info(f'grid: x={len(x)} y={len(y)} prefix={round(t0)}') vertical = [] diff --git a/javascript/black-teal-reimagined.css b/javascript/black-teal-reimagined.css index be6176ac4..ba8957c3e 100644 --- a/javascript/black-teal-reimagined.css +++ b/javascript/black-teal-reimagined.css @@ -816,6 +816,8 @@ svg.feather.feather-image, #txt2img_extra_search, #img2img_description, #img2img_extra_search, +#video_description, +#video_extra_search, #control_description, #control_extra_search { margin-top: 50px; diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index a9f055a32..b9f045a6b 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -9,6 +9,7 @@ const getENActiveTab = () => { if (gradioApp().getElementById('tab_txt2img').style.display === 'block') tabName = 'txt2img'; else if (gradioApp().getElementById('tab_img2img').style.display === 'block') tabName = 'img2img'; else if (gradioApp().getElementById('tab_control').style.display === 'block') tabName = 'control'; + else if (gradioApp().getElementById('tab_video').style.display === 'block') tabName = 'video'; // log('getENActiveTab', tabName); return tabName; }; @@ -491,7 +492,7 @@ function setupExtraNetworksForTab(tabname) { } async function showNetworks() { - for (const tabname of ['txt2img', 'img2img', 'control']) { + for (const tabname of ['txt2img', 'img2img', 'control', 'video']) { if (window.opts.extra_networks_show) gradioApp().getElementById(`${tabname}_extra_networks_btn`).click(); } log('showNetworks'); @@ -501,6 +502,7 @@ async function setupExtraNetworks() { setupExtraNetworksForTab('txt2img'); setupExtraNetworksForTab('img2img'); setupExtraNetworksForTab('control'); + setupExtraNetworksForTab('video'); function registerPrompt(tabname, id) { const textarea = gradioApp().querySelector(`#${id} > label > textarea`); @@ -515,6 +517,8 @@ async function setupExtraNetworks() { registerPrompt('img2img', 'img2img_neg_prompt'); registerPrompt('control', 'control_prompt'); registerPrompt('control', 'control_neg_prompt'); + registerPrompt('video', 'video_prompt'); + registerPrompt('video', 'video_neg_prompt'); log('initNetworks', window.opts.extra_networks_card_size); document.documentElement.style.setProperty('--card-size', `${window.opts.extra_networks_card_size}px`); } diff --git a/javascript/progressBar.js b/javascript/progressBar.js index 0bac99d6f..cfc9a9039 100644 --- a/javascript/progressBar.js +++ b/javascript/progressBar.js @@ -14,12 +14,14 @@ function checkPaused(state) { lastState.paused = state ? !state : !lastState.paused; const t_el = document.getElementById('txt2img_pause'); const i_el = document.getElementById('img2img_pause'); + const v_el = document.getElementById('video_pause'); if (t_el) t_el.innerText = lastState.paused ? 'Resume' : 'Pause'; if (i_el) i_el.innerText = lastState.paused ? 'Resume' : 'Pause'; + if (v_el) v_el.innerText = lastState.paused ? 'Resume' : 'Pause'; } function setProgress(res) { - const elements = ['txt2img_generate', 'img2img_generate', 'extras_generate', 'control_generate']; + const elements = ['txt2img_generate', 'img2img_generate', 'extras_generate', 'control_generate', 'video_generate']; const progress = res?.progress || 0; const job = res?.job || ''; let perc = ''; diff --git a/javascript/sdnext.css b/javascript/sdnext.css index d9aa365e7..ec5e4fa3d 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -113,6 +113,7 @@ button.custom-button { border-radius: var(--button-large-radius); padding: var(- min-width: unset; display: block !important; } #txt2img_prompt, #txt2img_neg_prompt, #img2img_prompt, #img2img_neg_prompt, #control_prompt, #control_neg_prompt, #video_prompt, #video_neg_prompt { display: contents; } #txt2img_actions_column, #img2img_actions_column, #control_actions, #video_actions { flex-flow: wrap; justify-content: space-between; } +#txt2img_seed, #img2img_seed, #control_seed, #video_seed { min-width: 90px !important } .interrogate { position: absolute; right: 2.8em; top: 0.2em; max-width: fit-content; background: none !important; z-index: 50; font-size: 1.5em !important; } .interrogate:hover { background: var(--button-primary-background-fill-hover) !important; } @@ -399,7 +400,8 @@ div:has(>#tab-gallery-folders) { flex-grow: 0 !important; background-color: var( #img2img_actions_column { display: flex; min-width: fit-content !important; flex-direction: row;justify-content: space-evenly; align-items: center;} #txt2img_generate_box, #img2img_generate_box, #txt2img_enqueue_wrapper,#img2img_enqueue_wrapper {display: flex;flex-direction: column;height: 4em !important;align-items: stretch;justify-content: space-evenly;} #img2img_interface, #img2img_results, #img2img_footer p { text-wrap: wrap; min-width: 100% !important; max-width: 100% !important;} /* maintain single column for from image operations on larger mobile devices */ - #txt2img_sampler, #txt2img_batch, #txt2img_seed_group, #txt2img_advanced, #txt2img_second_pass, #img2img_sampling_group, #img2img_resize_group, #img2img_batch_group, #img2img_seed_group, #img2img_denoise_group, #img2img_advanced_group { width: 100% !important; } /* fix from text/image UI elements to prevent them from moving around within the UI */ + #txt2img_sampler, #txt2img_batch, #txt2img_seed_group, #txt2img_advanced, #txt2img_second_pass, #img2img_sampling_group, #img2img_resize_group, #img2img_batch_group, #img2img_seed_group, #img2img_denoise_group, #img2img_advanced_group { width: 100% !important; } /* fix from text/image UI + elements to prevent them from moving around within the UI */ #img2img_resize_group .gradio-radio>div { display: flex; flex-direction: column; width: unset !important; } #inpaint_controls div { display:flex;flex-direction: row;} #inpaint_controls .gradio-radio>div { display: flex; flex-direction: column !important; } diff --git a/javascript/ui.js b/javascript/ui.js index badec0f01..9a5f2b794 100644 --- a/javascript/ui.js +++ b/javascript/ui.js @@ -155,6 +155,11 @@ function switch_to_control(...args) { return Array.from(arguments); } +function switch_to_video(...args) { + switchToTab('Video'); + return Array.from(arguments); +} + function switch_to_caption(...args) { switchToTab('Caption'); return Array.from(arguments); diff --git a/modules/lora/network.py b/modules/lora/network.py index 97feb76f1..c4768d9ad 100644 --- a/modules/lora/network.py +++ b/modules/lora/network.py @@ -17,6 +17,7 @@ class SdVersion(enum.Enum): SDXL = 4 SC = 5 F1 = 6 + HV = 7 class NetworkOnDisk: @@ -56,6 +57,8 @@ class NetworkOnDisk: return 'sd3' if base.startswith("flux"): return 'f1' + if base.startswith("hunyuan_video"): + return 'hv' if arch.startswith("stable-diffusion-v1"): return 'sd1' @@ -65,6 +68,8 @@ class NetworkOnDisk: return 'sc' if arch.startswith("flux"): return 'f1' + if arch.startswith("hunyuan-video"): + return 'hv' if "v1-5" in str(self.metadata.get('ss_sd_model_name', "")): return 'sd1' diff --git a/modules/modeldata.py b/modules/modeldata.py index 012372b3b..078a5b372 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -45,7 +45,7 @@ def get_model_type(pipe): model_type = 'cogvideox' elif "Sana" in name: model_type = 'sana' - elif 'HunyuanVideoPipeline' in name: + elif 'HunyuanVideoPipeline' in name or 'HunyuanSkyreels' in name: model_type = 'hunyuanvideo' else: model_type = name diff --git a/modules/modelloader.py b/modules/modelloader.py index f3621fdc3..2a1d6745a 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -64,6 +64,9 @@ def download_civit_meta(model_path: str, model_id): def download_civit_preview(model_path: str, preview_url: str): ext = os.path.splitext(preview_url)[1] preview_file = os.path.splitext(model_path)[0] + ext + if preview_file.endswith('.mp4'): + shared.log.warning(f'CivitAI download: url="{preview_url}" skip video') + return '' if os.path.exists(preview_file): return '' res = f'CivitAI download: url={preview_url} file="{preview_file}"' diff --git a/modules/processing.py b/modules/processing.py index 3a49f99ba..67d86021b 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -6,7 +6,7 @@ import numpy as np from PIL import Image, ImageOps from modules import shared, devices, errors, images, scripts, memstats, lowvram, script_callbacks, extra_networks, detailer, sd_models, sd_checkpoint, sd_vae, processing_helpers, timer, face_restoration, token_merge from modules.sd_hijack_hypertile import context_hypertile_vae, context_hypertile_unet -from modules.processing_class import StableDiffusionProcessing, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, StableDiffusionProcessingControl # pylint: disable=unused-import +from modules.processing_class import StableDiffusionProcessing, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, StableDiffusionProcessingControl, StableDiffusionProcessingVideo # pylint: disable=unused-import from modules.processing_info import create_infotext from modules.modeldata import model_data from modules import pag diff --git a/modules/processing_class.py b/modules/processing_class.py index 9b61c4539..a0d2104b5 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -360,6 +360,15 @@ class StableDiffusionProcessing: self.scripts = None +class StableDiffusionProcessingVideo(StableDiffusionProcessing): + def __init__(self, **kwargs): + self.prompt_template: str = None + self.frames: int = 1 + self.scheduler_shift: float = 0.0 + self.vae_tile_frames: int = 0 + debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access + super().__init__(**kwargs) + class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): def __init__(self, **kwargs): debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access @@ -592,9 +601,6 @@ class StableDiffusionProcessingControl(StableDiffusionProcessingImg2Img): self.hr_upscale_to_x, self.hr_upscale_to_y = 8 * int(self.width * scale / 8), 8 * int(self.height * scale / 8) else: self.hr_upscale_to_x, self.hr_upscale_to_y = self.hr_resize_x, self.hr_resize_y - # hypertile_set(self, hr=True) - # shared.state.job_count = 2 * self.n_iter - # shared.log.debug(f'Control refine: upscaler="{self.hr_upscaler}" scale={scale} fixed={not use_scale} size={self.hr_upscale_to_x}x{self.hr_upscale_to_y}') def switch_class(p: StableDiffusionProcessing, new_class: type, dct: dict = None): diff --git a/modules/processing_info.py b/modules/processing_info.py index 5677a538c..3b0e1bd24 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -50,7 +50,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No "Size": f"{p.width}x{p.height}" if hasattr(p, 'width') and hasattr(p, 'height') else None, "Sampler": p.sampler_name if p.sampler_name != 'Default' else None, "Seed": all_seeds[index], - "Seed resize from": None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}", + "Seed resize from": None if p.seed_resize_from_w <= 0 or p.seed_resize_from_h <= 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}", "CFG scale": p.cfg_scale if p.cfg_scale > 1.0 else None, "CFG rescale": p.diffusers_guidance_rescale if p.diffusers_guidance_rescale > 0 else None, "CFG end": p.cfg_end if p.cfg_end < 1.0 else None, diff --git a/modules/ui.py b/modules/ui.py index 371a26945..91fcf3cf3 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -147,7 +147,7 @@ def create_ui(startup_timer = None): timer.startup.record("ui-control") with gr.Blocks(analytics_enabled=False) as video_interface: - if shared.native and shared.cmd_opts.experimental: + if shared.native: from modules import ui_video ui_video.create_ui() timer.startup.record("ui-video") diff --git a/modules/ui_sections.py b/modules/ui_sections.py index 20b1cfc50..d2e73bdcc 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -121,18 +121,18 @@ def create_batch_inputs(tab, accordion=True): return batch_count, batch_size -def create_seed_inputs(tab, reuse_visible=True): - with gr.Accordion(open=False, label="Seed", elem_id=f"{tab}_seed_group", elem_classes=["small-accordion"]): +def create_seed_inputs(tab, reuse_visible=True, accordion=True, subseed_visible=True, seed_resize_visible=False): + with gr.Accordion(open=False, label="Seed", elem_id=f"{tab}_seed_group", elem_classes=["small-accordion"]) if accordion else gr.Group(): with gr.Row(elem_id=f"{tab}_seed_row", variant="compact"): seed = gr.Number(label='Initial seed', value=-1, elem_id=f"{tab}_seed", container=True) random_seed = ToolButton(ui_symbols.random, elem_id=f"{tab}_random_seed", label='Random seed') reuse_seed = ToolButton(ui_symbols.reuse, elem_id=f"{tab}_reuse_seed", label='Reuse seed', visible=reuse_visible) - with gr.Row(elem_id=f"{tab}_subseed_row", variant="compact", visible=True): + with gr.Row(elem_id=f"{tab}_subseed_row", variant="compact", visible=subseed_visible): subseed = gr.Number(label='Variation', value=-1, elem_id=f"{tab}_subseed", container=True) random_subseed = ToolButton(ui_symbols.random, elem_id=f"{tab}_random_subseed") reuse_subseed = ToolButton(ui_symbols.reuse, elem_id=f"{tab}_reuse_subseed", visible=reuse_visible) subseed_strength = gr.Slider(label='Variation strength', value=0.0, minimum=0, maximum=1, step=0.01, elem_id=f"{tab}_subseed_strength") - with gr.Row(visible=False): + with gr.Row(visible=seed_resize_visible): seed_resize_from_w = gr.Slider(minimum=0, maximum=4096, step=8, label="Resize seed from width", value=0, elem_id=f"{tab}_seed_resize_from_w") seed_resize_from_h = gr.Slider(minimum=0, maximum=4096, step=8, label="Resize seed from height", value=0, elem_id=f"{tab}_seed_resize_from_h") random_seed.click(fn=lambda: -1, show_progress=False, inputs=[], outputs=[seed]) @@ -150,7 +150,7 @@ def create_video_inputs(tab:str): ] with gr.Column(): video_codecs = ['None', 'GIF', 'PNG', 'MP4/MP4V', 'MP4/AVC1', 'MP4/JVT3', 'MKV/H264', 'AVI/DIVX', 'AVI/RGBA', 'MJPEG/MJPG', 'MPG/MPG1', 'AVR/AVR1'] - video_type = gr.Dropdown(label='Video type', choices=video_codecs, value='None', elem_id=f"{tab}_video_type") + video_type = gr.Dropdown(label='Save video', choices=video_codecs, value='None', elem_id=f"{tab}_video_type") with gr.Column(): video_duration = gr.Slider(label='Duration', minimum=0.25, maximum=300, step=0.25, value=2, visible=False, elem_id=f"{tab}_video_duration") video_loop = gr.Checkbox(label='Loop', value=True, visible=False, elem_id=f"{tab}_video_loop") diff --git a/modules/ui_video.py b/modules/ui_video.py index 130aebd0d..97529e188 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -1,40 +1,21 @@ -# TODO hunyuanvideo: seed, scheduler, scheduler_shift, guidance_scale=1.0, true_cfg_scale=6.0, num_inference_steps=30, prompt_template, vae, offloading +# TODO hunyuanvideo: prompt_template, lora +# TODO hunyuanvideo: teacache, pab, fastercache, paraattention, perflow # TODO modernui video tab -from dataclasses import dataclass import gradio as gr -from modules import shared, images, ui_common, ui_sections, sd_models, call_queue, generation_parameters_copypaste +from modules import shared, sd_models, timer, images, ui_common, ui_sections, ui_symbols, call_queue, generation_parameters_copypaste +from modules.ui_components import ToolButton from modules.video_models import hunyuan -@dataclass -class Model(): - name: str - repo: str - dit: str - - -MODELS = { - 'None': [], - 'Hunyuan Video': [ - Model('None', None, None), - Model('Hunyuan Video T2V', 'hunyuanvideo-community/HunyuanVideo', None), - Model('Hunyuan Video I2V', 'hunyuanvideo-community/HunyuanVideo', 'hunyuanvideo-community/HunyuanVideo-I2V'), # https://github.com/huggingface/diffusers/pull/10983 - Model('SkyReels Hunyuan T2V', 'hunyuanvideo-community/HunyuanVideo', 'Skywork/SkyReels-V1-Hunyuan-T2V'), # https://github.com/huggingface/diffusers/pull/10837 - Model('SkyReels Hunyuan I2V', 'hunyuanvideo-community/HunyuanVideo', 'Skywork/SkyReels-V1-Hunyuan-I2V'), - Model('Fast Hunyuan T2V', 'hunyuanvideo-community/HunyuanVideo', 'hunyuan-video-t2v-720p/transformers/mp_rank_00_model_states.pt'), # https://github.com/hao-ai-lab/FastVideo/blob/8a77cf22c9b9e7f931f42bc4b35d21fd91d24e45/fastvideo/models/hunyuan/inference.py#L213 - ] -} - - def engine_change(engine): - models = [model.name for model in MODELS.get(engine, [])] - return gr.update(choices=models, value=models[0] if len(models) > 0 else None) + found = [model.name for model in hunyuan.models.get(engine, [])] + return gr.update(choices=found, value=found[0] if len(found) > 0 else None) def model_change(engine, model): - models = [model.name for model in MODELS.get(engine, [])] - selected = [m for m in MODELS[engine] if m.name == model][0] if len(models) > 0 else None + found = [model.name for model in hunyuan.models.get(engine, [])] + selected = [m for m in hunyuan.models[engine] if m.name == model][0] if len(found) > 0 else None if selected: if 'None' in selected.name: sd_models.unload_model_weights() @@ -52,8 +33,8 @@ def model_change(engine, model): def run_video(*args): engine, model = args[2], args[3] - models = [model.name for model in MODELS.get(engine, [])] - selected = [m for m in MODELS[engine] if m.name == model][0] if len(models) > 0 else None + found = [model.name for model in hunyuan.models.get(engine, [])] + selected = [m for m in hunyuan.models[engine] if m.name == model][0] if len(found) > 0 else None if selected and 'Hunyuan' in selected.name: return hunyuan.generate(*args) shared.log.error(f'Video model not found: args={args}') @@ -61,38 +42,60 @@ def run_video(*args): def create_ui(): - shared.log.debug('UI initialize: txt2img') + shared.log.debug('UI initialize: video') with gr.Blocks(analytics_enabled=False) as _video_interface: - prompt, styles, _negative, generate, _reprocess, paste, _networks, _token_counter, _token_button, _token_counter_negative, _token_button_negative = ui_sections.create_toprow(is_img2img=False, id_part="video", negative_visible=False, reprocess_visible=False) + prompt, styles, negative, generate, _reprocess, paste, networks_button, _token_counter, _token_button, _token_counter_negative, _token_button_negative = ui_sections.create_toprow(is_img2img=False, id_part="video", negative_visible=True, reprocess_visible=False) prompt_image = gr.File(label="", elem_id="video_prompt_image", file_count="single", type="binary", visible=False) prompt_image.change(fn=images.image_data, inputs=[prompt_image], outputs=[prompt, prompt_image]) + with gr.Row(variant='compact', elem_id="video_extra_networks", elem_classes=["extra_networks_root"], visible=False) as extra_networks_ui: + from modules import ui_extra_networks + extra_networks_ui = ui_extra_networks.create_ui(extra_networks_ui, networks_button, 'video', skip_indexing=shared.opts.extra_network_skip_indexing) + timer.startup.record('ui-networks') + with gr.Row(elem_id="video_interface", equal_height=False): with gr.Column(variant='compact', elem_id="video_settings", scale=1): with gr.Row(): - engine = gr.Dropdown(label='Engine', choices=list(MODELS), value='None') - model = gr.Dropdown(label='Model', choices=[''], value=None) + engine = gr.Dropdown(label='Engine', choices=list(hunyuan.models), value='None', elem_id="video_engine") + model = gr.Dropdown(label='Model', choices=[''], value=None, elem_id="video_model") with gr.Row(): width, height = ui_sections.create_resolution_inputs('video', default_width=720, default_height=480) with gr.Row(): - frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=15) + frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=15, elem_id="video_frames") + seed = gr.Number(label='Initial seed', value=-1, elem_id="video_seed", container=True) + random_seed = ToolButton(ui_symbols.random, elem_id="video_random_seed", label='Random seed') + reuse_seed = ToolButton(ui_symbols.reuse, elem_id="video_reuse_seed", label='Reuse seed') + steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video") with gr.Row(): - with gr.Group(visible=False) as image_group: + sampler_shift = gr.Slider(label='Sampler shift', minimum=0.0, maximum=20.0, step=0.1, value=7.0, elem_id="video_scheduler_shift") + with gr.Row(): + guidance_scale = gr.Slider(label='Guidance scale', minimum=0.0, maximum=14.0, step=0.1, value=6.0, elem_id="video_guidance_scale") + guidance_true = gr.Slider(label='True guidance', minimum=0.0, maximum=14.0, step=0.1, value=1.0, elem_id="video_guidance_true") + with gr.Row(): + vae_type = gr.Dropdown(label='VAE decode', choices=['Default', 'Tiny', 'Remote'], value='Default', elem_id="video_vae_type") + vae_tile_frames = gr.Slider(label='Tile frames', minimum=1, maximum=64, step=1, value=16, elem_id="video_vae_tile_frames") + with gr.Row(): + with gr.Group(visible=False, elem_id='video_init_image') as image_group: gr.HTML("
  Init image") - image = gr.Image(elem_id="video_image", show_label=False, source="upload", interactive=True, type="pil", tool="select", image_mode="RGB", height=512) + init_image = gr.Image(elem_id="video_image", show_label=False, source="upload", interactive=True, type="pil", tool="select", image_mode="RGB", height=512) with gr.Row(): - save_frames = gr.Checkbox(label='Save image frames', value=False) + save_frames = gr.Checkbox(label='Save image frames', value=False, elem_id="video_save_frames") with gr.Row(): - cc, duration, loop, pad, interpolate = ui_sections.create_video_inputs(tab='video') + video_type, video_duration, video_loop, video_pad, video_interpolate = ui_sections.create_video_inputs(tab='video') override_settings = ui_common.create_override_inputs('video') # output panel with gallery gallery, gen_info, html_info, _html_info_formatted, html_log = ui_common.create_output_panel("video", prompt=prompt, preview=False, transfer=False, scale=2) + # connect reuse seed button + ui_common.connect_reuse_seed(seed, reuse_seed, gen_info, is_subseed=False) + random_seed.click(fn=lambda: -1, show_progress=False, inputs=[], outputs=[seed]) + # handle engine and model change + engine.change(fn=engine_change, inputs=[engine], outputs=[model]) + model.change(fn=model_change, inputs=[engine, model], outputs=[html_log, image_group]) + # setup extra networks + ui_extra_networks.setup_ui(extra_networks_ui, gallery) - # handle engine and model change - engine.change(fn=engine_change, inputs=[engine], outputs=[model]) - model.change(fn=model_change, inputs=[engine, model], outputs=[html_log, image_group]) # handle restore fields paste_fields = [ (prompt, "Prompt"), @@ -101,7 +104,7 @@ def create_ui(): (height, "Size-2"), (frames, "Frames"), ] - generation_parameters_copypaste.add_paste_fields("txt2img", None, paste_fields, override_settings) + generation_parameters_copypaste.add_paste_fields("video", None, paste_fields, override_settings) bindings = generation_parameters_copypaste.ParamBinding(paste_button=paste, tabname="video", source_text_component=prompt, source_image_component=None) generation_parameters_copypaste.register_paste_params_button(bindings) # hidden fields @@ -111,12 +114,18 @@ def create_ui(): video_args = [ task_id, ui_state, engine, model, - prompt, styles, + prompt, negative, styles, width, height, frames, - image, + steps, sampler_index, + sampler_shift, + seed, + guidance_scale, guidance_true, + init_image, + vae_type, vae_tile_frames, save_frames, - cc, duration, loop, pad, interpolate, + video_type, video_duration, video_loop, video_pad, video_interpolate, + override_settings, ] # generate function video_dict = dict( diff --git a/modules/video_models/hunyuan.py b/modules/video_models/hunyuan.py index 6fed27cb3..051ecde85 100644 --- a/modules/video_models/hunyuan.py +++ b/modules/video_models/hunyuan.py @@ -1,13 +1,260 @@ -from modules import shared +from dataclasses import dataclass +import os +import time +import torch +import transformers +import diffusers +from modules import shared, sd_models, sd_checkpoint, sd_samplers, processing, model_quant, devices, images, timer, ui_common + + +@dataclass +class Model(): + name: str + repo: str + dit: str + subfolder: str + +models = { + 'None': [], + 'Hunyuan Video': [ + Model(name='None', repo=None, dit=None, subfolder=None), + Model(name='Hunyuan Video T2V', repo='hunyuanvideo-community/HunyuanVideo', dit='hunyuanvideo-community/HunyuanVideo', subfolder='transformer'), + Model(name='Hunyuan Video I2V', repo='hunyuanvideo-community/HunyuanVideo-I2V', dit='hunyuanvideo-community/HunyuanVideo-I2V', subfolder='transformer'), # https://github.com/huggingface/diffusers/pull/10983 + Model(name='SkyReels Hunyuan T2V', repo='hunyuanvideo-community/HunyuanVideo', dit='Skywork/SkyReels-V1-Hunyuan-T2V', subfolder=None), # https://github.com/huggingface/diffusers/pull/10837 + Model(name='SkyReels Hunyuan I2V', repo='hunyuanvideo-community/HunyuanVideo', dit='Skywork/SkyReels-V1-Hunyuan-I2V', subfolder=None), + Model(name='Fast Hunyuan T2V', repo='hunyuanvideo-community/HunyuanVideo', dit='FastVideo/FastHunyuan-diffusers', subfolder='transformer'), # https://github.com/hao-ai-lab/FastVideo/blob/8a77cf22c9b9e7f931f42bc4b35d21fd91d24e45/fastvideo/models/hunyuan/inference.py#L213 + ] +} +debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None +loaded_model = None +prompt_template = { + "template": ( + "<|start_header_id|>system<|end_header_id|>" + "\nDescribe the video by detailing the following aspects: \n" + "1. The main content and theme of the video.\n" + "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects.\n" + "3. Actions, events, behaviors, temporal relationships, and physical movement changes of the objects.\n" + "4. Background environment, light, style and atmosphere.\n" + "5. Camera angles, movements, and transitions used in the video.\n" + "<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" + ), + "crop_start": 95, +} + + +def hijack_decode(*args, **kwargs): + t0 = time.time() + vae: diffusers.AutoencoderKLHunyuanVideo = shared.sd_model.vae + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) + res = shared.sd_model.vae.orig_decode(*args, **kwargs) + t1 = time.time() + timer.process.add('vae', t1-t0) + shared.log.debug(f'Video: vae={vae.__class__.__name__} tile={vae.tile_sample_min_width}:{vae.tile_sample_min_height}:{vae.tile_sample_min_num_frames} stride={vae.tile_sample_stride_width}:{vae.tile_sample_stride_height}:{vae.tile_sample_stride_num_frames} time={t1-t0:.2f}') + return res + + +def hijack_encode_prompt(*args, **kwargs): + t0 = time.time() + res = shared.sd_model.orig_encode_prompt(*args, **kwargs) + t1 = time.time() + timer.process.add('te', t1-t0) + shared.log.debug(f'Video: te={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}') + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + return res def load(selected): - msg = f'Video load: model="{selected.name}" repo="{selected.repo}" dit="{selected.dit}"' + if selected is None: + return + global loaded_model # pylint: disable=global-statement + if loaded_model == selected.name: + return + sd_models.unload_model_weights() + t0 = time.time() + + quant_args = model_quant.create_config(module='Model') + cls = diffusers.HunyuanVideoTransformer3DModel + try: + debug(f'Video load: module=transofrmer repo="{selected.dit}" subfolder="{selected.subfolder}" cls={cls.__name__} quant={quant_args is not None}') + transformer = cls.from_pretrained( + pretrained_model_name_or_path=selected.dit, + subfolder=selected.subfolder, + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + **quant_args + ) + except Exception as e: + shared.log.error(f'video load: module=transformer repo="{selected.dit}" subfolder="{selected.subfolder}" cls={cls.__name__} {e}') + + quant_args = model_quant.create_config(module='Text Encoder') + if 'I2V' in selected.repo: + cls = transformers.LlavaForConditionalGeneration + else: + cls = transformers.LlamaModel + try: + debug(f'Video load: module=te repo="{selected.repo}" cls={cls.__name__} quant={quant_args is not None}') + text_encoder = cls.from_pretrained( + pretrained_model_name_or_path=selected.repo, + subfolder="text_encoder", + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **quant_args + ) + except Exception as e: + shared.log.error(f'video load: module=te repo="{selected.repo}" cls={cls.__name__} {e}') + + cls = transformers.CLIPTextModel + try: + debug(f'Video load: module=clip repo="{selected.repo}" cls={cls.__name__} quant=False') + text_encoder_2 = transformers.CLIPTextModel.from_pretrained( + pretrained_model_name_or_path=selected.repo, + subfolder="text_encoder_2", + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + ) + except Exception as e: + shared.log.error(f'video load: module=clip repo="{selected.repo}" cls={cls.__name__} {e}') + + cls = diffusers.AutoencoderKLHunyuanVideo + try: + debug(f'Video load: module=vae repo="{selected.repo}" cls={cls.__name__} quant=False') + vae = diffusers.AutoencoderKLHunyuanVideo.from_pretrained( + pretrained_model_name_or_path=selected.repo, + subfolder="vae", + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + ) + except Exception as e: + shared.log.error(f'video load: module=vae repo="{selected.repo}" cls={cls.__name__} {e}') + + if selected.name == 'Hunyuan Video I2V': + cls = diffusers.HunyuanVideoImageToVideoPipeline + elif selected.name == 'SkyReels Hunyuan I2V': + cls = diffusers.HunyuanSkyreelsImageToVideoPipeline + else: + cls = diffusers.HunyuanVideoPipeline + try: + debug(f'Video load: module=pipe repo="{selected.repo}" cls={cls.__name__} quant=False') + shared.sd_model = cls.from_pretrained( + pretrained_model_name_or_path=selected.repo, + transformer=transformer, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + vae=vae, + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + ) + except Exception as e: + shared.log.error(f'video load: module=pipe repo="{selected.repo}" cls={cls.__name__} {e}') + + t1 = time.time() + sd_models.set_diffuser_options(shared.sd_model) + shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(selected.repo) + shared.sd_model.sd_model_hash = None + shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode + shared.sd_model.vae.decode = hijack_decode + shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt + shared.sd_model.encode_prompt = hijack_encode_prompt + shared.sd_model.vae.enable_slicing() + loaded_model = selected.name + msg = f'Video load: cls={shared.sd_model.__class__.__name__} model="{selected.name}" time={t1-t0:.2f}' shared.log.info(msg) return msg def generate(*args, **kwargs): - # TODO hunyuanvideo: check if loaded - shared.log.debug(f'Video generate: args={args} kwargs={kwargs}') - return [], '', '', 'TBD' + task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args + if engine is None or model is None or engine == 'None' or model == 'None': + shared.log.error('Video: model not selected') + return [], '', '', 'Video model not selected' + if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__: + found = [model.name for model in models.get(engine, [])] + selected = [m for m in models[engine] if m.name == model][0] if len(found) > 0 else None + load(selected) + if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__: + shared.log.error('Video: model not loaded') + return [], '', '', 'Video model not loaded' + debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') + + p = processing.StableDiffusionProcessingVideo( + sd_model=shared.sd_model, + styles=styles, + seed=int(seed), + sampler_name = processing.get_sampler_name(sampler_index), + sampler_shift=float(sampler_shift), + steps=int(steps), + width=16 * int(width // 16), + height=16 * int(height // 16), + frames=int(frames), + init_image=init_image, + cfg_scale=float(guidance_scale), + diffusers_guidance_rescale=float(guidance_true), + vae_type=vae_type, + vae_tile_frames=int(vae_tile_frames), + override_settings=override_settings, + ) + p.scripts = None + p.script_args = args + p.state = ui_state + p.do_not_save_grid = True + p.do_not_save_samples = not save_frames + if 'I2V' in model: + if init_image is None: + shared.log.error('Video: init image not set') + return [], '', '', 'Error: init image not set' + p.task_args['image'] = init_image + + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + devices.torch_gc(force=True) + + # handle sampler and seed + if p.sampler_name != 'Default': + shared.sd_model.scheduler = sd_samplers.create_sampler(p.sampler_name, shared.sd_model) + p.sampler_name = 'Default' # avoid double creation + if hasattr(shared.sd_model.scheduler, '_shift') and sampler_shift > 0: + shared.sd_model.scheduler._shift = sampler_shift # pylint: disable=protected-access + + # handle vae + if vae_tile_frames > p.frames: + shared.sd_model.vae.tile_sample_min_num_frames = vae_tile_frames + shared.sd_model.vae.use_framewise_decoding = True + shared.sd_model.vae.enable_tiling() + else: + shared.sd_model.vae.use_framewise_decoding = False + shared.sd_model.vae.disable_tiling() + + # set args + processing.fix_seed(p) + p.prompt = shared.prompt_styles.apply_styles_to_prompt(prompt, p.styles) + p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(negative, p.styles) + p.task_args['width'] = p.width + p.task_args['height'] = p.height + p.task_args['num_inference_steps'] = p.steps + p.task_args['num_frames'] = p.frames + p.task_args['generator'] = torch.manual_seed(p.seed) + p.task_args['guidance_scale'] = p.cfg_scale + p.task_args['true_cfg_scale'] = p.diffusers_guidance_rescale + p.task_args['prompt_template'] = prompt_template + p.task_args['output_type'] = 'pil' + p.task_args['prompt'] = p.prompt + p.task_args['negative_prompt'] = p.negative_prompt + p.ops.append('video') + debug(f'Video: task_args={p.task_args}') + + # run processing + shared.state.disable_preview = True + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}') + t0 = time.time() + processed = processing.process_images(p) + t1 = time.time() + shared.state.disable_preview = False + + p.close() + if processed is None or len(processed.images) == 0: + return [], '', '', 'Error: processing failed' + shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') + if video_type != 'None': + images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) + + generation_info_js = processed.js() if processed is not None else '' + return processed.images, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)