From 58b0ab9da6db4807719ed4ae165e756e95b9ca22 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 5 Oct 2025 13:30:22 -0400 Subject: [PATCH] unified video save Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 9 +++++---- javascript/control.js | 1 - javascript/history.js | 2 +- modules/video.py | 4 ++-- modules/video_models/video_run.py | 27 +++++++++++++++++++++++---- modules/video_models/video_save.py | 23 +++++++++++++++++++++-- modules/video_models/video_ui.py | 24 +++++++++++++++++------- 7 files changed, 69 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddd8b3a5d..0b2124b1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,10 +89,11 @@ - **xyz-grid** add guidance section - **Video** - use shared T5 text encoder for video models when possible - - **FramePack** add job state tracking - - **LTXVideo** fix model selection in ltx tab - - **LTXVideo** fix run with offloading - - **WAN** fix run with offloading + - unified video save code across all video models + also avoids creation of temporary files for each frame unless user wants to save them + - add job state tracking for video generation + - improve offloading for **ltx** and **wan** + - fix model selection in ltx tab - **Experimental** - `new` command line flag enables new `pydantic` and `albumentations` packages - **modular pipelines**: enable in *settings -> model options* diff --git a/javascript/control.js b/javascript/control.js index c94c61a67..e0bb550ba 100644 --- a/javascript/control.js +++ b/javascript/control.js @@ -4,7 +4,6 @@ function controlInputMode(inputMode, ...args) { const tab = gradioApp().querySelector('#control-tab-input button.selected'); if (!tab) return ['Image', ...args]; inputMode = tab.innerText; - console.log('HERE0', tab, inputMode); return [inputMode, ...args]; } diff --git a/javascript/history.js b/javascript/history.js index ff2db4831..2a40b0859 100644 --- a/javascript/history.js +++ b/javascript/history.js @@ -45,7 +45,7 @@ function refreshHistory() { if (inferenceTypes.some((type) => entry.job.toLowerCase().startsWith(type))) entry.type = 'inference'; else if (ioTypes.some((type) => entry.job.toLowerCase().startsWith(type))) entry.type = 'io'; else entry.type = 'default'; - ts.push({ start, end: end.timestamp, label: entry.job, type: entry.type }); + if (start && end.timestamp) ts.push({ start, end: end.timestamp, label: entry.job, type: entry.type }); } } if (!ts.length) return; diff --git a/modules/video.py b/modules/video.py index fe7d88c36..b95ae7aa1 100644 --- a/modules/video.py +++ b/modules/video.py @@ -29,7 +29,7 @@ def save_video_atomic(images, filename, video_type: str = 'none', duration: floa except Exception as e: shared.log.error(f'Save video: cv2: {e}') return - jobid = shared.state.begin('Save video') + savejob = shared.state.begin('Save video') os.makedirs(os.path.dirname(filename), exist_ok=True) if video_type.lower() in ['gif', 'png']: append = images.copy() @@ -57,7 +57,7 @@ def save_video_atomic(images, filename, video_type: str = 'none', duration: floa video_writer.write(img) size = os.path.getsize(filename) shared.log.info(f'Save video: file="{filename}" frames={len(frames)} duration={duration} fourcc={fourcc} size={size}') - shared.state.end(jobid) + shared.state.end(savejob) def save_video(p, images, filename = None, video_type: str = 'none', duration: float = 2.0, loop: bool = False, interpolate: int = 0, scale: float = 1.0, pad: int = 1, change: float = 0.3, sync: bool = False): diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 33d4f5d13..1dccc3611 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -1,16 +1,17 @@ import os import time from modules import shared, errors, sd_models, processing, devices, images, ui_common -from modules.video_models import models_def, video_utils, video_load, video_vae, video_overrides +from modules.video_models import models_def, video_utils, video_load, video_vae, video_overrides, video_save debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None def generate(*args, **kwargs): - task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, init_strength, last_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args + task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, init_strength, last_image, vae_type, vae_tile_frames, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, override_settings = args if engine is None or model is None or engine == 'None' or model == 'None': return video_utils.queue_err('model not selected') + # videojob = shared.state.begin('Video') found = [model.name for model in models_def.models.get(engine, [])] selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None if not shared.sd_loaded: @@ -51,7 +52,7 @@ def generate(*args, **kwargs): p.script_args = None p.state = ui_state p.do_not_save_grid = True - p.do_not_save_samples = not save_frames + p.do_not_save_samples = not mp4_frames p.outpath_samples = shared.opts.outdir_samples or shared.opts.outdir_video if 'T2V' in model: if init_image is not None: @@ -119,6 +120,24 @@ def generate(*args, **kwargs): if processed is None or len(processed.images) == 0: return video_utils.queue_err('processing failed') shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') - video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) + + # video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) # legacy video save from list of images + pixels = video_save.images_to_tensor(processed.images) + _num_frames, video_file = video_save.save_video( + pixels=pixels, + mp4_fps=mp4_fps, + mp4_codec=mp4_codec, + mp4_opt=mp4_opt, + mp4_ext=mp4_ext, + mp4_sf=mp4_sf, + mp4_video=mp4_video, + mp4_frames=mp4_frames, + mp4_interpolate=mp4_interpolate, + metadata={}, + ) + if not mp4_frames: + processed.images = [] + generation_info_js = processed.js() if processed is not None else '' + # shared.state.end(videojob) return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) diff --git a/modules/video_models/video_save.py b/modules/video_models/video_save.py index 2ce71f0af..41095922f 100644 --- a/modules/video_models/video_save.py +++ b/modules/video_models/video_save.py @@ -2,6 +2,7 @@ import os import time import datetime import cv2 +import numpy as np import torch import einops from modules import shared, errors ,timer, rife @@ -13,6 +14,18 @@ def get_video_filename(frames:int, codec:str): return output_filename +def images_to_tensor(images): + if images is None or len(images) == 0: + return None + array = [torch.from_numpy(np.array(image)) for image in images] + tensor = torch.stack(array, dim=0) # n h w c + tensor = tensor.unsqueeze(0) # 1, n, h, w, c + tensor = tensor.permute(0, 4, 1, 2, 3).contiguous() # 1, c, n, h, w + tensor = (tensor.float() / 127.5) - 1.0 # from [0,255] to [-1,1] + # shared.log.debug(f'Video output: images={len(images)} tensor={tensor.shape}') + return tensor + + def atomic_save_video(filename, tensor:torch.Tensor, fps:float=24, codec:str='libx264', pix_fmt:str='yuv420p', options:str='', metadata:dict={}, pbar=None): try: import av @@ -21,6 +34,7 @@ def atomic_save_video(filename, tensor:torch.Tensor, fps:float=24, codec:str='li shared.log.error(f'Video: {e}') return + savejob = shared.state.begin('Save video') frames, height, width, _channels = tensor.shape rate = round(fps) options_str = options @@ -54,6 +68,7 @@ def atomic_save_video(filename, tensor:torch.Tensor, fps:float=24, codec:str='li for packet in stream.encode(): # flush container.mux(packet) shared.state.outputs(filename) + shared.state.end(savejob) def save_video( @@ -73,13 +88,16 @@ def save_video( output_video = None if pixels is None: return 0, output_video + if not torch.is_tensor(pixels): + shared.log.error(f'Video: type={type(pixels)} not a tensor') + return 0, output_video t_save = time.time() n, _c, t, h, w = pixels.shape size = pixels.element_size() * pixels.numel() shared.log.debug(f'Video: video={mp4_video} export={mp4_frames} safetensors={mp4_sf} interpolate={mp4_interpolate}') shared.log.debug(f'Video: encode={t} raw={size} latent={pixels.shape} fps={mp4_fps} codec={mp4_codec} ext={mp4_ext} options="{mp4_opt}"') - jobid = shared.state.begin('Save video') try: + preparejob = shared.state.begin('Prepare video') if stream is not None: stream.output_queue.push(('progress', (None, 'Saving video...'))) if mp4_interpolate > 0: @@ -111,6 +129,8 @@ def save_video( shared.state.outputs(fn) cv2.imwrite(fn, image) + shared.state.end(preparejob) + if mp4_video and (mp4_codec != 'none'): output_video = f'{output_filename}.{mp4_ext}' atomic_save_video(output_video, tensor=x, fps=mp4_fps, codec=mp4_codec, options=mp4_opt, metadata=metadata, pbar=pbar) @@ -125,5 +145,4 @@ def save_video( shared.log.error(f'Video save: raw={size} {e}') errors.display(e, 'video') timer.process.add('save', time.time()-t_save) - shared.state.end(jobid) return t, output_video diff --git a/modules/video_models/video_ui.py b/modules/video_models/video_ui.py index 910f665a4..faea8b6ef 100644 --- a/modules/video_models/video_ui.py +++ b/modules/video_models/video_ui.py @@ -112,20 +112,31 @@ def create_ui(prompt, negative, styles, overrides): init_image = gr.Image(elem_id="video_image", show_label=False, type="pil", image_mode="RGB", width=256, height=256) gr.HTML("
  Last image") last_image = gr.Image(elem_id="video_last", show_label=False, type="pil", image_mode="RGB", width=256, height=256) + vlm_enhance, vlm_model, vlm_system_prompt = ui_video_vlm.create_ui(prompt_element=prompt, image_element=init_image) - with gr.Accordion(open=False, label="Output", elem_id='video_output_accordion'): + + with gr.Accordion(label="Video", open=False, elem_id='video_output_accordion'): with gr.Row(): - save_frames = gr.Checkbox(label='Save image frames', value=False, elem_id="video_save_frames") + mp4_fps = gr.Slider(label="FPS", minimum=1, maximum=60, value=24, step=1) + mp4_interpolate = gr.Slider(label="Video interpolation", minimum=0, maximum=10, value=0, step=1) with gr.Row(): - video_type, video_duration, video_loop, video_pad, video_interpolate = ui_sections.create_video_inputs(tab='video', show_always=True) + mp4_codec = gr.Dropdown(label="Video codec", choices=['none', 'libx264'], value='libx264', type='value') + ui_common.create_refresh_button(mp4_codec, video_utils.get_codecs, elem_id="framepack_mp4_codec_refresh") + mp4_ext = gr.Textbox(label="Video format", value='mp4', elem_id="framepack_mp4_ext") + mp4_opt = gr.Textbox(label="Video options", value='crf:16', elem_id="framepack_mp4_ext") + with gr.Row(): + mp4_video = gr.Checkbox(label='Video save video', value=True, elem_id="framepack_mp4_video") + mp4_frames = gr.Checkbox(label='Video save frames', value=False, elem_id="framepack_mp4_frames") + mp4_sf = gr.Checkbox(label='Video save safetensors', value=False, elem_id="framepack_mp4_sf") + # output panel with gallery and video tabs with gr.Column(elem_id='video-output-column', scale=2) as _column_output: with gr.Tabs(elem_classes=['video-output-tabs'], elem_id='video-output-tabs'): - with gr.Tab('Frames', id='out-gallery'): - gallery, gen_info, html_info, _html_info_formatted, html_log = ui_common.create_output_panel("video", prompt=prompt, preview=False, transfer=False, scale=2) with gr.Tab('Video', id='out-video'): video = gr.Video(label="Output", show_label=False, elem_id='control_output_video', elem_classes=['control-image'], height=512, autoplay=False) + with gr.Tab('Frames', id='out-gallery'): + gallery, gen_info, html_info, _html_info_formatted, html_log = ui_common.create_output_panel("video", prompt=prompt, preview=False, transfer=False, scale=2) # connect reuse seed button ui_common.connect_reuse_seed(seed, reuse_seed, gen_info, is_subseed=False) @@ -151,8 +162,7 @@ def create_ui(prompt, negative, styles, overrides): guidance_scale, guidance_true, init_image, init_strength, last_image, vae_type, vae_tile_frames, - save_frames, - video_type, video_duration, video_loop, video_pad, video_interpolate, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, overrides, ] video_outputs = [