Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-20 01:31:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

unified video save

Browse Source 
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
2025-10-05 13:30:22 -04:00
parent d7d86ed286
commit 58b0ab9da6
7 changed files with 69 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+5 -4
View File
@@ -89,10 +89,11 @@
- **xyz-grid** add guidance section
- **Video**
- use shared T5 text encoder for video models when possible
- **FramePack** add job state tracking
- **LTXVideo** fix model selection in ltx tab
- **LTXVideo** fix run with offloading
- **WAN** fix run with offloading
- unified video save code across all video models
also avoids creation of temporary files for each frame unless user wants to save them
- add job state tracking for video generation
- improve offloading for **ltx** and **wan**
- fix model selection in ltx tab
- **Experimental**
- `new` command line flag enables new `pydantic` and `albumentations` packages
- **modular pipelines**: enable in *settings -> model options*
javascript/control.js
-1
View File
@@ -4,7 +4,6 @@ function controlInputMode(inputMode, ...args) {
const tab = gradioApp().querySelector('#control-tab-input button.selected');
if (!tab) return ['Image', ...args];
inputMode = tab.innerText;
console.log('HERE0', tab, inputMode);
return [inputMode, ...args];
}
javascript/history.js
+1 -1
View File
@@ -45,7 +45,7 @@ function refreshHistory() {
if (inferenceTypes.some((type) => entry.job.toLowerCase().startsWith(type))) entry.type = 'inference';
else if (ioTypes.some((type) => entry.job.toLowerCase().startsWith(type))) entry.type = 'io';
else entry.type = 'default';
ts.push({ start, end: end.timestamp, label: entry.job, type: entry.type });
if (start && end.timestamp) ts.push({ start, end: end.timestamp, label: entry.job, type: entry.type });
}
}
if (!ts.length) return;
modules/video.py
+2 -2
View File
@@ -29,7 +29,7 @@ def save_video_atomic(images, filename, video_type: str = 'none', duration: floa
except Exception as e:
shared.log.error(f'Save video: cv2: {e}')
return
jobid = shared.state.begin('Save video')
savejob = shared.state.begin('Save video')
os.makedirs(os.path.dirname(filename), exist_ok=True)
if video_type.lower() in ['gif', 'png']:
append = images.copy()
@@ -57,7 +57,7 @@ def save_video_atomic(images, filename, video_type: str = 'none', duration: floa
video_writer.write(img)
size = os.path.getsize(filename)
shared.log.info(f'Save video: file="{filename}" frames={len(frames)} duration={duration} fourcc={fourcc} size={size}')
shared.state.end(jobid)
shared.state.end(savejob)
def save_video(p, images, filename = None, video_type: str = 'none', duration: float = 2.0, loop: bool = False, interpolate: int = 0, scale: float = 1.0, pad: int = 1, change: float = 0.3, sync: bool = False):
modules/video_models/video_run.py
+23 -4
View File
@@ -1,16 +1,17 @@
import os
import time
from modules import shared, errors, sd_models, processing, devices, images, ui_common
from modules.video_models import models_def, video_utils, video_load, video_vae, video_overrides
from modules.video_models import models_def, video_utils, video_load, video_vae, video_overrides, video_save
debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
def generate(*args, **kwargs):
task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, init_strength, last_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args
task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, init_strength, last_image, vae_type, vae_tile_frames, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, override_settings = args
if engine is None or model is None or engine == 'None' or model == 'None':
return video_utils.queue_err('model not selected')
# videojob = shared.state.begin('Video')
found = [model.name for model in models_def.models.get(engine, [])]
selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None
if not shared.sd_loaded:
@@ -51,7 +52,7 @@ def generate(*args, **kwargs):
p.script_args = None
p.state = ui_state
p.do_not_save_grid = True
p.do_not_save_samples = not save_frames
p.do_not_save_samples = not mp4_frames
p.outpath_samples = shared.opts.outdir_samples or shared.opts.outdir_video
if 'T2V' in model:
if init_image is not None:
@@ -119,6 +120,24 @@ def generate(*args, **kwargs):
if processed is None or len(processed.images) == 0:
return video_utils.queue_err('processing failed')
shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}')
video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate)
# video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) # legacy video save from list of images
pixels = video_save.images_to_tensor(processed.images)
_num_frames, video_file = video_save.save_video(
pixels=pixels,
mp4_fps=mp4_fps,
mp4_codec=mp4_codec,
mp4_opt=mp4_opt,
mp4_ext=mp4_ext,
mp4_sf=mp4_sf,
mp4_video=mp4_video,
mp4_frames=mp4_frames,
mp4_interpolate=mp4_interpolate,
metadata={},
)
if not mp4_frames:
processed.images = []
generation_info_js = processed.js() if processed is not None else ''
# shared.state.end(videojob)
return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments)
modules/video_models/video_save.py
+21 -2
View File
@@ -2,6 +2,7 @@ import os
import time
import datetime
import cv2
import numpy as np
import torch
import einops
from modules import shared, errors ,timer, rife
@@ -13,6 +14,18 @@ def get_video_filename(frames:int, codec:str):
return output_filename
def images_to_tensor(images):
if images is None or len(images) == 0:
return None
array = [torch.from_numpy(np.array(image)) for image in images]
tensor = torch.stack(array, dim=0) # n h w c
tensor = tensor.unsqueeze(0) # 1, n, h, w, c
tensor = tensor.permute(0, 4, 1, 2, 3).contiguous() # 1, c, n, h, w
tensor = (tensor.float() / 127.5) - 1.0 # from [0,255] to [-1,1]
# shared.log.debug(f'Video output: images={len(images)} tensor={tensor.shape}')
return tensor
def atomic_save_video(filename, tensor:torch.Tensor, fps:float=24, codec:str='libx264', pix_fmt:str='yuv420p', options:str='', metadata:dict={}, pbar=None):
try:
import av
@@ -21,6 +34,7 @@ def atomic_save_video(filename, tensor:torch.Tensor, fps:float=24, codec:str='li
shared.log.error(f'Video: {e}')
return
savejob = shared.state.begin('Save video')
frames, height, width, _channels = tensor.shape
rate = round(fps)
options_str = options
@@ -54,6 +68,7 @@ def atomic_save_video(filename, tensor:torch.Tensor, fps:float=24, codec:str='li
for packet in stream.encode(): # flush
container.mux(packet)
shared.state.outputs(filename)
shared.state.end(savejob)
def save_video(
@@ -73,13 +88,16 @@ def save_video(
output_video = None
if pixels is None:
return 0, output_video
if not torch.is_tensor(pixels):
shared.log.error(f'Video: type={type(pixels)} not a tensor')
return 0, output_video
t_save = time.time()
n, _c, t, h, w = pixels.shape
size = pixels.element_size() * pixels.numel()
shared.log.debug(f'Video: video={mp4_video} export={mp4_frames} safetensors={mp4_sf} interpolate={mp4_interpolate}')
shared.log.debug(f'Video: encode={t} raw={size} latent={pixels.shape} fps={mp4_fps} codec={mp4_codec} ext={mp4_ext} options="{mp4_opt}"')
jobid = shared.state.begin('Save video')
try:
preparejob = shared.state.begin('Prepare video')
if stream is not None:
stream.output_queue.push(('progress', (None, 'Saving video...')))
if mp4_interpolate > 0:
@@ -111,6 +129,8 @@ def save_video(
shared.state.outputs(fn)
cv2.imwrite(fn, image)
shared.state.end(preparejob)
if mp4_video and (mp4_codec != 'none'):
output_video = f'{output_filename}.{mp4_ext}'
atomic_save_video(output_video, tensor=x, fps=mp4_fps, codec=mp4_codec, options=mp4_opt, metadata=metadata, pbar=pbar)
@@ -125,5 +145,4 @@ def save_video(
shared.log.error(f'Video save: raw={size} {e}')
errors.display(e, 'video')
timer.process.add('save', time.time()-t_save)
shared.state.end(jobid)
return t, output_video
modules/video_models/video_ui.py
+17 -7
View File
@@ -112,20 +112,31 @@ def create_ui(prompt, negative, styles, overrides):
init_image = gr.Image(elem_id="video_image", show_label=False, type="pil", image_mode="RGB", width=256, height=256)
gr.HTML("<br>&nbsp Last image")
last_image = gr.Image(elem_id="video_last", show_label=False, type="pil", image_mode="RGB", width=256, height=256)
vlm_enhance, vlm_model, vlm_system_prompt = ui_video_vlm.create_ui(prompt_element=prompt, image_element=init_image)
with gr.Accordion(open=False, label="Output", elem_id='video_output_accordion'):
with gr.Accordion(label="Video", open=False, elem_id='video_output_accordion'):
with gr.Row():
save_frames = gr.Checkbox(label='Save image frames', value=False, elem_id="video_save_frames")
mp4_fps = gr.Slider(label="FPS", minimum=1, maximum=60, value=24, step=1)
mp4_interpolate = gr.Slider(label="Video interpolation", minimum=0, maximum=10, value=0, step=1)
with gr.Row():
video_type, video_duration, video_loop, video_pad, video_interpolate = ui_sections.create_video_inputs(tab='video', show_always=True)
mp4_codec = gr.Dropdown(label="Video codec", choices=['none', 'libx264'], value='libx264', type='value')
ui_common.create_refresh_button(mp4_codec, video_utils.get_codecs, elem_id="framepack_mp4_codec_refresh")
mp4_ext = gr.Textbox(label="Video format", value='mp4', elem_id="framepack_mp4_ext")
mp4_opt = gr.Textbox(label="Video options", value='crf:16', elem_id="framepack_mp4_ext")
with gr.Row():
mp4_video = gr.Checkbox(label='Video save video', value=True, elem_id="framepack_mp4_video")
mp4_frames = gr.Checkbox(label='Video save frames', value=False, elem_id="framepack_mp4_frames")
mp4_sf = gr.Checkbox(label='Video save safetensors', value=False, elem_id="framepack_mp4_sf")
# output panel with gallery and video tabs
with gr.Column(elem_id='video-output-column', scale=2) as _column_output:
with gr.Tabs(elem_classes=['video-output-tabs'], elem_id='video-output-tabs'):
with gr.Tab('Frames', id='out-gallery'):
gallery, gen_info, html_info, _html_info_formatted, html_log = ui_common.create_output_panel("video", prompt=prompt, preview=False, transfer=False, scale=2)
with gr.Tab('Video', id='out-video'):
video = gr.Video(label="Output", show_label=False, elem_id='control_output_video', elem_classes=['control-image'], height=512, autoplay=False)
with gr.Tab('Frames', id='out-gallery'):
gallery, gen_info, html_info, _html_info_formatted, html_log = ui_common.create_output_panel("video", prompt=prompt, preview=False, transfer=False, scale=2)
# connect reuse seed button
ui_common.connect_reuse_seed(seed, reuse_seed, gen_info, is_subseed=False)
@@ -151,8 +162,7 @@ def create_ui(prompt, negative, styles, overrides):
guidance_scale, guidance_true,
init_image, init_strength, last_image,
vae_type, vae_tile_frames,
save_frames,
video_type, video_duration, video_loop, video_pad, video_interpolate,
mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf,
overrides,
]
video_outputs = [