From d239bfcde182f266c1f25947a02a5108afd3c7ec Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 21 Aug 2026 10:26:17 +0200 Subject: [PATCH] placeholder video upscale Signed-off-by: Vladimir Mandic --- extensions-builtin/sdnext-modernui | 2 +- modules/framepack/framepack_ui.py | 3 ++- modules/framepack/framepack_worker.py | 3 +++ modules/framepack/framepack_wrappers.py | 3 ++- modules/ltx/ltx_process.py | 8 +++++++ modules/ltx/ltx_ui.py | 6 ++++-- modules/minimax/minimax_ui.py | 6 ++++-- modules/minimax/minimax_video.py | 6 +++++- modules/ui_video.py | 18 +++++++++++----- modules/video_models/video_run.py | 10 ++++++++- modules/video_models/video_save.py | 9 ++++++++ modules/video_models/video_ui.py | 28 +++++++++++++++++++------ modules/video_models/video_upscale.py | 7 +++++++ 13 files changed, 89 insertions(+), 20 deletions(-) create mode 100644 modules/video_models/video_upscale.py diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 6349f496f..f670238ad 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 6349f496f19ed4170a11a20224db487f0adfcd20 +Subproject commit f670238ad36f4c30f499153b132149b8c15188d9 diff --git a/modules/framepack/framepack_ui.py b/modules/framepack/framepack_ui.py index 48aa77983..c9fb51340 100644 --- a/modules/framepack/framepack_ui.py +++ b/modules/framepack/framepack_ui.py @@ -12,7 +12,7 @@ def change_sections(duration, mp4_fps, mp4_interpolate, latent_ws, variant): return gr.update(value=f'Target video: {num_frames} frames in {num_sections} sections'), gr.update(lines=max(2, 2*num_sections//3)) -def create_ui(prompt, negative, styles, _overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb): +def create_ui(prompt, negative, styles, _overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler): with gr.Row(): with gr.Column(variant='compact', elem_id="framepack_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): @@ -107,6 +107,7 @@ def create_ui(prompt, negative, styles, _overrides, script_inputs, mp4_fps, mp4_ shift, use_teacache, use_cfgzero, use_preview, mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate, + mp4_scale, mp4_upscaler, attention, vae_type, variant, vlm_enhance, vlm_model, vlm_system_prompt, ] diff --git a/modules/framepack/framepack_worker.py b/modules/framepack/framepack_worker.py index ae782657b..332243296 100644 --- a/modules/framepack/framepack_worker.py +++ b/modules/framepack/framepack_worker.py @@ -42,6 +42,7 @@ def worker( shift, use_teacache, use_cfgzero, use_preview, mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate, + mp4_scale, mp4_upscaler, vae_type, variant, metadata: dict | None = None, @@ -347,6 +348,8 @@ def worker( mp4_frames=mp4_frames, mp4_thumb=mp4_thumb, mp4_interpolate=0, + upscale_scale=mp4_scale, + upscale_upscaler=mp4_upscaler, pbar=pbar, stream=stream, metadata=metadata, diff --git a/modules/framepack/framepack_wrappers.py b/modules/framepack/framepack_wrappers.py index e4ef64d3a..94eb5155d 100644 --- a/modules/framepack/framepack_wrappers.py +++ b/modules/framepack/framepack_wrappers.py @@ -114,7 +114,7 @@ def unload_model(): yield gr.update(), gr.update(), 'Model unloaded' -def run_framepack(task_id, _ui_state, init_image, end_image, start_weight, end_weight, vision_weight, prompt, system_prompt, optimized_prompt, section_prompt, negative_prompt, styles, seed, resolution, duration, latent_ws, steps, cfg_scale, cfg_distilled, cfg_rescale, shift, use_teacache, use_cfgzero, use_preview, mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate, attention, vae_type, variant, vlm_enhance, vlm_model, vlm_system_prompt, *_args, **_kwargs): +def run_framepack(task_id, _ui_state, init_image, end_image, start_weight, end_weight, vision_weight, prompt, system_prompt, optimized_prompt, section_prompt, negative_prompt, styles, seed, resolution, duration, latent_ws, steps, cfg_scale, cfg_distilled, cfg_rescale, shift, use_teacache, use_cfgzero, use_preview, mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate, mp4_scale, mp4_upscaler, attention, vae_type, variant, vlm_enhance, vlm_model, vlm_system_prompt, *_args, **_kwargs): variant = variant or 'Bi-Directional' if variant == 'None': log.error('FramePack: no model selected') @@ -186,6 +186,7 @@ def run_framepack(task_id, _ui_state, init_image, end_image, start_weight, end_w shift, use_teacache, use_cfgzero, use_preview, mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate, + mp4_scale, mp4_upscaler, vae_type, variant, ) diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index c05989e37..687b1f3a6 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -166,6 +166,8 @@ def run(model: str, *, mp4_frames: bool = False, mp4_sf: bool = False, mp4_thumb: bool = True, + mp4_scale: float = 1.0, + mp4_upscaler: str = '', override_settings=None, ui_state=None, scripts=None, @@ -629,6 +631,8 @@ def run(model: str, *, mp4_thumb=mp4_thumb, mp4_interpolate=mp4_interpolate, aac_sample_rate=get_audio_rate(p), + upscale_scale=mp4_scale, + upscale_upscaler=mp4_upscaler, metadata={}, ) @@ -705,6 +709,8 @@ def run_ltx(task_id, mp4_frames: bool, mp4_sf: bool, mp4_thumb: bool, + mp4_scale: float, + mp4_upscaler: str, audio_enable: bool, _overrides, *args, @@ -760,6 +766,8 @@ def run_ltx(task_id, mp4_frames=mp4_frames, mp4_sf=mp4_sf, mp4_thumb=mp4_thumb, + mp4_scale=mp4_scale, + mp4_upscaler=mp4_upscaler, override_settings=_overrides, ui_state=_ui_state, script_args=args, diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index bd6cd709f..c3ece8685 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -58,7 +58,7 @@ def _model_change(model_name: str): ) -def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb): +def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler): with gr.Row(): with gr.Column(variant='compact', elem_id="ltx_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): @@ -170,7 +170,9 @@ def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_i refine_enable, refine_strength, ltx_condition_strength, ltx_init_image, last_image, condition_files, condition_video, condition_video_frames, condition_video_skip, decode_timestep, image_cond_noise_scale, - mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, + mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_scale, mp4_upscaler, audio_enable, overrides, ] diff --git a/modules/minimax/minimax_ui.py b/modules/minimax/minimax_ui.py index 2be8e60a7..bd03d2c7c 100644 --- a/modules/minimax/minimax_ui.py +++ b/modules/minimax/minimax_ui.py @@ -10,7 +10,7 @@ from modules.minimax import minimax_video, minimax_references debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None -def create_ui(prompt, _negative, styles, overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb): +def create_ui(prompt, _negative, styles, overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler): with gr.Row(): with gr.Column(variant='compact', elem_id="minimax_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): @@ -82,7 +82,9 @@ def create_ui(prompt, _negative, styles, overrides, script_inputs, mp4_fps, mp4_ steps, seed, init_image, last_image, reference_media, video_shift, audio_shift, - mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, + mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_scale, mp4_upscaler, audio_enable, overrides, ] diff --git a/modules/minimax/minimax_video.py b/modules/minimax/minimax_video.py index 0804f8128..bbdf78470 100644 --- a/modules/minimax/minimax_video.py +++ b/modules/minimax/minimax_video.py @@ -74,7 +74,9 @@ def generate(task_id, _ui_state, seed, init_image, last_image, reference_media, video_shift, audio_shift, - mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, + mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_scale, mp4_upscaler, audio_enable, _overrides, *args, @@ -181,6 +183,8 @@ def generate(task_id, _ui_state, mp4_thumb=mp4_thumb, mp4_interpolate=mp4_interpolate, aac_sample_rate=aac_sample_rate, + upscale_scale=mp4_scale, + upscale_upscaler=mp4_upscaler, metadata={}, ) _n, _c, _t, h, w = pixels.shape diff --git a/modules/ui_video.py b/modules/ui_video.py index ac302cfa4..72beae8f7 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -34,7 +34,7 @@ def create_ui(): overrides = ui_common.create_override_inputs('video') with gr.Tab('Output', id='video-outputs-tab') as _video_outputs_tab: from modules.video_models import video_ui - mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb = video_ui.create_ui_outputs() + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler = video_ui.create_ui_outputs() with gr.Tab('Extras', id='video-extras-tab', elem_id='video_extras') as _video_extras_tab: video_script_inputs = scripts_manager.scripts_video.setup_ui(parent='video', accordion=True) with gr.Tab('Models', id='video-core-tab') as video_core_tab: @@ -42,14 +42,18 @@ def create_ui(): engine, model, steps, sampler_index, width, height, frames, seed = video_ui.create_ui( prompt, negative, styles, overrides, video_script_inputs, - mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, + mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_scale, mp4_upscaler, ) with gr.Tab('MiniMax', id='minimax-tab') as minimax_tab: from modules.minimax import minimax_ui minimax_ui.create_ui( prompt, negative, styles, overrides, video_script_inputs, - mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, + mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_scale, mp4_upscaler, ) with gr.Tab('LTXVideo', id='ltx-tab') as ltx_tab: @@ -57,14 +61,18 @@ def create_ui(): ltx_ui.create_ui( prompt, negative, styles, overrides, video_script_inputs, - mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, + mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_scale, mp4_upscaler, ) with gr.Tab('FramePack', id='framepack-tab') as framepack_tab: from modules.framepack import framepack_ui framepack_ui.create_ui( prompt, negative, styles, overrides, video_script_inputs, - mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, + mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_scale, mp4_upscaler, ) paste_fields = [ diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 6c80412fc..97f76150c 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -125,6 +125,8 @@ def run(selected: models_def.Model, *, mp4_frames: bool = False, mp4_sf: bool = False, mp4_thumb: bool = True, + mp4_scale: float = 1.0, + mp4_upscaler: str = '', override_settings=None, engine: str | None = None, ui_state=None, @@ -348,6 +350,8 @@ def run(selected: models_def.Model, *, mp4_frames=mp4_frames, mp4_thumb=mp4_thumb, mp4_interpolate=mp4_interpolate, + upscale_scale=mp4_scale, + upscale_upscaler=mp4_upscaler, metadata={}, ) out_w, out_h = video_utils.pixel_size(processed.images, fallback=(p.width, p.height)) @@ -363,7 +367,9 @@ def generate(task_id, ui_state, seed, guidance_scale, guidance_true, init_image, init_strength, last_image, vae_type, vae_tile_frames, audio, - mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, + mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_scale, mp4_upscaler, override_settings, *args, **kwargs ): @@ -404,6 +410,8 @@ def generate(task_id, ui_state, mp4_frames=mp4_frames, mp4_sf=mp4_sf, mp4_thumb=mp4_thumb, + mp4_scale=mp4_scale, + mp4_upscaler=mp4_upscaler, override_settings=override_settings, engine=engine, ui_state=ui_state, diff --git a/modules/video_models/video_save.py b/modules/video_models/video_save.py index 028964aa6..916daf979 100644 --- a/modules/video_models/video_save.py +++ b/modules/video_models/video_save.py @@ -9,6 +9,7 @@ from PIL import Image from modules import shared, errors ,timer, rife, processing from modules.logger import log from modules.video_models.video_utils import check_av +from modules.video_models.video_upscale import upscale_video def get_audio_rate(p=None, default: int = 24000) -> int: @@ -285,6 +286,8 @@ def save_video( mp4_thumb: bool = True, # save thumbnail mp4_interpolate: int = 0, # rife interpolation aac_sample_rate: int = 24000, # audio sample rate + upscale_scale: float = 1.0, # upscale scale + upscale_upscaler: str = "", # upscale upscaler stream=None, # async progress reporting stream metadata: dict | None = None, # metadata for video pbar=None, # progress bar for video @@ -320,6 +323,12 @@ def save_video( if not torch.is_tensor(pixels): log.error(f'Video: type={type(pixels)} not a tensor') return 0, output_video, None + + if upscale_upscaler is not None and len(upscale_upscaler) > 0: + t_upscale = time.time() + pixels = upscale_video(pixels, scale=upscale_scale, upscaler=upscale_upscaler) + timer.process.add('upscale', time.time()-t_upscale) + t_save = time.time() if pixels.ndim == 4: pixels = pixels.unsqueeze(0) diff --git a/modules/video_models/video_ui.py b/modules/video_models/video_ui.py index c67549da1..87c3d39e6 100644 --- a/modules/video_models/video_ui.py +++ b/modules/video_models/video_ui.py @@ -55,6 +55,14 @@ def model_load(engine, model): yield msg +def refresh_upscalers(): + exclude = ['latent', 'interpolation', 'vips'] + from modules import modelloader + upscalers = modelloader.load_upscalers() + upscalers = [x for x in upscalers if all(e not in x.lower() for e in exclude)] + return upscalers + + def create_ui_outputs(): from modules.video_models import video_codecs default_codec = 'libx264' @@ -68,7 +76,7 @@ def create_ui_outputs(): with gr.Column(variant='compact', elem_id="video_outputs", elem_classes=['settings-column'], scale=1): with gr.Row(): mp4_fps = gr.Slider(label="Target FPS", minimum=1, maximum=60, value=24, step=1) - mp4_interpolate = gr.Slider(label="Video interpolation", minimum=0, maximum=10, value=0, step=1) + mp4_interpolate = gr.Slider(label="Interpolated frames", minimum=0, maximum=10, value=0, step=1, elem_id="video_outputs_interpolate") with gr.Row(): mp4_codec = gr.Dropdown(label="Video codec", choices=video_codecs.get_codec_list(), value=default_codec, type='value') ui_common.create_refresh_button(mp4_codec, video_utils.get_codecs, elem_id="video_mp4_codec_refresh") @@ -77,16 +85,22 @@ def create_ui_outputs(): # mp4_ext = gr.Textbox(label="Video format", value='mp4', elem_id="video_mp4_ext") mp4_ext = gr.Dropdown(label="Video format", choices=video_codecs.get_codec_allowed_exts(default_codec), value=video_codecs.get_codec_ext(default_codec), elem_id="video_mp4_ext") mp4_opt = gr.Textbox(label="FFmpeg options", value=video_codecs.get_codec_options(default_codec), elem_id="video_mp4_opt") - with gr.Row(): + with gr.Row(elem_id="video_outputs_save"): mp4_video = gr.Checkbox(label='Save: video', value=True, elem_id="video_mp4_video") + mp4_thumb = gr.Checkbox(label='Save: thumbnail', value=True, elem_id="video_mp4_thumb") mp4_frames = gr.Checkbox(label='Save: frames', value=False, elem_id="video_mp4_frames") mp4_sf = gr.Checkbox(label='Save: safetensors', value=False, elem_id="video_mp4_sf") - mp4_thumb = gr.Checkbox(label='Save: thumbnail', value=True, elem_id="video_mp4_thumb") mp4_codec.change(fn=on_codec_change, inputs=[mp4_codec], outputs=[mp4_info, mp4_ext, mp4_opt], show_progress='hidden') - return mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb + with gr.Group(elem_id="video_outputs_upscale"): + with gr.Row(): + upscale_scale = gr.Slider(label="Video scale", minimum=1, maximum=4, value=1, step=0.1, elem_id="video_outputs_upscale_scale") + with gr.Row(): + upscale_upscaler = gr.Dropdown(label="Video Upscaler", choices=['None'], value='None', type='value', elem_id="video_outputs_upscale_upscaler") + _upscale_upscaler_btn = ui_common.create_refresh_button(upscale_upscaler, refresh_upscalers) + return mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, upscale_scale, upscale_upscaler -def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb): +def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler): with gr.Row(): with gr.Column(variant='compact', elem_id="video_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): @@ -154,7 +168,9 @@ def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_i guidance_scale, guidance_true, init_image, init_strength, last_image, vae_type, vae_tile_frames, audio, - mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, + mp4_video, mp4_frames, mp4_sf, mp4_thumb, + mp4_scale, mp4_upscaler, overrides, ] video_outputs = [ diff --git a/modules/video_models/video_upscale.py b/modules/video_models/video_upscale.py new file mode 100644 index 000000000..5417677fc --- /dev/null +++ b/modules/video_models/video_upscale.py @@ -0,0 +1,7 @@ +import torch +from modules.logger import log + + +def upscale_video(pixels: torch.Tensor, scale: float = 1.0, upscaler: str = ""): + log.debug(f'Upscale video: scale={scale} upscaler="{upscaler}" shape={list(pixels.shape)} TODO') + return pixels