placeholder video upscale

Signed-off-by: Vladimir Mandic <mandic00@live.com>

modules/framepack/framepack_ui.py

@@ -12,7 +12,7 @@ def change_sections(duration, mp4_fps, mp4_interpolate, latent_ws, variant):
     return gr.update(value=f'Target video: {num_frames} frames in {num_sections} sections'), gr.update(lines=max(2, 2*num_sections//3))
 
 
-def create_ui(prompt, negative, styles, _overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb):
+def create_ui(prompt, negative, styles, _overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler):
     with gr.Row():
         with gr.Column(variant='compact', elem_id="framepack_settings", elem_classes=['settings-column'], scale=1):
             with gr.Row():
@@ -107,6 +107,7 @@ def create_ui(prompt, negative, styles, _overrides, script_inputs, mp4_fps, mp4_
         shift,
         use_teacache, use_cfgzero, use_preview,
         mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate,
+        mp4_scale, mp4_upscaler,
         attention, vae_type, variant,
         vlm_enhance, vlm_model, vlm_system_prompt,
     ]

modules/framepack/framepack_worker.py

@@ -42,6 +42,7 @@ def worker(
         shift,
         use_teacache, use_cfgzero, use_preview,
         mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate,
+        mp4_scale, mp4_upscaler,
         vae_type,
         variant,
         metadata: dict | None = None,
@@ -347,6 +348,8 @@ def worker(
                 mp4_frames=mp4_frames,
                 mp4_thumb=mp4_thumb,
                 mp4_interpolate=0,
+                upscale_scale=mp4_scale,
+                upscale_upscaler=mp4_upscaler,
                 pbar=pbar,
                 stream=stream,
                 metadata=metadata,

modules/framepack/framepack_wrappers.py

@@ -114,7 +114,7 @@ def unload_model():
     yield gr.update(), gr.update(), 'Model unloaded'
 
 
-def run_framepack(task_id, _ui_state, init_image, end_image, start_weight, end_weight, vision_weight, prompt, system_prompt, optimized_prompt, section_prompt, negative_prompt, styles, seed, resolution, duration, latent_ws, steps, cfg_scale, cfg_distilled, cfg_rescale, shift, use_teacache, use_cfgzero, use_preview, mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate, attention, vae_type, variant, vlm_enhance, vlm_model, vlm_system_prompt, *_args, **_kwargs):
+def run_framepack(task_id, _ui_state, init_image, end_image, start_weight, end_weight, vision_weight, prompt, system_prompt, optimized_prompt, section_prompt, negative_prompt, styles, seed, resolution, duration, latent_ws, steps, cfg_scale, cfg_distilled, cfg_rescale, shift, use_teacache, use_cfgzero, use_preview, mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate, mp4_scale, mp4_upscaler, attention, vae_type, variant, vlm_enhance, vlm_model, vlm_system_prompt, *_args, **_kwargs):
     variant = variant or 'Bi-Directional'
     if variant == 'None':
         log.error('FramePack: no model selected')
@@ -186,6 +186,7 @@ def run_framepack(task_id, _ui_state, init_image, end_image, start_weight, end_w
             shift,
             use_teacache, use_cfgzero, use_preview,
             mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate,
+            mp4_scale, mp4_upscaler,
             vae_type, variant,
         )
 

modules/ltx/ltx_process.py

@@ -166,6 +166,8 @@ def run(model: str, *,
         mp4_frames: bool = False,
         mp4_sf: bool = False,
         mp4_thumb: bool = True,
+        mp4_scale: float = 1.0,
+        mp4_upscaler: str = '',
         override_settings=None,
         ui_state=None,
         scripts=None,
@@ -629,6 +631,8 @@ def run(model: str, *,
                 mp4_thumb=mp4_thumb,
                 mp4_interpolate=mp4_interpolate,
                 aac_sample_rate=get_audio_rate(p),
+                upscale_scale=mp4_scale,
+                upscale_upscaler=mp4_upscaler,
                 metadata={},
             )
 
@@ -705,6 +709,8 @@ def run_ltx(task_id,
             mp4_frames: bool,
             mp4_sf: bool,
             mp4_thumb: bool,
+            mp4_scale: float,
+            mp4_upscaler: str,
             audio_enable: bool,
             _overrides,
             *args,
@@ -760,6 +766,8 @@ def run_ltx(task_id,
                       mp4_frames=mp4_frames,
                       mp4_sf=mp4_sf,
                       mp4_thumb=mp4_thumb,
+                      mp4_scale=mp4_scale,
+                      mp4_upscaler=mp4_upscaler,
                       override_settings=_overrides,
                       ui_state=_ui_state,
                       script_args=args,

modules/ltx/ltx_ui.py

@@ -58,7 +58,7 @@ def _model_change(model_name: str):
     )
 
 
-def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb):
+def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler):
     with gr.Row():
         with gr.Column(variant='compact', elem_id="ltx_settings", elem_classes=['settings-column'], scale=1):
             with gr.Row():
@@ -170,7 +170,9 @@ def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_i
         refine_enable, refine_strength,
         ltx_condition_strength, ltx_init_image, last_image, condition_files, condition_video, condition_video_frames, condition_video_skip,
         decode_timestep, image_cond_noise_scale,
-        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt,
+        mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+        mp4_scale, mp4_upscaler,
         audio_enable,
         overrides,
     ]

modules/minimax/minimax_ui.py

@@ -10,7 +10,7 @@ from modules.minimax import minimax_video, minimax_references
 debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
 
 
-def create_ui(prompt, _negative, styles, overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb):
+def create_ui(prompt, _negative, styles, overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler):
     with gr.Row():
         with gr.Column(variant='compact', elem_id="minimax_settings", elem_classes=['settings-column'], scale=1):
             with gr.Row():
@@ -82,7 +82,9 @@ def create_ui(prompt, _negative, styles, overrides, script_inputs, mp4_fps, mp4_
         steps, seed,
         init_image, last_image, reference_media,
         video_shift, audio_shift,
-        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt,
+        mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+        mp4_scale, mp4_upscaler,
         audio_enable,
         overrides,
     ]

modules/minimax/minimax_video.py

@@ -74,7 +74,9 @@ def generate(task_id, _ui_state,
              seed,
              init_image, last_image, reference_media,
              video_shift, audio_shift,
-             mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+             mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt,
+             mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+             mp4_scale, mp4_upscaler,
              audio_enable,
             _overrides,
             *args,
@@ -181,6 +183,8 @@ def generate(task_id, _ui_state,
                 mp4_thumb=mp4_thumb,
                 mp4_interpolate=mp4_interpolate,
                 aac_sample_rate=aac_sample_rate,
+                upscale_scale=mp4_scale,
+                upscale_upscaler=mp4_upscaler,
                 metadata={},
             )
             _n, _c, _t, h, w = pixels.shape

modules/ui_video.py

@@ -34,7 +34,7 @@ def create_ui():
                 overrides = ui_common.create_override_inputs('video')
                 with gr.Tab('Output', id='video-outputs-tab') as _video_outputs_tab:
                     from modules.video_models import video_ui
-                    mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb = video_ui.create_ui_outputs()
+                    mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler = video_ui.create_ui_outputs()
                 with gr.Tab('Extras', id='video-extras-tab', elem_id='video_extras') as _video_extras_tab:
                     video_script_inputs = scripts_manager.scripts_video.setup_ui(parent='video', accordion=True)
                 with gr.Tab('Models', id='video-core-tab') as video_core_tab:
@@ -42,14 +42,18 @@ def create_ui():
                     engine, model, steps, sampler_index, width, height, frames, seed = video_ui.create_ui(
                         prompt, negative, styles,
                         overrides, video_script_inputs,
-                        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+                        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt,
+                        mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+                        mp4_scale, mp4_upscaler,
                     )
                 with gr.Tab('MiniMax', id='minimax-tab') as minimax_tab:
                     from modules.minimax import minimax_ui
                     minimax_ui.create_ui(
                         prompt, negative, styles,
                         overrides, video_script_inputs,
-                        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+                        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt,
+                        mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+                        mp4_scale, mp4_upscaler,
                     )
 
                 with gr.Tab('LTXVideo', id='ltx-tab') as ltx_tab:
@@ -57,14 +61,18 @@ def create_ui():
                     ltx_ui.create_ui(
                         prompt, negative, styles,
                         overrides, video_script_inputs,
-                        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+                        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt,
+                        mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+                        mp4_scale, mp4_upscaler,
                     )
                 with gr.Tab('FramePack', id='framepack-tab') as framepack_tab:
                     from modules.framepack import framepack_ui
                     framepack_ui.create_ui(
                         prompt, negative, styles,
                         overrides, video_script_inputs,
-                        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+                        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt,
+                        mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+                        mp4_scale, mp4_upscaler,
                     )
 
         paste_fields = [

modules/video_models/video_run.py

@@ -125,6 +125,8 @@ def run(selected: models_def.Model, *,
         mp4_frames: bool = False,
         mp4_sf: bool = False,
         mp4_thumb: bool = True,
+        mp4_scale: float = 1.0,
+        mp4_upscaler: str = '',
         override_settings=None,
         engine: str | None = None,
         ui_state=None,
@@ -348,6 +350,8 @@ def run(selected: models_def.Model, *,
         mp4_frames=mp4_frames,
         mp4_thumb=mp4_thumb,
         mp4_interpolate=mp4_interpolate,
+        upscale_scale=mp4_scale,
+        upscale_upscaler=mp4_upscaler,
         metadata={},
     )
     out_w, out_h = video_utils.pixel_size(processed.images, fallback=(p.width, p.height))
@@ -363,7 +367,9 @@ def generate(task_id, ui_state,
              seed, guidance_scale, guidance_true,
              init_image, init_strength, last_image,
              vae_type, vae_tile_frames, audio,
-             mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+             mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt,
+             mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+             mp4_scale, mp4_upscaler,
              override_settings,
              *args, **kwargs
             ):
@@ -404,6 +410,8 @@ def generate(task_id, ui_state,
                   mp4_frames=mp4_frames,
                   mp4_sf=mp4_sf,
                   mp4_thumb=mp4_thumb,
+                  mp4_scale=mp4_scale,
+                  mp4_upscaler=mp4_upscaler,
                   override_settings=override_settings,
                   engine=engine,
                   ui_state=ui_state,

modules/video_models/video_save.py

@@ -9,6 +9,7 @@ from PIL import Image
 from modules import shared, errors ,timer, rife, processing
 from modules.logger import log
 from modules.video_models.video_utils import check_av
+from modules.video_models.video_upscale import upscale_video
 
 
 def get_audio_rate(p=None, default: int = 24000) -> int:
@@ -285,6 +286,8 @@ def save_video(
     mp4_thumb: bool = True,  # save thumbnail
     mp4_interpolate: int = 0,  # rife interpolation
     aac_sample_rate: int = 24000,  # audio sample rate
+    upscale_scale: float = 1.0,  # upscale scale
+    upscale_upscaler: str = "",  # upscale upscaler
     stream=None,  # async progress reporting stream
     metadata: dict | None = None,  # metadata for video
     pbar=None,  # progress bar for video
@@ -320,6 +323,12 @@ def save_video(
     if not torch.is_tensor(pixels):
         log.error(f'Video: type={type(pixels)} not a tensor')
         return 0, output_video, None
+
+    if upscale_upscaler is not None and len(upscale_upscaler) > 0:
+        t_upscale = time.time()
+        pixels = upscale_video(pixels, scale=upscale_scale, upscaler=upscale_upscaler)
+        timer.process.add('upscale', time.time()-t_upscale)
+
     t_save = time.time()
     if pixels.ndim == 4:
         pixels = pixels.unsqueeze(0)

modules/video_models/video_ui.py

@@ -55,6 +55,14 @@ def model_load(engine, model):
     yield msg
 
 
+def refresh_upscalers():
+    exclude = ['latent', 'interpolation', 'vips']
+    from modules import modelloader
+    upscalers = modelloader.load_upscalers()
+    upscalers = [x for x in upscalers if all(e not in x.lower() for e in exclude)]
+    return upscalers
+
+
 def create_ui_outputs():
     from modules.video_models import video_codecs
     default_codec = 'libx264'
@@ -68,7 +76,7 @@ def create_ui_outputs():
         with gr.Column(variant='compact', elem_id="video_outputs", elem_classes=['settings-column'], scale=1):
             with gr.Row():
                 mp4_fps = gr.Slider(label="Target FPS", minimum=1, maximum=60, value=24, step=1)
-                mp4_interpolate = gr.Slider(label="Video interpolation", minimum=0, maximum=10, value=0, step=1)
+                mp4_interpolate = gr.Slider(label="Interpolated frames", minimum=0, maximum=10, value=0, step=1, elem_id="video_outputs_interpolate")
             with gr.Row():
                 mp4_codec = gr.Dropdown(label="Video codec", choices=video_codecs.get_codec_list(), value=default_codec, type='value')
                 ui_common.create_refresh_button(mp4_codec, video_utils.get_codecs, elem_id="video_mp4_codec_refresh")
@@ -77,16 +85,22 @@ def create_ui_outputs():
                 # mp4_ext = gr.Textbox(label="Video format", value='mp4', elem_id="video_mp4_ext")
                 mp4_ext = gr.Dropdown(label="Video format", choices=video_codecs.get_codec_allowed_exts(default_codec), value=video_codecs.get_codec_ext(default_codec), elem_id="video_mp4_ext")
                 mp4_opt = gr.Textbox(label="FFmpeg options", value=video_codecs.get_codec_options(default_codec), elem_id="video_mp4_opt")
-            with gr.Row():
+            with gr.Row(elem_id="video_outputs_save"):
                 mp4_video = gr.Checkbox(label='Save: video', value=True, elem_id="video_mp4_video")
+                mp4_thumb = gr.Checkbox(label='Save: thumbnail', value=True, elem_id="video_mp4_thumb")
                 mp4_frames = gr.Checkbox(label='Save: frames', value=False, elem_id="video_mp4_frames")
                 mp4_sf = gr.Checkbox(label='Save: safetensors', value=False, elem_id="video_mp4_sf")
-                mp4_thumb = gr.Checkbox(label='Save: thumbnail', value=True, elem_id="video_mp4_thumb")
             mp4_codec.change(fn=on_codec_change, inputs=[mp4_codec], outputs=[mp4_info, mp4_ext, mp4_opt], show_progress='hidden')
-    return mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb
+            with gr.Group(elem_id="video_outputs_upscale"):
+                with gr.Row():
+                    upscale_scale = gr.Slider(label="Video scale", minimum=1, maximum=4, value=1, step=0.1, elem_id="video_outputs_upscale_scale")
+                with gr.Row():
+                    upscale_upscaler = gr.Dropdown(label="Video Upscaler", choices=['None'], value='None', type='value', elem_id="video_outputs_upscale_upscaler")
+                    _upscale_upscaler_btn = ui_common.create_refresh_button(upscale_upscaler, refresh_upscalers)
+    return mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, upscale_scale, upscale_upscaler
 
 
-def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb):
+def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler):
     with gr.Row():
         with gr.Column(variant='compact', elem_id="video_settings", elem_classes=['settings-column'], scale=1):
             with gr.Row():
@@ -154,7 +168,9 @@ def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_i
         guidance_scale, guidance_true,
         init_image, init_strength, last_image,
         vae_type, vae_tile_frames, audio,
-        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+        mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt,
+        mp4_video, mp4_frames, mp4_sf, mp4_thumb,
+        mp4_scale, mp4_upscaler,
         overrides,
     ]
     video_outputs = [

modules/video_models/video_upscale.py

@@ -0,0 +1,7 @@
+import torch
+from modules.logger import log
+
+
+def upscale_video(pixels: torch.Tensor, scale: float = 1.0, upscaler: str = ""):
+    log.debug(f'Upscale video: scale={scale} upscaler="{upscaler}" shape={list(pixels.shape)} TODO')
+    return pixels