From 84c1ed66b3101a33b905749355a821da3dd2876d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 21 Aug 2026 20:28:44 +0200 Subject: [PATCH] video upscaling Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 9 ++++-- extensions-builtin/sd-extension-chainner | 2 +- modules/upscaler_spandrel.py | 2 +- modules/video_models/models_def.py | 6 ++-- modules/video_models/video_upscale.py | 37 +++++++++++++++--------- scripts/postprocessing_upscale.py | 5 +++- wiki | 2 +- 7 files changed, 40 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d03df2e9..442734c6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ ## Highlights for 2026-08-21 Time for a new release, this is a larger one! -Main focus is improving video workflows which also brings full support for new [MiniMax H3](https://vladmandic.github.io/sdnext-docs/MiniMax) and [LTXVideo-2.5](https://vladmandic.github.io/sdnext-docs/LTX) +Main focus is improving video workflows which also brings full support for new [MiniMax H3](https://vladmandic.github.io/sdnext-docs/MiniMax) and [LTXVideo-2.5](https://vladmandic.github.io/sdnext-docs/LTX) +and improves general video processing with flexible video upscaling, updated interpolation, etc. *What else?* - [Detailer.next](https://vladmandic.github.io/sdnext-docs/Detailer) with new support for *vision-language models* and *per-class prompts* @@ -58,12 +59,16 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa - **Video** - reorganized *video* tab - better support for video codeces and formats + - add flexible video upscaling as video generation step + see [Video wiki page](https://vladmandic.github.io/sdnext-docs/Video) for details and recommended upscaling models + - improved interpolation - add *generate forever* button - **Upscalers** - update *spandrel* integration moving forward, spandrel engine will be main upscaling engine for sdnext when downloading any upscaling models manually, place them in `models/Spandrel` folder - - add several compact/light upscalers that are better suited for video upscaling + - update *chainner* integration + - add several low-latency upscalers that are better suited for video upscaling - **API** - full support for video generation using api new endpoints: `/sdapi/v1/video`, `/sdapi/v1/video/models`, `/sdapi/v1/video/file` diff --git a/extensions-builtin/sd-extension-chainner b/extensions-builtin/sd-extension-chainner index bd5f5a3cf..78f29b45f 160000 --- a/extensions-builtin/sd-extension-chainner +++ b/extensions-builtin/sd-extension-chainner @@ -1 +1 @@ -Subproject commit bd5f5a3cf736bc3cd687dfeea19567645927f7db +Subproject commit 78f29b45f248e6029c0924862de8a3f06819c053 diff --git a/modules/upscaler_spandrel.py b/modules/upscaler_spandrel.py index 67dd795c4..0a9f938db 100644 --- a/modules/upscaler_spandrel.py +++ b/modules/upscaler_spandrel.py @@ -92,7 +92,7 @@ class UpscalerSpandrel(Upscaler): self.model = spandrel.ModelLoader().load_from_file(model.local_data_path) self.model.to(devices.device).eval() - def do_upscale(self, img: Image.Image | torch.Tensor | np.ndarray, selected_model=None, output_type='pil', quiet=False): + def do_upscale(self, img: Image.Image | torch.Tensor | np.ndarray, selected_model: str | None = None, output_type='pil', quiet=False): try: if (self.model is None) or (self.selected != selected_model): self.load_model(selected_model) diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index 5edeae458..75075c483 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -537,7 +537,7 @@ try: ], 'Mochi Video': [ Model(name='None'), - Model(name='Mochi 1 T2V', + Model(name='Mochi 1 20B T2V', url='https://huggingface.co/genmo/mochi-1-preview', repo='genmo/mochi-1-preview', repo_cls='MochiPipeline', @@ -546,7 +546,7 @@ try: ], 'Latte Video': [ Model(name='None'), - Model(name='Latte 1 T2V', + Model(name='Latte 1 1B T2V', url='https://huggingface.co/maxin-cn/Latte-1', repo='maxin-cn/Latte-1', repo_cls='LattePipeline', @@ -555,7 +555,7 @@ try: ], 'Allegro Video': [ Model(name='None'), - Model(name='Allegro T2V', + Model(name='Allegro 2.8B T2V', url='https://huggingface.co/rhymes-ai/Allegro', repo='rhymes-ai/Allegro', repo_cls='AllegroPipeline', diff --git a/modules/video_models/video_upscale.py b/modules/video_models/video_upscale.py index 2be09c5c2..be00a1c3d 100644 --- a/modules/video_models/video_upscale.py +++ b/modules/video_models/video_upscale.py @@ -1,10 +1,14 @@ import time import inspect import torch -from modules.logger import log +import rich.progress as rp +from modules.logger import log, console from modules import shared, upscaler +pbar = rp.Progress(rp.TextColumn('[cyan]Upscale:'), rp.BarColumn(), rp.MofNCompleteColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=console) + + def load_upscaler(upscaler_name: str) -> upscaler.UpscalerData | None: upscalers = [x for x in shared.sd_upscalers if x.name.lower().replace('-', ' ') == upscaler_name.lower().replace('-', ' ')] # use inspect to check if upscaler.scaler method has output_type param, if not, then it is an old upscaler and we should not use it for video @@ -43,19 +47,24 @@ def upscale_video(pixels: torch.Tensor, scale: float = 1.0, upscaler_name: str = return pixels outputs = [] t0 = time.time() - for idx in range(frames.shape[2]): - frame = frames[:, :, idx, :, :] # BCHW - w = int(frame.shape[-1] * scale) - h = int(frame.shape[-2] * scale) - # upscale - frame = model.scaler.do_upscale(frame, model.name, output_type='tensor', quiet=True) - frame = frame * 2.0 - 1.0 # upscaler returns 0:1, need -1:1 for video - if frame.ndim == 3: - frame = frame.unsqueeze(0) - # interpolate to exact size - if frame.shape[-1] != w or frame.shape[-2] != h: - frame = torch.nn.functional.interpolate(frame, size=(h, w), mode='lanczos', align_corners=False, antialias=True) - outputs.append(frame) + with pbar: + num_frames = frames.shape[2] + task = pbar.add_task(total=num_frames, description='starting...') + for idx in range(num_frames): + pbar.update(task, advance=1, description=f'frame {idx + 1}/{num_frames}') + frame = frames[:, :, idx, :, :] # BCHW + w = int(frame.shape[-1] * scale) + h = int(frame.shape[-2] * scale) + # upscale + frame = model.scaler.do_upscale(frame, model.name, output_type='tensor', quiet=True) + frame = frame * 2.0 - 1.0 # upscaler returns 0:1, need -1:1 for video + if frame.ndim == 3: + frame = frame.unsqueeze(0) + # interpolate to exact size + if frame.shape[-1] != w or frame.shape[-2] != h: + frame = torch.nn.functional.interpolate(frame, size=(h, w), mode='lanczos', align_corners=False, antialias=True) + outputs.append(frame) + pbar.remove_task(task) outputs = torch.stack(outputs, dim=2) t1 = time.time() frames = outputs.shape[2] diff --git a/scripts/postprocessing_upscale.py b/scripts/postprocessing_upscale.py index 9726b23b3..fcb491bd4 100644 --- a/scripts/postprocessing_upscale.py +++ b/scripts/postprocessing_upscale.py @@ -72,7 +72,10 @@ class ScriptPostprocessingUpscale(scripts_postprocessing.ScriptPostprocessing): upscaler_1_name = None upscaler1 = next(iter([x for x in shared.sd_upscalers if x.name == upscaler_1_name]), None) if not upscaler1: - if upscaler_1_name is not None: + from modules.modelloader import load_upscalers # refresh loadable upscalers if not found + load_upscalers() + upscaler1 = next(iter([x for x in shared.sd_upscalers if x.name == upscaler_1_name]), None) + if not upscaler1 and upscaler_1_name is not None: log.warning(f"Could not find upscaler: {upscaler_1_name or ''}") return upscaled_image = self.upscale(pp.image, pp.info, upscaler1, upscale_mode, upscale_by, upscale_to_width, upscale_to_height, upscale_crop) diff --git a/wiki b/wiki index a7de1bc0c..b4c702bea 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit a7de1bc0c8619a31da3e14688aeec17988ac8024 +Subproject commit b4c702beae32f3e1f85656ed0c832506a4a23cb6