From dc4c58d0cb62b6b269f9ed1a87160747b1045d8f Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sat, 25 Apr 2026 21:14:18 +0100 Subject: [PATCH 1/8] feat(rife): upgrade vendored RIFE to Practical-RIFE v4.25 - Vendor IFNet_HDv3 v4.25 (5 IFBlocks, Head encoder, feat channel) and v4 warplayer with explicit (tenFlow_div, backwarp_tenGrid) signature - Rewrite RifeModel.inference for the new forward signature with per-(H,W,device,dtype) caching of tenFlow_div and backwarp_tenGrid - Force fp32 inference: bf16 produced visible checkerboard at the new IFNet's depth (was hidden by v3.9's shallower architecture) - Crop padded frames in interpolate_nchw before output (was missing, produced gray bar on non-128-aligned inputs) - Drop training scaffolding (AdamW, EPE/SOBEL, update method) - Log obsolete legacy v3.9 weights file on first v4.25 load instead of silently deleting user data - Default download URL is HolyWu vs-rife mirror (MIT, byte-identical upstream weights); swap to project-hosted URL before merge --- modules/rife/__init__.py | 15 ++-- modules/rife/model_ifnet.py | 145 ++++++++++++++++++------------------ modules/rife/model_rife.py | 72 ++++++++---------- modules/rife/warplayer.py | 21 ++---- 4 files changed, 123 insertions(+), 130 deletions(-) diff --git a/modules/rife/__init__.py b/modules/rife/__init__.py index b6c712e0c..e26c7aafc 100644 --- a/modules/rife/__init__.py +++ b/modules/rife/__init__.py @@ -16,16 +16,21 @@ from modules import devices, shared, paths from modules.logger import log -model_url = 'https://github.com/vladmandic/rife/raw/main/model/flownet-v46.pkl' +# Practical-RIFE v4.25 weights (MIT). Default URL points at the upstream HolyWu mirror; +# can be swapped to a self-hosted mirror without any other code change. +model_url = 'https://github.com/HolyWu/vs-rife/releases/download/model/flownet_v4.25.pkl' model: RifeModel = None -def load(model_path: str = 'rife/flownet-v46.pkl'): +def load(model_path: str = 'rife/flownet_v4.25.pkl'): global model # pylint: disable=global-statement if model is None: from modules import modelloader model_dir = os.path.join(paths.models_path, 'RIFE') - model_path = modelloader.load_file_from_url(url=model_url, model_dir=model_dir, file_name='flownet-v46.pkl') + model_path = modelloader.load_file_from_url(url=model_url, model_dir=model_dir, file_name='flownet_v4.25.pkl') + legacy_path = os.path.join(model_dir, 'flownet-v46.pkl') + if os.path.exists(legacy_path): + log.info(f'Video interpolate: legacy v3.9 weights at "{legacy_path}" are no longer used and can be deleted') log.debug(f'Video interpolate: model="{model_path}"') model = RifeModel() model.load_model(model_path, -1) @@ -144,8 +149,8 @@ def interpolate_nchw(images: list, count: int = 2, scale: float = 1.0): I1 = f_pad(frame.unsqueeze(0)) for i in range(count-1): output = model.inference(I0, I1, (i+1) * 1. / (count), scale) - interpolated.append(output) - interpolated.append(I1) + interpolated.append(output[:, :, :h, :w]) + interpolated.append(I1[:, :, :h, :w]) pbar.update(1) t1 = time.time() diff --git a/modules/rife/model_ifnet.py b/modules/rife/model_ifnet.py index df32b59a1..06ab8ef2c 100644 --- a/modules/rife/model_ifnet.py +++ b/modules/rife/model_ifnet.py @@ -1,49 +1,60 @@ -import os -import sys import torch import torch.nn as nn import torch.nn.functional as F -sys.path.append(os.path.dirname(__file__)) -from warplayer import warp # pylint: disable=wrong-import-position - - -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +from modules.rife.warplayer import warp def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1): return nn.Sequential( - nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, - padding=padding, dilation=dilation, bias=True), - nn.LeakyReLU(0.2, True) + nn.Conv2d( + in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=True + ), + nn.LeakyReLU(0.2, True), ) -def conv_bn(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1): - return nn.Sequential( - nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, - padding=padding, dilation=dilation, bias=False), - nn.BatchNorm2d(out_planes), - nn.LeakyReLU(0.2, True) - ) + +class Head(nn.Module): + def __init__(self): + super(Head, self).__init__() + self.cnn0 = nn.Conv2d(3, 16, 3, 2, 1) + self.cnn1 = nn.Conv2d(16, 16, 3, 1, 1) + self.cnn2 = nn.Conv2d(16, 16, 3, 1, 1) + self.cnn3 = nn.ConvTranspose2d(16, 4, 4, 2, 1) + self.relu = nn.LeakyReLU(0.2, True) + + def forward(self, x, feat=False): + x = x.clamp(0.0, 1.0) + x0 = self.cnn0(x) + x = self.relu(x0) + x1 = self.cnn1(x) + x = self.relu(x1) + x2 = self.cnn2(x) + x = self.relu(x2) + x3 = self.cnn3(x) + if feat: + return [x0, x1, x2, x3] + return x3 + class ResConv(nn.Module): def __init__(self, c, dilation=1): - super().__init__() - self.conv = nn.Conv2d(c, c, 3, 1, dilation, dilation=dilation, groups=1\ -) + super(ResConv, self).__init__() + self.conv = nn.Conv2d(c, c, 3, 1, dilation, dilation=dilation, groups=1) self.beta = nn.Parameter(torch.ones((1, c, 1, 1)), requires_grad=True) self.relu = nn.LeakyReLU(0.2, True) def forward(self, x): return self.relu(self.conv(x) * self.beta + x) + class IFBlock(nn.Module): def __init__(self, in_planes, c=64): - super().__init__() + super(IFBlock, self).__init__() self.conv0 = nn.Sequential( - conv(in_planes, c//2, 3, 2, 1), - conv(c//2, c, 3, 2, 1), - ) + conv(in_planes, c // 2, 3, 2, 1), + conv(c // 2, c, 3, 2, 1), + ) self.convblock = nn.Sequential( ResConv(c), ResConv(c), @@ -54,45 +65,39 @@ class IFBlock(nn.Module): ResConv(c), ResConv(c), ) - self.lastconv = nn.Sequential( - nn.ConvTranspose2d(c, 4*6, 4, 2, 1), - nn.PixelShuffle(2) - ) + self.lastconv = nn.Sequential(nn.ConvTranspose2d(c, 4 * 13, 4, 2, 1), nn.PixelShuffle(2)) def forward(self, x, flow=None, scale=1): - x = F.interpolate(x, scale_factor= 1. / scale, mode="bilinear", align_corners=False) + x = F.interpolate(x, scale_factor=1.0 / scale, mode="bilinear") if flow is not None: - flow = F.interpolate(flow, scale_factor= 1. / scale, mode="bilinear", align_corners=False) * 1. / scale + flow = F.interpolate(flow, scale_factor=1.0 / scale, mode="bilinear") / scale x = torch.cat((x, flow), 1) feat = self.conv0(x) feat = self.convblock(feat) tmp = self.lastconv(feat) - tmp = F.interpolate(tmp, scale_factor=scale, mode="bilinear", align_corners=False) + tmp = F.interpolate(tmp, scale_factor=scale, mode="bilinear") flow = tmp[:, :4] * scale mask = tmp[:, 4:5] - return flow, mask + feat = tmp[:, 5:] + return flow, mask, feat + class IFNet(nn.Module): - def __init__(self): - super().__init__() - self.block0 = IFBlock(7, c=192) - self.block1 = IFBlock(8+4, c=128) - self.block2 = IFBlock(8+4, c=96) - self.block3 = IFBlock(8+4, c=64) - # self.contextnet = Contextnet() - # self.unet = Unet() + def __init__(self, scale=1, ensemble=False): + super(IFNet, self).__init__() + self.block0 = IFBlock(7 + 8, c=192) + self.block1 = IFBlock(8 + 4 + 8 + 8, c=128) + self.block2 = IFBlock(8 + 4 + 8 + 8, c=96) + self.block3 = IFBlock(8 + 4 + 8 + 8, c=64) + self.block4 = IFBlock(8 + 4 + 8 + 8, c=32) + self.encode = Head() + self.scale_list = [16 / scale, 8 / scale, 4 / scale, 2 / scale, 1 / scale] + if ensemble: + raise ValueError("rife: ensemble is not supported in v4.25") - def forward( self, x, timestep=0.5, scale_list=None, training=False, fastmode=True, ensemble=False): # pylint: disable=dangerous-default-value, unused-argument - if scale_list is None: - scale_list = [8, 4, 2, 1] - if training is False: - channel = x.shape[1] // 2 - img0 = x[:, :channel] - img1 = x[:, channel:] - if not torch.is_tensor(timestep): - timestep = (x[:, :1].clone() * 0 + 1) * timestep - else: - timestep = timestep.repeat(1, 1, img0.shape[2], img0.shape[3]) + def forward(self, img0, img1, timestep, tenFlow_div, backwarp_tenGrid, f0, f1): + img0 = img0.clamp(0.0, 1.0) + img1 = img1.clamp(0.0, 1.0) flow_list = [] merged = [] mask_list = [] @@ -100,28 +105,26 @@ class IFNet(nn.Module): warped_img1 = img1 flow = None mask = None - # loss_cons = 0 - block = [self.block0, self.block1, self.block2, self.block3] - for i in range(4): + block = [self.block0, self.block1, self.block2, self.block3, self.block4] + for i in range(5): if flow is None: - flow, mask = block[i](torch.cat((img0[:, :3], img1[:, :3], timestep), 1), None, scale=scale_list[i]) - if ensemble: - f1, m1 = block[i](torch.cat((img1[:, :3], img0[:, :3], 1-timestep), 1), None, scale=scale_list[i]) - flow = (flow + torch.cat((f1[:, 2:4], f1[:, :2]), 1)) / 2 - mask = (mask + (-m1)) / 2 + flow, mask, feat = block[i]( + torch.cat((img0, img1, f0, f1, timestep), 1), None, scale=self.scale_list[i] + ) else: - f0, m0 = block[i](torch.cat((warped_img0[:, :3], warped_img1[:, :3], timestep, mask), 1), flow, scale=scale_list[i]) - if ensemble: - f1, m1 = block[i](torch.cat((warped_img1[:, :3], warped_img0[:, :3], 1-timestep, -mask), 1), torch.cat((flow[:, 2:4], flow[:, :2]), 1), scale=scale_list[i]) # pylint: disable=invalid-unary-operand-type - f0 = (f0 + torch.cat((f1[:, 2:4], f1[:, :2]), 1)) / 2 - m0 = (m0 + (-m1)) / 2 - flow = flow + f0 - mask = mask + m0 + wf0 = warp(f0, flow[:, :2], tenFlow_div, backwarp_tenGrid) + wf1 = warp(f1, flow[:, 2:4], tenFlow_div, backwarp_tenGrid) + fd, m0, feat = block[i]( + torch.cat((warped_img0, warped_img1, wf0, wf1, timestep, mask, feat), 1), + flow, + scale=self.scale_list[i], + ) + mask = m0 + flow = flow + fd mask_list.append(mask) flow_list.append(flow) - warped_img0 = warp(img0, flow[:, :2]) - warped_img1 = warp(img1, flow[:, 2:4]) + warped_img0 = warp(img0, flow[:, :2], tenFlow_div, backwarp_tenGrid) + warped_img1 = warp(img1, flow[:, 2:4], tenFlow_div, backwarp_tenGrid) merged.append((warped_img0, warped_img1)) - mask_list[3] = torch.sigmoid(mask_list[3]) - merged[3] = merged[3][0] * mask_list[3] + merged[3][1] * (1 - mask_list[3]) - return flow_list, mask_list[3], merged + mask = torch.sigmoid(mask) + return warped_img0 * mask + warped_img1 * (1 - mask) diff --git a/modules/rife/model_rife.py b/modules/rife/model_rife.py index 52d359923..ee3aa1ecf 100644 --- a/modules/rife/model_rife.py +++ b/modules/rife/model_rife.py @@ -1,8 +1,5 @@ import torch -from torch.optim import AdamW -from torch.nn.parallel import DistributedDataParallel as DDP from modules.rife.model_ifnet import IFNet -from modules.rife.loss import EPE, SOBEL from modules import devices @@ -10,12 +7,11 @@ class RifeModel: def __init__(self, local_rank=-1): self.flownet = IFNet() self.device() - self.optimG = AdamW(self.flownet.parameters(), lr=1e-6, weight_decay=1e-4) - self.epe = EPE() - self.version = 3.9 - # self.vgg = VGGPerceptualLoss().to(device) - self.sobel = SOBEL() + self.version = 4.25 + self.tenFlow_div_cache = {} + self.backwarp_tenGrid_cache = {} if local_rank != -1: + from torch.nn.parallel import DistributedDataParallel as DDP self.flownet = DDP(self.flownet, device_ids=[local_rank], output_device=local_rank) def train(self): @@ -26,7 +22,7 @@ class RifeModel: def device(self): self.flownet.to(devices.device) - self.flownet.to(devices.dtype) + self.flownet.to(torch.float32) # bfloat16 produces visible checkerboard artifacts at the new IFNet's depth def load_model(self, model_file, rank=0): def convert(param): @@ -44,36 +40,30 @@ class RifeModel: if rank == 0: torch.save(self.flownet.state_dict(), model_file) - def inference(self, img0, img1, timestep=0.5, scale=1.0): - imgs = torch.cat((img0, img1), 1) - scale_list = [8/scale, 4/scale, 2/scale, 1/scale] - _flow, _mask, merged = self.flownet(imgs, timestep, scale_list) - return merged[3] + def grid_for(self, h, w, device, dtype): + key = (h, w, str(device), str(dtype)) + grid = self.backwarp_tenGrid_cache.get(key) + if grid is None: + tenHorizontal = torch.linspace(-1.0, 1.0, w, device=device, dtype=dtype).view(1, 1, 1, w).expand(1, -1, h, -1) + tenVertical = torch.linspace(-1.0, 1.0, h, device=device, dtype=dtype).view(1, 1, h, 1).expand(1, -1, -1, w) + grid = torch.cat([tenHorizontal, tenVertical], 1) + self.backwarp_tenGrid_cache[key] = grid + div = self.tenFlow_div_cache.get(key) + if div is None: + div = torch.tensor([(w - 1.0) / 2.0, (h - 1.0) / 2.0], device=device, dtype=dtype) + self.tenFlow_div_cache[key] = div + return grid, div - def update(self, imgs, gt, learning_rate=0, mul=1, training=True, flow_gt=None): # pylint: disable=unused-argument - for param_group in self.optimG.param_groups: - param_group['lr'] = learning_rate - # img0 = imgs[:, :3] - # img1 = imgs[:, 3:] - if training: - self.train() - else: - self.eval() - scale = [8, 4, 2, 1] - flow, mask, merged = self.flownet(torch.cat((imgs, gt), 1), scale=scale, training=training) - loss_l1 = (merged[3] - gt).abs().mean() - loss_smooth = self.sobel(flow[3], flow[3]*0).mean() - # loss_vgg = self.vgg(merged[2], gt) - if training: - self.optimG.zero_grad() - loss_G = loss_l1 + loss_smooth * 0.1 - loss_G.backward() - self.optimG.step() - # else: - # flow_teacher = flow[2] - return merged[3], { - 'mask': mask, - 'flow': flow[3][:, :2], - 'loss_l1': loss_l1, - 'loss_smooth': loss_smooth, - } + def inference(self, img0, img1, timestep=0.5, scale=1.0): + in_dtype = img0.dtype + img0 = img0.float() + img1 = img1.float() + n, _c, h, w = img0.shape + device = img0.device + backwarp_tenGrid, tenFlow_div = self.grid_for(h, w, device, torch.float32) + self.flownet.scale_list = [16 / scale, 8 / scale, 4 / scale, 2 / scale, 1 / scale] + f0 = self.flownet.encode(img0) + f1 = self.flownet.encode(img1) + timestep_t = torch.full((n, 1, h, w), timestep, device=device, dtype=torch.float32) + out = self.flownet(img0, img1, timestep_t, tenFlow_div, backwarp_tenGrid, f0, f1) + return out.to(in_dtype) diff --git a/modules/rife/warplayer.py b/modules/rife/warplayer.py index abcd6aa5c..a1427e5ba 100644 --- a/modules/rife/warplayer.py +++ b/modules/rife/warplayer.py @@ -1,17 +1,12 @@ import torch -from modules import devices +import torch.nn.functional as F -backwarp_tenGrid = {} +def warp(tenInput, tenFlow, tenFlow_div, backwarp_tenGrid): + dtype = tenInput.dtype + tenInput = tenInput.to(torch.float) + tenFlow = tenFlow.to(torch.float) - -def warp(tenInput, tenFlow): - k = (str(tenFlow.device), str(tenFlow.size())) - if k not in backwarp_tenGrid: - tenHorizontal = torch.linspace(-1.0, 1.0, tenFlow.shape[3], device=devices.device).view(1, 1, 1, tenFlow.shape[3]).expand(tenFlow.shape[0], -1, tenFlow.shape[2], -1) - tenVertical = torch.linspace(-1.0, 1.0, tenFlow.shape[2], device=devices.device).view(1, 1, tenFlow.shape[2], 1).expand(tenFlow.shape[0], -1, -1, tenFlow.shape[3]) - backwarp_tenGrid[k] = torch.cat([tenHorizontal, tenVertical], 1).to(devices.device) - tenFlow = torch.cat([tenFlow[:, 0:1, :, :] / ((tenInput.shape[3] - 1.0) / 2.0), - tenFlow[:, 1:2, :, :] / ((tenInput.shape[2] - 1.0) / 2.0)], 1) - grid = (backwarp_tenGrid[k] + tenFlow).permute(0, 2, 3, 1).to(devices.dtype) - return torch.nn.functional.grid_sample(input=tenInput, grid=grid, mode='bilinear', padding_mode='border', align_corners=True) + tenFlow = torch.cat([tenFlow[:, 0:1] / tenFlow_div[0], tenFlow[:, 1:2] / tenFlow_div[1]], 1) + g = (backwarp_tenGrid + tenFlow).permute(0, 2, 3, 1) + return F.grid_sample(input=tenInput, grid=g, mode="bilinear", padding_mode="border", align_corners=True).to(dtype) From 121b357ba0b01a18eeac959d5ed602bc8a7f9283 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sat, 25 Apr 2026 21:14:36 +0100 Subject: [PATCH 2/8] fix(video): normalize pixel range to [0,1] before RIFE video_save passes [-1,1]-range pixels to rife.interpolate_nchw, but RIFE v4.25's IFNet explicitly clamps inputs to [0,1] (Head and IFNet forward pass), turning every negative pixel value into zero. v3.9 silently extrapolated and produced soft artifacts; v4.25 produces washout. Convert to [0,1] before the RIFE call and back to [-1,1] for downstream save. --- modules/video_models/video_save.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/video_models/video_save.py b/modules/video_models/video_save.py index e75276935..8d194356a 100644 --- a/modules/video_models/video_save.py +++ b/modules/video_models/video_save.py @@ -273,9 +273,11 @@ def save_video( stream.output_queue.push(('progress', (None, 'Saving video...'))) if mp4_interpolate > 0: x = pixels.squeeze(0).permute(1, 0, 2, 3) + x = (x.clamp(-1., 1.) + 1.0) * 0.5 # RIFE expects [0, 1]; video pixels are [-1, 1] interpolated = rife.interpolate_nchw(x, count=mp4_interpolate+1) pixels = torch.stack(interpolated, dim=0) pixels = pixels.permute(1, 2, 0, 3, 4) + pixels = pixels * 2.0 - 1.0 # back to [-1, 1] for downstream save n, _c, t, h, w = pixels.shape x = torch.clamp(pixels.float(), -1., 1.) * 127.5 + 127.5 From a2cccddee0dc3f0972f4eb2537de1aaf259f8b80 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sat, 25 Apr 2026 21:15:06 +0100 Subject: [PATCH 3/8] fix(ltx): scale fps to preserve duration with RIFE interpolation LTX conditions the model on mp4_fps as the source frame rate, then RIFE inflates frame count at save. Without compensation, save fps stays equal to source fps and the video becomes (mp4_interpolate+1)x slow-motion (e.g. 121 frames at 24 fps with mp4_interpolate=1 saved as 242 frames at 24 fps = 10s instead of the intended 5s). Scale the saved fps by the interpolation factor at the LTX call site so output duration matches user intent. FramePack already pre-divides mp4_fps at generation time (compute- saving semantic baked into UI math), so it does not need this fix. The pipelines now produce duration-correct output via two different mechanisms; standardization is tracked for the RIFE-in-processing follow-up. --- modules/ltx/ltx_process.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 6717b63ab..1ec2ff53a 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -563,11 +563,14 @@ def run_ltx(task_id, except Exception: aac_sample_rate = 24000 + # LTX conditions the model on mp4_fps as the source frame rate; RIFE inflates frames at save time. + # Scale the saved fps by the interpolation factor so the output preserves the user's intended duration. + save_fps = mp4_fps * (mp4_interpolate + 1) if mp4_interpolate > 0 else mp4_fps num_frames, video_file, _thumb = save_video( p=p, pixels=pixels, audio=audio, - mp4_fps=mp4_fps, + mp4_fps=save_fps, mp4_codec=mp4_codec, mp4_opt=mp4_opt, mp4_ext=mp4_ext, From e42f1fd8fb9a131d7c15951530b4f97423bf6c02 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 26 Apr 2026 03:08:07 +0100 Subject: [PATCH 4/8] feat(video): video_interpolate as execution-time stage Promote RIFE interpolation from a save-time kwarg to a real stage of the processing pipeline so per-frame work (detailer, color correction, postprocess scripts) operates on source-rate frames and the inflated stream becomes the saved output. - new modules/processing_video.py with apply_video_interpolation, interpolation_factor, expand_infotexts; PIL/tensor/numpy dispatch - video_interpolate, video_interpolate_scale, video_interpolated fields on StableDiffusionProcessingVideo - process_images_inner runs the helper after the batch loop and inflates infotexts in lockstep - save_video in modules/video.py and modules/video_models/video_save.py short-circuit re-interpolation when p.video_interpolated is set; the user-facing kwarg still flows into metadata --- modules/processing.py | 8 ++ modules/processing_class.py | 3 + modules/processing_video.py | 125 +++++++++++++++++++++++++++++ modules/video.py | 2 + modules/video_models/video_save.py | 4 +- 5 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 modules/processing_video.py diff --git a/modules/processing.py b/modules/processing.py index d8e43466f..0306b20a8 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -536,6 +536,14 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if shared.state.interrupted: break + if getattr(p, 'video_interpolate', 0) > 0 and len(output_images) > 1: + from modules.processing_video import apply_video_interpolation, expand_infotexts + n_before = len(output_images) + output_images = apply_video_interpolation(p, output_images) + n_after = len(output_images) + if n_after > n_before and len(infotexts) == n_before: + infotexts = expand_infotexts(infotexts, max(0, p.video_interpolate - 1)) + if not p.xyz: if hasattr(shared.sd_model, 'restore_pipeline') and (shared.sd_model.restore_pipeline is not None): shared.sd_model.restore_pipeline() diff --git a/modules/processing_class.py b/modules/processing_class.py index 299f9f9b4..9e1ed3e4f 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -691,6 +691,9 @@ class StableDiffusionProcessingVideo(StableDiffusionProcessing): self.vae_tile_frames: int = kwargs.pop('vae_tile_frames', 0) self.video_engine: str = kwargs.pop('video_engine', None) self.video_model: str = kwargs.pop('video_model', None) + self.video_interpolate: int = kwargs.pop('video_interpolate', 0) + self.video_interpolate_scale: float = kwargs.pop('video_interpolate_scale', 1.0) + self.video_interpolated: bool = False self.scheduler_shift: float = 0.0 debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access super().__init__(**kwargs) diff --git a/modules/processing_video.py b/modules/processing_video.py new file mode 100644 index 000000000..c3138f368 --- /dev/null +++ b/modules/processing_video.py @@ -0,0 +1,125 @@ +""" +Video frame interpolation helper. + +Used by: +- modules.processing.process_images_inner (after process_samples) +- modules.framepack.framepack_worker (before final save_video) +- modules.ltx.ltx_process (before save_video) +- modules.video_models.video_run (before save_video) + +Resolves count and scale from explicit kwargs first, then from +StableDiffusionProcessingVideo.video_interpolate on `p`. Marks the +processing object so save_video can skip its own interpolation pass. + +Forwards count straight to the PIL primitive and count+1 to the tensor +primitive to match the legacy interpolate_frames and video_save.py call +shapes. +""" +from typing import Any +import numpy as np +import torch +from PIL import Image +from modules.logger import log + + +def frames_len(frames: Any): + if frames is None: + return None + if isinstance(frames, list): + return len(frames) + try: + return frames.shape[0] + except Exception: + return None + + +def apply_video_interpolation( + p: Any = None, + frames: Any = None, + count: int = 0, + scale: float = 0.0, + pad: int = 1, + change: float = 0.3, +): + """Inflate a frame stream by RIFE interpolation. + + Dispatches by frames type: + list[PIL.Image] -> rife.interpolate + 4-D torch.Tensor (N,C,H,W) -> rife.interpolate_nchw + np.ndarray (N,H,W,C) -> rife.interpolate_nchw via tensor convert + Sets p.video_interpolated = True after a successful run. + """ + if frames is None: + return frames + if count <= 0: + count = int(getattr(p, 'video_interpolate', 0) or 0) + if count <= 0: + return frames + if scale <= 0: + scale = float(getattr(p, 'video_interpolate_scale', 1.0) or 1.0) + if scale <= 0: + scale = 1.0 + + in_len = frames_len(frames) + in_type = 'unknown' + out = frames + try: + from modules import rife + if isinstance(frames, list) and len(frames) > 0 and isinstance(frames[0], Image.Image): + in_type = 'pil' + out = rife.interpolate(frames, count=count, scale=scale, pad=pad, change=change) + elif torch.is_tensor(frames): + in_type = 'tensor' + interpolated = rife.interpolate_nchw(frames, count=count + 1, scale=scale) + out = torch.cat(interpolated, dim=0) if isinstance(interpolated, list) else interpolated + elif isinstance(frames, np.ndarray): + in_type = 'numpy' + t = torch.from_numpy(frames).permute(0, 3, 1, 2).float() / 255.0 + interpolated = rife.interpolate_nchw(t, count=count + 1, scale=scale) + t_out = torch.cat(interpolated, dim=0) if isinstance(interpolated, list) else interpolated + out = (t_out.clamp(0., 1.) * 255.0).byte().permute(0, 2, 3, 1).cpu().numpy() + else: + log.warning(f'Video interpolation: unsupported type={type(frames).__name__}') + return frames + except Exception as e: + from modules import errors + log.error(f'Video interpolation: {e}') + errors.display(e, 'Video interpolation') + return frames + + if p is not None: + try: + p.video_interpolated = True + except Exception: + pass + + log.info(f'Video interpolation: type={in_type} input={in_len} output={frames_len(out)} count={count} scale={scale}') + return out + + +def interpolation_factor(p: Any) -> int: + """Per-source-frame multiplier the helper applied to p, or 1 if it did not run. + + Multiply mp4_fps by this to preserve duration when the helper ran before save. + """ + if p is None or not getattr(p, 'video_interpolated', False): + return 1 + n = int(getattr(p, 'video_interpolate', 0) or 0) + if n <= 0: + return 1 + return n + 1 + + +def expand_infotexts(infotexts: list, count: int) -> list: + """Inflate the per-frame infotext list to match apply_video_interpolation output. + + Each interpolated frame inherits the infotext of the prior source frame. + """ + if not infotexts or count <= 0: + return infotexts + out = [] + for txt in infotexts: + out.append(txt) + for _ in range(count): + out.append(txt) + return out diff --git a/modules/video.py b/modules/video.py index 1713fe20e..7f6cf60bb 100644 --- a/modules/video.py +++ b/modules/video.py @@ -64,6 +64,8 @@ def save_video_atomic(images, filename, video_type: str = 'none', duration: floa def save_video(p, images, filename = None, video_type: str = 'none', duration: float = 2.0, loop: bool = False, interpolate: int = 0, scale: float = 1.0, pad: int = 1, change: float = 0.3, sync: bool = False): if images is None or len(images) < 2 or video_type is None or video_type.lower() == 'none': return None + if interpolate > 0 and getattr(p, 'video_interpolated', False): + interpolate = 0 image = images[0] if p is not None: seed = p.all_seeds[0] if getattr(p, 'all_seeds', None) is not None else p.seed diff --git a/modules/video_models/video_save.py b/modules/video_models/video_save.py index 8d194356a..db01c605b 100644 --- a/modules/video_models/video_save.py +++ b/modules/video_models/video_save.py @@ -271,13 +271,13 @@ def save_video( preparejob = shared.state.begin('Prepare video') if stream is not None: stream.output_queue.push(('progress', (None, 'Saving video...'))) - if mp4_interpolate > 0: + if mp4_interpolate > 0 and not getattr(p, 'video_interpolated', False): x = pixels.squeeze(0).permute(1, 0, 2, 3) x = (x.clamp(-1., 1.) + 1.0) * 0.5 # RIFE expects [0, 1]; video pixels are [-1, 1] interpolated = rife.interpolate_nchw(x, count=mp4_interpolate+1) pixels = torch.stack(interpolated, dim=0) pixels = pixels.permute(1, 2, 0, 3, 4) - pixels = pixels * 2.0 - 1.0 # back to [-1, 1] for downstream save + pixels = pixels * 2.0 - 1.0 n, _c, t, h, w = pixels.shape x = torch.clamp(pixels.float(), -1., 1.) * 127.5 + 127.5 From 6acde69467d97ec8a54294d05f5bff83db58642e Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 26 Apr 2026 03:08:19 +0100 Subject: [PATCH 5/8] feat(video): route video pipelines through interpolation helper LTX, video_run, and framepack_worker bypass process_images_inner, so they call apply_video_interpolation explicitly before save_video. Save receives already-inflated frames; the sentinel guard skips its own pass. - LTX and video_run scale mp4_fps by interpolation_factor(p) so duration is preserved instead of stretched (LTX is conditioned on source fps) - FramePack pre-divides at gen time per get_latent_paddings, so save fps stays at mp4_fps; worker passes p=None so save call uses mp4_interpolate=0 to skip directly - replaces the inline (mp4_interpolate+1) fps math at LTX with the helper-driven equivalent --- modules/framepack/framepack_worker.py | 11 ++++++++++- modules/ltx/ltx_process.py | 15 ++++++++++++--- modules/video_models/video_run.py | 13 ++++++++++++- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/modules/framepack/framepack_worker.py b/modules/framepack/framepack_worker.py index fdd9637c9..09b62b6cb 100644 --- a/modules/framepack/framepack_worker.py +++ b/modules/framepack/framepack_worker.py @@ -322,6 +322,15 @@ def worker( if is_last_section: break + if mp4_interpolate > 0: + from modules.processing_video import apply_video_interpolation + # history_pixels is 5-D (N,C,T,H,W) in [-1,1]; RIFE needs 4-D (T,C,H,W) in [0,1] + x = history_pixels.squeeze(0).permute(1, 0, 2, 3) + x = (x.clamp(-1., 1.) + 1.0) * 0.5 + x = apply_video_interpolation(None, x, count=mp4_interpolate) + x = x * 2.0 - 1.0 + history_pixels = x.permute(1, 0, 2, 3).unsqueeze(0) + total_generated_frames, _video_filename, _thumb = save_video( p=None, pixels=history_pixels, @@ -334,7 +343,7 @@ def worker( mp4_sf=mp4_sf, mp4_video=mp4_video, mp4_frames=mp4_frames, - mp4_interpolate=mp4_interpolate, + mp4_interpolate=0, pbar=pbar, stream=stream, metadata=metadata, diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 1ec2ff53a..778b0452b 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -563,9 +563,18 @@ def run_ltx(task_id, except Exception: aac_sample_rate = 24000 - # LTX conditions the model on mp4_fps as the source frame rate; RIFE inflates frames at save time. - # Scale the saved fps by the interpolation factor so the output preserves the user's intended duration. - save_fps = mp4_fps * (mp4_interpolate + 1) if mp4_interpolate > 0 else mp4_fps + if mp4_interpolate > 0: + p.video_interpolate = mp4_interpolate + from modules.processing_video import apply_video_interpolation + # pixels is 5-D (N,C,T,H,W); RIFE needs 4-D (T,C,H,W) in [0,1] + x = pixels.squeeze(0).permute(1, 0, 2, 3) + x = (x.clamp(-1., 1.) + 1.0) * 0.5 + x = apply_video_interpolation(p, x, count=mp4_interpolate) + x = x * 2.0 - 1.0 + pixels = x.permute(1, 0, 2, 3).unsqueeze(0) + # LTX is conditioned on mp4_fps as the source rate; scale saved fps to keep duration constant + from modules.processing_video import interpolation_factor + save_fps = mp4_fps * interpolation_factor(p) num_frames, video_file, _thumb = save_video( p=p, pixels=pixels, diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 3454faf69..60acf1625 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -160,12 +160,23 @@ def generate(*args, **kwargs): else: audio = None + if mp4_interpolate > 0 and pixels is not None: + p.video_interpolate = mp4_interpolate + from modules.processing_video import apply_video_interpolation + # pixels is 5-D (N,C,T,H,W) in [-1,1]; RIFE needs 4-D (T,C,H,W) in [0,1] + x = pixels.squeeze(0).permute(1, 0, 2, 3) + x = (x.clamp(-1., 1.) + 1.0) * 0.5 + x = apply_video_interpolation(p, x, count=mp4_interpolate) + x = x * 2.0 - 1.0 + pixels = x.permute(1, 0, 2, 3).unsqueeze(0) + from modules.processing_video import interpolation_factor + save_fps = mp4_fps * interpolation_factor(p) _num_frames, video_file, _thumb = video_save.save_video( p=p, pixels=pixels, audio=audio, binary=processed.bytes, - mp4_fps=mp4_fps, + mp4_fps=save_fps, mp4_codec=mp4_codec, mp4_opt=mp4_opt, mp4_ext=mp4_ext, From 7d77b01c3d885b8d8f420e1277029bc00fc532da Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 26 Apr 2026 03:09:27 +0100 Subject: [PATCH 6/8] feat(video): wire scripts and control to video_interpolate field Set p.video_interpolate at run() entry so process_images_inner picks up the helper. Save calls keep their existing kwargs; the sentinel guard skips re-interpolation when the helper already ran. - animatediff, text2video, image2video, stablevideodiffusion route mp4_interpolate into p.video_interpolate - modules/control/run.py routes the request video_interpolate arg - xyz_grid intentionally untouched: its end-of-axis save_video stitches the cell slideshow, not per-cell videos; a future video_interpolate axis would set p.video_interpolate per cell via axis_options --- modules/control/run.py | 1 + scripts/animatediff.py | 1 + scripts/image2video.py | 1 + scripts/stablevideodiffusion.py | 1 + scripts/text2video.py | 1 + 5 files changed, 5 insertions(+) diff --git a/modules/control/run.py b/modules/control/run.py index 9b6befe0e..90baacb71 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -537,6 +537,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg p.is_tile = False p.init_control = inits or [] p.orig_init_images = inputs + p.video_interpolate = video_interpolate # TODO modernui: monkey-patch for missing tabs.select event if p.selected_scale_tab_before == 0 and p.resize_name_before != 'None' and p.scale_by_before != 1 and inputs is not None and len(inputs) > 0: diff --git a/scripts/animatediff.py b/scripts/animatediff.py index 12df8c3e3..1800a7218 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -248,6 +248,7 @@ class AnimateDiffScript(scripts_manager.Script): p.extra_generation_params['AnimateDiff'] = loaded_adapter p.do_not_save_grid = True p.ops.append('video') + p.video_interpolate = mp4_interpolate p.task_args['generator'] = None p.task_args['num_frames'] = frames p.task_args['num_inference_steps'] = p.steps diff --git a/scripts/image2video.py b/scripts/image2video.py index 2ec56cacf..4041db5f8 100644 --- a/scripts/image2video.py +++ b/scripts/image2video.py @@ -57,6 +57,7 @@ class VGenI2VScript(scripts_manager.Script): return None if p.init_images is None or len(p.init_images) == 0: return None + p.video_interpolate = mp4_interpolate model = [m for m in MODELS if m['name'] == model_name][0] repo_id = model['url'] log.debug(f'Image2Video: model={model_name} frames={num_frames}, video={video_type} duration={duration} loop={gif_loop} pad={mp4_pad} interpolate={mp4_interpolate}') diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index 5da5b50b1..3fa2a5cc2 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -69,6 +69,7 @@ class SVDScript(scripts_manager.Script): return frames def run(self, p: processing.StableDiffusionProcessing, model, num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + p.video_interpolate = mp4_interpolate image = getattr(p, 'init_images', None) if image is None or len(image) == 0: log.error('SVD: no init_images') diff --git a/scripts/text2video.py b/scripts/text2video.py index 8cea90dba..eaa737ac0 100644 --- a/scripts/text2video.py +++ b/scripts/text2video.py @@ -54,6 +54,7 @@ class ModelScopeScript(scripts_manager.Script): def run(self, p: processing.StableDiffusionProcessing, model_name, use_default, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument if model_name == 'None': return None + p.video_interpolate = mp4_interpolate model = [m for m in MODELS if m['name'] == model_name][0] log.debug(f'Text2Video: model={model} defaults={use_default} frames={num_frames}, video={video_type} duration={duration} loop={gif_loop} pad={mp4_pad} interpolate={mp4_interpolate}') From b48bf5236b6534eb8d15600ad4a7a4e17fab9a6b Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Mon, 27 Apr 2026 01:13:47 +0100 Subject: [PATCH 7/8] fix(ltx): handle PIL list output from refine path in interpolation wiring LTX refine pipe uses output_type='pil' so result.frames[0] returns a list, not a 5-D tensor. Convert via images_to_tensor before the helper sees it, mirroring what save_video already does for the same input. --- modules/ltx/ltx_process.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 778b0452b..28c58b2dd 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -563,10 +563,14 @@ def run_ltx(task_id, except Exception: aac_sample_rate = 24000 - if mp4_interpolate > 0: + if mp4_interpolate > 0 and pixels is not None: p.video_interpolate = mp4_interpolate from modules.processing_video import apply_video_interpolation - # pixels is 5-D (N,C,T,H,W); RIFE needs 4-D (T,C,H,W) in [0,1] + # refine path returns PIL list (output_type='pil'); decode path returns 5-D tensor + if isinstance(pixels, list) and len(pixels) > 0 and isinstance(pixels[0], Image.Image): + from modules.video_models.video_save import images_to_tensor + pixels = images_to_tensor(pixels) + # pixels is 5-D (N,C,T,H,W) in [-1,1]; RIFE needs 4-D (T,C,H,W) in [0,1] x = pixels.squeeze(0).permute(1, 0, 2, 3) x = (x.clamp(-1., 1.) + 1.0) * 0.5 x = apply_video_interpolation(p, x, count=mp4_interpolate) From ba4436916d249cc3dbdd85a8b74df68468320e3f Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Mon, 27 Apr 2026 01:32:30 +0100 Subject: [PATCH 8/8] fix(control): prevent duplicate save_video and unmask video module shadow When a video script (animatediff, text2video, image2video, stablevideodiffusion) runs via Control tab, both the script's save and control_run's end-of-run save fired. The latter crashed silently because the local `video` cv2 capture name at control_run:584 shadowed the modules.video import, so the duplicate was hidden and the gallery video link never propagated. - alias import as video_module to bypass the shadow - p.video_saved marker set by each script - control_run skips its end-of-run save when the marker is set --- modules/control/run.py | 5 +++-- scripts/animatediff.py | 1 + scripts/image2video.py | 1 + scripts/stablevideodiffusion.py | 1 + scripts/text2video.py | 1 + 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/control/run.py b/modules/control/run.py index 90baacb71..c201367f5 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -19,6 +19,7 @@ from modules.ui_common import infotext_to_html from modules.api import script from modules.generation_parameters_copypaste import create_override_settings_dict from modules.paths import resolve_output_path +from modules import video as video_module # alias avoids shadow by local `video` cv2 capture name at control_run:584 debug = os.environ.get('SD_CONTROL_DEBUG', None) is not None @@ -795,9 +796,9 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg image_txt = '' p.init_images = output_images # may be used for hires - if video_type != 'None' and isinstance(output_images, list) and 'video' in p.ops: + if video_type != 'None' and isinstance(output_images, list) and 'video' in p.ops and not getattr(p, 'video_saved', False): p.do_not_save_grid = True # pylint: disable=attribute-defined-outside-init - output_filename = video.save_video(p, filename=None, images=output_images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate, sync=True) + output_filename = video_module.save_video(p, filename=None, images=output_images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate, sync=True) if shared.opts.gradio_skip_video: output_filename = '' image_txt = f'| Frames {len(output_images)} | Size {output_images[0].width}x{output_images[0].height}' diff --git a/scripts/animatediff.py b/scripts/animatediff.py index 1800a7218..bb7570488 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -268,3 +268,4 @@ class AnimateDiffScript(scripts_manager.Script): if video_type != 'None': log.debug(f'AnimateDiff video: type={video_type} duration={duration} loop={gif_loop} pad={mp4_pad} interpolate={mp4_interpolate}') save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) + p.video_saved = True diff --git a/scripts/image2video.py b/scripts/image2video.py index 4041db5f8..05f82a037 100644 --- a/scripts/image2video.py +++ b/scripts/image2video.py @@ -112,4 +112,5 @@ class VGenI2VScript(scripts_manager.Script): shared.sd_model = orig_pipeline if video_type != 'None' and processed is not None: video.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) + p.video_saved = True return processed diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index 3fa2a5cc2..6e023dd46 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -123,4 +123,5 @@ class SVDScript(scripts_manager.Script): processed = processing.process_images(p) if video_type != 'None': video.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) + p.video_saved = True return processed diff --git a/scripts/text2video.py b/scripts/text2video.py index eaa737ac0..a3726f63d 100644 --- a/scripts/text2video.py +++ b/scripts/text2video.py @@ -91,4 +91,5 @@ class ModelScopeScript(scripts_manager.Script): if video_type != 'None': video.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) + p.video_saved = True return processed