Merge pull request #4798 from vladmandic/feat/rife-upgrade

Feat/rife upgrade

modules/control/run.py

@@ -19,6 +19,7 @@ from modules.ui_common import infotext_to_html
 from modules.api import script
 from modules.generation_parameters_copypaste import create_override_settings_dict
 from modules.paths import resolve_output_path
+from modules import video as video_module  # alias avoids shadow by local `video` cv2 capture name at control_run:584
 
 
 debug = os.environ.get('SD_CONTROL_DEBUG', None) is not None
@@ -537,6 +538,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
     p.is_tile = False
     p.init_control = inits or []
     p.orig_init_images = inputs
+    p.video_interpolate = video_interpolate
 
     # TODO modernui: monkey-patch for missing tabs.select event
     if p.selected_scale_tab_before == 0 and p.resize_name_before != 'None' and p.scale_by_before != 1 and inputs is not None and len(inputs) > 0:
@@ -794,9 +796,9 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
         image_txt = ''
         p.init_images = output_images # may be used for hires
 
-    if video_type != 'None' and isinstance(output_images, list) and 'video' in p.ops:
+    if video_type != 'None' and isinstance(output_images, list) and 'video' in p.ops and not getattr(p, 'video_saved', False):
         p.do_not_save_grid = True # pylint: disable=attribute-defined-outside-init
-        output_filename = video.save_video(p, filename=None, images=output_images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate, sync=True)
+        output_filename = video_module.save_video(p, filename=None, images=output_images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate, sync=True)
         if shared.opts.gradio_skip_video:
             output_filename = ''
         image_txt = f'| Frames {len(output_images)} | Size {output_images[0].width}x{output_images[0].height}'

modules/framepack/framepack_worker.py

@@ -322,6 +322,15 @@ def worker(
                 if is_last_section:
                     break
 
+            if mp4_interpolate > 0:
+                from modules.processing_video import apply_video_interpolation
+                # history_pixels is 5-D (N,C,T,H,W) in [-1,1]; RIFE needs 4-D (T,C,H,W) in [0,1]
+                x = history_pixels.squeeze(0).permute(1, 0, 2, 3)
+                x = (x.clamp(-1., 1.) + 1.0) * 0.5
+                x = apply_video_interpolation(None, x, count=mp4_interpolate)
+                x = x * 2.0 - 1.0
+                history_pixels = x.permute(1, 0, 2, 3).unsqueeze(0)
+
             total_generated_frames, _video_filename, _thumb = save_video(
                 p=None,
                 pixels=history_pixels,
@@ -334,7 +343,7 @@ def worker(
                 mp4_sf=mp4_sf,
                 mp4_video=mp4_video,
                 mp4_frames=mp4_frames,
-                mp4_interpolate=mp4_interpolate,
+                mp4_interpolate=0,
                 pbar=pbar,
                 stream=stream,
                 metadata=metadata,

modules/ltx/ltx_process.py

@@ -563,11 +563,27 @@ def run_ltx(task_id,
             except Exception:
                 aac_sample_rate = 24000
 
+            if mp4_interpolate > 0 and pixels is not None:
+                p.video_interpolate = mp4_interpolate
+                from modules.processing_video import apply_video_interpolation
+                # refine path returns PIL list (output_type='pil'); decode path returns 5-D tensor
+                if isinstance(pixels, list) and len(pixels) > 0 and isinstance(pixels[0], Image.Image):
+                    from modules.video_models.video_save import images_to_tensor
+                    pixels = images_to_tensor(pixels)
+                # pixels is 5-D (N,C,T,H,W) in [-1,1]; RIFE needs 4-D (T,C,H,W) in [0,1]
+                x = pixels.squeeze(0).permute(1, 0, 2, 3)
+                x = (x.clamp(-1., 1.) + 1.0) * 0.5
+                x = apply_video_interpolation(p, x, count=mp4_interpolate)
+                x = x * 2.0 - 1.0
+                pixels = x.permute(1, 0, 2, 3).unsqueeze(0)
+            # LTX is conditioned on mp4_fps as the source rate; scale saved fps to keep duration constant
+            from modules.processing_video import interpolation_factor
+            save_fps = mp4_fps * interpolation_factor(p)
             num_frames, video_file, _thumb = save_video(
                 p=p,
                 pixels=pixels,
                 audio=audio,
-                mp4_fps=mp4_fps,
+                mp4_fps=save_fps,
                 mp4_codec=mp4_codec,
                 mp4_opt=mp4_opt,
                 mp4_ext=mp4_ext,

modules/processing.py

@@ -536,6 +536,14 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
             if shared.state.interrupted:
                 break
 
+        if getattr(p, 'video_interpolate', 0) > 0 and len(output_images) > 1:
+            from modules.processing_video import apply_video_interpolation, expand_infotexts
+            n_before = len(output_images)
+            output_images = apply_video_interpolation(p, output_images)
+            n_after = len(output_images)
+            if n_after > n_before and len(infotexts) == n_before:
+                infotexts = expand_infotexts(infotexts, max(0, p.video_interpolate - 1))
+
         if not p.xyz:
             if hasattr(shared.sd_model, 'restore_pipeline') and (shared.sd_model.restore_pipeline is not None):
                 shared.sd_model.restore_pipeline()

modules/processing_class.py

@@ -691,6 +691,9 @@ class StableDiffusionProcessingVideo(StableDiffusionProcessing):
         self.vae_tile_frames: int = kwargs.pop('vae_tile_frames', 0)
         self.video_engine: str = kwargs.pop('video_engine', None)
         self.video_model: str = kwargs.pop('video_model', None)
+        self.video_interpolate: int = kwargs.pop('video_interpolate', 0)
+        self.video_interpolate_scale: float = kwargs.pop('video_interpolate_scale', 1.0)
+        self.video_interpolated: bool = False
         self.scheduler_shift: float = 0.0
         debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access
         super().__init__(**kwargs)

modules/processing_video.py

@@ -0,0 +1,125 @@
+"""
+Video frame interpolation helper.
+
+Used by:
+- modules.processing.process_images_inner (after process_samples)
+- modules.framepack.framepack_worker (before final save_video)
+- modules.ltx.ltx_process (before save_video)
+- modules.video_models.video_run (before save_video)
+
+Resolves count and scale from explicit kwargs first, then from
+StableDiffusionProcessingVideo.video_interpolate on `p`. Marks the
+processing object so save_video can skip its own interpolation pass.
+
+Forwards count straight to the PIL primitive and count+1 to the tensor
+primitive to match the legacy interpolate_frames and video_save.py call
+shapes.
+"""
+from typing import Any
+import numpy as np
+import torch
+from PIL import Image
+from modules.logger import log
+
+
+def frames_len(frames: Any):
+    if frames is None:
+        return None
+    if isinstance(frames, list):
+        return len(frames)
+    try:
+        return frames.shape[0]
+    except Exception:
+        return None
+
+
+def apply_video_interpolation(
+    p: Any = None,
+    frames: Any = None,
+    count: int = 0,
+    scale: float = 0.0,
+    pad: int = 1,
+    change: float = 0.3,
+):
+    """Inflate a frame stream by RIFE interpolation.
+
+    Dispatches by frames type:
+      list[PIL.Image]            -> rife.interpolate
+      4-D torch.Tensor (N,C,H,W) -> rife.interpolate_nchw
+      np.ndarray (N,H,W,C)       -> rife.interpolate_nchw via tensor convert
+    Sets p.video_interpolated = True after a successful run.
+    """
+    if frames is None:
+        return frames
+    if count <= 0:
+        count = int(getattr(p, 'video_interpolate', 0) or 0)
+    if count <= 0:
+        return frames
+    if scale <= 0:
+        scale = float(getattr(p, 'video_interpolate_scale', 1.0) or 1.0)
+    if scale <= 0:
+        scale = 1.0
+
+    in_len = frames_len(frames)
+    in_type = 'unknown'
+    out = frames
+    try:
+        from modules import rife
+        if isinstance(frames, list) and len(frames) > 0 and isinstance(frames[0], Image.Image):
+            in_type = 'pil'
+            out = rife.interpolate(frames, count=count, scale=scale, pad=pad, change=change)
+        elif torch.is_tensor(frames):
+            in_type = 'tensor'
+            interpolated = rife.interpolate_nchw(frames, count=count + 1, scale=scale)
+            out = torch.cat(interpolated, dim=0) if isinstance(interpolated, list) else interpolated
+        elif isinstance(frames, np.ndarray):
+            in_type = 'numpy'
+            t = torch.from_numpy(frames).permute(0, 3, 1, 2).float() / 255.0
+            interpolated = rife.interpolate_nchw(t, count=count + 1, scale=scale)
+            t_out = torch.cat(interpolated, dim=0) if isinstance(interpolated, list) else interpolated
+            out = (t_out.clamp(0., 1.) * 255.0).byte().permute(0, 2, 3, 1).cpu().numpy()
+        else:
+            log.warning(f'Video interpolation: unsupported type={type(frames).__name__}')
+            return frames
+    except Exception as e:
+        from modules import errors
+        log.error(f'Video interpolation: {e}')
+        errors.display(e, 'Video interpolation')
+        return frames
+
+    if p is not None:
+        try:
+            p.video_interpolated = True
+        except Exception:
+            pass
+
+    log.info(f'Video interpolation: type={in_type} input={in_len} output={frames_len(out)} count={count} scale={scale}')
+    return out
+
+
+def interpolation_factor(p: Any) -> int:
+    """Per-source-frame multiplier the helper applied to p, or 1 if it did not run.
+
+    Multiply mp4_fps by this to preserve duration when the helper ran before save.
+    """
+    if p is None or not getattr(p, 'video_interpolated', False):
+        return 1
+    n = int(getattr(p, 'video_interpolate', 0) or 0)
+    if n <= 0:
+        return 1
+    return n + 1
+
+
+def expand_infotexts(infotexts: list, count: int) -> list:
+    """Inflate the per-frame infotext list to match apply_video_interpolation output.
+
+    Each interpolated frame inherits the infotext of the prior source frame.
+    """
+    if not infotexts or count <= 0:
+        return infotexts
+    out = []
+    for txt in infotexts:
+        out.append(txt)
+        for _ in range(count):
+            out.append(txt)
+    return out

modules/rife/__init__.py

@@ -16,16 +16,21 @@ from modules import devices, shared, paths
 from modules.logger import log
 
 
-model_url = 'https://github.com/vladmandic/rife/raw/main/model/flownet-v46.pkl'
+# Practical-RIFE v4.25 weights (MIT). Default URL points at the upstream HolyWu mirror;
+# can be swapped to a self-hosted mirror without any other code change.
+model_url = 'https://github.com/HolyWu/vs-rife/releases/download/model/flownet_v4.25.pkl'
 model: RifeModel = None
 
 
-def load(model_path: str = 'rife/flownet-v46.pkl'):
+def load(model_path: str = 'rife/flownet_v4.25.pkl'):
     global model # pylint: disable=global-statement
     if model is None:
         from modules import modelloader
         model_dir = os.path.join(paths.models_path, 'RIFE')
-        model_path = modelloader.load_file_from_url(url=model_url, model_dir=model_dir, file_name='flownet-v46.pkl')
+        model_path = modelloader.load_file_from_url(url=model_url, model_dir=model_dir, file_name='flownet_v4.25.pkl')
+        legacy_path = os.path.join(model_dir, 'flownet-v46.pkl')
+        if os.path.exists(legacy_path):
+            log.info(f'Video interpolate: legacy v3.9 weights at "{legacy_path}" are no longer used and can be deleted')
         log.debug(f'Video interpolate: model="{model_path}"')
         model = RifeModel()
         model.load_model(model_path, -1)
@@ -144,8 +149,8 @@ def interpolate_nchw(images: list, count: int = 2, scale: float = 1.0):
                 I1 = f_pad(frame.unsqueeze(0))
                 for i in range(count-1):
                     output = model.inference(I0, I1, (i+1) * 1. / (count), scale)
-                    interpolated.append(output)
-                interpolated.append(I1)
+                    interpolated.append(output[:, :, :h, :w])
+                interpolated.append(I1[:, :, :h, :w])
                 pbar.update(1)
 
     t1 = time.time()

modules/rife/model_ifnet.py

@@ -1,49 +1,60 @@
-import os
-import sys
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
 
-sys.path.append(os.path.dirname(__file__))
-from warplayer import warp # pylint: disable=wrong-import-position
-
-
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+from modules.rife.warplayer import warp
 
 
 def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
     return nn.Sequential(
-        nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride,
-                  padding=padding, dilation=dilation, bias=True),
-        nn.LeakyReLU(0.2, True)
+        nn.Conv2d(
+            in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=True
+        ),
+        nn.LeakyReLU(0.2, True),
     )
 
-def conv_bn(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
-    return nn.Sequential(
-        nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride,
-                  padding=padding, dilation=dilation, bias=False),
-        nn.BatchNorm2d(out_planes),
-        nn.LeakyReLU(0.2, True)
-    )
+
+class Head(nn.Module):
+    def __init__(self):
+        super(Head, self).__init__()
+        self.cnn0 = nn.Conv2d(3, 16, 3, 2, 1)
+        self.cnn1 = nn.Conv2d(16, 16, 3, 1, 1)
+        self.cnn2 = nn.Conv2d(16, 16, 3, 1, 1)
+        self.cnn3 = nn.ConvTranspose2d(16, 4, 4, 2, 1)
+        self.relu = nn.LeakyReLU(0.2, True)
+
+    def forward(self, x, feat=False):
+        x = x.clamp(0.0, 1.0)
+        x0 = self.cnn0(x)
+        x = self.relu(x0)
+        x1 = self.cnn1(x)
+        x = self.relu(x1)
+        x2 = self.cnn2(x)
+        x = self.relu(x2)
+        x3 = self.cnn3(x)
+        if feat:
+            return [x0, x1, x2, x3]
+        return x3
+
 
 class ResConv(nn.Module):
     def __init__(self, c, dilation=1):
-        super().__init__()
-        self.conv = nn.Conv2d(c, c, 3, 1, dilation, dilation=dilation, groups=1\
-)
+        super(ResConv, self).__init__()
+        self.conv = nn.Conv2d(c, c, 3, 1, dilation, dilation=dilation, groups=1)
         self.beta = nn.Parameter(torch.ones((1, c, 1, 1)), requires_grad=True)
         self.relu = nn.LeakyReLU(0.2, True)
 
     def forward(self, x):
         return self.relu(self.conv(x) * self.beta + x)
 
+
 class IFBlock(nn.Module):
     def __init__(self, in_planes, c=64):
-        super().__init__()
+        super(IFBlock, self).__init__()
         self.conv0 = nn.Sequential(
-            conv(in_planes, c//2, 3, 2, 1),
-            conv(c//2, c, 3, 2, 1),
-            )
+            conv(in_planes, c // 2, 3, 2, 1),
+            conv(c // 2, c, 3, 2, 1),
+        )
         self.convblock = nn.Sequential(
             ResConv(c),
             ResConv(c),
@@ -54,45 +65,39 @@ class IFBlock(nn.Module):
             ResConv(c),
             ResConv(c),
         )
-        self.lastconv = nn.Sequential(
-            nn.ConvTranspose2d(c, 4*6, 4, 2, 1),
-            nn.PixelShuffle(2)
-        )
+        self.lastconv = nn.Sequential(nn.ConvTranspose2d(c, 4 * 13, 4, 2, 1), nn.PixelShuffle(2))
 
     def forward(self, x, flow=None, scale=1):
-        x = F.interpolate(x, scale_factor= 1. / scale, mode="bilinear", align_corners=False)
+        x = F.interpolate(x, scale_factor=1.0 / scale, mode="bilinear")
         if flow is not None:
-            flow = F.interpolate(flow, scale_factor= 1. / scale, mode="bilinear", align_corners=False) * 1. / scale
+            flow = F.interpolate(flow, scale_factor=1.0 / scale, mode="bilinear") / scale
             x = torch.cat((x, flow), 1)
         feat = self.conv0(x)
         feat = self.convblock(feat)
         tmp = self.lastconv(feat)
-        tmp = F.interpolate(tmp, scale_factor=scale, mode="bilinear", align_corners=False)
+        tmp = F.interpolate(tmp, scale_factor=scale, mode="bilinear")
         flow = tmp[:, :4] * scale
         mask = tmp[:, 4:5]
-        return flow, mask
+        feat = tmp[:, 5:]
+        return flow, mask, feat
+
 
 class IFNet(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.block0 = IFBlock(7, c=192)
-        self.block1 = IFBlock(8+4, c=128)
-        self.block2 = IFBlock(8+4, c=96)
-        self.block3 = IFBlock(8+4, c=64)
-        # self.contextnet = Contextnet()
-        # self.unet = Unet()
+    def __init__(self, scale=1, ensemble=False):
+        super(IFNet, self).__init__()
+        self.block0 = IFBlock(7 + 8, c=192)
+        self.block1 = IFBlock(8 + 4 + 8 + 8, c=128)
+        self.block2 = IFBlock(8 + 4 + 8 + 8, c=96)
+        self.block3 = IFBlock(8 + 4 + 8 + 8, c=64)
+        self.block4 = IFBlock(8 + 4 + 8 + 8, c=32)
+        self.encode = Head()
+        self.scale_list = [16 / scale, 8 / scale, 4 / scale, 2 / scale, 1 / scale]
+        if ensemble:
+            raise ValueError("rife: ensemble is not supported in v4.25")
 
-    def forward( self, x, timestep=0.5, scale_list=None, training=False, fastmode=True, ensemble=False): # pylint: disable=dangerous-default-value, unused-argument
-        if scale_list is None:
-            scale_list = [8, 4, 2, 1]
-        if training is False:
-            channel = x.shape[1] // 2
-            img0 = x[:, :channel]
-            img1 = x[:, channel:]
-        if not torch.is_tensor(timestep):
-            timestep = (x[:, :1].clone() * 0 + 1) * timestep
-        else:
-            timestep = timestep.repeat(1, 1, img0.shape[2], img0.shape[3])
+    def forward(self, img0, img1, timestep, tenFlow_div, backwarp_tenGrid, f0, f1):
+        img0 = img0.clamp(0.0, 1.0)
+        img1 = img1.clamp(0.0, 1.0)
         flow_list = []
         merged = []
         mask_list = []
@@ -100,28 +105,26 @@ class IFNet(nn.Module):
         warped_img1 = img1
         flow = None
         mask = None
-        # loss_cons = 0
-        block = [self.block0, self.block1, self.block2, self.block3]
-        for i in range(4):
+        block = [self.block0, self.block1, self.block2, self.block3, self.block4]
+        for i in range(5):
             if flow is None:
-                flow, mask = block[i](torch.cat((img0[:, :3], img1[:, :3], timestep), 1), None, scale=scale_list[i])
-                if ensemble:
-                    f1, m1 = block[i](torch.cat((img1[:, :3], img0[:, :3], 1-timestep), 1), None, scale=scale_list[i])
-                    flow = (flow + torch.cat((f1[:, 2:4], f1[:, :2]), 1)) / 2
-                    mask = (mask + (-m1)) / 2
+                flow, mask, feat = block[i](
+                    torch.cat((img0, img1, f0, f1, timestep), 1), None, scale=self.scale_list[i]
+                )
             else:
-                f0, m0 = block[i](torch.cat((warped_img0[:, :3], warped_img1[:, :3], timestep, mask), 1), flow, scale=scale_list[i])
-                if ensemble:
-                    f1, m1 = block[i](torch.cat((warped_img1[:, :3], warped_img0[:, :3], 1-timestep, -mask), 1), torch.cat((flow[:, 2:4], flow[:, :2]), 1), scale=scale_list[i]) # pylint: disable=invalid-unary-operand-type
-                    f0 = (f0 + torch.cat((f1[:, 2:4], f1[:, :2]), 1)) / 2
-                    m0 = (m0 + (-m1)) / 2
-                flow = flow + f0
-                mask = mask + m0
+                wf0 = warp(f0, flow[:, :2], tenFlow_div, backwarp_tenGrid)
+                wf1 = warp(f1, flow[:, 2:4], tenFlow_div, backwarp_tenGrid)
+                fd, m0, feat = block[i](
+                    torch.cat((warped_img0, warped_img1, wf0, wf1, timestep, mask, feat), 1),
+                    flow,
+                    scale=self.scale_list[i],
+                )
+                mask = m0
+                flow = flow + fd
             mask_list.append(mask)
             flow_list.append(flow)
-            warped_img0 = warp(img0, flow[:, :2])
-            warped_img1 = warp(img1, flow[:, 2:4])
+            warped_img0 = warp(img0, flow[:, :2], tenFlow_div, backwarp_tenGrid)
+            warped_img1 = warp(img1, flow[:, 2:4], tenFlow_div, backwarp_tenGrid)
             merged.append((warped_img0, warped_img1))
-        mask_list[3] = torch.sigmoid(mask_list[3])
-        merged[3] = merged[3][0] * mask_list[3] + merged[3][1] * (1 - mask_list[3])
-        return flow_list, mask_list[3], merged
+        mask = torch.sigmoid(mask)
+        return warped_img0 * mask + warped_img1 * (1 - mask)

modules/rife/model_rife.py

@@ -1,8 +1,5 @@
 import torch
-from torch.optim import AdamW
-from torch.nn.parallel import DistributedDataParallel as DDP
 from modules.rife.model_ifnet import IFNet
-from modules.rife.loss import EPE, SOBEL
 from modules import devices
 
 
@@ -10,12 +7,11 @@ class RifeModel:
     def __init__(self, local_rank=-1):
         self.flownet = IFNet()
         self.device()
-        self.optimG = AdamW(self.flownet.parameters(), lr=1e-6, weight_decay=1e-4)
-        self.epe = EPE()
-        self.version = 3.9
-        # self.vgg = VGGPerceptualLoss().to(device)
-        self.sobel = SOBEL()
+        self.version = 4.25
+        self.tenFlow_div_cache = {}
+        self.backwarp_tenGrid_cache = {}
         if local_rank != -1:
+            from torch.nn.parallel import DistributedDataParallel as DDP
             self.flownet = DDP(self.flownet, device_ids=[local_rank], output_device=local_rank)
 
     def train(self):
@@ -26,7 +22,7 @@ class RifeModel:
 
     def device(self):
         self.flownet.to(devices.device)
-        self.flownet.to(devices.dtype)
+        self.flownet.to(torch.float32)  # bfloat16 produces visible checkerboard artifacts at the new IFNet's depth
 
     def load_model(self, model_file, rank=0):
         def convert(param):
@@ -44,36 +40,30 @@ class RifeModel:
         if rank == 0:
             torch.save(self.flownet.state_dict(), model_file)
 
-    def inference(self, img0, img1, timestep=0.5, scale=1.0):
-        imgs = torch.cat((img0, img1), 1)
-        scale_list = [8/scale, 4/scale, 2/scale, 1/scale]
-        _flow, _mask, merged = self.flownet(imgs, timestep, scale_list)
-        return merged[3]
+    def grid_for(self, h, w, device, dtype):
+        key = (h, w, str(device), str(dtype))
+        grid = self.backwarp_tenGrid_cache.get(key)
+        if grid is None:
+            tenHorizontal = torch.linspace(-1.0, 1.0, w, device=device, dtype=dtype).view(1, 1, 1, w).expand(1, -1, h, -1)
+            tenVertical = torch.linspace(-1.0, 1.0, h, device=device, dtype=dtype).view(1, 1, h, 1).expand(1, -1, -1, w)
+            grid = torch.cat([tenHorizontal, tenVertical], 1)
+            self.backwarp_tenGrid_cache[key] = grid
+        div = self.tenFlow_div_cache.get(key)
+        if div is None:
+            div = torch.tensor([(w - 1.0) / 2.0, (h - 1.0) / 2.0], device=device, dtype=dtype)
+            self.tenFlow_div_cache[key] = div
+        return grid, div
 
-    def update(self, imgs, gt, learning_rate=0, mul=1, training=True, flow_gt=None): # pylint: disable=unused-argument
-        for param_group in self.optimG.param_groups:
-            param_group['lr'] = learning_rate
-        # img0 = imgs[:, :3]
-        # img1 = imgs[:, 3:]
-        if training:
-            self.train()
-        else:
-            self.eval()
-        scale = [8, 4, 2, 1]
-        flow, mask, merged = self.flownet(torch.cat((imgs, gt), 1), scale=scale, training=training)
-        loss_l1 = (merged[3] - gt).abs().mean()
-        loss_smooth = self.sobel(flow[3], flow[3]*0).mean()
-        # loss_vgg = self.vgg(merged[2], gt)
-        if training:
-            self.optimG.zero_grad()
-            loss_G = loss_l1 + loss_smooth * 0.1
-            loss_G.backward()
-            self.optimG.step()
-        # else:
-        #    flow_teacher = flow[2]
-        return merged[3], {
-            'mask': mask,
-            'flow': flow[3][:, :2],
-            'loss_l1': loss_l1,
-            'loss_smooth': loss_smooth,
-        }
+    def inference(self, img0, img1, timestep=0.5, scale=1.0):
+        in_dtype = img0.dtype
+        img0 = img0.float()
+        img1 = img1.float()
+        n, _c, h, w = img0.shape
+        device = img0.device
+        backwarp_tenGrid, tenFlow_div = self.grid_for(h, w, device, torch.float32)
+        self.flownet.scale_list = [16 / scale, 8 / scale, 4 / scale, 2 / scale, 1 / scale]
+        f0 = self.flownet.encode(img0)
+        f1 = self.flownet.encode(img1)
+        timestep_t = torch.full((n, 1, h, w), timestep, device=device, dtype=torch.float32)
+        out = self.flownet(img0, img1, timestep_t, tenFlow_div, backwarp_tenGrid, f0, f1)
+        return out.to(in_dtype)

modules/rife/warplayer.py

@@ -1,17 +1,12 @@
 import torch
-from modules import devices
+import torch.nn.functional as F
 
 
-backwarp_tenGrid = {}
+def warp(tenInput, tenFlow, tenFlow_div, backwarp_tenGrid):
+    dtype = tenInput.dtype
+    tenInput = tenInput.to(torch.float)
+    tenFlow = tenFlow.to(torch.float)
 
-
-def warp(tenInput, tenFlow):
-    k = (str(tenFlow.device), str(tenFlow.size()))
-    if k not in backwarp_tenGrid:
-        tenHorizontal = torch.linspace(-1.0, 1.0, tenFlow.shape[3], device=devices.device).view(1, 1, 1, tenFlow.shape[3]).expand(tenFlow.shape[0], -1, tenFlow.shape[2], -1)
-        tenVertical = torch.linspace(-1.0, 1.0, tenFlow.shape[2], device=devices.device).view(1, 1, tenFlow.shape[2], 1).expand(tenFlow.shape[0], -1, -1, tenFlow.shape[3])
-        backwarp_tenGrid[k] = torch.cat([tenHorizontal, tenVertical], 1).to(devices.device)
-    tenFlow = torch.cat([tenFlow[:, 0:1, :, :] / ((tenInput.shape[3] - 1.0) / 2.0),
-                         tenFlow[:, 1:2, :, :] / ((tenInput.shape[2] - 1.0) / 2.0)], 1)
-    grid = (backwarp_tenGrid[k] + tenFlow).permute(0, 2, 3, 1).to(devices.dtype)
-    return torch.nn.functional.grid_sample(input=tenInput, grid=grid, mode='bilinear', padding_mode='border', align_corners=True)
+    tenFlow = torch.cat([tenFlow[:, 0:1] / tenFlow_div[0], tenFlow[:, 1:2] / tenFlow_div[1]], 1)
+    g = (backwarp_tenGrid + tenFlow).permute(0, 2, 3, 1)
+    return F.grid_sample(input=tenInput, grid=g, mode="bilinear", padding_mode="border", align_corners=True).to(dtype)

modules/video.py

@@ -65,6 +65,8 @@ def save_video_atomic(images, filename, video_type: str = 'none', duration: floa
 def save_video(p, images, filename = None, video_type: str = 'none', duration: float = 2.0, loop: bool = False, interpolate: int = 0, scale: float = 1.0, pad: int = 1, change: float = 0.3, sync: bool = False):
     if images is None or len(images) < 2 or video_type is None or video_type.lower() == 'none':
         return None
+    if interpolate > 0 and getattr(p, 'video_interpolated', False):
+        interpolate = 0
     image = images[0]
     if p is not None:
         seed = p.all_seeds[0] if getattr(p, 'all_seeds', None) is not None else p.seed

modules/video_models/video_run.py

@@ -160,12 +160,23 @@ def generate(*args, **kwargs):
     else:
         audio = None
 
+    if mp4_interpolate > 0 and pixels is not None:
+        p.video_interpolate = mp4_interpolate
+        from modules.processing_video import apply_video_interpolation
+        # pixels is 5-D (N,C,T,H,W) in [-1,1]; RIFE needs 4-D (T,C,H,W) in [0,1]
+        x = pixels.squeeze(0).permute(1, 0, 2, 3)
+        x = (x.clamp(-1., 1.) + 1.0) * 0.5
+        x = apply_video_interpolation(p, x, count=mp4_interpolate)
+        x = x * 2.0 - 1.0
+        pixels = x.permute(1, 0, 2, 3).unsqueeze(0)
+    from modules.processing_video import interpolation_factor
+    save_fps = mp4_fps * interpolation_factor(p)
     _num_frames, video_file, _thumb = video_save.save_video(
         p=p,
         pixels=pixels,
         audio=audio,
         binary=processed.bytes,
-        mp4_fps=mp4_fps,
+        mp4_fps=save_fps,
         mp4_codec=mp4_codec,
         mp4_opt=mp4_opt,
         mp4_ext=mp4_ext,

modules/video_models/video_save.py

@@ -271,11 +271,13 @@ def save_video(
         preparejob = shared.state.begin('Prepare video')
         if stream is not None:
             stream.output_queue.push(('progress', (None, 'Saving video...')))
-        if mp4_interpolate > 0:
+        if mp4_interpolate > 0 and not getattr(p, 'video_interpolated', False):
             x = pixels.squeeze(0).permute(1, 0, 2, 3)
+            x = (x.clamp(-1., 1.) + 1.0) * 0.5  # RIFE expects [0, 1]; video pixels are [-1, 1]
             interpolated = rife.interpolate_nchw(x, count=mp4_interpolate+1)
             pixels = torch.stack(interpolated, dim=0)
             pixels = pixels.permute(1, 2, 0, 3, 4)
+            pixels = pixels * 2.0 - 1.0
 
         n, _c, t, h, w = pixels.shape
         x = torch.clamp(pixels.float(), -1., 1.) * 127.5 + 127.5

scripts/animatediff.py

@@ -248,6 +248,7 @@ class AnimateDiffScript(scripts_manager.Script):
         p.extra_generation_params['AnimateDiff'] = loaded_adapter
         p.do_not_save_grid = True
         p.ops.append('video')
+        p.video_interpolate = mp4_interpolate
         p.task_args['generator'] = None
         p.task_args['num_frames'] = frames
         p.task_args['num_inference_steps'] = p.steps
@@ -267,3 +268,4 @@ class AnimateDiffScript(scripts_manager.Script):
         if video_type != 'None':
             log.debug(f'AnimateDiff video: type={video_type} duration={duration} loop={gif_loop} pad={mp4_pad} interpolate={mp4_interpolate}')
             save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
+            p.video_saved = True

scripts/image2video.py

@@ -57,6 +57,7 @@ class VGenI2VScript(scripts_manager.Script):
             return None
         if p.init_images is None or len(p.init_images) == 0:
             return None
+        p.video_interpolate = mp4_interpolate
         model = [m for m in MODELS if m['name'] == model_name][0]
         repo_id = model['url']
         log.debug(f'Image2Video: model={model_name} frames={num_frames}, video={video_type} duration={duration} loop={gif_loop} pad={mp4_pad} interpolate={mp4_interpolate}')
@@ -111,4 +112,5 @@ class VGenI2VScript(scripts_manager.Script):
         shared.sd_model = orig_pipeline
         if video_type != 'None' and processed is not None:
             video.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
+            p.video_saved = True
         return processed

scripts/stablevideodiffusion.py

@@ -69,6 +69,7 @@ class SVDScript(scripts_manager.Script):
         return frames
 
     def run(self, p: processing.StableDiffusionProcessing, model, num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
+        p.video_interpolate = mp4_interpolate
         image = getattr(p, 'init_images', None)
         if image is None or len(image) == 0:
             log.error('SVD: no init_images')
@@ -122,4 +123,5 @@ class SVDScript(scripts_manager.Script):
         processed = processing.process_images(p)
         if video_type != 'None':
             video.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
+            p.video_saved = True
         return processed

scripts/text2video.py

@@ -54,6 +54,7 @@ class ModelScopeScript(scripts_manager.Script):
     def run(self, p: processing.StableDiffusionProcessing, model_name, use_default, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
         if model_name == 'None':
             return None
+        p.video_interpolate = mp4_interpolate
         model = [m for m in MODELS if m['name'] == model_name][0]
         log.debug(f'Text2Video: model={model} defaults={use_default} frames={num_frames}, video={video_type} duration={duration} loop={gif_loop} pad={mp4_pad} interpolate={mp4_interpolate}')
 
@@ -90,4 +91,5 @@ class ModelScopeScript(scripts_manager.Script):
 
         if video_type != 'None':
             video.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
+            p.video_saved = True
         return processed