improve memory reporting, explicit interpolation and minimax tracking

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log for SD.Next
 
-## Highlights for 2026-08-24
+## Highlights for 2026-08-25
 
 Time for a new release, this is a larger one!
 Main focus is improving video workflows which also brings full support for new [MiniMax H3](https://vladmandic.github.io/sdnext-docs/MiniMax) and [LTXVideo-2.5](https://vladmandic.github.io/sdnext-docs/LTX)  
@@ -17,7 +17,7 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa
 
 [Home](https://vladmandic.github.io/sdnext/) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic)  
 
-## Details for 2026-08-24
+## Details for 2026-08-25
 
 - **Models**
   - [MiniMax H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) available in *base* and *ref* variants  
@@ -50,6 +50,7 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa
   - update handlers for all authenticated workflows
   - update handlers for all hf-based progress bars
   - offload options take effect immediately without restart/reload
+  - improve gpu memory tracking and reporting
   - log long torch autotune operations
   - utilize `torch.accelerator` where available
   - add `SD_DIFFUSERS_DEBUG` and `SD_TRANSFORMERS_DEBUG` env variables to trace diffusers and transformers internal operations  
@@ -120,6 +121,7 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa
   - upscaler: avoid unnecessary multi-pass
   - ideogram4: fix callback
   - xyzgrid: fix seed/prompt/negative handling, thanks @QuintusAntonius
+  - vram: skip reset
 
 ## Update for 2026-08-07
 

modules/ltx/ltx_process.py

@@ -638,7 +638,7 @@ def run(model: str, *,
 
             out_w, out_h = video_utils.pixel_size(pixels, fallback=(p.width, p.height))
             total_time = max(time.time() - t0, 1e-6)
-            log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={num_frames/total_time:.2f} its={p.steps/total_time:.3f} resolution={out_w}x{out_h} time={total_time:.2f} timers={timer.process.dct(no_total=True)} memory={memstats.memory_stats()}')
+            log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={num_frames/total_time:.2f} its={p.steps/total_time:.3f} resolution={out_w}x{out_h} time={total_time:.2f}')
             # the decode paths never materialize PIL, so frames come back through the saved file
             images_out = pixels if isinstance(pixels, list) else []
             processed_out = processing.Processed(p, images_out, seed=p.seed, audio=audio_out)

modules/memstats.py

@@ -101,7 +101,7 @@ def gpu_stats():
         if stats.get('num_ooms', 0) > 0:
             shared.state.oom = True
         gpu['active'] = gb(stats.get('active_bytes.all.current', 0))
-        gpu['peak'] = gb(stats.get('active_bytes.all.peak', 0))
+        gpu['peak'] = gb(stats.get('reserved_bytes.all.peak', 0))
         gpu['retries'] = stats.get('num_alloc_retries', 0)
         gpu['oom'] = stats.get('num_ooms', 0)
     except Exception as e:
@@ -156,6 +156,8 @@ def memory_stats():
 
 
 def reset_stats():
+    # fn = f'{sys._getframe(3).f_code.co_name}:{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
+    # log.trace(f'Memory: reset {fn}')
     try:
         torch.cuda.reset_memory_stats()
     except Exception:

modules/minimax/minimax_video.py

@@ -4,7 +4,7 @@ from PIL import Image
 import numpy as np
 from modules.logger import log
 from modules import shared, devices, errors, processing, timer, progress, paths, sd_models, scripts_manager, call_queue, memstats, processing_video
-from modules.video_models import models_def, video_save, video_utils
+from modules.video_models import models_def, video_save, video_utils, video_upscale
 
 
 engine = 'MiniMax'
@@ -31,7 +31,7 @@ def load_model(model: str):
     sd_models.set_diffuser_options(shared.sd_model) # apply attention, offload, etc.
     loaded = f'repo={selected.repo} workflow={selected.workflow}'
     t1 = time.time()
-    timer.process.add('load', t1 - t0)
+    timer.video.add('load', t1 - t0)
     if shared.sd_model is not None:
         return selected.workflow
     return None
@@ -48,6 +48,7 @@ def unwrap_file(entry):
 
 def prepare_inputs(workflow: str | None, init_image: Image.Image | None, last_image: Image.Image | None, reference_media: list | None) -> dict:
     """The task args a workflow conditions on, resolved before the model load so a rejected request costs nothing."""
+    t_inputs = time.time()
     from modules.minimax import minimax_references
     if minimax_references.get_reference_caps(workflow) is not None:
         entries = [unwrap_file(entry) for entry in (reference_media or [])]
@@ -62,6 +63,7 @@ def prepare_inputs(workflow: str | None, init_image: Image.Image | None, last_im
     if reference_media:
         log.warning(f'Video: op=reference workflow={workflow} references not supported, ignoring: count={len(reference_media)}')
     log.debug(f'Prepare inputs: workflow={workflow} first={init_image} last={last_image}')
+    timer.video.ts('inputs', t_inputs)
     return task_args
 
 
@@ -92,6 +94,7 @@ def generate(task_id, _ui_state,
         progress.start_task(task_id)
         memstats.reset_stats()
         timer.process.reset()
+        timer.video.reset()
 
         # init vars
         p = None
@@ -168,16 +171,26 @@ def generate(task_id, _ui_state,
                 return None, "MiniMax: No frames generated"
 
             if mp4_interpolate > 0:
+                t_interpolate = time.time()
                 p.video_interpolate = mp4_interpolate
                 from modules.processing_video import apply_video_interpolation
                 # pixels is 5-D (N,C,T,H,W) in [-1,1]; RIFE needs 4-D (T,C,H,W) in [0,1]
                 x = pixels.squeeze(0).permute(1, 0, 2, 3)
                 x = (x.clamp(-1., 1.) + 1.0) * 0.5
-                x = apply_video_interpolation(p, x, count=mp4_interpolate) # sets p.video_interpolated otherwise main save_video would do it also
+                x = apply_video_interpolation(p, x, count=mp4_interpolate)
                 x = x * 2.0 - 1.0
                 pixels = x.permute(1, 0, 2, 3).unsqueeze(0)
+                timer.video.ts('interpolate', t_interpolate)
+                p.video_interpolated = True # notice so main save_video does not do it again
+
+            if mp4_upscaler is not None and len(mp4_upscaler) > 0:
+                t_upscale = time.time()
+                pixels = video_upscale.upscale_video(pixels, scale=mp4_scale, upscaler_name=mp4_upscaler)
+                timer.video.ts('upscale', t_upscale)
+                p.video_upscaled = True # notice so main save_video does not do it again
 
             save_fps = mp4_fps * processing_video.interpolation_factor(p)
+            t_save = time.time()
             num_frames, video_file, _thumb = video_save.save_video(
                 p=p,
                 pixels=pixels,
@@ -196,6 +209,7 @@ def generate(task_id, _ui_state,
                 upscale_upscaler=mp4_upscaler,
                 metadata={},
             )
+            timer.video.ts('save', t_save)
             _n, _c, _t, h, w = pixels.shape
             del pixels
             if audio is not None:
@@ -219,9 +233,12 @@ def generate(task_id, _ui_state,
         summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ')
         memory = shared.mem_mon.summary()
         total_time = max(t1 - t0, 1e-6)
+        timer.video.merge(timer.process)
+        timer.video.set('wall', total_time)
+        log.debug(f'Video: timers={timer.video.dct(no_total=True)}')
         fps = f'{num_frames/total_time:.2f}'
         its = f'{(steps)/total_time:.3f}'
-        log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={fps} its={its} resolution={resolution} time={total_time:.2f} timers={timer.process.dct(no_total=True)} memory={memstats.memory_stats()}')
+        log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={fps} its={its} resolution={resolution} time={total_time:.2f}')
 
         ui_text = f'Video | File {video_file} | Frames {num_frames} | Resolution {resolution} | f/s {fps} | it/s {its} ' + f"<div class='performance'><p>{summary} {memory}</p></div>"
         return video_file, ui_text

modules/processing.py

@@ -87,7 +87,6 @@ class Processed:
         self.info = info or create_infotext(p)
         self.infotexts = infotexts or [self.info]
         self.comments = comments or ''
-        memstats.reset_stats()
 
     def js(self):
         obj = {

modules/processing_video.py

@@ -93,7 +93,7 @@ def apply_video_interpolation(
         except Exception:
             pass
 
-    log.info(f'Video interpolation: type={in_type} input={in_len} output={frames_len(out)} count={count} scale={scale}')
+    log.info(f'Video interpolate: type={in_type} input={in_len} output={frames_len(out)} count={count} scale={scale}')
     return out
 
 

modules/rife/__init__.py

@@ -9,17 +9,18 @@ import numpy as np
 import torch
 from PIL import Image
 from torch.nn import functional as F
-from tqdm.rich import tqdm
+import rich.progress as rp
 from modules.rife.ssim import ssim_matlab
 from modules.rife.model_rife import RifeModel
 from modules import devices, shared, paths
-from modules.logger import log
+from modules.logger import log, console
 
 
 # Practical-RIFE v4.25 weights (MIT). Default URL points at the upstream HolyWu mirror;
 # can be swapped to a self-hosted mirror without any other code change.
 model_url = 'https://github.com/HolyWu/vs-rife/releases/download/model/flownet_v4.25.pkl'
 model: RifeModel = None
+pbar = rp.Progress(rp.TextColumn('[cyan]Interpolate:'), rp.BarColumn(), rp.MofNCompleteColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=console)
 
 
 def load(model_path: str = 'rife/flownet_v4.25.pkl'):
@@ -90,8 +91,10 @@ def interpolate(images: list, count: int = 2, scale: float = 1.0, pad: int = 1,
 
     I1 = f_pad(torch.from_numpy(np.transpose(frame, (2,0,1))).to(devices.device).unsqueeze(0).float() / 255.0)
     with torch.no_grad():
-        with tqdm(total=len(images), desc='Interpolate', unit='frame') as pbar:
-            for image in images:
+        with pbar:
+            task = pbar.add_task(total=len(images), description='starting...')
+            for idx, image in enumerate(images):
+                pbar.update(task, advance=1, description=f'frame {idx + 1}/{len(images)}')
                 frame = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
                 I0 = I1
                 I1 = f_pad(torch.from_numpy(np.transpose(frame, (2,0,1))).to(devices.device).unsqueeze(0).float() / 255.0)
@@ -114,6 +117,7 @@ def interpolate(images: list, count: int = 2, scale: float = 1.0, pad: int = 1,
                     buffer.put(mid[:h, :w])
                 buffer.put(frame)
                 pbar.update(1)
+            pbar.remove_task(task)
 
     for _i in range(pad): # fill ending frames
         buffer.put(frame)
@@ -143,15 +147,17 @@ def interpolate_nchw(images: list, count: int = 2, scale: float = 1.0):
 
     I1 = f_pad(images[0].unsqueeze(0))
     with torch.no_grad():
-        with tqdm(total=len(images), desc='Interpolate', unit='frame') as pbar:
-            for frame in images:
+        with pbar:
+            task = pbar.add_task(total=len(images), description='starting...')
+            for idx, frame in enumerate(images):
+                pbar.update(task, advance=1, description=f'frame {idx + 1}/{len(images)}')
                 I0 = I1
                 I1 = f_pad(frame.unsqueeze(0))
                 for i in range(count-1):
                     output = model.inference(I0, I1, (i+1) * 1. / (count), scale)
                     interpolated.append(output[:, :, :h, :w])
                 interpolated.append(I1[:, :, :h, :w])
-                pbar.update(1)
+            pbar.remove_task(task)
 
     t1 = time.time()
     log.info(f'Video interpolate: input={len(images)} frames={len(interpolated)} width={w} height={h} interpolate={count} scale={scale} time={round(t1 - t0, 2)}')

modules/timer.py

@@ -35,10 +35,20 @@ class Timer:
     def get(self, name):
         return self.records.get(name, 0)
 
+    def set(self, name, t):
+        self.records[name] = t
+
     def ts(self, name, t):
         elapsed = time.time() - t
         self.add(name, elapsed)
 
+    def merge(self, other):
+        for k, v in other.records.items():
+            if k not in self.records:
+                self.records[k] = 0
+            self.records[k] += v
+        self.total += other.total
+
     def record(self, category=None, extra_time=0, reset=True):
         e = self.elapsed(reset)
         if category is None:
@@ -91,6 +101,7 @@ class Timer:
 
 startup = Timer()
 process = Timer()
+video = Timer()
 launch = Timer()
 init = Timer()
 load = Timer()

modules/ui_definitions.py

@@ -156,7 +156,7 @@ def create_settings(cmd_opts):
         "group_offload_stream": OptionInfo(False, "Prefetch with streams", gr.Checkbox),
         'group_offload_record': OptionInfo(False, "Overlap stream transfers", gr.Checkbox),
         'group_offload_pin': OptionInfo(True, "Pin offload memory", gr.Checkbox),
-        'group_offload_blocks': OptionInfo(1, "Offload blocks", gr.Number),
+        'group_offload_blocks': OptionInfo(1, "Group offload blocks", gr.Number),
         "caption_offload_sep": OptionInfo("<h2>Caption Model Offloading</h2>", "", gr.HTML),
         "caption_offload": OptionInfo(True, "Offload caption models"),
         "caption_to_gpu": OptionInfo(True, "Load caption models direct to GPU"),

modules/video_models/video_save.py

@@ -324,17 +324,36 @@ def save_video(
         log.error(f'Video: type={type(pixels)} not a tensor')
         return 0, output_video, None
 
-    if upscale_upscaler is not None and len(upscale_upscaler) > 0:
-        t_upscale = time.time()
-        pixels = upscale_video(pixels, scale=upscale_scale, upscaler_name=upscale_upscaler)
-        timer.process.add('upscale', time.time()-t_upscale)
+    try:
+        if mp4_interpolate > 0 and not getattr(p, 'video_interpolated', False):
+            t_interpolate = time.time()
+            x = pixels.squeeze(0).permute(1, 0, 2, 3)
+            x = (x.clamp(-1., 1.) + 1.0) * 0.5  # RIFE expects [0, 1]; video pixels are [-1, 1]
+            interpolated = rife.interpolate_nchw(x, count=mp4_interpolate+1)
+            pixels = torch.stack(interpolated, dim=0)
+            pixels = pixels.permute(1, 2, 0, 3, 4)
+            pixels = pixels * 2.0 - 1.0
+            timer.process.ts('interpolate', t_interpolate)
+            p.video_interpolated = True
+    except Exception as e:
+        log.error(f'Video interpolate: {e}')
+        errors.display(e, 'video')
+
+    try:
+        if upscale_upscaler is not None and len(upscale_upscaler) > 0 and not getattr(p, 'video_upscaled', False):
+            t_upscale = time.time()
+            pixels = upscale_video(pixels, scale=upscale_scale, upscaler_name=upscale_upscaler)
+            timer.process.ts('upscale', t_upscale)
+            p.video_upscaled = True
+    except Exception as e:
+        log.error(f'Video upscale: {e}')
+        errors.display(e, 'video')
 
     t_save = time.time()
     if pixels.ndim == 4:
         pixels = pixels.unsqueeze(0)
     n, _c, t, h, w = pixels.shape
     size = pixels.element_size() * pixels.numel()
-    t_min, t_max = pixels.min().item(), pixels.max().item()
     log.debug(f'Video: video={mp4_video} export={mp4_frames} safetensors={mp4_sf} interpolate={mp4_interpolate}')
     if hasattr(audio, 'shape'):
         audio_txt = f'audio={audio.shape} aac={aac_sample_rate}' if audio is not None else 'no audio'
@@ -342,18 +361,11 @@ def save_video(
         audio_txt = f'audio={audio.get("format", None)} packets={len(audio.get("frames", []))} '
     else:
         audio_txt = None
-    log.debug(f'Video: encode={t} tensor={pixels.shape} min={t_min} max={t_max} bytes={size} {audio_txt} fps={mp4_fps} codec={mp4_codec} ext={mp4_ext} options="{mp4_opt}"')
+    log.debug(f'Video: encode={t} tensor={pixels.shape} bytes={size} {audio_txt} fps={mp4_fps} codec={mp4_codec} ext={mp4_ext} options="{mp4_opt}"')
     try:
         preparejob = shared.state.begin('Prepare video')
         if stream is not None:
             stream.output_queue.push(('progress', (None, 'Saving video...')))
-        if mp4_interpolate > 0 and not getattr(p, 'video_interpolated', False):
-            x = pixels.squeeze(0).permute(1, 0, 2, 3)
-            x = (x.clamp(-1., 1.) + 1.0) * 0.5  # RIFE expects [0, 1]; video pixels are [-1, 1]
-            interpolated = rife.interpolate_nchw(x, count=mp4_interpolate+1)
-            pixels = torch.stack(interpolated, dim=0)
-            pixels = pixels.permute(1, 2, 0, 3, 4)
-            pixels = pixels * 2.0 - 1.0
 
         if reclamp:
             x = torch.clamp(pixels.float(), -1., 1.) * 127.5 + 127.5