diff --git a/CHANGELOG.md b/CHANGELOG.md index 0786f4eb6..7218c78c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Highlights for 2026-08-24 +## Highlights for 2026-08-25 Time for a new release, this is a larger one! Main focus is improving video workflows which also brings full support for new [MiniMax H3](https://vladmandic.github.io/sdnext-docs/MiniMax) and [LTXVideo-2.5](https://vladmandic.github.io/sdnext-docs/LTX) @@ -17,7 +17,7 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa [Home](https://vladmandic.github.io/sdnext/) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic) -## Details for 2026-08-24 +## Details for 2026-08-25 - **Models** - [MiniMax H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) available in *base* and *ref* variants @@ -50,6 +50,7 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa - update handlers for all authenticated workflows - update handlers for all hf-based progress bars - offload options take effect immediately without restart/reload + - improve gpu memory tracking and reporting - log long torch autotune operations - utilize `torch.accelerator` where available - add `SD_DIFFUSERS_DEBUG` and `SD_TRANSFORMERS_DEBUG` env variables to trace diffusers and transformers internal operations @@ -120,6 +121,7 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa - upscaler: avoid unnecessary multi-pass - ideogram4: fix callback - xyzgrid: fix seed/prompt/negative handling, thanks @QuintusAntonius + - vram: skip reset ## Update for 2026-08-07 diff --git a/extensions-builtin/sdnq b/extensions-builtin/sdnq index b99d9af15..26e5fd7da 160000 --- a/extensions-builtin/sdnq +++ b/extensions-builtin/sdnq @@ -1 +1 @@ -Subproject commit b99d9af158a81fdd713dcbe31c9eb36f2332e54c +Subproject commit 26e5fd7da6a759b82c03b14510e73165591c870b diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 687b1f3a6..d1238dbf2 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -638,7 +638,7 @@ def run(model: str, *, out_w, out_h = video_utils.pixel_size(pixels, fallback=(p.width, p.height)) total_time = max(time.time() - t0, 1e-6) - log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={num_frames/total_time:.2f} its={p.steps/total_time:.3f} resolution={out_w}x{out_h} time={total_time:.2f} timers={timer.process.dct(no_total=True)} memory={memstats.memory_stats()}') + log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={num_frames/total_time:.2f} its={p.steps/total_time:.3f} resolution={out_w}x{out_h} time={total_time:.2f}') # the decode paths never materialize PIL, so frames come back through the saved file images_out = pixels if isinstance(pixels, list) else [] processed_out = processing.Processed(p, images_out, seed=p.seed, audio=audio_out) diff --git a/modules/memstats.py b/modules/memstats.py index 63c95db81..ebf079fcd 100644 --- a/modules/memstats.py +++ b/modules/memstats.py @@ -101,7 +101,7 @@ def gpu_stats(): if stats.get('num_ooms', 0) > 0: shared.state.oom = True gpu['active'] = gb(stats.get('active_bytes.all.current', 0)) - gpu['peak'] = gb(stats.get('active_bytes.all.peak', 0)) + gpu['peak'] = gb(stats.get('reserved_bytes.all.peak', 0)) gpu['retries'] = stats.get('num_alloc_retries', 0) gpu['oom'] = stats.get('num_ooms', 0) except Exception as e: @@ -156,6 +156,8 @@ def memory_stats(): def reset_stats(): + # fn = f'{sys._getframe(3).f_code.co_name}:{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access + # log.trace(f'Memory: reset {fn}') try: torch.cuda.reset_memory_stats() except Exception: diff --git a/modules/minimax/minimax_video.py b/modules/minimax/minimax_video.py index 50701bcf8..1cbfb6442 100644 --- a/modules/minimax/minimax_video.py +++ b/modules/minimax/minimax_video.py @@ -4,7 +4,7 @@ from PIL import Image import numpy as np from modules.logger import log from modules import shared, devices, errors, processing, timer, progress, paths, sd_models, scripts_manager, call_queue, memstats, processing_video -from modules.video_models import models_def, video_save, video_utils +from modules.video_models import models_def, video_save, video_utils, video_upscale engine = 'MiniMax' @@ -31,7 +31,7 @@ def load_model(model: str): sd_models.set_diffuser_options(shared.sd_model) # apply attention, offload, etc. loaded = f'repo={selected.repo} workflow={selected.workflow}' t1 = time.time() - timer.process.add('load', t1 - t0) + timer.video.add('load', t1 - t0) if shared.sd_model is not None: return selected.workflow return None @@ -48,6 +48,7 @@ def unwrap_file(entry): def prepare_inputs(workflow: str | None, init_image: Image.Image | None, last_image: Image.Image | None, reference_media: list | None) -> dict: """The task args a workflow conditions on, resolved before the model load so a rejected request costs nothing.""" + t_inputs = time.time() from modules.minimax import minimax_references if minimax_references.get_reference_caps(workflow) is not None: entries = [unwrap_file(entry) for entry in (reference_media or [])] @@ -62,6 +63,7 @@ def prepare_inputs(workflow: str | None, init_image: Image.Image | None, last_im if reference_media: log.warning(f'Video: op=reference workflow={workflow} references not supported, ignoring: count={len(reference_media)}') log.debug(f'Prepare inputs: workflow={workflow} first={init_image} last={last_image}') + timer.video.ts('inputs', t_inputs) return task_args @@ -92,6 +94,7 @@ def generate(task_id, _ui_state, progress.start_task(task_id) memstats.reset_stats() timer.process.reset() + timer.video.reset() # init vars p = None @@ -168,16 +171,26 @@ def generate(task_id, _ui_state, return None, "MiniMax: No frames generated" if mp4_interpolate > 0: + t_interpolate = time.time() p.video_interpolate = mp4_interpolate from modules.processing_video import apply_video_interpolation # pixels is 5-D (N,C,T,H,W) in [-1,1]; RIFE needs 4-D (T,C,H,W) in [0,1] x = pixels.squeeze(0).permute(1, 0, 2, 3) x = (x.clamp(-1., 1.) + 1.0) * 0.5 - x = apply_video_interpolation(p, x, count=mp4_interpolate) # sets p.video_interpolated otherwise main save_video would do it also + x = apply_video_interpolation(p, x, count=mp4_interpolate) x = x * 2.0 - 1.0 pixels = x.permute(1, 0, 2, 3).unsqueeze(0) + timer.video.ts('interpolate', t_interpolate) + p.video_interpolated = True # notice so main save_video does not do it again + + if mp4_upscaler is not None and len(mp4_upscaler) > 0: + t_upscale = time.time() + pixels = video_upscale.upscale_video(pixels, scale=mp4_scale, upscaler_name=mp4_upscaler) + timer.video.ts('upscale', t_upscale) + p.video_upscaled = True # notice so main save_video does not do it again save_fps = mp4_fps * processing_video.interpolation_factor(p) + t_save = time.time() num_frames, video_file, _thumb = video_save.save_video( p=p, pixels=pixels, @@ -196,6 +209,7 @@ def generate(task_id, _ui_state, upscale_upscaler=mp4_upscaler, metadata={}, ) + timer.video.ts('save', t_save) _n, _c, _t, h, w = pixels.shape del pixels if audio is not None: @@ -219,9 +233,12 @@ def generate(task_id, _ui_state, summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ') memory = shared.mem_mon.summary() total_time = max(t1 - t0, 1e-6) + timer.video.merge(timer.process) + timer.video.set('wall', total_time) + log.debug(f'Video: timers={timer.video.dct(no_total=True)}') fps = f'{num_frames/total_time:.2f}' its = f'{(steps)/total_time:.3f}' - log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={fps} its={its} resolution={resolution} time={total_time:.2f} timers={timer.process.dct(no_total=True)} memory={memstats.memory_stats()}') + log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={fps} its={its} resolution={resolution} time={total_time:.2f}') ui_text = f'Video | File {video_file} | Frames {num_frames} | Resolution {resolution} | f/s {fps} | it/s {its} ' + f"

{summary} {memory}

" return video_file, ui_text diff --git a/modules/processing.py b/modules/processing.py index f0579f17b..90f8015df 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -87,7 +87,6 @@ class Processed: self.info = info or create_infotext(p) self.infotexts = infotexts or [self.info] self.comments = comments or '' - memstats.reset_stats() def js(self): obj = { diff --git a/modules/processing_video.py b/modules/processing_video.py index c3138f368..2647c4b86 100644 --- a/modules/processing_video.py +++ b/modules/processing_video.py @@ -93,7 +93,7 @@ def apply_video_interpolation( except Exception: pass - log.info(f'Video interpolation: type={in_type} input={in_len} output={frames_len(out)} count={count} scale={scale}') + log.info(f'Video interpolate: type={in_type} input={in_len} output={frames_len(out)} count={count} scale={scale}') return out diff --git a/modules/rife/__init__.py b/modules/rife/__init__.py index e26c7aafc..306ef687a 100644 --- a/modules/rife/__init__.py +++ b/modules/rife/__init__.py @@ -9,17 +9,18 @@ import numpy as np import torch from PIL import Image from torch.nn import functional as F -from tqdm.rich import tqdm +import rich.progress as rp from modules.rife.ssim import ssim_matlab from modules.rife.model_rife import RifeModel from modules import devices, shared, paths -from modules.logger import log +from modules.logger import log, console # Practical-RIFE v4.25 weights (MIT). Default URL points at the upstream HolyWu mirror; # can be swapped to a self-hosted mirror without any other code change. model_url = 'https://github.com/HolyWu/vs-rife/releases/download/model/flownet_v4.25.pkl' model: RifeModel = None +pbar = rp.Progress(rp.TextColumn('[cyan]Interpolate:'), rp.BarColumn(), rp.MofNCompleteColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=console) def load(model_path: str = 'rife/flownet_v4.25.pkl'): @@ -90,8 +91,10 @@ def interpolate(images: list, count: int = 2, scale: float = 1.0, pad: int = 1, I1 = f_pad(torch.from_numpy(np.transpose(frame, (2,0,1))).to(devices.device).unsqueeze(0).float() / 255.0) with torch.no_grad(): - with tqdm(total=len(images), desc='Interpolate', unit='frame') as pbar: - for image in images: + with pbar: + task = pbar.add_task(total=len(images), description='starting...') + for idx, image in enumerate(images): + pbar.update(task, advance=1, description=f'frame {idx + 1}/{len(images)}') frame = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR) I0 = I1 I1 = f_pad(torch.from_numpy(np.transpose(frame, (2,0,1))).to(devices.device).unsqueeze(0).float() / 255.0) @@ -114,6 +117,7 @@ def interpolate(images: list, count: int = 2, scale: float = 1.0, pad: int = 1, buffer.put(mid[:h, :w]) buffer.put(frame) pbar.update(1) + pbar.remove_task(task) for _i in range(pad): # fill ending frames buffer.put(frame) @@ -143,15 +147,17 @@ def interpolate_nchw(images: list, count: int = 2, scale: float = 1.0): I1 = f_pad(images[0].unsqueeze(0)) with torch.no_grad(): - with tqdm(total=len(images), desc='Interpolate', unit='frame') as pbar: - for frame in images: + with pbar: + task = pbar.add_task(total=len(images), description='starting...') + for idx, frame in enumerate(images): + pbar.update(task, advance=1, description=f'frame {idx + 1}/{len(images)}') I0 = I1 I1 = f_pad(frame.unsqueeze(0)) for i in range(count-1): output = model.inference(I0, I1, (i+1) * 1. / (count), scale) interpolated.append(output[:, :, :h, :w]) interpolated.append(I1[:, :, :h, :w]) - pbar.update(1) + pbar.remove_task(task) t1 = time.time() log.info(f'Video interpolate: input={len(images)} frames={len(interpolated)} width={w} height={h} interpolate={count} scale={scale} time={round(t1 - t0, 2)}') diff --git a/modules/timer.py b/modules/timer.py index cdd443188..2cdecc87e 100644 --- a/modules/timer.py +++ b/modules/timer.py @@ -35,10 +35,20 @@ class Timer: def get(self, name): return self.records.get(name, 0) + def set(self, name, t): + self.records[name] = t + def ts(self, name, t): elapsed = time.time() - t self.add(name, elapsed) + def merge(self, other): + for k, v in other.records.items(): + if k not in self.records: + self.records[k] = 0 + self.records[k] += v + self.total += other.total + def record(self, category=None, extra_time=0, reset=True): e = self.elapsed(reset) if category is None: @@ -91,6 +101,7 @@ class Timer: startup = Timer() process = Timer() +video = Timer() launch = Timer() init = Timer() load = Timer() diff --git a/modules/ui_definitions.py b/modules/ui_definitions.py index ed0dab9c6..552161cca 100644 --- a/modules/ui_definitions.py +++ b/modules/ui_definitions.py @@ -156,7 +156,7 @@ def create_settings(cmd_opts): "group_offload_stream": OptionInfo(False, "Prefetch with streams", gr.Checkbox), 'group_offload_record': OptionInfo(False, "Overlap stream transfers", gr.Checkbox), 'group_offload_pin': OptionInfo(True, "Pin offload memory", gr.Checkbox), - 'group_offload_blocks': OptionInfo(1, "Offload blocks", gr.Number), + 'group_offload_blocks': OptionInfo(1, "Group offload blocks", gr.Number), "caption_offload_sep": OptionInfo("

Caption Model Offloading

", "", gr.HTML), "caption_offload": OptionInfo(True, "Offload caption models"), "caption_to_gpu": OptionInfo(True, "Load caption models direct to GPU"), diff --git a/modules/video_models/video_save.py b/modules/video_models/video_save.py index 0ff991d73..beff3dec1 100644 --- a/modules/video_models/video_save.py +++ b/modules/video_models/video_save.py @@ -324,17 +324,36 @@ def save_video( log.error(f'Video: type={type(pixels)} not a tensor') return 0, output_video, None - if upscale_upscaler is not None and len(upscale_upscaler) > 0: - t_upscale = time.time() - pixels = upscale_video(pixels, scale=upscale_scale, upscaler_name=upscale_upscaler) - timer.process.add('upscale', time.time()-t_upscale) + try: + if mp4_interpolate > 0 and not getattr(p, 'video_interpolated', False): + t_interpolate = time.time() + x = pixels.squeeze(0).permute(1, 0, 2, 3) + x = (x.clamp(-1., 1.) + 1.0) * 0.5 # RIFE expects [0, 1]; video pixels are [-1, 1] + interpolated = rife.interpolate_nchw(x, count=mp4_interpolate+1) + pixels = torch.stack(interpolated, dim=0) + pixels = pixels.permute(1, 2, 0, 3, 4) + pixels = pixels * 2.0 - 1.0 + timer.process.ts('interpolate', t_interpolate) + p.video_interpolated = True + except Exception as e: + log.error(f'Video interpolate: {e}') + errors.display(e, 'video') + + try: + if upscale_upscaler is not None and len(upscale_upscaler) > 0 and not getattr(p, 'video_upscaled', False): + t_upscale = time.time() + pixels = upscale_video(pixels, scale=upscale_scale, upscaler_name=upscale_upscaler) + timer.process.ts('upscale', t_upscale) + p.video_upscaled = True + except Exception as e: + log.error(f'Video upscale: {e}') + errors.display(e, 'video') t_save = time.time() if pixels.ndim == 4: pixels = pixels.unsqueeze(0) n, _c, t, h, w = pixels.shape size = pixels.element_size() * pixels.numel() - t_min, t_max = pixels.min().item(), pixels.max().item() log.debug(f'Video: video={mp4_video} export={mp4_frames} safetensors={mp4_sf} interpolate={mp4_interpolate}') if hasattr(audio, 'shape'): audio_txt = f'audio={audio.shape} aac={aac_sample_rate}' if audio is not None else 'no audio' @@ -342,18 +361,11 @@ def save_video( audio_txt = f'audio={audio.get("format", None)} packets={len(audio.get("frames", []))} ' else: audio_txt = None - log.debug(f'Video: encode={t} tensor={pixels.shape} min={t_min} max={t_max} bytes={size} {audio_txt} fps={mp4_fps} codec={mp4_codec} ext={mp4_ext} options="{mp4_opt}"') + log.debug(f'Video: encode={t} tensor={pixels.shape} bytes={size} {audio_txt} fps={mp4_fps} codec={mp4_codec} ext={mp4_ext} options="{mp4_opt}"') try: preparejob = shared.state.begin('Prepare video') if stream is not None: stream.output_queue.push(('progress', (None, 'Saving video...'))) - if mp4_interpolate > 0 and not getattr(p, 'video_interpolated', False): - x = pixels.squeeze(0).permute(1, 0, 2, 3) - x = (x.clamp(-1., 1.) + 1.0) * 0.5 # RIFE expects [0, 1]; video pixels are [-1, 1] - interpolated = rife.interpolate_nchw(x, count=mp4_interpolate+1) - pixels = torch.stack(interpolated, dim=0) - pixels = pixels.permute(1, 2, 0, 3, 4) - pixels = pixels * 2.0 - 1.0 if reclamp: x = torch.clamp(pixels.float(), -1., 1.) * 127.5 + 127.5 diff --git a/wiki b/wiki index eff72c57f..f5b364b49 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit eff72c57fe723d43129721e33b244d824e6dc674 +Subproject commit f5b364b491558c95a0583fa7bfefd514c7903ac9