diff --git a/CHANGELOG.md b/CHANGELOG.md index 895ede350..1d03df2e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Highlights for 2026-08-19 +## Highlights for 2026-08-21 Time for a new release, this is a larger one! Main focus is improving video workflows which also brings full support for new [MiniMax H3](https://vladmandic.github.io/sdnext-docs/MiniMax) and [LTXVideo-2.5](https://vladmandic.github.io/sdnext-docs/LTX) @@ -15,7 +15,7 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa [Home](https://vladmandic.github.io/sdnext/) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic) -## Details for 2026-08-19 +## Details for 2026-08-21 - **Models** - [MiniMax H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) available in *base* and *ref* variants @@ -59,6 +59,11 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa - reorganized *video* tab - better support for video codeces and formats - add *generate forever* button +- **Upscalers** + - update *spandrel* integration + moving forward, spandrel engine will be main upscaling engine for sdnext + when downloading any upscaling models manually, place them in `models/Spandrel` folder + - add several compact/light upscalers that are better suited for video upscaling - **API** - full support for video generation using api new endpoints: `/sdapi/v1/video`, `/sdapi/v1/video/models`, `/sdapi/v1/video/file` diff --git a/TODO.md b/TODO.md index 81adad48d..76b356d1c 100644 --- a/TODO.md +++ b/TODO.md @@ -4,7 +4,6 @@ - Update LTX wiki, @CalamitousFelicitousness - Productize benchmark tool, @CalamitousFelicitousness -- Nunchaku-Lite Krea2 errors, @vladmandic - Inpaint: https://discord.com/channels/1101998836328697867/1130536562422186044/1506850651035144322, @vladmandic - Lora: new handler, @CalamitousFelicitousness - Control tab verify overrides handling, @vladmandic @@ -32,6 +31,7 @@ ### Unassigned +- Incorporate [prompting guides](https://github.com/CalamitousFelicitousness/ai-prompting-guides) - Video models: add to Reference - UI Lite vs Expert mode - Auto handle scheduler `prediction_type` diff --git a/extensions-builtin/sdnq b/extensions-builtin/sdnq index b8b1dd312..1f31513fa 160000 --- a/extensions-builtin/sdnq +++ b/extensions-builtin/sdnq @@ -1 +1 @@ -Subproject commit b8b1dd3126a14b561506ed23b1dc5c49c45eab96 +Subproject commit 1f31513fa0ef8b5866d1fc030d274530175a4837 diff --git a/modules/control/proc/mediapipe_face_util.py b/modules/control/proc/mediapipe_face_util.py index 40ec0d204..785f5cce3 100644 --- a/modules/control/proc/mediapipe_face_util.py +++ b/modules/control/proc/mediapipe_face_util.py @@ -1,6 +1,6 @@ from typing import Mapping import numpy as np -from modules.shared import log +from modules.logger import log try: import mediapipe as mp diff --git a/modules/framepack/framepack_install.py b/modules/framepack/framepack_install.py index 92522fc7b..121b6c680 100644 --- a/modules/framepack/framepack_install.py +++ b/modules/framepack/framepack_install.py @@ -2,7 +2,7 @@ import os import shutil import git as gitpython from installer import install, git -from modules.shared import log +from modules.logger import log def rename(src:str, dst:str): diff --git a/modules/postprocess/nvvfx_model.py b/modules/postprocess/nvvfx_model.py deleted file mode 100644 index f933f5c65..000000000 --- a/modules/postprocess/nvvfx_model.py +++ /dev/null @@ -1,104 +0,0 @@ -import torch -import numpy as np -from PIL import Image -from modules import shared, devices -from modules.logger import log -from modules.upscaler import Upscaler, UpscalerData - - -class UpscalerDiffusion(Upscaler): - def __init__(self, dirname): # pylint: disable=super-init-not-called - self.name = "nVidia VFX" - self.user_path = dirname - """ - self.scalers = [ - UpscalerData(name="nVidia VFX 1x Denoise Ultra", path="", upscaler=self, model=None, scale=1), - UpscalerData(name="nVidia VFX 1x Deblur Ultra", path="", upscaler=self, model=None, scale=1), - UpscalerData(name="nVidia VFX 1x Denoise High", path="", upscaler=self, model=None, scale=1), - UpscalerData(name="nVidia VFX 1x Deblur High", path="", upscaler=self, model=None, scale=1), - UpscalerData(name="nVidia VFX 2x Ultra", path="", upscaler=self, model=None, scale=2), - UpscalerData(name="nVidia VFX 4x Ultra", path="", upscaler=self, model=None, scale=4), - UpscalerData(name="nVidia VFX 2x High", path="", upscaler=self, model=None, scale=2), - UpscalerData(name="nVidia VFX 4x High", path="", upscaler=self, model=None, scale=4), - ] - """ - self.scalers = [] - self.models = {} - - def load_model(self, path: str): - scaler: UpscalerData = [x for x in self.scalers if x.data_path == path or x.name == path] - if len(scaler) == 0: - log.error(f"Upscaler cannot match model: type={self.name} model={path}") - return None - scaler = scaler[0] - if self.models.get(path, None) is not None: - log.debug(f"Upscaler cached: type={scaler.name} model={path}") - return self.models[path] - from installer import install - install('nvidia-vfx') - - def callback(self, _step: int, _timestep: int, _latents: torch.FloatTensor): - pass - - def do_upscale(self, img: Image.Image, selected_model): - devices.torch_gc() - self.load_model(selected_model) - - frame = torch.from_numpy(np.array(img)).permute(2, 0, 1).float().to(devices.device) / 255.0 - frame = frame.to(devices.device) - - try: - import nvvfx - except Exception as e: - log.error(f"Upscaler: failed to import nvvfx: {e}") - return img - - config_map = { - "nVidia VFX 1x Denoise Ultra": nvvfx.VideoSuperRes.QualityLevel.DENOISE_ULTRA, - "nVidia VFX 1x Deblur Ultra": nvvfx.VideoSuperRes.QualityLevel.DEBLUR_ULTRA, - "nVidia VFX 1x Denoise High": nvvfx.VideoSuperRes.QualityLevel.DENOISE_HIGH, - "nVidia VFX 1x Deblur High": nvvfx.VideoSuperRes.QualityLevel.DEBLUR_HIGH, - "nVidia VFX 2x Ultra": nvvfx.VideoSuperRes.QualityLevel.ULTRA, - "nVidia VFX 4x Ultra": nvvfx.VideoSuperRes.QualityLevel.ULTRA, - "nVidia VFX 2x High": nvvfx.VideoSuperRes.QualityLevel.HIGH, - "nVidia VFX 4x High": nvvfx.VideoSuperRes.QualityLevel.HIGH, - } - quality = config_map.get(selected_model, None) - log.info(f'Upscaler: type="{self.name}" model="{selected_model}" version={nvvfx.__version__} sdk={nvvfx.get_sdk_version()} quality={quality}') - if self.models.get(selected_model, None) is not None: - vsr = self.models[selected_model] - else: - vsr = nvvfx.VideoSuperRes(quality=quality) - self.models[selected_model] = vsr - if '2x' in selected_model: - vsr.output_width = img.width * 2 - vsr.output_height = img.height * 2 - elif '4x' in selected_model: - vsr.output_width = img.width * 4 - vsr.output_height = img.height * 4 - elif 'Denoise' in selected_model or 'Deblur' in selected_model or '1x' in selected_model: - vsr.output_width = img.width - vsr.output_height = img.height - else: - log.error(f"Upscaler: unknown model: {selected_model}") - return img - vsr.input_width = img.width - vsr.input_height = img.height - - log.debug(f"Upscaler: {vsr}") - try: - vsr.load() - except Exception as e: - log.error(f"Upscaler: failed to load model: {selected_model} error={e}") - return img - self.models[selected_model] = vsr - - result = vsr.run(frame) - result = torch.from_dlpack(result.image).clone() - image = Image.fromarray((result.permute(1, 2, 0).contiguous().cpu().numpy() * 255).astype(np.uint8)) - - if shared.opts.upscaler_unload and selected_model in self.models: - del self.models[selected_model] - log.debug(f"Upscaler unloaded: type={self.name} model={selected_model}") - devices.torch_gc(force=True) - return image diff --git a/modules/progress.py b/modules/progress.py index 409c2d69f..292e28728 100644 --- a/modules/progress.py +++ b/modules/progress.py @@ -18,9 +18,10 @@ debug_log = log.trace if debug else lambda *args, **kwargs: None def start_task(id_task): global current_task # pylint: disable=global-statement - current_task = id_task - pending_tasks.pop(id_task, None) - log.debug(f'State: start id={id_task} pending={len(pending_tasks)} finished={len(finished_tasks)}') + if current_task != id_task: + log.debug(f'State: start id={id_task} pending={len(pending_tasks)} finished={len(finished_tasks)}') + current_task = id_task + pending_tasks.pop(id_task, None) def record_results(id_task, res): @@ -31,8 +32,8 @@ def record_results(id_task, res): def finish_task(id_task): global current_task # pylint: disable=global-statement - log.debug(f'State: end id={id_task}') if current_task == id_task: + log.debug(f'State: end id={id_task}') current_task = None if id_task not in finished_tasks: finished_tasks.append(id_task) diff --git a/modules/upscaler.py b/modules/upscaler.py index b02633f58..4ed470ac9 100644 --- a/modules/upscaler.py +++ b/modules/upscaler.py @@ -145,10 +145,10 @@ class Upscaler: if info is None: log.error(f'Upscaler cannot match model: type={self.name} model="{path}"') return None - if info.local_data_path.startswith("http"): + if info.local_data_path is not None and info.local_data_path.startswith("http"): from modules.modelloader import load_file_from_url info.local_data_path = load_file_from_url(url=info.data_path, model_dir=self.model_download_path, progress=True) - if not os.path.isfile(info.local_data_path): + if info.local_data_path is not None and not os.path.isfile(info.local_data_path): log.error(f'Upscaler cannot find model: type={self.name} model="{info.local_data_path}"') return None return info @@ -162,16 +162,33 @@ class UpscalerData: scaler: Upscaler | None = None model: None - def __init__(self, name: str, path: str | None = None, upscaler: Upscaler | None = None, scale: int = 4, model=None): + def __init__(self, name: str, path: str | None = None, upscaler: Upscaler | None = None, scale: int = 0, model=None): self.name = name self.data_path = path self.local_data_path = path self.scaler = upscaler + if scale > 0: + self.scale = scale + elif '2x' in name.lower(): + self.scale = 2 + elif '3x' in name.lower(): + self.scale = 3 + elif '4x' in name.lower(): + self.scale = 4 + elif '4x' in name.lower(): + self.scale = 4 + elif '8x' in name.lower(): + self.scale = 8 + else: + self.scale = 1 self.scale = scale self.model = model def __str__(self): - return f"UpscalerData(name={self.name}, path={self.data_path}, scale={self.scale})" + return f'UpscalerData(name="{self.name}" path="{self.data_path}" scale={self.scale})' + + def __repr__(self): + return f'UpscalerData(name="{self.name}" path="{self.data_path}" scale={self.scale})' def compile_upscaler(model): diff --git a/modules/upscaler_algo.py b/modules/upscaler_algo.py index 8b3d54ef9..ea4c8ce6b 100644 --- a/modules/upscaler_algo.py +++ b/modules/upscaler_algo.py @@ -1,7 +1,7 @@ import time from PIL import Image from modules.upscaler import Upscaler, UpscalerData -from modules.shared import log +from modules.logger import log class UpscalerDCC(Upscaler): diff --git a/modules/upscaler_nvvfx.py b/modules/upscaler_nvvfx.py new file mode 100644 index 000000000..fb9cd1552 --- /dev/null +++ b/modules/upscaler_nvvfx.py @@ -0,0 +1,79 @@ +import os +import time +import numpy as np +import torch +from PIL import Image +from modules.upscaler import Upscaler, UpscalerData +from modules import devices, shared, errors +from modules.logger import log + + +class UpscalerNVVFX(Upscaler): + def __init__(self, dirname=None): # pylint: disable=unused-argument + super().__init__(False) + self.name = "nVidia VFX" + self.scalers = [ + UpscalerData("nVidia VFX bicubic", None, self, scale=0), + UpscalerData("nVidia VFX low", None, self, scale=1), + UpscalerData("nVidia VFX medium", None, self, scale=2), + UpscalerData("nVidia VFX high", None, self, scale=3), + UpscalerData("nVidia VFX ultra", None, self, scale=4), + UpscalerData("nVidia VFX denoise low", None, self, scale=8), + UpscalerData("nVidia VFX denoise medium", None, self, scale=9), + UpscalerData("nVidia VFX denoise high", None, self, scale=10), + UpscalerData("nVidia VFX denoise ultra", None, self, scale=11), + UpscalerData("nVidia VFX deblur low", None, self, scale=12), + UpscalerData("nVidia VFX deblur medium", None, self, scale=13), + UpscalerData("nVidia VFX deblur high", None, self, scale=14), + UpscalerData("nVidia VFX deblur ultra", None, self, scale=15), + UpscalerData("nVidia VFX highbitrate low", None, self, scale=16), + UpscalerData("nVidia VFX highbitrate medium", None, self, scale=17), + UpscalerData("nVidia VFX highbitrate high", None, self, scale=18), + UpscalerData("nVidia VFX highbitrate ultra", None, self, scale=19), + ] + + def upscale(self, img: Image.Image | torch.Tensor, scale, selected_model: str | None = None): # nvvfx overrides upscale instead of do_upscale because it handles scale directly + if selected_model is None: + return img + from installer import install + install('nvidia-vfx') + os.environ["NV_VFX_LOG_LEVEL"] = "4" + os.environ["NV_VFX_DEBUG"] = "1" + try: + import nvvfx + except Exception as e: + log.error(f"Upscaler: nvvfx {e}") + errors.display(e, "Upscaler: nvvfx error") + return img + + jobid = shared.state.begin('Upscale') + try: + t0 = time.time() + upscaler = self.find_model(selected_model) + + quality = nvvfx.VideoSuperRes.QualityLevel(upscaler.scale) + vsr = nvvfx.VideoSuperRes(quality=quality) + vsr.input_width = img.width + vsr.input_height = img.height + _scale = 1.0 if 'DEBLUR' in quality.name or 'DENOISE' in quality.name else scale + vsr.output_width = int(img.width * _scale) + vsr.output_height = int(img.height * _scale) + log.debug(f'Upscaler: id={upscaler.scale} scale={_scale} version={nvvfx.__version__} sdk={nvvfx.get_sdk_version()} vsr={vsr}') + vsr.load() + + tensor = torch.from_numpy(np.array(img)).permute(2, 0, 1).float().contiguous().to(devices.device) / 255.0 + result = vsr.run(tensor) + tensor = torch.from_dlpack(result.image).clone() + tensor = 255.0 * tensor.permute(1, 2, 0).contiguous().cpu() + upscaled = Image.fromarray(tensor.numpy().astype(np.uint8)) + + vsr.close() + t1 = time.time() + log.debug(f'Upscale: name="{selected_model}" input={img.size} output={upscaled.size} time={t1 - t0:.2f}') + except nvvfx.NvVFXError as e: + log.error(f"Upscaler: nvvfx {e}") + errors.display(e, "Upscaler: nvvfx error") + upscaled = img + + shared.state.end(jobid) + return upscaled diff --git a/modules/upscaler_simple.py b/modules/upscaler_simple.py index 5c5c299af..bffcec827 100644 --- a/modules/upscaler_simple.py +++ b/modules/upscaler_simple.py @@ -1,6 +1,6 @@ from PIL import Image from modules.upscaler import Upscaler, UpscalerData -from modules.shared import log +from modules.logger import log class UpscalerNone(Upscaler): diff --git a/modules/upscaler_spandrel.py b/modules/upscaler_spandrel.py index f1974edce..56a67c574 100644 --- a/modules/upscaler_spandrel.py +++ b/modules/upscaler_spandrel.py @@ -1,14 +1,20 @@ import os import time +import torch +import numpy as np from PIL import Image from modules.upscaler import Upscaler, UpscalerData -from modules import devices, paths -from modules.shared import log +from modules import devices, paths, errors +from modules.logger import log MODELS = { "Spandrel 4x RealPLKSR NomosWebPhoto": "https://huggingface.co/vladmandic/sdnext-upscalers/resolve/main/4xNomosWebPhoto_RealPLKSR.safetensors", "Spandrel 2x RealPLKSR AnimeSharpV2": "https://huggingface.co/vladmandic/sdnext-upscalers/resolve/main/2x-AnimeSharpV2_RPLKSR_Sharp.pth", + "Spandrel 2x RealESRGAN Compact": "https://huggingface.co/vladmandic/sdnext-upscalers/resolve/main/RealESRGAN-2x-Compact.pth", + "Spandrel 2x RealESRGAN UltraCompact": "https://huggingface.co/vladmandic/sdnext-upscalers/resolve/main/RealESRGAN-2x-UltraCompact.pth", + "Spandrel 2x RealSAFMN++": "https://huggingface.co/vladmandic/sdnext-upscalers/resolve/main/Real-SAFMN-x2.pth", + "Spandrel 4x RealSAFMN++": "https://huggingface.co/vladmandic/sdnext-upscalers/resolve/main/Real-SAFMN-x4-v2.pth", } class UpscalerSpandrel(Upscaler): @@ -19,37 +25,58 @@ class UpscalerSpandrel(Upscaler): self.user_path = os.path.join(paths.models_path, 'Spandrel') self.selected = None self.model = None - self.scalers = [] - for model_name, model_path in MODELS.items(): - scaler = UpscalerData(name=model_name, path=model_path, upscaler=self) - self.scalers.append(scaler) + self.scalers = self.find_scalers() + found = [os.path.basename(s.data_path) for s in self.scalers] + for k, v in MODELS.items(): + fn = os.path.basename(v) + if fn not in found: + scaler = UpscalerData(name=k, path=v, upscaler=self) + self.scalers.append(scaler) + else: + for s in self.scalers: # update name of existing scaler if it was found + if os.path.basename(s.data_path) == fn: + s.name = k + break def process(self, img: Image.Image) -> Image.Image: - from modules.image import convert - tensor = convert.to_tensor(img).unsqueeze(0).to(devices.device) - img = img.convert('RGB') + if isinstance(img, Image.Image): + from modules.image import convert + img = img.convert('RGB') + tensor = convert.to_tensor(img).unsqueeze(0).to(devices.device) + elif isinstance(img, np.ndarray): + from modules.image import convert + tensor = convert.to_tensor(img).unsqueeze(0).to(devices.device) + elif isinstance(img, torch.Tensor): + tensor = img.to(devices.device) + else: + log.error(f'Spandrel: unsupported input type={type(img)}') + return img t0 = time.time() with devices.inference_context(): tensor = self.model(tensor) tensor = tensor.clamp(0, 1).squeeze(0).cpu() t1 = time.time() upscaled = convert.to_pil(tensor) - log.debug(f'Upscale: name="{self.selected}" input={img.size} output={upscaled.size} time={t1 - t0:.2f}') + log.debug(f'Upscale: name="{self.selected}" input={img.size} output={upscaled.size} time={t1 - t0:.3f}') return upscaled - def do_upscale(self, img: Image.Image, selected_model=None): + def load_model(self, path: str): from installer import install - if selected_model is None: - return img + if path is None: + return install('spandrel') + import spandrel + self.selected = path + model = self.find_model(path) + self.model = spandrel.ModelLoader().load_from_file(model.local_data_path) + self.model.to(devices.device).eval() + + def do_upscale(self, img: Image.Image | torch.Tensor | np.ndarray, selected_model=None): try: - import spandrel if (self.model is None) or (self.selected != selected_model): - self.selected = selected_model - model = self.find_model(selected_model) - self.model = spandrel.ModelLoader().load_from_file(model.local_data_path) - self.model.to(devices.device).eval() + self.load_model(selected_model) return self.process(img) except Exception as e: log.error(f'Spandrel: {e}') + errors.display(e, "Spandrel") return img diff --git a/pipelines/qwen/qwen_pruning.py b/pipelines/qwen/qwen_pruning.py index e2a642af1..96c491f41 100644 --- a/pipelines/qwen/qwen_pruning.py +++ b/pipelines/qwen/qwen_pruning.py @@ -1,5 +1,5 @@ def check_qwen_pruning(repo_id, subfolder): - from modules.shared import log + from modules.logger import log if 'pruning' not in repo_id.lower(): return repo_id, subfolder if '2509' in (repo_id or '') or '2509' in (subfolder or ''):