diff --git a/cli/api-video.py b/cli/api-video.py new file mode 100644 index 000000000..33c6451fb --- /dev/null +++ b/cli/api-video.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python +# python cli/api-video.py --prompt "a paper boat drifting down a rain gutter" --frames 17 --steps 8 --output /tmp/video.mp4 +import os +import time +import base64 +import logging +import argparse +import threading +import requests +import urllib3 + +sd_url = os.environ.get('SDAPI_URL', "http://127.0.0.1:7860") +sd_username = os.environ.get('SDAPI_USR', None) +sd_password = os.environ.get('SDAPI_PWD', None) +options = { + "send_video": True, + "send_thumbnail": False, +} + +logging.basicConfig(level = logging.INFO, format = '%(asctime)s %(levelname)s: %(message)s') +log = logging.getLogger(__name__) +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +def auth(): + if sd_username is not None and sd_password is not None: + return requests.auth.HTTPBasicAuth(sd_username, sd_password) + return None + + +def get(endpoint: str, params: dict | None = None, timeout: int = 60): + req = requests.get(f'{sd_url}{endpoint}', params=params, timeout=timeout, verify=False, auth=auth()) + if req.status_code != 200: + return { 'error': req.status_code, 'reason': req.reason, 'url': req.url } + return req.json() + + +def post(endpoint: str, dct: dict | None = None, timeout: int = 3600): + req = requests.post(f'{sd_url}{endpoint}', json=dct, timeout=timeout, verify=False, auth=auth()) + if req.status_code != 200: + res = { 'error': req.status_code, 'reason': req.reason, 'url': req.url } + try: + res['detail'] = req.json().get('detail', None) + except Exception: + pass + return res + return req.json() + + +def encode(f: str): + with open(f, 'rb') as file: + return base64.b64encode(file.read()).decode() + + +def list_models(): + data = get('/sdapi/v1/video/models') + if isinstance(data, dict) and 'error' in data: + log.error(f'video models: {data}') + return + for item in data: + loaded = ' loaded=true' if item.get('loaded') else '' + log.info(f'engine="{item["engine"]}" model="{item["name"]}" mode={item["mode"]}{loaded}') + log.info(f'video models: {len(data)}') + + +def watch_progress(stop_event: threading.Event): + while not stop_event.is_set(): + status = get('/sdapi/v1/progress', params={ 'skip_current_image': True }) + if 'error' not in status: + state = status.get('state') or {} + log.info(f'progress={status.get("progress", 0):.2f} eta={status.get("eta_relative", 0):.1f} step={state.get("sampling_step", 0)}/{state.get("sampling_steps", 0)} info="{status.get("textinfo") or ""}"') + stop_event.wait(5) + + +def save_output(data: dict, output: str): + if data.get('video'): + with open(output, 'wb') as f: + f.write(base64.b64decode(data['video'])) + log.info(f'video saved: filename={output}') + elif data.get('still') and data.get('frames'): + with open(output, 'wb') as f: + f.write(base64.b64decode(data['frames'][0])) + log.info(f'still saved: filename={output}') + elif data.get('video_path'): + req = requests.get(f'{sd_url}/sdapi/v1/video/file', params={ 'file': data['video_path'] }, timeout=300, verify=False, auth=auth()) + if req.status_code == 200: + with open(output, 'wb') as f: + f.write(req.content) + log.info(f'video fetched: filename={output} size={len(req.content)}') + else: + log.error(f'video fetch failed: code={req.status_code} reason={req.reason}') + else: + log.warning('no video output received') + + +def generate(args): # pylint: disable=redefined-outer-name + t0 = time.time() + if args.engine: + options['engine'] = args.engine + if args.model: + options['model'] = args.model + options['prompt'] = args.prompt + options['negative_prompt'] = args.negative + options['width'] = int(args.width) + options['height'] = int(args.height) + options['frames'] = int(args.frames) + options['steps'] = int(args.steps) + options['seed'] = int(args.seed) + options['sampler_name'] = args.sampler + options['mp4_fps'] = int(args.fps) + options['mp4_interpolate'] = int(args.interpolate) + options['audio'] = bool(args.audio) + if args.init: + options['init_image'] = encode(args.init) + if args.last: + options['last_image'] = encode(args.last) + if args.reference: + options['references'] = [encode(f) for f in args.reference] + stop_event = threading.Event() + if args.progress: + threading.Thread(target=watch_progress, args=(stop_event,), daemon=True).start() + data = post('/sdapi/v1/video', options, timeout=int(args.timeout)) + stop_event.set() + t1 = time.time() + if 'error' in data: + log.error(f'generate failed: {data}') + return + log.info(f'video received: frames={data.get("frames_count")} fps={data.get("fps")} duration={data.get("duration")} audio={data.get("has_audio")} still={data.get("still")} path={data.get("video_path")} time={t1-t0:.2f}') + if args.output: + save_output(data, args.output) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description = 'api-video') + parser.add_argument('--list', action='store_true', help='list video engines and models') + parser.add_argument('--engine', required=False, default=None, help='video engine; omit with --model to use the loaded checkpoint') + parser.add_argument('--model', required=False, default=None, help='video model name within the engine') + parser.add_argument('--prompt', required=False, default='', help='prompt text') + parser.add_argument('--negative', required=False, default='', help='negative prompt text') + parser.add_argument('--width', required=False, default=832, help='video width') + parser.add_argument('--height', required=False, default=480, help='video height') + parser.add_argument('--frames', required=False, default=17, help='number of frames; 1 for a still image') + parser.add_argument('--steps', required=False, default=20, help='number of steps') + parser.add_argument('--seed', required=False, default=-1, help='initial seed') + parser.add_argument('--sampler', required=False, default='Default', help='sampler name') + parser.add_argument('--fps', required=False, default=24, help='frames per second') + parser.add_argument('--interpolate', required=False, default=0, help='rife interpolation passes') + parser.add_argument('--audio', action=argparse.BooleanOptionalAction, default=True, help='generate audio on supported models') + parser.add_argument('--init', required=False, default=None, help='init image file') + parser.add_argument('--last', required=False, default=None, help='last frame image file') + parser.add_argument('--reference', required=False, default=None, action='append', help='reference image file for reference workflows; repeat in the order the model should read them') + parser.add_argument('--output', required=False, default=None, help='output video file') + parser.add_argument('--progress', action='store_true', help='poll and log progress during generation') + parser.add_argument('--timeout', required=False, default=3600, help='request timeout in seconds') + args = parser.parse_args() + log.info(f'api-video: {args}') + if args.list: + list_models() + else: + generate(args) diff --git a/data/reference-base.json b/data/reference-base.json index 96b18d90d..9ab483474 100644 --- a/data/reference-base.json +++ b/data/reference-base.json @@ -374,6 +374,23 @@ "size": 75.64, "date": "2025 September" }, + "MiniMaxAI MiniMax-H3": { + "path": "MiniMaxAI/MiniMax-H3", + "preview": "MiniMaxAI--MiniMax-H3.jpg", + "desc": "MiniMax-H3 generates video with synchronized stereo audio in a single denoising pass through a 33B single-stream transformer with a Qwen3-VL conditioner. In image tabs the model runs in experimental still mode, keeping the first frame of a minimal generation.", + "extras": "sampler: Default", + "size": 134, + "date": "2026 August" + }, + "MiniMaxAI MiniMax-H3 Ref2VA": { + "path": "MiniMaxAI/MiniMax-H3", + "subfolder": "ref2va", + "preview": "MiniMaxAI--MiniMax-H3.jpg", + "desc": "The omni-reference variant of MiniMax-H3, sharing one repository with the base model as a separate checkpoint partition. Video with synchronized stereo audio is conditioned on reference images for identity and appearance, with reference rows held clean while video rows denoise.", + "extras": "sampler: Default", + "size": 134, + "date": "2026 August" + }, "Freepik F-Lite": { "path": "Freepik/F-Lite", "preview": "Freepik--F-Lite.jpg", diff --git a/data/reference-quantized.json b/data/reference-quantized.json index 132fdfe6e..57f3f39e4 100644 --- a/data/reference-quantized.json +++ b/data/reference-quantized.json @@ -79,6 +79,14 @@ "date": "2025 October", "size": 23.53 }, + "MiniMaxAI MiniMax-H3 sdnq-uint4": { + "path": "OzzyGT/MiniMax_H3_sdnq_dynamic_4bit", + "preview": "MiniMaxAI--MiniMax-H3.jpg", + "desc": "Quantization of MiniMaxAI/MiniMax-H3 using SDNQ: dynamic 4-bit uint. Video with synchronized audio; in image tabs the model runs in experimental still mode.", + "extras": "sampler: Default", + "size": 51, + "date": "2026 August" + }, "Z-Image-Turbo sdnq-svd-uint4": { "path": "Disty0/Z-Image-Turbo-SDNQ-uint4-svd-r32", "preview": "Disty0--Z-Image-Turbo-SDNQ-uint4-svd-r32.jpg", diff --git a/installer.py b/installer.py index 1bf4d1dd1..b37458157 100644 --- a/installer.py +++ b/installer.py @@ -584,7 +584,7 @@ def check_diffusers(): t_start = time.time() if args.skip_all: return - target_commit = "6f2010e8bbe61fd2a81a659b858e298edcba8fab" # diffusers commit hash == 0.40.0.dev0 == 08-04-2026 + target_commit = "9f169d98d0bce392a889c3b6524d0d97734dfc0e" # diffusers commit hash == 0.40.0.dev0 == 08-05-2026 # if args.use_rocm or args.use_zluda: # sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now pkg = package_spec('diffusers') diff --git a/modules/api/api.py b/modules/api/api.py index 97a5404f7..70b98019e 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -6,7 +6,7 @@ from fastapi.security import HTTPBasic, HTTPBasicCredentials from fastapi.exceptions import HTTPException from modules import errors, shared, paths from modules.logger import log -from modules.api import models, endpoints, script, helpers, server, generate, process, control, docs, gpu +from modules.api import models, endpoints, script, helpers, server, generate, process, control, video, docs, gpu errors.install() @@ -33,6 +33,7 @@ class Api: self.generate = generate.APIGenerate(queue_lock) self.process = process.APIProcess(queue_lock) self.control = control.APIControl(queue_lock) + self.video = video.APIVideo(queue_lock) # compatibility api self.text2imgapi = self.generate.post_text2img self.img2imgapi = self.generate.post_img2img @@ -68,6 +69,7 @@ class Api: self.add_api_route("/sdapi/v1/txt2img", self.generate.post_text2img, methods=["POST"], response_model=models.ResTxt2Img, tags=["Generation"]) self.add_api_route("/sdapi/v1/img2img", self.generate.post_img2img, methods=["POST"], response_model=models.ResImg2Img, tags=["Generation"]) self.add_api_route("/sdapi/v1/control", self.control.post_control, methods=["POST"], response_model=control.ResControl, tags=["Generation"]) + self.add_api_route("/sdapi/v1/video", self.video.post_video, methods=["POST"], response_model=video.ResVideo, tags=["Generation"]) self.add_api_route("/sdapi/v1/process", self.process.extras_single_image_api, methods=["POST"], response_model=models.ResProcessImage, tags=["Processing"]) self.add_api_route("/sdapi/v1/extra-single-image", self.process.extras_single_image_api, methods=["POST"], response_model=models.ResProcessImage, tags=["Processing"]) self.add_api_route("/sdapi/v1/process-batch", self.process.extras_batch_images_api, methods=["POST"], response_model=models.ResProcessBatch, tags=["Processing"]) @@ -104,9 +106,11 @@ class Api: self.add_api_route("/sdapi/v1/extra-network-detail", endpoints.get_extra_network_detail, methods=["GET"], response_model=models.ItemExtraNetworkFull, tags=["Enumerators"]) self.add_api_route("/sdapi/v1/extra-network-details", endpoints.get_extra_network_details, methods=["GET"], response_model=models.ResExtraNetworkDetails, tags=["Enumerators"]) self.add_api_route("/sdapi/v1/unets", endpoints.get_unets, methods=["GET"], response_model=list[models.ItemUNet], tags=["Enumerators"]) + self.add_api_route("/sdapi/v1/video/models", self.video.get_video_models, methods=["GET"], response_model=list[video.ItemVideoModel], tags=["Enumerators"]) # functional api self.add_api_route("/sdapi/v1/file", endpoints.get_file, methods=["GET"], tags=["Functional"]) + self.add_api_route("/sdapi/v1/video/file", self.video.get_video_file, methods=["GET"], tags=["Functional"]) self.add_api_route("/sdapi/v1/delete-image", endpoints.get_deleteimage, methods=["DELETE"], tags=["Functional"]) self.add_api_route("/sdapi/v1/delete-file", endpoints.get_deletefile, methods=["DELETE"], tags=["Functional"]) self.add_api_route("/sdapi/v1/png-info", endpoints.get_pnginfo, methods=["GET"], response_model=models.ResImageInfo, tags=["Functional"]) diff --git a/modules/api/helpers.py b/modules/api/helpers.py index 487499a80..f5087e5e8 100644 --- a/modules/api/helpers.py +++ b/modules/api/helpers.py @@ -1,4 +1,5 @@ import io +import os import base64 from PIL import Image, PngImagePlugin import piexif @@ -90,6 +91,24 @@ def encode_pil_to_base64(image): return b64 +MAX_B64_BYTES = 256 * 1024 * 1024 # base64 expands ~4/3 and the response is built in memory; larger artifacts are fetched by path instead + + +def encode_file_to_base64(fn: str, max_bytes: int = MAX_B64_BYTES) -> str | None: + try: + if fn is None or not os.path.isfile(fn): + return None + size = os.path.getsize(fn) + if size > max_bytes: + log.warning(f'API cannot encode file: fn="{fn}" size={size} max={max_bytes}') + return None + with open(fn, 'rb') as f: + return base64.b64encode(f.read()).decode('ascii') + except Exception as e: + log.warning(f'API cannot encode file: fn="{fn}" {e}') + return None + + def upscaler_to_index(name: str): try: return [x.name.lower() for x in shared.sd_upscalers].index(name.lower()) diff --git a/modules/api/mime.py b/modules/api/mime.py index 7b244f152..824af43be 100644 --- a/modules/api/mime.py +++ b/modules/api/mime.py @@ -10,3 +10,6 @@ def register(): mimetypes.add_type('image/webp', '.webp') mimetypes.add_type('image/jxl', '.jxl') mimetypes.add_type('font/ttf', '.ttf') + mimetypes.add_type('video/mp4', '.mp4') + mimetypes.add_type('video/webm', '.webm') + mimetypes.add_type('video/x-matroska', '.mkv') diff --git a/modules/api/validate.py b/modules/api/validate.py index 9738c0efb..7de27254a 100644 --- a/modules/api/validate.py +++ b/modules/api/validate.py @@ -12,6 +12,7 @@ request_cost = { "/sdapi/v1/txt2img": 5, "/sdapi/v1/img2img": 5, "/sdapi/v1/control": 5, + "/sdapi/v1/video": 5, } log_cost = { "/.well-known/appspecific/com.chrome.devtools.json": -1, diff --git a/modules/api/video.py b/modules/api/video.py new file mode 100644 index 000000000..d415d45f7 --- /dev/null +++ b/modules/api/video.py @@ -0,0 +1,275 @@ +from types import SimpleNamespace +from threading import Lock +from pydantic import BaseModel, Field # pylint: disable=no-name-in-module +from fastapi.exceptions import HTTPException +from modules import errors, shared, scripts_manager, ui +from modules.api import script, helpers +from modules.paths import resolve_output_path +from modules.video_models import models_def, video_load, video_run + + +errors.install() + + +class ReqVideo(BaseModel): + engine: str | None = Field(default=None, title="Engine", description="Video engine family; omit together with model to use the currently loaded checkpoint") + model: str | None = Field(default=None, title="Model", description="Video model name within the engine; see GET /sdapi/v1/video/models") + prompt: str = Field(default="", title="Prompt", description="Text prompt") + negative_prompt: str = Field(default="", title="Negative prompt", description="Negative text prompt") + styles: list[str] = Field(default=[], title="Styles", description="Prompt style names to apply") + width: int = Field(default=832, ge=64, le=4096, title="Width", description="Output width; snapped to the model canvas multiple") + height: int = Field(default=480, ge=64, le=4096, title="Height", description="Output height; snapped to the model canvas multiple") + frames: int = Field(default=17, ge=1, le=1024, title="Frames", description="Number of frames; 1 produces a single still image on workflow models") + steps: int = Field(default=50, ge=1, le=200, title="Steps", description="Number of inference steps") + sampler_name: str = Field(default="Default", title="Sampler", description="Sampler name; Default keeps the model scheduler") + sampler_shift: float = Field(default=-1.0, title="Sampler shift", description="Scheduler flow shift; -1 keeps the model default") + dynamic_shift: bool = Field(default=False, title="Dynamic shift", description="Enable dynamic scheduler shifting") + seed: int = Field(default=-1, title="Seed", description="Generation seed; -1 for random") + guidance_scale: float = Field(default=-1.0, title="Guidance scale", description="CFG scale; -1 keeps the model default") + guidance_true: float = Field(default=-1.0, title="True guidance", description="True CFG scale; -1 keeps the model default") + init_image: str | None = Field(default=None, title="Init image", description="Base64, data URI, or upload reference for the first-frame image") + init_strength: float = Field(default=0.8, ge=0.0, le=1.0, title="Init strength", description="Denoising strength for the init image") + last_image: str | None = Field(default=None, title="Last image", description="Base64, data URI, or upload reference for the last-frame image") + references: list[str] = Field(default=[], title="References", description="Reference images for a reference workflow, in the order the model reads them; base64, data URIs, or upload references. At most 9, each within a 1:4 to 4:1 aspect ratio. Rejected on models that do not condition on references") + vae_type: str = Field(default="Default", title="VAE type", description="Decode variant: Default, Tiny, Remote, or Upscale") + vae_tile_frames: int = Field(default=16, ge=1, le=64, title="VAE tile frames", description="Frames per VAE decode tile") + audio: bool = Field(default=True, title="Audio", description="Generate audio on models that support it") + mp4_fps: int = Field(default=24, ge=1, le=60, title="FPS", description="Frames per second of the saved video") + mp4_interpolate: int = Field(default=0, ge=0, le=10, title="Interpolation", description="RIFE interpolation passes between frames") + mp4_codec: str = Field(default="libx264", title="Codec", description="Video codec; none skips video encoding") + mp4_ext: str = Field(default="mp4", title="Container", description="Container extension; the muxer is inferred from it") + mp4_opt: str = Field(default="crf:16", title="Codec options", description="Encoder options as comma-separated key:value pairs") + mp4_video: bool = Field(default=True, title="Save video", description="Write the video container to disk") + mp4_frames: bool = Field(default=False, title="Save frames", description="Write individual frame images to disk") + mp4_sf: bool = Field(default=False, title="Save safetensors", description="Write raw frames as a safetensors file") + mp4_thumb: bool = Field(default=True, title="Save thumbnail", description="Write a thumbnail image next to the video") + override_settings: dict = Field(default={}, title="Override settings", description="Setting overrides applied for this generation only") + script_args: list = Field(default=[], title="Script args", description="Positional arguments for a selectable script") + alwayson_scripts: dict = Field(default={}, title="Always-on scripts", description="Per-script argument overrides, keyed by script name") + send_video: bool = Field(default=True, title="Send video", description="Return the video base64-encoded in the response") + send_frames: bool = Field(default=False, title="Send frames", description="Return every frame base64-encoded in the response") + send_thumbnail: bool = Field(default=True, title="Send thumbnail", description="Return the thumbnail base64-encoded in the response") + extra: dict | None = Field(default={}, exclude=True, title="Extra", description="Extra attributes set on the processing object") + + +class ResVideo(BaseModel): + video: str | None = Field(default=None, title="Video", description="Base64-encoded video file; empty when not requested, above the size cap, or in still mode") + video_path: str | None = Field(default=None, title="Video path", description="Server path of the saved video; fetch via GET /sdapi/v1/video/file") + thumbnail: str | None = Field(default=None, title="Thumbnail", description="Base64-encoded thumbnail image") + thumbnail_path: str | None = Field(default=None, title="Thumbnail path", description="Server path of the saved thumbnail") + frames: list[str] = Field(default=[], title="Frames", description="Base64-encoded frames; always populated in still mode") + frames_count: int = Field(default=0, title="Frame count", description="Number of frames written, after interpolation") + fps: float = Field(default=0.0, title="FPS", description="Effective frames per second of the saved video") + duration: float = Field(default=0.0, title="Duration", description="Video duration in seconds") + has_audio: bool = Field(default=False, title="Has audio", description="Whether the video carries an audio track") + still: bool = Field(default=False, title="Still", description="Single-frame result; the product is in frames and no video was written") + params: dict = Field(default={}, title="Parameters", description="Echo of the request parameters used for generation") + info: str = Field(default="", title="Info", description="Generation info string with seed, sampler, and pipeline details") + + +class ItemVideoModel(BaseModel): + engine: str = Field(title="Engine", description="Video engine family") + name: str = Field(title="Name", description="Model name; pass together with engine to select it") + repo: str = Field(default="", title="Repo", description="Model repository or path") + url: str = Field(default="", title="URL", description="Model information page") + mode: str = Field(title="Mode", description="Input mode: workflow, t2v, i2v, flf2v, vace, or animate") + workflow: str | None = Field(default=None, title="Workflow", description="Modular workflow name when the model dispatches on inputs; ref2va conditions on references and ignores the keyframe images") + base: bool = Field(default=False, title="Base", description="Also listed in the base checkpoint dropdown") + loaded: bool = Field(default=False, title="Loaded", description="Currently loaded through the video registry") + + +def model_mode(m: models_def.Model) -> str: + # mirrors the dispatch order in video_run.run: workflow models route on inputs, the rest on name markers + if m.workflow is not None: + return 'workflow' + if 'T2V' in m.name: + return 't2v' + if 'I2V' in m.name: + return 'i2v' + if 'FLF2V' in m.name: + return 'flf2v' + if 'VACE' in m.name: + return 'vace' + if 'Animate' in m.name: + return 'animate' + return 't2v' + + +class APIVideo: + def __init__(self, queue_lock: Lock): + self.queue_lock = queue_lock + self.default_script_arg_video = [] + + def prepare_scripts(self, p_stub, req: ReqVideo): + script_runner = scripts_manager.scripts_video + if not script_runner.scripts: + script_runner.initialize_scripts(is_img2img=False, is_control=False, is_video=True) + ui.create_ui(None) + if not self.default_script_arg_video: + self.default_script_arg_video = script.init_default_script_args(script_runner) + script_args = script.init_script_args(p_stub, req, self.default_script_arg_video, None, None, script_runner) + return script_runner, script_args + + def sanitize_b64(self, req: ReqVideo): + def sanitize_str(args: list): + for idx in range(0, len(args)): + if isinstance(args[idx], str) and len(args[idx]) >= 1000: + args[idx] = f"" + for name in ('init_image', 'last_image'): + val = getattr(req, name, None) + if isinstance(val, str) and len(val) >= 1000: + setattr(req, name, f"") + if req.references: + sanitize_str(req.references) + if req.script_args: + sanitize_str(req.script_args) + if req.alwayson_scripts: + for script_obj in req.alwayson_scripts.values(): + if script_obj and "args" in script_obj and script_obj["args"]: + sanitize_str(script_obj["args"]) + + def post_video(self, req: ReqVideo): + """Generate a video, or a single still frame, using a video model. + + Omit `engine` and `model` to drive the currently loaded checkpoint when it is + video-capable; this covers models loaded from local folders that have no registry + entry. Pass both names to select a registry model, which is loaded on demand; + `GET /sdapi/v1/video/models` enumerates the valid pairs. + + `frames` of 1 on a workflow model produces a single still image returned in `frames`. + Disk outputs are controlled by `mp4_video`, `mp4_frames`, `mp4_sf`, and `mp4_thumb`; + response payloads are controlled independently by `send_video`, `send_frames`, and + `send_thumbnail`. Artifacts above the base64 size cap return `video` empty with + `video_path` set; fetch those via `GET /sdapi/v1/video/file`. + + `init_image` and `last_image` accept base64 data, data URIs, or upload references. + Models whose workflow is `ref2va` condition on `references` instead: an ordered list of + images the prompt addresses as ``, `` and so on, following list + order. A single reference may also be passed as `init_image`. Reference images do not + set the output canvas, and `last_image` is ignored. + + Progress is reported on `GET /sdapi/v1/progress`; `POST /sdapi/v1/interrupt` cancels. + Switching checkpoints via `override_settings` is not supported here; use + `POST /sdapi/v1/checkpoint` before generating. + """ + try: + selected, needs_load = video_run.resolve_model(req.engine, req.model) + except video_run.VideoError as e: + raise HTTPException(status_code=e.code, detail=str(e)) from e + sampler_name = helpers.validate_sampler_name(req.sampler_name) + init_image = helpers.decode_base64_to_image(req.init_image) if req.init_image else None + last_image = helpers.decode_base64_to_image(req.last_image) if req.last_image else None + references = [helpers.decode_base64_to_image(x) for x in (req.references or [])] + overrides = dict(req.override_settings or {}) + for key in ('sd_model_checkpoint', 'sd_model_refiner'): + if key in overrides: + raise HTTPException(status_code=400, detail=f"{key} override is not supported here: switch models via POST /sdapi/v1/checkpoint before generating") + p_stub = SimpleNamespace(per_script_args={}) + script_runner, script_args = self.prepare_scripts(p_stub, req) + extra = getattr(req, 'extra', {}) or {} + + with self.queue_lock: + jobid = shared.state.begin('API-VID', api=True) + try: + res = video_run.run( + selected, + prompt=req.prompt, + negative=req.negative_prompt, + styles=req.styles, + width=req.width, + height=req.height, + frames=req.frames, + steps=req.steps, + sampler_name=sampler_name, + sampler_shift=req.sampler_shift, + dynamic_shift=req.dynamic_shift, + seed=req.seed, + guidance_scale=req.guidance_scale, + guidance_true=req.guidance_true, + init_image=init_image, + init_strength=req.init_strength, + last_image=last_image, + references=references, + vae_type=req.vae_type, + vae_tile_frames=req.vae_tile_frames, + audio=req.audio, + mp4_fps=req.mp4_fps, + mp4_interpolate=req.mp4_interpolate, + mp4_codec=req.mp4_codec, + mp4_ext=req.mp4_ext, + mp4_opt=req.mp4_opt, + mp4_video=req.mp4_video, + mp4_frames=req.mp4_frames, + mp4_sf=req.mp4_sf, + mp4_thumb=req.mp4_thumb, + override_settings=overrides, + engine=req.engine, + scripts=script_runner, + script_args=script_args, + per_script_args=p_stub.per_script_args, + extra_p=extra, + needs_load=needs_load, + ) + except video_run.VideoError as e: + raise HTTPException(status_code=e.code, detail=str(e)) from e + finally: + shared.state.end(jobid, api=False) + + send_frames = req.send_frames or res.still # a still request has no other product to return + b64_frames = list(map(helpers.encode_pil_to_base64, res.images)) if send_frames else [] + video_b64 = helpers.encode_file_to_base64(res.video_path) if req.send_video and res.video_path else None + thumb_b64 = helpers.encode_file_to_base64(res.thumb_path) if req.send_thumbnail and res.thumb_path else None + duration = round(res.num_frames / res.fps, 3) if res.fps > 0 else 0.0 + self.sanitize_b64(req) + params = {k: v for k, v in vars(req).items() if k != 'extra'} + return ResVideo( + video=video_b64, + video_path=res.video_path, + thumbnail=thumb_b64, + thumbnail_path=res.thumb_path, + frames=b64_frames, + frames_count=res.num_frames, + fps=res.fps, + duration=duration, + has_audio=res.has_audio, + still=res.still, + params=params, + info=res.processed.info, + ) + + def get_video_models(self, engine: str | None = None): + """List video engines and models; optionally filter by engine.""" + items = [] + for family, rows in models_def.models.items(): + if engine is not None and family.lower() != engine.lower(): + continue + for m in rows: + if m.name == 'None': + continue + items.append(ItemVideoModel( + engine=family, + name=m.name, + repo=m.repo or '', + url=m.url or '', + mode=model_mode(m), + workflow=m.workflow, + base=m.base, + loaded=(m.name == video_load.loaded_model), + )) + return items + + def get_video_file(self, file: str): + """Serve a video artifact produced by this endpoint; the path must resolve inside the video output directory.""" + import mimetypes + from pathlib import Path + from starlette.responses import FileResponse + if not file or not file.strip(): + raise HTTPException(status_code=400, detail="file path is required") + root = Path(resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_video)).resolve() + target = Path(file).resolve() + if root not in target.parents: + raise HTTPException(status_code=403, detail=f"file {file}: must be inside the video output directory") + if not target.is_file(): + raise HTTPException(status_code=404, detail=f"file not found: {file}") + media_type = mimetypes.guess_type(target.name)[0] or 'application/octet-stream' + return FileResponse(str(target), media_type=media_type, filename=target.name) diff --git a/modules/modeldata.py b/modules/modeldata.py index 439123a05..ee8122b15 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -156,6 +156,8 @@ def get_model_type(pipe): model_type = 'mochivideo' elif "Allegro" in name: model_type = 'allegrovideo' + elif 'MiniMaxH3' in name: + model_type = 'minimaxh3' # cloud models elif 'GoogleVeo' in name: model_type = 'veo3' diff --git a/modules/modelloader.py b/modules/modelloader.py index a06f296eb..b12337f99 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -152,10 +152,13 @@ def load_diffusers_models(clear=True): continue name = name.replace("--", "/") friendly = os.path.join(place, name) - has_index = os.path.exists(os.path.join(folder, 'model_index.json')) + index_file = os.path.join(folder, 'model_index.json') + if not os.path.exists(index_file): + index_file = os.path.join(folder, 'modular_model_index.json') # modular pipelines carry their own index flavor + has_index = os.path.exists(index_file) if has_index: # direct download of diffusers model - repo = { 'name': name, 'filename': name, 'friendly': friendly, 'folder': folder, 'path': folder, 'hash': None, 'mtime': os.path.getmtime(folder), 'model_info': os.path.join(folder, 'model_info.json'), 'model_index': os.path.join(folder, 'model_index.json') } + repo = { 'name': name, 'filename': name, 'friendly': friendly, 'folder': folder, 'path': folder, 'hash': None, 'mtime': os.path.getmtime(folder), 'model_info': os.path.join(folder, 'model_info.json'), 'model_index': index_file } diffuser_repos.append(repo) continue @@ -168,6 +171,8 @@ def load_diffusers_models(clear=True): mtime = os.path.getmtime(commit) info = os.path.join(commit, "model_info.json") index = os.path.join(commit, "model_index.json") + if not os.path.exists(index): + index = os.path.join(commit, "modular_model_index.json") # modular pipelines carry their own index flavor config = os.path.join(commit, "config.json") if (not os.path.exists(index)) and (not os.path.exists(info)) and (not os.path.exists(config)): debug(f'Diffusers skip model no info: {name}') diff --git a/modules/processing_args.py b/modules/processing_args.py index eca6b4ce5..0755f169a 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -172,7 +172,7 @@ def task_specific_kwargs(p, model): def get_params(model): if hasattr(model, 'blocks') and hasattr(model.blocks, 'inputs'): # modular pipeline possible = [input_param.name for input_param in model.blocks.inputs] - return possible + return possible + ['output'] # __call__ param selecting which state values to return, not a block input else: signature = inspect.signature(type(model).__call__, follow_wrapped=True) possible = list(signature.parameters) @@ -325,6 +325,14 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:l args['negative_prompt'] = args['negative_prompt'][0] if len(args['negative_prompt']) > 0 else '' if isinstance(args['generator'], list) and len(args['generator']) > 0: args['generator'] = args['generator'][0] + if 'MiniMaxH3' in model.__class__.__name__: + if isinstance(args.get('prompt', None), list): # packs one request into one sequence, str only + args['prompt'] = args['prompt'][0] if len(args['prompt']) > 0 else '' + if not str(args.get('prompt', '') or '').strip(): + args['prompt'] = ' ' # an empty prompt tokenizes to zero tokens, which the conditioner cannot reshape + args.pop('negative_prompt', None) # guidance-distilled, no negative prompt + if isinstance(args.get('generator', None), list) and len(args['generator']) > 0: + args['generator'] = args['generator'][0] # >1-element list breaks the audio noise draw # set callbacks if 'prior_callback_steps' in possible: # Wuerstchen / Cascade diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index d032d5bc9..1e10e56c5 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -191,6 +191,8 @@ def process_base(p: processing.StableDiffusionProcessing): output = SimpleNamespace(images=output) if isinstance(output, Image.Image): output = SimpleNamespace(images=[output]) + if not hasattr(output, 'frames') and hasattr(output, 'videos'): + output.frames = output.videos # modular video pipelines emit videos, not frames if hasattr(output, 'image'): output.images = output.image if hasattr(output, 'images'): @@ -473,9 +475,13 @@ def process_decode(p: processing.StableDiffusionProcessing, output): log.debug(f'Generated: bytes={len(output.bytes)}') return output audio = getattr(output, 'audio', None) + if audio is not None: + p.audio_sampling_rate = getattr(output, 'sampling_rate', None) if not hasattr(output, 'images') and hasattr(output, 'frames'): log.debug(f'Generated: frames={len(output.frames[0])}') output.images = output.frames[0] + if getattr(p, 'video_still', False) and hasattr(output, 'images') and output.images is not None: + output.images = output.images[:1] # only the first frame derives from real latents; the rest decode from padding if output.images is not None and len(output.images) > 0 and isinstance(output.images[0], Image.Image): sd_models.offload_ondemand(shared.sd_model) # in-pipe decode paths return materialized frames; the vae seam in processing_vae never runs return attach_audio(output.images, audio) @@ -534,6 +540,13 @@ def update_pipeline(sd_model, p: processing.StableDiffusionProcessing): log.warning('Processing: op=update model not loaded') return None updated_model = sd_model + if 'MiniMaxH3' in sd_model.__class__.__name__ and not isinstance(p, processing.StableDiffusionProcessingVideo): + # image tabs run the model in still mode; the video tab applies its own overrides + from modules.video_models import video_modular + video_modular.apply_minimax_overrides(p, sd_model, still=True, audio=False) + if getattr(p, 'detailer_enabled', False): + log.warning(f'Processing: cls={sd_model.__class__.__name__} detailer not supported') + p.detailer_enabled = False if sd_models.get_diffusers_task(sd_model) == sd_models.DiffusersTaskType.INPAINTING and getattr(p, 'image_mask', None) is None and p.task_args.get('image_mask', None) is None and getattr(p, 'mask', None) is None: log.warning('Processing: mode=inpaint mask=None') updated_model = sd_models.set_diffuser_pipe(sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) @@ -551,19 +564,9 @@ def update_pipeline(sd_model, p: processing.StableDiffusionProcessing): def validate_pipeline(p: processing.StableDiffusionProcessing): - from modules.video_models.models_def import models as video_models - models_cls = [] - for family in video_models: - for m in video_models[family]: - if m.repo_cls is not None: - if isinstance(m.repo_cls, str): - models_cls.append(m.repo_cls) - else: - models_cls.append(m.repo_cls.__name__) - if m.custom is not None: - models_cls.append(m.custom) - is_video_model = shared.sd_model.__class__.__name__ in models_cls - override_video_pipelines = ['WanPipeline', 'WanImageToVideoPipeline', 'WanVACEPipeline'] + from modules.video_models import models_def + is_video_model = shared.sd_model.__class__.__name__ in models_def.pipeline_classes() + override_video_pipelines = ['WanPipeline', 'WanImageToVideoPipeline', 'WanVACEPipeline', 'MiniMaxH3ModularPipeline'] is_video_pipeline = ('video' in p.__class__.__name__.lower()) or (shared.sd_model.__class__.__name__ in override_video_pipelines) if is_video_model and not is_video_pipeline: log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} video model with non-video pipeline') @@ -571,6 +574,11 @@ def validate_pipeline(p: processing.StableDiffusionProcessing): elif not is_video_model and is_video_pipeline: log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} non-video model with video pipeline') return False + if getattr(shared.sd_model, 'sdnext_video_workflow', None) == 'ref2va' and p.task_args.get('references', None) is None: + # the reference workflow loads its own transformer partition alone: without references the pipeline + # dispatches to the keyframe path and reaches a transformer that was never loaded + log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} reference workflow requires reference images: use the video tab or the video api') + return False return True diff --git a/modules/sd_detect.py b/modules/sd_detect.py index cde46ca06..bf3327be1 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -115,6 +115,8 @@ def guess_by_name(fn, current_guess): new_guess = 'Cosmos' elif 'f-lite' in fn.lower(): new_guess = 'FLite' + elif 'minimax' in fn.lower(): + new_guess = 'MiniMaxH3' elif 'wan' in fn.lower(): new_guess = 'WanAI' if 'chronoedit' in fn.lower(): diff --git a/modules/sd_models.py b/modules/sd_models.py index 0fdd1ce21..87691e8e1 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -248,7 +248,7 @@ def move_model(model, device=None, force=False): for name, m in model.components.items(): if not hasattr(m, "_hf_hook"): # not accelerate hook break - if not isinstance(m, torch.nn.Module) or name in model._exclude_from_cpu_offload: # pylint: disable=protected-access + if not isinstance(m, torch.nn.Module) or name in getattr(model, '_exclude_from_cpu_offload', []): # modular pipelines lack the attr continue for module in m.modules(): set_execution_device(module, device) @@ -493,6 +493,10 @@ def load_diffuser_force(detected_model_type: str, checkpoint_info: CheckpointInf from pipelines.model_wanai import load_wan sd_model = load_wan(checkpoint_info, diffusers_load_config) allow_post_quant = False + elif model_type in ['MiniMaxH3']: + from pipelines.model_minimax import load_minimax + sd_model = load_minimax(checkpoint_info, diffusers_load_config) + allow_post_quant = False elif model_type in ['ChronoEdit']: from pipelines.model_chrono import load_chrono sd_model = load_chrono(checkpoint_info, diffusers_load_config) @@ -1602,7 +1606,7 @@ def hf_auth_check(checkpoint_info: CheckpointInfo, force:bool=False): try: if (checkpoint_info.path.endswith('.safetensors') and os.path.isfile(checkpoint_info.path)): # skip check for single-file safetensors models return True - if (os.path.exists(checkpoint_info.path) and os.path.isdir(checkpoint_info.path) and os.path.isfile(os.path.join(checkpoint_info.path, 'model_index.json'))): # skip check for local diffusers folders + if os.path.exists(checkpoint_info.path) and os.path.isdir(checkpoint_info.path) and any(os.path.isfile(os.path.join(checkpoint_info.path, f)) for f in ('model_index.json', 'modular_model_index.json')): # skip check for local diffusers folders return True except Exception: pass @@ -1640,9 +1644,13 @@ def save_model(name: str, path: str | None = None, shard: str = "5GB", overwrite torch.cuda.synchronize() except Exception: pass + jobid = shared.state.begin('Save model') try: t0 = time.time() log.info(f'Save model: path="{model_name}" cls={shared.sd_model.__class__.__name__} start') + if hasattr(shared.sd_model, '_component_specs'): # modular pipeline: the saved index must reference the destination folder, not the source repos; save_sdnq_model lives in the sdnq submodule and does not pass this flag + import functools + shared.sd_model.save_pretrained = functools.partial(shared.sd_model.save_pretrained, overwrite_modular_index=True) save_sdnq_model( model=shared.sd_model, model_path=model_name, @@ -1656,6 +1664,10 @@ def save_model(name: str, path: str | None = None, shard: str = "5GB", overwrite log.error(f'Save model: path="{model_name}" {e}') errors.display(e, 'Save model') return f'Error: {e}' + finally: + if 'save_pretrained' in vars(shared.sd_model): + del shared.sd_model.save_pretrained # drop the instance shadow, restoring the class method + shared.state.end(jobid) def list_hfcache(): diff --git a/modules/sd_offload.py b/modules/sd_offload.py index c7e608aac..89eddae81 100644 --- a/modules/sd_offload.py +++ b/modules/sd_offload.py @@ -25,6 +25,7 @@ no_split_module_classes = [ "Linear", "Conv1d", "Conv2d", "Conv3d", "ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d", "Embedding", "SDNQLinear", "SDNQConv1d", "SDNQConv2d", "SDNQConv3d", "SDNQConvTranspose1d", "SDNQConvTranspose2d", "SDNQConvTranspose3d", "SDNQEmbedding", "WanTransformerBlock", + "MiniMaxH3TransformerBlock", "MiniMaxH3TokenRefinerBlock", ] accelerate_dtype_byte_size = None move_stream = None @@ -163,6 +164,7 @@ def apply_group_offload_component(module, module_name: str, main: bool, op: str module = accelerate.hooks.remove_hook_from_module(module, recurse=True) remove_group_offload_component(module) module.requires_grad_(False) + log.debug(f'Setting {op}: offload=group op=apply module={module_name} pin={cfg["use_stream"] and not cfg["low_cpu_mem_usage"]}') # before the apply: pinning large components takes a while and would otherwise run silently apply_group_offloading(module, onload_device=devices.device, offload_device=devices.cpu, **cfg) module.sdnext_group_offload_sig = sig return True @@ -395,6 +397,11 @@ def set_diffuser_offload(sd_model, op:str='model', quiet:bool=False, force:bool= accelerate_dtype_byte_size = accelerate.utils.modeling.dtype_byte_size accelerate.utils.modeling.dtype_byte_size = dtype_byte_size + if sd_models.get_diffusers_task(sd_model) == sd_models.DiffusersTaskType.MODULAR and shared.opts.diffusers_offload_mode in {'model', 'sequential', 'group'}: + apply_modular_group_offload(sd_model, op=op) + process_timer.add('offload', time.time() - t0) + return + if shared.opts.diffusers_offload_mode == "none": apply_none_offload(sd_model, op=op, quiet=quiet) diff --git a/modules/sd_vae.py b/modules/sd_vae.py index ac436ad58..75dad5592 100644 --- a/modules/sd_vae.py +++ b/modules/sd_vae.py @@ -24,6 +24,7 @@ vae_scale_override = { 'WanPipeline': 16, 'ChronoEditPipeline': 16, 'AutoencoderKLWan': 16, + 'AutoencoderKLMiniMaxH3': 16, } @@ -53,6 +54,8 @@ def get_vae_scale_factor(model: DiffusionPipeline | None = None): vae_scale_factor = 8 if model is not None and hasattr(model, 'patch_size'): patch_size = model.patch_size + if isinstance(patch_size, (tuple, list)): # 3d patch sizes are (t, h, w); spatial term is last + patch_size = patch_size[-1] if debug: log.trace(f'VAE: cls={model.__class__.__name__ if model else "None"} scale={vae_scale_factor} patch={patch_size}') return vae_scale_factor * patch_size diff --git a/modules/shared_items.py b/modules/shared_items.py index a69edfeba..3ebfb5baf 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -55,6 +55,7 @@ pipelines = { 'PixArtAlpha': getattr(diffusers, 'PixArtAlphaPipeline', None), 'PixArtSigma': getattr(diffusers, 'PixArtSigmaPipeline', None), 'PRXPixel': getattr(diffusers, 'PRXPixelPipeline', None), + 'MiniMaxH3': getattr(diffusers, 'MiniMaxH3ModularPipeline', None), 'Qwen': getattr(diffusers, 'QwenImagePipeline', None), 'Sana': getattr(diffusers, 'SanaPipeline', None), 'WanAI': getattr(diffusers, 'WanPipeline', None), diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index e1fa4d026..795f228be 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -25,9 +25,11 @@ class Model: image_hijack: bool = True vae_hijack: bool = True vae_remote: bool = False + workflow: str = None + base: bool = False # also registered as a base model: caches into the diffusers folder so the model scan lists it in the dropdown def __str__(self): - return f'name="{self.name}" url="{self.url}" repo="{self.repo}" repo_cls="{self.repo_cls}" dit="{self.dit}" dit_cls="{self.dit_cls}" dit_folder="{self.dit_folder}" te="{self.te}" te_cls="{self.te_cls}" te_folder="{self.te_folder}" te_hijack={self.te_hijack} vae_hijack={self.vae_hijack} vae_remote={self.vae_remote}' + return f'name="{self.name}" url="{self.url}" repo="{self.repo}" repo_cls="{self.repo_cls}" dit="{self.dit}" dit_cls="{self.dit_cls}" dit_folder="{self.dit_folder}" te="{self.te}" te_cls="{self.te_cls}" te_folder="{self.te_folder}" te_hijack={self.te_hijack} vae_hijack={self.vae_hijack} vae_remote={self.vae_remote} workflow="{self.workflow}" base={self.base}' def getpipe(package, name, _default=None): @@ -630,6 +632,45 @@ try: te_cls='Qwen2_5_VLForConditionalGeneration', dit_cls='Kandinsky5Transformer3DModel'), ], + 'MiniMax': [ + Model(name='None'), + Model(name='MiniMax H3 SDNQ uint4', + url='https://huggingface.co/MiniMaxAI/MiniMax-H3', + repo='OzzyGT/MiniMax_H3_sdnq_dynamic_4bit', + repo_cls='MiniMaxH3ModularPipeline', + workflow='fl2va', + base=True, + te_cls=None, + dit_cls=None, + te_hijack=False, + image_hijack=False, + vae_hijack=False, + vae_remote=False), + Model(name='MiniMax H3', + url='https://huggingface.co/MiniMaxAI/MiniMax-H3', + repo='MiniMaxAI/MiniMax-H3', + repo_cls='MiniMaxH3ModularPipeline', + workflow='fl2va', + base=True, + te_cls=None, + dit_cls=None, + te_hijack=False, + image_hijack=False, + vae_hijack=False, + vae_remote=False), + Model(name='MiniMax H3 Ref2VA', + url='https://huggingface.co/MiniMaxAI/MiniMax-H3', + repo='MiniMaxAI/MiniMax-H3', + repo_cls='MiniMaxH3ModularPipeline', + workflow='ref2va', + base=True, + te_cls=None, + dit_cls=None, + te_hijack=False, + image_hijack=False, + vae_hijack=False, + vae_remote=False), + ], 'Google Veo': [ Model(name='Google Veo 3.1 T2V', url='https://gemini.google/overview/video-generation/', @@ -659,3 +700,47 @@ try: except Exception as e: models = {} log.error(f'Networks: type="video" {e}') + + +def engines() -> list[str]: + """Engine families with at least one real model, sentinel rows excluded.""" + return [engine for engine, rows in models.items() if any(row.name != 'None' for row in rows)] + + +def model_names(engine: str) -> list[str]: + """Real model names for an engine, sentinel rows excluded.""" + return [row.name for row in models.get(engine, []) if row.name != 'None'] + + +def find(engine: str, name: str) -> Model | None: + """Case-insensitive exact-name lookup; the 'None' sentinel rows are not models.""" + for family, rows in models.items(): + if family.lower() != (engine or '').lower(): + continue + for row in rows: + if row.name != 'None' and row.name.lower() == (name or '').lower(): + return row + return None + + +def workflow_for_class(cls_name: str) -> str | None: + """Workflow of the first registry row whose pipeline class matches, if any.""" + for rows in models.values(): + for row in rows: + if row.workflow is not None and row.repo_cls is not None: + row_cls = row.repo_cls if isinstance(row.repo_cls, str) else row.repo_cls.__name__ + if row_cls == cls_name: + return row.workflow + return None + + +def pipeline_classes() -> set[str]: + """Class names of every registry pipeline; repo_cls is a string before load and a class after video_load resolves it in place.""" + classes = set() + for rows in models.values(): + for row in rows: + if row.repo_cls is not None: + classes.add(row.repo_cls if isinstance(row.repo_cls, str) else row.repo_cls.__name__) + if row.custom is not None: + classes.add(row.custom) + return classes diff --git a/modules/video_models/video_load.py b/modules/video_models/video_load.py index 66419d6c5..ad3fc2237 100644 --- a/modules/video_models/video_load.py +++ b/modules/video_models/video_load.py @@ -6,7 +6,7 @@ import transformers import diffusers from modules import shared, errors, sd_models, sd_checkpoint, model_quant, devices, sd_hijack_te, sd_hijack_vae from modules.logger import log -from modules.video_models import models_def, video_utils, video_overrides, video_cache +from modules.video_models import models_def, video_utils, video_overrides, video_cache, video_modular def _loader(component): @@ -151,7 +151,9 @@ def load_model(selected: models_def.Model): # model try: - if selected.repo_cls is None: + if selected.workflow is not None or video_modular.is_modular(selected.repo_cls): + shared.sd_model = video_modular.load_modular(selected, offline_args) + elif selected.repo_cls is None: shared.sd_model = load_custom(selected.repo) else: log.debug(f'Load video: module=pipe repo="{selected.repo}" cls={selected.repo_cls.__name__}') @@ -210,6 +212,8 @@ def load_model(selected: models_def.Model): shared.sd_model = model_quant.do_post_load_quant(shared.sd_model, allow=False) sd_models.set_diffuser_offload(shared.sd_model) + if video_modular.is_modular(shared.sd_model): + video_modular.install_state_hook(shared.sd_model) loaded_model = selected.name msg = f'Load video: cls={shared.sd_model.__class__.__name__} model="{selected.name}" time={t1-t0:.2f}' diff --git a/modules/video_models/video_modular.py b/modules/video_models/video_modular.py new file mode 100644 index 000000000..0732defe8 --- /dev/null +++ b/modules/video_models/video_modular.py @@ -0,0 +1,214 @@ +import time +import logging +import torch +from modules import shared, errors, devices, model_quant +from modules.logger import log + + +MIN_LATENT_FRAMES = 7 # decoder floor: fewer latent frames leave the chunked decode with nothing to emit + + +def is_modular(obj) -> bool: + if obj is None: + return False + cls = obj if isinstance(obj, type) else obj.__class__ + try: + import diffusers + modular_cls = getattr(diffusers, 'ModularPipeline', None) + if isinstance(modular_cls, type) and issubclass(cls, modular_cls): + return True + except Exception: + pass + return 'Modular' in cls.__name__ + + +def load_modular_pipe(repo_cls, repo: str, workflow: str | None = None, revision: str | None = None, offline_args: dict | None = None, base: bool = False): + if repo_cls is None or isinstance(repo_cls, str): + log.error(f'Load modular: repo="{repo}" cls="{repo_cls}" pipeline class not found: diffusers too old') + return None + offline_args = offline_args or {} + cache_dir = shared.opts.diffusers_dir if base else shared.opts.hfcache_dir # base models live in the diffusers folder so the model scan lists them; video-only models stay out of the dropdown + try: + t0 = time.time() + log.debug(f'Load modular: repo="{repo}" cls={repo_cls.__name__} workflow={workflow} base={base}') + pipe = repo_cls.from_pretrained( + repo, + revision=revision, + cache_dir=cache_dir, + **offline_args, + ) + # workflow selection stays out of from_pretrained: pruning the blocks tree to one task + # would disable runtime auto-dispatch between them; only the component fetch is restricted + load_kwargs = {} + quant_config = {} + quant_args = model_quant.create_config(module='Model') + if 'quantization_config' in quant_args: + quant_config['transformer'] = quant_args['quantization_config'] + quant_config['transformer_ref'] = quant_args['quantization_config'] + te_args = model_quant.create_config(module='TE', modules_to_not_convert=['.model.visual']) # the conditioner's vision tower stays unquantized: quantized vision blocks have no validated precedent and only run for keyframe workflows + if 'quantization_config' in te_args: + quant_config['text_encoder'] = te_args['quantization_config'] + if quant_config: + # per-component dict without a default entry: only the listed components quantize while + # loading, everything else loads unquantized + load_kwargs['quantization_config'] = quant_config + log.debug(f'Load modular: quant={next(iter(quant_config.values())).__class__.__name__} modules={list(quant_config)}') + pipe.load_components( + workflow=workflow, + dtype=devices.dtype, + cache_dir=cache_dir, + **load_kwargs, + **offline_args, + ) + loaded = [name for name, component in pipe.components.items() if component is not None] + pipe.sdnext_video_workflow = workflow # lets a pipe loaded outside the video registry report its own workflow + if hasattr(pipe, 'min_duration') and hasattr(pipe, 'fps'): + pipe.sdnext_supported_min_frames = int(pipe.min_duration * pipe.fps) # fresh pipes report the true floor; still mode gates per instance + log.debug(f'Load modular: cls={pipe.__class__.__name__} workflow={workflow} components={loaded} time={time.time()-t0:.2f}') + return pipe + except Exception as e: + log.error(f'Load modular: repo="{repo}" workflow={workflow} {e}') + errors.display(e, 'video') + return None + + +def load_modular(selected, offline_args: dict): + return load_modular_pipe(selected.repo_cls, selected.repo, workflow=selected.workflow, revision=selected.repo_revision, offline_args=offline_args, base=selected.base) + + +def apply_minimax_overrides(p, pipe, still: bool = False, audio: bool = True): + """Per-generation constraints shared by the video tab and the image path: canvas and frame + alignment, the bespoke scheduler guard, tiling, and the audio/still toggles.""" + if still: + audio = False # a sub-second soundtrack is pure waste on a kept single frame + multiple = pipe.canvas_multiple + p.task_args['width'] = multiple * (p.width // multiple) + p.task_args['height'] = multiple * (p.height // multiple) + set_still(pipe, still) + if still: + frames = 5 # two latent frames; decode pads to the decoder floor and only the first frame is kept + log.info(f'Video modular: cls={pipe.__class__.__name__} mode=still experimental') + else: + frames = max(getattr(p, 'frames', 1), getattr(pipe, 'sdnext_supported_min_frames', 120)) + while frames % pipe.vae_frames_per_chunk != pipe.vae_latents_per_chunk: # frame counts align to 17n+5 + frames += 1 + max_frames = int(pipe.max_duration * pipe.fps) + while frames > max_frames: + frames -= pipe.vae_frames_per_chunk + if frames != getattr(p, 'frames', None): + log.debug(f'Video modular: cls={pipe.__class__.__name__} frames={getattr(p, "frames", None)} aligned={frames}') + p.frames = frames + p.task_args['num_frames'] = frames + p.steps = max(2, p.steps) + p.task_args['num_inference_steps'] = p.steps + pipe.num_timesteps = p.steps - 1 # sigma grid includes the terminal point; feeds the progress total + if p.sampler_name not in ('None', 'Default'): + log.warning(f'Video modular: cls={pipe.__class__.__name__} sampler={p.sampler_name} unsupported: using model default') + p.sampler_name = 'Default' # the model default is the bespoke scheduler pair, which discrete samplers must not replace + pipe.vae.enable_tiling() # model always tiles; the shared vae params path may have disabled it + set_audio(pipe, audio) + p.task_args['output'] = ['videos', 'audio', 'sampling_rate'] if audio else ['videos'] + p.task_args['output_type'] = 'pil' # the image path otherwise requests latent output, which the decode block rejects + p.video_still = still + + +def set_still(pipe, enabled: bool = True): + """Toggle sub-floor generation for single-frame output. The duration floor is lifted only + while the instance flag is set, so other pipes of the class and later normal runs keep the + supported floor; decoded latents below the decoder floor are padded by duplicating the + trailing latent. The causal VAE keeps padding out of frame 0.""" + cls = type(pipe) + if getattr(cls, 'sdnext_min_duration_orig', None) is None: + orig = cls.min_duration + cls.sdnext_min_duration_orig = orig + cls.min_duration = property(lambda self: 0.0 if getattr(self, 'sdnext_still_mode', False) else orig.fget(self)) + pipe.sdnext_still_mode = enabled + if not enabled: + return + vae = getattr(pipe, 'vae', None) + if vae is not None and getattr(vae, 'sdnext_orig_decode', None) is None: + vae.sdnext_orig_decode = vae.decode + def padded_decode(z, *args, **kwargs): + if z.ndim == 5 and z.shape[2] < MIN_LATENT_FRAMES: + pad = z[:, :, -1:].repeat(1, 1, MIN_LATENT_FRAMES - z.shape[2], 1, 1) + z = torch.cat([z, pad], dim=2) + return vae.sdnext_orig_decode(z, *args, **kwargs) + vae.decode = padded_decode + + +def set_audio(pipe, enabled: bool): + """Pop or restore the audio decode block. The joint denoise still carries the audio rows + (a few percent of the sequence), but without the block the audio VAE never runs. + Operates on the backing block tree: the public blocks property deep-copies per access.""" + blocks = getattr(pipe, '_blocks', None) # pylint: disable=protected-access + decode = blocks.sub_blocks.get('decode', None) if blocks is not None and hasattr(blocks, 'sub_blocks') else None + sub = getattr(decode, 'sub_blocks', None) + if sub is None: + return + if enabled and 'audio' not in sub: + stashed = getattr(pipe, 'sdnext_audio_decode_block', None) + if stashed is not None: + sub.insert('audio', stashed, len(sub)) + log.debug(f'Video modular: cls={pipe.__class__.__name__} audio=enabled') + elif not enabled and 'audio' in sub: + pipe.sdnext_audio_decode_block = sub.pop('audio') + log.debug(f'Video modular: cls={pipe.__class__.__name__} audio=disabled') + + +class InterruptLogFilter(logging.Filter): + """Drops the per-block error dumps the modular runner logs when an interrupt raises through it.""" + def filter(self, record): + return 'Interrupted...' not in record.getMessage() + + +def install_state_hook(pipe): + runner_log = logging.getLogger('diffusers.modular_pipelines.modular_pipeline') + if not any(isinstance(f, InterruptLogFilter) for f in runner_log.filters): + runner_log.addFilter(InterruptLogFilter()) + + def set_phase(phase: str): + # every stage runs inside one pipeline call, so the forward hooks are the only + # place the current stage is visible; state.begin clears the label per job + if getattr(pipe, 'sdnext_phase', None) != phase: + pipe.sdnext_phase = phase + shared.state.textinfo = phase + log.debug(f'Video modular: cls={pipe.__class__.__name__} phase="{phase}"') + + def state_hook(module, args): # pylint: disable=unused-argument + set_phase('Generate') + if shared.state.sampling_steps == 0 and getattr(pipe, 'num_timesteps', 0) > 0: + shared.state.sampling_steps = pipe.num_timesteps + if shared.state.paused: + log.debug('Sampling paused') + while shared.state.paused: + if shared.state.interrupted or shared.state.skipped: + raise AssertionError('Interrupted...') + time.sleep(0.1) + shared.state.step() + if shared.state.interrupted or shared.state.skipped: + raise AssertionError('Interrupted...') + + def encode_hook(module, args): # pylint: disable=unused-argument + set_phase('Text encode') + if shared.state.interrupted or shared.state.skipped: + raise AssertionError('Interrupted...') + + def decode_hook(module, args): # pylint: disable=unused-argument + set_phase('Decode') + if shared.state.interrupted or shared.state.skipped: # fires per tile, so tiled decodes abort promptly + raise AssertionError('Interrupted...') + + for name in ('transformer', 'transformer_ref'): + module = getattr(pipe, name, None) + if module is None or getattr(module, 'sdnext_state_hook', None) is not None: + continue + module.sdnext_state_hook = module.register_forward_pre_hook(state_hook) + text_encoder = getattr(pipe, 'text_encoder', None) + if text_encoder is not None: + target = getattr(text_encoder, 'model', text_encoder) # conditioning calls the inner model directly + if isinstance(target, torch.nn.Module) and getattr(target, 'sdnext_state_hook', None) is None: + target.sdnext_state_hook = target.register_forward_pre_hook(encode_hook) + for name in ('vae', 'audio_vae'): + decoder = getattr(getattr(pipe, name, None), 'decoder', None) # decode entry points bypass forward, the inner decoder does not + if isinstance(decoder, torch.nn.Module) and getattr(decoder, 'sdnext_state_hook', None) is None: + decoder.sdnext_state_hook = decoder.register_forward_pre_hook(decode_hook) diff --git a/modules/video_models/video_overrides.py b/modules/video_models/video_overrides.py index cdd9c9036..2c9b8c79e 100644 --- a/modules/video_models/video_overrides.py +++ b/modules/video_models/video_overrides.py @@ -4,6 +4,7 @@ import diffusers from modules import shared, processing, devices from modules.logger import log from modules.video_models.models_def import Model +from modules.video_models import video_modular debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None @@ -104,3 +105,6 @@ def set_overrides(p: processing.StableDiffusionProcessingVideo, selected: Model) if 'Kandinsky 5.0 Lite 10s' in selected.name: # p.task_args['time_length'] = 10 shared.sd_model.transformer.set_attention_backend("flex") + # MiniMax H3 + if 'MiniMaxH3' in cls: + video_modular.apply_minimax_overrides(p, shared.sd_model, still=getattr(p, 'video_still', False), audio=getattr(p, 'video_audio', True)) diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 195543c85..92296b4b5 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -1,53 +1,158 @@ import os import copy import time +from dataclasses import dataclass from modules import shared, errors, sd_models, processing, devices, images, ui_common, scripts_manager from modules.logger import log -from modules.video_models import models_def, video_utils, video_load, video_vae, video_overrides, video_save +from modules.video_models import models_def, video_utils, video_load, video_vae, video_overrides, video_save, video_modular from modules.paths import resolve_output_path debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None +MAX_IMAGE_REFERENCES = 9 # mirrors the reference setup block's own limit; reading it off the pipe would deep-copy the block tree per access -def generate(task_id, ui_state, - engine, model, - prompt, negative, styles, - width, height, frames, steps, - sampler_index, sampler_shift, dynamic_shift, - seed, guidance_scale, guidance_true, - init_image, init_strength, last_image, - vae_type, vae_tile_frames, - mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, - override_settings, - *args, **kwargs - ): +class VideoError(Exception): + """Video generation failure; code follows HTTP semantics so API callers can map it directly.""" + def __init__(self, msg: str, code: int = 500): + super().__init__(msg) + self.code = code - if engine is None or model is None or engine == 'None' or model == 'None': - return video_utils.queue_err('model not selected') - # videojob = shared.state.begin('Video') - found = [model.name for model in models_def.models.get(engine, [])] - selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None - if not shared.sd_loaded: - debug('Video: model not yet loaded') - video_load.load_model(selected) - if selected.name != video_load.loaded_model: - debug('Video: force reload') - video_load.load_model(selected) - if not shared.sd_loaded: - debug('Video: model still not loaded') - return video_utils.queue_err('model not loaded') - debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') + +@dataclass +class VideoResult: + images: list # PIL frames as produced; callers decide what to surface + video_path: str | None + thumb_path: str | None + num_frames: int + fps: float # effective save fps after interpolation + has_audio: bool + still: bool + processed: processing.Processed + + +def resolve_model(engine: str | None, model: str | None) -> tuple[models_def.Model, bool]: + """Return (selected, needs_load): a registry row when both names are given, or a synthesized + row describing the already-loaded pipeline when both are omitted.""" + engine_given = engine not in (None, '', 'None') + model_given = model not in (None, '', 'None') + if engine_given != model_given: + raise VideoError('video model selection requires both engine and model', 400) + if engine_given: + selected = models_def.find(engine, model) + if selected is None: + available = models_def.model_names(engine) or models_def.engines() + raise VideoError(f'video model not found: engine="{engine}" model="{model}" available={available}', 404) + return selected, True + cls = shared.sd_model.__class__.__name__ if shared.sd_loaded else None + if not shared.sd_loaded or cls not in models_def.pipeline_classes(): + raise VideoError(f'no video model loaded: cls={cls} select engine and model or load a video-capable checkpoint first', 400) + pipe = shared.sd_model + workflow = getattr(pipe, 'sdnext_video_workflow', None) + if workflow is None and video_modular.is_modular(pipe): + workflow = models_def.workflow_for_class(cls) or 'auto' # modular pipes dispatch on inputs, so any workflow marker selects the modular branch + ckpt = getattr(pipe, 'sd_checkpoint_info', None) + selected = models_def.Model( + name=getattr(ckpt, 'title', None) or cls, + repo=getattr(ckpt, 'name', None), + repo_cls=type(pipe), + workflow=workflow, + base=True, + ) + return selected, False + + +def validate_references(selected: models_def.Model, references: list | None, init_image) -> list | None: + """Return the ordered reference images for a reference workflow, None for every other model. + Reference conditioning is exclusive to ref2va: its partition holds no keyframe transformer, and + a mismatched request would only fail once the pipeline reached a component it never loaded. + Checks run before the model load so a rejected request costs nothing.""" + workflow = getattr(selected, 'workflow', None) + if workflow != 'ref2va': + if references: + raise VideoError(f'reference images require a ref2va model: model="{selected.name}" workflow={workflow}', 400) + return None + refs = list(references) if references else ([init_image] if init_image is not None else []) + if len(refs) == 0: + raise VideoError('No reference image provided. The ref2va workflow conditions on reference images, so at least one is required.', 400) + if len(refs) > MAX_IMAGE_REFERENCES: + raise VideoError(f'too many reference images: count={len(refs)} max={MAX_IMAGE_REFERENCES}', 400) + for index, image in enumerate(refs): + size = getattr(image, 'size', None) + if size is None or len(size) != 2: + raise VideoError(f'reference {index + 1} is not an image: type={type(image).__name__}', 400) + width, height = size + if width > 4 * height or height > 4 * width: # the same bound the pipeline enforces, raised here where it is free + raise VideoError(f'reference {index + 1} aspect ratio out of range: size={width}x{height} supported=1:4..4:1', 400) + return refs + + +def run(selected: models_def.Model, *, + prompt: str, + negative: str = '', + styles: list | None = None, + width: int = 832, + height: int = 480, + frames: int = 17, + steps: int = 50, + sampler_name: str = 'Default', + sampler_shift: float = -1.0, + dynamic_shift: bool = False, + seed: int = -1, + guidance_scale: float = -1.0, + guidance_true: float = -1.0, + init_image=None, + init_strength: float = 0.8, + last_image=None, + references: list | None = None, + vae_type: str = 'Default', + vae_tile_frames: int = 16, + audio: bool = True, + mp4_fps: int = 24, + mp4_interpolate: int = 0, + mp4_codec: str = 'libx264', + mp4_ext: str = 'mp4', + mp4_opt: str = 'crf:16', + mp4_video: bool = True, + mp4_frames: bool = False, + mp4_sf: bool = False, + mp4_thumb: bool = True, + override_settings=None, + engine: str | None = None, + ui_state=None, + scripts=None, + script_args=(), + per_script_args: dict | None = None, + extra_p: dict | None = None, + needs_load: bool = True, + ) -> VideoResult: + + refs = validate_references(selected, references, init_image) + + if needs_load: + if not shared.sd_loaded: + debug('Video: model not yet loaded') + video_load.load_model(selected) + if selected.name != video_load.loaded_model: + debug('Video: force reload') + video_load.load_model(selected) + if not shared.sd_loaded: + debug('Video: model still not loaded') + raise VideoError('model not loaded', 500) + + if isinstance(override_settings, (list, tuple)): # the ui override control emits "setting: value" pairs; always empty on the video tab since the control stays hidden + from modules.generation_parameters_copypaste import create_override_settings_dict + override_settings = create_override_settings_dict(override_settings) p = processing.StableDiffusionProcessingVideo( sd_model=shared.sd_model, - video_engine=engine, - video_model=model, + video_engine=engine or 'Loaded', + video_model=selected.name, prompt=prompt, negative_prompt=negative, - styles=styles, + styles=styles or [], seed=int(seed), - sampler_name = processing.get_sampler_name(sampler_index), + sampler_name=sampler_name, sampler_shift=float(sampler_shift), steps=int(steps), width=16 * int(width // 16), @@ -59,6 +164,7 @@ def generate(task_id, ui_state, cfg_true=float(guidance_true), vae_type=vae_type, vae_tile_frames=int(vae_tile_frames), + video_audio=bool(audio), override_settings=override_settings, ) if p.vae_type == 'Remote' and not selected.vae_remote: @@ -66,49 +172,74 @@ def generate(task_id, ui_state, p.vae_type = 'Default' p.state = ui_state - p.scripts = scripts_manager.scripts_video - p.script_args = args - processed: processing.Processed = scripts_manager.scripts_video.run(p, *args) + if per_script_args: + p.per_script_args.update(per_script_args) + for k, v in (extra_p or {}).items(): + setattr(p, k, v) + p.scripts = scripts if scripts is not None else scripts_manager.scripts_video + p.script_args = tuple(script_args) + p.scripts.run(p, *script_args) p.do_not_save_grid = True p.do_not_save_samples = not mp4_frames p.outpath_samples = resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_video) - if 'T2V' in model: + if getattr(selected, 'workflow', None) is not None: + # modular workflows dispatch on which inputs are present; keyframes pass through + # unresized since the pipeline defines its own canvas placement per anchor + p.video_still = int(frames) <= 1 + if refs is not None: + from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3ImageReference + # references outrank the keyframe inputs in every block, so those stay unset; the reference + # encoder reads the image array as (height, width, 3) and never converts it itself + p.task_args['references'] = [MiniMaxH3ImageReference(image=image.convert('RGB')) for image in refs] + if last_image is not None: + log.warning(f'Video: op=reference model="{selected.name}" last frame not supported, ignoring') + else: + if init_image is not None: + p.task_args['image'] = init_image + if last_image is not None: + p.task_args['last_image'] = last_image + if p.video_still: + p.do_not_save_samples = False # the still is the product; save it like an image result + elif int(mp4_fps) != 24: + log.warning(f'Video: model="{selected.name}" fps={mp4_fps} model output is fixed at 24') + log.debug(f'Video: op=modular workflow={selected.workflow} still={p.video_still} init={init_image} last={last_image} references={len(refs) if refs else 0}') + elif 'T2V' in selected.name: if init_image is not None: log.warning('Video: op=T2V init image not supported') - elif 'I2V' in model: + elif 'I2V' in selected.name: if init_image is None: - return video_utils.queue_err('No input image provided. Please upload or select an image.') + raise VideoError('No input image provided. Please upload or select an image.', 400) p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') if last_image is not None and video_utils.supports_last_frame(shared.sd_model): p.task_args['last_image'] = images.resize_image(resize_mode=2, im=last_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') log.debug(f'Video: op=FLF2V init={init_image} last={last_image} resized={p.task_args["image"]}') elif last_image is not None: - log.warning(f'Video: op=I2V model="{model}" last frame not supported, ignoring') + log.warning(f'Video: op=I2V model="{selected.name}" last frame not supported, ignoring') else: log.debug(f'Video: op=I2V init={init_image} resized={p.task_args["image"]}') - elif 'FLF2V' in model: + elif 'FLF2V' in selected.name: if init_image is None: - return video_utils.queue_err('No input image provided. Please upload or select an image.') + raise VideoError('No input image provided. Please upload or select an image.', 400) if last_image is None: - return video_utils.queue_err('No last frame image provided. Please upload or select an image.') + raise VideoError('No last frame image provided. Please upload or select an image.', 400) p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') p.task_args['last_image'] = images.resize_image(resize_mode=2, im=last_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') log.debug(f'Video: op=FLF2V init={init_image} last={last_image} resized={p.task_args["image"]}') - elif 'VACE' in model: + elif 'VACE' in selected.name: if init_image is not None: p.task_args['reference_images'] = [images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')] log.debug(f'Video: op=VACE reference={init_image} resized={p.task_args["reference_images"]}') - elif 'Animate' in model: + elif 'Animate' in selected.name: if init_image is None: - return video_utils.queue_err('No input image provided. Please upload or select an image.') + raise VideoError('No input image provided. Please upload or select an image.', 400) p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') p.task_args['mode'] = 'animate' p.task_args['pose_video'] = [] # input pose video to condition the generation on. must be a list of PIL images. p.task_args['face_video'] = [] # input face video to condition the generation on. must be a list of PIL images. log.debug(f'Video: op=Animate init={p.task_args["image"]} pose={p.task_args["pose_video"]} face={p.task_args["face_video"]}') else: - log.warning(f'Video: unknown model type "{model}"') + log.warning(f'Video: unknown model type "{selected.name}"') # cleanup memory shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) @@ -162,19 +293,23 @@ def generate(task_id, ui_state, # done if err: - return video_utils.queue_err(err) + raise VideoError(err, 500) if processed is None or (len(processed.images) == 0 and processed.bytes is None): - return video_utils.queue_err('processing failed') + raise VideoError('processing failed', 500) log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') + if getattr(p, 'video_still', False): + stills = processed.images[:1] # already trimmed in process_decode; defensive + return VideoResult(images=stills, video_path=None, thumb_path=None, num_frames=len(stills), fps=0.0, has_audio=False, still=True, processed=processed) + if hasattr(processed, 'images') and processed.images is not None: pixels = video_save.images_to_tensor(processed.images) else: pixels = None if hasattr(processed, 'audio') and processed.audio is not None: - audio = processed.audio[0].float().cpu() + waveform = processed.audio[0].float().cpu() else: - audio = None + waveform = None if mp4_interpolate > 0 and pixels is not None: p.video_interpolate = mp4_interpolate @@ -187,10 +322,11 @@ def generate(task_id, ui_state, pixels = x.permute(1, 0, 2, 3).unsqueeze(0) from modules.processing_video import interpolation_factor save_fps = mp4_fps * interpolation_factor(p) - _num_frames, video_file, _thumb = video_save.save_video( + num_frames, video_file, thumb_file = video_save.save_video( p=p, pixels=pixels, - audio=audio, + audio=waveform, + aac_sample_rate=getattr(p, 'audio_sampling_rate', None) or 24000, binary=processed.bytes, mp4_fps=save_fps, mp4_codec=mp4_codec, @@ -203,9 +339,68 @@ def generate(task_id, ui_state, mp4_interpolate=mp4_interpolate, metadata={}, ) - if not mp4_frames: - processed.images = [] + return VideoResult(images=processed.images, video_path=video_file, thumb_path=thumb_file, num_frames=num_frames, fps=float(save_fps), has_audio=waveform is not None, still=False, processed=processed) - generation_info_js = processed.js() if processed is not None else '' - # shared.state.end(videojob) - return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) + +def generate(task_id, ui_state, + engine, model, + prompt, negative, styles, + width, height, frames, steps, + sampler_index, sampler_shift, dynamic_shift, + seed, guidance_scale, guidance_true, + init_image, init_strength, last_image, + vae_type, vae_tile_frames, audio, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, + override_settings, + *args, **kwargs + ): + # gradio adapter around run(): the positional signature is frozen since external callers bind to it + if engine is None or model is None or engine == 'None' or model == 'None': + return video_utils.queue_err('model not selected') + selected = models_def.find(engine, model) + if selected is None: + return video_utils.queue_err(f'model not found: engine="{engine}" model="{model}"') + debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') + try: + res = run(selected, + prompt=prompt, + negative=negative, + styles=styles, + width=width, + height=height, + frames=frames, + steps=steps, + sampler_name=processing.get_sampler_name(sampler_index), + sampler_shift=sampler_shift, + dynamic_shift=dynamic_shift, + seed=seed, + guidance_scale=guidance_scale, + guidance_true=guidance_true, + init_image=init_image, + init_strength=init_strength, + last_image=last_image, + vae_type=vae_type, + vae_tile_frames=vae_tile_frames, + audio=audio, + mp4_fps=mp4_fps, + mp4_interpolate=mp4_interpolate, + mp4_codec=mp4_codec, + mp4_ext=mp4_ext, + mp4_opt=mp4_opt, + mp4_video=mp4_video, + mp4_frames=mp4_frames, + mp4_sf=mp4_sf, + mp4_thumb=mp4_thumb, + override_settings=override_settings, + engine=engine, + ui_state=ui_state, + script_args=args, + ) + except VideoError as e: + return video_utils.queue_err(str(e)) + generation_info_js = res.processed.js() + html_log = ui_common.plaintext_to_html(res.processed.comments) + if res.still: + return res.images, None, generation_info_js, res.processed.info, html_log + result_images = res.images if mp4_frames else [] + return result_images, res.video_path, generation_info_js, res.processed.info, html_log diff --git a/modules/video_models/video_ui.py b/modules/video_models/video_ui.py index 4bcc5b618..e3bdf2a78 100644 --- a/modules/video_models/video_ui.py +++ b/modules/video_models/video_ui.py @@ -58,34 +58,6 @@ def model_load(engine, model): return msg -def run_video(*args): - engine, model = args[2], args[3] - debug(f'Video run: engine="{engine}" model="{model}"') - selected = get_selected(engine, model) - if not selected or engine is None or model is None or engine == 'None' or model == 'None': - return video_utils.queue_err('model not selected') - debug(f'Video run: {str(selected)}') - if selected and 'Hunyuan' in selected.name: - return video_run.generate(*args) - elif selected and 'LTX' in selected.name: - return video_run.generate(*args) - elif selected and 'Mochi' in selected.name: - return video_run.generate(*args) - elif selected and 'Cog' in selected.name: - return video_run.generate(*args) - elif selected and 'Allegro' in selected.name: - return video_run.generate(*args) - elif selected and 'WAN' in selected.name: - return video_run.generate(*args) - elif selected and 'Latte' in selected.name: - return video_run.generate(*args) - elif selected and 'anisora' in selected.name.lower(): - return video_run.generate(*args) - elif selected and 'Kandinsky' in selected.name: - return video_run.generate(*args) - return video_utils.queue_err(f'model not found: engine="{engine}" model="{model}"') - - def create_ui_outputs(): with gr.Row(): with gr.Column(variant='compact', elem_id="video_outputs", elem_classes=['settings-column'], scale=1): @@ -120,6 +92,7 @@ def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_i with gr.Row(): sampler_shift = gr.Slider(label='Sampler shift', minimum=-1.0, maximum=20.0, step=0.1, value=-1.0, elem_id="video_scheduler_shift") dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift") + audio = gr.Checkbox(label='Audio', value=True, elem_id="video_audio") with gr.Row(): guidance_scale = gr.Slider(label='Guidance scale', minimum=-1.0, maximum=14.0, step=0.1, value=-1.0, elem_id="video_guidance_scale") guidance_true = gr.Slider(label='True guidance', minimum=-1.0, maximum=14.0, step=0.1, value=-1.0, elem_id="video_guidance_true") @@ -170,7 +143,7 @@ def create_ui(prompt, negative, styles, overrides, script_inputs, mp4_fps, mp4_i seed, guidance_scale, guidance_true, init_image, init_strength, last_image, - vae_type, vae_tile_frames, + vae_type, vae_tile_frames, audio, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, overrides, ] diff --git a/pipelines/model_minimax.py b/pipelines/model_minimax.py new file mode 100644 index 000000000..bba966856 --- /dev/null +++ b/pipelines/model_minimax.py @@ -0,0 +1,32 @@ +import diffusers +from modules import shared, devices, sd_models +from modules.logger import log + + +def load_minimax(checkpoint_info, diffusers_load_config=None): # pylint: disable=unused-argument + from modules.video_models import video_modular, video_load + repo_id = sd_models.path_to_repo(checkpoint_info) + sd_models.hf_auth_check(checkpoint_info) + if repo_id is None or repo_id.lower() == 'none': + return None + offline_args = {'local_files_only': True} if shared.opts.offline_mode else {} + workflow = (getattr(checkpoint_info, 'subfolder', None) or 'fl2va').lower() # one repo holds both checkpoint partitions; reference entries select ref2va via the subfolder tag + log.debug(f'Load model: type=MiniMaxH3 repo="{repo_id}" workflow={workflow} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype}') + + pipe = video_modular.load_modular_pipe( + getattr(diffusers, 'MiniMaxH3ModularPipeline', None), + repo_id, + workflow=workflow, + offline_args=offline_args, + base=True, + ) + if pipe is None: + return None + + video_modular.install_state_hook(pipe) + video_load.loaded_model = None # image-path load invalidates the video tab's name cache + if hasattr(pipe, 'vae') and hasattr(pipe.vae, 'enable_tiling'): + pipe.vae.enable_tiling() + + devices.torch_gc() + return pipe diff --git a/test/full-test.sh b/test/full-test.sh index bf129f336..5813b16f3 100644 --- a/test/full-test.sh +++ b/test/full-test.sh @@ -30,3 +30,7 @@ echo control-preprocess python cli/api-preprocess.py --input ui/assets/logo-bg-0.jpg --model "Zoe Depth" echo control-controlnet python cli/api-control.py --prompt "cute robot" --input ui/assets/logo-bg-0.jpg --type controlnet --control "Zoe Depth:Xinsir Union XL:0.5" +echo video-models +python cli/api-video.py --list +echo video +python cli/api-video.py --prompt "a paper boat drifting down a rain gutter" --frames 17 --steps 8 --output /tmp/sdnext-test-video.mp4 diff --git a/test/test-video-api.py b/test/test-video-api.py new file mode 100644 index 000000000..040cf0523 --- /dev/null +++ b/test/test-video-api.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python +""" +API tests for video generation. + +Tests: +- GET /sdapi/v1/video/models — engine/model enumeration and mode derivation +- POST /sdapi/v1/video — request validation errors (partial pair, unknown model/sampler, checkpoint override, unknown script) +- POST /sdapi/v1/video — reference rules (wrong workflow, missing, over limit, aspect) +- POST /sdapi/v1/video — still mode (frames=1) against the currently loaded model +- POST /sdapi/v1/video — video generation against the currently loaded model +- POST /sdapi/v1/video — wire switches and GET /sdapi/v1/video/file serving + +Requires a running SD.Next instance. Generation categories require a video-capable +model loaded (for example MiniMax-H3 via the base checkpoint dropdown) and are +skipped otherwise; enumeration and validation run against any instance. A loaded +model that conditions on references is detected by the still probe, and every +later request against it carries one. + +Usage: + python test/test-video-api.py [--url URL] [--steps STEPS] [--frames FRAMES] +""" + +import os +import sys +import base64 +import struct +import time +import zlib +import argparse +import requests +import urllib3 + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +VALID_MODES = {'workflow', 't2v', 'i2v', 'flf2v', 'vace', 'animate'} + + +def png_b64(width: int, height: int) -> str: + """Minimal grey RGB PNG, so reference tests need no image library.""" + def chunk(tag: bytes, payload: bytes) -> bytes: + return struct.pack('>I', len(payload)) + tag + payload + struct.pack('>I', zlib.crc32(tag + payload) & 0xffffffff) + scanlines = b''.join(b'\x00' + b'\x7f\x7f\x7f' * width for _ in range(height)) + header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + data = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(scanlines)) + chunk(b'IEND', b'') + return base64.b64encode(data).decode() + + +class VideoAPITest: + """Test harness for the video generation API.""" + + def __init__(self, base_url, steps=8, frames=17, timeout=3600): + self.base_url = base_url.rstrip('/') + self.steps = steps + self.frames = frames + self.timeout = timeout + self.video_capable = None # set by the still-mode probe + self.ref2va = False # set by the same probe when the loaded model conditions on references + self.results = { + 'enumerate': {'passed': 0, 'failed': 0, 'skipped': 0, 'tests': []}, + 'validation': {'passed': 0, 'failed': 0, 'skipped': 0, 'tests': []}, + 'still': {'passed': 0, 'failed': 0, 'skipped': 0, 'tests': []}, + 'generation': {'passed': 0, 'failed': 0, 'skipped': 0, 'tests': []}, + 'wire': {'passed': 0, 'failed': 0, 'skipped': 0, 'tests': []}, + } + self._category = 'enumerate' + + def _get(self, endpoint, params=None): + try: + r = requests.get(f'{self.base_url}{endpoint}', params=params, timeout=self.timeout, verify=False) + if r.status_code != 200: + res = {'error': r.status_code, 'reason': r.reason} + try: + res['detail'] = r.json().get('detail', None) + except Exception: + pass + return res + return r.json() + except requests.exceptions.ConnectionError: + return {'error': 'connection_refused', 'reason': 'Server not running'} + except Exception as e: + return {'error': 'exception', 'reason': str(e)} + + def _post(self, endpoint, data): + try: + r = requests.post(f'{self.base_url}{endpoint}', json=data, timeout=self.timeout, verify=False) + if r.status_code != 200: + res = {'error': r.status_code, 'reason': r.reason} + try: + res['detail'] = r.json().get('detail', None) + except Exception: + pass + return res + return r.json() + except requests.exceptions.ConnectionError: + return {'error': 'connection_refused', 'reason': 'Server not running'} + except Exception as e: + return {'error': 'exception', 'reason': str(e)} + + def record(self, passed, name, detail=''): + status = 'PASS' if passed else 'FAIL' + self.results[self._category]['passed' if passed else 'failed'] += 1 + self.results[self._category]['tests'].append((status, name)) + msg = f' {status}: {name}' + if detail: + msg += f' ({detail})' + print(msg) + + def skip(self, name, reason): + self.results[self._category]['skipped'] += 1 + self.results[self._category]['tests'].append(('SKIP', name)) + print(f' SKIP: {name} ({reason})') + + def _video(self, extra_params=None, prompt='a red fox in the snow'): + payload = { + 'prompt': prompt, + 'steps': self.steps, + 'frames': self.frames, + 'width': 640, + 'height': 384, + 'seed': 42, + } + if extra_params: + payload.update(extra_params) + if self.ref2va and 'references' not in payload and 'engine' not in payload: + payload['references'] = [png_b64(64, 64)] # requests aimed at the loaded model carry a reference when that model needs one + t0 = time.time() + data = self._post('/sdapi/v1/video', payload) + return data, time.time() - t0 + + # ========================================================================= + # Tests: Enumeration + # ========================================================================= + + def test_enumerate(self): + self._category = 'enumerate' + print("\n--- Enumeration Tests ---") + data = self._get('/sdapi/v1/video/models') + if isinstance(data, dict) and 'error' in data: + self.record(False, 'models_list', f'error: {data}') + return [] + self.record(len(data) > 0, 'models_list', f'{len(data)} models') + bad_modes = [item['name'] for item in data if item.get('mode') not in VALID_MODES] + self.record(len(bad_modes) == 0, 'models_modes', 'all valid' if not bad_modes else f'invalid: {bad_modes}') + minimax = [item for item in data if item['engine'] == 'MiniMax'] + if minimax: + self.record(all(item['base'] for item in minimax), 'models_minimax_base', f'{len(minimax)} rows') + self.record(all(item['mode'] == 'workflow' for item in minimax), 'models_minimax_workflow') + else: + self.skip('models_minimax', 'no MiniMax rows in registry') + filtered = self._get('/sdapi/v1/video/models', params={'engine': 'MiniMax'}) + if isinstance(filtered, list): + self.record(all(item['engine'] == 'MiniMax' for item in filtered), 'models_engine_filter', f'{len(filtered)} rows') + else: + self.record(False, 'models_engine_filter', f'error: {filtered}') + return data + + # ========================================================================= + # Tests: Validation + # ========================================================================= + + def test_validation(self, models): + self._category = 'validation' + print("\n--- Validation Tests ---") + data, _elapsed = self._video({'engine': 'MiniMax'}) + self.record(data.get('error') == 400, 'partial_pair_rejected', f'code={data.get("error")}') + data, _elapsed = self._video({'engine': 'NoSuchEngine', 'model': 'NoSuchModel'}) + self.record(data.get('error') == 404, 'unknown_model_rejected', f'code={data.get("error")} detail={data.get("detail")}') + # a valid registry pair fails on the sampler before any model load happens + if models: + pair = {'engine': models[0]['engine'], 'model': models[0]['name']} + data, _elapsed = self._video({**pair, 'sampler_name': 'NoSuchSampler'}) + self.record(data.get('error') == 404, 'unknown_sampler_rejected', f'code={data.get("error")}') + data, _elapsed = self._video({**pair, 'override_settings': {'sd_model_checkpoint': 'other-model'}}) + self.record(data.get('error') == 400, 'checkpoint_override_rejected', f'code={data.get("error")}') + data, _elapsed = self._video({**pair, 'alwayson_scripts': {'no-such-script': {'args': []}}}) + self.record(data.get('error') == 422, 'unknown_script_rejected', f'code={data.get("error")}') + else: + self.skip('unknown_sampler_rejected', 'no registry models to pair with') + self.check_references(models) + + def check_references(self, models): + # every reference rule is checked before the model load, so these stay fast on a cold registry row + keyframe = next((m for m in models if m.get('workflow') not in (None, 'ref2va')), None) + reference = next((m for m in models if m.get('workflow') == 'ref2va'), None) + if keyframe: + pair = {'engine': keyframe['engine'], 'model': keyframe['name']} + data, elapsed = self._video({**pair, 'references': [png_b64(64, 64)]}) + self.record(data.get('error') == 400, 'references_wrong_workflow_rejected', f'code={data.get("error")} time={elapsed:.2f}s') + else: + self.skip('references_wrong_workflow_rejected', 'no keyframe workflow model in registry') + if not reference: + for name in ('references_required', 'references_over_limit', 'references_aspect_rejected'): + self.skip(name, 'no ref2va model in registry') + return + pair = {'engine': reference['engine'], 'model': reference['name']} + data, elapsed = self._video(pair) + self.record(data.get('error') == 400, 'references_required', f'code={data.get("error")} time={elapsed:.2f}s') + data, elapsed = self._video({**pair, 'references': [png_b64(64, 64)] * 10}) + self.record(data.get('error') == 400, 'references_over_limit', f'code={data.get("error")} time={elapsed:.2f}s') + data, elapsed = self._video({**pair, 'references': [png_b64(8, 64)]}) + self.record(data.get('error') == 400, 'references_aspect_rejected', f'code={data.get("error")} time={elapsed:.2f}s') + + # ========================================================================= + # Tests: Still mode (doubles as the video-capability probe) + # ========================================================================= + + def test_still(self): + self._category = 'still' + print("\n--- Still Mode Tests ---") + data, elapsed = self._video({'frames': 1}) + if data.get('error') == 400 and 'ref2va' in str(data.get('detail', '')): + self.ref2va = True # the loaded model conditions on references; every later request carries one + data, elapsed = self._video({'frames': 1}) + if data.get('error') == 400: + self.video_capable = False + self.skip('still_generation', f'no video-capable model loaded: {data.get("detail")}') + return + if 'error' in data: + self.video_capable = False + self.record(False, 'still_generation', f'error: {data}') + return + self.video_capable = True + self.record(data.get('still') is True, 'still_flag', f'time={elapsed:.1f}s') + self.record(data.get('video') is None, 'still_no_video') + self.record(len(data.get('frames') or []) == 1, 'still_single_frame', f'frames={len(data.get("frames") or [])}') + + # ========================================================================= + # Tests: Generation with the loaded model + # ========================================================================= + + def test_generation(self): + self._category = 'generation' + print("\n--- Generation Tests ---") + if not self.video_capable: + self.skip('video_generation', 'no video-capable model loaded') + return None + data, elapsed = self._video() + if 'error' in data: + self.record(False, 'video_generation', f'error: {data}') + return None + self.record(data.get('frames_count', 0) > 0, 'video_frames_count', f'frames={data.get("frames_count")} time={elapsed:.1f}s') + video_b64 = data.get('video') + decoded = len(base64.b64decode(video_b64)) if video_b64 else 0 + self.record(decoded > 1000, 'video_payload', f'bytes={decoded}') + self.record(data.get('fps', 0) > 0 and data.get('duration', 0) > 0, 'video_timing', f'fps={data.get("fps")} duration={data.get("duration")}') + self.record(isinstance(data.get('has_audio'), bool), 'video_audio_flag', f'has_audio={data.get("has_audio")}') + self.record(bool(data.get('info')), 'video_info') + return data + + # ========================================================================= + # Tests: Wire switches and file serving + # ========================================================================= + + def test_wire(self): + self._category = 'wire' + print("\n--- Wire Tests ---") + if not self.video_capable: + self.skip('wire_all', 'no video-capable model loaded') + return + data, _elapsed = self._video({'send_video': False, 'send_thumbnail': False}) + if 'error' in data: + self.record(False, 'wire_send_video_off', f'error: {data}') + return + self.record(data.get('video') is None, 'wire_send_video_off') + path = data.get('video_path') + self.record(bool(path), 'wire_video_path', f'path={path}') + if path: + r = requests.get(f'{self.base_url}/sdapi/v1/video/file', params={'file': path}, timeout=300, verify=False) + ctype = r.headers.get('content-type', '') + self.record(r.status_code == 200 and ctype.startswith('video/'), 'wire_file_endpoint', f'code={r.status_code} type={ctype} bytes={len(r.content)}') + r = requests.get(f'{self.base_url}/sdapi/v1/video/file', params={'file': '/etc/passwd'}, timeout=60, verify=False) + self.record(r.status_code == 403, 'wire_file_jail', f'code={r.status_code}') + + # ========================================================================= + # Runner + # ========================================================================= + + def run_all(self): + print("=" * 60) + print("Video API Test Suite") + print(f"Server: {self.base_url}") + print(f"Steps: {self.steps} Frames: {self.frames}") + print("=" * 60) + + models = self.test_enumerate() + self.test_validation(models) + self.test_still() + self.test_generation() + self.test_wire() + + print("\n" + "=" * 60) + print("Results") + print("=" * 60) + total_passed = 0 + total_failed = 0 + total_skipped = 0 + for cat, data in self.results.items(): + total_passed += data['passed'] + total_failed += data['failed'] + total_skipped += data['skipped'] + status = 'PASS' if data['failed'] == 0 else 'FAIL' + print(f" {cat}: {data['passed']} passed, {data['failed']} failed, {data['skipped']} skipped [{status}]") + print(f" Total: {total_passed} passed, {total_failed} failed, {total_skipped} skipped") + print("=" * 60) + return total_failed == 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Video API Tests (enumeration, validation, generation, file serving)') + parser.add_argument('--url', default=os.environ.get('SDAPI_URL', 'http://127.0.0.1:7860'), help='server URL') + parser.add_argument('--steps', type=int, default=8, help='generation steps (lower = faster tests)') + parser.add_argument('--frames', type=int, default=17, help='frame count for video tests') + args = parser.parse_args() + test = VideoAPITest(args.url, args.steps, args.frames) + success = test.run_all() + sys.exit(0 if success else 1) diff --git a/ui/locale/locale_en.json b/ui/locale/locale_en.json index c281da8fd..532f1c81c 100644 --- a/ui/locale/locale_en.json +++ b/ui/locale/locale_en.json @@ -60,6 +60,8 @@ ], "a": [ {"id":"","label":"Active dictionaries","localized":"","hint":"Select which tag dictionaries are used for prompt autocompletion.
Dictionaries not yet downloaded locally will be fetched automatically when the autocomplete engine loads them.","ui":"script_autocomplete"}, + {"id":"video_audio","label":"Audio","localized":"","hint":"Generate synchronized audio for video models with audio support
When disabled, audio decode and muxing are skipped and the audio component stays off the device","ui":"video"}, + {"id":"ltx_audio_accordion","label":"Audio","localized":"","hint":"Audio track settings for video models with audio support","ui":"video"}, {"id":"txt2img_advanced","label":"Advanced","localized":"","hint":"Advanced settings used to run image generation","ui":"txt2img"}, {"id":"txt2img_adapters","label":"Adapters","localized":"","hint":"Settings related to IP Adapters","ui":"txt2img"}, {"id":"component-981","label":"Apply to model","localized":"","hint":"","ui":"script_layerdiffuse"}, @@ -485,7 +487,7 @@ {"id":"","label":"Fixed","localized":"","hint":"Resize image to target resolution. Unless height and width match, you will get incorrect aspect ratio","ui":"txt2img"}, {"id":"","label":"Folder","localized":"","hint":"","ui":"control"}, {"id":"video_params_framepack","label":"FramePack","localized":"","hint":"","ui":"video"}, - {"id":"","label":"Frames","localized":"","hint":"","ui":"video"}, + {"id":"video_frames","label":"Frames","localized":"","hint":"Number of frames to generate
Values are aligned to the frame grid of the selected model
On MiniMax H3, a value of 1 generates a single still image (experimental)","ui":"video"}, {"id":"","label":"Fallback guidance","localized":"","hint":"","ui":"txt2img"}, {"id":"","label":"FreeU","localized":"","hint":"","ui":"settings_advanced"}, {"id":"","label":"Faster Cache","localized":"","hint":"","ui":"settings_advanced"}, diff --git a/ui/progressBar.ts b/ui/progressBar.ts index a2e187ad6..9ed8293e6 100644 --- a/ui/progressBar.ts +++ b/ui/progressBar.ts @@ -42,7 +42,7 @@ export function checkPaused(state) { export function setProgress(res?: any) { const elements = ['txt2img_generate', 'img2img_generate', 'extras_generate', 'control_generate', 'video_generate', 'framepack_generate']; const progress = res?.progress || 0; - const job = res?.job || ''; + const job = res?.textinfo || res?.job || ''; // stage label when the backend reports one, job name otherwise let perc: string; let eta = ''; if (job === 'VAE') perc = 'Decode'; @@ -62,7 +62,7 @@ export function setProgress(res?: any) { const elPerf = document.getElementById('control-performance'); let hint = ''; if (elPerf && res) { - const jobTxt = res.job && res.job !== '' ? ` | Job ${res.job}` : ''; + const jobTxt = res.job && res.job !== '' ? ` | Job ${res.job}${res.textinfo ? `: ${res.textinfo}` : ''}` : ''; const batchTxt = res.batch > 0 ? ` | Batch ${res.batch}/${res.batches}` : ''; const stateTxt = res.queued ? 'Queued' : res.paused ? 'Paused' : res.completed ? 'Completed' : res.active ? 'Active' : 'Idle'; // eslint-disable-line no-nested-ternary const stepsTxt = res.step > 0 ? ` | Step ${res.step}/${res.steps}` : ''; @@ -92,7 +92,7 @@ export function setProgress(res?: any) { } const el = document.getElementById('control-performance'); if (el && res) { - const jobTxt = res.job && res.job !== '' ? ` | Job ${res.job}` : ''; + const jobTxt = res.job && res.job !== '' ? ` | Job ${res.job}${res.textinfo ? `: ${res.textinfo}` : ''}` : ''; const batchTxt = res.batch > 0 ? ` | Batch ${res.batch}/${res.batches}` : ''; const stateTxt = res.queued ? 'Queued' : res.paused ? 'Paused' : res.completed ? 'Completed' : res.active ? 'Active' : 'Idle'; // eslint-disable-line no-nested-ternary const stepsTxt = res.step > 0 ? ` | Step ${res.step}/${res.steps}` : '';