diff --git a/cli/api-video.py b/cli/api-video.py index adc354a0d..33c6451fb 100644 --- a/cli/api-video.py +++ b/cli/api-video.py @@ -114,6 +114,8 @@ def generate(args): # pylint: disable=redefined-outer-name options['init_image'] = encode(args.init) if args.last: options['last_image'] = encode(args.last) + if args.reference: + options['references'] = [encode(f) for f in args.reference] stop_event = threading.Event() if args.progress: threading.Thread(target=watch_progress, args=(stop_event,), daemon=True).start() @@ -146,6 +148,7 @@ if __name__ == "__main__": parser.add_argument('--audio', action=argparse.BooleanOptionalAction, default=True, help='generate audio on supported models') parser.add_argument('--init', required=False, default=None, help='init image file') parser.add_argument('--last', required=False, default=None, help='last frame image file') + parser.add_argument('--reference', required=False, default=None, action='append', help='reference image file for reference workflows; repeat in the order the model should read them') parser.add_argument('--output', required=False, default=None, help='output video file') parser.add_argument('--progress', action='store_true', help='poll and log progress during generation') parser.add_argument('--timeout', required=False, default=3600, help='request timeout in seconds') diff --git a/data/reference-base.json b/data/reference-base.json index 94ce9cc51..9ab483474 100644 --- a/data/reference-base.json +++ b/data/reference-base.json @@ -382,6 +382,15 @@ "size": 134, "date": "2026 August" }, + "MiniMaxAI MiniMax-H3 Ref2VA": { + "path": "MiniMaxAI/MiniMax-H3", + "subfolder": "ref2va", + "preview": "MiniMaxAI--MiniMax-H3.jpg", + "desc": "The omni-reference variant of MiniMax-H3, sharing one repository with the base model as a separate checkpoint partition. Video with synchronized stereo audio is conditioned on reference images for identity and appearance, with reference rows held clean while video rows denoise.", + "extras": "sampler: Default", + "size": 134, + "date": "2026 August" + }, "Freepik F-Lite": { "path": "Freepik/F-Lite", "preview": "Freepik--F-Lite.jpg", diff --git a/modules/api/video.py b/modules/api/video.py index 414951900..d415d45f7 100644 --- a/modules/api/video.py +++ b/modules/api/video.py @@ -30,6 +30,7 @@ class ReqVideo(BaseModel): init_image: str | None = Field(default=None, title="Init image", description="Base64, data URI, or upload reference for the first-frame image") init_strength: float = Field(default=0.8, ge=0.0, le=1.0, title="Init strength", description="Denoising strength for the init image") last_image: str | None = Field(default=None, title="Last image", description="Base64, data URI, or upload reference for the last-frame image") + references: list[str] = Field(default=[], title="References", description="Reference images for a reference workflow, in the order the model reads them; base64, data URIs, or upload references. At most 9, each within a 1:4 to 4:1 aspect ratio. Rejected on models that do not condition on references") vae_type: str = Field(default="Default", title="VAE type", description="Decode variant: Default, Tiny, Remote, or Upscale") vae_tile_frames: int = Field(default=16, ge=1, le=64, title="VAE tile frames", description="Frames per VAE decode tile") audio: bool = Field(default=True, title="Audio", description="Generate audio on models that support it") @@ -72,7 +73,7 @@ class ItemVideoModel(BaseModel): repo: str = Field(default="", title="Repo", description="Model repository or path") url: str = Field(default="", title="URL", description="Model information page") mode: str = Field(title="Mode", description="Input mode: workflow, t2v, i2v, flf2v, vace, or animate") - workflow: str | None = Field(default=None, title="Workflow", description="Modular workflow name when the model dispatches on inputs") + workflow: str | None = Field(default=None, title="Workflow", description="Modular workflow name when the model dispatches on inputs; ref2va conditions on references and ignores the keyframe images") base: bool = Field(default=False, title="Base", description="Also listed in the base checkpoint dropdown") loaded: bool = Field(default=False, title="Loaded", description="Currently loaded through the video registry") @@ -118,6 +119,8 @@ class APIVideo: val = getattr(req, name, None) if isinstance(val, str) and len(val) >= 1000: setattr(req, name, f"") + if req.references: + sanitize_str(req.references) if req.script_args: sanitize_str(req.script_args) if req.alwayson_scripts: @@ -140,6 +143,11 @@ class APIVideo: `video_path` set; fetch those via `GET /sdapi/v1/video/file`. `init_image` and `last_image` accept base64 data, data URIs, or upload references. + Models whose workflow is `ref2va` condition on `references` instead: an ordered list of + images the prompt addresses as ``, `` and so on, following list + order. A single reference may also be passed as `init_image`. Reference images do not + set the output canvas, and `last_image` is ignored. + Progress is reported on `GET /sdapi/v1/progress`; `POST /sdapi/v1/interrupt` cancels. Switching checkpoints via `override_settings` is not supported here; use `POST /sdapi/v1/checkpoint` before generating. @@ -151,6 +159,7 @@ class APIVideo: sampler_name = helpers.validate_sampler_name(req.sampler_name) init_image = helpers.decode_base64_to_image(req.init_image) if req.init_image else None last_image = helpers.decode_base64_to_image(req.last_image) if req.last_image else None + references = [helpers.decode_base64_to_image(x) for x in (req.references or [])] overrides = dict(req.override_settings or {}) for key in ('sd_model_checkpoint', 'sd_model_refiner'): if key in overrides: @@ -180,6 +189,7 @@ class APIVideo: init_image=init_image, init_strength=req.init_strength, last_image=last_image, + references=references, vae_type=req.vae_type, vae_tile_frames=req.vae_tile_frames, audio=req.audio, diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index ba16e0e02..1e10e56c5 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -574,6 +574,11 @@ def validate_pipeline(p: processing.StableDiffusionProcessing): elif not is_video_model and is_video_pipeline: log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} non-video model with video pipeline') return False + if getattr(shared.sd_model, 'sdnext_video_workflow', None) == 'ref2va' and p.task_args.get('references', None) is None: + # the reference workflow loads its own transformer partition alone: without references the pipeline + # dispatches to the keyframe path and reaches a transformer that was never loaded + log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} reference workflow requires reference images: use the video tab or the video api') + return False return True diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index db7b1d42e..795f228be 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -658,6 +658,18 @@ try: image_hijack=False, vae_hijack=False, vae_remote=False), + Model(name='MiniMax H3 Ref2VA', + url='https://huggingface.co/MiniMaxAI/MiniMax-H3', + repo='MiniMaxAI/MiniMax-H3', + repo_cls='MiniMaxH3ModularPipeline', + workflow='ref2va', + base=True, + te_cls=None, + dit_cls=None, + te_hijack=False, + image_hijack=False, + vae_hijack=False, + vae_remote=False), ], 'Google Veo': [ Model(name='Google Veo 3.1 T2V', diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 3c1e10d35..92296b4b5 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -9,6 +9,7 @@ from modules.paths import resolve_output_path debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None +MAX_IMAGE_REFERENCES = 9 # mirrors the reference setup block's own limit; reading it off the pipe would deep-copy the block tree per access class VideoError(Exception): @@ -61,6 +62,31 @@ def resolve_model(engine: str | None, model: str | None) -> tuple[models_def.Mod return selected, False +def validate_references(selected: models_def.Model, references: list | None, init_image) -> list | None: + """Return the ordered reference images for a reference workflow, None for every other model. + Reference conditioning is exclusive to ref2va: its partition holds no keyframe transformer, and + a mismatched request would only fail once the pipeline reached a component it never loaded. + Checks run before the model load so a rejected request costs nothing.""" + workflow = getattr(selected, 'workflow', None) + if workflow != 'ref2va': + if references: + raise VideoError(f'reference images require a ref2va model: model="{selected.name}" workflow={workflow}', 400) + return None + refs = list(references) if references else ([init_image] if init_image is not None else []) + if len(refs) == 0: + raise VideoError('No reference image provided. The ref2va workflow conditions on reference images, so at least one is required.', 400) + if len(refs) > MAX_IMAGE_REFERENCES: + raise VideoError(f'too many reference images: count={len(refs)} max={MAX_IMAGE_REFERENCES}', 400) + for index, image in enumerate(refs): + size = getattr(image, 'size', None) + if size is None or len(size) != 2: + raise VideoError(f'reference {index + 1} is not an image: type={type(image).__name__}', 400) + width, height = size + if width > 4 * height or height > 4 * width: # the same bound the pipeline enforces, raised here where it is free + raise VideoError(f'reference {index + 1} aspect ratio out of range: size={width}x{height} supported=1:4..4:1', 400) + return refs + + def run(selected: models_def.Model, *, prompt: str, negative: str = '', @@ -78,6 +104,7 @@ def run(selected: models_def.Model, *, init_image=None, init_strength: float = 0.8, last_image=None, + references: list | None = None, vae_type: str = 'Default', vae_tile_frames: int = 16, audio: bool = True, @@ -100,6 +127,8 @@ def run(selected: models_def.Model, *, needs_load: bool = True, ) -> VideoResult: + refs = validate_references(selected, references, init_image) + if needs_load: if not shared.sd_loaded: debug('Video: model not yet loaded') @@ -158,15 +187,23 @@ def run(selected: models_def.Model, *, # modular workflows dispatch on which inputs are present; keyframes pass through # unresized since the pipeline defines its own canvas placement per anchor p.video_still = int(frames) <= 1 - if init_image is not None: - p.task_args['image'] = init_image - if last_image is not None: - p.task_args['last_image'] = last_image + if refs is not None: + from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3ImageReference + # references outrank the keyframe inputs in every block, so those stay unset; the reference + # encoder reads the image array as (height, width, 3) and never converts it itself + p.task_args['references'] = [MiniMaxH3ImageReference(image=image.convert('RGB')) for image in refs] + if last_image is not None: + log.warning(f'Video: op=reference model="{selected.name}" last frame not supported, ignoring') + else: + if init_image is not None: + p.task_args['image'] = init_image + if last_image is not None: + p.task_args['last_image'] = last_image if p.video_still: p.do_not_save_samples = False # the still is the product; save it like an image result elif int(mp4_fps) != 24: log.warning(f'Video: model="{selected.name}" fps={mp4_fps} model output is fixed at 24') - log.debug(f'Video: op=modular workflow={selected.workflow} still={p.video_still} init={init_image} last={last_image}') + log.debug(f'Video: op=modular workflow={selected.workflow} still={p.video_still} init={init_image} last={last_image} references={len(refs) if refs else 0}') elif 'T2V' in selected.name: if init_image is not None: log.warning('Video: op=T2V init image not supported') diff --git a/pipelines/model_minimax.py b/pipelines/model_minimax.py index 45f1b8f7d..bba966856 100644 --- a/pipelines/model_minimax.py +++ b/pipelines/model_minimax.py @@ -10,12 +10,13 @@ def load_minimax(checkpoint_info, diffusers_load_config=None): # pylint: disable if repo_id is None or repo_id.lower() == 'none': return None offline_args = {'local_files_only': True} if shared.opts.offline_mode else {} - log.debug(f'Load model: type=MiniMaxH3 repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype}') + workflow = (getattr(checkpoint_info, 'subfolder', None) or 'fl2va').lower() # one repo holds both checkpoint partitions; reference entries select ref2va via the subfolder tag + log.debug(f'Load model: type=MiniMaxH3 repo="{repo_id}" workflow={workflow} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype}') pipe = video_modular.load_modular_pipe( getattr(diffusers, 'MiniMaxH3ModularPipeline', None), repo_id, - workflow='fl2va', + workflow=workflow, offline_args=offline_args, base=True, ) diff --git a/test/test-video-api.py b/test/test-video-api.py index f57cd6067..040cf0523 100644 --- a/test/test-video-api.py +++ b/test/test-video-api.py @@ -5,13 +5,16 @@ API tests for video generation. Tests: - GET /sdapi/v1/video/models — engine/model enumeration and mode derivation - POST /sdapi/v1/video — request validation errors (partial pair, unknown model/sampler, checkpoint override, unknown script) +- POST /sdapi/v1/video — reference rules (wrong workflow, missing, over limit, aspect) - POST /sdapi/v1/video — still mode (frames=1) against the currently loaded model - POST /sdapi/v1/video — video generation against the currently loaded model - POST /sdapi/v1/video — wire switches and GET /sdapi/v1/video/file serving Requires a running SD.Next instance. Generation categories require a video-capable model loaded (for example MiniMax-H3 via the base checkpoint dropdown) and are -skipped otherwise; enumeration and validation run against any instance. +skipped otherwise; enumeration and validation run against any instance. A loaded +model that conditions on references is detected by the still probe, and every +later request against it carries one. Usage: python test/test-video-api.py [--url URL] [--steps STEPS] [--frames FRAMES] @@ -20,7 +23,9 @@ Usage: import os import sys import base64 +import struct import time +import zlib import argparse import requests import urllib3 @@ -30,6 +35,16 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) VALID_MODES = {'workflow', 't2v', 'i2v', 'flf2v', 'vace', 'animate'} +def png_b64(width: int, height: int) -> str: + """Minimal grey RGB PNG, so reference tests need no image library.""" + def chunk(tag: bytes, payload: bytes) -> bytes: + return struct.pack('>I', len(payload)) + tag + payload + struct.pack('>I', zlib.crc32(tag + payload) & 0xffffffff) + scanlines = b''.join(b'\x00' + b'\x7f\x7f\x7f' * width for _ in range(height)) + header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + data = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(scanlines)) + chunk(b'IEND', b'') + return base64.b64encode(data).decode() + + class VideoAPITest: """Test harness for the video generation API.""" @@ -39,6 +54,7 @@ class VideoAPITest: self.frames = frames self.timeout = timeout self.video_capable = None # set by the still-mode probe + self.ref2va = False # set by the same probe when the loaded model conditions on references self.results = { 'enumerate': {'passed': 0, 'failed': 0, 'skipped': 0, 'tests': []}, 'validation': {'passed': 0, 'failed': 0, 'skipped': 0, 'tests': []}, @@ -105,6 +121,8 @@ class VideoAPITest: } if extra_params: payload.update(extra_params) + if self.ref2va and 'references' not in payload and 'engine' not in payload: + payload['references'] = [png_b64(64, 64)] # requests aimed at the loaded model carry a reference when that model needs one t0 = time.time() data = self._post('/sdapi/v1/video', payload) return data, time.time() - t0 @@ -158,6 +176,29 @@ class VideoAPITest: self.record(data.get('error') == 422, 'unknown_script_rejected', f'code={data.get("error")}') else: self.skip('unknown_sampler_rejected', 'no registry models to pair with') + self.check_references(models) + + def check_references(self, models): + # every reference rule is checked before the model load, so these stay fast on a cold registry row + keyframe = next((m for m in models if m.get('workflow') not in (None, 'ref2va')), None) + reference = next((m for m in models if m.get('workflow') == 'ref2va'), None) + if keyframe: + pair = {'engine': keyframe['engine'], 'model': keyframe['name']} + data, elapsed = self._video({**pair, 'references': [png_b64(64, 64)]}) + self.record(data.get('error') == 400, 'references_wrong_workflow_rejected', f'code={data.get("error")} time={elapsed:.2f}s') + else: + self.skip('references_wrong_workflow_rejected', 'no keyframe workflow model in registry') + if not reference: + for name in ('references_required', 'references_over_limit', 'references_aspect_rejected'): + self.skip(name, 'no ref2va model in registry') + return + pair = {'engine': reference['engine'], 'model': reference['name']} + data, elapsed = self._video(pair) + self.record(data.get('error') == 400, 'references_required', f'code={data.get("error")} time={elapsed:.2f}s') + data, elapsed = self._video({**pair, 'references': [png_b64(64, 64)] * 10}) + self.record(data.get('error') == 400, 'references_over_limit', f'code={data.get("error")} time={elapsed:.2f}s') + data, elapsed = self._video({**pair, 'references': [png_b64(8, 64)]}) + self.record(data.get('error') == 400, 'references_aspect_rejected', f'code={data.get("error")} time={elapsed:.2f}s') # ========================================================================= # Tests: Still mode (doubles as the video-capability probe) @@ -167,6 +208,9 @@ class VideoAPITest: self._category = 'still' print("\n--- Still Mode Tests ---") data, elapsed = self._video({'frames': 1}) + if data.get('error') == 400 and 'ref2va' in str(data.get('detail', '')): + self.ref2va = True # the loaded model conditions on references; every later request carries one + data, elapsed = self._video({'frames': 1}) if data.get('error') == 400: self.video_capable = False self.skip('still_generation', f'no video-capable model loaded: {data.get("detail")}')