From 3e8f0372ad81a6ede238572e14a10e3380e5c2f2 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 9 Aug 2026 23:15:15 +0100 Subject: [PATCH] feat(video): condition minimax h3 ref2va on reference images The ref2va checkpoint partition conditions on reference images instead of keyframes, so it gets its own registry row and reference card, and the video core marshals PIL images into task_args as MiniMaxH3ImageReference. Images are converted to RGB first, since the reference encoder reads the array raw. The keyframe path is unchanged. Validation runs before the model load in one funnel shared by the tab and the API, so a rejected request costs nothing: references on a non-reference model, a reference model with nothing to condition on, more than nine images, non-images, and aspect outside 1:4 to 4:1 all return 400. The image path rejects a reference pipe without references instead of reaching a transformer that was never loaded. --- cli/api-video.py | 3 ++ data/reference-base.json | 9 ++++++ modules/api/video.py | 12 +++++++- modules/processing_diffusers.py | 5 ++++ modules/video_models/models_def.py | 12 ++++++++ modules/video_models/video_run.py | 47 ++++++++++++++++++++++++++---- pipelines/model_minimax.py | 5 ++-- test/test-video-api.py | 46 ++++++++++++++++++++++++++++- 8 files changed, 130 insertions(+), 9 deletions(-) diff --git a/cli/api-video.py b/cli/api-video.py index adc354a0d..33c6451fb 100644 --- a/cli/api-video.py +++ b/cli/api-video.py @@ -114,6 +114,8 @@ def generate(args): # pylint: disable=redefined-outer-name options['init_image'] = encode(args.init) if args.last: options['last_image'] = encode(args.last) + if args.reference: + options['references'] = [encode(f) for f in args.reference] stop_event = threading.Event() if args.progress: threading.Thread(target=watch_progress, args=(stop_event,), daemon=True).start() @@ -146,6 +148,7 @@ if __name__ == "__main__": parser.add_argument('--audio', action=argparse.BooleanOptionalAction, default=True, help='generate audio on supported models') parser.add_argument('--init', required=False, default=None, help='init image file') parser.add_argument('--last', required=False, default=None, help='last frame image file') + parser.add_argument('--reference', required=False, default=None, action='append', help='reference image file for reference workflows; repeat in the order the model should read them') parser.add_argument('--output', required=False, default=None, help='output video file') parser.add_argument('--progress', action='store_true', help='poll and log progress during generation') parser.add_argument('--timeout', required=False, default=3600, help='request timeout in seconds') diff --git a/data/reference-base.json b/data/reference-base.json index 94ce9cc51..9ab483474 100644 --- a/data/reference-base.json +++ b/data/reference-base.json @@ -382,6 +382,15 @@ "size": 134, "date": "2026 August" }, + "MiniMaxAI MiniMax-H3 Ref2VA": { + "path": "MiniMaxAI/MiniMax-H3", + "subfolder": "ref2va", + "preview": "MiniMaxAI--MiniMax-H3.jpg", + "desc": "The omni-reference variant of MiniMax-H3, sharing one repository with the base model as a separate checkpoint partition. Video with synchronized stereo audio is conditioned on reference images for identity and appearance, with reference rows held clean while video rows denoise.", + "extras": "sampler: Default", + "size": 134, + "date": "2026 August" + }, "Freepik F-Lite": { "path": "Freepik/F-Lite", "preview": "Freepik--F-Lite.jpg", diff --git a/modules/api/video.py b/modules/api/video.py index 414951900..d415d45f7 100644 --- a/modules/api/video.py +++ b/modules/api/video.py @@ -30,6 +30,7 @@ class ReqVideo(BaseModel): init_image: str | None = Field(default=None, title="Init image", description="Base64, data URI, or upload reference for the first-frame image") init_strength: float = Field(default=0.8, ge=0.0, le=1.0, title="Init strength", description="Denoising strength for the init image") last_image: str | None = Field(default=None, title="Last image", description="Base64, data URI, or upload reference for the last-frame image") + references: list[str] = Field(default=[], title="References", description="Reference images for a reference workflow, in the order the model reads them; base64, data URIs, or upload references. At most 9, each within a 1:4 to 4:1 aspect ratio. Rejected on models that do not condition on references") vae_type: str = Field(default="Default", title="VAE type", description="Decode variant: Default, Tiny, Remote, or Upscale") vae_tile_frames: int = Field(default=16, ge=1, le=64, title="VAE tile frames", description="Frames per VAE decode tile") audio: bool = Field(default=True, title="Audio", description="Generate audio on models that support it") @@ -72,7 +73,7 @@ class ItemVideoModel(BaseModel): repo: str = Field(default="", title="Repo", description="Model repository or path") url: str = Field(default="", title="URL", description="Model information page") mode: str = Field(title="Mode", description="Input mode: workflow, t2v, i2v, flf2v, vace, or animate") - workflow: str | None = Field(default=None, title="Workflow", description="Modular workflow name when the model dispatches on inputs") + workflow: str | None = Field(default=None, title="Workflow", description="Modular workflow name when the model dispatches on inputs; ref2va conditions on references and ignores the keyframe images") base: bool = Field(default=False, title="Base", description="Also listed in the base checkpoint dropdown") loaded: bool = Field(default=False, title="Loaded", description="Currently loaded through the video registry") @@ -118,6 +119,8 @@ class APIVideo: val = getattr(req, name, None) if isinstance(val, str) and len(val) >= 1000: setattr(req, name, f"") + if req.references: + sanitize_str(req.references) if req.script_args: sanitize_str(req.script_args) if req.alwayson_scripts: @@ -140,6 +143,11 @@ class APIVideo: `video_path` set; fetch those via `GET /sdapi/v1/video/file`. `init_image` and `last_image` accept base64 data, data URIs, or upload references. + Models whose workflow is `ref2va` condition on `references` instead: an ordered list of + images the prompt addresses as ``, `` and so on, following list + order. A single reference may also be passed as `init_image`. Reference images do not + set the output canvas, and `last_image` is ignored. + Progress is reported on `GET /sdapi/v1/progress`; `POST /sdapi/v1/interrupt` cancels. Switching checkpoints via `override_settings` is not supported here; use `POST /sdapi/v1/checkpoint` before generating. @@ -151,6 +159,7 @@ class APIVideo: sampler_name = helpers.validate_sampler_name(req.sampler_name) init_image = helpers.decode_base64_to_image(req.init_image) if req.init_image else None last_image = helpers.decode_base64_to_image(req.last_image) if req.last_image else None + references = [helpers.decode_base64_to_image(x) for x in (req.references or [])] overrides = dict(req.override_settings or {}) for key in ('sd_model_checkpoint', 'sd_model_refiner'): if key in overrides: @@ -180,6 +189,7 @@ class APIVideo: init_image=init_image, init_strength=req.init_strength, last_image=last_image, + references=references, vae_type=req.vae_type, vae_tile_frames=req.vae_tile_frames, audio=req.audio, diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index ba16e0e02..1e10e56c5 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -574,6 +574,11 @@ def validate_pipeline(p: processing.StableDiffusionProcessing): elif not is_video_model and is_video_pipeline: log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} non-video model with video pipeline') return False + if getattr(shared.sd_model, 'sdnext_video_workflow', None) == 'ref2va' and p.task_args.get('references', None) is None: + # the reference workflow loads its own transformer partition alone: without references the pipeline + # dispatches to the keyframe path and reaches a transformer that was never loaded + log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} reference workflow requires reference images: use the video tab or the video api') + return False return True diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index db7b1d42e..795f228be 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -658,6 +658,18 @@ try: image_hijack=False, vae_hijack=False, vae_remote=False), + Model(name='MiniMax H3 Ref2VA', + url='https://huggingface.co/MiniMaxAI/MiniMax-H3', + repo='MiniMaxAI/MiniMax-H3', + repo_cls='MiniMaxH3ModularPipeline', + workflow='ref2va', + base=True, + te_cls=None, + dit_cls=None, + te_hijack=False, + image_hijack=False, + vae_hijack=False, + vae_remote=False), ], 'Google Veo': [ Model(name='Google Veo 3.1 T2V', diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 3c1e10d35..92296b4b5 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -9,6 +9,7 @@ from modules.paths import resolve_output_path debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None +MAX_IMAGE_REFERENCES = 9 # mirrors the reference setup block's own limit; reading it off the pipe would deep-copy the block tree per access class VideoError(Exception): @@ -61,6 +62,31 @@ def resolve_model(engine: str | None, model: str | None) -> tuple[models_def.Mod return selected, False +def validate_references(selected: models_def.Model, references: list | None, init_image) -> list | None: + """Return the ordered reference images for a reference workflow, None for every other model. + Reference conditioning is exclusive to ref2va: its partition holds no keyframe transformer, and + a mismatched request would only fail once the pipeline reached a component it never loaded. + Checks run before the model load so a rejected request costs nothing.""" + workflow = getattr(selected, 'workflow', None) + if workflow != 'ref2va': + if references: + raise VideoError(f'reference images require a ref2va model: model="{selected.name}" workflow={workflow}', 400) + return None + refs = list(references) if references else ([init_image] if init_image is not None else []) + if len(refs) == 0: + raise VideoError('No reference image provided. The ref2va workflow conditions on reference images, so at least one is required.', 400) + if len(refs) > MAX_IMAGE_REFERENCES: + raise VideoError(f'too many reference images: count={len(refs)} max={MAX_IMAGE_REFERENCES}', 400) + for index, image in enumerate(refs): + size = getattr(image, 'size', None) + if size is None or len(size) != 2: + raise VideoError(f'reference {index + 1} is not an image: type={type(image).__name__}', 400) + width, height = size + if width > 4 * height or height > 4 * width: # the same bound the pipeline enforces, raised here where it is free + raise VideoError(f'reference {index + 1} aspect ratio out of range: size={width}x{height} supported=1:4..4:1', 400) + return refs + + def run(selected: models_def.Model, *, prompt: str, negative: str = '', @@ -78,6 +104,7 @@ def run(selected: models_def.Model, *, init_image=None, init_strength: float = 0.8, last_image=None, + references: list | None = None, vae_type: str = 'Default', vae_tile_frames: int = 16, audio: bool = True, @@ -100,6 +127,8 @@ def run(selected: models_def.Model, *, needs_load: bool = True, ) -> VideoResult: + refs = validate_references(selected, references, init_image) + if needs_load: if not shared.sd_loaded: debug('Video: model not yet loaded') @@ -158,15 +187,23 @@ def run(selected: models_def.Model, *, # modular workflows dispatch on which inputs are present; keyframes pass through # unresized since the pipeline defines its own canvas placement per anchor p.video_still = int(frames) <= 1 - if init_image is not None: - p.task_args['image'] = init_image - if last_image is not None: - p.task_args['last_image'] = last_image + if refs is not None: + from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3ImageReference + # references outrank the keyframe inputs in every block, so those stay unset; the reference + # encoder reads the image array as (height, width, 3) and never converts it itself + p.task_args['references'] = [MiniMaxH3ImageReference(image=image.convert('RGB')) for image in refs] + if last_image is not None: + log.warning(f'Video: op=reference model="{selected.name}" last frame not supported, ignoring') + else: + if init_image is not None: + p.task_args['image'] = init_image + if last_image is not None: + p.task_args['last_image'] = last_image if p.video_still: p.do_not_save_samples = False # the still is the product; save it like an image result elif int(mp4_fps) != 24: log.warning(f'Video: model="{selected.name}" fps={mp4_fps} model output is fixed at 24') - log.debug(f'Video: op=modular workflow={selected.workflow} still={p.video_still} init={init_image} last={last_image}') + log.debug(f'Video: op=modular workflow={selected.workflow} still={p.video_still} init={init_image} last={last_image} references={len(refs) if refs else 0}') elif 'T2V' in selected.name: if init_image is not None: log.warning('Video: op=T2V init image not supported') diff --git a/pipelines/model_minimax.py b/pipelines/model_minimax.py index 45f1b8f7d..bba966856 100644 --- a/pipelines/model_minimax.py +++ b/pipelines/model_minimax.py @@ -10,12 +10,13 @@ def load_minimax(checkpoint_info, diffusers_load_config=None): # pylint: disable if repo_id is None or repo_id.lower() == 'none': return None offline_args = {'local_files_only': True} if shared.opts.offline_mode else {} - log.debug(f'Load model: type=MiniMaxH3 repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype}') + workflow = (getattr(checkpoint_info, 'subfolder', None) or 'fl2va').lower() # one repo holds both checkpoint partitions; reference entries select ref2va via the subfolder tag + log.debug(f'Load model: type=MiniMaxH3 repo="{repo_id}" workflow={workflow} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype}') pipe = video_modular.load_modular_pipe( getattr(diffusers, 'MiniMaxH3ModularPipeline', None), repo_id, - workflow='fl2va', + workflow=workflow, offline_args=offline_args, base=True, ) diff --git a/test/test-video-api.py b/test/test-video-api.py index f57cd6067..040cf0523 100644 --- a/test/test-video-api.py +++ b/test/test-video-api.py @@ -5,13 +5,16 @@ API tests for video generation. Tests: - GET /sdapi/v1/video/models — engine/model enumeration and mode derivation - POST /sdapi/v1/video — request validation errors (partial pair, unknown model/sampler, checkpoint override, unknown script) +- POST /sdapi/v1/video — reference rules (wrong workflow, missing, over limit, aspect) - POST /sdapi/v1/video — still mode (frames=1) against the currently loaded model - POST /sdapi/v1/video — video generation against the currently loaded model - POST /sdapi/v1/video — wire switches and GET /sdapi/v1/video/file serving Requires a running SD.Next instance. Generation categories require a video-capable model loaded (for example MiniMax-H3 via the base checkpoint dropdown) and are -skipped otherwise; enumeration and validation run against any instance. +skipped otherwise; enumeration and validation run against any instance. A loaded +model that conditions on references is detected by the still probe, and every +later request against it carries one. Usage: python test/test-video-api.py [--url URL] [--steps STEPS] [--frames FRAMES] @@ -20,7 +23,9 @@ Usage: import os import sys import base64 +import struct import time +import zlib import argparse import requests import urllib3 @@ -30,6 +35,16 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) VALID_MODES = {'workflow', 't2v', 'i2v', 'flf2v', 'vace', 'animate'} +def png_b64(width: int, height: int) -> str: + """Minimal grey RGB PNG, so reference tests need no image library.""" + def chunk(tag: bytes, payload: bytes) -> bytes: + return struct.pack('>I', len(payload)) + tag + payload + struct.pack('>I', zlib.crc32(tag + payload) & 0xffffffff) + scanlines = b''.join(b'\x00' + b'\x7f\x7f\x7f' * width for _ in range(height)) + header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + data = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(scanlines)) + chunk(b'IEND', b'') + return base64.b64encode(data).decode() + + class VideoAPITest: """Test harness for the video generation API.""" @@ -39,6 +54,7 @@ class VideoAPITest: self.frames = frames self.timeout = timeout self.video_capable = None # set by the still-mode probe + self.ref2va = False # set by the same probe when the loaded model conditions on references self.results = { 'enumerate': {'passed': 0, 'failed': 0, 'skipped': 0, 'tests': []}, 'validation': {'passed': 0, 'failed': 0, 'skipped': 0, 'tests': []}, @@ -105,6 +121,8 @@ class VideoAPITest: } if extra_params: payload.update(extra_params) + if self.ref2va and 'references' not in payload and 'engine' not in payload: + payload['references'] = [png_b64(64, 64)] # requests aimed at the loaded model carry a reference when that model needs one t0 = time.time() data = self._post('/sdapi/v1/video', payload) return data, time.time() - t0 @@ -158,6 +176,29 @@ class VideoAPITest: self.record(data.get('error') == 422, 'unknown_script_rejected', f'code={data.get("error")}') else: self.skip('unknown_sampler_rejected', 'no registry models to pair with') + self.check_references(models) + + def check_references(self, models): + # every reference rule is checked before the model load, so these stay fast on a cold registry row + keyframe = next((m for m in models if m.get('workflow') not in (None, 'ref2va')), None) + reference = next((m for m in models if m.get('workflow') == 'ref2va'), None) + if keyframe: + pair = {'engine': keyframe['engine'], 'model': keyframe['name']} + data, elapsed = self._video({**pair, 'references': [png_b64(64, 64)]}) + self.record(data.get('error') == 400, 'references_wrong_workflow_rejected', f'code={data.get("error")} time={elapsed:.2f}s') + else: + self.skip('references_wrong_workflow_rejected', 'no keyframe workflow model in registry') + if not reference: + for name in ('references_required', 'references_over_limit', 'references_aspect_rejected'): + self.skip(name, 'no ref2va model in registry') + return + pair = {'engine': reference['engine'], 'model': reference['name']} + data, elapsed = self._video(pair) + self.record(data.get('error') == 400, 'references_required', f'code={data.get("error")} time={elapsed:.2f}s') + data, elapsed = self._video({**pair, 'references': [png_b64(64, 64)] * 10}) + self.record(data.get('error') == 400, 'references_over_limit', f'code={data.get("error")} time={elapsed:.2f}s') + data, elapsed = self._video({**pair, 'references': [png_b64(8, 64)]}) + self.record(data.get('error') == 400, 'references_aspect_rejected', f'code={data.get("error")} time={elapsed:.2f}s') # ========================================================================= # Tests: Still mode (doubles as the video-capability probe) @@ -167,6 +208,9 @@ class VideoAPITest: self._category = 'still' print("\n--- Still Mode Tests ---") data, elapsed = self._video({'frames': 1}) + if data.get('error') == 400 and 'ref2va' in str(data.get('detail', '')): + self.ref2va = True # the loaded model conditions on references; every later request carries one + data, elapsed = self._video({'frames': 1}) if data.get('error') == 400: self.video_capable = False self.skip('still_generation', f'no video-capable model loaded: {data.get("detail")}')