feat(video): condition minimax h3 ref2va on reference images

The ref2va checkpoint partition conditions on reference images instead
of keyframes, so it gets its own registry row and reference card, and
the video core marshals PIL images into task_args as
MiniMaxH3ImageReference. Images are converted to RGB first, since the
reference encoder reads the array raw. The keyframe path is unchanged.

Validation runs before the model load in one funnel shared by the tab
and the API, so a rejected request costs nothing: references on a
non-reference model, a reference model with nothing to condition on,
more than nine images, non-images, and aspect outside 1:4 to 4:1 all
return 400. The image path rejects a reference pipe without references
instead of reaching a transformer that was never loaded.

cli/api-video.py

@@ -114,6 +114,8 @@ def generate(args): # pylint: disable=redefined-outer-name
         options['init_image'] = encode(args.init)
     if args.last:
         options['last_image'] = encode(args.last)
+    if args.reference:
+        options['references'] = [encode(f) for f in args.reference]
     stop_event = threading.Event()
     if args.progress:
         threading.Thread(target=watch_progress, args=(stop_event,), daemon=True).start()
@@ -146,6 +148,7 @@ if __name__ == "__main__":
     parser.add_argument('--audio', action=argparse.BooleanOptionalAction, default=True, help='generate audio on supported models')
     parser.add_argument('--init', required=False, default=None, help='init image file')
     parser.add_argument('--last', required=False, default=None, help='last frame image file')
+    parser.add_argument('--reference', required=False, default=None, action='append', help='reference image file for reference workflows; repeat in the order the model should read them')
     parser.add_argument('--output', required=False, default=None, help='output video file')
     parser.add_argument('--progress', action='store_true', help='poll and log progress during generation')
     parser.add_argument('--timeout', required=False, default=3600, help='request timeout in seconds')

data/reference-base.json

@@ -382,6 +382,15 @@
     "size": 134,
     "date": "2026 August"
   },
+  "MiniMaxAI MiniMax-H3 Ref2VA": {
+    "path": "MiniMaxAI/MiniMax-H3",
+    "subfolder": "ref2va",
+    "preview": "MiniMaxAI--MiniMax-H3.jpg",
+    "desc": "The omni-reference variant of MiniMax-H3, sharing one repository with the base model as a separate checkpoint partition. Video with synchronized stereo audio is conditioned on reference images for identity and appearance, with reference rows held clean while video rows denoise.",
+    "extras": "sampler: Default",
+    "size": 134,
+    "date": "2026 August"
+  },
   "Freepik F-Lite": {
     "path": "Freepik/F-Lite",
     "preview": "Freepik--F-Lite.jpg",

modules/api/video.py

@@ -30,6 +30,7 @@ class ReqVideo(BaseModel):
     init_image: str | None = Field(default=None, title="Init image", description="Base64, data URI, or upload reference for the first-frame image")
     init_strength: float = Field(default=0.8, ge=0.0, le=1.0, title="Init strength", description="Denoising strength for the init image")
     last_image: str | None = Field(default=None, title="Last image", description="Base64, data URI, or upload reference for the last-frame image")
+    references: list[str] = Field(default=[], title="References", description="Reference images for a reference workflow, in the order the model reads them; base64, data URIs, or upload references. At most 9, each within a 1:4 to 4:1 aspect ratio. Rejected on models that do not condition on references")
     vae_type: str = Field(default="Default", title="VAE type", description="Decode variant: Default, Tiny, Remote, or Upscale")
     vae_tile_frames: int = Field(default=16, ge=1, le=64, title="VAE tile frames", description="Frames per VAE decode tile")
     audio: bool = Field(default=True, title="Audio", description="Generate audio on models that support it")
@@ -72,7 +73,7 @@ class ItemVideoModel(BaseModel):
     repo: str = Field(default="", title="Repo", description="Model repository or path")
     url: str = Field(default="", title="URL", description="Model information page")
     mode: str = Field(title="Mode", description="Input mode: workflow, t2v, i2v, flf2v, vace, or animate")
-    workflow: str | None = Field(default=None, title="Workflow", description="Modular workflow name when the model dispatches on inputs")
+    workflow: str | None = Field(default=None, title="Workflow", description="Modular workflow name when the model dispatches on inputs; ref2va conditions on references and ignores the keyframe images")
     base: bool = Field(default=False, title="Base", description="Also listed in the base checkpoint dropdown")
     loaded: bool = Field(default=False, title="Loaded", description="Currently loaded through the video registry")
 
@@ -118,6 +119,8 @@ class APIVideo:
             val = getattr(req, name, None)
             if isinstance(val, str) and len(val) >= 1000:
                 setattr(req, name, f"<str {len(val)}>")
+        if req.references:
+            sanitize_str(req.references)
         if req.script_args:
             sanitize_str(req.script_args)
         if req.alwayson_scripts:
@@ -140,6 +143,11 @@ class APIVideo:
         `video_path` set; fetch those via `GET /sdapi/v1/video/file`.
 
         `init_image` and `last_image` accept base64 data, data URIs, or upload references.
+        Models whose workflow is `ref2va` condition on `references` instead: an ordered list of
+        images the prompt addresses as `<Picture 1>`, `<Picture 2>` and so on, following list
+        order. A single reference may also be passed as `init_image`. Reference images do not
+        set the output canvas, and `last_image` is ignored.
+
         Progress is reported on `GET /sdapi/v1/progress`; `POST /sdapi/v1/interrupt` cancels.
         Switching checkpoints via `override_settings` is not supported here; use
         `POST /sdapi/v1/checkpoint` before generating.
@@ -151,6 +159,7 @@ class APIVideo:
         sampler_name = helpers.validate_sampler_name(req.sampler_name)
         init_image = helpers.decode_base64_to_image(req.init_image) if req.init_image else None
         last_image = helpers.decode_base64_to_image(req.last_image) if req.last_image else None
+        references = [helpers.decode_base64_to_image(x) for x in (req.references or [])]
         overrides = dict(req.override_settings or {})
         for key in ('sd_model_checkpoint', 'sd_model_refiner'):
             if key in overrides:
@@ -180,6 +189,7 @@ class APIVideo:
                     init_image=init_image,
                     init_strength=req.init_strength,
                     last_image=last_image,
+                    references=references,
                     vae_type=req.vae_type,
                     vae_tile_frames=req.vae_tile_frames,
                     audio=req.audio,

modules/processing_diffusers.py

@@ -574,6 +574,11 @@ def validate_pipeline(p: processing.StableDiffusionProcessing):
     elif not is_video_model and is_video_pipeline:
         log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} non-video model with video pipeline')
         return False
+    if getattr(shared.sd_model, 'sdnext_video_workflow', None) == 'ref2va' and p.task_args.get('references', None) is None:
+        # the reference workflow loads its own transformer partition alone: without references the pipeline
+        # dispatches to the keyframe path and reaches a transformer that was never loaded
+        log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} reference workflow requires reference images: use the video tab or the video api')
+        return False
     return True
 
 

modules/video_models/models_def.py

@@ -658,6 +658,18 @@ try:
                 image_hijack=False,
                 vae_hijack=False,
                 vae_remote=False),
+            Model(name='MiniMax H3 Ref2VA',
+                url='https://huggingface.co/MiniMaxAI/MiniMax-H3',
+                repo='MiniMaxAI/MiniMax-H3',
+                repo_cls='MiniMaxH3ModularPipeline',
+                workflow='ref2va',
+                base=True,
+                te_cls=None,
+                dit_cls=None,
+                te_hijack=False,
+                image_hijack=False,
+                vae_hijack=False,
+                vae_remote=False),
         ],
         'Google Veo': [
             Model(name='Google Veo 3.1 T2V',

modules/video_models/video_run.py

@@ -9,6 +9,7 @@ from modules.paths import resolve_output_path
 
 
 debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
+MAX_IMAGE_REFERENCES = 9 # mirrors the reference setup block's own limit; reading it off the pipe would deep-copy the block tree per access
 
 
 class VideoError(Exception):
@@ -61,6 +62,31 @@ def resolve_model(engine: str | None, model: str | None) -> tuple[models_def.Mod
     return selected, False
 
 
+def validate_references(selected: models_def.Model, references: list | None, init_image) -> list | None:
+    """Return the ordered reference images for a reference workflow, None for every other model.
+    Reference conditioning is exclusive to ref2va: its partition holds no keyframe transformer, and
+    a mismatched request would only fail once the pipeline reached a component it never loaded.
+    Checks run before the model load so a rejected request costs nothing."""
+    workflow = getattr(selected, 'workflow', None)
+    if workflow != 'ref2va':
+        if references:
+            raise VideoError(f'reference images require a ref2va model: model="{selected.name}" workflow={workflow}', 400)
+        return None
+    refs = list(references) if references else ([init_image] if init_image is not None else [])
+    if len(refs) == 0:
+        raise VideoError('No reference image provided. The ref2va workflow conditions on reference images, so at least one is required.', 400)
+    if len(refs) > MAX_IMAGE_REFERENCES:
+        raise VideoError(f'too many reference images: count={len(refs)} max={MAX_IMAGE_REFERENCES}', 400)
+    for index, image in enumerate(refs):
+        size = getattr(image, 'size', None)
+        if size is None or len(size) != 2:
+            raise VideoError(f'reference {index + 1} is not an image: type={type(image).__name__}', 400)
+        width, height = size
+        if width > 4 * height or height > 4 * width: # the same bound the pipeline enforces, raised here where it is free
+            raise VideoError(f'reference {index + 1} aspect ratio out of range: size={width}x{height} supported=1:4..4:1', 400)
+    return refs
+
+
 def run(selected: models_def.Model, *,
         prompt: str,
         negative: str = '',
@@ -78,6 +104,7 @@ def run(selected: models_def.Model, *,
         init_image=None,
         init_strength: float = 0.8,
         last_image=None,
+        references: list | None = None,
         vae_type: str = 'Default',
         vae_tile_frames: int = 16,
         audio: bool = True,
@@ -100,6 +127,8 @@ def run(selected: models_def.Model, *,
         needs_load: bool = True,
        ) -> VideoResult:
 
+    refs = validate_references(selected, references, init_image)
+
     if needs_load:
         if not shared.sd_loaded:
             debug('Video: model not yet loaded')
@@ -158,15 +187,23 @@ def run(selected: models_def.Model, *,
         # modular workflows dispatch on which inputs are present; keyframes pass through
         # unresized since the pipeline defines its own canvas placement per anchor
         p.video_still = int(frames) <= 1
-        if init_image is not None:
-            p.task_args['image'] = init_image
-        if last_image is not None:
-            p.task_args['last_image'] = last_image
+        if refs is not None:
+            from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3ImageReference
+            # references outrank the keyframe inputs in every block, so those stay unset; the reference
+            # encoder reads the image array as (height, width, 3) and never converts it itself
+            p.task_args['references'] = [MiniMaxH3ImageReference(image=image.convert('RGB')) for image in refs]
+            if last_image is not None:
+                log.warning(f'Video: op=reference model="{selected.name}" last frame not supported, ignoring')
+        else:
+            if init_image is not None:
+                p.task_args['image'] = init_image
+            if last_image is not None:
+                p.task_args['last_image'] = last_image
         if p.video_still:
             p.do_not_save_samples = False # the still is the product; save it like an image result
         elif int(mp4_fps) != 24:
             log.warning(f'Video: model="{selected.name}" fps={mp4_fps} model output is fixed at 24')
-        log.debug(f'Video: op=modular workflow={selected.workflow} still={p.video_still} init={init_image} last={last_image}')
+        log.debug(f'Video: op=modular workflow={selected.workflow} still={p.video_still} init={init_image} last={last_image} references={len(refs) if refs else 0}')
     elif 'T2V' in selected.name:
         if init_image is not None:
             log.warning('Video: op=T2V init image not supported')

pipelines/model_minimax.py

@@ -10,12 +10,13 @@ def load_minimax(checkpoint_info, diffusers_load_config=None): # pylint: disable
     if repo_id is None or repo_id.lower() == 'none':
         return None
     offline_args = {'local_files_only': True} if shared.opts.offline_mode else {}
-    log.debug(f'Load model: type=MiniMaxH3 repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype}')
+    workflow = (getattr(checkpoint_info, 'subfolder', None) or 'fl2va').lower() # one repo holds both checkpoint partitions; reference entries select ref2va via the subfolder tag
+    log.debug(f'Load model: type=MiniMaxH3 repo="{repo_id}" workflow={workflow} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype}')
 
     pipe = video_modular.load_modular_pipe(
         getattr(diffusers, 'MiniMaxH3ModularPipeline', None),
         repo_id,
-        workflow='fl2va',
+        workflow=workflow,
         offline_args=offline_args,
         base=True,
     )

test/test-video-api.py

@@ -5,13 +5,16 @@ API tests for video generation.
 Tests:
 - GET /sdapi/v1/video/models — engine/model enumeration and mode derivation
 - POST /sdapi/v1/video — request validation errors (partial pair, unknown model/sampler, checkpoint override, unknown script)
+- POST /sdapi/v1/video — reference rules (wrong workflow, missing, over limit, aspect)
 - POST /sdapi/v1/video — still mode (frames=1) against the currently loaded model
 - POST /sdapi/v1/video — video generation against the currently loaded model
 - POST /sdapi/v1/video — wire switches and GET /sdapi/v1/video/file serving
 
 Requires a running SD.Next instance. Generation categories require a video-capable
 model loaded (for example MiniMax-H3 via the base checkpoint dropdown) and are
-skipped otherwise; enumeration and validation run against any instance.
+skipped otherwise; enumeration and validation run against any instance. A loaded
+model that conditions on references is detected by the still probe, and every
+later request against it carries one.
 
 Usage:
     python test/test-video-api.py [--url URL] [--steps STEPS] [--frames FRAMES]
@@ -20,7 +23,9 @@ Usage:
 import os
 import sys
 import base64
+import struct
 import time
+import zlib
 import argparse
 import requests
 import urllib3
@@ -30,6 +35,16 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 VALID_MODES = {'workflow', 't2v', 'i2v', 'flf2v', 'vace', 'animate'}
 
 
+def png_b64(width: int, height: int) -> str:
+    """Minimal grey RGB PNG, so reference tests need no image library."""
+    def chunk(tag: bytes, payload: bytes) -> bytes:
+        return struct.pack('>I', len(payload)) + tag + payload + struct.pack('>I', zlib.crc32(tag + payload) & 0xffffffff)
+    scanlines = b''.join(b'\x00' + b'\x7f\x7f\x7f' * width for _ in range(height))
+    header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
+    data = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(scanlines)) + chunk(b'IEND', b'')
+    return base64.b64encode(data).decode()
+
+
 class VideoAPITest:
     """Test harness for the video generation API."""
 
@@ -39,6 +54,7 @@ class VideoAPITest:
         self.frames = frames
         self.timeout = timeout
         self.video_capable = None # set by the still-mode probe
+        self.ref2va = False # set by the same probe when the loaded model conditions on references
         self.results = {
             'enumerate': {'passed': 0, 'failed': 0, 'skipped': 0, 'tests': []},
             'validation': {'passed': 0, 'failed': 0, 'skipped': 0, 'tests': []},
@@ -105,6 +121,8 @@ class VideoAPITest:
         }
         if extra_params:
             payload.update(extra_params)
+        if self.ref2va and 'references' not in payload and 'engine' not in payload:
+            payload['references'] = [png_b64(64, 64)] # requests aimed at the loaded model carry a reference when that model needs one
         t0 = time.time()
         data = self._post('/sdapi/v1/video', payload)
         return data, time.time() - t0
@@ -158,6 +176,29 @@ class VideoAPITest:
             self.record(data.get('error') == 422, 'unknown_script_rejected', f'code={data.get("error")}')
         else:
             self.skip('unknown_sampler_rejected', 'no registry models to pair with')
+        self.check_references(models)
+
+    def check_references(self, models):
+        # every reference rule is checked before the model load, so these stay fast on a cold registry row
+        keyframe = next((m for m in models if m.get('workflow') not in (None, 'ref2va')), None)
+        reference = next((m for m in models if m.get('workflow') == 'ref2va'), None)
+        if keyframe:
+            pair = {'engine': keyframe['engine'], 'model': keyframe['name']}
+            data, elapsed = self._video({**pair, 'references': [png_b64(64, 64)]})
+            self.record(data.get('error') == 400, 'references_wrong_workflow_rejected', f'code={data.get("error")} time={elapsed:.2f}s')
+        else:
+            self.skip('references_wrong_workflow_rejected', 'no keyframe workflow model in registry')
+        if not reference:
+            for name in ('references_required', 'references_over_limit', 'references_aspect_rejected'):
+                self.skip(name, 'no ref2va model in registry')
+            return
+        pair = {'engine': reference['engine'], 'model': reference['name']}
+        data, elapsed = self._video(pair)
+        self.record(data.get('error') == 400, 'references_required', f'code={data.get("error")} time={elapsed:.2f}s')
+        data, elapsed = self._video({**pair, 'references': [png_b64(64, 64)] * 10})
+        self.record(data.get('error') == 400, 'references_over_limit', f'code={data.get("error")} time={elapsed:.2f}s')
+        data, elapsed = self._video({**pair, 'references': [png_b64(8, 64)]})
+        self.record(data.get('error') == 400, 'references_aspect_rejected', f'code={data.get("error")} time={elapsed:.2f}s')
 
     # =========================================================================
     # Tests: Still mode (doubles as the video-capability probe)
@@ -167,6 +208,9 @@ class VideoAPITest:
         self._category = 'still'
         print("\n--- Still Mode Tests ---")
         data, elapsed = self._video({'frames': 1})
+        if data.get('error') == 400 and 'ref2va' in str(data.get('detail', '')):
+            self.ref2va = True # the loaded model conditions on references; every later request carries one
+            data, elapsed = self._video({'frames': 1})
         if data.get('error') == 400:
             self.video_capable = False
             self.skip('still_generation', f'no video-capable model loaded: {data.get("detail")}')