From 005fc5c86e9370fb1ed12a504a80eeea019fb1e9 Mon Sep 17 00:00:00 2001
From: CalamitousFelicitousness <iowasovereign@gmail.com>
Date: Sun, 16 Aug 2026 07:34:35 +0100
Subject: [PATCH] refactor(video): accept video and audio references in the
 shared core

The core took reference images only, so no api caller could send the video and
audio references the ref2va workflow conditions on, and the marshalling that
handles them existed solely in the MiniMax tab.

validate_references now gates on the workflow and hands the entries to the
architecture that owns them, which accepts decoded images and local file paths
in any mix and preserves their order, since order fixes the labels a prompt
addresses. reference_caps exposes the same limits the validation enforces, so a
client reads them instead of mirroring the numbers.

- MAX_IMAGE_REFERENCES is gone: the limits now cover all three kinds and a total
- the run body no longer builds reference objects or knows their class
- an image is converted where it is built rather than at the call site, so a
  reference decoded from a file and one posted as base64 arrive the same way
- pipeline args summarize a reference list by kind, since a decoded video would
  otherwise print its frames into the per-generation log line
- the video endpoint documents what it actually accepts: images alone, because
  video and audio decode from files rather than from the wire, and an upload
  reference only where an extension provides the store that resolves one
---
 modules/api/video.py              | 13 +++++---
 modules/processing_args.py        |  2 ++
 modules/video_models/video_run.py | 38 ++++++++++------------
 test/test-video-references.py     | 52 +++++++++++++++++++++++++++++++
 4 files changed, 79 insertions(+), 26 deletions(-)

diff --git a/modules/api/video.py b/modules/api/video.py
index 37cdf3be0..2b34357d2 100644
--- a/modules/api/video.py
+++ b/modules/api/video.py
@@ -27,10 +27,10 @@ class ReqVideo(BaseModel):
     seed: int = Field(default=-1, title="Seed", description="Generation seed; -1 for random")
     guidance_scale: float = Field(default=-1.0, title="Guidance scale", description="CFG scale; -1 keeps the model default")
     guidance_true: float = Field(default=-1.0, title="True guidance", description="True CFG scale; -1 keeps the model default")
-    init_image: str | None = Field(default=None, title="Init image", description="Base64, data URI, or upload reference for the first-frame image")
+    init_image: str | None = Field(default=None, title="Init image", description="Base64 or data URI for the first-frame image; an upload reference resolves only where an extension provides the upload store")
     init_strength: float = Field(default=0.8, ge=0.0, le=1.0, title="Init strength", description="Denoising strength for the init image")
-    last_image: str | None = Field(default=None, title="Last image", description="Base64, data URI, or upload reference for the last-frame image")
-    references: list[str] = Field(default=[], title="References", description="Reference images for a reference workflow, in the order the model reads them; base64, data URIs, or upload references. At most 9, each within a 1:4 to 4:1 aspect ratio. Rejected on models that do not condition on references")
+    last_image: str | None = Field(default=None, title="Last image", description="Base64 or data URI for the last-frame image; an upload reference resolves only where an extension provides the upload store")
+    references: list[str] = Field(default=[], title="References", description="Reference images for a reference workflow, in the order the model reads them; base64 or data URIs, or upload references where an extension provides the upload store. Images only: the video core also conditions on video and audio references, which this endpoint cannot carry. At most 9, each within a 1:4 to 4:1 aspect ratio. Rejected on models that do not condition on references")
     vae_type: str = Field(default="Default", title="VAE type", description="Decode variant: Default, Tiny, Remote, or Upscale")
     vae_tile_frames: int = Field(default=16, ge=1, le=64, title="VAE tile frames", description="Frames per VAE decode tile")
     audio: bool = Field(default=True, title="Audio", description="Generate audio on models that support it")
@@ -142,11 +142,14 @@ class APIVideo:
         `send_thumbnail`. Artifacts above the base64 size cap return `video` empty with
         `video_path` set; fetch those via `GET /sdapi/v1/video/file`.
 
-        `init_image` and `last_image` accept base64 data, data URIs, or upload references.
+        `init_image` and `last_image` accept base64 data or data URIs. An `upload:` reference
+        resolves only where an extension registers an upload store; without one it is rejected.
         Models whose workflow is `ref2va` condition on `references` instead: an ordered list of
         images the prompt addresses as `<Picture 1>`, `<Picture 2>` and so on, following list
         order. A single reference may also be passed as `init_image`. Reference images do not
-        set the output canvas, and `last_image` is ignored.
+        set the output canvas, and `last_image` is ignored. The workflow also conditions on video
+        and audio references, addressed as `<Video i>` and `<Audio i>`, but they decode from files
+        rather than from the wire, so this endpoint carries images alone.
 
         Progress is reported on `GET /sdapi/v1/progress`; `POST /sdapi/v1/interrupt` cancels.
         Switching checkpoints via `override_settings` is not supported here; use
diff --git a/modules/processing_args.py b/modules/processing_args.py
index f0e27a8ed..94a66153f 100644
--- a/modules/processing_args.py
+++ b/modules/processing_args.py
@@ -488,6 +488,8 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:l
             clean[k] = v.shape
         elif isinstance(v, list) and len(v) > 0 and (isinstance(v[0], torch.Tensor) or isinstance(v[0], np.ndarray)):
             clean[k] = [x.shape for x in v]
+        elif isinstance(v, list) and len(v) > 0 and hasattr(v[0], 'kind'): # media references carry decoded frames and waveforms
+            clean[k] = [getattr(x, 'kind', type(x).__name__) for x in v]
         elif not debug_enabled and k.endswith('_embeds'):
             del clean[k]
             clean['prompt'] = 'embeds'
diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py
index 861b743e4..cd52c950e 100644
--- a/modules/video_models/video_run.py
+++ b/modules/video_models/video_run.py
@@ -9,7 +9,7 @@ from modules.paths import resolve_output_path
 
 
 debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
-MAX_IMAGE_REFERENCES = 9 # mirrors the reference setup block's own limit; reading it off the pipe would deep-copy the block tree per access
+REFERENCE_WORKFLOWS = ('ref2va',) # workflows that condition on references; the resolver and the limits it enforces live with the architecture
 
 
 class VideoError(Exception):
@@ -62,29 +62,27 @@ def resolve_model(engine: str | None, model: str | None) -> tuple[models_def.Mod
     return selected, False
 
 
+def reference_caps(workflow: str | None):
+    """Reference limits of a workflow, None when it conditions on none. The seam a client reads
+    instead of mirroring the numbers."""
+    if workflow not in REFERENCE_WORKFLOWS:
+        return None
+    from modules.minimax import minimax_references
+    return minimax_references.get_reference_caps(workflow)
+
+
 def validate_references(selected: models_def.Model, references: list | None, init_image) -> list | None:
-    """Return the ordered reference images for a reference workflow, None for every other model.
+    """Return the ordered references a reference workflow conditions on, None for every other model.
     Reference conditioning is exclusive to ref2va: its partition holds no keyframe transformer, and
     a mismatched request would only fail once the pipeline reached a component it never loaded.
     Checks run before the model load so a rejected request costs nothing."""
     workflow = getattr(selected, 'workflow', None)
-    if workflow != 'ref2va':
+    if workflow not in REFERENCE_WORKFLOWS:
         if references:
-            raise VideoError(f'reference images require a ref2va model: model="{selected.name}" workflow={workflow}', 400)
+            raise VideoError(f'reference media requires a reference workflow: model="{selected.name}" workflow={workflow} supported={list(REFERENCE_WORKFLOWS)}', 400)
         return None
-    refs = list(references) if references else ([init_image] if init_image is not None else [])
-    if len(refs) == 0:
-        raise VideoError('No reference image provided. The ref2va workflow conditions on reference images, so at least one is required.', 400)
-    if len(refs) > MAX_IMAGE_REFERENCES:
-        raise VideoError(f'too many reference images: count={len(refs)} max={MAX_IMAGE_REFERENCES}', 400)
-    for index, image in enumerate(refs):
-        size = getattr(image, 'size', None)
-        if size is None or len(size) != 2:
-            raise VideoError(f'reference {index + 1} is not an image: type={type(image).__name__}', 400)
-        width, height = size
-        if width > 4 * height or height > 4 * width: # the same bound the pipeline enforces, raised here where it is free
-            raise VideoError(f'reference {index + 1} aspect ratio out of range: size={width}x{height} supported=1:4..4:1', 400)
-    return refs
+    from modules.minimax import minimax_references
+    return minimax_references.resolve(workflow, references, init_image)
 
 
 def run(selected: models_def.Model, *,
@@ -188,10 +186,8 @@ def run(selected: models_def.Model, *,
         # unresized since the pipeline defines its own canvas placement per anchor
         p.video_still = int(frames) <= 1
         if refs is not None:
-            from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3ImageReference
-            # references outrank the keyframe inputs in every block, so those stay unset; the reference
-            # encoder reads the image array as (height, width, 3) and never converts it itself
-            p.task_args['references'] = [MiniMaxH3ImageReference(image=image.convert('RGB')) for image in refs]
+            # references outrank the keyframe inputs in every block, so those stay unset
+            p.task_args['references'] = refs
             if last_image is not None:
                 log.warning(f'Video: op=reference model="{selected.name}" last frame not supported, ignoring')
         else:
diff --git a/test/test-video-references.py b/test/test-video-references.py
index fae7d0596..2290e7b3e 100644
--- a/test/test-video-references.py
+++ b/test/test-video-references.py
@@ -556,6 +556,47 @@ def test_short_video_is_rejected_after_the_decode():
     return expect_error(lambda: refs.resolve('ref2va', [image(), fn]), 'video too short')
 
 
+# ============================================================
+# Core seam
+# ============================================================
+
+def test_core_serves_the_caps():
+    from modules.video_models import video_run
+    assert video_run.reference_caps('ref2va') is caps(), 'the core served different caps than the resolver'
+    assert video_run.reference_caps('fl2va') is None, 'fl2va reported reference limits'
+    assert video_run.reference_caps(None) is None, 'a missing workflow reported reference limits'
+
+
+def test_core_gate_and_caps_table_agree():
+    from modules.video_models import video_run
+    assert set(video_run.REFERENCE_WORKFLOWS) == set(refs.REFERENCE_CAPS), f'{video_run.REFERENCE_WORKFLOWS} vs {sorted(refs.REFERENCE_CAPS)}'
+
+
+def test_core_gate_names_workflows_the_registry_carries():
+    from modules.video_models import models_def, video_run
+    known = {row.workflow for rows in models_def.models.values() for row in rows if getattr(row, 'workflow', None)}
+    assert set(video_run.REFERENCE_WORKFLOWS) <= known, f'{video_run.REFERENCE_WORKFLOWS} not among {sorted(known)}'
+
+
+def test_core_rejects_references_on_a_keyframe_model():
+    from modules.video_models import models_def, video_run
+    row = models_def.Model(name='test keyframe', workflow='fl2va')
+    expect_error(lambda: video_run.validate_references(row, [image()], None), 'requires a reference workflow')
+    assert video_run.validate_references(row, None, None) is None, 'a keyframe model resolved references'
+    assert video_run.validate_references(row, [], image()) is None, 'a keyframe model claimed its init image'
+
+
+def test_core_resolves_a_reference_model():
+    from modules.video_models import models_def, video_run
+    row = models_def.Model(name='test reference', workflow='ref2va')
+    expect_error(lambda: video_run.validate_references(row, None, None), 'No reference media provided')
+    if not has_diffusers():
+        return 'diffusers not installed'
+    built = video_run.validate_references(row, None, image()) # the init image stands in for a single reference
+    assert len(built) == 1 and built[0].kind == 'image', f'{built}'
+    return True
+
+
 # ============================================================
 # Runner
 # ============================================================
@@ -655,6 +696,17 @@ def run_all():
         ]:
             run_test(cat, fn)
 
+        log.warning('=== core seam ===')
+        cat = category('core')
+        for fn in [
+            test_core_serves_the_caps,
+            test_core_gate_and_caps_table_agree,
+            test_core_gate_names_workflows_the_registry_carries,
+            test_core_rejects_references_on_a_keyframe_model,
+            test_core_resolves_a_reference_model,
+        ]:
+            run_test(cat, fn)
+
     log.warning('=== Results ===')
     total_passed = 0
     total_failed = 0