diff --git a/modules/minimax/minimax_ui.py b/modules/minimax/minimax_ui.py
index 455901c5b..2be8e60a7 100644
--- a/modules/minimax/minimax_ui.py
+++ b/modules/minimax/minimax_ui.py
@@ -4,7 +4,7 @@ from modules import ui_sections, ui_symbols
from modules.ui_components import ToolButton
from modules.logger import log
from modules.video_models.models_def import models
-from modules.minimax import minimax_video
+from modules.minimax import minimax_video, minimax_references
debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
@@ -41,8 +41,9 @@ def create_ui(prompt, _negative, styles, overrides, script_inputs, mp4_fps, mp4_
with gr.Row():
last_image = gr.Image(label='Last image', elem_id='minimax_last_image', type='pil', image_mode='RGB', width=256, height=256)
with gr.Accordion(open=False, label="Reference media", elem_id='minimax_reference_accordion', visible=True) as reference_accordion:
- gr.HTML("""Upload up to 9 images, 3 videos, and 3 audio files
- The total number of files must not exceed 12
""", elem_id='minimax_reference_media_info', elem_classes=['smaller'])
+ caps = minimax_references.get_reference_caps('ref2va')
+ gr.HTML(f"""Upload up to {caps.max_images} images, {caps.max_videos} videos, and {caps.max_audios} audio files
+ The total number of files must not exceed {caps.max_references}
""", elem_id='minimax_reference_media_info', elem_classes=['smaller'])
reference_media = gr.Files(label="Reference media", interactive=True, elem_id="minimax_reference_media", visible=True)
with gr.Column(elem_id='minimax-output-column', scale=2) as _column_output:
diff --git a/modules/minimax/minimax_video.py b/modules/minimax/minimax_video.py
index 60e6ffe46..0804f8128 100644
--- a/modules/minimax/minimax_video.py
+++ b/modules/minimax/minimax_video.py
@@ -1,4 +1,3 @@
-import os
import time
from PIL import Image
import numpy as np
@@ -37,40 +36,32 @@ def load_model(model: str):
return None
-def prepare_inputs(workflow: str, p: processing.StableDiffusionProcessingVideo, init_image: Image.Image | None, last_image: Image.Image | None, reference_media: list | None):
- from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3ImageReference, MiniMaxH3VideoReference, MiniMaxH3AudioReference
- if workflow == 'fl2va':
- if init_image is not None:
- p.task_args['image'] = init_image
- if last_image is not None:
- p.task_args['last_image'] = last_image
- log.debug(f'Prepare inputs: workflow={workflow} first={init_image} last={last_image}')
- if workflow == 'ref2va':
- if reference_media is None or len(reference_media) == 0:
- return
- files = []
- references = []
- for fn in reference_media:
- try:
- if hasattr(fn, 'name'): # gradio tempfile wrapper as files end up uploaded and not embedded
- fn = fn.name
- if not os.path.exists(fn):
- log.warning(f'Prepare inputs: workflow={workflow} file="{fn}" not found')
- continue
- if fn.lower().endswith((".png", ".jpg", ".jpeg", ".webp")):
- files.append(fn)
- references.append(MiniMaxH3ImageReference.from_file(fn))
- elif fn.lower().endswith((".mp4", ".mov", ".avi")):
- files.append(fn)
- references.append(MiniMaxH3VideoReference.from_file(fn))
- elif fn.lower().endswith((".wav", ".mp3", ".flac", ".aac")):
- files.append(fn)
- references.append(MiniMaxH3AudioReference.from_file(fn))
- except Exception as e:
- log.error(f'Prepare inputs: workflow={workflow} file="{fn}" {e}')
- if len(references) > 0:
- p.task_args['references'] = references
- log.debug(f'Prepare inputs: workflow={workflow} files={files}')
+def unwrap_file(entry):
+ """The path behind a gradio file entry: an upload arrives as a tempfile wrapper or a dict, not a path."""
+ if hasattr(entry, 'name'):
+ return entry.name
+ if isinstance(entry, dict) and 'name' in entry:
+ return entry['name']
+ return entry
+
+
+def prepare_inputs(workflow: str | None, init_image: Image.Image | None, last_image: Image.Image | None, reference_media: list | None) -> dict:
+ """The task args a workflow conditions on, resolved before the model load so a rejected request costs nothing."""
+ from modules.minimax import minimax_references
+ if minimax_references.get_reference_caps(workflow) is not None:
+ entries = [unwrap_file(entry) for entry in (reference_media or [])]
+ references = minimax_references.resolve(workflow, entries, init_image)
+ log.debug(f'Prepare inputs: workflow={workflow} references={len(references)}')
+ return {'references': references}
+ task_args = {}
+ if init_image is not None:
+ task_args['image'] = init_image
+ if last_image is not None:
+ task_args['last_image'] = last_image
+ if reference_media:
+ log.warning(f'Video: op=reference workflow={workflow} references not supported, ignoring: count={len(reference_media)}')
+ log.debug(f'Prepare inputs: workflow={workflow} first={init_image} last={last_image}')
+ return task_args
def generate(task_id, _ui_state,
@@ -90,7 +81,7 @@ def generate(task_id, _ui_state,
**_kwargs,
):
video_utils.check_av()
- from modules.video_models import video_minimax
+ from modules.video_models import video_minimax, video_run
progress.add_task_to_queue(task_id)
with call_queue.get_lock():
@@ -100,12 +91,18 @@ def generate(task_id, _ui_state,
timer.process.reset()
# init vars
+ p = None
+ workflow = None # the incoming argument is the ui's display label, so the row and then the load supply the real one
pixels = None
num_frames = 0
video_file = None
aac_sample_rate = 32000
try:
+ # resolved off the registry row so a bad reference is rejected before the load, the same as on the api path
+ selected = models_def.find(engine, model)
+ workflow = getattr(selected, 'workflow', None)
+ task_args = prepare_inputs(workflow, init_image, last_image, reference_media)
workflow = load_model(model) # override workflow based on loaded model
if not workflow:
progress.finish_task(task_id)
@@ -135,7 +132,7 @@ def generate(task_id, _ui_state,
p.scripts = scripts_manager.scripts_video
p.script_args = args
- prepare_inputs(workflow, p, init_image, last_image, reference_media)
+ p.task_args.update(task_args)
_processed: processing.Processed = scripts_manager.scripts_video.run(p, *args)
processed = processing.process_images(p)
@@ -191,14 +188,18 @@ def generate(task_id, _ui_state,
if audio is not None:
del audio
+ except video_run.VideoError as e: # a rejected input, so the reason belongs in the output box and not only in the log
+ log.error(f'Video: engine="{engine}" model="{model}" workflow={workflow} {e}')
+ return None, f'Error: {e}'
except Exception as e:
log.error(f'Video: engine="{engine}" model="{model}" workflow={workflow} {e}')
errors.display(e, 'Video')
finally:
- jobid = getattr(shared.sd_model, 'sdnext_phaseid', None) # previous jobid if any
+ jobid = getattr(shared.sd_model, 'sdnext_phaseid', None) if shared.sd_loaded else None # sd_model loads on access, and a request rejected before the load must not trigger one
shared.state.end(jobid) # clear the previous job if exists
progress.finish_task(task_id)
- p.close()
+ if p is not None: # a request rejected before the processing object exists has nothing to close
+ p.close()
t1 = time.time()
resolution = f'{w}x{h}' if num_frames > 0 else None
diff --git a/test/test-video-references.py b/test/test-video-references.py
index 2290e7b3e..edd1d4cd8 100644
--- a/test/test-video-references.py
+++ b/test/test-video-references.py
@@ -145,6 +145,12 @@ def stub_video_reference(frames: int, fps: float):
return types.SimpleNamespace(frames=[None] * frames, fps=fps, kind='video')
+def make_png(name: str, width: int = 64, height: int = 64) -> str:
+ fn = os.path.join(tmpdir, name)
+ image(width, height).save(fn)
+ return fn
+
+
def make_wav(name: str, seconds: float = 1.0, rate: int = 32000, channels: int = 1) -> str:
fn = os.path.join(tmpdir, name)
with wave.open(fn, 'wb') as handle:
@@ -545,6 +551,13 @@ def test_built_image_is_rgb():
assert built[0].image.mode == 'RGB', f'mode={built[0].image.mode}'
+def test_corrupt_file_is_rejected_cleanly():
+ if not has_diffusers():
+ return 'diffusers not installed'
+ # an unreadable file has to surface as a rejection naming it, not as a traceback out of the decoder
+ return expect_error(lambda: refs.resolve('ref2va', [touch('corrupt.png')]), 'decode failed')
+
+
def test_short_video_is_rejected_after_the_decode():
if not has_av():
return 'av not installed'
@@ -597,6 +610,53 @@ def test_core_resolves_a_reference_model():
return True
+# ============================================================
+# Tab delegation
+# ============================================================
+
+def test_tab_unwraps_gradio_file_entries():
+ from modules.minimax import minimax_video
+ fn = touch('unwrap.png')
+ assert minimax_video.unwrap_file(fn) == fn, 'a plain path was rewritten'
+ assert minimax_video.unwrap_file(types.SimpleNamespace(name=fn)) == fn, 'a tempfile wrapper was not unwrapped'
+ assert minimax_video.unwrap_file({'name': fn}) == fn, 'a dict entry was not unwrapped'
+
+
+def test_tab_keeps_keyframes_on_a_keyframe_workflow():
+ from modules.minimax import minimax_video
+ task_args = minimax_video.prepare_inputs('fl2va', image(), image(), None)
+ assert sorted(task_args) == ['image', 'last_image'], f'{sorted(task_args)}'
+
+
+def test_tab_reports_references_it_cannot_use():
+ # uploads survive the accordion hiding when the row changes, and dropping them silently reads as a working request
+ from modules.minimax import minimax_video
+ seen = []
+ original = minimax_video.log.warning
+ minimax_video.log.warning = lambda msg, *a, **k: seen.append(str(msg))
+ try:
+ task_args = minimax_video.prepare_inputs('fl2va', image(), None, [touch('stale.png')])
+ finally:
+ minimax_video.log.warning = original
+ assert any('not supported' in message for message in seen), f'{seen}'
+ assert 'references' not in task_args, 'a keyframe workflow claimed the references'
+
+
+def test_tab_rejections_name_the_reason():
+ from modules.minimax import minimax_video
+ return expect_error(lambda: minimax_video.prepare_inputs('ref2va', None, None, [touch('notes.txt')]), 'unsupported media type')
+
+
+def test_tab_resolves_references_through_the_shared_funnel():
+ from modules.minimax import minimax_video
+ if not has_diffusers():
+ return 'diffusers not installed'
+ task_args = minimax_video.prepare_inputs('ref2va', image(), None, [types.SimpleNamespace(name=make_png('tab.png'))])
+ assert list(task_args) == ['references'], f'{list(task_args)}'
+ assert [reference.kind for reference in task_args['references']] == ['image'], f'{task_args}'
+ return True
+
+
# ============================================================
# Runner
# ============================================================
@@ -692,6 +752,7 @@ def run_all():
test_built_references_keep_the_request_order,
test_built_video_carries_its_frame_rate,
test_built_image_is_rgb,
+ test_corrupt_file_is_rejected_cleanly,
test_short_video_is_rejected_after_the_decode,
]:
run_test(cat, fn)
@@ -707,6 +768,17 @@ def run_all():
]:
run_test(cat, fn)
+ log.warning('=== tab delegation ===')
+ cat = category('tab')
+ for fn in [
+ test_tab_unwraps_gradio_file_entries,
+ test_tab_keeps_keyframes_on_a_keyframe_workflow,
+ test_tab_reports_references_it_cannot_use,
+ test_tab_rejections_name_the_reason,
+ test_tab_resolves_references_through_the_shared_funnel,
+ ]:
+ run_test(cat, fn)
+
log.warning('=== Results ===')
total_passed = 0
total_failed = 0