mirror of
https://github.com/vladmandic/automatic
synced 2026-09-20 01:31:13 +02:00
Merge pull request #5034 from vladmandic/feat/video-mixed-references
Feat/video mixed references
This commit is contained in:
Vladimir Mandic
GitHub
@@ -27,10 +27,10 @@ class ReqVideo(BaseModel):
|
||||
seed: int = Field(default=-1, title="Seed", description="Generation seed; -1 for random")
|
||||
guidance_scale: float = Field(default=-1.0, title="Guidance scale", description="CFG scale; -1 keeps the model default")
|
||||
guidance_true: float = Field(default=-1.0, title="True guidance", description="True CFG scale; -1 keeps the model default")
|
||||
init_image: str | None = Field(default=None, title="Init image", description="Base64, data URI, or upload reference for the first-frame image")
|
||||
init_image: str | None = Field(default=None, title="Init image", description="Base64 or data URI for the first-frame image; an upload reference resolves only where an extension provides the upload store")
|
||||
init_strength: float = Field(default=0.8, ge=0.0, le=1.0, title="Init strength", description="Denoising strength for the init image")
|
||||
last_image: str | None = Field(default=None, title="Last image", description="Base64, data URI, or upload reference for the last-frame image")
|
||||
references: list[str] = Field(default=[], title="References", description="Reference images for a reference workflow, in the order the model reads them; base64, data URIs, or upload references. At most 9, each within a 1:4 to 4:1 aspect ratio. Rejected on models that do not condition on references")
|
||||
last_image: str | None = Field(default=None, title="Last image", description="Base64 or data URI for the last-frame image; an upload reference resolves only where an extension provides the upload store")
|
||||
references: list[str] = Field(default=[], title="References", description="Reference images for a reference workflow, in the order the model reads them; base64 or data URIs, or upload references where an extension provides the upload store. Images only: the video core also conditions on video and audio references, which this endpoint cannot carry. At most 9, each within a 1:4 to 4:1 aspect ratio. Rejected on models that do not condition on references")
|
||||
vae_type: str = Field(default="Default", title="VAE type", description="Decode variant: Default, Tiny, Remote, or Upscale")
|
||||
vae_tile_frames: int = Field(default=16, ge=1, le=64, title="VAE tile frames", description="Frames per VAE decode tile")
|
||||
audio: bool = Field(default=True, title="Audio", description="Generate audio on models that support it")
|
||||
@@ -142,11 +142,14 @@ class APIVideo:
|
||||
`send_thumbnail`. Artifacts above the base64 size cap return `video` empty with
|
||||
`video_path` set; fetch those via `GET /sdapi/v1/video/file`.
|
||||
|
||||
`init_image` and `last_image` accept base64 data, data URIs, or upload references.
|
||||
`init_image` and `last_image` accept base64 data or data URIs. An `upload:` reference
|
||||
resolves only where an extension registers an upload store; without one it is rejected.
|
||||
Models whose workflow is `ref2va` condition on `references` instead: an ordered list of
|
||||
images the prompt addresses as `<Picture 1>`, `<Picture 2>` and so on, following list
|
||||
order. A single reference may also be passed as `init_image`. Reference images do not
|
||||
set the output canvas, and `last_image` is ignored.
|
||||
set the output canvas, and `last_image` is ignored. The workflow also conditions on video
|
||||
and audio references, addressed as `<Video i>` and `<Audio i>`, but they decode from files
|
||||
rather than from the wire, so this endpoint carries images alone.
|
||||
|
||||
Progress is reported on `GET /sdapi/v1/progress`; `POST /sdapi/v1/interrupt` cancels.
|
||||
Switching checkpoints via `override_settings` is not supported here; use
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
import os
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from PIL import Image
|
||||
from modules.logger import log
|
||||
|
||||
|
||||
debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
|
||||
REFERENCE_FPS = 24.0 # references are resampled onto the model's own frame rate before anything reads them
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReferenceCaps:
|
||||
"""Limits a reference workflow enforces, mirrored from the pipeline's own setup step so a bad
|
||||
request is rejected before the model load instead of after it. Frozen because the registry rows
|
||||
these describe are rewritten in place by the loader."""
|
||||
max_images: int = 9
|
||||
max_videos: int = 3
|
||||
max_audios: int = 3 # audio references only: a video's own soundtrack takes an <Audio i> label but no slot here
|
||||
max_references: int = 12 # not the sum of the three: 9 images and 3 videos is legal, 9 and 3 and 3 is not
|
||||
image_aspect: float = 4.0 # widest side ratio, rejected strictly so exactly 4:1 passes
|
||||
video_min_frames: int = 13 # after the resample: the conditioner samples at 2 fps and merges frames in pairs, so a shorter reference has no pair to merge
|
||||
video_max_seconds: float = 16.0 # a reference is truncated to the generated video, which tops out at 14.375s, so beyond this the decode is thrown away
|
||||
video_max_bytes: int = 2 * 1024 * 1024 * 1024 # decoding costs frames*width*height*3 plus a stacking copy, and the result is held across the model load
|
||||
audio_max_channels: int = 2 # mono is upmixed to stereo, more than two channels has no downmix
|
||||
audio_sample_rate: int = 32000 # the audio vae's own rate; any other rate resamples through torchaudio, which sdnext does not install
|
||||
|
||||
|
||||
# limits are per workflow rather than per model: the registry rows that carry ref2va differ only in which repo they load
|
||||
REFERENCE_CAPS = {
|
||||
'ref2va': ReferenceCaps(),
|
||||
}
|
||||
|
||||
|
||||
def get_reference_caps(workflow):
|
||||
"""Reference limits of a workflow, None for the workflows that condition on none."""
|
||||
return REFERENCE_CAPS.get(workflow, None)
|
||||
|
||||
|
||||
def reference_error(msg: str, code: int = 400):
|
||||
from modules.video_models.video_run import VideoError
|
||||
return VideoError(msg, code)
|
||||
|
||||
|
||||
def source_of(entry) -> str:
|
||||
return f'file="{entry}"' if isinstance(entry, (str, os.PathLike)) else f'type={type(entry).__name__}'
|
||||
|
||||
|
||||
def classify_entries(entries: list) -> list:
|
||||
"""(label, kind, entry) per reference in the order given, labelled from 1 to match the model's own numbering."""
|
||||
from modules.video_models import video_utils
|
||||
items = []
|
||||
for index, entry in enumerate(entries):
|
||||
label = index + 1
|
||||
if isinstance(entry, Image.Image):
|
||||
items.append((label, 'image', entry))
|
||||
continue
|
||||
if not isinstance(entry, (str, os.PathLike)):
|
||||
raise reference_error(f'reference {label} unsupported input: type={type(entry).__name__} expected an image or a local file path')
|
||||
fn = str(entry)
|
||||
if fn.lower().startswith(('http://', 'https://')):
|
||||
# the reference classes fetch a url and decode whatever comes back, so a request never gets to name one
|
||||
raise reference_error(f'reference {label} not a local file: url="{fn}"')
|
||||
if not os.path.isfile(fn):
|
||||
raise reference_error(f'reference {label} file not found: file="{fn}"')
|
||||
kind = video_utils.classify_extension(fn)
|
||||
if kind is None:
|
||||
supported = [ext for extensions in video_utils.MEDIA_EXTENSIONS.values() for ext in extensions]
|
||||
raise reference_error(f'reference {label} unsupported media type: file="{fn}" supported={supported}')
|
||||
items.append((label, kind, fn))
|
||||
return items
|
||||
|
||||
|
||||
def check_counts(caps: ReferenceCaps, kinds: list):
|
||||
"""Per kind, then the total, then the pairing rule: the order the setup step itself checks in."""
|
||||
for kind, limit in (('image', caps.max_images), ('video', caps.max_videos), ('audio', caps.max_audios)):
|
||||
count = kinds.count(kind)
|
||||
if count > limit:
|
||||
raise reference_error(f'too many {kind} references: count={count} max={limit}')
|
||||
if len(kinds) > caps.max_references:
|
||||
raise reference_error(f'too many references: count={len(kinds)} max={caps.max_references}')
|
||||
if set(kinds) == {'audio'}: # an audio reference goes to the audio vae alone and conditions no picture on its own
|
||||
raise reference_error(f'audio references must be paired with an image or video reference: count={len(kinds)}')
|
||||
|
||||
|
||||
def check_video_probe(caps: ReferenceCaps, label: int, fn: str, probe):
|
||||
if not probe.fps:
|
||||
raise reference_error(f'reference {label} video frame rate unknown: file="{fn}"')
|
||||
if probe.duration is not None and probe.duration > caps.video_max_seconds:
|
||||
raise reference_error(f'reference {label} video too long: seconds={probe.duration:.1f} max={caps.video_max_seconds:g}')
|
||||
frames = probe.frames or (math.ceil(probe.duration * probe.fps) if probe.duration else None)
|
||||
if frames and probe.width and probe.height:
|
||||
size = frames * probe.width * probe.height * 3
|
||||
if size > caps.video_max_bytes:
|
||||
raise reference_error(f'reference {label} video too large to decode: estimate={size / 1024 ** 3:.1f}GB max={caps.video_max_bytes / 1024 ** 3:.1f}GB size={probe.width}x{probe.height} frames={frames}')
|
||||
if probe.width and probe.height and (probe.width > caps.image_aspect * probe.height or probe.height > caps.image_aspect * probe.width):
|
||||
raise reference_error(f'reference {label} aspect ratio out of range: size={probe.width}x{probe.height} supported=1:{caps.image_aspect:g}..{caps.image_aspect:g}:1')
|
||||
|
||||
|
||||
def check_audio_probe(caps: ReferenceCaps, label: int, probe):
|
||||
from modules.video_models import video_utils
|
||||
if probe.channels is not None and probe.channels > caps.audio_max_channels:
|
||||
raise reference_error(f'reference {label} too many audio channels: channels={probe.channels} max={caps.audio_max_channels}')
|
||||
if probe.sample_rate != caps.audio_sample_rate and not video_utils.has_torchaudio():
|
||||
raise reference_error(f'reference {label} resampling {probe.sample_rate}Hz to {caps.audio_sample_rate}Hz requires torchaudio')
|
||||
|
||||
|
||||
def preflight_probe(caps: ReferenceCaps, items: list):
|
||||
"""Header checks for the file entries, each one standing in for a failure that would otherwise
|
||||
land after the model load. A container that reports no duration is decoded unbounded."""
|
||||
from modules.video_models import video_utils
|
||||
for label, kind, entry in items:
|
||||
if kind == 'image':
|
||||
continue
|
||||
probe = video_utils.probe_media(entry, kind)
|
||||
if probe is None:
|
||||
raise reference_error(f'reference {label} unreadable: file="{entry}"')
|
||||
debug(f'Video: op=reference probe={label} kind={kind} file="{entry}" fps={probe.fps} frames={probe.frames} seconds={probe.duration} channels={probe.channels} rate={probe.sample_rate}')
|
||||
if kind == 'video':
|
||||
check_video_probe(caps, label, entry, probe)
|
||||
elif probe.sample_rate is None:
|
||||
raise reference_error(f'reference {label} has no audio stream: file="{entry}"')
|
||||
if probe.sample_rate is not None: # a video's own soundtrack goes through the same normalization an audio reference does
|
||||
check_audio_probe(caps, label, probe)
|
||||
|
||||
|
||||
def build_reference_objects(items: list) -> list:
|
||||
from diffusers.utils import load_image
|
||||
from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3AudioReference, MiniMaxH3ImageReference, MiniMaxH3VideoReference
|
||||
references = []
|
||||
for label, kind, entry in items:
|
||||
try:
|
||||
if kind == 'image':
|
||||
# the reference encoder reads the image array raw, and load_image is what applies the exif transpose and the rgb conversion
|
||||
references.append(MiniMaxH3ImageReference(image=load_image(entry)))
|
||||
elif kind == 'video':
|
||||
references.append(MiniMaxH3VideoReference.from_file(entry))
|
||||
else:
|
||||
references.append(MiniMaxH3AudioReference.from_file(entry))
|
||||
except Exception as e:
|
||||
raise reference_error(f'reference {label} decode failed: {source_of(entry)} {e}') from e
|
||||
return references
|
||||
|
||||
|
||||
def check_decoded(caps: ReferenceCaps, items: list, references: list):
|
||||
"""The checks that need the decoded media: an image's real size, and a video's frame count at the
|
||||
rate it is resampled to, which is what the conditioner counts."""
|
||||
for (label, kind, _entry), reference in zip(items, references):
|
||||
if kind == 'image':
|
||||
width, height = reference.image.size
|
||||
if width <= 0 or height <= 0:
|
||||
raise reference_error(f'reference {label} image size invalid: size={width}x{height}')
|
||||
if width > caps.image_aspect * height or height > caps.image_aspect * width:
|
||||
raise reference_error(f'reference {label} aspect ratio out of range: size={width}x{height} supported=1:{caps.image_aspect:g}..{caps.image_aspect:g}:1')
|
||||
elif kind == 'video':
|
||||
frames, fps = len(reference.frames), float(reference.fps or REFERENCE_FPS)
|
||||
resampled = math.floor(frames * REFERENCE_FPS / fps + 0.5) if fps > 0 else 0 # the rounding the resample itself uses
|
||||
if resampled < caps.video_min_frames:
|
||||
raise reference_error(f'reference {label} video too short: frames={frames} fps={fps:g} min={caps.video_min_frames}@{REFERENCE_FPS:g}fps')
|
||||
|
||||
|
||||
def resolve(workflow: str, references: list | None, init_image=None) -> list:
|
||||
"""The ordered reference objects a reference workflow conditions on, from decoded images and
|
||||
local file paths. Order is preserved exactly: it fixes the <Picture i>, <Video i> and <Audio i>
|
||||
labels a prompt addresses, and the shared clock the references are laid on."""
|
||||
caps = get_reference_caps(workflow)
|
||||
if caps is None:
|
||||
raise reference_error(f'workflow conditions on no references: workflow={workflow}')
|
||||
entries = list(references) if references else ([init_image] if init_image is not None else [])
|
||||
if len(entries) == 0:
|
||||
# keep the workflow named here: the api test probes for a reference server by matching it in the rejection
|
||||
raise reference_error(f'No reference media provided. The {workflow} workflow conditions on references, so at least one is required.')
|
||||
items = classify_entries(entries)
|
||||
kinds = [kind for _label, kind, _entry in items]
|
||||
check_counts(caps, kinds)
|
||||
if 'video' in kinds or 'audio' in kinds:
|
||||
from modules.video_models import video_utils
|
||||
if not video_utils.check_av():
|
||||
raise reference_error('video and audio references require the av package', 500)
|
||||
preflight_probe(caps, items)
|
||||
built = build_reference_objects(items)
|
||||
check_decoded(caps, items, built)
|
||||
log.debug(f'Video: op=reference workflow={workflow} images={kinds.count("image")} videos={kinds.count("video")} audio={kinds.count("audio")} total={len(built)}')
|
||||
return built
|
||||
@@ -4,7 +4,7 @@ from modules import ui_sections, ui_symbols
|
||||
from modules.ui_components import ToolButton
|
||||
from modules.logger import log
|
||||
from modules.video_models.models_def import models
|
||||
from modules.minimax import minimax_video
|
||||
from modules.minimax import minimax_video, minimax_references
|
||||
|
||||
|
||||
debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
|
||||
@@ -41,8 +41,9 @@ def create_ui(prompt, _negative, styles, overrides, script_inputs, mp4_fps, mp4_
|
||||
with gr.Row():
|
||||
last_image = gr.Image(label='Last image', elem_id='minimax_last_image', type='pil', image_mode='RGB', width=256, height=256)
|
||||
with gr.Accordion(open=False, label="Reference media", elem_id='minimax_reference_accordion', visible=True) as reference_accordion:
|
||||
gr.HTML("""Upload up to 9 images, 3 videos, and 3 audio files<br>
|
||||
The total number of files must not exceed 12<br><br>""", elem_id='minimax_reference_media_info', elem_classes=['smaller'])
|
||||
caps = minimax_references.get_reference_caps('ref2va')
|
||||
gr.HTML(f"""Upload up to {caps.max_images} images, {caps.max_videos} videos, and {caps.max_audios} audio files<br>
|
||||
The total number of files must not exceed {caps.max_references}<br><br>""", elem_id='minimax_reference_media_info', elem_classes=['smaller'])
|
||||
reference_media = gr.Files(label="Reference media", interactive=True, elem_id="minimax_reference_media", visible=True)
|
||||
|
||||
with gr.Column(elem_id='minimax-output-column', scale=2) as _column_output:
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import os
|
||||
import time
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
@@ -37,40 +36,32 @@ def load_model(model: str):
|
||||
return None
|
||||
|
||||
|
||||
def prepare_inputs(workflow: str, p: processing.StableDiffusionProcessingVideo, init_image: Image.Image | None, last_image: Image.Image | None, reference_media: list | None):
|
||||
from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3ImageReference, MiniMaxH3VideoReference, MiniMaxH3AudioReference
|
||||
if workflow == 'fl2va':
|
||||
if init_image is not None:
|
||||
p.task_args['image'] = init_image
|
||||
if last_image is not None:
|
||||
p.task_args['last_image'] = last_image
|
||||
log.debug(f'Prepare inputs: workflow={workflow} first={init_image} last={last_image}')
|
||||
if workflow == 'ref2va':
|
||||
if reference_media is None or len(reference_media) == 0:
|
||||
return
|
||||
files = []
|
||||
references = []
|
||||
for fn in reference_media:
|
||||
try:
|
||||
if hasattr(fn, 'name'): # gradio tempfile wrapper as files end up uploaded and not embedded
|
||||
fn = fn.name
|
||||
if not os.path.exists(fn):
|
||||
log.warning(f'Prepare inputs: workflow={workflow} file="{fn}" not found')
|
||||
continue
|
||||
if fn.lower().endswith((".png", ".jpg", ".jpeg", ".webp")):
|
||||
files.append(fn)
|
||||
references.append(MiniMaxH3ImageReference.from_file(fn))
|
||||
elif fn.lower().endswith((".mp4", ".mov", ".avi")):
|
||||
files.append(fn)
|
||||
references.append(MiniMaxH3VideoReference.from_file(fn))
|
||||
elif fn.lower().endswith((".wav", ".mp3", ".flac", ".aac")):
|
||||
files.append(fn)
|
||||
references.append(MiniMaxH3AudioReference.from_file(fn))
|
||||
except Exception as e:
|
||||
log.error(f'Prepare inputs: workflow={workflow} file="{fn}" {e}')
|
||||
if len(references) > 0:
|
||||
p.task_args['references'] = references
|
||||
log.debug(f'Prepare inputs: workflow={workflow} files={files}')
|
||||
def unwrap_file(entry):
|
||||
"""The path behind a gradio file entry: an upload arrives as a tempfile wrapper or a dict, not a path."""
|
||||
if hasattr(entry, 'name'):
|
||||
return entry.name
|
||||
if isinstance(entry, dict) and 'name' in entry:
|
||||
return entry['name']
|
||||
return entry
|
||||
|
||||
|
||||
def prepare_inputs(workflow: str | None, init_image: Image.Image | None, last_image: Image.Image | None, reference_media: list | None) -> dict:
|
||||
"""The task args a workflow conditions on, resolved before the model load so a rejected request costs nothing."""
|
||||
from modules.minimax import minimax_references
|
||||
if minimax_references.get_reference_caps(workflow) is not None:
|
||||
entries = [unwrap_file(entry) for entry in (reference_media or [])]
|
||||
references = minimax_references.resolve(workflow, entries, init_image)
|
||||
log.debug(f'Prepare inputs: workflow={workflow} references={len(references)}')
|
||||
return {'references': references}
|
||||
task_args = {}
|
||||
if init_image is not None:
|
||||
task_args['image'] = init_image
|
||||
if last_image is not None:
|
||||
task_args['last_image'] = last_image
|
||||
if reference_media:
|
||||
log.warning(f'Video: op=reference workflow={workflow} references not supported, ignoring: count={len(reference_media)}')
|
||||
log.debug(f'Prepare inputs: workflow={workflow} first={init_image} last={last_image}')
|
||||
return task_args
|
||||
|
||||
|
||||
def generate(task_id, _ui_state,
|
||||
@@ -90,7 +81,7 @@ def generate(task_id, _ui_state,
|
||||
**_kwargs,
|
||||
):
|
||||
video_utils.check_av()
|
||||
from modules.video_models import video_minimax
|
||||
from modules.video_models import video_minimax, video_run
|
||||
progress.add_task_to_queue(task_id)
|
||||
|
||||
with call_queue.get_lock():
|
||||
@@ -100,12 +91,18 @@ def generate(task_id, _ui_state,
|
||||
timer.process.reset()
|
||||
|
||||
# init vars
|
||||
p = None
|
||||
workflow = None # the incoming argument is the ui's display label, so the row and then the load supply the real one
|
||||
pixels = None
|
||||
num_frames = 0
|
||||
video_file = None
|
||||
aac_sample_rate = 32000
|
||||
|
||||
try:
|
||||
# resolved off the registry row so a bad reference is rejected before the load, the same as on the api path
|
||||
selected = models_def.find(engine, model)
|
||||
workflow = getattr(selected, 'workflow', None)
|
||||
task_args = prepare_inputs(workflow, init_image, last_image, reference_media)
|
||||
workflow = load_model(model) # override workflow based on loaded model
|
||||
if not workflow:
|
||||
progress.finish_task(task_id)
|
||||
@@ -135,7 +132,7 @@ def generate(task_id, _ui_state,
|
||||
p.scripts = scripts_manager.scripts_video
|
||||
p.script_args = args
|
||||
|
||||
prepare_inputs(workflow, p, init_image, last_image, reference_media)
|
||||
p.task_args.update(task_args)
|
||||
|
||||
_processed: processing.Processed = scripts_manager.scripts_video.run(p, *args)
|
||||
processed = processing.process_images(p)
|
||||
@@ -191,14 +188,18 @@ def generate(task_id, _ui_state,
|
||||
if audio is not None:
|
||||
del audio
|
||||
|
||||
except video_run.VideoError as e: # a rejected input, so the reason belongs in the output box and not only in the log
|
||||
log.error(f'Video: engine="{engine}" model="{model}" workflow={workflow} {e}')
|
||||
return None, f'Error: {e}'
|
||||
except Exception as e:
|
||||
log.error(f'Video: engine="{engine}" model="{model}" workflow={workflow} {e}')
|
||||
errors.display(e, 'Video')
|
||||
finally:
|
||||
jobid = getattr(shared.sd_model, 'sdnext_phaseid', None) # previous jobid if any
|
||||
jobid = getattr(shared.sd_model, 'sdnext_phaseid', None) if shared.sd_loaded else None # sd_model loads on access, and a request rejected before the load must not trigger one
|
||||
shared.state.end(jobid) # clear the previous job if exists
|
||||
progress.finish_task(task_id)
|
||||
p.close()
|
||||
if p is not None: # a request rejected before the processing object exists has nothing to close
|
||||
p.close()
|
||||
|
||||
t1 = time.time()
|
||||
resolution = f'{w}x{h}' if num_frames > 0 else None
|
||||
|
||||
@@ -488,6 +488,8 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:l
|
||||
clean[k] = v.shape
|
||||
elif isinstance(v, list) and len(v) > 0 and (isinstance(v[0], torch.Tensor) or isinstance(v[0], np.ndarray)):
|
||||
clean[k] = [x.shape for x in v]
|
||||
elif isinstance(v, list) and len(v) > 0 and hasattr(v[0], 'kind'): # media references carry decoded frames and waveforms
|
||||
clean[k] = [getattr(x, 'kind', type(x).__name__) for x in v]
|
||||
elif not debug_enabled and k.endswith('_embeds'):
|
||||
del clean[k]
|
||||
clean['prompt'] = 'embeds'
|
||||
|
||||
@@ -9,7 +9,7 @@ from modules.paths import resolve_output_path
|
||||
|
||||
|
||||
debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
|
||||
MAX_IMAGE_REFERENCES = 9 # mirrors the reference setup block's own limit; reading it off the pipe would deep-copy the block tree per access
|
||||
REFERENCE_WORKFLOWS = ('ref2va',) # workflows that condition on references; the resolver and the limits it enforces live with the architecture
|
||||
|
||||
|
||||
class VideoError(Exception):
|
||||
@@ -62,29 +62,27 @@ def resolve_model(engine: str | None, model: str | None) -> tuple[models_def.Mod
|
||||
return selected, False
|
||||
|
||||
|
||||
def reference_caps(workflow: str | None):
|
||||
"""Reference limits of a workflow, None when it conditions on none. The seam a client reads
|
||||
instead of mirroring the numbers."""
|
||||
if workflow not in REFERENCE_WORKFLOWS:
|
||||
return None
|
||||
from modules.minimax import minimax_references
|
||||
return minimax_references.get_reference_caps(workflow)
|
||||
|
||||
|
||||
def validate_references(selected: models_def.Model, references: list | None, init_image) -> list | None:
|
||||
"""Return the ordered reference images for a reference workflow, None for every other model.
|
||||
"""Return the ordered references a reference workflow conditions on, None for every other model.
|
||||
Reference conditioning is exclusive to ref2va: its partition holds no keyframe transformer, and
|
||||
a mismatched request would only fail once the pipeline reached a component it never loaded.
|
||||
Checks run before the model load so a rejected request costs nothing."""
|
||||
workflow = getattr(selected, 'workflow', None)
|
||||
if workflow != 'ref2va':
|
||||
if workflow not in REFERENCE_WORKFLOWS:
|
||||
if references:
|
||||
raise VideoError(f'reference images require a ref2va model: model="{selected.name}" workflow={workflow}', 400)
|
||||
raise VideoError(f'reference media requires a reference workflow: model="{selected.name}" workflow={workflow} supported={list(REFERENCE_WORKFLOWS)}', 400)
|
||||
return None
|
||||
refs = list(references) if references else ([init_image] if init_image is not None else [])
|
||||
if len(refs) == 0:
|
||||
raise VideoError('No reference image provided. The ref2va workflow conditions on reference images, so at least one is required.', 400)
|
||||
if len(refs) > MAX_IMAGE_REFERENCES:
|
||||
raise VideoError(f'too many reference images: count={len(refs)} max={MAX_IMAGE_REFERENCES}', 400)
|
||||
for index, image in enumerate(refs):
|
||||
size = getattr(image, 'size', None)
|
||||
if size is None or len(size) != 2:
|
||||
raise VideoError(f'reference {index + 1} is not an image: type={type(image).__name__}', 400)
|
||||
width, height = size
|
||||
if width > 4 * height or height > 4 * width: # the same bound the pipeline enforces, raised here where it is free
|
||||
raise VideoError(f'reference {index + 1} aspect ratio out of range: size={width}x{height} supported=1:4..4:1', 400)
|
||||
return refs
|
||||
from modules.minimax import minimax_references
|
||||
return minimax_references.resolve(workflow, references, init_image)
|
||||
|
||||
|
||||
def run(selected: models_def.Model, *,
|
||||
@@ -188,10 +186,8 @@ def run(selected: models_def.Model, *,
|
||||
# unresized since the pipeline defines its own canvas placement per anchor
|
||||
p.video_still = int(frames) <= 1
|
||||
if refs is not None:
|
||||
from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3ImageReference
|
||||
# references outrank the keyframe inputs in every block, so those stay unset; the reference
|
||||
# encoder reads the image array as (height, width, 3) and never converts it itself
|
||||
p.task_args['references'] = [MiniMaxH3ImageReference(image=image.convert('RGB')) for image in refs]
|
||||
# references outrank the keyframe inputs in every block, so those stay unset
|
||||
p.task_args['references'] = refs
|
||||
if last_image is not None:
|
||||
log.warning(f'Video: op=reference model="{selected.name}" last frame not supported, ignoring')
|
||||
else:
|
||||
|
||||
@@ -2,6 +2,8 @@ import os
|
||||
import sys
|
||||
import time
|
||||
import inspect
|
||||
import importlib.util
|
||||
from dataclasses import dataclass
|
||||
from PIL import Image
|
||||
from installer import install
|
||||
from modules import shared, sd_models, timer, errors, devices
|
||||
@@ -10,6 +12,24 @@ from modules.video_models.video_codecs import codecs_config
|
||||
|
||||
|
||||
debug = log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
|
||||
MEDIA_EXTENSIONS = {
|
||||
'image': ('.png', '.jpg', '.jpeg', '.webp'),
|
||||
'video': ('.mp4', '.mov', '.avi'),
|
||||
'audio': ('.wav', '.mp3', '.flac', '.aac'),
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaProbe:
|
||||
"""What a container header reports, without decoding any of it."""
|
||||
kind: str
|
||||
fps: float | None = None
|
||||
frames: int | None = None
|
||||
duration: float | None = None # seconds
|
||||
width: int | None = None
|
||||
height: int | None = None
|
||||
channels: int | None = None
|
||||
sample_rate: int | None = None
|
||||
|
||||
|
||||
def queue_err(msg):
|
||||
@@ -44,6 +64,55 @@ def check_av():
|
||||
return av
|
||||
|
||||
|
||||
def has_torchaudio():
|
||||
# never installed on demand: torchaudio wheels pin a torch build and would replace it under the running server
|
||||
try:
|
||||
return importlib.util.find_spec('torchaudio') is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def classify_extension(fn: str):
|
||||
"""Media kind of a filename, None when the extension is not one sdnext reads."""
|
||||
lower = str(fn).lower()
|
||||
for kind, extensions in MEDIA_EXTENSIONS.items():
|
||||
if lower.endswith(extensions):
|
||||
return kind
|
||||
return None
|
||||
|
||||
|
||||
def probe_media(fn: str, kind: str):
|
||||
"""Container metadata for a media file, None when it cannot be opened. Reads headers only, so
|
||||
a file too large or too short to use is rejected before anything decodes it."""
|
||||
av = check_av()
|
||||
if not av:
|
||||
return None
|
||||
probe = MediaProbe(kind=kind)
|
||||
try:
|
||||
with av.open(fn) as container:
|
||||
if kind == 'video' and container.streams.video: # an audio file with cover art carries a video stream that is not frames
|
||||
stream = container.streams.video[0]
|
||||
rate = stream.average_rate or stream.guessed_rate # average_rate is a Fraction and can be a falsy 0/1, which is why the decoder falls back the same way
|
||||
probe.fps = float(rate) if rate else None
|
||||
probe.frames = stream.frames or None # 0 means the container carries no count, not an empty file
|
||||
probe.width, probe.height = stream.codec_context.width, stream.codec_context.height
|
||||
if stream.duration is not None and stream.time_base is not None:
|
||||
probe.duration = float(stream.duration * stream.time_base) # stream durations are in time_base units
|
||||
elif container.duration is not None:
|
||||
probe.duration = container.duration / 1000000 # container durations are in AV_TIME_BASE units
|
||||
elif probe.frames and probe.fps:
|
||||
probe.duration = probe.frames / probe.fps
|
||||
if container.streams.audio:
|
||||
stream = container.streams.audio[0]
|
||||
# the soundtrack decoder converts to planar float keeping the container's own rate and layout, so these are the values it yields
|
||||
probe.channels = getattr(stream, 'channels', None) or getattr(getattr(stream, 'layout', None), 'nb_channels', None)
|
||||
probe.sample_rate = int(stream.codec_context.sample_rate)
|
||||
except Exception as e:
|
||||
debug(f'Video probe: file="{fn}" {e}')
|
||||
return None
|
||||
return probe
|
||||
|
||||
|
||||
def hijack_encode_image(*args, **kwargs):
|
||||
t0 = time.time()
|
||||
try:
|
||||
|
||||
@@ -208,6 +208,7 @@ class VideoAPITest:
|
||||
self._category = 'still'
|
||||
print("\n--- Still Mode Tests ---")
|
||||
data, elapsed = self._video({'frames': 1})
|
||||
# matches the workflow named in the resolver's nothing-to-condition-on rejection; rewording it there skips every later category
|
||||
if data.get('error') == 400 and 'ref2va' in str(data.get('detail', '')):
|
||||
self.ref2va = True # the loaded model conditions on references; every later request carries one
|
||||
data, elapsed = self._video({'frames': 1})
|
||||
|
||||
@@ -0,0 +1,802 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
Offline unit tests for mixed-media reference validation in modules.minimax.minimax_references.
|
||||
|
||||
The resolver turns decoded images and local file paths into the reference objects a ref2va
|
||||
workflow conditions on, rejecting a bad request before the model load. Its checks run cheapest
|
||||
first, so the count rules never open a file and the decode rules never run on a request the
|
||||
counts already rejected.
|
||||
|
||||
Covers:
|
||||
|
||||
- ``ReferenceCaps`` values and immutability, and the workflow lookup that serves them
|
||||
- classification of every extension sdnext reads, plus the rejections: an unknown type, a url,
|
||||
a missing file, an unsupported extension
|
||||
- the count rules in the order the pipeline itself checks them, per kind before the total
|
||||
- the pairing rule that an all-audio request has nothing to condition
|
||||
- order preservation across a mixed request
|
||||
- the aspect and frame-count rules that need the decoded media
|
||||
- container header probing, and the guards for a video too long, too large, or too short
|
||||
- the two soft dependencies, forced to be absent so their rejections are observed rather than
|
||||
assumed
|
||||
|
||||
No running server required, and no model is loaded. The decode cases need ``av`` and the
|
||||
construction cases need ``diffusers``; both skip with a reason when absent.
|
||||
|
||||
Usage:
|
||||
python test/test-video-references.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import math
|
||||
import wave
|
||||
import struct
|
||||
import types
|
||||
import tempfile
|
||||
|
||||
script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, script_dir)
|
||||
os.chdir(script_dir)
|
||||
|
||||
os.environ['SD_INSTALL_QUIET'] = '1'
|
||||
|
||||
# Bootstrap cmd_args before any module that pulls in shared.py.
|
||||
import modules.cmd_args # pylint: disable=wrong-import-position
|
||||
import installer # pylint: disable=wrong-import-position
|
||||
orig_argv = sys.argv
|
||||
sys.argv = [sys.argv[0]]
|
||||
try:
|
||||
modules.cmd_args.parse_args()
|
||||
finally:
|
||||
sys.argv = orig_argv
|
||||
installer.add_args(modules.cmd_args.parser)
|
||||
modules.cmd_args.parsed, _ = modules.cmd_args.parser.parse_known_args([])
|
||||
|
||||
from PIL import Image # pylint: disable=wrong-import-position
|
||||
from modules.errors import log # pylint: disable=wrong-import-position
|
||||
from modules import shared # pylint: disable=wrong-import-position,unused-import
|
||||
from modules.video_models import video_utils # pylint: disable=wrong-import-position
|
||||
from modules.minimax import minimax_references as refs # pylint: disable=wrong-import-position
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Test infrastructure
|
||||
# ============================================================
|
||||
|
||||
results: dict[str, dict] = {}
|
||||
tmpdir = None
|
||||
|
||||
|
||||
def category(name: str):
|
||||
if name not in results:
|
||||
results[name] = {'passed': 0, 'failed': 0, 'skipped': 0, 'tests': []}
|
||||
return name
|
||||
|
||||
|
||||
def record(cat: str, passed: bool, name: str, detail: str = ''):
|
||||
status = 'PASS' if passed else 'FAIL'
|
||||
results[cat]['passed' if passed else 'failed'] += 1
|
||||
results[cat]['tests'].append((status, name))
|
||||
msg = f' {status}: {name}'
|
||||
if detail:
|
||||
msg += f' ({detail})'
|
||||
if passed:
|
||||
log.info(msg)
|
||||
else:
|
||||
log.error(msg)
|
||||
|
||||
|
||||
def skip(cat: str, name: str, reason: str):
|
||||
results[cat]['skipped'] += 1
|
||||
results[cat]['tests'].append(('SKIP', name))
|
||||
log.warning(f' SKIP: {name} ({reason})')
|
||||
|
||||
|
||||
def run_test(cat: str, fn):
|
||||
name = fn.__name__
|
||||
try:
|
||||
ok = fn()
|
||||
if ok is False:
|
||||
record(cat, False, name)
|
||||
elif isinstance(ok, str):
|
||||
skip(cat, name, ok)
|
||||
else:
|
||||
record(cat, True, name)
|
||||
except AssertionError as e:
|
||||
record(cat, False, name, str(e))
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
record(cat, False, name, f'exception: {type(e).__name__}: {e}')
|
||||
|
||||
|
||||
def expect_error(fn, fragment: str, code: int = 400):
|
||||
"""Run fn, assert it rejects with the given code and a message naming the reason."""
|
||||
try:
|
||||
fn()
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
assert getattr(e, 'code', None) == code, f'expected code={code} got code={getattr(e, "code", None)}: {e}'
|
||||
assert fragment in str(e), f'expected "{fragment}" in "{e}"'
|
||||
return True
|
||||
raise AssertionError(f'expected a rejection containing "{fragment}"')
|
||||
|
||||
|
||||
def touch(name: str) -> str:
|
||||
"""An empty file with a real extension: enough for every check that runs before the decode."""
|
||||
fn = os.path.join(tmpdir, name)
|
||||
with open(fn, 'wb'):
|
||||
pass
|
||||
return fn
|
||||
|
||||
|
||||
def image(width: int = 64, height: int = 64):
|
||||
return Image.new('RGB', (width, height))
|
||||
|
||||
|
||||
def caps():
|
||||
return refs.get_reference_caps('ref2va')
|
||||
|
||||
|
||||
def stub_image_reference(width: int, height: int):
|
||||
"""What check_decoded reads off an image reference, without constructing the real one."""
|
||||
return types.SimpleNamespace(image=image(width, height), kind='image')
|
||||
|
||||
|
||||
def stub_video_reference(frames: int, fps: float):
|
||||
return types.SimpleNamespace(frames=[None] * frames, fps=fps, kind='video')
|
||||
|
||||
|
||||
def make_png(name: str, width: int = 64, height: int = 64) -> str:
|
||||
fn = os.path.join(tmpdir, name)
|
||||
image(width, height).save(fn)
|
||||
return fn
|
||||
|
||||
|
||||
def make_wav(name: str, seconds: float = 1.0, rate: int = 32000, channels: int = 1) -> str:
|
||||
fn = os.path.join(tmpdir, name)
|
||||
with wave.open(fn, 'wb') as handle:
|
||||
handle.setnchannels(channels)
|
||||
handle.setsampwidth(2)
|
||||
handle.setframerate(rate)
|
||||
count = int(seconds * rate) * channels
|
||||
handle.writeframes(struct.pack('<' + 'h' * count, *([0] * count)))
|
||||
return fn
|
||||
|
||||
|
||||
def make_mp4(name: str, frames: int = 20, width: int = 64, height: int = 64, fps: int = 24) -> str | None:
|
||||
av = video_utils.check_av()
|
||||
if not av:
|
||||
return None
|
||||
import numpy as np
|
||||
fn = os.path.join(tmpdir, name)
|
||||
try:
|
||||
with av.open(fn, mode='w') as container:
|
||||
stream = container.add_stream('libx264', rate=fps)
|
||||
stream.width, stream.height, stream.pix_fmt = width, height, 'yuv420p'
|
||||
for index in range(frames):
|
||||
array = np.full((height, width, 3), (index * 8) % 256, dtype=np.uint8)
|
||||
for packet in stream.encode(av.VideoFrame.from_ndarray(array, format='rgb24')):
|
||||
container.mux(packet)
|
||||
for packet in stream.encode():
|
||||
container.mux(packet)
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
log.warning(f'test fixture: file="{fn}" {e}')
|
||||
return None
|
||||
return fn
|
||||
|
||||
|
||||
def has_av() -> bool:
|
||||
return bool(video_utils.check_av())
|
||||
|
||||
|
||||
def has_diffusers() -> bool:
|
||||
try:
|
||||
from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3ImageReference # pylint: disable=unused-import
|
||||
return True
|
||||
except Exception: # pylint: disable=broad-except
|
||||
return False
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Caps
|
||||
# ============================================================
|
||||
|
||||
def test_caps_mirror_the_pipeline_limits():
|
||||
c = caps()
|
||||
assert c is not None, 'ref2va has no caps'
|
||||
assert (c.max_images, c.max_videos, c.max_audios, c.max_references) == (9, 3, 3, 12), f'{c}'
|
||||
assert c.image_aspect == 4.0, f'{c.image_aspect}'
|
||||
assert c.video_min_frames == 13, f'{c.video_min_frames}'
|
||||
assert c.audio_sample_rate == 32000, f'{c.audio_sample_rate}'
|
||||
assert c.audio_max_channels == 2, f'{c.audio_max_channels}'
|
||||
|
||||
|
||||
def test_caps_total_is_not_the_sum_of_the_kinds():
|
||||
c = caps()
|
||||
assert c.max_references < c.max_images + c.max_videos + c.max_audios, 'the total limit has to bind'
|
||||
|
||||
|
||||
def test_caps_are_immutable():
|
||||
try:
|
||||
caps().max_images = 99
|
||||
except Exception: # pylint: disable=broad-except
|
||||
return True
|
||||
raise AssertionError('caps accepted a write')
|
||||
|
||||
|
||||
def test_caps_lookup_misses_on_keyframe_workflows():
|
||||
assert refs.get_reference_caps('fl2va') is None, 'fl2va reported reference limits'
|
||||
assert refs.get_reference_caps(None) is None, 'a missing workflow reported reference limits'
|
||||
|
||||
|
||||
def test_resolve_rejects_a_workflow_without_caps():
|
||||
return expect_error(lambda: refs.resolve('fl2va', [image()]), 'conditions on no references')
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Classification
|
||||
# ============================================================
|
||||
|
||||
def test_every_supported_extension_classifies():
|
||||
for kind, extensions in video_utils.MEDIA_EXTENSIONS.items():
|
||||
for ext in extensions:
|
||||
got = video_utils.classify_extension(f'sample{ext}')
|
||||
assert got == kind, f'{ext} classified as {got}, expected {kind}'
|
||||
|
||||
|
||||
def test_classification_ignores_case():
|
||||
assert video_utils.classify_extension('SAMPLE.MP4') == 'video', 'uppercase extension missed'
|
||||
|
||||
|
||||
def test_unknown_extension_has_no_kind():
|
||||
assert video_utils.classify_extension('notes.txt') is None, 'txt classified as media'
|
||||
|
||||
|
||||
def test_decoded_image_classifies_without_a_file():
|
||||
items = refs.classify_entries([image()])
|
||||
assert [kind for _label, kind, _entry in items] == ['image'], f'{items}'
|
||||
|
||||
|
||||
def test_paths_classify_by_extension():
|
||||
entries = [touch('a.png'), touch('b.mp4'), touch('c.wav')]
|
||||
items = refs.classify_entries(entries)
|
||||
assert [kind for _label, kind, _entry in items] == ['image', 'video', 'audio'], f'{items}'
|
||||
|
||||
|
||||
def test_unsupported_extension_is_rejected():
|
||||
return expect_error(lambda: refs.classify_entries([touch('notes.txt')]), 'unsupported media type')
|
||||
|
||||
|
||||
def test_url_is_rejected_before_any_fetch():
|
||||
# the reference classes download a url and decode whatever comes back, so a request never names one
|
||||
return expect_error(lambda: refs.classify_entries(['https://example.com/clip.mp4']), 'not a local file')
|
||||
|
||||
|
||||
def test_missing_file_is_rejected():
|
||||
return expect_error(lambda: refs.classify_entries([os.path.join(tmpdir, 'absent.png')]), 'file not found')
|
||||
|
||||
|
||||
def test_unsupported_input_type_is_rejected():
|
||||
return expect_error(lambda: refs.classify_entries([7]), 'unsupported input')
|
||||
|
||||
|
||||
def test_labels_are_one_based_and_follow_the_request():
|
||||
items = refs.classify_entries([image(), touch('d.mp4'), image()])
|
||||
assert [label for label, _kind, _entry in items] == [1, 2, 3], f'{items}'
|
||||
|
||||
|
||||
def test_order_is_preserved_across_kinds():
|
||||
entries = [touch('o1.mp4'), image(), touch('o2.wav'), touch('o3.png'), image(), touch('o4.mov')]
|
||||
items = refs.classify_entries(entries)
|
||||
assert [kind for _label, kind, _entry in items] == ['video', 'image', 'audio', 'image', 'image', 'video'], f'{items}'
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Counts
|
||||
# ============================================================
|
||||
|
||||
def test_too_many_images():
|
||||
return expect_error(lambda: refs.check_counts(caps(), ['image'] * 10), 'too many image references')
|
||||
|
||||
|
||||
def test_too_many_videos():
|
||||
return expect_error(lambda: refs.check_counts(caps(), ['video'] * 4), 'too many video references')
|
||||
|
||||
|
||||
def test_too_many_audio():
|
||||
return expect_error(lambda: refs.check_counts(caps(), ['audio'] * 4), 'too many audio references')
|
||||
|
||||
|
||||
def test_total_limit_binds_when_every_kind_is_legal():
|
||||
kinds = ['image'] * 9 + ['video'] * 3 + ['audio'] # 13 references, no kind over its own limit
|
||||
return expect_error(lambda: refs.check_counts(caps(), kinds), 'too many references')
|
||||
|
||||
|
||||
def test_per_kind_is_reported_before_the_total():
|
||||
kinds = ['image'] * 9 + ['video'] * 4 # over both, and the pipeline names the kind first
|
||||
return expect_error(lambda: refs.check_counts(caps(), kinds), 'too many video references')
|
||||
|
||||
|
||||
def test_the_kind_limits_accept_their_boundary():
|
||||
refs.check_counts(caps(), ['image'] * 9)
|
||||
refs.check_counts(caps(), ['image'] * 9 + ['video'] * 3) # exactly the total
|
||||
|
||||
|
||||
def test_audio_alone_is_rejected():
|
||||
for count in (1, 2, 3):
|
||||
expect_error(lambda n=count: refs.check_counts(caps(), ['audio'] * n), 'must be paired')
|
||||
|
||||
|
||||
def test_audio_paired_with_a_picture_passes():
|
||||
refs.check_counts(caps(), ['image', 'audio', 'audio', 'audio'])
|
||||
|
||||
|
||||
def test_counts_run_before_any_file_is_opened():
|
||||
# every entry is an empty file, so reaching the decode would fail differently than the count rule
|
||||
entries = [touch(f'count{index}.mp4') for index in range(4)]
|
||||
return expect_error(lambda: refs.resolve('ref2va', entries), 'too many video references')
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Nothing to condition on
|
||||
# ============================================================
|
||||
|
||||
def test_empty_request_names_the_workflow():
|
||||
# the api test detects a reference server by matching the workflow in this rejection
|
||||
return expect_error(lambda: refs.resolve('ref2va', []), 'ref2va')
|
||||
|
||||
|
||||
def test_init_image_stands_in_for_a_single_reference():
|
||||
items = refs.classify_entries([image()])
|
||||
assert len(items) == 1, f'{items}'
|
||||
return expect_error(lambda: refs.resolve('ref2va', None), 'No reference media provided')
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Checks that need the decoded media
|
||||
# ============================================================
|
||||
|
||||
def test_image_aspect_accepts_the_boundary():
|
||||
refs.check_decoded(caps(), [(1, 'image', None)], [stub_image_reference(64, 16)]) # exactly 4:1
|
||||
|
||||
|
||||
def test_image_aspect_rejects_beyond_the_boundary():
|
||||
expect_error(lambda: refs.check_decoded(caps(), [(1, 'image', None)], [stub_image_reference(8, 64)]), 'aspect ratio out of range')
|
||||
expect_error(lambda: refs.check_decoded(caps(), [(1, 'image', None)], [stub_image_reference(64, 8)]), 'aspect ratio out of range')
|
||||
|
||||
|
||||
def test_video_frame_floor_counts_at_the_resampled_rate():
|
||||
# 13 frames at 24 fps is the floor; the same 13 frames at 12 fps resample up to 26 and clear it
|
||||
refs.check_decoded(caps(), [(1, 'video', None)], [stub_video_reference(13, 24)])
|
||||
refs.check_decoded(caps(), [(1, 'video', None)], [stub_video_reference(7, 12)])
|
||||
expect_error(lambda: refs.check_decoded(caps(), [(1, 'video', None)], [stub_video_reference(12, 24)]), 'video too short')
|
||||
|
||||
|
||||
def test_video_frame_floor_uses_the_rounding_the_resample_uses():
|
||||
c = caps()
|
||||
for frames, fps in ((13, 24), (7, 12), (26, 48)):
|
||||
resampled = math.floor(frames * refs.REFERENCE_FPS / fps + 0.5)
|
||||
assert resampled >= c.video_min_frames, f'{frames}@{fps} resampled to {resampled}'
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Container probing
|
||||
# ============================================================
|
||||
|
||||
def test_probe_reads_video_headers():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
fn = make_mp4('probe.mp4', frames=20)
|
||||
if fn is None:
|
||||
return 'no h264 encoder to build a fixture'
|
||||
probe = video_utils.probe_media(fn, 'video')
|
||||
assert probe is not None, 'probe returned nothing'
|
||||
assert abs(probe.fps - 24.0) < 0.01, f'fps={probe.fps}'
|
||||
assert (probe.width, probe.height) == (64, 64), f'{probe.width}x{probe.height}'
|
||||
assert probe.duration is not None and probe.duration < 2.0, f'duration={probe.duration}'
|
||||
|
||||
|
||||
def test_probe_reads_audio_headers():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
probe = video_utils.probe_media(make_wav('probe.wav', seconds=0.5, rate=44100), 'audio')
|
||||
assert probe is not None, 'probe returned nothing'
|
||||
assert probe.sample_rate == 44100, f'rate={probe.sample_rate}'
|
||||
assert probe.channels == 1, f'channels={probe.channels}'
|
||||
|
||||
|
||||
def test_probe_returns_nothing_for_an_unreadable_file():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
assert video_utils.probe_media(touch('broken.mp4'), 'video') is None, 'an empty container probed clean'
|
||||
|
||||
|
||||
def test_unreadable_video_is_rejected():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
return expect_error(lambda: refs.resolve('ref2va', [image(), touch('empty.mp4')]), 'unreadable')
|
||||
|
||||
|
||||
def test_video_too_long_is_rejected():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
fn = make_mp4('long.mp4', frames=24 * 20, width=32, height=32) # 20 seconds, past what a generation can use
|
||||
if fn is None:
|
||||
return 'no h264 encoder to build a fixture'
|
||||
return expect_error(lambda: refs.resolve('ref2va', [image(), fn]), 'video too long')
|
||||
|
||||
|
||||
def test_video_aspect_is_rejected_from_the_header():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
fn = make_mp4('wide.mp4', frames=20, width=16, height=128)
|
||||
if fn is None:
|
||||
return 'no h264 encoder to build a fixture'
|
||||
return expect_error(lambda: refs.resolve('ref2va', [image(), fn]), 'aspect ratio out of range')
|
||||
|
||||
|
||||
def test_video_too_large_to_decode_is_rejected():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
probe = types.SimpleNamespace(kind='video', fps=24.0, frames=24 * 15, duration=15.0, width=7680, height=4320, channels=None, sample_rate=None)
|
||||
return expect_error(lambda: refs.check_video_probe(caps(), 1, 'huge.mp4', probe), 'too large to decode')
|
||||
|
||||
|
||||
def test_audio_channels_over_stereo_are_rejected():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
fn = make_wav('surround.wav', seconds=0.5, channels=6)
|
||||
probe = video_utils.probe_media(fn, 'audio')
|
||||
if probe is None or probe.channels != 6:
|
||||
return 'no six channel probe available'
|
||||
return expect_error(lambda: refs.check_audio_probe(caps(), 1, probe), 'too many audio channels')
|
||||
|
||||
|
||||
def test_audio_file_without_a_stream_is_rejected():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
fn = make_mp4('silent_as_audio.mp4', frames=20)
|
||||
if fn is None:
|
||||
return 'no h264 encoder to build a fixture'
|
||||
target = os.path.join(tmpdir, 'silent.wav')
|
||||
os.replace(fn, target) # a video container behind an audio extension: classified audio, and it carries no soundtrack
|
||||
return expect_error(lambda: refs.resolve('ref2va', [image(), target]), 'has no audio stream')
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Soft dependencies, forced absent
|
||||
# ============================================================
|
||||
|
||||
def test_missing_torchaudio_rejects_a_resample():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
fn = make_wav('offrate.wav', seconds=0.5, rate=44100)
|
||||
original = video_utils.has_torchaudio
|
||||
video_utils.has_torchaudio = lambda: False
|
||||
try:
|
||||
return expect_error(lambda: refs.resolve('ref2va', [image(), fn]), 'requires torchaudio')
|
||||
finally:
|
||||
video_utils.has_torchaudio = original
|
||||
|
||||
|
||||
def test_matching_rate_needs_no_torchaudio():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
fn = make_wav('onrate.wav', seconds=0.5, rate=32000)
|
||||
probe = video_utils.probe_media(fn, 'audio')
|
||||
assert probe is not None, 'probe returned nothing'
|
||||
original = video_utils.has_torchaudio
|
||||
video_utils.has_torchaudio = lambda: False
|
||||
try:
|
||||
refs.check_audio_probe(caps(), 1, probe) # the vae's own rate, so nothing resamples
|
||||
finally:
|
||||
video_utils.has_torchaudio = original
|
||||
|
||||
|
||||
def test_missing_av_rejects_media_references():
|
||||
original = video_utils.check_av
|
||||
video_utils.check_av = lambda: False
|
||||
try:
|
||||
return expect_error(lambda: refs.resolve('ref2va', [image(), touch('noav.mp4')]), 'require the av package', code=500)
|
||||
finally:
|
||||
video_utils.check_av = original
|
||||
|
||||
|
||||
def test_missing_av_leaves_image_requests_alone():
|
||||
original = video_utils.check_av
|
||||
video_utils.check_av = lambda: False
|
||||
try:
|
||||
items = refs.classify_entries([image(), image()])
|
||||
refs.check_counts(caps(), [kind for _label, kind, _entry in items])
|
||||
finally:
|
||||
video_utils.check_av = original
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Construction
|
||||
# ============================================================
|
||||
|
||||
def test_built_references_keep_the_request_order():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
if not has_diffusers():
|
||||
return 'diffusers not installed'
|
||||
video = make_mp4('build.mp4', frames=30)
|
||||
if video is None:
|
||||
return 'no h264 encoder to build a fixture'
|
||||
audio = make_wav('build.wav', seconds=1.0, rate=32000)
|
||||
built = refs.resolve('ref2va', [video, image(), audio])
|
||||
assert [reference.kind for reference in built] == ['video', 'image', 'audio'], f'{[r.kind for r in built]}'
|
||||
|
||||
|
||||
def test_built_video_carries_its_frame_rate():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
if not has_diffusers():
|
||||
return 'diffusers not installed'
|
||||
fn = make_mp4('rate.mp4', frames=30)
|
||||
if fn is None:
|
||||
return 'no h264 encoder to build a fixture'
|
||||
built = refs.resolve('ref2va', [image(), fn])
|
||||
reference = built[1]
|
||||
assert abs(float(reference.fps) - 24.0) < 0.01, f'fps={reference.fps}'
|
||||
assert len(reference.frames) == 30, f'frames={len(reference.frames)}'
|
||||
assert reference.audio is None, 'a silent video reported a soundtrack' # silent containers are legal
|
||||
|
||||
|
||||
def test_built_image_is_rgb():
|
||||
if not has_diffusers():
|
||||
return 'diffusers not installed'
|
||||
# the reference encoder reads the array raw, so a non-rgb upload has to be converted on the way in
|
||||
built = refs.resolve('ref2va', [Image.new('RGBA', (64, 64))])
|
||||
assert built[0].image.mode == 'RGB', f'mode={built[0].image.mode}'
|
||||
|
||||
|
||||
def test_corrupt_file_is_rejected_cleanly():
|
||||
if not has_diffusers():
|
||||
return 'diffusers not installed'
|
||||
# an unreadable file has to surface as a rejection naming it, not as a traceback out of the decoder
|
||||
return expect_error(lambda: refs.resolve('ref2va', [touch('corrupt.png')]), 'decode failed')
|
||||
|
||||
|
||||
def test_short_video_is_rejected_after_the_decode():
|
||||
if not has_av():
|
||||
return 'av not installed'
|
||||
if not has_diffusers():
|
||||
return 'diffusers not installed'
|
||||
fn = make_mp4('short.mp4', frames=8)
|
||||
if fn is None:
|
||||
return 'no h264 encoder to build a fixture'
|
||||
return expect_error(lambda: refs.resolve('ref2va', [image(), fn]), 'video too short')
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Core seam
|
||||
# ============================================================
|
||||
|
||||
def test_core_serves_the_caps():
|
||||
from modules.video_models import video_run
|
||||
assert video_run.reference_caps('ref2va') is caps(), 'the core served different caps than the resolver'
|
||||
assert video_run.reference_caps('fl2va') is None, 'fl2va reported reference limits'
|
||||
assert video_run.reference_caps(None) is None, 'a missing workflow reported reference limits'
|
||||
|
||||
|
||||
def test_core_gate_and_caps_table_agree():
|
||||
from modules.video_models import video_run
|
||||
assert set(video_run.REFERENCE_WORKFLOWS) == set(refs.REFERENCE_CAPS), f'{video_run.REFERENCE_WORKFLOWS} vs {sorted(refs.REFERENCE_CAPS)}'
|
||||
|
||||
|
||||
def test_core_gate_names_workflows_the_registry_carries():
|
||||
from modules.video_models import models_def, video_run
|
||||
known = {row.workflow for rows in models_def.models.values() for row in rows if getattr(row, 'workflow', None)}
|
||||
assert set(video_run.REFERENCE_WORKFLOWS) <= known, f'{video_run.REFERENCE_WORKFLOWS} not among {sorted(known)}'
|
||||
|
||||
|
||||
def test_core_rejects_references_on_a_keyframe_model():
|
||||
from modules.video_models import models_def, video_run
|
||||
row = models_def.Model(name='test keyframe', workflow='fl2va')
|
||||
expect_error(lambda: video_run.validate_references(row, [image()], None), 'requires a reference workflow')
|
||||
assert video_run.validate_references(row, None, None) is None, 'a keyframe model resolved references'
|
||||
assert video_run.validate_references(row, [], image()) is None, 'a keyframe model claimed its init image'
|
||||
|
||||
|
||||
def test_core_resolves_a_reference_model():
|
||||
from modules.video_models import models_def, video_run
|
||||
row = models_def.Model(name='test reference', workflow='ref2va')
|
||||
expect_error(lambda: video_run.validate_references(row, None, None), 'No reference media provided')
|
||||
if not has_diffusers():
|
||||
return 'diffusers not installed'
|
||||
built = video_run.validate_references(row, None, image()) # the init image stands in for a single reference
|
||||
assert len(built) == 1 and built[0].kind == 'image', f'{built}'
|
||||
return True
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Tab delegation
|
||||
# ============================================================
|
||||
|
||||
def test_tab_unwraps_gradio_file_entries():
|
||||
from modules.minimax import minimax_video
|
||||
fn = touch('unwrap.png')
|
||||
assert minimax_video.unwrap_file(fn) == fn, 'a plain path was rewritten'
|
||||
assert minimax_video.unwrap_file(types.SimpleNamespace(name=fn)) == fn, 'a tempfile wrapper was not unwrapped'
|
||||
assert minimax_video.unwrap_file({'name': fn}) == fn, 'a dict entry was not unwrapped'
|
||||
|
||||
|
||||
def test_tab_keeps_keyframes_on_a_keyframe_workflow():
|
||||
from modules.minimax import minimax_video
|
||||
task_args = minimax_video.prepare_inputs('fl2va', image(), image(), None)
|
||||
assert sorted(task_args) == ['image', 'last_image'], f'{sorted(task_args)}'
|
||||
|
||||
|
||||
def test_tab_reports_references_it_cannot_use():
|
||||
# uploads survive the accordion hiding when the row changes, and dropping them silently reads as a working request
|
||||
from modules.minimax import minimax_video
|
||||
seen = []
|
||||
original = minimax_video.log.warning
|
||||
minimax_video.log.warning = lambda msg, *a, **k: seen.append(str(msg))
|
||||
try:
|
||||
task_args = minimax_video.prepare_inputs('fl2va', image(), None, [touch('stale.png')])
|
||||
finally:
|
||||
minimax_video.log.warning = original
|
||||
assert any('not supported' in message for message in seen), f'{seen}'
|
||||
assert 'references' not in task_args, 'a keyframe workflow claimed the references'
|
||||
|
||||
|
||||
def test_tab_rejections_name_the_reason():
|
||||
from modules.minimax import minimax_video
|
||||
return expect_error(lambda: minimax_video.prepare_inputs('ref2va', None, None, [touch('notes.txt')]), 'unsupported media type')
|
||||
|
||||
|
||||
def test_tab_resolves_references_through_the_shared_funnel():
|
||||
from modules.minimax import minimax_video
|
||||
if not has_diffusers():
|
||||
return 'diffusers not installed'
|
||||
task_args = minimax_video.prepare_inputs('ref2va', image(), None, [types.SimpleNamespace(name=make_png('tab.png'))])
|
||||
assert list(task_args) == ['references'], f'{list(task_args)}'
|
||||
assert [reference.kind for reference in task_args['references']] == ['image'], f'{task_args}'
|
||||
return True
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Runner
|
||||
# ============================================================
|
||||
|
||||
def run_all():
|
||||
global tmpdir # pylint: disable=global-statement
|
||||
with tempfile.TemporaryDirectory(prefix='sdnext-refs-') as path:
|
||||
tmpdir = path
|
||||
|
||||
log.warning('=== caps ===')
|
||||
cat = category('caps')
|
||||
for fn in [
|
||||
test_caps_mirror_the_pipeline_limits,
|
||||
test_caps_total_is_not_the_sum_of_the_kinds,
|
||||
test_caps_are_immutable,
|
||||
test_caps_lookup_misses_on_keyframe_workflows,
|
||||
test_resolve_rejects_a_workflow_without_caps,
|
||||
]:
|
||||
run_test(cat, fn)
|
||||
|
||||
log.warning('=== classification ===')
|
||||
cat = category('classification')
|
||||
for fn in [
|
||||
test_every_supported_extension_classifies,
|
||||
test_classification_ignores_case,
|
||||
test_unknown_extension_has_no_kind,
|
||||
test_decoded_image_classifies_without_a_file,
|
||||
test_paths_classify_by_extension,
|
||||
test_unsupported_extension_is_rejected,
|
||||
test_url_is_rejected_before_any_fetch,
|
||||
test_missing_file_is_rejected,
|
||||
test_unsupported_input_type_is_rejected,
|
||||
test_labels_are_one_based_and_follow_the_request,
|
||||
test_order_is_preserved_across_kinds,
|
||||
]:
|
||||
run_test(cat, fn)
|
||||
|
||||
log.warning('=== counts ===')
|
||||
cat = category('counts')
|
||||
for fn in [
|
||||
test_too_many_images,
|
||||
test_too_many_videos,
|
||||
test_too_many_audio,
|
||||
test_total_limit_binds_when_every_kind_is_legal,
|
||||
test_per_kind_is_reported_before_the_total,
|
||||
test_the_kind_limits_accept_their_boundary,
|
||||
test_audio_alone_is_rejected,
|
||||
test_audio_paired_with_a_picture_passes,
|
||||
test_counts_run_before_any_file_is_opened,
|
||||
test_empty_request_names_the_workflow,
|
||||
test_init_image_stands_in_for_a_single_reference,
|
||||
]:
|
||||
run_test(cat, fn)
|
||||
|
||||
log.warning('=== decoded media ===')
|
||||
cat = category('decoded')
|
||||
for fn in [
|
||||
test_image_aspect_accepts_the_boundary,
|
||||
test_image_aspect_rejects_beyond_the_boundary,
|
||||
test_video_frame_floor_counts_at_the_resampled_rate,
|
||||
test_video_frame_floor_uses_the_rounding_the_resample_uses,
|
||||
]:
|
||||
run_test(cat, fn)
|
||||
|
||||
log.warning('=== probing ===')
|
||||
cat = category('probing')
|
||||
for fn in [
|
||||
test_probe_reads_video_headers,
|
||||
test_probe_reads_audio_headers,
|
||||
test_probe_returns_nothing_for_an_unreadable_file,
|
||||
test_unreadable_video_is_rejected,
|
||||
test_video_too_long_is_rejected,
|
||||
test_video_aspect_is_rejected_from_the_header,
|
||||
test_video_too_large_to_decode_is_rejected,
|
||||
test_audio_channels_over_stereo_are_rejected,
|
||||
test_audio_file_without_a_stream_is_rejected,
|
||||
]:
|
||||
run_test(cat, fn)
|
||||
|
||||
log.warning('=== soft dependencies ===')
|
||||
cat = category('dependencies')
|
||||
for fn in [
|
||||
test_missing_torchaudio_rejects_a_resample,
|
||||
test_matching_rate_needs_no_torchaudio,
|
||||
test_missing_av_rejects_media_references,
|
||||
test_missing_av_leaves_image_requests_alone,
|
||||
]:
|
||||
run_test(cat, fn)
|
||||
|
||||
log.warning('=== construction ===')
|
||||
cat = category('construction')
|
||||
for fn in [
|
||||
test_built_references_keep_the_request_order,
|
||||
test_built_video_carries_its_frame_rate,
|
||||
test_built_image_is_rgb,
|
||||
test_corrupt_file_is_rejected_cleanly,
|
||||
test_short_video_is_rejected_after_the_decode,
|
||||
]:
|
||||
run_test(cat, fn)
|
||||
|
||||
log.warning('=== core seam ===')
|
||||
cat = category('core')
|
||||
for fn in [
|
||||
test_core_serves_the_caps,
|
||||
test_core_gate_and_caps_table_agree,
|
||||
test_core_gate_names_workflows_the_registry_carries,
|
||||
test_core_rejects_references_on_a_keyframe_model,
|
||||
test_core_resolves_a_reference_model,
|
||||
]:
|
||||
run_test(cat, fn)
|
||||
|
||||
log.warning('=== tab delegation ===')
|
||||
cat = category('tab')
|
||||
for fn in [
|
||||
test_tab_unwraps_gradio_file_entries,
|
||||
test_tab_keeps_keyframes_on_a_keyframe_workflow,
|
||||
test_tab_reports_references_it_cannot_use,
|
||||
test_tab_rejections_name_the_reason,
|
||||
test_tab_resolves_references_through_the_shared_funnel,
|
||||
]:
|
||||
run_test(cat, fn)
|
||||
|
||||
log.warning('=== Results ===')
|
||||
total_passed = 0
|
||||
total_failed = 0
|
||||
total_skipped = 0
|
||||
for cat_name, info in results.items():
|
||||
ok = info['failed'] == 0
|
||||
status = 'PASS' if ok else 'FAIL'
|
||||
log.info(f" {cat_name}: {info['passed']} passed, {info['failed']} failed, {info['skipped']} skipped [{status}]")
|
||||
total_passed += info['passed']
|
||||
total_failed += info['failed']
|
||||
total_skipped += info['skipped']
|
||||
log.warning(f'Total: {total_passed} passed, {total_failed} failed, {total_skipped} skipped')
|
||||
return total_failed == 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import time
|
||||
t0 = time.time()
|
||||
success = run_all()
|
||||
log.warning(f'Total time: {time.time() - t0:.2f}s')
|
||||
sys.exit(0 if success else 1)
|
||||
Reference in New Issue
Block a user