diff --git a/modules/api/video.py b/modules/api/video.py index ff2345214..9afc66385 100644 --- a/modules/api/video.py +++ b/modules/api/video.py @@ -72,27 +72,14 @@ class ItemVideoModel(BaseModel): name: str = Field(title="Name", description="Model name; pass together with engine to select it") repo: str = Field(default="", title="Repo", description="Model repository or path") url: str = Field(default="", title="URL", description="Model information page") - mode: str = Field(title="Mode", description="Input mode: workflow, t2v, i2v, flf2v, vace, or animate") + mode: str = Field(title="Mode", description="Input mode: workflow, t2v, i2v, flf2v, vace, animate, condition, or unknown; condition models accept conditioning the generic path does not wire and run as text to video here") workflow: str | None = Field(default=None, title="Workflow", description="Modular workflow name when the model dispatches on inputs; ref2va conditions on references and ignores the keyframe images") base: bool = Field(default=False, title="Base", description="Also listed in the base checkpoint dropdown") loaded: bool = Field(default=False, title="Loaded", description="Currently loaded through the video registry") def model_mode(m: models_def.Model) -> str: - # mirrors the dispatch order in video_run.run: workflow models route on inputs, the rest on name markers - if m.workflow is not None: - return 'workflow' - if 'T2V' in m.name: - return 't2v' - if 'I2V' in m.name: - return 'i2v' - if 'FLF2V' in m.name: - return 'flf2v' - if 'VACE' in m.name: - return 'vace' - if 'Animate' in m.name: - return 'animate' - return 't2v' + return models_def.dispatch_mode(m) class APIVideo: diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index b8872fdab..5edeae458 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -826,3 +826,61 @@ def pipeline_classes() -> set[str]: if row.custom is not None: classes.add(row.custom) return classes + + +NAME_MODES = ( # markers a row name carries to declare its inputs, in the order run() tests them + ('T2V', 't2v'), + ('I2V', 'i2v'), + ('FLF2V', 'flf2v'), + ('VACE', 'vace'), + ('Animate', 'animate'), +) +CLASS_MODES = { # the mode a pipeline class implies, for rows whose name declares nothing + 'HunyuanVideoPipeline': 't2v', + 'HunyuanVideo15Pipeline': 't2v', + 'HunyuanVideoImageToVideoPipeline': 'i2v', + 'HunyuanVideo15ImageToVideoPipeline': 'i2v', + 'HunyuanSkyreelsImageToVideoPipeline': 'i2v', + 'LTXPipeline': 't2v', + 'LTX2Pipeline': 't2v', + 'LTXImageToVideoPipeline': 'i2v', + 'LTX2ImageToVideoPipeline': 'i2v', + 'LTXConditionPipeline': 'condition', + 'LTX2ConditionPipeline': 'condition', + 'WanPipeline': 't2v', + 'WanImageToVideoPipeline': 'i2v', + 'WanVACEPipeline': 'vace', + 'WanAnimatePipeline': 'animate', + 'SkyReelsV2Pipeline': 't2v', + 'SkyReelsV2DiffusionForcingPipeline': 't2v', + 'SkyReelsV2ImageToVideoPipeline': 'i2v', + 'SkyReelsV2DiffusionForcingImageToVideoPipeline': 'i2v', + 'MochiPipeline': 't2v', + 'LattePipeline': 't2v', + 'AllegroPipeline': 't2v', + 'CogVideoXPipeline': 't2v', + 'CogVideoXImageToVideoPipeline': 'i2v', + 'Cosmos2VideoToWorldPipeline': 'i2v', + 'SanaVideoPipeline': 't2v', + 'Kandinsky5T2VPipeline': 't2v', + 'Kandinsky5I2VPipeline': 'i2v', + 'MiniMaxH3ModularPipeline': 'workflow', + 'GoogleVeoVideoPipeline': 't2v', +} + + +def dispatch_mode(row: Model) -> str: + """How a row's inputs are wired: workflow, t2v, i2v, flf2v, vace, animate, condition, or unknown. + + Name markers are read before the pipeline class, since one class serves several modes: six + LTXConditionPipeline rows are named T2V or I2V and generate as such. + """ + if row is None: + return 'unknown' + if row.workflow is not None: + return 'workflow' + for marker, mode in NAME_MODES: + if marker in (row.name or ''): + return mode + cls = row.repo_cls if isinstance(row.repo_cls, str) else getattr(row.repo_cls, '__name__', None) + return CLASS_MODES.get(cls or row.custom, 'unknown') diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 453e6738a..5f36ea8d0 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -181,7 +181,8 @@ def run(selected: models_def.Model, *, p.do_not_save_grid = True p.do_not_save_samples = not mp4_frames p.outpath_samples = resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_video) - if getattr(selected, 'workflow', None) is not None: + mode = models_def.dispatch_mode(selected) + if mode == 'workflow': # modular workflows dispatch on which inputs are present; keyframes pass through # unresized since the pipeline defines its own canvas placement per anchor p.video_still = int(frames) <= 1 @@ -200,10 +201,10 @@ def run(selected: models_def.Model, *, elif int(mp4_fps) != 24: log.warning(f'Video: model="{selected.name}" fps={mp4_fps} model output is fixed at 24') log.debug(f'Video: op=modular workflow={selected.workflow} still={p.video_still} init={init_image} last={last_image} references={len(refs) if refs else 0}') - elif 'T2V' in selected.name: + elif mode == 't2v': if init_image is not None: log.warning('Video: op=T2V init image not supported') - elif 'I2V' in selected.name: + elif mode == 'i2v': if init_image is None: raise VideoError('No input image provided. Please upload or select an image.', 400) p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') @@ -214,7 +215,7 @@ def run(selected: models_def.Model, *, log.warning(f'Video: op=I2V model="{selected.name}" last frame not supported, ignoring') else: log.debug(f'Video: op=I2V init={init_image} resized={p.task_args["image"]}') - elif 'FLF2V' in selected.name: + elif mode == 'flf2v': if init_image is None: raise VideoError('No input image provided. Please upload or select an image.', 400) if last_image is None: @@ -222,11 +223,11 @@ def run(selected: models_def.Model, *, p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') p.task_args['last_image'] = images.resize_image(resize_mode=2, im=last_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') log.debug(f'Video: op=FLF2V init={init_image} last={last_image} resized={p.task_args["image"]}') - elif 'VACE' in selected.name: + elif mode == 'vace': if init_image is not None: p.task_args['reference_images'] = [images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')] log.debug(f'Video: op=VACE reference={init_image} resized={p.task_args["reference_images"]}') - elif 'Animate' in selected.name: + elif mode == 'animate': if init_image is None: raise VideoError('No input image provided. Please upload or select an image.', 400) p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') @@ -234,6 +235,11 @@ def run(selected: models_def.Model, *, p.task_args['pose_video'] = [] # input pose video to condition the generation on. must be a list of PIL images. p.task_args['face_video'] = [] # input face video to condition the generation on. must be a list of PIL images. log.debug(f'Video: op=Animate init={p.task_args["image"]} pose={p.task_args["pose_video"]} face={p.task_args["face_video"]}') + elif mode == 'condition': + # the conditioning inputs these models accept are wired on the ltx tab, not here + log.warning(f'Video: op=condition model="{selected.name}" conditioning not supported here, running text to video') + if init_image is not None: + log.warning(f'Video: op=condition model="{selected.name}" init image not supported, ignoring') else: log.warning(f'Video: unknown model type "{selected.name}"')