mirror of
https://github.com/vladmandic/automatic
synced 2026-09-19 09:14:35 +02:00
feat(api): add video generation endpoint
Add POST /sdapi/v1/video plus GET /sdapi/v1/video/models and GET /sdapi/v1/video/file. The generation body is extracted from the gradio handler into a keyword-only core, video_run.run, which returns a structured result and raises typed errors; the positional generate signature is unchanged and now adapts to the core. Omitting engine and model drives the currently loaded checkpoint when it is video-capable, which covers models loaded from local folders without a registry entry. - registry helpers in models_def (find, engines, pipeline_classes, workflow_for_class); validate_pipeline reuses the shared class set - modular pipes stamp their workflow so out-of-registry loads dispatch onto the modular branch - disk switches (mp4_*) and wire switches (send_*) are independent; artifacts above the base64 cap fall back to path plus the file route, which is jailed to the video output directory and serves video/mp4 with range support - always-on video scripts get bootstrapped default args, matching the txt2img handler; missing bootstrap raised a TypeError per frame - checkpoint overrides are rejected with a pointer to the checkpoint endpoint; unknown engine, model and sampler names return 404 with the valid choices - cli/api-video.py client, test/test-video-api.py suite and a full-test.sh entry; video mimetypes registered; rate-limit cost set - remove the unreferenced video_ui.run_video dispatcher
This commit is contained in:
CalamitousFelicitousness
+5
-1
@@ -6,7 +6,7 @@ from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
||||
from fastapi.exceptions import HTTPException
|
||||
from modules import errors, shared, paths
|
||||
from modules.logger import log
|
||||
from modules.api import models, endpoints, script, helpers, server, generate, process, control, docs, gpu
|
||||
from modules.api import models, endpoints, script, helpers, server, generate, process, control, video, docs, gpu
|
||||
|
||||
|
||||
errors.install()
|
||||
@@ -33,6 +33,7 @@ class Api:
|
||||
self.generate = generate.APIGenerate(queue_lock)
|
||||
self.process = process.APIProcess(queue_lock)
|
||||
self.control = control.APIControl(queue_lock)
|
||||
self.video = video.APIVideo(queue_lock)
|
||||
# compatibility api
|
||||
self.text2imgapi = self.generate.post_text2img
|
||||
self.img2imgapi = self.generate.post_img2img
|
||||
@@ -68,6 +69,7 @@ class Api:
|
||||
self.add_api_route("/sdapi/v1/txt2img", self.generate.post_text2img, methods=["POST"], response_model=models.ResTxt2Img, tags=["Generation"])
|
||||
self.add_api_route("/sdapi/v1/img2img", self.generate.post_img2img, methods=["POST"], response_model=models.ResImg2Img, tags=["Generation"])
|
||||
self.add_api_route("/sdapi/v1/control", self.control.post_control, methods=["POST"], response_model=control.ResControl, tags=["Generation"])
|
||||
self.add_api_route("/sdapi/v1/video", self.video.post_video, methods=["POST"], response_model=video.ResVideo, tags=["Generation"])
|
||||
self.add_api_route("/sdapi/v1/process", self.process.extras_single_image_api, methods=["POST"], response_model=models.ResProcessImage, tags=["Processing"])
|
||||
self.add_api_route("/sdapi/v1/extra-single-image", self.process.extras_single_image_api, methods=["POST"], response_model=models.ResProcessImage, tags=["Processing"])
|
||||
self.add_api_route("/sdapi/v1/process-batch", self.process.extras_batch_images_api, methods=["POST"], response_model=models.ResProcessBatch, tags=["Processing"])
|
||||
@@ -104,9 +106,11 @@ class Api:
|
||||
self.add_api_route("/sdapi/v1/extra-network-detail", endpoints.get_extra_network_detail, methods=["GET"], response_model=models.ItemExtraNetworkFull, tags=["Enumerators"])
|
||||
self.add_api_route("/sdapi/v1/extra-network-details", endpoints.get_extra_network_details, methods=["GET"], response_model=models.ResExtraNetworkDetails, tags=["Enumerators"])
|
||||
self.add_api_route("/sdapi/v1/unets", endpoints.get_unets, methods=["GET"], response_model=list[models.ItemUNet], tags=["Enumerators"])
|
||||
self.add_api_route("/sdapi/v1/video/models", self.video.get_video_models, methods=["GET"], response_model=list[video.ItemVideoModel], tags=["Enumerators"])
|
||||
|
||||
# functional api
|
||||
self.add_api_route("/sdapi/v1/file", endpoints.get_file, methods=["GET"], tags=["Functional"])
|
||||
self.add_api_route("/sdapi/v1/video/file", self.video.get_video_file, methods=["GET"], tags=["Functional"])
|
||||
self.add_api_route("/sdapi/v1/delete-image", endpoints.get_deleteimage, methods=["DELETE"], tags=["Functional"])
|
||||
self.add_api_route("/sdapi/v1/delete-file", endpoints.get_deletefile, methods=["DELETE"], tags=["Functional"])
|
||||
self.add_api_route("/sdapi/v1/png-info", endpoints.get_pnginfo, methods=["GET"], response_model=models.ResImageInfo, tags=["Functional"])
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import io
|
||||
import os
|
||||
import base64
|
||||
from PIL import Image, PngImagePlugin
|
||||
import piexif
|
||||
@@ -90,6 +91,24 @@ def encode_pil_to_base64(image):
|
||||
return b64
|
||||
|
||||
|
||||
MAX_B64_BYTES = 256 * 1024 * 1024 # base64 expands ~4/3 and the response is built in memory; larger artifacts are fetched by path instead
|
||||
|
||||
|
||||
def encode_file_to_base64(fn: str, max_bytes: int = MAX_B64_BYTES) -> str | None:
|
||||
try:
|
||||
if fn is None or not os.path.isfile(fn):
|
||||
return None
|
||||
size = os.path.getsize(fn)
|
||||
if size > max_bytes:
|
||||
log.warning(f'API cannot encode file: fn="{fn}" size={size} max={max_bytes}')
|
||||
return None
|
||||
with open(fn, 'rb') as f:
|
||||
return base64.b64encode(f.read()).decode('ascii')
|
||||
except Exception as e:
|
||||
log.warning(f'API cannot encode file: fn="{fn}" {e}')
|
||||
return None
|
||||
|
||||
|
||||
def upscaler_to_index(name: str):
|
||||
try:
|
||||
return [x.name.lower() for x in shared.sd_upscalers].index(name.lower())
|
||||
|
||||
@@ -10,3 +10,6 @@ def register():
|
||||
mimetypes.add_type('image/webp', '.webp')
|
||||
mimetypes.add_type('image/jxl', '.jxl')
|
||||
mimetypes.add_type('font/ttf', '.ttf')
|
||||
mimetypes.add_type('video/mp4', '.mp4')
|
||||
mimetypes.add_type('video/webm', '.webm')
|
||||
mimetypes.add_type('video/x-matroska', '.mkv')
|
||||
|
||||
@@ -12,6 +12,7 @@ request_cost = {
|
||||
"/sdapi/v1/txt2img": 5,
|
||||
"/sdapi/v1/img2img": 5,
|
||||
"/sdapi/v1/control": 5,
|
||||
"/sdapi/v1/video": 5,
|
||||
}
|
||||
log_cost = {
|
||||
"/.well-known/appspecific/com.chrome.devtools.json": -1,
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
from types import SimpleNamespace
|
||||
from threading import Lock
|
||||
from pydantic import BaseModel, Field # pylint: disable=no-name-in-module
|
||||
from fastapi.exceptions import HTTPException
|
||||
from modules import errors, shared, scripts_manager, ui
|
||||
from modules.api import script, helpers
|
||||
from modules.paths import resolve_output_path
|
||||
from modules.video_models import models_def, video_load, video_run
|
||||
|
||||
|
||||
errors.install()
|
||||
|
||||
|
||||
class ReqVideo(BaseModel):
|
||||
engine: str | None = Field(default=None, title="Engine", description="Video engine family; omit together with model to use the currently loaded checkpoint")
|
||||
model: str | None = Field(default=None, title="Model", description="Video model name within the engine; see GET /sdapi/v1/video/models")
|
||||
prompt: str = Field(default="", title="Prompt", description="Text prompt")
|
||||
negative_prompt: str = Field(default="", title="Negative prompt", description="Negative text prompt")
|
||||
styles: list[str] = Field(default=[], title="Styles", description="Prompt style names to apply")
|
||||
width: int = Field(default=832, ge=64, le=4096, title="Width", description="Output width; snapped to the model canvas multiple")
|
||||
height: int = Field(default=480, ge=64, le=4096, title="Height", description="Output height; snapped to the model canvas multiple")
|
||||
frames: int = Field(default=17, ge=1, le=1024, title="Frames", description="Number of frames; 1 produces a single still image on workflow models")
|
||||
steps: int = Field(default=50, ge=1, le=200, title="Steps", description="Number of inference steps")
|
||||
sampler_name: str = Field(default="Default", title="Sampler", description="Sampler name; Default keeps the model scheduler")
|
||||
sampler_shift: float = Field(default=-1.0, title="Sampler shift", description="Scheduler flow shift; -1 keeps the model default")
|
||||
dynamic_shift: bool = Field(default=False, title="Dynamic shift", description="Enable dynamic scheduler shifting")
|
||||
seed: int = Field(default=-1, title="Seed", description="Generation seed; -1 for random")
|
||||
guidance_scale: float = Field(default=-1.0, title="Guidance scale", description="CFG scale; -1 keeps the model default")
|
||||
guidance_true: float = Field(default=-1.0, title="True guidance", description="True CFG scale; -1 keeps the model default")
|
||||
init_image: str | None = Field(default=None, title="Init image", description="Base64, data URI, or upload reference for the first-frame image")
|
||||
init_strength: float = Field(default=0.8, ge=0.0, le=1.0, title="Init strength", description="Denoising strength for the init image")
|
||||
last_image: str | None = Field(default=None, title="Last image", description="Base64, data URI, or upload reference for the last-frame image")
|
||||
vae_type: str = Field(default="Default", title="VAE type", description="Decode variant: Default, Tiny, Remote, or Upscale")
|
||||
vae_tile_frames: int = Field(default=16, ge=1, le=64, title="VAE tile frames", description="Frames per VAE decode tile")
|
||||
audio: bool = Field(default=True, title="Audio", description="Generate audio on models that support it")
|
||||
mp4_fps: int = Field(default=24, ge=1, le=60, title="FPS", description="Frames per second of the saved video")
|
||||
mp4_interpolate: int = Field(default=0, ge=0, le=10, title="Interpolation", description="RIFE interpolation passes between frames")
|
||||
mp4_codec: str = Field(default="libx264", title="Codec", description="Video codec; none skips video encoding")
|
||||
mp4_ext: str = Field(default="mp4", title="Container", description="Container extension; the muxer is inferred from it")
|
||||
mp4_opt: str = Field(default="crf:16", title="Codec options", description="Encoder options as comma-separated key:value pairs")
|
||||
mp4_video: bool = Field(default=True, title="Save video", description="Write the video container to disk")
|
||||
mp4_frames: bool = Field(default=False, title="Save frames", description="Write individual frame images to disk")
|
||||
mp4_sf: bool = Field(default=False, title="Save safetensors", description="Write raw frames as a safetensors file")
|
||||
mp4_thumb: bool = Field(default=True, title="Save thumbnail", description="Write a thumbnail image next to the video")
|
||||
override_settings: dict = Field(default={}, title="Override settings", description="Setting overrides applied for this generation only")
|
||||
script_args: list = Field(default=[], title="Script args", description="Positional arguments for a selectable script")
|
||||
alwayson_scripts: dict = Field(default={}, title="Always-on scripts", description="Per-script argument overrides, keyed by script name")
|
||||
send_video: bool = Field(default=True, title="Send video", description="Return the video base64-encoded in the response")
|
||||
send_frames: bool = Field(default=False, title="Send frames", description="Return every frame base64-encoded in the response")
|
||||
send_thumbnail: bool = Field(default=True, title="Send thumbnail", description="Return the thumbnail base64-encoded in the response")
|
||||
extra: dict | None = Field(default={}, exclude=True, title="Extra", description="Extra attributes set on the processing object")
|
||||
|
||||
|
||||
class ResVideo(BaseModel):
|
||||
video: str | None = Field(default=None, title="Video", description="Base64-encoded video file; empty when not requested, above the size cap, or in still mode")
|
||||
video_path: str | None = Field(default=None, title="Video path", description="Server path of the saved video; fetch via GET /sdapi/v1/video/file")
|
||||
thumbnail: str | None = Field(default=None, title="Thumbnail", description="Base64-encoded thumbnail image")
|
||||
thumbnail_path: str | None = Field(default=None, title="Thumbnail path", description="Server path of the saved thumbnail")
|
||||
frames: list[str] = Field(default=[], title="Frames", description="Base64-encoded frames; always populated in still mode")
|
||||
frames_count: int = Field(default=0, title="Frame count", description="Number of frames written, after interpolation")
|
||||
fps: float = Field(default=0.0, title="FPS", description="Effective frames per second of the saved video")
|
||||
duration: float = Field(default=0.0, title="Duration", description="Video duration in seconds")
|
||||
has_audio: bool = Field(default=False, title="Has audio", description="Whether the video carries an audio track")
|
||||
still: bool = Field(default=False, title="Still", description="Single-frame result; the product is in frames and no video was written")
|
||||
params: dict = Field(default={}, title="Parameters", description="Echo of the request parameters used for generation")
|
||||
info: str = Field(default="", title="Info", description="Generation info string with seed, sampler, and pipeline details")
|
||||
|
||||
|
||||
class ItemVideoModel(BaseModel):
|
||||
engine: str = Field(title="Engine", description="Video engine family")
|
||||
name: str = Field(title="Name", description="Model name; pass together with engine to select it")
|
||||
repo: str = Field(default="", title="Repo", description="Model repository or path")
|
||||
url: str = Field(default="", title="URL", description="Model information page")
|
||||
mode: str = Field(title="Mode", description="Input mode: workflow, t2v, i2v, flf2v, vace, or animate")
|
||||
workflow: str | None = Field(default=None, title="Workflow", description="Modular workflow name when the model dispatches on inputs")
|
||||
base: bool = Field(default=False, title="Base", description="Also listed in the base checkpoint dropdown")
|
||||
loaded: bool = Field(default=False, title="Loaded", description="Currently loaded through the video registry")
|
||||
|
||||
|
||||
def model_mode(m: models_def.Model) -> str:
|
||||
# mirrors the dispatch order in video_run.run: workflow models route on inputs, the rest on name markers
|
||||
if m.workflow is not None:
|
||||
return 'workflow'
|
||||
if 'T2V' in m.name:
|
||||
return 't2v'
|
||||
if 'I2V' in m.name:
|
||||
return 'i2v'
|
||||
if 'FLF2V' in m.name:
|
||||
return 'flf2v'
|
||||
if 'VACE' in m.name:
|
||||
return 'vace'
|
||||
if 'Animate' in m.name:
|
||||
return 'animate'
|
||||
return 't2v'
|
||||
|
||||
|
||||
class APIVideo:
|
||||
def __init__(self, queue_lock: Lock):
|
||||
self.queue_lock = queue_lock
|
||||
self.default_script_arg_video = []
|
||||
|
||||
def prepare_scripts(self, p_stub, req: ReqVideo):
|
||||
script_runner = scripts_manager.scripts_video
|
||||
if not script_runner.scripts:
|
||||
script_runner.initialize_scripts(is_img2img=False, is_control=False, is_video=True)
|
||||
ui.create_ui(None)
|
||||
if not self.default_script_arg_video:
|
||||
self.default_script_arg_video = script.init_default_script_args(script_runner)
|
||||
script_args = script.init_script_args(p_stub, req, self.default_script_arg_video, None, None, script_runner)
|
||||
return script_runner, script_args
|
||||
|
||||
def sanitize_b64(self, req: ReqVideo):
|
||||
def sanitize_str(args: list):
|
||||
for idx in range(0, len(args)):
|
||||
if isinstance(args[idx], str) and len(args[idx]) >= 1000:
|
||||
args[idx] = f"<str {len(args[idx])}>"
|
||||
for name in ('init_image', 'last_image'):
|
||||
val = getattr(req, name, None)
|
||||
if isinstance(val, str) and len(val) >= 1000:
|
||||
setattr(req, name, f"<str {len(val)}>")
|
||||
if req.script_args:
|
||||
sanitize_str(req.script_args)
|
||||
if req.alwayson_scripts:
|
||||
for script_obj in req.alwayson_scripts.values():
|
||||
if script_obj and "args" in script_obj and script_obj["args"]:
|
||||
sanitize_str(script_obj["args"])
|
||||
|
||||
def post_video(self, req: ReqVideo):
|
||||
"""Generate a video, or a single still frame, using a video model.
|
||||
|
||||
Omit `engine` and `model` to drive the currently loaded checkpoint when it is
|
||||
video-capable; this covers models loaded from local folders that have no registry
|
||||
entry. Pass both names to select a registry model, which is loaded on demand;
|
||||
`GET /sdapi/v1/video/models` enumerates the valid pairs.
|
||||
|
||||
`frames` of 1 on a workflow model produces a single still image returned in `frames`.
|
||||
Disk outputs are controlled by `mp4_video`, `mp4_frames`, `mp4_sf`, and `mp4_thumb`;
|
||||
response payloads are controlled independently by `send_video`, `send_frames`, and
|
||||
`send_thumbnail`. Artifacts above the base64 size cap return `video` empty with
|
||||
`video_path` set; fetch those via `GET /sdapi/v1/video/file`.
|
||||
|
||||
`init_image` and `last_image` accept base64 data, data URIs, or upload references.
|
||||
Progress is reported on `GET /sdapi/v1/progress`; `POST /sdapi/v1/interrupt` cancels.
|
||||
Switching checkpoints via `override_settings` is not supported here; use
|
||||
`POST /sdapi/v1/checkpoint` before generating.
|
||||
"""
|
||||
try:
|
||||
selected, needs_load = video_run.resolve_model(req.engine, req.model)
|
||||
except video_run.VideoError as e:
|
||||
raise HTTPException(status_code=e.code, detail=str(e)) from e
|
||||
sampler_name = helpers.validate_sampler_name(req.sampler_name)
|
||||
init_image = helpers.decode_base64_to_image(req.init_image) if req.init_image else None
|
||||
last_image = helpers.decode_base64_to_image(req.last_image) if req.last_image else None
|
||||
overrides = dict(req.override_settings or {})
|
||||
for key in ('sd_model_checkpoint', 'sd_model_refiner'):
|
||||
if key in overrides:
|
||||
raise HTTPException(status_code=400, detail=f"{key} override is not supported here: switch models via POST /sdapi/v1/checkpoint before generating")
|
||||
p_stub = SimpleNamespace(per_script_args={})
|
||||
script_runner, script_args = self.prepare_scripts(p_stub, req)
|
||||
extra = getattr(req, 'extra', {}) or {}
|
||||
|
||||
with self.queue_lock:
|
||||
jobid = shared.state.begin('API-VID', api=True)
|
||||
try:
|
||||
res = video_run.run(
|
||||
selected,
|
||||
prompt=req.prompt,
|
||||
negative=req.negative_prompt,
|
||||
styles=req.styles,
|
||||
width=req.width,
|
||||
height=req.height,
|
||||
frames=req.frames,
|
||||
steps=req.steps,
|
||||
sampler_name=sampler_name,
|
||||
sampler_shift=req.sampler_shift,
|
||||
dynamic_shift=req.dynamic_shift,
|
||||
seed=req.seed,
|
||||
guidance_scale=req.guidance_scale,
|
||||
guidance_true=req.guidance_true,
|
||||
init_image=init_image,
|
||||
init_strength=req.init_strength,
|
||||
last_image=last_image,
|
||||
vae_type=req.vae_type,
|
||||
vae_tile_frames=req.vae_tile_frames,
|
||||
audio=req.audio,
|
||||
mp4_fps=req.mp4_fps,
|
||||
mp4_interpolate=req.mp4_interpolate,
|
||||
mp4_codec=req.mp4_codec,
|
||||
mp4_ext=req.mp4_ext,
|
||||
mp4_opt=req.mp4_opt,
|
||||
mp4_video=req.mp4_video,
|
||||
mp4_frames=req.mp4_frames,
|
||||
mp4_sf=req.mp4_sf,
|
||||
mp4_thumb=req.mp4_thumb,
|
||||
override_settings=overrides,
|
||||
engine=req.engine,
|
||||
scripts=script_runner,
|
||||
script_args=script_args,
|
||||
per_script_args=p_stub.per_script_args,
|
||||
extra_p=extra,
|
||||
needs_load=needs_load,
|
||||
)
|
||||
except video_run.VideoError as e:
|
||||
raise HTTPException(status_code=e.code, detail=str(e)) from e
|
||||
finally:
|
||||
shared.state.end(jobid, api=False)
|
||||
|
||||
send_frames = req.send_frames or res.still # a still request has no other product to return
|
||||
b64_frames = list(map(helpers.encode_pil_to_base64, res.images)) if send_frames else []
|
||||
video_b64 = helpers.encode_file_to_base64(res.video_path) if req.send_video and res.video_path else None
|
||||
thumb_b64 = helpers.encode_file_to_base64(res.thumb_path) if req.send_thumbnail and res.thumb_path else None
|
||||
duration = round(res.num_frames / res.fps, 3) if res.fps > 0 else 0.0
|
||||
self.sanitize_b64(req)
|
||||
params = {k: v for k, v in vars(req).items() if k != 'extra'}
|
||||
return ResVideo(
|
||||
video=video_b64,
|
||||
video_path=res.video_path,
|
||||
thumbnail=thumb_b64,
|
||||
thumbnail_path=res.thumb_path,
|
||||
frames=b64_frames,
|
||||
frames_count=res.num_frames,
|
||||
fps=res.fps,
|
||||
duration=duration,
|
||||
has_audio=res.has_audio,
|
||||
still=res.still,
|
||||
params=params,
|
||||
info=res.processed.info,
|
||||
)
|
||||
|
||||
def get_video_models(self, engine: str | None = None):
|
||||
"""List video engines and models; optionally filter by engine."""
|
||||
items = []
|
||||
for family, rows in models_def.models.items():
|
||||
if engine is not None and family.lower() != engine.lower():
|
||||
continue
|
||||
for m in rows:
|
||||
if m.name == 'None':
|
||||
continue
|
||||
items.append(ItemVideoModel(
|
||||
engine=family,
|
||||
name=m.name,
|
||||
repo=m.repo or '',
|
||||
url=m.url or '',
|
||||
mode=model_mode(m),
|
||||
workflow=m.workflow,
|
||||
base=m.base,
|
||||
loaded=(m.name == video_load.loaded_model),
|
||||
))
|
||||
return items
|
||||
|
||||
def get_video_file(self, file: str):
|
||||
"""Serve a video artifact produced by this endpoint; the path must resolve inside the video output directory."""
|
||||
import mimetypes
|
||||
from pathlib import Path
|
||||
from starlette.responses import FileResponse
|
||||
if not file or not file.strip():
|
||||
raise HTTPException(status_code=400, detail="file path is required")
|
||||
root = Path(resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_video)).resolve()
|
||||
target = Path(file).resolve()
|
||||
if root not in target.parents:
|
||||
raise HTTPException(status_code=403, detail=f"file {file}: must be inside the video output directory")
|
||||
if not target.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"file not found: {file}")
|
||||
media_type = mimetypes.guess_type(target.name)[0] or 'application/octet-stream'
|
||||
return FileResponse(str(target), media_type=media_type, filename=target.name)
|
||||
Reference in New Issue
Block a user