mirror of
https://github.com/vladmandic/automatic
synced 2026-09-18 16:54:33 +02:00
feat(api): add caption API endpoints and documentation
Add comprehensive caption/interrogate API with documentation: - GET /sdapi/v1/interrogate: List available interrogation models - POST /sdapi/v1/interrogate: Interrogate with OpenCLIP/BLIP/DeepDanbooru - POST /sdapi/v1/vqa: Caption with Vision-Language Models (VLM) - GET /sdapi/v1/vqa: List available VLM models - POST /sdapi/v1/vqa/batch: Batch caption multiple images - POST /sdapi/v1/tagger: Tag images with WaifuDiffusion/DeepBooru Updates: - Add detailed docstrings with usage examples - Fix analyze_image response parsing for Gradio update dicts - Add request/response models for all endpoints
This commit is contained in:
CalamitousFelicitousness
+7
-3
@@ -76,7 +76,7 @@ class Api:
|
||||
# enumerator api
|
||||
self.add_api_route("/sdapi/v1/preprocessors", self.process.get_preprocess, methods=["GET"], response_model=List[process.ItemPreprocess])
|
||||
self.add_api_route("/sdapi/v1/masking", self.process.get_mask, methods=["GET"], response_model=process.ItemMask)
|
||||
self.add_api_route("/sdapi/v1/interrogate", endpoints.get_interrogate, methods=["GET"], response_model=List[str])
|
||||
self.add_api_route("/sdapi/v1/interrogate", endpoints.get_interrogate, methods=["GET"], response_model=List[str], tags=["Caption"])
|
||||
self.add_api_route("/sdapi/v1/samplers", endpoints.get_samplers, methods=["GET"], response_model=List[models.ItemSampler])
|
||||
self.add_api_route("/sdapi/v1/upscalers", endpoints.get_upscalers, methods=["GET"], response_model=List[models.ItemUpscaler])
|
||||
self.add_api_route("/sdapi/v1/sd-models", endpoints.get_sd_models, methods=["GET"], response_model=List[models.ItemModel])
|
||||
@@ -91,8 +91,12 @@ class Api:
|
||||
|
||||
# functional api
|
||||
self.add_api_route("/sdapi/v1/png-info", endpoints.post_pnginfo, methods=["POST"], response_model=models.ResImageInfo)
|
||||
self.add_api_route("/sdapi/v1/interrogate", endpoints.post_interrogate, methods=["POST"])
|
||||
self.add_api_route("/sdapi/v1/vqa", endpoints.post_vqa, methods=["POST"])
|
||||
self.add_api_route("/sdapi/v1/interrogate", endpoints.post_interrogate, methods=["POST"], response_model=models.ResInterrogate, tags=["Caption"])
|
||||
self.add_api_route("/sdapi/v1/vqa", endpoints.post_vqa, methods=["POST"], response_model=models.ResVQA, tags=["Caption"])
|
||||
self.add_api_route("/sdapi/v1/vqa/models", endpoints.get_vqa_models, methods=["GET"], response_model=List[models.ItemVLMModel], tags=["Caption"])
|
||||
self.add_api_route("/sdapi/v1/vqa/prompts", endpoints.get_vqa_prompts, methods=["GET"], response_model=models.ResVLMPrompts, tags=["Caption"])
|
||||
self.add_api_route("/sdapi/v1/tagger", endpoints.post_tagger, methods=["POST"], response_model=models.ResTagger, tags=["Caption"])
|
||||
self.add_api_route("/sdapi/v1/tagger/models", endpoints.get_tagger_models, methods=["GET"], response_model=List[models.ItemTaggerModel], tags=["Caption"])
|
||||
self.add_api_route("/sdapi/v1/checkpoint", endpoints.get_checkpoint, methods=["GET"])
|
||||
self.add_api_route("/sdapi/v1/checkpoint", endpoints.set_checkpoint, methods=["POST"])
|
||||
self.add_api_route("/sdapi/v1/refresh-checkpoints", endpoints.post_refresh_checkpoints, methods=["POST"])
|
||||
|
||||
+253
-2
@@ -92,6 +92,20 @@ def get_extra_networks(page: Optional[str] = None, name: Optional[str] = None, f
|
||||
return res
|
||||
|
||||
def get_interrogate():
|
||||
"""
|
||||
List available interrogation models.
|
||||
|
||||
Returns model identifiers for use with POST /sdapi/v1/interrogate.
|
||||
|
||||
**Model Types:**
|
||||
- `deepdanbooru`: Anime-style image tagger returning comma-separated tags
|
||||
- OpenCLIP models: Format `architecture/pretrained_dataset` (e.g., `ViT-L-14/openai`)
|
||||
|
||||
**Example Response:**
|
||||
```json
|
||||
["deepdanbooru", "ViT-L-14/openai", "ViT-H-14/laion2b_s32b_b79k"]
|
||||
```
|
||||
"""
|
||||
from modules.interrogate.openclip import refresh_clip_models
|
||||
return ['deepdanbooru'] + refresh_clip_models()
|
||||
|
||||
@@ -103,6 +117,28 @@ def get_schedulers():
|
||||
return all_schedulers
|
||||
|
||||
def post_interrogate(req: models.ReqInterrogate):
|
||||
"""
|
||||
Interrogate an image using OpenCLIP/BLIP or DeepDanbooru.
|
||||
|
||||
Analyze image using CLIP model via OpenCLIP to generate Stable Diffusion prompts.
|
||||
|
||||
**DeepDanbooru Mode** (`model="deepdanbooru"`):
|
||||
- Specialized for anime/illustration images
|
||||
- Returns comma-separated tags with confidence
|
||||
|
||||
**OpenCLIP Mode** (any other model):
|
||||
- Uses CLIP for image-text matching, BLIP for captioning
|
||||
- **Modes:**
|
||||
- `best`: Highest quality, combines multiple techniques
|
||||
- `fast`: Quick results with fewer iterations
|
||||
- `classic`: Traditional CLIP interrogator style
|
||||
- `caption`: BLIP caption only
|
||||
- `negative`: Generate negative prompt suggestions
|
||||
- Set `analyze=True` for detailed breakdown (medium, artist, movement, trending, flavor)
|
||||
|
||||
**Error Codes:**
|
||||
- 404: Image not provided or model not found
|
||||
"""
|
||||
if req.image is None or len(req.image) < 64:
|
||||
raise HTTPException(status_code=404, detail="Image not found")
|
||||
image = helpers.decode_base64_to_image(req.image)
|
||||
@@ -122,17 +158,232 @@ def post_interrogate(req: models.ReqInterrogate):
|
||||
if not req.analyze:
|
||||
return models.ResInterrogate(caption=caption)
|
||||
else:
|
||||
medium, artist, movement, trending, flavor, _ = analyze_image(image, clip_model=req.clip_model, blip_model=req.blip_model)
|
||||
analyze_results = analyze_image(image, clip_model=req.clip_model, blip_model=req.blip_model)
|
||||
# Extract top-ranked item from each Gradio update dict
|
||||
def get_top_item(result):
|
||||
if isinstance(result, dict) and 'value' in result:
|
||||
value = result['value']
|
||||
if isinstance(value, dict) and value:
|
||||
return next(iter(value.keys())) # First key = top ranked
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return None
|
||||
medium = get_top_item(analyze_results[0])
|
||||
artist = get_top_item(analyze_results[1])
|
||||
movement = get_top_item(analyze_results[2])
|
||||
trending = get_top_item(analyze_results[3])
|
||||
flavor = get_top_item(analyze_results[4])
|
||||
return models.ResInterrogate(caption=caption, medium=medium, artist=artist, movement=movement, trending=trending, flavor=flavor)
|
||||
|
||||
def post_vqa(req: models.ReqVQA):
|
||||
"""
|
||||
Caption an image using Vision-Language Models (VLM).
|
||||
|
||||
Analyze image using vision language model for flexible image understanding.
|
||||
Supports 60+ models including Google Gemma, Alibaba Qwen, Microsoft Florence, Moondream.
|
||||
|
||||
**Common Tasks:**
|
||||
|
||||
1. **General Captioning**
|
||||
- `question="Short Caption"` or `"Normal Caption"` or `"Long Caption"`
|
||||
|
||||
2. **Object Detection** (Florence-2):
|
||||
- `question="<OD>"` - Returns bounding boxes
|
||||
- `question="<DENSE_REGION_CAPTION>"` - Captions for regions
|
||||
|
||||
3. **OCR/Text Recognition** (Florence-2):
|
||||
- `question="<OCR>"` - Extract text from image
|
||||
|
||||
4. **Point/Detect** (Moondream):
|
||||
- `question="Point at the cat"` - Returns coordinates
|
||||
- `question="Detect all faces"` - Returns all instances
|
||||
|
||||
**Annotated Images:**
|
||||
Set `include_annotated=True` to receive an annotated image with detection results.
|
||||
Returns Base64 PNG with bounding boxes and points drawn for:
|
||||
- Florence-2: Object detection, phrase grounding, region proposals
|
||||
- Moondream 2/3: Point detection, object detection, gaze detection
|
||||
|
||||
**Model Selection:**
|
||||
- Small/Fast: Florence 2 Base, SmolVLM 0.5B, FastVLM 0.5B
|
||||
- Balanced: Qwen 2.5 VL 3B, Florence 2 Large, Gemma 3 4B
|
||||
- High Quality: Qwen 3 VL 8B, JoyCaption Beta, Moondream 3
|
||||
|
||||
Use GET /sdapi/v1/vqa/models for complete model list.
|
||||
"""
|
||||
if req.image is None or len(req.image) < 64:
|
||||
raise HTTPException(status_code=404, detail="Image not found")
|
||||
image = helpers.decode_base64_to_image(req.image)
|
||||
image = image.convert('RGB')
|
||||
from modules.interrogate import vqa
|
||||
answer = vqa.interrogate(req.question, req.system, '', image, req.model)
|
||||
return models.ResVQA(answer=answer)
|
||||
# Return annotated image if requested and available
|
||||
annotated_b64 = None
|
||||
if req.include_annotated:
|
||||
annotated_img = vqa.get_last_annotated_image()
|
||||
if annotated_img is not None:
|
||||
annotated_b64 = helpers.encode_pil_to_base64(annotated_img)
|
||||
return models.ResVQA(answer=answer, annotated_image=annotated_b64)
|
||||
|
||||
def get_vqa_models():
|
||||
"""
|
||||
List available VLM models for captioning.
|
||||
|
||||
Returns all Vision-Language Models available for POST /sdapi/v1/vqa.
|
||||
|
||||
**Response includes:**
|
||||
- `name`: Display name
|
||||
- `repo`: HuggingFace repository ID
|
||||
- `prompts`: Available prompts/tasks
|
||||
- `capabilities`: Model features (caption, vqa, detection, ocr, thinking)
|
||||
"""
|
||||
from modules.interrogate import vqa
|
||||
models_list = []
|
||||
for name, repo in vqa.vlm_models.items():
|
||||
prompts = vqa.get_prompts_for_model(name)
|
||||
capabilities = ["caption", "vqa"]
|
||||
# Detect additional capabilities based on model name
|
||||
name_lower = name.lower()
|
||||
if 'florence' in name_lower or 'promptgen' in name_lower:
|
||||
capabilities.extend(["detection", "ocr"])
|
||||
if 'moondream' in name_lower:
|
||||
capabilities.append("detection")
|
||||
if vqa.is_thinking_model(name):
|
||||
capabilities.append("thinking")
|
||||
models_list.append({
|
||||
"name": name,
|
||||
"repo": repo,
|
||||
"prompts": prompts,
|
||||
"capabilities": list(set(capabilities))
|
||||
})
|
||||
return models_list
|
||||
|
||||
def get_vqa_prompts(model: Optional[str] = None):
|
||||
"""
|
||||
List available prompts/tasks for VLM models.
|
||||
|
||||
**Query Parameters:**
|
||||
- `model` (optional): Filter prompts for a specific model
|
||||
|
||||
**Prompt Categories:**
|
||||
- Common: Use Prompt, Short/Normal/Long Caption
|
||||
- Florence: Phrase Grounding, Object Detection, OCR, Dense Region Caption
|
||||
- Moondream: Point at..., Detect all..., Detect Gaze
|
||||
"""
|
||||
from modules.interrogate import vqa
|
||||
if model:
|
||||
prompts = vqa.get_prompts_for_model(model)
|
||||
return {"available": prompts}
|
||||
return {
|
||||
"common": vqa.vlm_prompts_common,
|
||||
"florence": vqa.vlm_prompts_florence,
|
||||
"moondream": vqa.vlm_prompts_moondream,
|
||||
"moondream2_only": vqa.vlm_prompts_moondream2
|
||||
}
|
||||
|
||||
def get_tagger_models():
|
||||
"""
|
||||
List available tagger models.
|
||||
|
||||
Returns WaifuDiffusion and DeepBooru models for image tagging.
|
||||
|
||||
**WaifuDiffusion Models:**
|
||||
- `wd-eva02-large-tagger-v3` (recommended)
|
||||
- `wd-vit-tagger-v3`, `wd-convnext-tagger-v3`, `wd-swinv2-tagger-v3`
|
||||
|
||||
**DeepBooru:**
|
||||
- Legacy tagger for anime images
|
||||
"""
|
||||
from modules.interrogate import waifudiffusion
|
||||
models_list = []
|
||||
# Add WaifuDiffusion models
|
||||
for name in waifudiffusion.get_models():
|
||||
models_list.append({"name": name, "type": "waifudiffusion"})
|
||||
# Add DeepBooru
|
||||
models_list.append({"name": "deepbooru", "type": "deepbooru"})
|
||||
return models_list
|
||||
|
||||
def post_tagger(req: models.ReqTagger):
|
||||
"""
|
||||
Tag an image using WaifuDiffusion or DeepBooru.
|
||||
|
||||
Generate anime/illustration tags for images.
|
||||
|
||||
**WaifuDiffusion Models:**
|
||||
- `wd-eva02-large-tagger-v3` (recommended)
|
||||
- `wd-vit-tagger-v3`, `wd-convnext-tagger-v3`, `wd-swinv2-tagger-v3`
|
||||
|
||||
**DeepBooru:**
|
||||
- Legacy tagger, use `deepbooru` or `deepdanbooru`
|
||||
|
||||
**Thresholds:**
|
||||
- `threshold`: General tag confidence (default: 0.5)
|
||||
- `character_threshold`: Character identification (default: 0.85, WaifuDiffusion only)
|
||||
"""
|
||||
if req.image is None or len(req.image) < 64:
|
||||
raise HTTPException(status_code=404, detail="Image not found")
|
||||
image = helpers.decode_base64_to_image(req.image)
|
||||
image = image.convert('RGB')
|
||||
from modules.interrogate import tagger
|
||||
# Determine if using DeepBooru
|
||||
is_deepbooru = req.model.lower() in ('deepbooru', 'deepdanbooru')
|
||||
# Store original settings and apply request settings
|
||||
original_opts = {
|
||||
'tagger_threshold': shared.opts.tagger_threshold,
|
||||
'tagger_max_tags': shared.opts.tagger_max_tags,
|
||||
'tagger_include_rating': shared.opts.tagger_include_rating,
|
||||
'tagger_sort_alpha': shared.opts.tagger_sort_alpha,
|
||||
'tagger_use_spaces': shared.opts.tagger_use_spaces,
|
||||
'tagger_escape_brackets': shared.opts.tagger_escape_brackets,
|
||||
'tagger_exclude_tags': shared.opts.tagger_exclude_tags,
|
||||
'tagger_show_scores': shared.opts.tagger_show_scores,
|
||||
}
|
||||
# WaifuDiffusion-specific settings (not applicable to DeepBooru)
|
||||
if not is_deepbooru:
|
||||
original_opts['waifudiffusion_character_threshold'] = shared.opts.waifudiffusion_character_threshold
|
||||
original_opts['waifudiffusion_model'] = shared.opts.waifudiffusion_model
|
||||
try:
|
||||
shared.opts.tagger_threshold = req.threshold
|
||||
shared.opts.tagger_max_tags = req.max_tags
|
||||
shared.opts.tagger_include_rating = req.include_rating
|
||||
shared.opts.tagger_sort_alpha = req.sort_alpha
|
||||
shared.opts.tagger_use_spaces = req.use_spaces
|
||||
shared.opts.tagger_escape_brackets = req.escape_brackets
|
||||
shared.opts.tagger_exclude_tags = req.exclude_tags
|
||||
shared.opts.tagger_show_scores = req.show_scores
|
||||
# WaifuDiffusion-specific settings (not applicable to DeepBooru)
|
||||
if not is_deepbooru:
|
||||
shared.opts.waifudiffusion_character_threshold = req.character_threshold
|
||||
shared.opts.waifudiffusion_model = req.model
|
||||
tags = tagger.tag(image, model_name='DeepBooru' if is_deepbooru else None)
|
||||
# Parse scores if requested - format is "(tag:0.95)" from WaifuDiffusion/DeepBooru
|
||||
scores = None
|
||||
if req.show_scores:
|
||||
scores = {}
|
||||
for item in tags.split(', '):
|
||||
item = item.strip()
|
||||
# Format: "(tag:0.95)" - tag and score wrapped in parentheses
|
||||
if item.startswith('(') and item.endswith(')') and ':' in item:
|
||||
inner = item[1:-1] # Remove outer parentheses
|
||||
tag, score_str = inner.rsplit(':', 1)
|
||||
try:
|
||||
scores[tag.strip()] = float(score_str.strip())
|
||||
except ValueError:
|
||||
pass
|
||||
elif ':' in item:
|
||||
# Fallback format: "tag:0.95" without parentheses
|
||||
tag, score_str = item.rsplit(':', 1)
|
||||
try:
|
||||
scores[tag.strip()] = float(score_str.strip())
|
||||
except ValueError:
|
||||
pass
|
||||
if not scores:
|
||||
scores = None
|
||||
return models.ResTagger(tags=tags, scores=scores)
|
||||
finally:
|
||||
# Restore original settings
|
||||
for key, value in original_opts.items():
|
||||
setattr(shared.opts, key, value)
|
||||
|
||||
def post_unload_checkpoint():
|
||||
from modules import sd_models
|
||||
|
||||
+70
-14
@@ -366,31 +366,87 @@ class ResStatus(BaseModel):
|
||||
progress: Optional[float] = Field(default=None, title="Progress", description="The progress with a range of 0 to 1")
|
||||
|
||||
class ReqInterrogate(BaseModel):
|
||||
image: str = Field(default="", title="Image", description="Image to work on, must be a Base64 string containing the image's data.")
|
||||
clip_model: str = Field(default="", title="CLiP Model", description="The interrogate model used.")
|
||||
blip_model: str = Field(default="", title="BLiP Model", description="The interrogate model used.")
|
||||
"""Request model for OpenCLIP/BLIP image interrogation.
|
||||
|
||||
Analyze image using CLIP model via OpenCLIP to generate prompts,
|
||||
or use DeepDanbooru for anime-style tagging.
|
||||
"""
|
||||
image: str = Field(default="", title="Image", description="Image to interrogate. Must be a Base64 encoded string containing the image data (PNG/JPEG).")
|
||||
model: str = Field(default="ViT-L-14/openai", title="Model", description="OpenCLIP model to use. Use 'deepdanbooru' or 'deepbooru' for anime tagging. Get available models from GET /sdapi/v1/interrogate.")
|
||||
clip_model: str = Field(default="ViT-L-14/openai", title="CLIP Model", description="OpenCLIP model for image encoding. Format: 'architecture/pretrained_dataset'.")
|
||||
blip_model: str = Field(default="blip-large", title="BLIP Model", description="BLIP/BLIP2 captioning model for generating base captions. Options: blip-base, blip-large, blip2-opt-2.7b, blip2-opt-6.7b.")
|
||||
mode: str = Field(default="best", title="Mode", description="Interrogation mode: 'best' (highest quality), 'fast' (quick results), 'classic' (traditional style), 'caption' (BLIP caption only), 'negative' (negative prompt generation).")
|
||||
analyze: bool = Field(default=False, title="Analyze", description="If True, returns detailed image analysis breakdown (medium, artist, movement, trending, flavor) in addition to caption.")
|
||||
|
||||
InterrogateRequest = ReqInterrogate # alias for backwards compatibility
|
||||
|
||||
class ResInterrogate(BaseModel):
|
||||
caption: Optional[str] = Field(default=None, title="Caption", description="The generated caption for the image.")
|
||||
medium: Optional[str] = Field(default=None, title="Medium", description="Image medium.")
|
||||
artist: Optional[str] = Field(default=None, title="Medium", description="Image artist.")
|
||||
movement: Optional[str] = Field(default=None, title="Medium", description="Image movement.")
|
||||
trending: Optional[str] = Field(default=None, title="Medium", description="Image trending.")
|
||||
flavor: Optional[str] = Field(default=None, title="Medium", description="Image flavor.")
|
||||
"""Response model for image interrogation results."""
|
||||
caption: Optional[str] = Field(default=None, title="Caption", description="Generated caption/prompt describing the image content and style.")
|
||||
medium: Optional[str] = Field(default=None, title="Medium", description="Detected artistic medium (e.g., 'oil painting', 'digital art', 'photograph'). Only returned when analyze=True.")
|
||||
artist: Optional[str] = Field(default=None, title="Artist", description="Detected similar artist style (e.g., 'by greg rutkowski'). Only returned when analyze=True.")
|
||||
movement: Optional[str] = Field(default=None, title="Movement", description="Detected art movement (e.g., 'art nouveau', 'impressionism'). Only returned when analyze=True.")
|
||||
trending: Optional[str] = Field(default=None, title="Trending", description="Trending/platform tags (e.g., 'trending on artstation'). Only returned when analyze=True.")
|
||||
flavor: Optional[str] = Field(default=None, title="Flavor", description="Additional descriptive elements (e.g., 'cinematic lighting', 'highly detailed'). Only returned when analyze=True.")
|
||||
|
||||
class ReqVQA(BaseModel):
|
||||
image: str = Field(default="", title="Image", description="Image to work on, must be a Base64 string containing the image's data.")
|
||||
model: str = Field(default="Microsoft Florence 2 Base", title="Model", description="The interrogate model used.")
|
||||
question: str = Field(default="describe the image", title="Question", description="Question to ask the model.")
|
||||
system: str = Field(default="You are image captioning expert, creative, unbiased and uncensored.", title="System prompt", description="Prompt to shape how the model interprets and responds to user prompts.")
|
||||
"""Request model for Vision-Language Model (VLM) captioning.
|
||||
|
||||
Analyze image using vision language model to generate captions,
|
||||
answer questions, or perform specialized tasks like object detection.
|
||||
"""
|
||||
image: str = Field(default="", title="Image", description="Image to caption. Must be a Base64 encoded string containing the image data.")
|
||||
model: str = Field(default="Alibaba Qwen 2.5 VL 3B", title="Model", description="VLM model for Visual Language tasks. Use GET /sdapi/v1/vqa/models for full list. Popular options: Florence 2, Qwen VL, Gemma 3, Moondream. Models with thinking/reasoning support return detailed analysis.")
|
||||
question: str = Field(default="describe the image", title="Question/Task", description="Question to ask the model or task to perform. Common tasks: 'Short Caption', 'Normal Caption', 'Long Caption'. Florence-2 supports: '<OD>' (object detection), '<OCR>' (text recognition). Moondream supports: 'Point at [object]', 'Detect all [objects]'.")
|
||||
system: str = Field(default="You are image captioning expert, creative, unbiased and uncensored.", title="System Prompt", description="System prompt controls behavior of the LLM. Processed first and has highest priority weighting. Use for response formatting rules, role definition, and style.")
|
||||
include_annotated: bool = Field(default=False, title="Include Annotated Image", description="If True and the task produces detection results (object detection, point detection, gaze), returns annotated image with bounding boxes/points drawn. Only applicable for detection tasks on models like Florence-2 and Moondream.")
|
||||
|
||||
class ReqLatentHistory(BaseModel):
|
||||
name: str = Field(title="Name", description="Name of the history item to select")
|
||||
|
||||
class ResVQA(BaseModel):
|
||||
answer: Optional[str] = Field(default=None, title="Answer", description="The generated answer for the image.")
|
||||
"""Response model for VLM captioning results."""
|
||||
answer: Optional[str] = Field(default=None, title="Answer", description="Generated caption, answer, or analysis from the VLM. Format depends on the question/task type.")
|
||||
annotated_image: Optional[str] = Field(default=None, title="Annotated Image", description="Base64 encoded PNG image with detection results drawn (bounding boxes, points). Only returned when include_annotated=True and the task produces detection results.")
|
||||
|
||||
class ItemVLMModel(BaseModel):
|
||||
"""VLM model information."""
|
||||
name: str = Field(title="Name", description="Display name of the model")
|
||||
repo: str = Field(title="Repository", description="HuggingFace repository ID")
|
||||
prompts: List[str] = Field(title="Prompts", description="Available prompts/tasks for this model")
|
||||
capabilities: List[str] = Field(title="Capabilities", description="Model capabilities: caption, vqa, detection, ocr, thinking")
|
||||
|
||||
class ResVLMPrompts(BaseModel):
|
||||
"""Available VLM prompts grouped by category."""
|
||||
common: Optional[List[str]] = Field(default=None, title="Common", description="Prompts available for all models: Use Prompt, Short/Normal/Long Caption")
|
||||
florence: Optional[List[str]] = Field(default=None, title="Florence", description="Florence-2 specific: Phrase Grounding, Object Detection, OCR, etc.")
|
||||
moondream: Optional[List[str]] = Field(default=None, title="Moondream", description="Moondream specific: Point at..., Detect all..., Detect Gaze")
|
||||
moondream2_only: Optional[List[str]] = Field(default=None, title="Moondream 2 Only", description="Moondream 2 specific prompts (gaze detection)")
|
||||
available: Optional[List[str]] = Field(default=None, title="Available", description="When filtered by model, the available prompts for that model")
|
||||
|
||||
class ItemTaggerModel(BaseModel):
|
||||
"""Tagger model information."""
|
||||
name: str = Field(title="Name", description="Model name")
|
||||
type: str = Field(title="Type", description="Model type: waifudiffusion or deepbooru")
|
||||
|
||||
class ReqTagger(BaseModel):
|
||||
"""Request model for image tagging."""
|
||||
image: str = Field(default="", title="Image", description="Image to tag. Must be a Base64 encoded string.")
|
||||
model: str = Field(default="wd-eva02-large-tagger-v3", title="Model", description="Tagger model to use. WaifuDiffusion models (wd-*) or 'deepbooru'/'deepdanbooru'.")
|
||||
threshold: float = Field(default=0.5, title="Threshold", description="General tag confidence threshold (0-1). Tags below this confidence are excluded.", ge=0.0, le=1.0)
|
||||
character_threshold: float = Field(default=0.85, title="Character Threshold", description="Character tag confidence threshold (0-1). Higher values for more precise character identification. WaifuDiffusion only - ignored for DeepBooru.", ge=0.0, le=1.0)
|
||||
max_tags: int = Field(default=74, title="Max Tags", description="Maximum number of tags to return.", ge=1, le=512)
|
||||
include_rating: bool = Field(default=False, title="Include Rating", description="Include rating tags (safe, questionable, explicit) in results.")
|
||||
sort_alpha: bool = Field(default=False, title="Sort Alphabetically", description="Sort tags alphabetically instead of by confidence.")
|
||||
use_spaces: bool = Field(default=False, title="Use Spaces", description="Replace underscores with spaces in tag names.")
|
||||
escape_brackets: bool = Field(default=True, title="Escape Brackets", description="Escape parentheses in tags for prompt compatibility.")
|
||||
exclude_tags: str = Field(default="", title="Exclude Tags", description="Comma-separated list of tags to exclude from results.")
|
||||
show_scores: bool = Field(default=False, title="Show Scores", description="Include confidence scores with each tag in the output.")
|
||||
|
||||
class ResTagger(BaseModel):
|
||||
"""Response model for image tagging results."""
|
||||
tags: str = Field(title="Tags", description="Comma-separated list of detected tags")
|
||||
scores: Optional[dict] = Field(default=None, title="Scores", description="Tag confidence scores (when show_scores=True)")
|
||||
|
||||
class ResTrain(BaseModel):
|
||||
info: str = Field(title="Train info", description="Response string from train embedding task.")
|
||||
|
||||
Reference in New Issue
Block a user