diff --git a/modules/api/api.py b/modules/api/api.py index eeff16e10..80d3c6d34 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -76,7 +76,7 @@ class Api: # enumerator api self.add_api_route("/sdapi/v1/preprocessors", self.process.get_preprocess, methods=["GET"], response_model=List[process.ItemPreprocess]) self.add_api_route("/sdapi/v1/masking", self.process.get_mask, methods=["GET"], response_model=process.ItemMask) - self.add_api_route("/sdapi/v1/interrogate", endpoints.get_interrogate, methods=["GET"], response_model=List[str]) + self.add_api_route("/sdapi/v1/interrogate", endpoints.get_interrogate, methods=["GET"], response_model=List[str], tags=["Caption"]) self.add_api_route("/sdapi/v1/samplers", endpoints.get_samplers, methods=["GET"], response_model=List[models.ItemSampler]) self.add_api_route("/sdapi/v1/upscalers", endpoints.get_upscalers, methods=["GET"], response_model=List[models.ItemUpscaler]) self.add_api_route("/sdapi/v1/sd-models", endpoints.get_sd_models, methods=["GET"], response_model=List[models.ItemModel]) @@ -91,8 +91,12 @@ class Api: # functional api self.add_api_route("/sdapi/v1/png-info", endpoints.post_pnginfo, methods=["POST"], response_model=models.ResImageInfo) - self.add_api_route("/sdapi/v1/interrogate", endpoints.post_interrogate, methods=["POST"]) - self.add_api_route("/sdapi/v1/vqa", endpoints.post_vqa, methods=["POST"]) + self.add_api_route("/sdapi/v1/interrogate", endpoints.post_interrogate, methods=["POST"], response_model=models.ResInterrogate, tags=["Caption"]) + self.add_api_route("/sdapi/v1/vqa", endpoints.post_vqa, methods=["POST"], response_model=models.ResVQA, tags=["Caption"]) + self.add_api_route("/sdapi/v1/vqa/models", endpoints.get_vqa_models, methods=["GET"], response_model=List[models.ItemVLMModel], tags=["Caption"]) + self.add_api_route("/sdapi/v1/vqa/prompts", endpoints.get_vqa_prompts, methods=["GET"], response_model=models.ResVLMPrompts, tags=["Caption"]) + self.add_api_route("/sdapi/v1/tagger", endpoints.post_tagger, methods=["POST"], response_model=models.ResTagger, tags=["Caption"]) + self.add_api_route("/sdapi/v1/tagger/models", endpoints.get_tagger_models, methods=["GET"], response_model=List[models.ItemTaggerModel], tags=["Caption"]) self.add_api_route("/sdapi/v1/checkpoint", endpoints.get_checkpoint, methods=["GET"]) self.add_api_route("/sdapi/v1/checkpoint", endpoints.set_checkpoint, methods=["POST"]) self.add_api_route("/sdapi/v1/refresh-checkpoints", endpoints.post_refresh_checkpoints, methods=["POST"]) diff --git a/modules/api/endpoints.py b/modules/api/endpoints.py index 8b9b47258..4b96ffdb9 100644 --- a/modules/api/endpoints.py +++ b/modules/api/endpoints.py @@ -92,6 +92,20 @@ def get_extra_networks(page: Optional[str] = None, name: Optional[str] = None, f return res def get_interrogate(): + """ + List available interrogation models. + + Returns model identifiers for use with POST /sdapi/v1/interrogate. + + **Model Types:** + - `deepdanbooru`: Anime-style image tagger returning comma-separated tags + - OpenCLIP models: Format `architecture/pretrained_dataset` (e.g., `ViT-L-14/openai`) + + **Example Response:** + ```json + ["deepdanbooru", "ViT-L-14/openai", "ViT-H-14/laion2b_s32b_b79k"] + ``` + """ from modules.interrogate.openclip import refresh_clip_models return ['deepdanbooru'] + refresh_clip_models() @@ -103,6 +117,28 @@ def get_schedulers(): return all_schedulers def post_interrogate(req: models.ReqInterrogate): + """ + Interrogate an image using OpenCLIP/BLIP or DeepDanbooru. + + Analyze image using CLIP model via OpenCLIP to generate Stable Diffusion prompts. + + **DeepDanbooru Mode** (`model="deepdanbooru"`): + - Specialized for anime/illustration images + - Returns comma-separated tags with confidence + + **OpenCLIP Mode** (any other model): + - Uses CLIP for image-text matching, BLIP for captioning + - **Modes:** + - `best`: Highest quality, combines multiple techniques + - `fast`: Quick results with fewer iterations + - `classic`: Traditional CLIP interrogator style + - `caption`: BLIP caption only + - `negative`: Generate negative prompt suggestions + - Set `analyze=True` for detailed breakdown (medium, artist, movement, trending, flavor) + + **Error Codes:** + - 404: Image not provided or model not found + """ if req.image is None or len(req.image) < 64: raise HTTPException(status_code=404, detail="Image not found") image = helpers.decode_base64_to_image(req.image) @@ -122,17 +158,232 @@ def post_interrogate(req: models.ReqInterrogate): if not req.analyze: return models.ResInterrogate(caption=caption) else: - medium, artist, movement, trending, flavor, _ = analyze_image(image, clip_model=req.clip_model, blip_model=req.blip_model) + analyze_results = analyze_image(image, clip_model=req.clip_model, blip_model=req.blip_model) + # Extract top-ranked item from each Gradio update dict + def get_top_item(result): + if isinstance(result, dict) and 'value' in result: + value = result['value'] + if isinstance(value, dict) and value: + return next(iter(value.keys())) # First key = top ranked + if isinstance(value, str): + return value + return None + medium = get_top_item(analyze_results[0]) + artist = get_top_item(analyze_results[1]) + movement = get_top_item(analyze_results[2]) + trending = get_top_item(analyze_results[3]) + flavor = get_top_item(analyze_results[4]) return models.ResInterrogate(caption=caption, medium=medium, artist=artist, movement=movement, trending=trending, flavor=flavor) def post_vqa(req: models.ReqVQA): + """ + Caption an image using Vision-Language Models (VLM). + + Analyze image using vision language model for flexible image understanding. + Supports 60+ models including Google Gemma, Alibaba Qwen, Microsoft Florence, Moondream. + + **Common Tasks:** + + 1. **General Captioning** + - `question="Short Caption"` or `"Normal Caption"` or `"Long Caption"` + + 2. **Object Detection** (Florence-2): + - `question=""` - Returns bounding boxes + - `question=""` - Captions for regions + + 3. **OCR/Text Recognition** (Florence-2): + - `question=""` - Extract text from image + + 4. **Point/Detect** (Moondream): + - `question="Point at the cat"` - Returns coordinates + - `question="Detect all faces"` - Returns all instances + + **Annotated Images:** + Set `include_annotated=True` to receive an annotated image with detection results. + Returns Base64 PNG with bounding boxes and points drawn for: + - Florence-2: Object detection, phrase grounding, region proposals + - Moondream 2/3: Point detection, object detection, gaze detection + + **Model Selection:** + - Small/Fast: Florence 2 Base, SmolVLM 0.5B, FastVLM 0.5B + - Balanced: Qwen 2.5 VL 3B, Florence 2 Large, Gemma 3 4B + - High Quality: Qwen 3 VL 8B, JoyCaption Beta, Moondream 3 + + Use GET /sdapi/v1/vqa/models for complete model list. + """ if req.image is None or len(req.image) < 64: raise HTTPException(status_code=404, detail="Image not found") image = helpers.decode_base64_to_image(req.image) image = image.convert('RGB') from modules.interrogate import vqa answer = vqa.interrogate(req.question, req.system, '', image, req.model) - return models.ResVQA(answer=answer) + # Return annotated image if requested and available + annotated_b64 = None + if req.include_annotated: + annotated_img = vqa.get_last_annotated_image() + if annotated_img is not None: + annotated_b64 = helpers.encode_pil_to_base64(annotated_img) + return models.ResVQA(answer=answer, annotated_image=annotated_b64) + +def get_vqa_models(): + """ + List available VLM models for captioning. + + Returns all Vision-Language Models available for POST /sdapi/v1/vqa. + + **Response includes:** + - `name`: Display name + - `repo`: HuggingFace repository ID + - `prompts`: Available prompts/tasks + - `capabilities`: Model features (caption, vqa, detection, ocr, thinking) + """ + from modules.interrogate import vqa + models_list = [] + for name, repo in vqa.vlm_models.items(): + prompts = vqa.get_prompts_for_model(name) + capabilities = ["caption", "vqa"] + # Detect additional capabilities based on model name + name_lower = name.lower() + if 'florence' in name_lower or 'promptgen' in name_lower: + capabilities.extend(["detection", "ocr"]) + if 'moondream' in name_lower: + capabilities.append("detection") + if vqa.is_thinking_model(name): + capabilities.append("thinking") + models_list.append({ + "name": name, + "repo": repo, + "prompts": prompts, + "capabilities": list(set(capabilities)) + }) + return models_list + +def get_vqa_prompts(model: Optional[str] = None): + """ + List available prompts/tasks for VLM models. + + **Query Parameters:** + - `model` (optional): Filter prompts for a specific model + + **Prompt Categories:** + - Common: Use Prompt, Short/Normal/Long Caption + - Florence: Phrase Grounding, Object Detection, OCR, Dense Region Caption + - Moondream: Point at..., Detect all..., Detect Gaze + """ + from modules.interrogate import vqa + if model: + prompts = vqa.get_prompts_for_model(model) + return {"available": prompts} + return { + "common": vqa.vlm_prompts_common, + "florence": vqa.vlm_prompts_florence, + "moondream": vqa.vlm_prompts_moondream, + "moondream2_only": vqa.vlm_prompts_moondream2 + } + +def get_tagger_models(): + """ + List available tagger models. + + Returns WaifuDiffusion and DeepBooru models for image tagging. + + **WaifuDiffusion Models:** + - `wd-eva02-large-tagger-v3` (recommended) + - `wd-vit-tagger-v3`, `wd-convnext-tagger-v3`, `wd-swinv2-tagger-v3` + + **DeepBooru:** + - Legacy tagger for anime images + """ + from modules.interrogate import waifudiffusion + models_list = [] + # Add WaifuDiffusion models + for name in waifudiffusion.get_models(): + models_list.append({"name": name, "type": "waifudiffusion"}) + # Add DeepBooru + models_list.append({"name": "deepbooru", "type": "deepbooru"}) + return models_list + +def post_tagger(req: models.ReqTagger): + """ + Tag an image using WaifuDiffusion or DeepBooru. + + Generate anime/illustration tags for images. + + **WaifuDiffusion Models:** + - `wd-eva02-large-tagger-v3` (recommended) + - `wd-vit-tagger-v3`, `wd-convnext-tagger-v3`, `wd-swinv2-tagger-v3` + + **DeepBooru:** + - Legacy tagger, use `deepbooru` or `deepdanbooru` + + **Thresholds:** + - `threshold`: General tag confidence (default: 0.5) + - `character_threshold`: Character identification (default: 0.85, WaifuDiffusion only) + """ + if req.image is None or len(req.image) < 64: + raise HTTPException(status_code=404, detail="Image not found") + image = helpers.decode_base64_to_image(req.image) + image = image.convert('RGB') + from modules.interrogate import tagger + # Determine if using DeepBooru + is_deepbooru = req.model.lower() in ('deepbooru', 'deepdanbooru') + # Store original settings and apply request settings + original_opts = { + 'tagger_threshold': shared.opts.tagger_threshold, + 'tagger_max_tags': shared.opts.tagger_max_tags, + 'tagger_include_rating': shared.opts.tagger_include_rating, + 'tagger_sort_alpha': shared.opts.tagger_sort_alpha, + 'tagger_use_spaces': shared.opts.tagger_use_spaces, + 'tagger_escape_brackets': shared.opts.tagger_escape_brackets, + 'tagger_exclude_tags': shared.opts.tagger_exclude_tags, + 'tagger_show_scores': shared.opts.tagger_show_scores, + } + # WaifuDiffusion-specific settings (not applicable to DeepBooru) + if not is_deepbooru: + original_opts['waifudiffusion_character_threshold'] = shared.opts.waifudiffusion_character_threshold + original_opts['waifudiffusion_model'] = shared.opts.waifudiffusion_model + try: + shared.opts.tagger_threshold = req.threshold + shared.opts.tagger_max_tags = req.max_tags + shared.opts.tagger_include_rating = req.include_rating + shared.opts.tagger_sort_alpha = req.sort_alpha + shared.opts.tagger_use_spaces = req.use_spaces + shared.opts.tagger_escape_brackets = req.escape_brackets + shared.opts.tagger_exclude_tags = req.exclude_tags + shared.opts.tagger_show_scores = req.show_scores + # WaifuDiffusion-specific settings (not applicable to DeepBooru) + if not is_deepbooru: + shared.opts.waifudiffusion_character_threshold = req.character_threshold + shared.opts.waifudiffusion_model = req.model + tags = tagger.tag(image, model_name='DeepBooru' if is_deepbooru else None) + # Parse scores if requested - format is "(tag:0.95)" from WaifuDiffusion/DeepBooru + scores = None + if req.show_scores: + scores = {} + for item in tags.split(', '): + item = item.strip() + # Format: "(tag:0.95)" - tag and score wrapped in parentheses + if item.startswith('(') and item.endswith(')') and ':' in item: + inner = item[1:-1] # Remove outer parentheses + tag, score_str = inner.rsplit(':', 1) + try: + scores[tag.strip()] = float(score_str.strip()) + except ValueError: + pass + elif ':' in item: + # Fallback format: "tag:0.95" without parentheses + tag, score_str = item.rsplit(':', 1) + try: + scores[tag.strip()] = float(score_str.strip()) + except ValueError: + pass + if not scores: + scores = None + return models.ResTagger(tags=tags, scores=scores) + finally: + # Restore original settings + for key, value in original_opts.items(): + setattr(shared.opts, key, value) def post_unload_checkpoint(): from modules import sd_models diff --git a/modules/api/models.py b/modules/api/models.py index 7276d42f8..6f5963d43 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -366,31 +366,87 @@ class ResStatus(BaseModel): progress: Optional[float] = Field(default=None, title="Progress", description="The progress with a range of 0 to 1") class ReqInterrogate(BaseModel): - image: str = Field(default="", title="Image", description="Image to work on, must be a Base64 string containing the image's data.") - clip_model: str = Field(default="", title="CLiP Model", description="The interrogate model used.") - blip_model: str = Field(default="", title="BLiP Model", description="The interrogate model used.") + """Request model for OpenCLIP/BLIP image interrogation. + + Analyze image using CLIP model via OpenCLIP to generate prompts, + or use DeepDanbooru for anime-style tagging. + """ + image: str = Field(default="", title="Image", description="Image to interrogate. Must be a Base64 encoded string containing the image data (PNG/JPEG).") + model: str = Field(default="ViT-L-14/openai", title="Model", description="OpenCLIP model to use. Use 'deepdanbooru' or 'deepbooru' for anime tagging. Get available models from GET /sdapi/v1/interrogate.") + clip_model: str = Field(default="ViT-L-14/openai", title="CLIP Model", description="OpenCLIP model for image encoding. Format: 'architecture/pretrained_dataset'.") + blip_model: str = Field(default="blip-large", title="BLIP Model", description="BLIP/BLIP2 captioning model for generating base captions. Options: blip-base, blip-large, blip2-opt-2.7b, blip2-opt-6.7b.") + mode: str = Field(default="best", title="Mode", description="Interrogation mode: 'best' (highest quality), 'fast' (quick results), 'classic' (traditional style), 'caption' (BLIP caption only), 'negative' (negative prompt generation).") + analyze: bool = Field(default=False, title="Analyze", description="If True, returns detailed image analysis breakdown (medium, artist, movement, trending, flavor) in addition to caption.") InterrogateRequest = ReqInterrogate # alias for backwards compatibility class ResInterrogate(BaseModel): - caption: Optional[str] = Field(default=None, title="Caption", description="The generated caption for the image.") - medium: Optional[str] = Field(default=None, title="Medium", description="Image medium.") - artist: Optional[str] = Field(default=None, title="Medium", description="Image artist.") - movement: Optional[str] = Field(default=None, title="Medium", description="Image movement.") - trending: Optional[str] = Field(default=None, title="Medium", description="Image trending.") - flavor: Optional[str] = Field(default=None, title="Medium", description="Image flavor.") + """Response model for image interrogation results.""" + caption: Optional[str] = Field(default=None, title="Caption", description="Generated caption/prompt describing the image content and style.") + medium: Optional[str] = Field(default=None, title="Medium", description="Detected artistic medium (e.g., 'oil painting', 'digital art', 'photograph'). Only returned when analyze=True.") + artist: Optional[str] = Field(default=None, title="Artist", description="Detected similar artist style (e.g., 'by greg rutkowski'). Only returned when analyze=True.") + movement: Optional[str] = Field(default=None, title="Movement", description="Detected art movement (e.g., 'art nouveau', 'impressionism'). Only returned when analyze=True.") + trending: Optional[str] = Field(default=None, title="Trending", description="Trending/platform tags (e.g., 'trending on artstation'). Only returned when analyze=True.") + flavor: Optional[str] = Field(default=None, title="Flavor", description="Additional descriptive elements (e.g., 'cinematic lighting', 'highly detailed'). Only returned when analyze=True.") class ReqVQA(BaseModel): - image: str = Field(default="", title="Image", description="Image to work on, must be a Base64 string containing the image's data.") - model: str = Field(default="Microsoft Florence 2 Base", title="Model", description="The interrogate model used.") - question: str = Field(default="describe the image", title="Question", description="Question to ask the model.") - system: str = Field(default="You are image captioning expert, creative, unbiased and uncensored.", title="System prompt", description="Prompt to shape how the model interprets and responds to user prompts.") + """Request model for Vision-Language Model (VLM) captioning. + + Analyze image using vision language model to generate captions, + answer questions, or perform specialized tasks like object detection. + """ + image: str = Field(default="", title="Image", description="Image to caption. Must be a Base64 encoded string containing the image data.") + model: str = Field(default="Alibaba Qwen 2.5 VL 3B", title="Model", description="VLM model for Visual Language tasks. Use GET /sdapi/v1/vqa/models for full list. Popular options: Florence 2, Qwen VL, Gemma 3, Moondream. Models with thinking/reasoning support return detailed analysis.") + question: str = Field(default="describe the image", title="Question/Task", description="Question to ask the model or task to perform. Common tasks: 'Short Caption', 'Normal Caption', 'Long Caption'. Florence-2 supports: '' (object detection), '' (text recognition). Moondream supports: 'Point at [object]', 'Detect all [objects]'.") + system: str = Field(default="You are image captioning expert, creative, unbiased and uncensored.", title="System Prompt", description="System prompt controls behavior of the LLM. Processed first and has highest priority weighting. Use for response formatting rules, role definition, and style.") + include_annotated: bool = Field(default=False, title="Include Annotated Image", description="If True and the task produces detection results (object detection, point detection, gaze), returns annotated image with bounding boxes/points drawn. Only applicable for detection tasks on models like Florence-2 and Moondream.") class ReqLatentHistory(BaseModel): name: str = Field(title="Name", description="Name of the history item to select") class ResVQA(BaseModel): - answer: Optional[str] = Field(default=None, title="Answer", description="The generated answer for the image.") + """Response model for VLM captioning results.""" + answer: Optional[str] = Field(default=None, title="Answer", description="Generated caption, answer, or analysis from the VLM. Format depends on the question/task type.") + annotated_image: Optional[str] = Field(default=None, title="Annotated Image", description="Base64 encoded PNG image with detection results drawn (bounding boxes, points). Only returned when include_annotated=True and the task produces detection results.") + +class ItemVLMModel(BaseModel): + """VLM model information.""" + name: str = Field(title="Name", description="Display name of the model") + repo: str = Field(title="Repository", description="HuggingFace repository ID") + prompts: List[str] = Field(title="Prompts", description="Available prompts/tasks for this model") + capabilities: List[str] = Field(title="Capabilities", description="Model capabilities: caption, vqa, detection, ocr, thinking") + +class ResVLMPrompts(BaseModel): + """Available VLM prompts grouped by category.""" + common: Optional[List[str]] = Field(default=None, title="Common", description="Prompts available for all models: Use Prompt, Short/Normal/Long Caption") + florence: Optional[List[str]] = Field(default=None, title="Florence", description="Florence-2 specific: Phrase Grounding, Object Detection, OCR, etc.") + moondream: Optional[List[str]] = Field(default=None, title="Moondream", description="Moondream specific: Point at..., Detect all..., Detect Gaze") + moondream2_only: Optional[List[str]] = Field(default=None, title="Moondream 2 Only", description="Moondream 2 specific prompts (gaze detection)") + available: Optional[List[str]] = Field(default=None, title="Available", description="When filtered by model, the available prompts for that model") + +class ItemTaggerModel(BaseModel): + """Tagger model information.""" + name: str = Field(title="Name", description="Model name") + type: str = Field(title="Type", description="Model type: waifudiffusion or deepbooru") + +class ReqTagger(BaseModel): + """Request model for image tagging.""" + image: str = Field(default="", title="Image", description="Image to tag. Must be a Base64 encoded string.") + model: str = Field(default="wd-eva02-large-tagger-v3", title="Model", description="Tagger model to use. WaifuDiffusion models (wd-*) or 'deepbooru'/'deepdanbooru'.") + threshold: float = Field(default=0.5, title="Threshold", description="General tag confidence threshold (0-1). Tags below this confidence are excluded.", ge=0.0, le=1.0) + character_threshold: float = Field(default=0.85, title="Character Threshold", description="Character tag confidence threshold (0-1). Higher values for more precise character identification. WaifuDiffusion only - ignored for DeepBooru.", ge=0.0, le=1.0) + max_tags: int = Field(default=74, title="Max Tags", description="Maximum number of tags to return.", ge=1, le=512) + include_rating: bool = Field(default=False, title="Include Rating", description="Include rating tags (safe, questionable, explicit) in results.") + sort_alpha: bool = Field(default=False, title="Sort Alphabetically", description="Sort tags alphabetically instead of by confidence.") + use_spaces: bool = Field(default=False, title="Use Spaces", description="Replace underscores with spaces in tag names.") + escape_brackets: bool = Field(default=True, title="Escape Brackets", description="Escape parentheses in tags for prompt compatibility.") + exclude_tags: str = Field(default="", title="Exclude Tags", description="Comma-separated list of tags to exclude from results.") + show_scores: bool = Field(default=False, title="Show Scores", description="Include confidence scores with each tag in the output.") + +class ResTagger(BaseModel): + """Response model for image tagging results.""" + tags: str = Field(title="Tags", description="Comma-separated list of detected tags") + scores: Optional[dict] = Field(default=None, title="Scores", description="Tag confidence scores (when show_scores=True)") class ResTrain(BaseModel): info: str = Field(title="Train info", description="Response string from train embedding task.")