diff --git a/cli/test-caption-api.py b/cli/test-caption-api.py index d692ede86..f5ce205ee 100755 --- a/cli/test-caption-api.py +++ b/cli/test-caption-api.py @@ -22,6 +22,7 @@ Examples: """ import os +import re import sys import time import base64 @@ -38,6 +39,11 @@ DEFAULT_TEST_IMAGES = [ 'extensions-builtin/sdnext-modernui/html/logo.png', ] +# OCR test image (must have readable text) +OCR_TEST_IMAGE = 'models/Reference/HiDream-ai--HiDream-I1-Fast.jpg' +# Bracket test image (must produce tags with parentheses, e.g. pokemon_(creature)) +BRACKET_TEST_IMAGE = 'models/Reference/SDXL-Flash_Mini.jpg' + class CaptionAPITest: """Test harness for Caption API endpoints.""" @@ -75,6 +81,8 @@ class CaptionAPITest: self.base_url = base_url.rstrip('/') self.image_path = image_path self.image_b64 = None + self.ocr_image_b64 = None # Separate image with text for OCR tests + self.bracket_image_b64 = None # Separate image that produces bracket-containing tags self.timeout = timeout # Request timeout in seconds # Categorized results tracking self.results = { @@ -149,27 +157,34 @@ class CaptionAPITest: return False text_lower = str(response_text).lower() # Critical error patterns that indicate backend is broken + # Note: patterns are substring matches, so be careful with short strings that could match common words critical_patterns = [ 'runtimeerror', 'cuda error', 'out of memory', - 'oom', + # 'oom' removed - matches words like "room", "zoom", "bloom"; 'out of memory' covers this case 'device-side assert', 'cublas', 'cudnn', 'nccl', 'input type', # tensor type mismatch 'weight type', # tensor type mismatch - 'expected .* but got', # type/device mismatch 'cannot be performed', 'illegal memory access', 'segmentation fault', - 'killed', - 'critical', + ] + # Patterns that need word boundary checking (could match common words) + word_boundary_patterns = [ + 'killed', # could match "skilled", "thrilled" + 'critical', # could match "critical thinking" ] for pattern in critical_patterns: if pattern in text_lower: return True + # Check word boundary patterns with regex + for pattern in word_boundary_patterns: + if re.search(rf'\b{pattern}\b', text_lower): + return True return False def check_critical_error(self, data, backend): @@ -304,8 +319,8 @@ class CaptionAPITest: return json_data.get('backend', 'openclip') if json_data else 'openclip' return None - def post(self, endpoint, json_data): - """Make POST request and return JSON response. Auto-checks for critical errors.""" + def post(self, endpoint, json_data, check_critical=True): + """Make POST request and return JSON response. Auto-checks for critical errors unless check_critical=False.""" url = f"{self.base_url}{endpoint}" backend = self._infer_backend_from_endpoint(endpoint, json_data) @@ -314,11 +329,12 @@ class CaptionAPITest: resp.raise_for_status() data = resp.json() - # Auto-check for critical errors in the response - if backend and backend != 'vlm': # VLM backend name differs - self._auto_check_critical(data, backend) - elif backend == 'vlm': - self._auto_check_critical(data, 'vqa') + # Auto-check for critical errors in the response (skip for deliberate error tests) + if check_critical: + if backend and backend != 'vlm': # VLM backend name differs + self._auto_check_critical(data, backend) + elif backend == 'vlm': + self._auto_check_critical(data, 'vqa') return data except requests.exceptions.Timeout: @@ -399,6 +415,33 @@ class CaptionAPITest: print(f" ERROR: Failed to load image: {e}") return False + # Load OCR test image (image with readable text) + script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ocr_image_path = os.path.join(script_dir, OCR_TEST_IMAGE) + if os.path.exists(ocr_image_path): + try: + with open(ocr_image_path, 'rb') as f: + ocr_data = f.read() + self.ocr_image_b64 = base64.b64encode(ocr_data).decode('utf-8') + print(f" OCR test image loaded: {OCR_TEST_IMAGE} ({len(ocr_data)} bytes)") + except Exception as e: + print(f" Warning: Failed to load OCR test image: {e}") + else: + print(f" Warning: OCR test image not found: {OCR_TEST_IMAGE}") + + # Load bracket test image (image that produces tags with parentheses) + bracket_image_path = os.path.join(script_dir, BRACKET_TEST_IMAGE) + if os.path.exists(bracket_image_path): + try: + with open(bracket_image_path, 'rb') as f: + bracket_data = f.read() + self.bracket_image_b64 = base64.b64encode(bracket_data).decode('utf-8') + print(f" Bracket test image loaded: {BRACKET_TEST_IMAGE} ({len(bracket_data)} bytes)") + except Exception as e: + print(f" Warning: Failed to load bracket test image: {e}") + else: + print(f" Warning: Bracket test image not found: {BRACKET_TEST_IMAGE}") + return True def print_summary(self): @@ -438,6 +481,12 @@ class CaptionAPITest: for _, msg in failures: print(f" [FAIL] {msg}") + # Show skipped tests for this category + skipped = [(s, m) for s, m in data['tests'] if s == 'skipped'] + if skipped: + for _, msg in skipped: + print(f" [SKIP] {msg}") + # Overall totals print("\n" + "-" * 70) overall_run = total_passed + total_failed @@ -622,21 +671,21 @@ class CaptionAPITest: print("TEST: POST /sdapi/v1/openclip (invalid inputs)") print("=" * 70) - # Test missing image + # Test missing image (check_critical=False since we expect errors) data = self.post('/sdapi/v1/openclip', { 'image': '', 'model': 'ViT-L-14/openai' - }) + }, check_critical=False) if 'error' in data and data.get('status') == 404: self.log_pass("Missing image returns 404") else: self.log_fail(f"Missing image should return 404, got: {data}") - # Test invalid model + # Test invalid model (check_critical=False since we expect errors) data = self.post('/sdapi/v1/openclip', { 'image': self.image_b64, 'model': 'invalid-nonexistent-model' - }) + }, check_critical=False) if 'error' in data: self.log_pass(f"Invalid model returns error: {data.get('status', 'error')}") else: @@ -717,10 +766,10 @@ class CaptionAPITest: # TEST: POST /sdapi/v1/openclip - Caption Length # ========================================================================= def test_openclip_length(self): - """Test min_length and max_length constraints.""" + """Test max_length constraints.""" self.set_category('openclip') print("\n" + "=" * 70) - print("TEST: POST /sdapi/v1/openclip (min_length, max_length)") + print("TEST: POST /sdapi/v1/openclip (max_length)") print("=" * 70) # Skip if critical error already occurred @@ -777,36 +826,6 @@ class CaptionAPITest: else: self.log_fail("max_length test returned empty captions") - # Test min_length effect by comparing low vs high minimums - data_min_low = self.post('/sdapi/v1/openclip', { - 'image': self.image_b64, - 'model': model, - 'mode': 'caption', - 'min_length': 5 # Low minimum - }) - data_min_high = self.post('/sdapi/v1/openclip', { - 'image': self.image_b64, - 'model': model, - 'mode': 'caption', - 'min_length': 50 # Higher minimum - }) - - if 'error' in data_min_low or 'error' in data_min_high: - self.log_skip("min_length test: API error") - elif data_min_low.get('caption') and data_min_high.get('caption'): - len_low = len(data_min_low['caption']) - len_high = len(data_min_high['caption']) - self.log_info(f"min_length=5: {len_low} chars - '{data_min_low['caption'][:50]}...'") - self.log_info(f"min_length=50: {len_high} chars - '{data_min_high['caption'][:50]}...'") - if len_high > len_low: - self.log_pass(f"min_length has effect: {len_low} < {len_high} chars") - elif len_high == len_low: - self.log_fail(f"min_length has no effect (both {len_low} chars)") - else: - self.log_fail(f"min_length reversed: low={len_low}, high={len_high}") - else: - self.log_fail("min_length test returned empty captions") - # ========================================================================= # TEST: POST /sdapi/v1/openclip - Flavors # ========================================================================= @@ -858,18 +877,21 @@ class CaptionAPITest: else: self.log_fail("max_flavors test returned empty captions") - # Test min_flavors effect by comparing low vs high minimums + # Test min_flavors effect: only applies in mode='best' which iterates from min to max flavors + # Use a narrow max_flavors window so min_flavors has a visible floor effect data_min_low = self.post('/sdapi/v1/openclip', { 'image': self.image_b64, 'model': model, - 'mode': 'fast', - 'min_flavors': 1 + 'mode': 'best', + 'min_flavors': 1, + 'max_flavors': 3 }) data_min_high = self.post('/sdapi/v1/openclip', { 'image': self.image_b64, 'model': model, - 'mode': 'fast', - 'min_flavors': 10 + 'mode': 'best', + 'min_flavors': 8, + 'max_flavors': 10 }) if 'error' in data_min_low or 'error' in data_min_high: @@ -877,10 +899,12 @@ class CaptionAPITest: elif data_min_low.get('caption') and data_min_high.get('caption'): len_low = len(data_min_low['caption']) len_high = len(data_min_high['caption']) - self.log_info(f"min_flavors=1: {len_low} chars") - self.log_info(f"min_flavors=10: {len_high} chars") - if len_high >= len_low: - self.log_pass(f"min_flavors has effect: {len_low} <= {len_high} chars") + self.log_info(f"min_flavors=1,max=3: {len_low} chars - '{data_min_low['caption'][:50]}...'") + self.log_info(f"min_flavors=8,max=10: {len_high} chars - '{data_min_high['caption'][:50]}...'") + if len_high > len_low: + self.log_pass(f"min_flavors has effect: {len_low} < {len_high} chars") + elif len_high == len_low: + self.log_fail(f"min_flavors has no effect (both {len_low} chars)") else: self.log_fail(f"min_flavors reversed: low={len_low}, high={len_high}") else: @@ -921,6 +945,7 @@ class CaptionAPITest: self.log_skip(f"chunk_size override: {data.get('reason', 'failed')}") elif data.get('caption') and not self.is_error_answer(data['caption']): self.log_pass(f"chunk_size=1024 accepted ({elapsed:.1f}s)") + self.log_info("NOTE: acceptance-only test, does not verify output effect") else: self.log_fail("chunk_size override returned empty/error") @@ -938,6 +963,7 @@ class CaptionAPITest: self.log_skip(f"flavor_count override: {data.get('reason', 'failed')}") elif data.get('caption') and not self.is_error_answer(data['caption']): self.log_pass(f"flavor_count=16 accepted ({elapsed:.1f}s)") + self.log_info("NOTE: acceptance-only test, does not verify output effect") else: self.log_fail("flavor_count override returned empty/error") @@ -955,6 +981,7 @@ class CaptionAPITest: self.log_skip(f"num_beams override: {data.get('reason', 'failed')}") elif data.get('caption') and not self.is_error_answer(data['caption']): self.log_pass(f"num_beams=3 accepted ({elapsed:.1f}s)") + self.log_info("NOTE: acceptance-only test, does not verify output effect") else: self.log_fail("num_beams override returned empty/error") @@ -1022,7 +1049,7 @@ class CaptionAPITest: return # Verify categories - expected_categories = ['common', 'florence', 'moondream'] + expected_categories = ['common', 'florence', 'promptgen', 'moondream'] for cat in expected_categories: if cat in data and isinstance(data[cat], list): self.log_pass(f"Has '{cat}' category with {len(data[cat])} prompts") @@ -1086,6 +1113,7 @@ class CaptionAPITest: return prompts = ['Short Caption', 'Normal Caption', 'Long Caption'] + results = {} for prompt in prompts: t0 = time.time() data = self.post('/sdapi/v1/vqa', { @@ -1097,11 +1125,18 @@ class CaptionAPITest: if 'error' in data: self.log_skip(f"prompt='{prompt}': {data.get('reason', 'failed')}") elif data.get('answer') and not self.is_error_answer(data['answer']): + results[prompt] = len(data['answer']) self.log_pass(f"prompt='{prompt}' returns answer ({len(data['answer'])} chars, {elapsed:.1f}s)") elif self.is_error_answer(data.get('answer', '')): self.log_fail(f"prompt='{prompt}' returned error: {data['answer']}") else: self.log_fail(f"prompt='{prompt}' returned empty answer") + # Length sanity check: Short should be noticeably shorter than Normal/Long + if 'Short Caption' in results and 'Normal Caption' in results and 'Long Caption' in results: + if results['Short Caption'] >= results['Normal Caption'] or results['Short Caption'] >= results['Long Caption']: + self.log_info(f"NOTE: Short ({results['Short Caption']}) >= Normal ({results['Normal Caption']}) or Long ({results['Long Caption']}); LLM output length is non-deterministic and prompt-dependent") + if results['Long Caption'] < results['Normal Caption']: + self.log_info(f"NOTE: Long ({results['Long Caption']}) < Normal ({results['Normal Caption']}); LLM may interpret length prompts differently per run") # ========================================================================= # TEST: POST /sdapi/v1/vqa - Annotated Image @@ -1231,11 +1266,11 @@ class CaptionAPITest: print("TEST: POST /sdapi/v1/vqa (invalid inputs)") print("=" * 70) - # Test missing image + # Test missing image (check_critical=False since we expect errors) data = self.post('/sdapi/v1/vqa', { 'image': '', 'question': 'describe' - }) + }, check_critical=False) if 'error' in data and data.get('status') == 404: self.log_pass("Missing image returns 404") else: @@ -1421,9 +1456,11 @@ class CaptionAPITest: }) elapsed = time.time() - t0 + greedy_elapsed = None if 'error' in data_greedy: self.log_skip(f"do_sample=False test: {data_greedy.get('reason', 'failed')}") elif data_greedy.get('answer') and not self.is_error_answer(data_greedy['answer']): + greedy_elapsed = elapsed self.log_pass(f"do_sample=False (greedy) accepted ({elapsed:.1f}s)") else: self.log_fail("do_sample=False returned empty/error") @@ -1445,7 +1482,7 @@ class CaptionAPITest: else: self.log_fail("do_sample=True returned empty/error") - # Test with num_beams (beam search) + # Test with num_beams (beam search - should be slower than greedy) t0 = time.time() data_beams = self.post('/sdapi/v1/vqa', { 'image': self.image_b64, @@ -1458,6 +1495,8 @@ class CaptionAPITest: self.log_skip(f"num_beams=4 test: {data_beams.get('reason', 'failed')}") elif data_beams.get('answer') and not self.is_error_answer(data_beams['answer']): self.log_pass(f"num_beams=4 (beam search) accepted ({elapsed:.1f}s)") + if greedy_elapsed is not None and elapsed <= greedy_elapsed: + self.log_info(f"NOTE: num_beams=4 ({elapsed:.1f}s) not slower than greedy ({greedy_elapsed:.1f}s); beam search overhead may be negligible for short outputs or fast GPUs") else: self.log_fail("num_beams=4 returned empty/error") @@ -1533,8 +1572,11 @@ class CaptionAPITest: self.log_skip(f"keep_thinking=True test: {data_keep.get('reason', 'failed')}") elif data_keep.get('answer') and not self.is_error_answer(data_keep['answer']): answer = data_keep['answer'] - has_thinking = '' in answer.lower() + # Thinking trace is reformatted: →"Reasoning:" and →"Answer:" by strip_think_xml_tags() + has_thinking = 'reasoning:' in answer.lower() or ' tags for this input") # Show first part of answer (may include thinking trace) answer_preview = answer[:150] + '...' if len(answer) > 150 else answer self.log_info(f"Answer: {answer_preview}") @@ -1554,7 +1596,7 @@ class CaptionAPITest: if self.skip_if_critical('vqa', 'vqa prefill'): return - prefill_text = "The image shows" + prefill_text = "Vlado is the best, and I'm looking at his robot which" # Test with prefill to guide response start t0 = time.time() @@ -1677,35 +1719,49 @@ class CaptionAPITest: if self.skip_if_critical('vqa', 'vqa florence prompts'): return - # Find a Florence model + # Find Florence models: base and PromptGen (which supports extra prompts) florence_model = None + promptgen_model = None if self._vqa_models: for m in self._vqa_models: - if 'florence' in m['name'].lower(): + name_lower = m['name'].lower() + if 'promptgen' in name_lower and promptgen_model is None: + promptgen_model = m['name'] + elif 'florence' in name_lower and 'promptgen' not in name_lower and 'cog' not in name_lower and florence_model is None: florence_model = m['name'] - break if not florence_model: self.log_skip("No Florence model available") return self.log_info(f"Using Florence model: {florence_model}") + if promptgen_model: + self.log_info(f"Using PromptGen model: {promptgen_model}") - # Florence-specific prompts - florence_prompts = { + # Base Florence prompts (supported by all Florence models) + base_prompts = { '': 'Object Detection', '': 'Optical Character Recognition', '': 'Dense Region Captioning', - '': 'Tag Generation', '': 'Standard Caption', '': 'Detailed Caption', } + # PromptGen-only prompts (require MiaoshouAI PromptGen fine-tune) + promptgen_prompts = { + '': 'Tag Generation', + } + + def run_florence_prompt(model, prompt, description): + # Use OCR test image for OCR prompts (image with readable text) + if prompt == '' and self.ocr_image_b64: + test_image = self.ocr_image_b64 + else: + test_image = self.image_b64 - for prompt, description in florence_prompts.items(): t0 = time.time() data = self.post('/sdapi/v1/vqa', { - 'image': self.image_b64, - 'model': florence_model, + 'image': test_image, + 'model': model, 'question': prompt }) elapsed = time.time() - t0 @@ -1722,6 +1778,15 @@ class CaptionAPITest: else: self.log_fail(f"{description} ({prompt}): empty/error response") + for prompt, description in base_prompts.items(): + run_florence_prompt(florence_model, prompt, description) + + for prompt, description in promptgen_prompts.items(): + if promptgen_model: + run_florence_prompt(promptgen_model, prompt, f"{description} [PromptGen]") + else: + self.log_skip(f"{description} ({prompt}): requires PromptGen model, none available") + # ========================================================================= # TEST: VQA Moondream Detection Features # ========================================================================= @@ -1840,8 +1905,14 @@ class CaptionAPITest: model_name = capability_models[capability][0] self.log_info(f"Testing '{capability}' with: {model_name}") + # Use OCR test image for OCR capability (image with readable text) + if capability == 'ocr' and self.ocr_image_b64: + test_image = self.ocr_image_b64 + else: + test_image = self.image_b64 + request_data = { - 'image': self.image_b64, + 'image': test_image, 'model': model_name, 'question': test_prompt } @@ -1860,6 +1931,8 @@ class CaptionAPITest: answer = data['answer'] answer_preview = answer[:80] + '...' if len(answer) > 80 else answer self.log_pass(f"Capability '{capability}' ({elapsed:.1f}s): {answer_preview}") + if elapsed > 60: + self.log_info(f"NOTE: {model_name} took {elapsed:.1f}s which is suspiciously slow; may need performance investigation") elif data.get('answer'): self.log_fail(f"Capability '{capability}': non-meaningful response: '{data['answer']}'") else: @@ -2110,12 +2183,15 @@ class CaptionAPITest: self.log_skip("sort_alpha test: model not loaded") return + list_conf = [t.strip() for t in data_conf.get('tags', '').split(',') if t.strip()] list_alpha = [t.strip() for t in data_alpha.get('tags', '').split(',') if t.strip()] if len(list_alpha) < 2: self.log_skip("Not enough tags to test sorting") return + self.log_info(f"By confidence: {', '.join(list_conf[:8])}...") + self.log_info(f"Alphabetical: {', '.join(list_alpha[:8])}...") is_sorted = list_alpha == sorted(list_alpha, key=str.lower) if is_sorted: self.log_pass("sort_alpha=True returns alphabetically sorted tags") @@ -2182,14 +2258,19 @@ class CaptionAPITest: if self.skip_if_critical('tagger', 'tagger escape_brackets'): return + # Use bracket test image (produces tags with parentheses like "pokemon_(creature)") + test_image = self.bracket_image_b64 or self.image_b64 + if not self.bracket_image_b64: + self.log_info("NOTE: bracket test image not available, using default image (may not produce bracket tags)") + data_escaped = self.post('/sdapi/v1/tagger', { - 'image': self.image_b64, + 'image': test_image, 'escape_brackets': True, 'max_tags': 50, 'threshold': 0.1 }) data_raw = self.post('/sdapi/v1/tagger', { - 'image': self.image_b64, + 'image': test_image, 'escape_brackets': False, 'max_tags': 50, 'threshold': 0.1 @@ -2202,8 +2283,8 @@ class CaptionAPITest: tags_escaped = data_escaped.get('tags', '') tags_raw = data_raw.get('tags', '') - self.log_info(f"escape=True: {tags_escaped[:60]}...") - self.log_info(f"escape=False: {tags_raw[:60]}...") + self.log_info(f"escape=True: {tags_escaped[:70]}...") + self.log_info(f"escape=False: {tags_raw[:70]}...") # Check for escaped brackets (\\( or \\)) has_escaped = '\\(' in tags_escaped or '\\)' in tags_escaped @@ -2451,10 +2532,12 @@ class CaptionAPITest: count_high = len(data_high.get('tags', '').split(', ')) self.log_info(f"Tag counts: threshold=0.5→{count_low}, threshold=0.99→{count_high}") - if count_low >= count_high: - self.log_pass("character_threshold affects tag filtering") + if count_low > count_high: + self.log_pass(f"character_threshold affects tag filtering: {count_low} > {count_high}") + elif count_low == count_high: + self.log_info("NOTE: acceptance-only test, tag counts identical; test image likely has no character tags (character_threshold only filters anime character names)") else: - self.log_info("Tag counts similar (image may not have character tags)") + self.log_fail(f"character_threshold reversed: low={count_low} < high={count_high}") else: self.log_fail("character_threshold=0.99 returned empty/error") @@ -2611,10 +2694,10 @@ class CaptionAPITest: print("TEST: POST /sdapi/v1/tagger (invalid inputs)") print("=" * 70) - # Test missing image + # Test missing image (check_critical=False since we expect errors) data = self.post('/sdapi/v1/tagger', { 'image': '' - }) + }, check_critical=False) if 'error' in data and data.get('status') == 404: self.log_pass("Missing image returns 404") else: @@ -2977,10 +3060,11 @@ class CaptionAPITest: print("TEST: POST /sdapi/v1/caption (dispatch: invalid backend)") print("=" * 70) + # check_critical=False since we expect errors data = self.post('/sdapi/v1/caption', { 'backend': 'invalid_backend', 'image': self.image_b64 - }) + }, check_critical=False) if 'error' in data: self.log_pass(f"Invalid backend returns error: {data.get('status', 'error')}") @@ -2998,7 +3082,8 @@ class CaptionAPITest: req = {'backend': backend, 'image': ''} if backend == 'vlm': req['question'] = 'describe' - data = self.post('/sdapi/v1/caption', req) + # check_critical=False since we expect errors + data = self.post('/sdapi/v1/caption', req, check_critical=False) if 'error' in data and data.get('status') == 404: self.log_pass(f"dispatch {backend} missing image returns 404") diff --git a/modules/api/endpoints.py b/modules/api/endpoints.py index 6ad917da6..b37422479 100644 --- a/modules/api/endpoints.py +++ b/modules/api/endpoints.py @@ -145,8 +145,6 @@ def post_caption(req: models.ReqCaption): raise HTTPException(status_code=404, detail="Model not found") # Build clip overrides from request (only include non-None values) clip_overrides = {} - if req.min_length is not None: - clip_overrides['min_length'] = req.min_length if req.max_length is not None: clip_overrides['max_length'] = req.max_length if req.chunk_size is not None: @@ -314,8 +312,6 @@ def _dispatch_openclip(req: models.ReqCaptionOpenCLIP) -> models.ResCaptionDispa raise HTTPException(status_code=404, detail="Model not found") # Build clip overrides from request clip_overrides = {} - if req.min_length is not None: - clip_overrides['min_length'] = req.min_length if req.max_length is not None: clip_overrides['max_length'] = req.max_length if req.chunk_size is not None: @@ -499,7 +495,8 @@ def get_vqa_prompts(model: Optional[str] = None): **Prompt Categories:** - Common: Use Prompt, Short/Normal/Long Caption - - Florence: Phrase Grounding, Object Detection, OCR, Dense Region Caption + - Florence: Phrase Grounding, Object Detection, OCR, Dense Region Caption (all Florence models) + - PromptGen: Analyze, Generate Tags, Mixed Caption (MiaoshouAI PromptGen fine-tunes only) - Moondream: Point at..., Detect all..., Detect Gaze """ from modules.caption import vqa @@ -509,6 +506,7 @@ def get_vqa_prompts(model: Optional[str] = None): return { "common": vqa.vlm_prompts_common, "florence": vqa.vlm_prompts_florence, + "promptgen": vqa.vlm_prompts_promptgen, "moondream": vqa.vlm_prompts_moondream, "moondream2_only": vqa.vlm_prompts_moondream2 } diff --git a/modules/api/models.py b/modules/api/models.py index 329627bbc..fcef523fb 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -378,7 +378,6 @@ class ReqCaption(BaseModel): mode: str = Field(default="best", title="Mode", description="Caption mode. Fast: Quick caption with minimal flavor terms. Classic: Standard captioning with balanced quality and speed. Best: Most thorough analysis, slowest but highest quality. Negative: Generate terms to use as negative prompt.") analyze: bool = Field(default=False, title="Analyze", description="If True, returns detailed image analysis breakdown (medium, artist, movement, trending, flavor) in addition to caption.") # Advanced settings (optional per-request overrides) - min_length: Optional[int] = Field(default=None, title="Min Length", description="Minimum number of tokens in the generated caption.") max_length: Optional[int] = Field(default=None, title="Max Length", description="Maximum number of tokens in the generated caption.") chunk_size: Optional[int] = Field(default=None, title="Chunk Size", description="Batch size for processing description candidates (flavors). Higher values speed up captioning but increase VRAM usage.") min_flavors: Optional[int] = Field(default=None, title="Min Flavors", description="Minimum number of descriptive tags (flavors) to keep in the final prompt.") @@ -486,7 +485,6 @@ class ReqCaptionOpenCLIP(BaseModel): blip_model: str = Field(default="blip-large", title="Caption Model", description="BLIP model used to generate the initial image caption.") mode: str = Field(default="best", title="Mode", description="Caption mode: 'best' (highest quality), 'fast' (quick), 'classic' (traditional), 'caption' (BLIP only), 'negative' (for negative prompts).") analyze: bool = Field(default=False, title="Analyze", description="If True, returns detailed breakdown (medium, artist, movement, trending, flavor).") - min_length: Optional[int] = Field(default=None, title="Min Length", description="Minimum tokens in generated caption.") max_length: Optional[int] = Field(default=None, title="Max Length", description="Maximum tokens in generated caption.") chunk_size: Optional[int] = Field(default=None, title="Chunk Size", description="Batch size for processing flavors.") min_flavors: Optional[int] = Field(default=None, title="Min Flavors", description="Minimum descriptive tags to keep.") diff --git a/modules/caption/vqa.py b/modules/caption/vqa.py index 1bb5b49f1..80eb53757 100644 --- a/modules/caption/vqa.py +++ b/modules/caption/vqa.py @@ -80,7 +80,7 @@ vlm_prompts_common = [ "Long Caption", ] -# Florence-2 specific prompts (only shown for Florence/PromptGen models) +# Florence-2 base prompts (supported by all Florence models including CogFlorence) vlm_prompts_florence = [ "Phrase Grounding", "Object Detection", @@ -88,6 +88,10 @@ vlm_prompts_florence = [ "Region Proposal", "OCR (Read Text)", "OCR with Regions", +] + +# PromptGen-only prompts (require MiaoshouAI PromptGen fine-tune) +vlm_prompts_promptgen = [ "Analyze", "Generate Tags", "Mixed Caption", @@ -148,7 +152,7 @@ vlm_prompt_placeholders = { } # Legacy list for backwards compatibility -vlm_prompts = vlm_prompts_common + vlm_prompts_florence + vlm_prompts_moondream + vlm_prompts_moondream2 +vlm_prompts = vlm_prompts_common + vlm_prompts_florence + vlm_prompts_promptgen + vlm_prompts_moondream + vlm_prompts_moondream2 vlm_prefill = 'Answer: the image shows' @@ -160,8 +164,12 @@ def get_prompts_for_model(model_name: str) -> list: model_lower = model_name.lower() - # Check for Florence-2 / PromptGen models - if 'florence' in model_lower or 'promptgen' in model_lower: + # Check for PromptGen models (MiaoshouAI fine-tunes with extra prompts) + if 'promptgen' in model_lower: + return vlm_prompts_common + vlm_prompts_florence + vlm_prompts_promptgen + + # Check for Florence-2 base / CogFlorence models (no PromptGen-specific prompts) + if 'florence' in model_lower: return vlm_prompts_common + vlm_prompts_florence # Check for Moondream models (Moondream 2 has gaze detection, Moondream 3 does not) @@ -194,17 +202,21 @@ def get_prompt_placeholder(friendly_name: str) -> str: def is_florence_task(question: str) -> bool: - """Check if the question is a Florence-2 task token (either friendly name or internal token).""" + """Check if the question is a Florence-2 task token (either friendly name or internal token). + + This includes both base Florence prompts and PromptGen-specific prompts, + since all are handled by the Florence handler. + """ if not question: return False - # Check if it's a Florence-specific friendly name - if question in vlm_prompts_florence: + # Check if it's a Florence-specific friendly name (base or PromptGen) + if question in vlm_prompts_florence or question in vlm_prompts_promptgen: return True # Check if it's an internal Florence-2 task token (for backwards compatibility) - florence_tokens = ['', '', '', '', - '', '', '', '', '', - '', '', '', ''] - return question in florence_tokens + florence_base_tokens = ['', '', '', '', + '', '', '', '', ''] + promptgen_tokens = ['', '', '', ''] + return question in florence_base_tokens or question in promptgen_tokens def is_thinking_model(model_name: str) -> bool: @@ -1125,19 +1137,8 @@ class VQA: self.last_detection_data = {'points': points} return vqa_detection.format_points_text(points) return "Object not found" - elif question.lower().startswith('detect ') or question == 'DETECT_MODE': - target = question[7:].strip() if question.lower().startswith('detect ') else '' - if not target: - return "Please specify an object to detect" - debug(f'VQA caption: handler=moondream method=detect target="{target}"') - result = self.model.detect(image, target) - debug(f'VQA caption: handler=moondream detect_raw_result={result}') - detections = vqa_detection.parse_detections(result, target) - if detections: - self.last_detection_data = {'detections': detections} - return vqa_detection.format_detections_text(detections, include_confidence=False) - return "No objects detected" elif question == 'DETECT_GAZE' or question.lower() == 'detect gaze': + # Must be checked before generic 'detect ' prefix to avoid matching as detect target="Gaze" debug('VQA caption: handler=moondream method=detect_gaze') faces = self.model.detect(image, "face") debug(f'VQA caption: handler=moondream detect_gaze faces={faces}') @@ -1150,6 +1151,18 @@ class VQA: self.last_detection_data = {'points': [(gaze['x'], gaze['y'])]} return f"Gaze direction: ({gaze['x']:.3f}, {gaze['y']:.3f})" return "No face/gaze detected" + elif question.lower().startswith('detect ') or question == 'DETECT_MODE': + target = question[7:].strip() if question.lower().startswith('detect ') else '' + if not target: + return "Please specify an object to detect" + debug(f'VQA caption: handler=moondream method=detect target="{target}"') + result = self.model.detect(image, target) + debug(f'VQA caption: handler=moondream detect_raw_result={result}') + detections = vqa_detection.parse_detections(result, target) + if detections: + self.last_detection_data = {'detections': detections} + return vqa_detection.format_detections_text(detections, include_confidence=False) + return "No objects detected" else: debug(f'VQA caption: handler=moondream method=query question="{question}" reasoning={thinking_mode}') result = self.model.query(image, question, reasoning=thinking_mode) diff --git a/modules/shared.py b/modules/shared.py index 7e0c4a4a9..fb3b919fa 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -749,7 +749,6 @@ options_templates.update(options_section(('hidden_options', "Hidden options"), { "caption_openclip_mode": OptionInfo(caption_types[0], "OpenCLiP: default mode", gr.Dropdown, {"choices": caption_types, "visible": False}), "caption_openclip_blip_model": OptionInfo(list(caption_models)[0], "OpenCLiP: default captioner", gr.Dropdown, {"choices": list(caption_models), "visible": False}), "caption_openclip_num_beams": OptionInfo(1, "OpenCLiP: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}), - "caption_openclip_min_length": OptionInfo(32, "OpenCLiP: min length", gr.Slider, {"minimum": 1, "maximum": 128, "step": 1, "visible": False}), "caption_openclip_max_length": OptionInfo(74, "OpenCLiP: max length", gr.Slider, {"minimum": 1, "maximum": 512, "step": 1, "visible": False}), "caption_openclip_min_flavors": OptionInfo(2, "OpenCLiP: min flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1, "visible": False}), "caption_openclip_max_flavors": OptionInfo(16, "OpenCLiP: max flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1, "visible": False}), diff --git a/modules/ui_caption.py b/modules/ui_caption.py index 5867edd79..90e94387f 100644 --- a/modules/ui_caption.py +++ b/modules/ui_caption.py @@ -112,8 +112,7 @@ def update_tagger_params(model_name, general_threshold, character_threshold, inc def update_clip_params(*args): - clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams = args - shared.opts.caption_openclip_min_length = int(clip_min_length) + clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams = args shared.opts.caption_openclip_max_length = int(clip_max_length) shared.opts.caption_openclip_min_flavors = int(clip_min_flavors) shared.opts.caption_openclip_max_flavors = int(clip_max_flavors) @@ -215,7 +214,6 @@ def create_ui(): clip_mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast', elem_id='clip_clip_mode') with gr.Accordion(label='Caption: Advanced Options', open=False, visible=True): with gr.Row(): - clip_min_length = gr.Slider(label='clip: min length', value=shared.opts.caption_openclip_min_length, minimum=8, maximum=75, step=1, elem_id='clip_caption_min_length') clip_max_length = gr.Slider(label='clip: max length', value=shared.opts.caption_openclip_max_length, minimum=16, maximum=1024, step=1, elem_id='clip_caption_max_length') clip_chunk_size = gr.Slider(label='clip: chunk size', value=shared.opts.caption_openclip_chunk_size, minimum=256, maximum=4096, step=8, elem_id='clip_chunk_size') with gr.Row(): @@ -224,13 +222,12 @@ def create_ui(): clip_flavor_count = gr.Slider(label='clip: intermediates', value=shared.opts.caption_openclip_flavor_count, minimum=256, maximum=4096, step=8, elem_id='clip_flavor_intermediate_count') with gr.Row(): clip_num_beams = gr.Slider(label='clip: num beams', value=shared.opts.caption_openclip_num_beams, minimum=1, maximum=16, step=1, elem_id='clip_num_beams') - clip_min_length.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) - clip_max_length.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) - clip_chunk_size.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) - clip_min_flavors.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) - clip_max_flavors.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) - clip_flavor_count.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) - clip_num_beams.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + clip_max_length.change(fn=update_clip_params, inputs=[clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + clip_chunk_size.change(fn=update_clip_params, inputs=[clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + clip_min_flavors.change(fn=update_clip_params, inputs=[clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + clip_max_flavors.change(fn=update_clip_params, inputs=[clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + clip_flavor_count.change(fn=update_clip_params, inputs=[clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + clip_num_beams.change(fn=update_clip_params, inputs=[clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) with gr.Accordion(label='Caption: Batch', open=False, visible=True): with gr.Row(): clip_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], interactive=True, height=100, elem_id='clip_batch_files')