mirror of
https://github.com/vladmandic/automatic
synced 2026-09-19 17:24:32 +02:00
fix(caption): remove dead min_length param, split Florence/PromptGen prompts, fix gaze detection
- Remove caption_openclip_min_length from settings, API models, endpoints, and UI (clip_interrogator library has no min_length support; parameter was never functional) - Split vlm_prompts_florence into base Florence prompts and PromptGen-only prompts (GENERATE_TAGS, Analyze, Mixed Caption require MiaoshouAI PromptGen fine-tune) - Add 'promptgen' category to /vqa/prompts API endpoint - Fix gaze detection: move DETECT_GAZE check before generic 'detect ' prefix to prevent "Detect Gaze" matching as detect target="Gaze" - Update test suite: remove min_length tests, fix min_flavors to use mode='best', add acceptance-only notes, fix thinking trace detection, improve bracket/OCR tests, split Florence/PromptGen test coverage
This commit is contained in:
CalamitousFelicitousness
@@ -145,8 +145,6 @@ def post_caption(req: models.ReqCaption):
|
||||
raise HTTPException(status_code=404, detail="Model not found")
|
||||
# Build clip overrides from request (only include non-None values)
|
||||
clip_overrides = {}
|
||||
if req.min_length is not None:
|
||||
clip_overrides['min_length'] = req.min_length
|
||||
if req.max_length is not None:
|
||||
clip_overrides['max_length'] = req.max_length
|
||||
if req.chunk_size is not None:
|
||||
@@ -314,8 +312,6 @@ def _dispatch_openclip(req: models.ReqCaptionOpenCLIP) -> models.ResCaptionDispa
|
||||
raise HTTPException(status_code=404, detail="Model not found")
|
||||
# Build clip overrides from request
|
||||
clip_overrides = {}
|
||||
if req.min_length is not None:
|
||||
clip_overrides['min_length'] = req.min_length
|
||||
if req.max_length is not None:
|
||||
clip_overrides['max_length'] = req.max_length
|
||||
if req.chunk_size is not None:
|
||||
@@ -499,7 +495,8 @@ def get_vqa_prompts(model: Optional[str] = None):
|
||||
|
||||
**Prompt Categories:**
|
||||
- Common: Use Prompt, Short/Normal/Long Caption
|
||||
- Florence: Phrase Grounding, Object Detection, OCR, Dense Region Caption
|
||||
- Florence: Phrase Grounding, Object Detection, OCR, Dense Region Caption (all Florence models)
|
||||
- PromptGen: Analyze, Generate Tags, Mixed Caption (MiaoshouAI PromptGen fine-tunes only)
|
||||
- Moondream: Point at..., Detect all..., Detect Gaze
|
||||
"""
|
||||
from modules.caption import vqa
|
||||
@@ -509,6 +506,7 @@ def get_vqa_prompts(model: Optional[str] = None):
|
||||
return {
|
||||
"common": vqa.vlm_prompts_common,
|
||||
"florence": vqa.vlm_prompts_florence,
|
||||
"promptgen": vqa.vlm_prompts_promptgen,
|
||||
"moondream": vqa.vlm_prompts_moondream,
|
||||
"moondream2_only": vqa.vlm_prompts_moondream2
|
||||
}
|
||||
|
||||
@@ -378,7 +378,6 @@ class ReqCaption(BaseModel):
|
||||
mode: str = Field(default="best", title="Mode", description="Caption mode. Fast: Quick caption with minimal flavor terms. Classic: Standard captioning with balanced quality and speed. Best: Most thorough analysis, slowest but highest quality. Negative: Generate terms to use as negative prompt.")
|
||||
analyze: bool = Field(default=False, title="Analyze", description="If True, returns detailed image analysis breakdown (medium, artist, movement, trending, flavor) in addition to caption.")
|
||||
# Advanced settings (optional per-request overrides)
|
||||
min_length: Optional[int] = Field(default=None, title="Min Length", description="Minimum number of tokens in the generated caption.")
|
||||
max_length: Optional[int] = Field(default=None, title="Max Length", description="Maximum number of tokens in the generated caption.")
|
||||
chunk_size: Optional[int] = Field(default=None, title="Chunk Size", description="Batch size for processing description candidates (flavors). Higher values speed up captioning but increase VRAM usage.")
|
||||
min_flavors: Optional[int] = Field(default=None, title="Min Flavors", description="Minimum number of descriptive tags (flavors) to keep in the final prompt.")
|
||||
@@ -486,7 +485,6 @@ class ReqCaptionOpenCLIP(BaseModel):
|
||||
blip_model: str = Field(default="blip-large", title="Caption Model", description="BLIP model used to generate the initial image caption.")
|
||||
mode: str = Field(default="best", title="Mode", description="Caption mode: 'best' (highest quality), 'fast' (quick), 'classic' (traditional), 'caption' (BLIP only), 'negative' (for negative prompts).")
|
||||
analyze: bool = Field(default=False, title="Analyze", description="If True, returns detailed breakdown (medium, artist, movement, trending, flavor).")
|
||||
min_length: Optional[int] = Field(default=None, title="Min Length", description="Minimum tokens in generated caption.")
|
||||
max_length: Optional[int] = Field(default=None, title="Max Length", description="Maximum tokens in generated caption.")
|
||||
chunk_size: Optional[int] = Field(default=None, title="Chunk Size", description="Batch size for processing flavors.")
|
||||
min_flavors: Optional[int] = Field(default=None, title="Min Flavors", description="Minimum descriptive tags to keep.")
|
||||
|
||||
+36
-23
@@ -80,7 +80,7 @@ vlm_prompts_common = [
|
||||
"Long Caption",
|
||||
]
|
||||
|
||||
# Florence-2 specific prompts (only shown for Florence/PromptGen models)
|
||||
# Florence-2 base prompts (supported by all Florence models including CogFlorence)
|
||||
vlm_prompts_florence = [
|
||||
"Phrase Grounding",
|
||||
"Object Detection",
|
||||
@@ -88,6 +88,10 @@ vlm_prompts_florence = [
|
||||
"Region Proposal",
|
||||
"OCR (Read Text)",
|
||||
"OCR with Regions",
|
||||
]
|
||||
|
||||
# PromptGen-only prompts (require MiaoshouAI PromptGen fine-tune)
|
||||
vlm_prompts_promptgen = [
|
||||
"Analyze",
|
||||
"Generate Tags",
|
||||
"Mixed Caption",
|
||||
@@ -148,7 +152,7 @@ vlm_prompt_placeholders = {
|
||||
}
|
||||
|
||||
# Legacy list for backwards compatibility
|
||||
vlm_prompts = vlm_prompts_common + vlm_prompts_florence + vlm_prompts_moondream + vlm_prompts_moondream2
|
||||
vlm_prompts = vlm_prompts_common + vlm_prompts_florence + vlm_prompts_promptgen + vlm_prompts_moondream + vlm_prompts_moondream2
|
||||
|
||||
vlm_prefill = 'Answer: the image shows'
|
||||
|
||||
@@ -160,8 +164,12 @@ def get_prompts_for_model(model_name: str) -> list:
|
||||
|
||||
model_lower = model_name.lower()
|
||||
|
||||
# Check for Florence-2 / PromptGen models
|
||||
if 'florence' in model_lower or 'promptgen' in model_lower:
|
||||
# Check for PromptGen models (MiaoshouAI fine-tunes with extra prompts)
|
||||
if 'promptgen' in model_lower:
|
||||
return vlm_prompts_common + vlm_prompts_florence + vlm_prompts_promptgen
|
||||
|
||||
# Check for Florence-2 base / CogFlorence models (no PromptGen-specific prompts)
|
||||
if 'florence' in model_lower:
|
||||
return vlm_prompts_common + vlm_prompts_florence
|
||||
|
||||
# Check for Moondream models (Moondream 2 has gaze detection, Moondream 3 does not)
|
||||
@@ -194,17 +202,21 @@ def get_prompt_placeholder(friendly_name: str) -> str:
|
||||
|
||||
|
||||
def is_florence_task(question: str) -> bool:
|
||||
"""Check if the question is a Florence-2 task token (either friendly name or internal token)."""
|
||||
"""Check if the question is a Florence-2 task token (either friendly name or internal token).
|
||||
|
||||
This includes both base Florence prompts and PromptGen-specific prompts,
|
||||
since all are handled by the Florence handler.
|
||||
"""
|
||||
if not question:
|
||||
return False
|
||||
# Check if it's a Florence-specific friendly name
|
||||
if question in vlm_prompts_florence:
|
||||
# Check if it's a Florence-specific friendly name (base or PromptGen)
|
||||
if question in vlm_prompts_florence or question in vlm_prompts_promptgen:
|
||||
return True
|
||||
# Check if it's an internal Florence-2 task token (for backwards compatibility)
|
||||
florence_tokens = ['<CAPTION>', '<DETAILED_CAPTION>', '<MORE_DETAILED_CAPTION>', '<CAPTION_TO_PHRASE_GROUNDING>',
|
||||
'<OD>', '<DENSE_REGION_CAPTION>', '<REGION_PROPOSAL>', '<OCR>', '<OCR_WITH_REGION>',
|
||||
'<ANALYZE>', '<GENERATE_TAGS>', '<MIXED_CAPTION>', '<MIXED_CAPTION_PLUS>']
|
||||
return question in florence_tokens
|
||||
florence_base_tokens = ['<CAPTION>', '<DETAILED_CAPTION>', '<MORE_DETAILED_CAPTION>', '<CAPTION_TO_PHRASE_GROUNDING>',
|
||||
'<OD>', '<DENSE_REGION_CAPTION>', '<REGION_PROPOSAL>', '<OCR>', '<OCR_WITH_REGION>']
|
||||
promptgen_tokens = ['<ANALYZE>', '<GENERATE_TAGS>', '<MIXED_CAPTION>', '<MIXED_CAPTION_PLUS>']
|
||||
return question in florence_base_tokens or question in promptgen_tokens
|
||||
|
||||
|
||||
def is_thinking_model(model_name: str) -> bool:
|
||||
@@ -1125,19 +1137,8 @@ class VQA:
|
||||
self.last_detection_data = {'points': points}
|
||||
return vqa_detection.format_points_text(points)
|
||||
return "Object not found"
|
||||
elif question.lower().startswith('detect ') or question == 'DETECT_MODE':
|
||||
target = question[7:].strip() if question.lower().startswith('detect ') else ''
|
||||
if not target:
|
||||
return "Please specify an object to detect"
|
||||
debug(f'VQA caption: handler=moondream method=detect target="{target}"')
|
||||
result = self.model.detect(image, target)
|
||||
debug(f'VQA caption: handler=moondream detect_raw_result={result}')
|
||||
detections = vqa_detection.parse_detections(result, target)
|
||||
if detections:
|
||||
self.last_detection_data = {'detections': detections}
|
||||
return vqa_detection.format_detections_text(detections, include_confidence=False)
|
||||
return "No objects detected"
|
||||
elif question == 'DETECT_GAZE' or question.lower() == 'detect gaze':
|
||||
# Must be checked before generic 'detect ' prefix to avoid matching as detect target="Gaze"
|
||||
debug('VQA caption: handler=moondream method=detect_gaze')
|
||||
faces = self.model.detect(image, "face")
|
||||
debug(f'VQA caption: handler=moondream detect_gaze faces={faces}')
|
||||
@@ -1150,6 +1151,18 @@ class VQA:
|
||||
self.last_detection_data = {'points': [(gaze['x'], gaze['y'])]}
|
||||
return f"Gaze direction: ({gaze['x']:.3f}, {gaze['y']:.3f})"
|
||||
return "No face/gaze detected"
|
||||
elif question.lower().startswith('detect ') or question == 'DETECT_MODE':
|
||||
target = question[7:].strip() if question.lower().startswith('detect ') else ''
|
||||
if not target:
|
||||
return "Please specify an object to detect"
|
||||
debug(f'VQA caption: handler=moondream method=detect target="{target}"')
|
||||
result = self.model.detect(image, target)
|
||||
debug(f'VQA caption: handler=moondream detect_raw_result={result}')
|
||||
detections = vqa_detection.parse_detections(result, target)
|
||||
if detections:
|
||||
self.last_detection_data = {'detections': detections}
|
||||
return vqa_detection.format_detections_text(detections, include_confidence=False)
|
||||
return "No objects detected"
|
||||
else:
|
||||
debug(f'VQA caption: handler=moondream method=query question="{question}" reasoning={thinking_mode}')
|
||||
result = self.model.query(image, question, reasoning=thinking_mode)
|
||||
|
||||
@@ -749,7 +749,6 @@ options_templates.update(options_section(('hidden_options', "Hidden options"), {
|
||||
"caption_openclip_mode": OptionInfo(caption_types[0], "OpenCLiP: default mode", gr.Dropdown, {"choices": caption_types, "visible": False}),
|
||||
"caption_openclip_blip_model": OptionInfo(list(caption_models)[0], "OpenCLiP: default captioner", gr.Dropdown, {"choices": list(caption_models), "visible": False}),
|
||||
"caption_openclip_num_beams": OptionInfo(1, "OpenCLiP: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}),
|
||||
"caption_openclip_min_length": OptionInfo(32, "OpenCLiP: min length", gr.Slider, {"minimum": 1, "maximum": 128, "step": 1, "visible": False}),
|
||||
"caption_openclip_max_length": OptionInfo(74, "OpenCLiP: max length", gr.Slider, {"minimum": 1, "maximum": 512, "step": 1, "visible": False}),
|
||||
"caption_openclip_min_flavors": OptionInfo(2, "OpenCLiP: min flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1, "visible": False}),
|
||||
"caption_openclip_max_flavors": OptionInfo(16, "OpenCLiP: max flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1, "visible": False}),
|
||||
|
||||
+7
-10
@@ -112,8 +112,7 @@ def update_tagger_params(model_name, general_threshold, character_threshold, inc
|
||||
|
||||
|
||||
def update_clip_params(*args):
|
||||
clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams = args
|
||||
shared.opts.caption_openclip_min_length = int(clip_min_length)
|
||||
clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams = args
|
||||
shared.opts.caption_openclip_max_length = int(clip_max_length)
|
||||
shared.opts.caption_openclip_min_flavors = int(clip_min_flavors)
|
||||
shared.opts.caption_openclip_max_flavors = int(clip_max_flavors)
|
||||
@@ -215,7 +214,6 @@ def create_ui():
|
||||
clip_mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast', elem_id='clip_clip_mode')
|
||||
with gr.Accordion(label='Caption: Advanced Options', open=False, visible=True):
|
||||
with gr.Row():
|
||||
clip_min_length = gr.Slider(label='clip: min length', value=shared.opts.caption_openclip_min_length, minimum=8, maximum=75, step=1, elem_id='clip_caption_min_length')
|
||||
clip_max_length = gr.Slider(label='clip: max length', value=shared.opts.caption_openclip_max_length, minimum=16, maximum=1024, step=1, elem_id='clip_caption_max_length')
|
||||
clip_chunk_size = gr.Slider(label='clip: chunk size', value=shared.opts.caption_openclip_chunk_size, minimum=256, maximum=4096, step=8, elem_id='clip_chunk_size')
|
||||
with gr.Row():
|
||||
@@ -224,13 +222,12 @@ def create_ui():
|
||||
clip_flavor_count = gr.Slider(label='clip: intermediates', value=shared.opts.caption_openclip_flavor_count, minimum=256, maximum=4096, step=8, elem_id='clip_flavor_intermediate_count')
|
||||
with gr.Row():
|
||||
clip_num_beams = gr.Slider(label='clip: num beams', value=shared.opts.caption_openclip_num_beams, minimum=1, maximum=16, step=1, elem_id='clip_num_beams')
|
||||
clip_min_length.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
clip_max_length.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
clip_chunk_size.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
clip_min_flavors.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
clip_max_flavors.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
clip_flavor_count.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
clip_num_beams.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
clip_max_length.change(fn=update_clip_params, inputs=[clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
clip_chunk_size.change(fn=update_clip_params, inputs=[clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
clip_min_flavors.change(fn=update_clip_params, inputs=[clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
clip_max_flavors.change(fn=update_clip_params, inputs=[clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
clip_flavor_count.change(fn=update_clip_params, inputs=[clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
clip_num_beams.change(fn=update_clip_params, inputs=[clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
|
||||
with gr.Accordion(label='Caption: Batch', open=False, visible=True):
|
||||
with gr.Row():
|
||||
clip_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], interactive=True, height=100, elem_id='clip_batch_files')
|
||||
|
||||
Reference in New Issue
Block a user