From a8a9e6d83612ad13deaf7082132c2e767e42e99f Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Fri, 5 Dec 2025 01:38:28 +0000 Subject: [PATCH] fix(vqa): separate Moondream 2 and 3 task prompts Moondream 3 does not support gaze detection (detect_gaze method), so "Detect Gaze" task is now only shown for Moondream 2. --- modules/interrogate/vqa.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 5055ef600..4df0acaf0 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -100,10 +100,14 @@ vlm_prompts_florence = [ "Mixed Caption+", ] -# Moondream specific prompts (only shown for Moondream models) +# Moondream specific prompts (shared by Moondream 2 and 3) vlm_prompts_moondream = [ "Point at...", "Detect all...", +] + +# Moondream 2 only prompts (gaze detection not available in Moondream 3) +vlm_prompts_moondream2 = [ "Detect Gaze", ] @@ -150,7 +154,7 @@ vlm_prompt_placeholders = { } # Legacy list for backwards compatibility -vlm_prompts = vlm_prompts_common + vlm_prompts_florence + vlm_prompts_moondream +vlm_prompts = vlm_prompts_common + vlm_prompts_florence + vlm_prompts_moondream + vlm_prompts_moondream2 vlm_prefill = 'Answer: the image shows' @@ -166,9 +170,12 @@ def get_prompts_for_model(model_name: str) -> list: if 'florence' in model_lower or 'promptgen' in model_lower: return vlm_prompts_common + vlm_prompts_florence - # Check for Moondream models + # Check for Moondream models (Moondream 2 has gaze detection, Moondream 3 does not) if 'moondream' in model_lower: - return vlm_prompts_common + vlm_prompts_moondream + if 'moondream3' in model_lower or 'moondream 3' in model_lower: + return vlm_prompts_common + vlm_prompts_moondream + else: # Moondream 2 includes gaze detection + return vlm_prompts_common + vlm_prompts_moondream + vlm_prompts_moondream2 # Default: common prompts only return vlm_prompts_common @@ -224,7 +231,7 @@ def is_thinking_model(model_name: str) -> bool: def load_model(model_name: str = None): - """Pre-load VLM model into memory without running inference.""" + """Pre-load VLM model into memory.""" global processor, model, loaded, quant_args # pylint: disable=global-statement model_name = model_name or shared.opts.interrogate_vlm_model if model_name not in vlm_models: @@ -582,7 +589,7 @@ def qwen( ], } ] - # Add prefill for all models (only if provided) + # Add prefill if provided) prefill_value = vlm_prefill if prefill is None else prefill prefill_text = prefill_value.strip() @@ -725,7 +732,7 @@ def gemma( {"role": "system", "content": system_content}, {"role": "user", "content": user_content}, ] - # Add prefill for all models (only if provided) + # Add prefill if provided) prefill_value = vlm_prefill if prefill is None else prefill prefill_text = prefill_value.strip() use_prefill = len(prefill_text) > 0 @@ -928,7 +935,7 @@ def smol( ], } ] - # Add prefill for all models (only if provided) + # Add prefill if provided) prefill_value = vlm_prefill if prefill is None else prefill prefill_text = prefill_value.strip() use_prefill = len(prefill_text) > 0