diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index f349add87..5055ef600 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -52,8 +52,8 @@ vlm_models = { "MiaoshouAI PromptGen 2.0 Large": "Disty0/Florence-2-large-PromptGen-v2.0", # 1.5GB "CogFlorence 2.0 Large": "thwri/CogFlorence-2-Large-Freeze", # 1.6GB "CogFlorence 2.2 Large": "thwri/CogFlorence-2.2-Large", # 1.6GB - "Moondream 2": "vikhyatk/moondream2", # 3.7GB - "Moondream 3 Preview": "moondream/moondream3-preview", # 9.3GB (gated) + f"Moondream 2 {ui_symbols.reasoning}": "vikhyatk/moondream2", # 3.7GB + f"Moondream 3 Preview {ui_symbols.reasoning}": "moondream/moondream3-preview", # 9.3GB (gated) "Google Pix Textcaps": "google/pix2struct-textcaps-base", # 1.1GB "Google PaliGemma 2 3B": "google/paligemma2-3b-pt-224", "Salesforce BLIP Base": "Salesforce/blip-vqa-base", # 1.5GB @@ -216,6 +216,8 @@ def is_thinking_model(model_name: str) -> bool: 'thinking', # Qwen3-VL-*-Thinking models 'moondream3', # Moondream 3 supports thinking 'moondream 3', + 'moondream2', # Moondream 2 supports reasoning mode + 'moondream 2', 'mimo', ] return any(indicator in model_lower for indicator in thinking_indicators) @@ -348,7 +350,7 @@ def clean(response, question, prefill=None): r_text = response['reasoning'] if isinstance(r_text, dict) and 'text' in r_text: r_text = r_text['text'] - text_response += f"Reasoning:\n{r_text}\nAnswer:\n" + text_response += f"Reasoning:\n{r_text}\n\nAnswer:\n" if 'answer' in response: text_response += response['answer'] @@ -376,7 +378,7 @@ def clean(response, question, prefill=None): while any(s in response for s in strip): for s in strip: response = response.replace(s, '') - response = response.replace('\n\n', '\n').replace(' ', ' ').replace('* ', '- ').strip() + response = response.replace(' ', ' ').replace('* ', '- ').strip() # Handle prefill retention/removal if shared.opts.interrogate_vlm_keep_prefill: @@ -585,9 +587,8 @@ def qwen( prefill_text = prefill_value.strip() # Thinking models emit their own tags via the chat template - # Use manual toggle OR auto-detection based on model name + # Only models with thinking capability can use thinking mode is_thinking = is_thinking_model(model_name) - use_thinking = thinking_mode or is_thinking # Standardize prefill prefill_value = vlm_prefill if prefill is None else prefill @@ -647,10 +648,15 @@ def qwen( if debug_enabled: debug(f'VQA interrogate: handler=qwen response_before_clean="{response}"') # Clean up thinking tags + # Note: is in the prompt, not the response - only appears in generated output if len(response) > 0: text = response[0] if shared.opts.interrogate_vlm_keep_thinking: - text = text.replace('', 'Reasoning:\n').replace('', '\nAnswer:') + # Handle case where is in prompt (not response) but is in response + if '' in text and '' not in text: + text = 'Reasoning:\n' + text.replace('', '\n\nAnswer:') + else: + text = text.replace('', 'Reasoning:\n').replace('', '\n\nAnswer:') else: while '' in text: start = text.find('') @@ -786,7 +792,7 @@ def gemma( # Clean up thinking tags (if any remain) if shared.opts.interrogate_vlm_keep_thinking: - response = response.replace('', 'Reasoning:\n').replace('', '\nAnswer:') + response = response.replace('', 'Reasoning:\n').replace('', '\n\nAnswer:') else: text = response while '' in text: @@ -976,7 +982,7 @@ def smol( if len(response) > 0: text = response[0] if shared.opts.interrogate_vlm_keep_thinking: - text = text.replace('', 'Reasoning:\n').replace('', '\nAnswer:') + text = text.replace('', 'Reasoning:\n').replace('', '\n\nAnswer:') else: while '' in text: start = text.find('') @@ -1107,7 +1113,6 @@ def moondream(question: str, image: Image.Image, repo: str = None, model_name: s devices.torch_gc() sd_models.move_model(model, devices.device) question = question.replace('<', '').replace('>', '').replace('_', ' ') - encoded = model.encode_image(image) with devices.inference_context(): if question == 'CAPTION': response = model.caption(image, length="short")['caption'] @@ -1180,7 +1185,7 @@ def moondream(question: str, image: Image.Image, repo: str = None, model_name: s reasoning_text = result['reasoning'].get('text', '') if isinstance(result['reasoning'], dict) else str(result['reasoning']) debug(f'VQA interrogate: handler=moondream reasoning_text="{reasoning_text[:100]}..."') if shared.opts.interrogate_vlm_keep_thinking: - response = f"Reasoning:\n{reasoning_text}\nAnswer:\n{response}" + response = f"Reasoning:\n{reasoning_text}\n\nAnswer:\n{response}" # When keep_thinking is False, just use the answer (reasoning is discarded) return response diff --git a/modules/shared.py b/modules/shared.py index 01e32e2c0..ddd01442e 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -671,6 +671,7 @@ options_templates.update(options_section(('interrogate', "Interrogate"), { "interrogate_vlm_top_p": OptionInfo(0, "VLM: top-p", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.01, "visible": False}), "interrogate_vlm_keep_prefill": OptionInfo(False, "VLM: keep prefill text in output", gr.Checkbox), "interrogate_vlm_keep_thinking": OptionInfo(False, "VLM: keep reasoning trace in output", gr.Checkbox), + "interrogate_vlm_thinking_mode": OptionInfo(False, "VLM: enable thinking/reasoning mode", gr.Checkbox), "deepbooru_sep": OptionInfo("

DeepBooru

", "", gr.HTML), "deepbooru_score_threshold": OptionInfo(0.65, "DeepBooru: score threshold", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}), diff --git a/modules/ui_caption.py b/modules/ui_caption.py index ad4681e7d..a8e333eeb 100644 --- a/modules/ui_caption.py +++ b/modules/ui_caption.py @@ -30,7 +30,7 @@ def update_vlm_prompt_placeholder(question): def update_vlm_params(*args): - vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking = args + vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking, vlm_thinking_mode = args shared.opts.interrogate_vlm_max_length = int(vlm_max_tokens) shared.opts.interrogate_vlm_num_beams = int(vlm_num_beams) shared.opts.interrogate_vlm_temperature = float(vlm_temperature) @@ -39,6 +39,7 @@ def update_vlm_params(*args): shared.opts.interrogate_vlm_top_p = float(vlm_top_p) shared.opts.interrogate_vlm_keep_prefill = bool(vlm_keep_prefill) shared.opts.interrogate_vlm_keep_thinking = bool(vlm_keep_thinking) + shared.opts.interrogate_vlm_thinking_mode = bool(vlm_thinking_mode) shared.opts.save(shared.config_filename) @@ -87,20 +88,21 @@ def create_ui(): vlm_top_p = gr.Slider(label='Top-P', value=shared.opts.interrogate_vlm_top_p, minimum=0.0, maximum=1.0, step=0.01, elem_id='vlm_top_p') with gr.Row(): vlm_do_sample = gr.Checkbox(label='Use sample', value=shared.opts.interrogate_vlm_do_sample, elem_id='vlm_do_sample') - vlm_thinking_mode = gr.Checkbox(label='Thinking Mode', value=False, elem_id='vlm_thinking_mode') + vlm_thinking_mode = gr.Checkbox(label='Thinking Mode', value=shared.opts.interrogate_vlm_thinking_mode, elem_id='vlm_thinking_mode') with gr.Row(): vlm_keep_thinking = gr.Checkbox(label='Keep Thinking Trace', value=shared.opts.interrogate_vlm_keep_thinking, elem_id='vlm_keep_thinking') vlm_keep_prefill = gr.Checkbox(label='Keep Prefill', value=shared.opts.interrogate_vlm_keep_prefill, elem_id='vlm_keep_prefill') with gr.Row(): vlm_prefill = gr.Textbox(label='Prefill Text', value='', lines=1, elem_id='vlm_prefill', placeholder='Optional prefill text for model to continue from') - vlm_max_tokens.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking], outputs=[]) - vlm_num_beams.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking], outputs=[]) - vlm_temperature.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking], outputs=[]) - vlm_do_sample.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking], outputs=[]) - vlm_top_k.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking], outputs=[]) - vlm_top_p.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking], outputs=[]) - vlm_keep_prefill.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking], outputs=[]) - vlm_keep_thinking.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking], outputs=[]) + vlm_max_tokens.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking, vlm_thinking_mode], outputs=[]) + vlm_num_beams.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking, vlm_thinking_mode], outputs=[]) + vlm_temperature.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking, vlm_thinking_mode], outputs=[]) + vlm_do_sample.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking, vlm_thinking_mode], outputs=[]) + vlm_top_k.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking, vlm_thinking_mode], outputs=[]) + vlm_top_p.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking, vlm_thinking_mode], outputs=[]) + vlm_keep_prefill.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking, vlm_thinking_mode], outputs=[]) + vlm_keep_thinking.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking, vlm_thinking_mode], outputs=[]) + vlm_thinking_mode.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p, vlm_keep_prefill, vlm_keep_thinking, vlm_thinking_mode], outputs=[]) with gr.Accordion(label='Batch caption', open=False, visible=True): with gr.Row(): vlm_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], interactive=True, height=100, elem_id='vlm_batch_files')