From 863e172aad27dd0dc08686bf84b22a033f22292c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 12 Aug 2025 11:55:38 -0400 Subject: [PATCH] add Qwen/Qwen2.5-VL-3B-Instruct Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/interrogate/vqa.py | 9 +++++++-- scripts/prompt_enhance.py | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db01163cf..ce81cf4cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,7 @@ And (*as always*) many bugfixes and improvements to existing features! - **Other** - **prompt enhance** add `allura-org/Gemma-3-Glitter-4B` model support - **prompt enhance** add `Qwen/Qwen3-4B-Instruct-2507` model support + - **caption** add `Qwen/Qwen2.5-VL-3B-Instruct` model support - **schedulers** add **Flash FlowMatch** - **API** - add `/sdapi/v1/checkpoint` POST endpoint to simply load a model diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 07f7e7a15..6f419a4e5 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -29,8 +29,9 @@ vlm_models = { "Google Gemma 3n E4B": "google/gemma-3n-E4B-it", # 1.5GB "Google Pix Textcaps": "google/pix2struct-textcaps-base", # 1.1GB "Google PaliGemma 2 3B": "google/paligemma2-3b-pt-224", - "Alibaba Qwen VL2 2B": "Qwen/Qwen2-VL-2B-Instruct", + "Alibaba Qwen 2.0 VL 2B": "Qwen/Qwen2-VL-2B-Instruct", "Alibaba Qwen 2.5 Omni 3B": "Qwen/Qwen2.5-Omni-3B", + "Alibaba Qwen 2.5 VL 4B": "Qwen/Qwen2.5-VL-3B-Instruct", "Huggingface Smol VL2 0.5B": "HuggingFaceTB/SmolVLM-500M-Instruct", "Huggingface Smol VL2 2B": "HuggingFaceTB/SmolVLM-Instruct", "Salesforce BLIP Base": "Salesforce/blip-vqa-base", # 1.5GB @@ -122,7 +123,11 @@ def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str if model is None or loaded != repo: shared.log.debug(f'Interrogate load: vlm="{repo}"') model = None - model = transformers.Qwen2VLForConditionalGeneration.from_pretrained( + if '2.5' in repo: + cls_name = transformers.Qwen2_5_VLForConditionalGeneration + else: + cls_name = transformers.Qwen2VLForConditionalGeneration + model = cls_name.from_pretrained( repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index 1950f847e..494c52afa 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -48,6 +48,7 @@ class Options: 'Qwen/Qwen2.5-0.5B-Instruct': {}, 'Qwen/Qwen2.5-1.5B-Instruct': {}, 'Qwen/Qwen2.5-3B-Instruct': {}, + 'Qwen/Qwen2.5-VL-3B-Instruct': {}, 'microsoft/Phi-4-mini-instruct': {}, 'HuggingFaceTB/SmolLM2-135M-Instruct': {}, 'HuggingFaceTB/SmolLM2-360M-Instruct': {},