Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 17:24:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4314 from CalamitousFelicitousness/qwen3-vl

Browse Source 
Add Qwen3VL and expand prompt enhance img2prompt
This commit is contained in:
Vladimir Mandic
2025-10-26 09:42:52 -04:00
committed by GitHub
parent a291987a1e 73588c2000
commit 8b8bfa8e3d
2 changed files with 56 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/interrogate/vqa.py
+12 -2
View File
@@ -22,6 +22,12 @@ vlm_models = {
"Alibaba Qwen 2.0 VL 2B": "Qwen/Qwen2-VL-2B-Instruct",
"Alibaba Qwen 2.5 Omni 3B": "Qwen/Qwen2.5-Omni-3B",
"Alibaba Qwen 2.5 VL 3B": "Qwen/Qwen2.5-VL-3B-Instruct",
"Alibaba Qwen 3 VL 2B": "Qwen/Qwen3-VL-2B-Instruct",
"Alibaba Qwen 3 VL 2B Thinking": "Qwen/Qwen3-VL-2B-Thinking",
"Alibaba Qwen 3 VL 4B": "Qwen/Qwen3-VL-4B-Instruct",
"Alibaba Qwen 3 VL 4B Thinking": "Qwen/Qwen3-VL-4B-Thinking",
"Alibaba Qwen 3 VL 8B": "Qwen/Qwen3-VL-8B-Instruct",
"Alibaba Qwen 3 VL 8B Thinking": "Qwen/Qwen3-VL-8B-Thinking",
"Huggingface Smol VL2 0.5B": "HuggingFaceTB/SmolVLM-500M-Instruct",
"Huggingface Smol VL2 2B": "HuggingFaceTB/SmolVLM-Instruct",
"Apple FastVLM 0.5B": "apple/FastVLM-0.5B",
@@ -181,10 +187,14 @@ def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str
if model is None or loaded != repo:
shared.log.debug(f'Interrogate load: vlm="{repo}"')
model = None
if '2.5' in repo:
if 'Qwen3-VL' in repo or 'Qwen3VL' in repo:
cls_name = transformers.Qwen3VLForConditionalGeneration
elif 'Qwen2.5-VL' in repo or 'Qwen2_5_VL' in repo:
cls_name = transformers.Qwen2_5_VLForConditionalGeneration
else:
elif 'Qwen2-VL' in repo or 'Qwen2VL' in repo:
cls_name = transformers.Qwen2VLForConditionalGeneration
else:
cls_name = transformers.AutoModelForCausalLM
model = cls_name.from_pretrained(
repo,
torch_dtype=devices.dtype,
scripts/prompt_enhance.py
+44 -1
View File
@@ -32,6 +32,14 @@ def b64(image):
class Options:
img2img = [
'google/gemma-3-4b-it',
'allura-org/Gemma-3-Glitter-4B',
'Qwen/Qwen2.5-VL-3B-Instruct',
'Qwen/Qwen3-VL-2B-Instruct',
'Qwen/Qwen3-VL-2B-Thinking',
'Qwen/Qwen3-VL-4B-Instruct',
'Qwen/Qwen3-VL-4B-Thinking',
'Qwen/Qwen3-VL-8B-Instruct',
'Qwen/Qwen3-VL-8B-Thinking',
]
models = {
'google/gemma-3-1b-it': {},
@@ -49,6 +57,12 @@ class Options:
'Qwen/Qwen2.5-1.5B-Instruct': {},
'Qwen/Qwen2.5-3B-Instruct': {},
'Qwen/Qwen2.5-VL-3B-Instruct': {},
'Qwen/Qwen3-VL-2B-Instruct': {},
'Qwen/Qwen3-VL-2B-Thinking': {},
'Qwen/Qwen3-VL-4B-Instruct': {},
'Qwen/Qwen3-VL-4B-Thinking': {},
'Qwen/Qwen3-VL-8B-Instruct': {},
'Qwen/Qwen3-VL-8B-Thinking': {},
'microsoft/Phi-4-mini-instruct': {},
'HuggingFaceTB/SmolLM2-135M-Instruct': {},
'HuggingFaceTB/SmolLM2-360M-Instruct': {},
@@ -154,7 +168,17 @@ class Script(scripts_manager.Script):
load_args = { 'pretrained_model_name_or_path': model_repo if not gguf_args else model_gguf }
if model_subfolder:
load_args['subfolder'] = model_subfolder # Comma was incorrect here
self.llm = transformers.AutoModelForCausalLM.from_pretrained(
if 'Qwen3-VL' in model_repo or 'Qwen3VL' in model_repo:
cls_name = transformers.Qwen3VLForConditionalGeneration
elif 'Qwen2.5-VL' in model_repo or 'Qwen2_5_VL' in model_repo:
cls_name = transformers.Qwen2_5_VLForConditionalGeneration
elif 'Qwen2-VL' in model_repo or 'Qwen2VL' in model_repo:
cls_name = transformers.Qwen2VLForConditionalGeneration
else:
cls_name = transformers.AutoModelForCausalLM
self.llm = cls_name.from_pretrained(
**load_args,
trust_remote_code=True,
torch_dtype=devices.dtype,
@@ -288,6 +312,25 @@ class Script(scripts_manager.Script):
except Exception:
current_image = None
# Resize large images to match VQA performance (Qwen3-VL performance is sensitive to resolution)
# Create a copy to avoid modifying the original image used by img2img
if current_image is not None and isinstance(current_image, Image.Image):
original_size = (current_image.width, current_image.height)
needs_resize = current_image.width > 768 or current_image.height > 768
needs_rgb = current_image.mode != 'RGB'
if needs_resize or needs_rgb:
# Copy the image before any modifications to preserve the original
current_image = current_image.copy()
if needs_resize:
current_image.thumbnail((768, 768), Image.Resampling.LANCZOS)
debug_log(f'Prompt enhance: Resized image from {original_size} to {(current_image.width, current_image.height)}')
if needs_rgb:
current_image = current_image.convert('RGB')
debug_log('Prompt enhance: Converted image to RGB mode')
has_system = system is not None and len(system) > 4
mode = 'custom' if has_system else ''