diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index cb1780677..fcb7464bc 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -22,6 +22,12 @@ vlm_models = { "Alibaba Qwen 2.0 VL 2B": "Qwen/Qwen2-VL-2B-Instruct", "Alibaba Qwen 2.5 Omni 3B": "Qwen/Qwen2.5-Omni-3B", "Alibaba Qwen 2.5 VL 3B": "Qwen/Qwen2.5-VL-3B-Instruct", + "Alibaba Qwen 3 VL 2B": "Qwen/Qwen3-VL-2B-Instruct", + "Alibaba Qwen 3 VL 2B Thinking": "Qwen/Qwen3-VL-2B-Thinking", + "Alibaba Qwen 3 VL 4B": "Qwen/Qwen3-VL-4B-Instruct", + "Alibaba Qwen 3 VL 4B Thinking": "Qwen/Qwen3-VL-4B-Thinking", + "Alibaba Qwen 3 VL 8B": "Qwen/Qwen3-VL-8B-Instruct", + "Alibaba Qwen 3 VL 8B Thinking": "Qwen/Qwen3-VL-8B-Thinking", "Huggingface Smol VL2 0.5B": "HuggingFaceTB/SmolVLM-500M-Instruct", "Huggingface Smol VL2 2B": "HuggingFaceTB/SmolVLM-Instruct", "Apple FastVLM 0.5B": "apple/FastVLM-0.5B", @@ -181,10 +187,14 @@ def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str if model is None or loaded != repo: shared.log.debug(f'Interrogate load: vlm="{repo}"') model = None - if '2.5' in repo: + if 'Qwen3-VL' in repo or 'Qwen3VL' in repo: + cls_name = transformers.Qwen3VLForConditionalGeneration + elif 'Qwen2.5-VL' in repo or 'Qwen2_5_VL' in repo: cls_name = transformers.Qwen2_5_VLForConditionalGeneration - else: + elif 'Qwen2-VL' in repo or 'Qwen2VL' in repo: cls_name = transformers.Qwen2VLForConditionalGeneration + else: + cls_name = transformers.AutoModelForCausalLM model = cls_name.from_pretrained( repo, torch_dtype=devices.dtype, diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index 221d815dc..1925a05d8 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -32,6 +32,14 @@ def b64(image): class Options: img2img = [ 'google/gemma-3-4b-it', + 'allura-org/Gemma-3-Glitter-4B', + 'Qwen/Qwen2.5-VL-3B-Instruct', + 'Qwen/Qwen3-VL-2B-Instruct', + 'Qwen/Qwen3-VL-2B-Thinking', + 'Qwen/Qwen3-VL-4B-Instruct', + 'Qwen/Qwen3-VL-4B-Thinking', + 'Qwen/Qwen3-VL-8B-Instruct', + 'Qwen/Qwen3-VL-8B-Thinking', ] models = { 'google/gemma-3-1b-it': {}, @@ -49,6 +57,12 @@ class Options: 'Qwen/Qwen2.5-1.5B-Instruct': {}, 'Qwen/Qwen2.5-3B-Instruct': {}, 'Qwen/Qwen2.5-VL-3B-Instruct': {}, + 'Qwen/Qwen3-VL-2B-Instruct': {}, + 'Qwen/Qwen3-VL-2B-Thinking': {}, + 'Qwen/Qwen3-VL-4B-Instruct': {}, + 'Qwen/Qwen3-VL-4B-Thinking': {}, + 'Qwen/Qwen3-VL-8B-Instruct': {}, + 'Qwen/Qwen3-VL-8B-Thinking': {}, 'microsoft/Phi-4-mini-instruct': {}, 'HuggingFaceTB/SmolLM2-135M-Instruct': {}, 'HuggingFaceTB/SmolLM2-360M-Instruct': {}, @@ -154,7 +168,17 @@ class Script(scripts_manager.Script): load_args = { 'pretrained_model_name_or_path': model_repo if not gguf_args else model_gguf } if model_subfolder: load_args['subfolder'] = model_subfolder # Comma was incorrect here - self.llm = transformers.AutoModelForCausalLM.from_pretrained( + + if 'Qwen3-VL' in model_repo or 'Qwen3VL' in model_repo: + cls_name = transformers.Qwen3VLForConditionalGeneration + elif 'Qwen2.5-VL' in model_repo or 'Qwen2_5_VL' in model_repo: + cls_name = transformers.Qwen2_5_VLForConditionalGeneration + elif 'Qwen2-VL' in model_repo or 'Qwen2VL' in model_repo: + cls_name = transformers.Qwen2VLForConditionalGeneration + else: + cls_name = transformers.AutoModelForCausalLM + + self.llm = cls_name.from_pretrained( **load_args, trust_remote_code=True, torch_dtype=devices.dtype, @@ -288,6 +312,25 @@ class Script(scripts_manager.Script): except Exception: current_image = None + # Resize large images to match VQA performance (Qwen3-VL performance is sensitive to resolution) + # Create a copy to avoid modifying the original image used by img2img + if current_image is not None and isinstance(current_image, Image.Image): + original_size = (current_image.width, current_image.height) + needs_resize = current_image.width > 768 or current_image.height > 768 + needs_rgb = current_image.mode != 'RGB' + + if needs_resize or needs_rgb: + # Copy the image before any modifications to preserve the original + current_image = current_image.copy() + + if needs_resize: + current_image.thumbnail((768, 768), Image.Resampling.LANCZOS) + debug_log(f'Prompt enhance: Resized image from {original_size} to {(current_image.width, current_image.height)}') + + if needs_rgb: + current_image = current_image.convert('RGB') + debug_log('Prompt enhance: Converted image to RGB mode') + has_system = system is not None and len(system) > 4 mode = 'custom' if has_system else ''