From 1b80147881235fa6e38147e0f2fa4a197908f883 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sat, 25 Oct 2025 22:12:20 +0100 Subject: [PATCH 1/7] Add Qwen3-VL-4B-Instruct --- modules/interrogate/vqa.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 0821d5c4f..1cd2f69b6 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -22,6 +22,7 @@ vlm_models = { "Alibaba Qwen 2.0 VL 2B": "Qwen/Qwen2-VL-2B-Instruct", "Alibaba Qwen 2.5 Omni 3B": "Qwen/Qwen2.5-Omni-3B", "Alibaba Qwen 2.5 VL 4B": "Qwen/Qwen2.5-VL-3B-Instruct", + "Alibaba Qwen 3 VL 4B": "Qwen/Qwen3-VL-4B-Instruct", "Huggingface Smol VL2 0.5B": "HuggingFaceTB/SmolVLM-500M-Instruct", "Huggingface Smol VL2 2B": "HuggingFaceTB/SmolVLM-Instruct", "Apple FastVLM 0.5B": "apple/FastVLM-0.5B", @@ -181,7 +182,9 @@ def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str if model is None or loaded != repo: shared.log.debug(f'Interrogate load: vlm="{repo}"') model = None - if '2.5' in repo: + if 'Qwen3' in repo: + cls_name = transformers.Qwen3VLForConditionalGeneration + elif '2.5' in repo: cls_name = transformers.Qwen2_5_VLForConditionalGeneration else: cls_name = transformers.Qwen2VLForConditionalGeneration From 3fc9efa9ee588d472706af6cdb8786565e34be2e Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 26 Oct 2025 02:53:34 +0000 Subject: [PATCH 2/7] Add remaining Qwen3VL models up to 8B --- modules/interrogate/vqa.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 1cd2f69b6..0275773ce 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -22,7 +22,12 @@ vlm_models = { "Alibaba Qwen 2.0 VL 2B": "Qwen/Qwen2-VL-2B-Instruct", "Alibaba Qwen 2.5 Omni 3B": "Qwen/Qwen2.5-Omni-3B", "Alibaba Qwen 2.5 VL 4B": "Qwen/Qwen2.5-VL-3B-Instruct", + "Alibaba Qwen 3 VL 2B": "Qwen/Qwen3-VL-2B-Instruct", + "Alibaba Qwen 3 VL 2B Thinking": "Qwen/Qwen3-VL-2B-Thinking", "Alibaba Qwen 3 VL 4B": "Qwen/Qwen3-VL-4B-Instruct", + "Alibaba Qwen 3 VL 4B Thinking": "Qwen/Qwen3-VL-4B-Thinking", + "Alibaba Qwen 3 VL 8B": "Qwen/Qwen3-VL-8B-Instruct", + "Alibaba Qwen 3 VL 8B Thinking": "Qwen/Qwen3-VL-8B-Thinking", "Huggingface Smol VL2 0.5B": "HuggingFaceTB/SmolVLM-500M-Instruct", "Huggingface Smol VL2 2B": "HuggingFaceTB/SmolVLM-Instruct", "Apple FastVLM 0.5B": "apple/FastVLM-0.5B", From c72d993ff4a3bdccfb0d6c439809c77eaa6e01d7 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 26 Oct 2025 03:12:10 +0000 Subject: [PATCH 3/7] Prompt enhance update Added Qwen3-VL models to prompt enhance Added Qwen2.5-VL and Glitter Gemma 4B to img2img prompt enhance --- scripts/prompt_enhance.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index 221d815dc..87396136f 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -32,6 +32,14 @@ def b64(image): class Options: img2img = [ 'google/gemma-3-4b-it', + 'allura-org/Gemma-3-Glitter-4B', + 'Qwen/Qwen2.5-VL-3B-Instruct', + 'Qwen/Qwen3-VL-2B-Instruct', + 'Qwen/Qwen3-VL-2B-Thinking', + 'Qwen/Qwen3-VL-4B-Instruct', + 'Qwen/Qwen3-VL-4B-Thinking', + 'Qwen/Qwen3-VL-8B-Instruct', + 'Qwen/Qwen3-VL-8B-Thinking', ] models = { 'google/gemma-3-1b-it': {}, @@ -49,6 +57,12 @@ class Options: 'Qwen/Qwen2.5-1.5B-Instruct': {}, 'Qwen/Qwen2.5-3B-Instruct': {}, 'Qwen/Qwen2.5-VL-3B-Instruct': {}, + 'Qwen/Qwen3-VL-2B-Instruct': {}, + 'Qwen/Qwen3-VL-2B-Thinking': {}, + 'Qwen/Qwen3-VL-4B-Instruct': {}, + 'Qwen/Qwen3-VL-4B-Thinking': {}, + 'Qwen/Qwen3-VL-8B-Instruct': {}, + 'Qwen/Qwen3-VL-8B-Thinking': {}, 'microsoft/Phi-4-mini-instruct': {}, 'HuggingFaceTB/SmolLM2-135M-Instruct': {}, 'HuggingFaceTB/SmolLM2-360M-Instruct': {}, From 80bb331169fcf36f2c3ba0b18e4379d361a4a690 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 26 Oct 2025 06:01:33 +0000 Subject: [PATCH 4/7] Prompt enhance resizing and Qwen VL fix --- modules/interrogate/vqa.py | 6 +++--- scripts/prompt_enhance.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 0275773ce..750439f0c 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -187,11 +187,11 @@ def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str if model is None or loaded != repo: shared.log.debug(f'Interrogate load: vlm="{repo}"') model = None - if 'Qwen3' in repo: + if 'Qwen3-VL' in repo or 'Qwen3VL' in repo: cls_name = transformers.Qwen3VLForConditionalGeneration - elif '2.5' in repo: + elif 'Qwen2.5-VL' in repo or 'Qwen2_5_VL' in repo: cls_name = transformers.Qwen2_5_VLForConditionalGeneration - else: + elif 'Qwen2-VL' in repo or 'Qwen2VL' in repo: cls_name = transformers.Qwen2VLForConditionalGeneration model = cls_name.from_pretrained( repo, diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index 87396136f..ec5d0b3e6 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -168,7 +168,19 @@ class Script(scripts_manager.Script): load_args = { 'pretrained_model_name_or_path': model_repo if not gguf_args else model_gguf } if model_subfolder: load_args['subfolder'] = model_subfolder # Comma was incorrect here - self.llm = transformers.AutoModelForCausalLM.from_pretrained( + + # Determine model class based on model type + if 'Qwen3-VL' in model_repo or 'Qwen3VL' in model_repo: + model_cls = transformers.Qwen3VLForConditionalGeneration + # Use Qwen3-VL's optimized default attention + elif 'Qwen2.5-VL' in model_repo or 'Qwen2_5_VL' in model_repo: + model_cls = transformers.Qwen2_5_VLForConditionalGeneration + elif 'Qwen2-VL' in model_repo or 'Qwen2VL' in model_repo: + model_cls = transformers.Qwen2VLForConditionalGeneration + else: + model_cls = transformers.AutoModelForCausalLM + + self.llm = model_cls.from_pretrained( **load_args, trust_remote_code=True, torch_dtype=devices.dtype, @@ -302,6 +314,25 @@ class Script(scripts_manager.Script): except Exception: current_image = None + # Resize large images to match VQA performance (vision models are sensitive to resolution) + # Create a copy to avoid modifying the original image used by img2img + if current_image is not None and isinstance(current_image, Image.Image): + original_size = (current_image.width, current_image.height) + needs_resize = current_image.width > 768 or current_image.height > 768 + needs_rgb = current_image.mode != 'RGB' + + if needs_resize or needs_rgb: + # Copy the image before any modifications to preserve the original + current_image = current_image.copy() + + if needs_resize: + current_image.thumbnail((768, 768), Image.Resampling.LANCZOS) + debug_log(f'Prompt enhance: Resized image from {original_size} to {(current_image.width, current_image.height)}') + + if needs_rgb: + current_image = current_image.convert('RGB') + debug_log(f'Prompt enhance: Converted image to RGB mode') + has_system = system is not None and len(system) > 4 mode = 'custom' if has_system else '' From 1ac4214379982aedb33926bca858794002b8d9c4 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 26 Oct 2025 06:20:14 +0000 Subject: [PATCH 5/7] fix cls_name variable --- scripts/prompt_enhance.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index ec5d0b3e6..5c21fe27d 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -171,16 +171,16 @@ class Script(scripts_manager.Script): # Determine model class based on model type if 'Qwen3-VL' in model_repo or 'Qwen3VL' in model_repo: - model_cls = transformers.Qwen3VLForConditionalGeneration + cls_name = transformers.Qwen3VLForConditionalGeneration # Use Qwen3-VL's optimized default attention elif 'Qwen2.5-VL' in model_repo or 'Qwen2_5_VL' in model_repo: - model_cls = transformers.Qwen2_5_VLForConditionalGeneration + cls_name = transformers.Qwen2_5_VLForConditionalGeneration elif 'Qwen2-VL' in model_repo or 'Qwen2VL' in model_repo: - model_cls = transformers.Qwen2VLForConditionalGeneration + cls_name = transformers.Qwen2VLForConditionalGeneration else: - model_cls = transformers.AutoModelForCausalLM + cls_name = transformers.AutoModelForCausalLM - self.llm = model_cls.from_pretrained( + self.llm = cls_name.from_pretrained( **load_args, trust_remote_code=True, torch_dtype=devices.dtype, From 33f335a98c093e3032b688ee16a367d732be744b Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 26 Oct 2025 06:39:05 +0000 Subject: [PATCH 6/7] VQA class fix f-statement fix --- modules/interrogate/vqa.py | 2 ++ scripts/prompt_enhance.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index ead010828..fcb7464bc 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -193,6 +193,8 @@ def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str cls_name = transformers.Qwen2_5_VLForConditionalGeneration elif 'Qwen2-VL' in repo or 'Qwen2VL' in repo: cls_name = transformers.Qwen2VLForConditionalGeneration + else: + cls_name = transformers.AutoModelForCausalLM model = cls_name.from_pretrained( repo, torch_dtype=devices.dtype, diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index 5c21fe27d..5c709429d 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -331,7 +331,7 @@ class Script(scripts_manager.Script): if needs_rgb: current_image = current_image.convert('RGB') - debug_log(f'Prompt enhance: Converted image to RGB mode') + debug_log('Prompt enhance: Converted image to RGB mode') has_system = system is not None and len(system) > 4 mode = 'custom' if has_system else '' From 73588c2000e6aff3101e23ce4f647af2b9541de1 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 26 Oct 2025 06:52:39 +0000 Subject: [PATCH 7/7] Comment tweaks --- scripts/prompt_enhance.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index 5c709429d..1925a05d8 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -169,10 +169,8 @@ class Script(scripts_manager.Script): if model_subfolder: load_args['subfolder'] = model_subfolder # Comma was incorrect here - # Determine model class based on model type if 'Qwen3-VL' in model_repo or 'Qwen3VL' in model_repo: cls_name = transformers.Qwen3VLForConditionalGeneration - # Use Qwen3-VL's optimized default attention elif 'Qwen2.5-VL' in model_repo or 'Qwen2_5_VL' in model_repo: cls_name = transformers.Qwen2_5_VLForConditionalGeneration elif 'Qwen2-VL' in model_repo or 'Qwen2VL' in model_repo: @@ -314,7 +312,7 @@ class Script(scripts_manager.Script): except Exception: current_image = None - # Resize large images to match VQA performance (vision models are sensitive to resolution) + # Resize large images to match VQA performance (Qwen3-VL performance is sensitive to resolution) # Create a copy to avoid modifying the original image used by img2img if current_image is not None and isinstance(current_image, Image.Image): original_size = (current_image.width, current_image.height)