add gemma3

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -2,21 +2,30 @@
 
 ## Update for 2025-03-15
 
+### TODO
+  - Gemma3 requires `git+https://github.com/huggingface/transformers@v4.49.0-Gemma-3`
+
 - **Models**  
   - [THUDM CogView 4 6B](https://huggingface.co/THUDM/CogView4-6B)  
-    new foundation model for image generation based o T5-XXL text encoder and a flow-based diffusion transformer  
+    new foundation model for image generation based o GLM-4 text encoder and a flow-based diffusion transformer  
     fully supports offloading and on-the-fly quantization  
     simply select from *networks -> models -> reference*  
+    *note* cogview4 is compatible with flowmatching samplers  
   - New [zer0int CLiP-L](https://huggingface.co/zer0int/CLIP-Registers-Gated_MLP-ViT-L-14) models:  
     download text encoders into folder set in settings -> system paths -> text encoders (default is `models/Text-encoder`)  
     load using *settings -> text encoder*
     *tip*: add *sd_text_encoder* to your *settings -> user interface -> quicksettings* list to have it appear at the top of the ui  
+- **Caption/VLM**  
+  - [Google Gemma 3 4B](https://huggingface.co/google/gemma-3-4b-it)  
+    simply select from list of available models in caption tab  
+  - add option to set system prompt for vlm models that support it: *Gemma, Smol, Qwen*
 - **Wiki/Docs**
   - updated [Models](https://github.com/vladmandic/sdnext/wiki/Models) info  
   - Updated SD3  
 - **Other**
   - add remote vae info to metadata, thanks @iDeNoh  
   - add quantization support to **CogView-3Plus**
+  - update `diffusers`  
 - **Fixes**
   - fix installer not starting when older version of `rich` is installed  
   - fix circular imports when debug flags are enabled  

installer.py

@@ -538,7 +538,7 @@ def check_diffusers():
     t_start = time.time()
     if args.skip_all or args.skip_git:
         return
-    sha = 'b75b204a584e29ebf4e80a61be11458e9ed56e3e' # diffusers commit hash
+    sha = '82188cef0487837b8c70fc3f36ea63c05c85f341' # diffusers commit hash
     pkg = pkg_resources.working_set.by_key.get('diffusers', None)
     minor = int(pkg.version.split('.')[1] if pkg is not None else 0)
     cur = opts.get('diffusers_version', '') if minor > 0 else ''

modules/api/endpoints.py

@@ -113,7 +113,7 @@ def post_vqa(req: models.ReqVQA):
     image = helpers.decode_base64_to_image(req.image)
     image = image.convert('RGB')
     from modules.interrogate import vqa
-    answer = vqa.interrogate(req.question, '', image, req.model)
+    answer = vqa.interrogate(req.question, req.system, '', image, req.model)
     return models.ResVQA(answer=answer)
 
 def post_unload_checkpoint():

modules/interrogate/interrogate.py

@@ -28,7 +28,7 @@ def interrogate(image):
     elif shared.opts.interrogate_default_type == 'VLM':
         shared.log.info(f'Interrogate: type={shared.opts.interrogate_default_type} vlm="{shared.opts.interrogate_vlm_model}" prompt="{shared.opts.interrogate_vlm_prompt}"')
         from modules.interrogate import vqa
-        prompt = vqa.interrogate(image=image, model_name=shared.opts.interrogate_vlm_model, question=shared.opts.interrogate_vlm_prompt, prompt=None)
+        prompt = vqa.interrogate(image=image, model_name=shared.opts.interrogate_vlm_model, question=shared.opts.interrogate_vlm_prompt, prompt=None, system_prompt=shared.opts.interrogate_vlm_system)
         shared.log.debug(f'Interrogate: time={time.time()-t0:.2f} answer="{prompt}"')
         return prompt
     else:

modules/interrogate/vqa.py

@@ -22,19 +22,20 @@ vlm_models = {
     "CogFlorence 2.0 Large": "thwri/CogFlorence-2-Large-Freeze", # 1.6GB
     "CogFlorence 2.2 Large": "thwri/CogFlorence-2.2-Large", # 1.6GB
     "Moondream 2": "vikhyatk/moondream2", # 3.7GB
+    "Google Gemma 3 4B": "google/gemma-3-4b-it",
+    "Google Pix Textcaps": "google/pix2struct-textcaps-base", # 1.1GB
+    "Google PaliGemma 2 3B": "google/paligemma2-3b-pt-224",
     "Alibaba Qwen VL2 2B": "Qwen/Qwen2-VL-2B-Instruct",
     "Huggingface Smol VL2 0.5B": "HuggingFaceTB/SmolVLM-500M-Instruct",
     "Huggingface Smol VL2 2B": "HuggingFaceTB/SmolVLM-Instruct",
     "Salesforce BLIP Base": "Salesforce/blip-vqa-base", # 1.5GB
     "Salesforce BLIP Large": "Salesforce/blip-vqa-capfilt-large", # 1.5GB
-    "Google Pix Textcaps": "google/pix2struct-textcaps-base", # 1.1GB
     "Microsoft GIT TextCaps Base": "microsoft/git-base-textcaps", # 0.7GB
     "Microsoft GIT VQA Base": "microsoft/git-base-vqav2", # 0.7GB
     "Microsoft GIT VQA Large": "microsoft/git-large-vqav2", # 1.6GB
     "ToriiGate 0.4 2B": "Minthy/ToriiGate-v0.4-2B",
     "ToriiGate 0.4 7B": "Minthy/ToriiGate-v0.4-7B",
     "ViLT Base": "dandelin/vilt-b32-finetuned-vqa", # 0.5GB
-    "Google PaliGemma 2 3B": "google/paligemma2-3b-pt-224",
     "JoyCaption": "fancyfeast/llama-joycaption-alpha-two-hf-llava", # 0.7GB
     "JoyTag": "fancyfeast/joytag", # 17.4GB
     "AIDC Ovis2 1B": "AIDC-AI/Ovis2-1B",
@@ -44,6 +45,7 @@ vlm_models = {
     # "DeepSeek VL2 Tiny": "deepseek-ai/deepseek-vl2-tiny", # broken
     # "nVidia Eagle 2 1B": "nvidia/Eagle2-1B", # not compatible with latest transformers
 }
+vlm_system = 'You are image captioning expert, creative, unbiased and uncensored.'
 vlm_prompts = [
     '<CAPTION>',
     '<DETAILED_CAPTION>',
@@ -70,6 +72,7 @@ def b64(image):
 
 
 def clean(response, question):
+    strip = ['---', '\r', '\t', '**', '"', '“', '”', 'Assistant:', 'Caption:']
     if isinstance(response, dict):
         if 'task' in response:
             response = response['task']
@@ -81,12 +84,10 @@ def clean(response, question):
     question = question.replace('<', '').replace('>', '').replace('_', ' ')
     if question in response:
         response = response.split(question, 1)[1]
-    response = response.replace('\n', '').replace('\r', '').replace('\t', '').strip()
-    if response.startswith('"'):
-        response = response[1:]
-    if response.endswith('"'):
-        response = response[:-1]
-    response = response.replace('Assistant:', '').strip()
+    while any(s in response for s in strip):
+        for s in strip:
+            response = response.replace(s, '')
+    response = response.replace('\n\n', '\n').replace('  ', ' ').replace('*  ', '- ').strip()
     return response
 
 
@@ -106,7 +107,7 @@ def get_kwargs():
     return kwargs
 
 
-def qwen(question: str, image: Image.Image, repo: str = None):
+def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str = None):
     global processor, model, loaded # pylint: disable=global-statement
     if model is None or loaded != repo:
         shared.log.debug(f'Interrogate load: vlm="{repo}"')
@@ -118,12 +119,11 @@ def qwen(question: str, image: Image.Image, repo: str = None):
         loaded = repo
     model = model.to(devices.device, devices.dtype)
     question = question.replace('<', '').replace('>', '').replace('_', ' ')
+    system_prompt = system_prompt or shared.opts.vlm_system
     conversation = [
         {
             "role": "system",
-            "content": [
-                {"type": "text", "text": "You are image captioning expert, creative, unbiased and uncensored."}
-            ],
+            "content": [{"type": "text", "text": system_prompt}],
         },
         {
             "role": "user",
@@ -134,7 +134,6 @@ def qwen(question: str, image: Image.Image, repo: str = None):
         }
     ]
     text_prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
-    # '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>Describe this image.<|im_end|>\n<|im_start|>assistant\n'
     inputs = processor(text=[text_prompt], images=[image], padding=True, return_tensors="pt")
     inputs = inputs.to(devices.device, devices.dtype)
     output_ids = model.generate(
@@ -149,6 +148,47 @@ def qwen(question: str, image: Image.Image, repo: str = None):
     return response
 
 
+def gemma(question: str, image: Image.Image, repo: str = None, system_prompt: str = None):
+    global processor, model, loaded # pylint: disable=global-statement
+    if model is None or loaded != repo:
+        shared.log.debug(f'Interrogate load: vlm="{repo}"')
+        model = transformers.Gemma3ForConditionalGeneration.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
+        processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
+        loaded = repo
+    model = model.to(devices.device, devices.dtype)
+    question = question.replace('<', '').replace('>', '').replace('_', ' ')
+    system_prompt = system_prompt or shared.opts.vlm_system
+    conversation = [
+        {
+            "role": "system",
+            "content": [{"type": "text", "text": system_prompt}]
+        },
+        {
+            "role": "user",
+            "content": [
+                {"type": "image", "image": b64(image)},
+                {"type": "text", "text": question}
+            ]
+        }
+    ]
+    inputs = processor.apply_chat_template(
+        conversation,
+        add_generation_prompt=True,
+        tokenize=True,
+        return_dict=True,
+        return_tensors="pt",
+    ).to(device=devices.device, dtype=devices.dtype)
+    input_len = inputs["input_ids"].shape[-1]
+    with devices.inference_context():
+        generation = model.generate(
+            **inputs,
+            **get_kwargs(),
+        )
+        generation = generation[0][input_len:]
+    response = processor.decode(generation, skip_special_tokens=True)
+    return response
+
+
 def paligemma(question: str, image: Image.Image, repo: str = None):
     global processor, model, loaded # pylint: disable=global-statement
     if model is None or loaded != repo:
@@ -219,7 +259,7 @@ def ovis(question: str, image: Image.Image, repo: str = None):
     return response
 
 
-def smol(question: str, image: Image.Image, repo: str = None):
+def smol(question: str, image: Image.Image, repo: str = None, system_prompt: str = None):
     global processor, model, loaded # pylint: disable=global-statement
     if model is None or loaded != repo:
         shared.log.debug(f'Interrogate load: vlm="{repo}"')
@@ -233,12 +273,11 @@ def smol(question: str, image: Image.Image, repo: str = None):
         loaded = repo
     model.to(devices.device, devices.dtype)
     question = question.replace('<', '').replace('>', '').replace('_', ' ')
+    system_prompt = system_prompt or shared.opts.vlm_system
     conversation = [
         {
             "role": "system",
-            "content": [
-                {"type": "text", "text": "You are image captioning expert, creative, unbiased and uncensored."}
-            ],
+            "content": [{"type": "text", "text": system_prompt}],
         },
         {
             "role": "user",
@@ -249,7 +288,6 @@ def smol(question: str, image: Image.Image, repo: str = None):
         }
     ]
     text_prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
-    # '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>Describe this image.<|im_end|>\n<|im_start|>assistant\n'
     inputs = processor(text=text_prompt, images=[image], padding=True, return_tensors="pt")
     inputs = inputs.to(devices.device, devices.dtype)
     output_ids = model.generate(
@@ -410,7 +448,7 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str
     return response
 
 
-def interrogate(question, prompt, image, model_name, quiet:bool=False):
+def interrogate(question, system_prompt, prompt, image, model_name, quiet:bool=False):
     if not quiet:
         shared.state.begin('Interrogate')
     t0 = time.time()
@@ -457,9 +495,9 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False):
         elif 'florence' in vqa_model.lower():
             answer = florence(question, image, vqa_model)
         elif 'qwen' in vqa_model.lower() or 'torii' in vqa_model.lower():
-            answer = qwen(question, image, vqa_model)
+            answer = qwen(question, image, vqa_model, system_prompt)
         elif 'smol' in vqa_model.lower():
-            answer = smol(question, image, vqa_model)
+            answer = smol(question, image, vqa_model, system_prompt)
         elif 'joytag' in vqa_model.lower():
             from modules.interrogate import joytag
             answer = joytag.predict(image)
@@ -471,6 +509,8 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False):
             answer = deepseek.predict(question, image, vqa_model)
         elif 'paligemma' in vqa_model.lower():
             answer = paligemma(question, image, vqa_model)
+        elif 'gemma' in vqa_model.lower():
+            answer = gemma(question, image, vqa_model, system_prompt)
         elif 'ovis' in vqa_model.lower():
             answer = ovis(question, image, vqa_model)
         else:
@@ -481,7 +521,9 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False):
     if shared.opts.interrogate_offload and model is not None:
         model.to(devices.cpu)
     devices.torch_gc()
+    print('HERE1', answer)
     answer = clean(answer, question)
+    print('HERE2', answer)
     t1 = time.time()
     if not quiet:
         shared.log.debug(f'Interrogate: type=vlm model="{model_name}" repo="{vqa_model}" args={get_kwargs()} time={t1-t0:.2f}')
@@ -489,7 +531,7 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False):
     return answer
 
 
-def batch(model_name, batch_files, batch_folder, batch_str, question, prompt, write, append, recursive):
+def batch(model_name, system_prompt, batch_files, batch_folder, batch_str, question, prompt, write, append, recursive):
     class BatchWriter:
         def __init__(self, folder, mode='w'):
             self.folder = folder
@@ -536,7 +578,7 @@ def batch(model_name, batch_files, batch_folder, batch_str, question, prompt, wr
                 if shared.state.interrupted:
                     break
                 image = Image.open(file)
-                prompt = interrogate(question, prompt, image, model_name, quiet=True)
+                prompt = interrogate(question, system_prompt, prompt, image, model_name, quiet=True)
                 prompts.append(prompt)
                 if write:
                     writer.add(file, prompt)

modules/model_cogview.py

@@ -76,7 +76,7 @@ def load_cogview4(checkpoint_info, diffusers_load_config={}):
     )
 
     diffusers_load_config, quant_args = load_common(diffusers_load_config, module='Text Encoder')
-    text_encoder = transformers.T5EncoderModel.from_pretrained(
+    text_encoder = transformers.AutoModelForCausalLM.from_pretrained(
         repo_id,
         subfolder="text_encoder",
         cache_dir=shared.opts.diffusers_dir,
@@ -91,5 +91,9 @@ def load_cogview4(checkpoint_info, diffusers_load_config={}):
         cache_dir=shared.opts.diffusers_dir,
         **diffusers_load_config,
     )
+    if shared.opts.diffusers_eval:
+        pipe.text_encoder.eval()
+        pipe.transformer.eval()
+    pipe.enable_model_cpu_offload() # TODO cogview4: balanced offload does not work for GlmModel
     devices.torch_gc()
     return pipe

modules/pixelsmith/pixelsmith_pipeline.py

@@ -133,7 +133,6 @@ class PAGIdentitySelfAttnProcessor:
         value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
 
         # the output of sdp = (batch, num_heads, seq_len, head_dim)
-        # TODO: add support for attn.scale when we move to Torch 2.1
         hidden_states_org = F.scaled_dot_product_attention(
             query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
         )
@@ -248,7 +247,6 @@ class PAGCFGIdentitySelfAttnProcessor:
         value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
 
         # the output of sdp = (batch, num_heads, seq_len, head_dim)
-        # TODO: add support for attn.scale when we move to Torch 2.1
         hidden_states_org = F.scaled_dot_product_attention(
             query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
         )

modules/processing_vae.py

@@ -139,7 +139,7 @@ def full_vae_decode(latents, model):
     if latents_mean and latents_std:
         latents_mean = (torch.tensor(latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype))
         latents_std = (torch.tensor(latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype))
-        latents = latents * latents_std / scaling_factor + latents_mean
+        latents = ((latents * latents_std) / scaling_factor) + latents_mean
     else:
         latents = latents / scaling_factor
     if shift_factor:

modules/schedulers/scheduler_dpm_flowmatch.py

@@ -230,6 +230,7 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
         device: Union[str, torch.device] = None,
         sigmas: Optional[List[float]] = None,
         mu: Optional[float] = None,
+        timesteps: Optional[torch.Tensor] = None,
     ):
         """
         Sets the discrete timesteps used for the diffusion chain (to be run before inference).
@@ -355,12 +356,12 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
                 sigma_min = sigmas[-1]
                 sigmas = np.linspace(1.0, sigma_min, num_inference_steps)
             sigmas = torch.from_numpy(sigmas).to(dtype=torch.float64, device=device)
-        
+
         if self.config.use_dynamic_shifting:
             sigmas = self.time_shift(mu, 1.0, sigmas)
         else:
             sigmas = self.config.shift * sigmas / (1 + (self.config.shift - 1) * sigmas)
-        
+
         timesteps = sigmas * self.config.num_train_timesteps
         self.timesteps = timesteps.to(device=device)
         self.sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)])
@@ -517,7 +518,7 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
                     model_output = sample - sigma * model_output
                     d = (sample - model_output) / sigma
                     dt = sigma_next - sigma
-                    sample = sample + d * dt                
+                    sample = sample + d * dt
                 else:
                     # DPM-Solver2
                     sigma_mid = sigma.log().lerp(sigma_next.log(), 0.5).exp()
@@ -596,7 +597,7 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
         elif self.config.algorithm_type == "dpmsolver++2M":
             if self.config.solver_order == 2:
                 t, t_next = t_fn(sigma), t_fn(sigma_next)
-                h = t_next - t 
+                h = t_next - t
                 if self.model_outputs[-2] is None or sigma_next == 0:
                     sample = (sigma_fn(t_next) / sigma_fn(t)) * sample - (-h).expm1() * model_output
                 else:
@@ -703,7 +704,7 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
                     if self.config.use_noise_sampler:
                         sample = sample + self.noise_sampler(sigma_fn(t), sigma_fn(t_next)) * self.config.s_noise * su
                     else:
-                        sample = sample + noise * self.config.s_noise * su                    
+                        sample = sample + noise * self.config.s_noise * su
                     del x_2
                     del denoised_2
                     del d
@@ -745,13 +746,13 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
                     t, s = -sigma.log(), -sigma_next.log()
                     h = s - t
                     h_eta = h * 2
-                     
+
                     # 3. Delta timestep
                     dt = sigma_next - sigma
                     sample = sample + model_output * dt
 
                     sample = torch.exp(-h_eta) * sample + (-h_eta).expm1().neg() * model_output
-                
+
                     if self.h_2 is not None:
                         r0 = self.h_1 / h
                         r1 = self.h_2 / h
@@ -780,13 +781,13 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
                         sample = sample + self.noise_sampler(sigma, sigma_next) * sigma_next * (-2 * h).expm1().neg().sqrt() * self.config.s_noise
                     else:
                         sample = sample + noise * sigma_next * (-2 * h).expm1().neg().sqrt() * self.config.s_noise
-            
+
                     self.h_2 = self.h_1
                     self.h_1 = h
             if not self.config.use_noise_sampler and noise is not None:
                 del noise
         prev_sample = sample
-     
+
         # Cast sample back to expected dtype
         prev_sample = prev_sample.to(model_output.dtype)
 

modules/sd_offload.py

@@ -10,8 +10,9 @@ from modules.timer import process as process_timer
 
 
 debug_move = shared.log.trace if os.environ.get('SD_MOVE_DEBUG', None) is not None else lambda *args, **kwargs: None
-should_offload = ['sc', 'sd3', 'f1', 'hunyuandit', 'auraflow', 'omnigen', 'hunyuanvideo', 'cogvideox', 'mochi']
+should_offload = ['sc', 'sd3', 'f1', 'hunyuandit', 'auraflow', 'omnigen', 'hunyuanvideo', 'cogvideox', 'mochi', 'cogview4']
 offload_hook_instance = None
+balanced_offload_exclude = ['OmniGenPipeline', 'CogView4Pipeline']
 
 
 def get_signature(cls):
@@ -193,8 +194,7 @@ def apply_balanced_offload(sd_model, exclude=[]):
     if sd_model is None:
         return sd_model
     t0 = time.time()
-    excluded = ['OmniGenPipeline']
-    if sd_model.__class__.__name__ in excluded:
+    if sd_model.__class__.__name__ in balanced_offload_exclude:
         return sd_model
     cached = True
     checkpoint_name = sd_model.sd_checkpoint_info.name if getattr(sd_model, "sd_checkpoint_info", None) is not None else None

modules/sd_samplers.py

@@ -79,7 +79,7 @@ def create_sampler(name, model):
         shared.log.debug(f'Sampler: "{name}" config={config.options}')
         return sampler
     elif shared.native:
-        FlowModels = ['Flux', 'StableDiffusion3', 'Lumina', 'AuraFlow', 'Sana', 'HunyuanVideoPipeline']
+        FlowModels = ['Flux', 'StableDiffusion3', 'Lumina', 'AuraFlow', 'Sana', 'HunyuanVideoPipeline', 'CogView4Pipeline']
         if 'KDiffusion' in model.__class__.__name__:
             return None
         if not any(x in model.__class__.__name__ for x in FlowModels) and 'FlowMatch' in name:

modules/shared.py

@@ -19,7 +19,7 @@ from modules.dml import memory_providers, default_memory_provider, directml_do_h
 from modules.onnx_impl import initialize_onnx, execution_providers
 from modules.memstats import memory_stats, ram_stats # pylint: disable=unused-import
 from modules.interrogate.openclip import caption_models, caption_types, get_clip_models, refresh_clip_models, category_types
-from modules.interrogate.vqa import vlm_models, vlm_prompts
+from modules.interrogate.vqa import vlm_models, vlm_prompts, vlm_system
 from modules.ui_components import DropdownEditable
 from modules.options import OptionInfo
 import modules.memmon
@@ -855,6 +855,7 @@ options_templates.update(options_section(('interrogate', "Interrogate"), {
     "interrogate_vlm_sep": OptionInfo("<h2>VLM</h2>", "", gr.HTML),
     "interrogate_vlm_model": OptionInfo(list(vlm_models)[0], "VLM: default model", gr.Dropdown, {"choices": list(vlm_models)}),
     "interrogate_vlm_prompt": OptionInfo(vlm_prompts[2], "VLM: default prompt", DropdownEditable, {"choices": vlm_prompts }),
+    "interrogate_vlm_system": OptionInfo(vlm_system, "VLM: default prompt"),
     "interrogate_vlm_num_beams": OptionInfo(3, "VLM: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}),
     "interrogate_vlm_max_length": OptionInfo(512, "VLM: max length", gr.Slider, {"minimum": 1, "maximum": 4096, "step": 1, "visible": False}),
     "interrogate_vlm_do_sample": OptionInfo(False, "VLM: use sample method"),

modules/ui_caption.py

@@ -35,6 +35,8 @@ def create_ui():
             with gr.Tabs(elem_id="mode_caption"):
                 with gr.Tab("VLM Caption", elem_id="tab_vlm_caption"):
                     from modules.interrogate import vqa
+                    with gr.Row():
+                        vlm_system = gr.Textbox(label="System prompt", value=vqa.vlm_system, lines=1, elem_id='vlm_system')
                     with gr.Row():
                         vlm_question = gr.Dropdown(label="Predefined question", allow_custom_value=False, choices=vqa.vlm_prompts, value=vqa.vlm_prompts[2], elem_id='vlm_question')
                     with gr.Row():
@@ -114,7 +116,7 @@ def create_ui():
                         btn_clip_analyze_img = gr.Button("Analyze", variant='primary', elem_id="btn_clip_analyze_img")
         with gr.Column(variant='compact', elem_id='interrogate_output'):
             with gr.Row(elem_id='interrogate_output_prompt'):
-                prompt = gr.Textbox(label="Answer", lines=8, placeholder="ai generated image description")
+                prompt = gr.Textbox(label="Answer", lines=12, placeholder="ai generated image description")
             with gr.Row(elem_id='interrogate_output_classes'):
                 medium = gr.Label(elem_id="interrogate_label_medium", label="Medium", num_top_classes=5, visible=False)
                 artist = gr.Label(elem_id="interrogate_label_artist", label="Artist", num_top_classes=5, visible=False)
@@ -127,8 +129,8 @@ def create_ui():
     btn_clip_interrogate_img.click(openclip.interrogate_image, inputs=[image, clip_model, blip_model, clip_mode], outputs=[prompt])
     btn_clip_analyze_img.click(openclip.analyze_image, inputs=[image, clip_model, blip_model], outputs=[medium, artist, movement, trending, flavor])
     btn_clip_interrogate_batch.click(fn=openclip.interrogate_batch, inputs=[clip_batch_files, clip_batch_folder, clip_batch_str, clip_model, blip_model, clip_mode, clip_save_output, clip_save_append, clip_folder_recursive], outputs=[prompt])
-    btn_vlm_caption.click(fn=vqa.interrogate, inputs=[vlm_question, vlm_prompt, image, vlm_model], outputs=[prompt])
-    btn_vlm_caption_batch.click(fn=vqa.batch, inputs=[vlm_model, vlm_batch_files, vlm_batch_folder, vlm_batch_str, vlm_question, vlm_prompt, vlm_save_output, vlm_save_append, vlm_folder_recursive], outputs=[prompt])
+    btn_vlm_caption.click(fn=vqa.interrogate, inputs=[vlm_question, vlm_system, vlm_prompt, image, vlm_model], outputs=[prompt])
+    btn_vlm_caption_batch.click(fn=vqa.batch, inputs=[vlm_model, vlm_system, vlm_batch_files, vlm_batch_folder, vlm_batch_str, vlm_question, vlm_prompt, vlm_save_output, vlm_save_append, vlm_folder_recursive], outputs=[prompt])
 
     for tabname, button in copy_interrogate_buttons.items():
         generation_parameters_copypaste.register_paste_params_button(generation_parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=prompt, source_image_component=image,))

requirements.txt

@@ -41,18 +41,18 @@ torchsde==0.2.6
 antlr4-python3-runtime==4.9.3
 requests==2.32.3
 tqdm==4.67.1
-accelerate==1.3.0
+accelerate==1.5.2
 opencv-contrib-python-headless==4.9.0.80
 einops==0.4.1
 gradio==3.43.2
-huggingface_hub==0.28.1
+huggingface_hub==0.29.3
 numexpr==2.8.8
 numpy==1.26.4
 numba==0.59.1
 protobuf==4.25.3
 pytorch_lightning==1.9.4
 tokenizers==0.21.0
-transformers==4.48.3
+transformers==4.49.0
 urllib3==1.26.19
 Pillow==10.4.0
 timm==0.9.16