modernize clip interrogate

Signed-off-by: Vladimir Mandic <mandic00@live.com>
2026-09-19 09:14:35 +02:00 · 2025-02-16 19:37:09 -05:00
parent 5e12985c52
commit a4b3dc269e
5 changed files with 171 additions and 137 deletions
@@ -1,14 +1,12 @@
 # Change Log for SD.Next

-## Update for 2025-02-14
+## Update for 2025-02-16

 ### TODO

 - VLM ModernUI support
- CLiP Move settings
- CLiP Batch progress bar

-### Highlight for 2025-02-14
+### Highlight for 2025-02-16

 We're back with another update with over 50 commits!  
 - Starting with massive UI update with full [localization](https://vladmandic.github.io/sdnext-docs/Locale/) for 8 languages 
@@ -19,13 +17,13 @@ We're back with another update with over 50 commits!
 - And new **Mixture-of-Diffusers** regional tiling pipeline  
 - Follow-up to last weeks **interrogate/captioning** rewrite  
  now with redesigned captioning UI and  
-  added **JoyTag**, **JoyCaption**, **Google PaliGemma**, **ToriiGate 0.4** to list of supported models  
+  added **JoyTag**, **JoyCaption**, **PaliGemma**, **ToriiGate**, **Ovis2** to list of supported models  
 - Some changes to **prompt parsing** to allow more control as well as more flexibility when mouting SDNext server to custom URL  
 - Of course, cumulative fixes...

 *...and more* - see [changelog](https://github.com/vladmandic/sdnext/blob/dev/CHANGELOG.md) for full details!  

-### Details for 2025-02-14
+### Details for 2025-02-16

 - **User Interface**  
  - **Hints**  
@@ -64,19 +62,25 @@ We're back with another update with over 50 commits!
  - Redesigned captioning UI  
    split from Process tab into separate tab  
    split `clip` vs `vlm` models processing  
-    direct *send-to* buttons on all tabs  
-  - Add VLM advanced params: max-tokens, num-beams, temperature, top-k, top-p, do-sample  
-    params are saved in `config.json` and used when using quick interrogate  
+    direct *send-to* buttons on all tabs: txt/img/ctrl->process/caption, process/caption->txt/img/ctrl  
+  - Advanced params:
+    VLM: *max-tokens, num-beams, temperature, top-k, top-p, do-sample*  
+    CLiP: *min-length, max-length, chunk-size, min-flavors, max-flavors, flavor-count, num-beams*  
+    params are auto-saved in `config.json` and used when using quick interrogate  
    params that are set to 0 mean use model defaults  
-  - Add VLM batch processing  
+  - Batch processing: VLM and CLiP  
    for example, can be used to caption your training dataset in one go  
    add option to append to captions file, can be used to run multiple captioning models in sequence  
    add progress bar  
  - Add additional VLM models:  
    [JoyTag](https://huggingface.co/fancyfeast/joytag)  
    [JoyCaption 2](https://huggingface.co/fancyfeast/llama-joycaption-alpha-two-hf-llava)  
-    [Google PaliGemma 2](https://huggingface.co/google/paligemma2-3b-pt-224)  
-    [ToriiGate 0.4 7B](https://huggingface.co/Minthy/ToriiGate-v0.4-7B)  
+    [Google PaliGemma 2](https://huggingface.co/google/paligemma2-3b-pt-224) 3B  
+    [ToriiGate 0.4](https://huggingface.co/Minthy/ToriiGate-v0.4-7B) 7B  
+    [AIDC Ovis2](https://huggingface.co/AIDC-AI/Ovis2-1B) 1B/2B/4B  
+  - *Note* some models require `flash-attn` to be installed  
+    due to binary/build dependencies, it should not be done automatically,  
+    see [flash-attn](https://github.com/Dao-AILab/flash-attention) for installation instructions  
 - **Docker**  
  - updated **CUDA** receipe to `torch==2.6.0` with `cuda==12.6` and add prebuilt image  
  - added **ROCm** receipe and prebuilt image  
@@ -13,15 +13,6 @@ from torchvision.transforms.functional import InterpolationMode
 from modules import devices, paths, shared, lowvram, errors, sd_models


-config = {
-    "caption_max_length": 74,
-    "chunk_size": 1024,
-    "flavor_intermediate_count": 1024,
-    "min_flavors": 2,
-    "max_flavors": 8,
-    "clip_offload": True,
-    "caption_offload": True,
-}
 caption_models = {
    'blip-base': 'Salesforce/blip-image-captioning-base',
    'blip-large': 'Salesforce/blip-image-captioning-large',
@@ -244,21 +235,13 @@ class BatchWriter:
            self.file.close()


-def update_interrogate_params(caption_max_length:int=None, chunk_size:int=None, min_flavors:int=None, max_flavors:int=None, flavor_intermediate_count:int=None):
-    config["caption_max_length"] = int(caption_max_length or shared.opts.interrogate_clip_max_length)
-    config["clip_offload"] = shared.opts.interrogate_offload
-    config["caption_offload"] = shared.opts.interrogate_offload
-    config["min_flavors"] = int(min_flavors or shared.opts.interrogate_clip_min_flavors)
-    config["max_flavors"] = int(max_flavors or shared.opts.interrogate_clip_max_flavors)
-    if chunk_size is not None:
-        config["chunk_size"] = int(chunk_size)
-    if flavor_intermediate_count is not None:
-        config["flavor_intermediate_count"] = int(flavor_intermediate_count)
+def update_interrogate_params():
    if ci is not None:
-        ci.config.caption_max_length = config["caption_max_length"]
-        ci.config.chunk_size = config["chunk_size"]
-        ci.config.flavor_intermediate_count = config["flavor_intermediate_count"]
-    shared.log.debug(f'Interrogate: type={shared.opts.interrogate_default_type} config={config}')
+        ci.caption_max_length=shared.opts.interrogate_clip_max_length,
+        ci.chunk_size=shared.opts.interrogate_clip_chunk_size,
+        ci.flavor_intermediate_count=shared.opts.interrogate_clip_flavor_count,
+        ci.clip_offload=shared.opts.interrogate_offload,
+        ci.caption_offload=shared.opts.interrogate_offload,


 def get_clip_models():
@@ -288,11 +271,11 @@ def load_interrogator(clip_model, blip_model):
            clip_model_name=clip_model,
            caption_model_name=blip_model,
            quiet=True,
-            caption_max_length=config['caption_max_length'],
-            chunk_size=config['chunk_size'],
-            flavor_intermediate_count=config['flavor_intermediate_count'],
-            clip_offload=config['clip_offload'],
-            caption_offload=config['caption_offload'],
+            caption_max_length=shared.opts.interrogate_clip_max_length,
+            chunk_size=shared.opts.interrogate_clip_chunk_size,
+            flavor_intermediate_count=shared.opts.interrogate_clip_flavor_count,
+            clip_offload=shared.opts.interrogate_offload,
+            caption_offload=shared.opts.interrogate_offload,
        )
        ci = clip_interrogator.Interrogator(interrogator_config)
    elif clip_model != ci.config.clip_model_name or blip_model != ci.config.caption_model_name:
@@ -322,15 +305,15 @@ def interrogate(image, mode, caption=None):
        return ''
    image = image.convert("RGB")
    if mode == 'best':
-        prompt = ci.interrogate(image, caption=caption, min_flavors=config["min_flavors"], max_flavors=config["max_flavors"])
+        prompt = ci.interrogate(image, caption=caption, min_flavors=shared.opts.interrogate_clip_min_flavors, max_flavors=shared.opts.interrogate_clip_max_flavors, )
    elif mode == 'caption':
        prompt = ci.generate_caption(image) if caption is None else caption
    elif mode == 'classic':
-        prompt = ci.interrogate_classic(image, caption=caption, max_flavors=config["max_flavors"])
+        prompt = ci.interrogate_classic(image, caption=caption, max_flavors=shared.opts.interrogate_clip_max_flavors)
    elif mode == 'fast':
-        prompt = ci.interrogate_fast(image, caption=caption, max_flavors=config["max_flavors"])
+        prompt = ci.interrogate_fast(image, caption=caption, max_flavors=shared.opts.interrogate_clip_max_flavors)
    elif mode == 'negative':
-        prompt = ci.interrogate_negative(image, max_flavors=config["max_flavors"])
+        prompt = ci.interrogate_negative(image, max_flavors=shared.opts.interrogate_clip_max_flavors)
    else:
        raise RuntimeError(f"Unknown mode {mode}")
    return prompt
@@ -342,7 +325,7 @@ def interrogate_image(image, clip_model, blip_model, mode):
        if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram):
            lowvram.send_everything_to_cpu()
            devices.torch_gc()
-        if shared.native:
+        if shared.native and shared.sd_loaded:
            sd_models.apply_balanced_offload(shared.sd_model)
        load_interrogator(clip_model, blip_model)
        image = image.convert('RGB')
@@ -365,50 +348,35 @@ def interrogate_batch(batch_files, batch_folder, batch_str, clip_model, blip_mod
    if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str):
        files += [os.path.join(batch_str, f) for f in os.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))]
    if len(files) == 0:
-        shared.log.error('Interrogate batch no images')
+        shared.log.warning('Interrogate batch: type=clip no images')
        return ''
-    shared.state.begin('Batch interrogate')
+    shared.state.begin('Interrogate batch')
    prompts = []
-    try:
-        if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram):
-            lowvram.send_everything_to_cpu()
-            devices.torch_gc()
-        load_interrogator(clip_model, blip_model)
-        shared.log.info(f'Interrogate batch: images={len(files)} mode={mode} config={ci.config}')
-        captions = []
-        # first pass: generate captions
+
+    load_interrogator(clip_model, blip_model)
+    if write:
+        file_mode = 'w' if not append else 'a'
+        writer = BatchWriter(os.path.dirname(files[0]), mode=file_mode)
+    import rich.progress as rp
+    pbar = rp.Progress(rp.TextColumn('[cyan]Caption:'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console)
+    with pbar:
+        task = pbar.add_task(total=len(files), description='starting...')
        for file in files:
-            caption = ""
+            pbar.update(task, advance=1, description=file)
            try:
                if shared.state.interrupted:
                    break
                image = Image.open(file).convert('RGB')
-                caption = ci.generate_caption(image)
-            except Exception as e:
-                shared.log.error(f'Interrogate caption: {e}')
-            finally:
-                captions.append(caption)
-        # second pass: interrogate
-        if write:
-            mode = 'w' if not append else 'a'
-            writer = BatchWriter(os.path.dirname(files[0]), mode=mode)
-        for idx, file in enumerate(files):
-            try:
-                if shared.state.interrupted:
-                    break
-                image = Image.open(file).convert('RGB')
-                prompt = interrogate(image, mode, caption=captions[idx])
+                prompt = interrogate(image, mode)
                prompts.append(prompt)
                if write:
                    writer.add(file, prompt)
            except OSError as e:
                shared.log.error(f'Interrogate batch: {e}')
-        if write:
-            writer.close()
-        ci.config.quiet = False
-        unload_clip_model()
-    except Exception as e:
-        shared.log.error(f'Interrogate batch: {e}')
+    if write:
+        writer.close()
+    ci.config.quiet = False
+    unload_clip_model()
    shared.state.end()
    return '\n\n'.join(prompts)

@@ -7,7 +7,7 @@ import torch
 import transformers
 import transformers.dynamic_module_utils
 from PIL import Image
-from modules import shared, devices, errors
+from modules import shared, devices, errors, sd_models

 processor = None
 model = None
@@ -37,6 +37,9 @@ vlm_models = {
    "Google PaliGemma 2 3B": "google/paligemma2-3b-pt-224",
    "JoyCaption": "fancyfeast/llama-joycaption-alpha-two-hf-llava", # 0.7GB
    "JoyTag": "fancyfeast/joytag", # 17.4GB
+    "AIDC Ovis2 1B": "AIDC-AI/Ovis2-1B",
+    "AIDC Ovis2 2B": "AIDC-AI/Ovis2-2B",
+    "AIDC Ovis2 4B": "AIDC-AI/Ovis2-4B",
    # "DeepSeek VL2 Tiny": "deepseek-ai/deepseek-vl2-tiny", # broken
    # "nVidia Eagle 2 1B": "nvidia/Eagle2-1B", # not compatible with latest transformers
 }
@@ -74,7 +77,7 @@ def clean(response, question):
        response = json.dumps(response)
    if isinstance(response, list):
        response = response[0]
-    question = question.replace('<', '').replace('>', '')
+    question = question.replace('<', '').replace('>', '').replace('_', ' ')
    if question in response:
        response = response.split(question, 1)[1]
    response = response.replace('\n', '').replace('\r', '').replace('\t', '').strip()
@@ -110,9 +113,7 @@ def qwen(question: str, image: Image.Image, repo: str = None):
        processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
        loaded = repo
    model = model.to(devices.device, devices.dtype)
-    if len(question) < 2:
-        question = "Describe the image."
-    question = question.replace('<', '').replace('>', '')
+    question = question.replace('<', '').replace('>', '').replace('_', ' ')
    conversation = [
        {
            "role": "system",
@@ -152,9 +153,7 @@ def paligemma(question: str, image: Image.Image, repo: str = None):
        model = transformers.PaliGemmaForConditionalGeneration.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype)
        loaded = repo
    model = model.to(devices.device, devices.dtype)
-    if len(question) < 2:
-        question = "Describe the image."
-    question = question.replace('<', '').replace('>', '')
+    question = question.replace('<', '').replace('>', '').replace('_', ' ')
    model_inputs = processor(text=question, images=image, return_tensors="pt").to(devices.device, devices.dtype)
    input_len = model_inputs["input_ids"].shape[-1]
    with devices.inference_context():
@@ -167,6 +166,45 @@ def paligemma(question: str, image: Image.Image, repo: str = None):
    return response


+def ovis(question: str, image: Image.Image, repo: str = None):
+    try:
+        import flash_attn # pylint: disable=unused-import
+    except Exception:
+        shared.log.error(f'Interrogate: vlm="{repo}" flash-attn is not available')
+        return ''
+    global model, loaded # pylint: disable=global-statement
+    if model is None or loaded != repo:
+        shared.log.debug(f'Interrogate load: vlm="{repo}"')
+        model = transformers.AutoModelForCausalLM.from_pretrained(repo, torch_dtype=devices.dtype, multimodal_max_length=32768, trust_remote_code=True)
+        loaded = repo
+    model = model.to(devices.device, devices.dtype)
+    text_tokenizer = model.get_text_tokenizer()
+    visual_tokenizer = model.get_visual_tokenizer()
+    max_partition = 9
+    question = question.replace('<', '').replace('>', '').replace('_', ' ')
+    question = f'<image>\n{question}'
+    _prompt, input_ids, pixel_values = model.preprocess_inputs(question, [image], max_partition=max_partition)
+    attention_mask = torch.ne(input_ids, text_tokenizer.pad_token_id)
+    input_ids = input_ids.unsqueeze(0).to(device=model.device)
+    attention_mask = attention_mask.unsqueeze(0).to(device=model.device)
+    if pixel_values is not None:
+        pixel_values = pixel_values.to(dtype=visual_tokenizer.dtype, device=visual_tokenizer.device)
+    pixel_values = [pixel_values]
+    with devices.inference_context():
+        output_ids = model.generate(
+            input_ids,
+            pixel_values=pixel_values,
+            attention_mask=attention_mask,
+            repetition_penalty=None,
+            eos_token_id=model.generation_config.eos_token_id,
+            pad_token_id=text_tokenizer.pad_token_id,
+            use_cache=True,
+            **get_kwargs())
+        response = text_tokenizer.decode(output_ids[0], skip_special_tokens=True)
+        print(f'Output:\n{response}')
+    return response
+
+
 def smol(question: str, image: Image.Image, repo: str = None):
    global processor, model, loaded # pylint: disable=global-statement
    if model is None or loaded != repo:
@@ -180,9 +218,7 @@ def smol(question: str, image: Image.Image, repo: str = None):
        processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
        loaded = repo
    model.to(devices.device, devices.dtype)
-    if len(question) < 2:
-        question = "Describe the image."
-    question = question.replace('<', '').replace('>', '')
+    question = question.replace('<', '').replace('>', '').replace('_', ' ')
    conversation = [
        {
            "role": "system",
@@ -298,9 +334,7 @@ def moondream(question: str, image: Image.Image, repo: str = None):
        loaded = repo
        model.eval()
    model.to(devices.device, devices.dtype)
-    if len(question) < 2:
-        question = "Describe the image."
-    question = question.replace('<', '').replace('>', '')
+    question = question.replace('<', '').replace('>', '').replace('_', ' ')
    encoded = model.encode_image(image)
    with devices.inference_context():
        response = model.answer_question(encoded, question, processor)
@@ -331,7 +365,6 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str
        task = question.split('>', 1)[0] + '>'
    else:
        task = '<MORE_DETAILED_CAPTION>'
-        # question = task + question
    inputs = processor(text=task, images=image, return_tensors="pt")
    input_ids = inputs['input_ids'].to(devices.device)
    pixel_values = inputs['pixel_values'].to(devices.device, devices.dtype)
@@ -348,7 +381,7 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str

 def interrogate(question, prompt, image, model_name, quiet:bool=False):
    if not quiet:
-        shared.state.begin('Caption')
+        shared.state.begin('Interrogate')
    t0 = time.time()
    if isinstance(image, list):
        image = image[0] if len(image) > 0 else None
@@ -362,6 +395,10 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False):
        image = image.convert('RGB')
    if prompt is not None and len(prompt) > 0:
        question = prompt
+    if len(question) < 2:
+        question = "Describe the image."
+    if shared.native and shared.sd_loaded:
+        sd_models.apply_balanced_offload(shared.sd_model)
    from modules import modelloader
    modelloader.hf_login()
    try:
@@ -402,6 +439,8 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False):
            answer = deepseek.predict(question, image, vqa_model)
        elif 'paligemma' in vqa_model.lower():
            answer = paligemma(question, image, vqa_model)
+        elif 'ovis' in vqa_model.lower():
+            answer = ovis(question, image, vqa_model)
        else:
            answer = 'unknown model'
    except Exception as e:
@@ -443,16 +482,16 @@ def batch(model_name, batch_files, batch_folder, batch_str, question, prompt, wr
    if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str):
        files += [os.path.join(batch_str, f) for f in os.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))]
    if len(files) == 0:
-        shared.log.error('Interrogate batch no images')
+        shared.log.warning('Interrogate batch: type=vlm no images')
        return ''
-    shared.state.begin('Caption batch')
+    shared.state.begin('Interrogate batch')
    prompts = []
    if write:
        mode = 'w' if not append else 'a'
        writer = BatchWriter(os.path.dirname(files[0]), mode=mode)
-    import rich.progress as rp
    orig_offload = shared.opts.interrogate_offload
    shared.opts.interrogate_offload = False
+    import rich.progress as rp
    pbar = rp.Progress(rp.TextColumn('[cyan]Caption:'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console)
    with pbar:
        task = pbar.add_task(total=len(files), description='starting...')
@@ -917,11 +917,13 @@ options_templates.update(options_section(('interrogate', "Interrogate"), {
    "interrogate_clip_model": OptionInfo("ViT-L-14/openai", "CLiP: default model", gr.Dropdown, lambda: {"choices": get_clip_models()}, refresh=refresh_clip_models),
    "interrogate_clip_mode": OptionInfo(caption_types[0], "CLiP: default mode", gr.Dropdown, {"choices": caption_types}),
    "interrogate_blip_model": OptionInfo(list(caption_models)[0], "CLiP: default captioner", gr.Dropdown, {"choices": list(caption_models)}),
-    "interrogate_clip_num_beams": OptionInfo(1, "CLiP: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1}),
-    "interrogate_clip_min_length": OptionInfo(32, "CLiP: min length", gr.Slider, {"minimum": 1, "maximum": 128, "step": 1}),
-    "interrogate_clip_max_length": OptionInfo(74, "CLiP: max length", gr.Slider, {"minimum": 1, "maximum": 512, "step": 1}),
-    "interrogate_clip_min_flavors": OptionInfo(2, "CLiP: min flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1}),
-    "interrogate_clip_max_flavors": OptionInfo(8, "CLiP: max flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1}),
+    "interrogate_clip_num_beams": OptionInfo(1, "CLiP: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}),
+    "interrogate_clip_min_length": OptionInfo(32, "CLiP: min length", gr.Slider, {"minimum": 1, "maximum": 128, "step": 1,  "visible": False}),
+    "interrogate_clip_max_length": OptionInfo(74, "CLiP: max length", gr.Slider, {"minimum": 1, "maximum": 512, "step": 1,  "visible": False}),
+    "interrogate_clip_min_flavors": OptionInfo(2, "CLiP: min flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1,  "visible": False}),
+    "interrogate_clip_max_flavors": OptionInfo(16, "CLiP: max flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1,  "visible": False}),
+    "interrogate_clip_flavor_count": OptionInfo(16, "CLiP: intermediate flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1,  "visible": False}),
+    "interrogate_clip_chunk_size": OptionInfo(1024, "CLiP: chunk size", gr.Slider, {"minimum": 256, "maximum": 4096, "step": 8,  "visible": False}),
    "interrogate_clip_skip_categories": OptionInfo(["artists", "movements", "flavors"], "CLiP: skip categories", gr.CheckboxGroup, lambda: {"choices": category_types()}, refresh=category_types),

    "interrogate_vlm_sep": OptionInfo("<h2>VLM</h2>", "", gr.HTML),
@@ -14,46 +14,28 @@ def update_vlm_params(*args):
    shared.opts.save(shared.config_filename)


+def update_clip_params(*args):
+    """
+    "interrogate_clip_num_beams": OptionInfo(1, "CLiP: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}),
+    """
+    clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams = args
+    shared.opts.interrogate_clip_min_length = int(clip_min_length)
+    shared.opts.interrogate_clip_max_length = int(clip_max_length)
+    shared.opts.interrogate_clip_min_flavors = int(clip_min_flavors)
+    shared.opts.interrogate_clip_max_flavors = int(clip_max_flavors)
+    shared.opts.interrogate_clip_num_beams = int(clip_num_beams)
+    shared.opts.interrogate_clip_flavor_count = int(clip_flavor_count)
+    shared.opts.interrogate_clip_chunk_size = int(clip_chunk_size)
+    shared.opts.save(shared.config_filename)
+    openclip.update_interrogate_params()
+
+
 def create_ui():
    with gr.Row(equal_height=False, variant='compact', elem_classes="caption"):
        with gr.Column(variant='compact'):
            with gr.Row():
                image = gr.Image(type='pil', label="Image")
            with gr.Tabs(elem_id="mode_caption"):
-                with gr.Tab("CLiP Interrogate"):
-                    with gr.Row():
-                        clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model', elem_id='clip_clip_model')
-                        ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models')
-                        blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model', elem_id='clip_blip_model')
-                        clip_mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast', elem_id='clip_clip_mode')
-                    with gr.Accordion(label='Advanced', open=False, visible=True):
-                        with gr.Row():
-                            clip_caption_max_length = gr.Slider(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=512, elem_id='clip_caption_max_length')
-                            clip_chunk_size = gr.Slider(label='Chunk size', value=1024, minimum=256, maximum=4096, elem_id='clip_chunk_size')
-                        with gr.Row():
-                            clip_min_flavors = gr.Slider(label='Min flavors', value=2, minimum=1, maximum=16, elem_id='clip_min_flavors')
-                            clip_max_flavors = gr.Slider(label='Max flavors', value=8, minimum=1, maximum=64, elem_id='clip_max_flavors')
-                            clip_flavor_intermediate_count = gr.Slider(label='Intermediates', value=1024, minimum=256, maximum=4096, elem_id='clip_flavor_intermediate_count')
-                        clip_caption_max_length.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
-                        clip_chunk_size.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
-                        clip_min_flavors.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
-                        clip_max_flavors.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
-                        clip_flavor_intermediate_count.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
-                    with gr.Accordion(label='Batch', open=False, visible=True):
-                        with gr.Row():
-                            clip_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_files')
-                        with gr.Row():
-                            clip_batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_folder')
-                        with gr.Row():
-                            clip_batch_str = gr.Text(label="Folder", value="", interactive=True, elem_id='clip_batch_str')
-                        with gr.Row():
-                            clip_save_output = gr.Checkbox(label='Save caption files', value=True, elem_id="clip_save_output")
-                            clip_save_append = gr.Checkbox(label='Append caption files', value=False, elem_id="clip_save_append")
-                        with gr.Row():
-                            btn_clip_interrogate_batch = gr.Button("Batch interrogate", variant='primary', elem_id="btn_clip_interrogate_batch")
-                    with gr.Row():
-                        btn_clip_interrogate_img = gr.Button("Interrogate", variant='primary', elem_id="btn_clip_interrogate_img")
-                        btn_clip_analyze_img = gr.Button("Analyze", variant='primary', elem_id="btn_clip_analyze_img")
                with gr.Tab("VLM Caption"):
                    from modules.interrogate import vqa
                    with gr.Row():
@@ -62,7 +44,7 @@ def create_ui():
                        vlm_prompt = gr.Textbox(label="Prompt", placeholder="optionally enter custom prompt", lines=2, elem_id='vlm_prompt')
                    with gr.Row(elem_id='interrogate_buttons_query'):
                        vlm_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model', elem_id='vlm_model')
-                    with gr.Accordion(label='Advanced', open=False, visible=True):
+                    with gr.Accordion(label='Advanced options', open=False, visible=True):
                        with gr.Row():
                            vlm_max_tokens = gr.Slider(label='Max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1, elem_id='vlm_max_tokens')
                            vlm_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1, elem_id='vlm_num_beams')
@@ -78,7 +60,7 @@ def create_ui():
                        vlm_do_sample.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
                        vlm_top_k.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
                        vlm_top_p.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
-                    with gr.Accordion(label='Batch', open=False, visible=True):
+                    with gr.Accordion(label='Batch caption', open=False, visible=True):
                        with gr.Row():
                            vlm_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100, elem_id='vlm_batch_files')
                        with gr.Row():
@@ -92,6 +74,45 @@ def create_ui():
                            btn_vlm_caption_batch = gr.Button("Batch caption", variant='primary', elem_id="btn_vlm_caption_batch")
                    with gr.Row():
                        btn_vlm_caption = gr.Button("Caption", variant='primary', elem_id="btn_vlm_caption")
+                with gr.Tab("CLiP Interrogate"):
+                    with gr.Row():
+                        clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model', elem_id='clip_clip_model')
+                        ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models')
+                        blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model', elem_id='clip_blip_model')
+                        clip_mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast', elem_id='clip_clip_mode')
+                    with gr.Accordion(label='Advanced options', open=False, visible=True):
+                        with gr.Row():
+                            clip_min_length = gr.Slider(label='Min length', value=shared.opts.interrogate_clip_min_length, minimum=8, maximum=75, step=1, elem_id='clip_caption_min_length')
+                            clip_max_length = gr.Slider(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=1024, step=1, elem_id='clip_caption_max_length')
+                            clip_chunk_size = gr.Slider(label='Chunk size', value=shared.opts.interrogate_clip_chunk_size, minimum=256, maximum=4096, step=8, elem_id='clip_chunk_size')
+                        with gr.Row():
+                            clip_min_flavors = gr.Slider(label='Min flavors', value=shared.opts.interrogate_clip_min_flavors, minimum=1, maximum=16, step=1, elem_id='clip_min_flavors')
+                            clip_max_flavors = gr.Slider(label='Max flavors', value=shared.opts.interrogate_clip_max_flavors, minimum=1, maximum=64, step=1, elem_id='clip_max_flavors')
+                            clip_flavor_count = gr.Slider(label='Intermediates', value=shared.opts.interrogate_clip_flavor_count, minimum=256, maximum=4096, step=8, elem_id='clip_flavor_intermediate_count')
+                        with gr.Row():
+                            clip_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_clip_num_beams, minimum=1, maximum=16, step=1, elem_id='clip_num_beams')
+                        clip_min_length.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
+                        clip_max_length.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
+                        clip_chunk_size.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
+                        clip_min_flavors.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
+                        clip_max_flavors.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
+                        clip_flavor_count.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
+                        clip_num_beams.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[])
+                    with gr.Accordion(label='Batch interogate', open=False, visible=True):
+                        with gr.Row():
+                            clip_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_files')
+                        with gr.Row():
+                            clip_batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_folder')
+                        with gr.Row():
+                            clip_batch_str = gr.Text(label="Folder", value="", interactive=True, elem_id='clip_batch_str')
+                        with gr.Row():
+                            clip_save_output = gr.Checkbox(label='Save caption files', value=True, elem_id="clip_save_output")
+                            clip_save_append = gr.Checkbox(label='Append caption files', value=False, elem_id="clip_save_append")
+                        with gr.Row():
+                            btn_clip_interrogate_batch = gr.Button("Batch interrogate", variant='primary', elem_id="btn_clip_interrogate_batch")
+                    with gr.Row():
+                        btn_clip_interrogate_img = gr.Button("Interrogate", variant='primary', elem_id="btn_clip_interrogate_img")
+                        btn_clip_analyze_img = gr.Button("Analyze", variant='primary', elem_id="btn_clip_analyze_img")
        with gr.Column(variant='compact'):
            with gr.Row():
                prompt = gr.Textbox(label="Answer", lines=8, placeholder="ai generated image description")