diff --git a/CHANGELOG.md b/CHANGELOG.md
index 537c24247..d4110af98 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
## Update for 2025-02-14
+### TODO
+
+- VLM ModernUI support
+- CLiP Move settings
+- CLiP Batch progress bar
+
### Highlight for 2025-02-14
We're back with another update with over 50 commits!
@@ -58,10 +64,18 @@ We're back with another update with over 50 commits!
split from Process tab into separate tab
split `clip` vs `vlm` models processing
direct *send-to* buttons on all tabs
- - [JoyTag](https://huggingface.co/fancyfeast/joytag)
- - [JoyCaption 2](https://huggingface.co/fancyfeast/llama-joycaption-alpha-two-hf-llava)
- - [Google PaliGemma 2](https://huggingface.co/google/paligemma2-3b-pt-224)
- - [ToriiGate 0.4 7B](https://huggingface.co/Minthy/ToriiGate-v0.4-7B),
+ - Add VLM advanced params: max-tokens, num-beams, temperature, top-k, top-p, do-sample
+ params are saved in `config.json` and used when using quick interrogate
+ params that are set to 0 mean use model defaults
+ - Add VLM batch processing
+ for example, can be used to caption your training dataset in one go
+ add option to append to captions file, can be used to run multiple captioning models in sequence
+ add progress bar
+ - Add additional VLM models:
+ [JoyTag](https://huggingface.co/fancyfeast/joytag)
+ [JoyCaption 2](https://huggingface.co/fancyfeast/llama-joycaption-alpha-two-hf-llava)
+ [Google PaliGemma 2](https://huggingface.co/google/paligemma2-3b-pt-224)
+ [ToriiGate 0.4 7B](https://huggingface.co/Minthy/ToriiGate-v0.4-7B)
- **Docker**
- updated **CUDA** receipe to `torch==2.6.0` with `cuda==12.6` and add prebuilt image
- added **ROCm** receipe and prebuilt image
diff --git a/modules/interrogate/openclip.py b/modules/interrogate/openclip.py
index fd10d09ab..3449dba28 100644
--- a/modules/interrogate/openclip.py
+++ b/modules/interrogate/openclip.py
@@ -228,13 +228,15 @@ class InterrogateModels:
# --------- interrrogate ui
class BatchWriter:
- def __init__(self, folder):
+ def __init__(self, folder, mode='w'):
self.folder = folder
- self.csv, self.file = None, None
+ self.csv = None
+ self.file = None
+ self.mode = mode
def add(self, file, prompt):
txt_file = os.path.splitext(file)[0] + ".txt"
- with open(os.path.join(self.folder, txt_file), 'w', encoding='utf-8') as f:
+ with open(os.path.join(self.folder, txt_file), self.mode, encoding='utf-8') as f:
f.write(prompt)
def close(self):
@@ -354,7 +356,7 @@ def interrogate_image(image, clip_model, blip_model, mode):
return prompt
-def interrogate_batch(batch_files, batch_folder, batch_str, clip_model, blip_model, mode, write):
+def interrogate_batch(batch_files, batch_folder, batch_str, clip_model, blip_model, mode, write, append):
files = []
if batch_files is not None:
files += [f.name for f in batch_files]
@@ -388,7 +390,8 @@ def interrogate_batch(batch_files, batch_folder, batch_str, clip_model, blip_mod
captions.append(caption)
# second pass: interrogate
if write:
- writer = BatchWriter(os.path.dirname(files[0]))
+ mode = 'w' if not append else 'a'
+ writer = BatchWriter(os.path.dirname(files[0]), mode=mode)
for idx, file in enumerate(files):
try:
if shared.state.interrupted:
diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py
index 8ce6e5e77..f98a566b4 100644
--- a/modules/interrogate/vqa.py
+++ b/modules/interrogate/vqa.py
@@ -1,4 +1,5 @@
import io
+import os
import time
import json
import base64
@@ -77,10 +78,30 @@ def clean(response, question):
if question in response:
response = response.split(question, 1)[1]
response = response.replace('\n', '').replace('\r', '').replace('\t', '').strip()
+ if response.startswith('"'):
+ response = response[1:]
+ if response.endswith('"'):
+ response = response[:-1]
response = response.replace('Assistant:', '').strip()
return response
+def get_kwargs():
+ kwargs = {
+ 'max_new_tokens': shared.opts.interrogate_vlm_max_length,
+ 'do_sample': shared.opts.interrogate_vlm_do_sample,
+ }
+ if shared.opts.interrogate_vlm_num_beams > 0:
+ kwargs['num_beams'] = shared.opts.interrogate_vlm_num_beams
+ if shared.opts.interrogate_vlm_temperature > 0:
+ kwargs['temperature'] = shared.opts.interrogate_vlm_temperature
+ if shared.opts.interrogate_vlm_top_k > 0:
+ kwargs['top_k'] = shared.opts.interrogate_vlm_top_k
+ if shared.opts.interrogate_vlm_top_p > 0:
+ kwargs['top_p'] = shared.opts.interrogate_vlm_top_p
+ return kwargs
+
+
def qwen(question: str, image: Image.Image, repo: str = None):
global processor, model, loaded # pylint: disable=global-statement
if model is None or loaded != repo:
@@ -113,7 +134,7 @@ def qwen(question: str, image: Image.Image, repo: str = None):
inputs = inputs.to(devices.device, devices.dtype)
output_ids = model.generate(
**inputs,
- max_new_tokens=shared.opts.interrogate_vlm_max_length,
+ **get_kwargs(),
)
generated_ids = [
output_ids[len(input_ids) :]
@@ -139,8 +160,7 @@ def paligemma(question: str, image: Image.Image, repo: str = None):
with devices.inference_context():
generation = model.generate(
**model_inputs,
- max_new_tokens=shared.opts.interrogate_vlm_max_length,
- do_sample=shared.opts.interrogate_vlm_do_sample,
+ **get_kwargs(),
)
generation = generation[0][input_len:]
response = processor.decode(generation, skip_special_tokens=True)
@@ -184,7 +204,7 @@ def smol(question: str, image: Image.Image, repo: str = None):
inputs = inputs.to(devices.device, devices.dtype)
output_ids = model.generate(
**inputs,
- max_new_tokens=shared.opts.interrogate_vlm_max_length,
+ **get_kwargs(),
)
response = processor.batch_decode(output_ids,skip_special_tokens=True)
return response
@@ -297,7 +317,7 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str
return R
revision = None
if '@' in repo:
- repo, revision = model.split('@')
+ repo, revision = repo.split('@')
if model is None or loaded != repo:
shared.log.debug(f'Interrogate load: vlm="{repo}" path="{shared.opts.hfcache_dir}"')
transformers.dynamic_module_utils.get_imports = get_imports
@@ -319,16 +339,16 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str
generated_ids = model.generate(
input_ids=input_ids,
pixel_values=pixel_values,
- max_new_tokens=shared.opts.interrogate_vlm_max_length,
- num_beams=shared.opts.interrogate_vlm_num_beams,
- do_sample=shared.opts.interrogate_vlm_do_sample,
+ **get_kwargs()
)
generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
response = processor.post_process_generation(generated_text, task="task", image_size=(image.width, image.height))
return response
-def interrogate(question, prompt, image, model_name):
+def interrogate(question, prompt, image, model_name, quiet:bool=False):
+ if not quiet:
+ shared.state.begin('Caption')
t0 = time.time()
if isinstance(image, list):
image = image[0] if len(image) > 0 else None
@@ -337,7 +357,7 @@ def interrogate(question, prompt, image, model_name):
if image is None:
return ''
if image.width > 768 or image.height > 768:
- image.thumbnail((768, 768), Image.Resampling.HAMMING)
+ image.thumbnail((768, 768), Image.Resampling.LANCZOS)
if image.mode != 'RGB':
image = image.convert('RGB')
if prompt is not None and len(prompt) > 0:
@@ -392,5 +412,64 @@ def interrogate(question, prompt, image, model_name):
devices.torch_gc()
answer = clean(answer, question)
t1 = time.time()
- shared.log.debug(f'Interrogate: type=vlm model="{model_name}" repo="{vqa_model}" time={t1-t0:.2f}')
+ if not quiet:
+ shared.log.debug(f'Interrogate: type=vlm model="{model_name}" repo="{vqa_model}" args={get_kwargs()} time={t1-t0:.2f}')
+ shared.state.end()
return answer
+
+
+def batch(model_name, batch_files, batch_folder, batch_str, question, prompt, write, append):
+ class BatchWriter:
+ def __init__(self, folder, mode='w'):
+ self.folder = folder
+ self.csv = None
+ self.file = None
+ self.mode = mode
+
+ def add(self, file, prompt):
+ txt_file = os.path.splitext(file)[0] + ".txt"
+ with open(os.path.join(self.folder, txt_file), self.mode, encoding='utf-8') as f:
+ f.write(prompt)
+
+ def close(self):
+ if self.file is not None:
+ self.file.close()
+
+ files = []
+ if batch_files is not None:
+ files += [f.name for f in batch_files]
+ if batch_folder is not None:
+ files += [f.name for f in batch_folder]
+ if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str):
+ files += [os.path.join(batch_str, f) for f in os.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))]
+ if len(files) == 0:
+ shared.log.error('Interrogate batch no images')
+ return ''
+ shared.state.begin('Caption batch')
+ prompts = []
+ if write:
+ mode = 'w' if not append else 'a'
+ writer = BatchWriter(os.path.dirname(files[0]), mode=mode)
+ import rich.progress as rp
+ orig_offload = shared.opts.interrogate_offload
+ shared.opts.interrogate_offload = False
+ pbar = rp.Progress(rp.TextColumn('[cyan]Caption:'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console)
+ with pbar:
+ task = pbar.add_task(total=len(files), description='starting...')
+ for file in files:
+ pbar.update(task, advance=1, description=file)
+ try:
+ if shared.state.interrupted:
+ break
+ image = Image.open(file)
+ prompt = interrogate(question, prompt, image, model_name, quiet=True)
+ prompts.append(prompt)
+ if write:
+ writer.add(file, prompt)
+ except Exception as e:
+ shared.log.error(f'Interrogate batch: {e}')
+ if write:
+ writer.close()
+ shared.opts.interrogate_offload = orig_offload
+ shared.state.end()
+ return '\n\n'.join(prompts)
diff --git a/modules/shared.py b/modules/shared.py
index b8f5e3fd7..9d8633eeb 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -909,7 +909,7 @@ options_templates.update(options_section(('control', "Control Options"), {
options_templates.update(options_section(('interrogate', "Interrogate"), {
"interrogate_default_type": OptionInfo("OpenCLiP", "Default type", gr.Radio, {"choices": ["OpenCLiP", "VLM", "DeepBooru"]}),
- "interrogate_offload": OptionInfo(True, "Interrogate: offload models "),
+ "interrogate_offload": OptionInfo(True, "Offload models "),
"interrogate_score": OptionInfo(False, "Include scores in results when available"),
"interrogate_clip_sep": OptionInfo("
OpenCLiP
", "", gr.HTML),
@@ -929,7 +929,9 @@ options_templates.update(options_section(('interrogate', "Interrogate"), {
"interrogate_vlm_num_beams": OptionInfo(3, "VLM: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}),
"interrogate_vlm_max_length": OptionInfo(512, "VLM: max length", gr.Slider, {"minimum": 1, "maximum": 4096, "step": 1, "visible": False}),
"interrogate_vlm_do_sample": OptionInfo(False, "VLM: use sample method"),
- "interrogate_vlm_temperature": OptionInfo(0.6, "VLM: num beams", gr.Slider, {"minimum": 0.1, "maximum": 1.0, "step": 0.11, "visible": False}),
+ "interrogate_vlm_temperature": OptionInfo(0, "VLM: num beams", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.01, "visible": False}),
+ "interrogate_vlm_top_k": OptionInfo(0, "VLM: top-k", gr.Slider, {"minimum": 0, "maximum": 99, "step": 1, "visible": False}),
+ "interrogate_vlm_top_p": OptionInfo(0, "VLM: top-p", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.01, "visible": False}),
"deepbooru_sep": OptionInfo("DeepBooru
", "", gr.HTML),
"deepbooru_score_threshold": OptionInfo(0.65, "DeepBooru: score threshold", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}),
diff --git a/modules/ui_caption.py b/modules/ui_caption.py
index a306122e0..dbf00a916 100644
--- a/modules/ui_caption.py
+++ b/modules/ui_caption.py
@@ -4,11 +4,14 @@ from modules.interrogate import openclip
def update_vlm_params(*args):
- vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample = args
+ vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p = args
shared.opts.interrogate_vlm_max_length = vlm_max_tokens
shared.opts.interrogate_vlm_num_beams = vlm_num_beams
shared.opts.interrogate_vlm_temperature = vlm_temperature
shared.opts.interrogate_vlm_do_sample = vlm_do_sample
+ shared.opts.interrogate_vlm_top_k = vlm_top_k
+ shared.opts.interrogate_vlm_top_p = vlm_top_p
+ shared.opts.save(shared.config_filename)
def create_ui():
@@ -19,67 +22,80 @@ def create_ui():
with gr.Tabs(elem_id="mode_caption"):
with gr.Tab("CLiP Interrogate"):
with gr.Row():
- clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model')
+ clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model', elem_id='clip_clip_model')
ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models')
- blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model')
- mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast')
+ blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model', elem_id='clip_blip_model')
+ clip_mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast', elem_id='clip_clip_mode')
with gr.Accordion(label='Advanced', open=False, visible=True):
with gr.Row():
- caption_max_length = gr.Slider(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=512, min_width=300)
- chunk_size = gr.Slider(label='Chunk size', value=1024, minimum=256, maximum=4096, min_width=300)
+ clip_caption_max_length = gr.Slider(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=512, elem_id='clip_caption_max_length')
+ clip_chunk_size = gr.Slider(label='Chunk size', value=1024, minimum=256, maximum=4096, elem_id='clip_chunk_size')
with gr.Row():
- min_flavors = gr.Slider(label='Min flavors', value=2, minimum=1, maximum=16, min_width=300)
- max_flavors = gr.Slider(label='Max flavors', value=8, minimum=1, maximum=64, min_width=300)
- flavor_intermediate_count = gr.Slider(label='Intermediates', value=1024, minimum=256, maximum=4096)
- caption_max_length.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
- chunk_size.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
- min_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
- max_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
- flavor_intermediate_count.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
+ clip_min_flavors = gr.Slider(label='Min flavors', value=2, minimum=1, maximum=16, elem_id='clip_min_flavors')
+ clip_max_flavors = gr.Slider(label='Max flavors', value=8, minimum=1, maximum=64, elem_id='clip_max_flavors')
+ clip_flavor_intermediate_count = gr.Slider(label='Intermediates', value=1024, minimum=256, maximum=4096, elem_id='clip_flavor_intermediate_count')
+ clip_caption_max_length.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
+ clip_chunk_size.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
+ clip_min_flavors.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
+ clip_max_flavors.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
+ clip_flavor_intermediate_count.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
with gr.Accordion(label='Batch', open=False, visible=True):
with gr.Row():
- batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100)
+ clip_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_files')
with gr.Row():
- batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100)
+ clip_batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_folder')
with gr.Row():
- batch_str = gr.Text(label="Folder", value="", interactive=True)
+ clip_batch_str = gr.Text(label="Folder", value="", interactive=True, elem_id='clip_batch_str')
with gr.Row():
- batch = gr.Text(label="Prompts", lines=10)
+ clip_save_output = gr.Checkbox(label='Save caption files', value=True, elem_id="clip_save_output")
+ clip_save_append = gr.Checkbox(label='Append caption files', value=False, elem_id="clip_save_append")
with gr.Row():
- clip_model = gr.Dropdown([], value='ViT-L-14/openai', label='CLiP Batch Model')
- ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models')
- with gr.Row(elem_id='interrogate_buttons_batch'):
- btn_interrogate_batch = gr.Button("Batch interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
- with gr.Row():
- save_output = gr.Checkbox(label='Save output', value=True, elem_id="extras_save_output")
- with gr.Row(elem_id='interrogate_buttons_image'):
- btn_interrogate_img = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
- btn_analyze_img = gr.Button("Analyze", elem_id="interrogate_btn_analyze", variant='primary')
+ btn_clip_interrogate_batch = gr.Button("Batch interrogate", variant='primary', elem_id="btn_clip_interrogate_batch")
+ with gr.Row():
+ btn_clip_interrogate_img = gr.Button("Interrogate", variant='primary', elem_id="btn_clip_interrogate_img")
+ btn_clip_analyze_img = gr.Button("Analyze", variant='primary', elem_id="btn_clip_analyze_img")
with gr.Tab("VLM Caption"):
from modules.interrogate import vqa
with gr.Row():
- vqa_question = gr.Dropdown(label="Predefined question", allow_custom_value=False, choices=vqa.vlm_prompts, value=vqa.vlm_prompts[2])
+ vlm_question = gr.Dropdown(label="Predefined question", allow_custom_value=False, choices=vqa.vlm_prompts, value=vqa.vlm_prompts[2], elem_id='vlm_question')
with gr.Row():
- vqa_prompt = gr.Textbox(label="Prompt", placeholder="optionally enter custom prompt", lines=2)
+ vlm_prompt = gr.Textbox(label="Prompt", placeholder="optionally enter custom prompt", lines=2, elem_id='vlm_prompt')
with gr.Row(elem_id='interrogate_buttons_query'):
- vqa_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model')
+ vlm_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model', elem_id='vlm_model')
with gr.Accordion(label='Advanced', open=False, visible=True):
with gr.Row():
- vlm_max_tokens = gr.Slider(label='Max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1)
- vlm_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1)
- vlm_temperature = gr.Slider(label='Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01)
+ vlm_max_tokens = gr.Slider(label='Max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1, elem_id='vlm_max_tokens')
+ vlm_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1, elem_id='vlm_num_beams')
+ vlm_temperature = gr.Slider(label='Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01, elem_id='vlm_temperature')
with gr.Row():
- vlm_do_sample = gr.Checkbox(label='Use sample', value=shared.opts.interrogate_vlm_do_sample)
- vlm_max_tokens.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[])
- vlm_num_beams.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[])
- vlm_temperature.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[])
- vlm_do_sample.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[])
- with gr.Row(elem_id='interrogate_buttons_query'):
- vqa_submit = gr.Button("Caption", elem_id="interrogate_btn_interrogate", variant='primary')
+ vlm_top_k = gr.Slider(label='Top-K', value=shared.opts.interrogate_vlm_top_k, minimum=0, maximum=99, step=1, elem_id='vlm_top_k')
+ vlm_top_p = gr.Slider(label='Top-P', value=shared.opts.interrogate_vlm_top_p, minimum=0.0, maximum=1.0, step=0.01, elem_id='vlm_top_p')
+ with gr.Row():
+ vlm_do_sample = gr.Checkbox(label='Use sample', value=shared.opts.interrogate_vlm_do_sample, elem_id='vlm_do_sample')
+ vlm_max_tokens.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
+ vlm_num_beams.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
+ vlm_temperature.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
+ vlm_do_sample.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
+ vlm_top_k.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
+ vlm_top_p.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
+ with gr.Accordion(label='Batch', open=False, visible=True):
+ with gr.Row():
+ vlm_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100, elem_id='vlm_batch_files')
+ with gr.Row():
+ vlm_batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100, elem_id='vlm_batch_folder')
+ with gr.Row():
+ vlm_batch_str = gr.Text(label="Folder", value="", interactive=True, elem_id='vlm_batch_str')
+ with gr.Row():
+ vlm_save_output = gr.Checkbox(label='Save caption files', value=True, elem_id="vlm_save_output")
+ vlm_save_append = gr.Checkbox(label='Append caption files', value=False, elem_id="vlm_save_append")
+ with gr.Row(elem_id='interrogate_buttons_batch'):
+ btn_vlm_caption_batch = gr.Button("Batch caption", variant='primary', elem_id="btn_vlm_caption_batch")
+ with gr.Row():
+ btn_vlm_caption = gr.Button("Caption", variant='primary', elem_id="btn_vlm_caption")
with gr.Column(variant='compact'):
with gr.Row():
prompt = gr.Textbox(label="Answer", lines=8, placeholder="ai generated image description")
- with gr.Row(elem_id="interrogate_labels"):
+ with gr.Row():
medium = gr.Label(elem_id="interrogate_label_medium", label="Medium", num_top_classes=5, visible=False)
artist = gr.Label(elem_id="interrogate_label_artist", label="Artist", num_top_classes=5, visible=False)
movement = gr.Label(elem_id="interrogate_label_movement", label="Movement", num_top_classes=5, visible=False)
@@ -88,10 +104,11 @@ def create_ui():
with gr.Row(elem_id='copy_buttons_interrogate'):
copy_interrogate_buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "control", "extras"])
- btn_interrogate_img.click(openclip.interrogate_image, inputs=[image, clip_model, blip_model, mode], outputs=[prompt])
- btn_analyze_img.click(openclip.analyze_image, inputs=[image, clip_model, blip_model], outputs=[medium, artist, movement, trending, flavor])
- btn_interrogate_batch.click(fn=openclip.interrogate_batch, inputs=[batch_files, batch_folder, batch_str, clip_model, blip_model, mode, save_output], outputs=[batch])
- vqa_submit.click(vqa.interrogate, inputs=[vqa_question, vqa_prompt, image, vqa_model], outputs=[prompt])
+ btn_clip_interrogate_img.click(openclip.interrogate_image, inputs=[image, clip_model, blip_model, clip_mode], outputs=[prompt])
+ btn_clip_analyze_img.click(openclip.analyze_image, inputs=[image, clip_model, blip_model], outputs=[medium, artist, movement, trending, flavor])
+ btn_clip_interrogate_batch.click(fn=openclip.interrogate_batch, inputs=[clip_batch_files, clip_batch_folder, clip_batch_str, clip_model, blip_model, clip_mode, clip_save_output, clip_save_append], outputs=[prompt])
+ btn_vlm_caption.click(fn=vqa.interrogate, inputs=[vlm_question, vlm_prompt, image, vlm_model], outputs=[prompt])
+ btn_vlm_caption_batch.click(fn=vqa.batch, inputs=[vlm_model, vlm_batch_files, vlm_batch_folder, vlm_batch_str, vlm_question, vlm_prompt, vlm_save_output, vlm_save_append], outputs=[prompt])
for tabname, button in copy_interrogate_buttons.items():
generation_parameters_copypaste.register_paste_params_button(generation_parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=prompt, source_image_component=image,))
diff --git a/requirements.txt b/requirements.txt
index e15d141a2..f61890cbe 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -40,7 +40,7 @@ compel==2.0.3
torchsde==0.2.6
antlr4-python3-runtime==4.9.3
requests==2.32.3
-tqdm==4.66.5
+tqdm==4.67.1
accelerate==1.3.0
opencv-contrib-python-headless==4.9.0.80
einops==0.4.1