mirror of
https://github.com/vladmandic/automatic
synced 2026-09-19 17:24:32 +02:00
vlm advanced settings and batch processing
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
+18
-4
@@ -2,6 +2,12 @@
|
||||
|
||||
## Update for 2025-02-14
|
||||
|
||||
### TODO
|
||||
|
||||
- VLM ModernUI support
|
||||
- CLiP Move settings
|
||||
- CLiP Batch progress bar
|
||||
|
||||
### Highlight for 2025-02-14
|
||||
|
||||
We're back with another update with over 50 commits!
|
||||
@@ -58,10 +64,18 @@ We're back with another update with over 50 commits!
|
||||
split from Process tab into separate tab
|
||||
split `clip` vs `vlm` models processing
|
||||
direct *send-to* buttons on all tabs
|
||||
- [JoyTag](https://huggingface.co/fancyfeast/joytag)
|
||||
- [JoyCaption 2](https://huggingface.co/fancyfeast/llama-joycaption-alpha-two-hf-llava)
|
||||
- [Google PaliGemma 2](https://huggingface.co/google/paligemma2-3b-pt-224)
|
||||
- [ToriiGate 0.4 7B](https://huggingface.co/Minthy/ToriiGate-v0.4-7B),
|
||||
- Add VLM advanced params: max-tokens, num-beams, temperature, top-k, top-p, do-sample
|
||||
params are saved in `config.json` and used when using quick interrogate
|
||||
params that are set to 0 mean use model defaults
|
||||
- Add VLM batch processing
|
||||
for example, can be used to caption your training dataset in one go
|
||||
add option to append to captions file, can be used to run multiple captioning models in sequence
|
||||
add progress bar
|
||||
- Add additional VLM models:
|
||||
[JoyTag](https://huggingface.co/fancyfeast/joytag)
|
||||
[JoyCaption 2](https://huggingface.co/fancyfeast/llama-joycaption-alpha-two-hf-llava)
|
||||
[Google PaliGemma 2](https://huggingface.co/google/paligemma2-3b-pt-224)
|
||||
[ToriiGate 0.4 7B](https://huggingface.co/Minthy/ToriiGate-v0.4-7B)
|
||||
- **Docker**
|
||||
- updated **CUDA** receipe to `torch==2.6.0` with `cuda==12.6` and add prebuilt image
|
||||
- added **ROCm** receipe and prebuilt image
|
||||
|
||||
@@ -228,13 +228,15 @@ class InterrogateModels:
|
||||
# --------- interrrogate ui
|
||||
|
||||
class BatchWriter:
|
||||
def __init__(self, folder):
|
||||
def __init__(self, folder, mode='w'):
|
||||
self.folder = folder
|
||||
self.csv, self.file = None, None
|
||||
self.csv = None
|
||||
self.file = None
|
||||
self.mode = mode
|
||||
|
||||
def add(self, file, prompt):
|
||||
txt_file = os.path.splitext(file)[0] + ".txt"
|
||||
with open(os.path.join(self.folder, txt_file), 'w', encoding='utf-8') as f:
|
||||
with open(os.path.join(self.folder, txt_file), self.mode, encoding='utf-8') as f:
|
||||
f.write(prompt)
|
||||
|
||||
def close(self):
|
||||
@@ -354,7 +356,7 @@ def interrogate_image(image, clip_model, blip_model, mode):
|
||||
return prompt
|
||||
|
||||
|
||||
def interrogate_batch(batch_files, batch_folder, batch_str, clip_model, blip_model, mode, write):
|
||||
def interrogate_batch(batch_files, batch_folder, batch_str, clip_model, blip_model, mode, write, append):
|
||||
files = []
|
||||
if batch_files is not None:
|
||||
files += [f.name for f in batch_files]
|
||||
@@ -388,7 +390,8 @@ def interrogate_batch(batch_files, batch_folder, batch_str, clip_model, blip_mod
|
||||
captions.append(caption)
|
||||
# second pass: interrogate
|
||||
if write:
|
||||
writer = BatchWriter(os.path.dirname(files[0]))
|
||||
mode = 'w' if not append else 'a'
|
||||
writer = BatchWriter(os.path.dirname(files[0]), mode=mode)
|
||||
for idx, file in enumerate(files):
|
||||
try:
|
||||
if shared.state.interrupted:
|
||||
|
||||
+90
-11
@@ -1,4 +1,5 @@
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
import base64
|
||||
@@ -77,10 +78,30 @@ def clean(response, question):
|
||||
if question in response:
|
||||
response = response.split(question, 1)[1]
|
||||
response = response.replace('\n', '').replace('\r', '').replace('\t', '').strip()
|
||||
if response.startswith('"'):
|
||||
response = response[1:]
|
||||
if response.endswith('"'):
|
||||
response = response[:-1]
|
||||
response = response.replace('Assistant:', '').strip()
|
||||
return response
|
||||
|
||||
|
||||
def get_kwargs():
|
||||
kwargs = {
|
||||
'max_new_tokens': shared.opts.interrogate_vlm_max_length,
|
||||
'do_sample': shared.opts.interrogate_vlm_do_sample,
|
||||
}
|
||||
if shared.opts.interrogate_vlm_num_beams > 0:
|
||||
kwargs['num_beams'] = shared.opts.interrogate_vlm_num_beams
|
||||
if shared.opts.interrogate_vlm_temperature > 0:
|
||||
kwargs['temperature'] = shared.opts.interrogate_vlm_temperature
|
||||
if shared.opts.interrogate_vlm_top_k > 0:
|
||||
kwargs['top_k'] = shared.opts.interrogate_vlm_top_k
|
||||
if shared.opts.interrogate_vlm_top_p > 0:
|
||||
kwargs['top_p'] = shared.opts.interrogate_vlm_top_p
|
||||
return kwargs
|
||||
|
||||
|
||||
def qwen(question: str, image: Image.Image, repo: str = None):
|
||||
global processor, model, loaded # pylint: disable=global-statement
|
||||
if model is None or loaded != repo:
|
||||
@@ -113,7 +134,7 @@ def qwen(question: str, image: Image.Image, repo: str = None):
|
||||
inputs = inputs.to(devices.device, devices.dtype)
|
||||
output_ids = model.generate(
|
||||
**inputs,
|
||||
max_new_tokens=shared.opts.interrogate_vlm_max_length,
|
||||
**get_kwargs(),
|
||||
)
|
||||
generated_ids = [
|
||||
output_ids[len(input_ids) :]
|
||||
@@ -139,8 +160,7 @@ def paligemma(question: str, image: Image.Image, repo: str = None):
|
||||
with devices.inference_context():
|
||||
generation = model.generate(
|
||||
**model_inputs,
|
||||
max_new_tokens=shared.opts.interrogate_vlm_max_length,
|
||||
do_sample=shared.opts.interrogate_vlm_do_sample,
|
||||
**get_kwargs(),
|
||||
)
|
||||
generation = generation[0][input_len:]
|
||||
response = processor.decode(generation, skip_special_tokens=True)
|
||||
@@ -184,7 +204,7 @@ def smol(question: str, image: Image.Image, repo: str = None):
|
||||
inputs = inputs.to(devices.device, devices.dtype)
|
||||
output_ids = model.generate(
|
||||
**inputs,
|
||||
max_new_tokens=shared.opts.interrogate_vlm_max_length,
|
||||
**get_kwargs(),
|
||||
)
|
||||
response = processor.batch_decode(output_ids,skip_special_tokens=True)
|
||||
return response
|
||||
@@ -297,7 +317,7 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str
|
||||
return R
|
||||
revision = None
|
||||
if '@' in repo:
|
||||
repo, revision = model.split('@')
|
||||
repo, revision = repo.split('@')
|
||||
if model is None or loaded != repo:
|
||||
shared.log.debug(f'Interrogate load: vlm="{repo}" path="{shared.opts.hfcache_dir}"')
|
||||
transformers.dynamic_module_utils.get_imports = get_imports
|
||||
@@ -319,16 +339,16 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str
|
||||
generated_ids = model.generate(
|
||||
input_ids=input_ids,
|
||||
pixel_values=pixel_values,
|
||||
max_new_tokens=shared.opts.interrogate_vlm_max_length,
|
||||
num_beams=shared.opts.interrogate_vlm_num_beams,
|
||||
do_sample=shared.opts.interrogate_vlm_do_sample,
|
||||
**get_kwargs()
|
||||
)
|
||||
generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
|
||||
response = processor.post_process_generation(generated_text, task="task", image_size=(image.width, image.height))
|
||||
return response
|
||||
|
||||
|
||||
def interrogate(question, prompt, image, model_name):
|
||||
def interrogate(question, prompt, image, model_name, quiet:bool=False):
|
||||
if not quiet:
|
||||
shared.state.begin('Caption')
|
||||
t0 = time.time()
|
||||
if isinstance(image, list):
|
||||
image = image[0] if len(image) > 0 else None
|
||||
@@ -337,7 +357,7 @@ def interrogate(question, prompt, image, model_name):
|
||||
if image is None:
|
||||
return ''
|
||||
if image.width > 768 or image.height > 768:
|
||||
image.thumbnail((768, 768), Image.Resampling.HAMMING)
|
||||
image.thumbnail((768, 768), Image.Resampling.LANCZOS)
|
||||
if image.mode != 'RGB':
|
||||
image = image.convert('RGB')
|
||||
if prompt is not None and len(prompt) > 0:
|
||||
@@ -392,5 +412,64 @@ def interrogate(question, prompt, image, model_name):
|
||||
devices.torch_gc()
|
||||
answer = clean(answer, question)
|
||||
t1 = time.time()
|
||||
shared.log.debug(f'Interrogate: type=vlm model="{model_name}" repo="{vqa_model}" time={t1-t0:.2f}')
|
||||
if not quiet:
|
||||
shared.log.debug(f'Interrogate: type=vlm model="{model_name}" repo="{vqa_model}" args={get_kwargs()} time={t1-t0:.2f}')
|
||||
shared.state.end()
|
||||
return answer
|
||||
|
||||
|
||||
def batch(model_name, batch_files, batch_folder, batch_str, question, prompt, write, append):
|
||||
class BatchWriter:
|
||||
def __init__(self, folder, mode='w'):
|
||||
self.folder = folder
|
||||
self.csv = None
|
||||
self.file = None
|
||||
self.mode = mode
|
||||
|
||||
def add(self, file, prompt):
|
||||
txt_file = os.path.splitext(file)[0] + ".txt"
|
||||
with open(os.path.join(self.folder, txt_file), self.mode, encoding='utf-8') as f:
|
||||
f.write(prompt)
|
||||
|
||||
def close(self):
|
||||
if self.file is not None:
|
||||
self.file.close()
|
||||
|
||||
files = []
|
||||
if batch_files is not None:
|
||||
files += [f.name for f in batch_files]
|
||||
if batch_folder is not None:
|
||||
files += [f.name for f in batch_folder]
|
||||
if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str):
|
||||
files += [os.path.join(batch_str, f) for f in os.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))]
|
||||
if len(files) == 0:
|
||||
shared.log.error('Interrogate batch no images')
|
||||
return ''
|
||||
shared.state.begin('Caption batch')
|
||||
prompts = []
|
||||
if write:
|
||||
mode = 'w' if not append else 'a'
|
||||
writer = BatchWriter(os.path.dirname(files[0]), mode=mode)
|
||||
import rich.progress as rp
|
||||
orig_offload = shared.opts.interrogate_offload
|
||||
shared.opts.interrogate_offload = False
|
||||
pbar = rp.Progress(rp.TextColumn('[cyan]Caption:'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console)
|
||||
with pbar:
|
||||
task = pbar.add_task(total=len(files), description='starting...')
|
||||
for file in files:
|
||||
pbar.update(task, advance=1, description=file)
|
||||
try:
|
||||
if shared.state.interrupted:
|
||||
break
|
||||
image = Image.open(file)
|
||||
prompt = interrogate(question, prompt, image, model_name, quiet=True)
|
||||
prompts.append(prompt)
|
||||
if write:
|
||||
writer.add(file, prompt)
|
||||
except Exception as e:
|
||||
shared.log.error(f'Interrogate batch: {e}')
|
||||
if write:
|
||||
writer.close()
|
||||
shared.opts.interrogate_offload = orig_offload
|
||||
shared.state.end()
|
||||
return '\n\n'.join(prompts)
|
||||
|
||||
+4
-2
@@ -909,7 +909,7 @@ options_templates.update(options_section(('control', "Control Options"), {
|
||||
|
||||
options_templates.update(options_section(('interrogate', "Interrogate"), {
|
||||
"interrogate_default_type": OptionInfo("OpenCLiP", "Default type", gr.Radio, {"choices": ["OpenCLiP", "VLM", "DeepBooru"]}),
|
||||
"interrogate_offload": OptionInfo(True, "Interrogate: offload models "),
|
||||
"interrogate_offload": OptionInfo(True, "Offload models "),
|
||||
"interrogate_score": OptionInfo(False, "Include scores in results when available"),
|
||||
|
||||
"interrogate_clip_sep": OptionInfo("<h2>OpenCLiP</h2>", "", gr.HTML),
|
||||
@@ -929,7 +929,9 @@ options_templates.update(options_section(('interrogate', "Interrogate"), {
|
||||
"interrogate_vlm_num_beams": OptionInfo(3, "VLM: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}),
|
||||
"interrogate_vlm_max_length": OptionInfo(512, "VLM: max length", gr.Slider, {"minimum": 1, "maximum": 4096, "step": 1, "visible": False}),
|
||||
"interrogate_vlm_do_sample": OptionInfo(False, "VLM: use sample method"),
|
||||
"interrogate_vlm_temperature": OptionInfo(0.6, "VLM: num beams", gr.Slider, {"minimum": 0.1, "maximum": 1.0, "step": 0.11, "visible": False}),
|
||||
"interrogate_vlm_temperature": OptionInfo(0, "VLM: num beams", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.01, "visible": False}),
|
||||
"interrogate_vlm_top_k": OptionInfo(0, "VLM: top-k", gr.Slider, {"minimum": 0, "maximum": 99, "step": 1, "visible": False}),
|
||||
"interrogate_vlm_top_p": OptionInfo(0, "VLM: top-p", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.01, "visible": False}),
|
||||
|
||||
"deepbooru_sep": OptionInfo("<h2>DeepBooru</h2>", "", gr.HTML),
|
||||
"deepbooru_score_threshold": OptionInfo(0.65, "DeepBooru: score threshold", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}),
|
||||
|
||||
+62
-45
@@ -4,11 +4,14 @@ from modules.interrogate import openclip
|
||||
|
||||
|
||||
def update_vlm_params(*args):
|
||||
vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample = args
|
||||
vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p = args
|
||||
shared.opts.interrogate_vlm_max_length = vlm_max_tokens
|
||||
shared.opts.interrogate_vlm_num_beams = vlm_num_beams
|
||||
shared.opts.interrogate_vlm_temperature = vlm_temperature
|
||||
shared.opts.interrogate_vlm_do_sample = vlm_do_sample
|
||||
shared.opts.interrogate_vlm_top_k = vlm_top_k
|
||||
shared.opts.interrogate_vlm_top_p = vlm_top_p
|
||||
shared.opts.save(shared.config_filename)
|
||||
|
||||
|
||||
def create_ui():
|
||||
@@ -19,67 +22,80 @@ def create_ui():
|
||||
with gr.Tabs(elem_id="mode_caption"):
|
||||
with gr.Tab("CLiP Interrogate"):
|
||||
with gr.Row():
|
||||
clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model')
|
||||
clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model', elem_id='clip_clip_model')
|
||||
ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models')
|
||||
blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model')
|
||||
mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast')
|
||||
blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model', elem_id='clip_blip_model')
|
||||
clip_mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast', elem_id='clip_clip_mode')
|
||||
with gr.Accordion(label='Advanced', open=False, visible=True):
|
||||
with gr.Row():
|
||||
caption_max_length = gr.Slider(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=512, min_width=300)
|
||||
chunk_size = gr.Slider(label='Chunk size', value=1024, minimum=256, maximum=4096, min_width=300)
|
||||
clip_caption_max_length = gr.Slider(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=512, elem_id='clip_caption_max_length')
|
||||
clip_chunk_size = gr.Slider(label='Chunk size', value=1024, minimum=256, maximum=4096, elem_id='clip_chunk_size')
|
||||
with gr.Row():
|
||||
min_flavors = gr.Slider(label='Min flavors', value=2, minimum=1, maximum=16, min_width=300)
|
||||
max_flavors = gr.Slider(label='Max flavors', value=8, minimum=1, maximum=64, min_width=300)
|
||||
flavor_intermediate_count = gr.Slider(label='Intermediates', value=1024, minimum=256, maximum=4096)
|
||||
caption_max_length.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
|
||||
chunk_size.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
|
||||
min_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
|
||||
max_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
|
||||
flavor_intermediate_count.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
|
||||
clip_min_flavors = gr.Slider(label='Min flavors', value=2, minimum=1, maximum=16, elem_id='clip_min_flavors')
|
||||
clip_max_flavors = gr.Slider(label='Max flavors', value=8, minimum=1, maximum=64, elem_id='clip_max_flavors')
|
||||
clip_flavor_intermediate_count = gr.Slider(label='Intermediates', value=1024, minimum=256, maximum=4096, elem_id='clip_flavor_intermediate_count')
|
||||
clip_caption_max_length.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
|
||||
clip_chunk_size.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
|
||||
clip_min_flavors.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
|
||||
clip_max_flavors.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
|
||||
clip_flavor_intermediate_count.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[])
|
||||
with gr.Accordion(label='Batch', open=False, visible=True):
|
||||
with gr.Row():
|
||||
batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100)
|
||||
clip_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_files')
|
||||
with gr.Row():
|
||||
batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100)
|
||||
clip_batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_folder')
|
||||
with gr.Row():
|
||||
batch_str = gr.Text(label="Folder", value="", interactive=True)
|
||||
clip_batch_str = gr.Text(label="Folder", value="", interactive=True, elem_id='clip_batch_str')
|
||||
with gr.Row():
|
||||
batch = gr.Text(label="Prompts", lines=10)
|
||||
clip_save_output = gr.Checkbox(label='Save caption files', value=True, elem_id="clip_save_output")
|
||||
clip_save_append = gr.Checkbox(label='Append caption files', value=False, elem_id="clip_save_append")
|
||||
with gr.Row():
|
||||
clip_model = gr.Dropdown([], value='ViT-L-14/openai', label='CLiP Batch Model')
|
||||
ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models')
|
||||
with gr.Row(elem_id='interrogate_buttons_batch'):
|
||||
btn_interrogate_batch = gr.Button("Batch interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
|
||||
with gr.Row():
|
||||
save_output = gr.Checkbox(label='Save output', value=True, elem_id="extras_save_output")
|
||||
with gr.Row(elem_id='interrogate_buttons_image'):
|
||||
btn_interrogate_img = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
|
||||
btn_analyze_img = gr.Button("Analyze", elem_id="interrogate_btn_analyze", variant='primary')
|
||||
btn_clip_interrogate_batch = gr.Button("Batch interrogate", variant='primary', elem_id="btn_clip_interrogate_batch")
|
||||
with gr.Row():
|
||||
btn_clip_interrogate_img = gr.Button("Interrogate", variant='primary', elem_id="btn_clip_interrogate_img")
|
||||
btn_clip_analyze_img = gr.Button("Analyze", variant='primary', elem_id="btn_clip_analyze_img")
|
||||
with gr.Tab("VLM Caption"):
|
||||
from modules.interrogate import vqa
|
||||
with gr.Row():
|
||||
vqa_question = gr.Dropdown(label="Predefined question", allow_custom_value=False, choices=vqa.vlm_prompts, value=vqa.vlm_prompts[2])
|
||||
vlm_question = gr.Dropdown(label="Predefined question", allow_custom_value=False, choices=vqa.vlm_prompts, value=vqa.vlm_prompts[2], elem_id='vlm_question')
|
||||
with gr.Row():
|
||||
vqa_prompt = gr.Textbox(label="Prompt", placeholder="optionally enter custom prompt", lines=2)
|
||||
vlm_prompt = gr.Textbox(label="Prompt", placeholder="optionally enter custom prompt", lines=2, elem_id='vlm_prompt')
|
||||
with gr.Row(elem_id='interrogate_buttons_query'):
|
||||
vqa_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model')
|
||||
vlm_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model', elem_id='vlm_model')
|
||||
with gr.Accordion(label='Advanced', open=False, visible=True):
|
||||
with gr.Row():
|
||||
vlm_max_tokens = gr.Slider(label='Max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1)
|
||||
vlm_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1)
|
||||
vlm_temperature = gr.Slider(label='Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01)
|
||||
vlm_max_tokens = gr.Slider(label='Max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1, elem_id='vlm_max_tokens')
|
||||
vlm_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1, elem_id='vlm_num_beams')
|
||||
vlm_temperature = gr.Slider(label='Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01, elem_id='vlm_temperature')
|
||||
with gr.Row():
|
||||
vlm_do_sample = gr.Checkbox(label='Use sample', value=shared.opts.interrogate_vlm_do_sample)
|
||||
vlm_max_tokens.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[])
|
||||
vlm_num_beams.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[])
|
||||
vlm_temperature.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[])
|
||||
vlm_do_sample.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[])
|
||||
with gr.Row(elem_id='interrogate_buttons_query'):
|
||||
vqa_submit = gr.Button("Caption", elem_id="interrogate_btn_interrogate", variant='primary')
|
||||
vlm_top_k = gr.Slider(label='Top-K', value=shared.opts.interrogate_vlm_top_k, minimum=0, maximum=99, step=1, elem_id='vlm_top_k')
|
||||
vlm_top_p = gr.Slider(label='Top-P', value=shared.opts.interrogate_vlm_top_p, minimum=0.0, maximum=1.0, step=0.01, elem_id='vlm_top_p')
|
||||
with gr.Row():
|
||||
vlm_do_sample = gr.Checkbox(label='Use sample', value=shared.opts.interrogate_vlm_do_sample, elem_id='vlm_do_sample')
|
||||
vlm_max_tokens.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
|
||||
vlm_num_beams.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
|
||||
vlm_temperature.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
|
||||
vlm_do_sample.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
|
||||
vlm_top_k.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
|
||||
vlm_top_p.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[])
|
||||
with gr.Accordion(label='Batch', open=False, visible=True):
|
||||
with gr.Row():
|
||||
vlm_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100, elem_id='vlm_batch_files')
|
||||
with gr.Row():
|
||||
vlm_batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100, elem_id='vlm_batch_folder')
|
||||
with gr.Row():
|
||||
vlm_batch_str = gr.Text(label="Folder", value="", interactive=True, elem_id='vlm_batch_str')
|
||||
with gr.Row():
|
||||
vlm_save_output = gr.Checkbox(label='Save caption files', value=True, elem_id="vlm_save_output")
|
||||
vlm_save_append = gr.Checkbox(label='Append caption files', value=False, elem_id="vlm_save_append")
|
||||
with gr.Row(elem_id='interrogate_buttons_batch'):
|
||||
btn_vlm_caption_batch = gr.Button("Batch caption", variant='primary', elem_id="btn_vlm_caption_batch")
|
||||
with gr.Row():
|
||||
btn_vlm_caption = gr.Button("Caption", variant='primary', elem_id="btn_vlm_caption")
|
||||
with gr.Column(variant='compact'):
|
||||
with gr.Row():
|
||||
prompt = gr.Textbox(label="Answer", lines=8, placeholder="ai generated image description")
|
||||
with gr.Row(elem_id="interrogate_labels"):
|
||||
with gr.Row():
|
||||
medium = gr.Label(elem_id="interrogate_label_medium", label="Medium", num_top_classes=5, visible=False)
|
||||
artist = gr.Label(elem_id="interrogate_label_artist", label="Artist", num_top_classes=5, visible=False)
|
||||
movement = gr.Label(elem_id="interrogate_label_movement", label="Movement", num_top_classes=5, visible=False)
|
||||
@@ -88,10 +104,11 @@ def create_ui():
|
||||
with gr.Row(elem_id='copy_buttons_interrogate'):
|
||||
copy_interrogate_buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "control", "extras"])
|
||||
|
||||
btn_interrogate_img.click(openclip.interrogate_image, inputs=[image, clip_model, blip_model, mode], outputs=[prompt])
|
||||
btn_analyze_img.click(openclip.analyze_image, inputs=[image, clip_model, blip_model], outputs=[medium, artist, movement, trending, flavor])
|
||||
btn_interrogate_batch.click(fn=openclip.interrogate_batch, inputs=[batch_files, batch_folder, batch_str, clip_model, blip_model, mode, save_output], outputs=[batch])
|
||||
vqa_submit.click(vqa.interrogate, inputs=[vqa_question, vqa_prompt, image, vqa_model], outputs=[prompt])
|
||||
btn_clip_interrogate_img.click(openclip.interrogate_image, inputs=[image, clip_model, blip_model, clip_mode], outputs=[prompt])
|
||||
btn_clip_analyze_img.click(openclip.analyze_image, inputs=[image, clip_model, blip_model], outputs=[medium, artist, movement, trending, flavor])
|
||||
btn_clip_interrogate_batch.click(fn=openclip.interrogate_batch, inputs=[clip_batch_files, clip_batch_folder, clip_batch_str, clip_model, blip_model, clip_mode, clip_save_output, clip_save_append], outputs=[prompt])
|
||||
btn_vlm_caption.click(fn=vqa.interrogate, inputs=[vlm_question, vlm_prompt, image, vlm_model], outputs=[prompt])
|
||||
btn_vlm_caption_batch.click(fn=vqa.batch, inputs=[vlm_model, vlm_batch_files, vlm_batch_folder, vlm_batch_str, vlm_question, vlm_prompt, vlm_save_output, vlm_save_append], outputs=[prompt])
|
||||
|
||||
for tabname, button in copy_interrogate_buttons.items():
|
||||
generation_parameters_copypaste.register_paste_params_button(generation_parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=prompt, source_image_component=image,))
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@ compel==2.0.3
|
||||
torchsde==0.2.6
|
||||
antlr4-python3-runtime==4.9.3
|
||||
requests==2.32.3
|
||||
tqdm==4.66.5
|
||||
tqdm==4.67.1
|
||||
accelerate==1.3.0
|
||||
opencv-contrib-python-headless==4.9.0.80
|
||||
einops==0.4.1
|
||||
|
||||
Reference in New Issue
Block a user