diff --git a/CHANGELOG.md b/CHANGELOG.md index 537c24247..d4110af98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Update for 2025-02-14 +### TODO + +- VLM ModernUI support +- CLiP Move settings +- CLiP Batch progress bar + ### Highlight for 2025-02-14 We're back with another update with over 50 commits! @@ -58,10 +64,18 @@ We're back with another update with over 50 commits! split from Process tab into separate tab split `clip` vs `vlm` models processing direct *send-to* buttons on all tabs - - [JoyTag](https://huggingface.co/fancyfeast/joytag) - - [JoyCaption 2](https://huggingface.co/fancyfeast/llama-joycaption-alpha-two-hf-llava) - - [Google PaliGemma 2](https://huggingface.co/google/paligemma2-3b-pt-224) - - [ToriiGate 0.4 7B](https://huggingface.co/Minthy/ToriiGate-v0.4-7B), + - Add VLM advanced params: max-tokens, num-beams, temperature, top-k, top-p, do-sample + params are saved in `config.json` and used when using quick interrogate + params that are set to 0 mean use model defaults + - Add VLM batch processing + for example, can be used to caption your training dataset in one go + add option to append to captions file, can be used to run multiple captioning models in sequence + add progress bar + - Add additional VLM models: + [JoyTag](https://huggingface.co/fancyfeast/joytag) + [JoyCaption 2](https://huggingface.co/fancyfeast/llama-joycaption-alpha-two-hf-llava) + [Google PaliGemma 2](https://huggingface.co/google/paligemma2-3b-pt-224) + [ToriiGate 0.4 7B](https://huggingface.co/Minthy/ToriiGate-v0.4-7B) - **Docker** - updated **CUDA** receipe to `torch==2.6.0` with `cuda==12.6` and add prebuilt image - added **ROCm** receipe and prebuilt image diff --git a/modules/interrogate/openclip.py b/modules/interrogate/openclip.py index fd10d09ab..3449dba28 100644 --- a/modules/interrogate/openclip.py +++ b/modules/interrogate/openclip.py @@ -228,13 +228,15 @@ class InterrogateModels: # --------- interrrogate ui class BatchWriter: - def __init__(self, folder): + def __init__(self, folder, mode='w'): self.folder = folder - self.csv, self.file = None, None + self.csv = None + self.file = None + self.mode = mode def add(self, file, prompt): txt_file = os.path.splitext(file)[0] + ".txt" - with open(os.path.join(self.folder, txt_file), 'w', encoding='utf-8') as f: + with open(os.path.join(self.folder, txt_file), self.mode, encoding='utf-8') as f: f.write(prompt) def close(self): @@ -354,7 +356,7 @@ def interrogate_image(image, clip_model, blip_model, mode): return prompt -def interrogate_batch(batch_files, batch_folder, batch_str, clip_model, blip_model, mode, write): +def interrogate_batch(batch_files, batch_folder, batch_str, clip_model, blip_model, mode, write, append): files = [] if batch_files is not None: files += [f.name for f in batch_files] @@ -388,7 +390,8 @@ def interrogate_batch(batch_files, batch_folder, batch_str, clip_model, blip_mod captions.append(caption) # second pass: interrogate if write: - writer = BatchWriter(os.path.dirname(files[0])) + mode = 'w' if not append else 'a' + writer = BatchWriter(os.path.dirname(files[0]), mode=mode) for idx, file in enumerate(files): try: if shared.state.interrupted: diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 8ce6e5e77..f98a566b4 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -1,4 +1,5 @@ import io +import os import time import json import base64 @@ -77,10 +78,30 @@ def clean(response, question): if question in response: response = response.split(question, 1)[1] response = response.replace('\n', '').replace('\r', '').replace('\t', '').strip() + if response.startswith('"'): + response = response[1:] + if response.endswith('"'): + response = response[:-1] response = response.replace('Assistant:', '').strip() return response +def get_kwargs(): + kwargs = { + 'max_new_tokens': shared.opts.interrogate_vlm_max_length, + 'do_sample': shared.opts.interrogate_vlm_do_sample, + } + if shared.opts.interrogate_vlm_num_beams > 0: + kwargs['num_beams'] = shared.opts.interrogate_vlm_num_beams + if shared.opts.interrogate_vlm_temperature > 0: + kwargs['temperature'] = shared.opts.interrogate_vlm_temperature + if shared.opts.interrogate_vlm_top_k > 0: + kwargs['top_k'] = shared.opts.interrogate_vlm_top_k + if shared.opts.interrogate_vlm_top_p > 0: + kwargs['top_p'] = shared.opts.interrogate_vlm_top_p + return kwargs + + def qwen(question: str, image: Image.Image, repo: str = None): global processor, model, loaded # pylint: disable=global-statement if model is None or loaded != repo: @@ -113,7 +134,7 @@ def qwen(question: str, image: Image.Image, repo: str = None): inputs = inputs.to(devices.device, devices.dtype) output_ids = model.generate( **inputs, - max_new_tokens=shared.opts.interrogate_vlm_max_length, + **get_kwargs(), ) generated_ids = [ output_ids[len(input_ids) :] @@ -139,8 +160,7 @@ def paligemma(question: str, image: Image.Image, repo: str = None): with devices.inference_context(): generation = model.generate( **model_inputs, - max_new_tokens=shared.opts.interrogate_vlm_max_length, - do_sample=shared.opts.interrogate_vlm_do_sample, + **get_kwargs(), ) generation = generation[0][input_len:] response = processor.decode(generation, skip_special_tokens=True) @@ -184,7 +204,7 @@ def smol(question: str, image: Image.Image, repo: str = None): inputs = inputs.to(devices.device, devices.dtype) output_ids = model.generate( **inputs, - max_new_tokens=shared.opts.interrogate_vlm_max_length, + **get_kwargs(), ) response = processor.batch_decode(output_ids,skip_special_tokens=True) return response @@ -297,7 +317,7 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str return R revision = None if '@' in repo: - repo, revision = model.split('@') + repo, revision = repo.split('@') if model is None or loaded != repo: shared.log.debug(f'Interrogate load: vlm="{repo}" path="{shared.opts.hfcache_dir}"') transformers.dynamic_module_utils.get_imports = get_imports @@ -319,16 +339,16 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str generated_ids = model.generate( input_ids=input_ids, pixel_values=pixel_values, - max_new_tokens=shared.opts.interrogate_vlm_max_length, - num_beams=shared.opts.interrogate_vlm_num_beams, - do_sample=shared.opts.interrogate_vlm_do_sample, + **get_kwargs() ) generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0] response = processor.post_process_generation(generated_text, task="task", image_size=(image.width, image.height)) return response -def interrogate(question, prompt, image, model_name): +def interrogate(question, prompt, image, model_name, quiet:bool=False): + if not quiet: + shared.state.begin('Caption') t0 = time.time() if isinstance(image, list): image = image[0] if len(image) > 0 else None @@ -337,7 +357,7 @@ def interrogate(question, prompt, image, model_name): if image is None: return '' if image.width > 768 or image.height > 768: - image.thumbnail((768, 768), Image.Resampling.HAMMING) + image.thumbnail((768, 768), Image.Resampling.LANCZOS) if image.mode != 'RGB': image = image.convert('RGB') if prompt is not None and len(prompt) > 0: @@ -392,5 +412,64 @@ def interrogate(question, prompt, image, model_name): devices.torch_gc() answer = clean(answer, question) t1 = time.time() - shared.log.debug(f'Interrogate: type=vlm model="{model_name}" repo="{vqa_model}" time={t1-t0:.2f}') + if not quiet: + shared.log.debug(f'Interrogate: type=vlm model="{model_name}" repo="{vqa_model}" args={get_kwargs()} time={t1-t0:.2f}') + shared.state.end() return answer + + +def batch(model_name, batch_files, batch_folder, batch_str, question, prompt, write, append): + class BatchWriter: + def __init__(self, folder, mode='w'): + self.folder = folder + self.csv = None + self.file = None + self.mode = mode + + def add(self, file, prompt): + txt_file = os.path.splitext(file)[0] + ".txt" + with open(os.path.join(self.folder, txt_file), self.mode, encoding='utf-8') as f: + f.write(prompt) + + def close(self): + if self.file is not None: + self.file.close() + + files = [] + if batch_files is not None: + files += [f.name for f in batch_files] + if batch_folder is not None: + files += [f.name for f in batch_folder] + if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str): + files += [os.path.join(batch_str, f) for f in os.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))] + if len(files) == 0: + shared.log.error('Interrogate batch no images') + return '' + shared.state.begin('Caption batch') + prompts = [] + if write: + mode = 'w' if not append else 'a' + writer = BatchWriter(os.path.dirname(files[0]), mode=mode) + import rich.progress as rp + orig_offload = shared.opts.interrogate_offload + shared.opts.interrogate_offload = False + pbar = rp.Progress(rp.TextColumn('[cyan]Caption:'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) + with pbar: + task = pbar.add_task(total=len(files), description='starting...') + for file in files: + pbar.update(task, advance=1, description=file) + try: + if shared.state.interrupted: + break + image = Image.open(file) + prompt = interrogate(question, prompt, image, model_name, quiet=True) + prompts.append(prompt) + if write: + writer.add(file, prompt) + except Exception as e: + shared.log.error(f'Interrogate batch: {e}') + if write: + writer.close() + shared.opts.interrogate_offload = orig_offload + shared.state.end() + return '\n\n'.join(prompts) diff --git a/modules/shared.py b/modules/shared.py index b8f5e3fd7..9d8633eeb 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -909,7 +909,7 @@ options_templates.update(options_section(('control', "Control Options"), { options_templates.update(options_section(('interrogate', "Interrogate"), { "interrogate_default_type": OptionInfo("OpenCLiP", "Default type", gr.Radio, {"choices": ["OpenCLiP", "VLM", "DeepBooru"]}), - "interrogate_offload": OptionInfo(True, "Interrogate: offload models "), + "interrogate_offload": OptionInfo(True, "Offload models "), "interrogate_score": OptionInfo(False, "Include scores in results when available"), "interrogate_clip_sep": OptionInfo("

OpenCLiP

", "", gr.HTML), @@ -929,7 +929,9 @@ options_templates.update(options_section(('interrogate', "Interrogate"), { "interrogate_vlm_num_beams": OptionInfo(3, "VLM: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}), "interrogate_vlm_max_length": OptionInfo(512, "VLM: max length", gr.Slider, {"minimum": 1, "maximum": 4096, "step": 1, "visible": False}), "interrogate_vlm_do_sample": OptionInfo(False, "VLM: use sample method"), - "interrogate_vlm_temperature": OptionInfo(0.6, "VLM: num beams", gr.Slider, {"minimum": 0.1, "maximum": 1.0, "step": 0.11, "visible": False}), + "interrogate_vlm_temperature": OptionInfo(0, "VLM: num beams", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.01, "visible": False}), + "interrogate_vlm_top_k": OptionInfo(0, "VLM: top-k", gr.Slider, {"minimum": 0, "maximum": 99, "step": 1, "visible": False}), + "interrogate_vlm_top_p": OptionInfo(0, "VLM: top-p", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.01, "visible": False}), "deepbooru_sep": OptionInfo("

DeepBooru

", "", gr.HTML), "deepbooru_score_threshold": OptionInfo(0.65, "DeepBooru: score threshold", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}), diff --git a/modules/ui_caption.py b/modules/ui_caption.py index a306122e0..dbf00a916 100644 --- a/modules/ui_caption.py +++ b/modules/ui_caption.py @@ -4,11 +4,14 @@ from modules.interrogate import openclip def update_vlm_params(*args): - vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample = args + vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p = args shared.opts.interrogate_vlm_max_length = vlm_max_tokens shared.opts.interrogate_vlm_num_beams = vlm_num_beams shared.opts.interrogate_vlm_temperature = vlm_temperature shared.opts.interrogate_vlm_do_sample = vlm_do_sample + shared.opts.interrogate_vlm_top_k = vlm_top_k + shared.opts.interrogate_vlm_top_p = vlm_top_p + shared.opts.save(shared.config_filename) def create_ui(): @@ -19,67 +22,80 @@ def create_ui(): with gr.Tabs(elem_id="mode_caption"): with gr.Tab("CLiP Interrogate"): with gr.Row(): - clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model') + clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model', elem_id='clip_clip_model') ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models') - blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model') - mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast') + blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model', elem_id='clip_blip_model') + clip_mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast', elem_id='clip_clip_mode') with gr.Accordion(label='Advanced', open=False, visible=True): with gr.Row(): - caption_max_length = gr.Slider(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=512, min_width=300) - chunk_size = gr.Slider(label='Chunk size', value=1024, minimum=256, maximum=4096, min_width=300) + clip_caption_max_length = gr.Slider(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=512, elem_id='clip_caption_max_length') + clip_chunk_size = gr.Slider(label='Chunk size', value=1024, minimum=256, maximum=4096, elem_id='clip_chunk_size') with gr.Row(): - min_flavors = gr.Slider(label='Min flavors', value=2, minimum=1, maximum=16, min_width=300) - max_flavors = gr.Slider(label='Max flavors', value=8, minimum=1, maximum=64, min_width=300) - flavor_intermediate_count = gr.Slider(label='Intermediates', value=1024, minimum=256, maximum=4096) - caption_max_length.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) - chunk_size.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) - min_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) - max_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) - flavor_intermediate_count.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) + clip_min_flavors = gr.Slider(label='Min flavors', value=2, minimum=1, maximum=16, elem_id='clip_min_flavors') + clip_max_flavors = gr.Slider(label='Max flavors', value=8, minimum=1, maximum=64, elem_id='clip_max_flavors') + clip_flavor_intermediate_count = gr.Slider(label='Intermediates', value=1024, minimum=256, maximum=4096, elem_id='clip_flavor_intermediate_count') + clip_caption_max_length.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[]) + clip_chunk_size.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[]) + clip_min_flavors.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[]) + clip_max_flavors.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[]) + clip_flavor_intermediate_count.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[]) with gr.Accordion(label='Batch', open=False, visible=True): with gr.Row(): - batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100) + clip_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_files') with gr.Row(): - batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100) + clip_batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_folder') with gr.Row(): - batch_str = gr.Text(label="Folder", value="", interactive=True) + clip_batch_str = gr.Text(label="Folder", value="", interactive=True, elem_id='clip_batch_str') with gr.Row(): - batch = gr.Text(label="Prompts", lines=10) + clip_save_output = gr.Checkbox(label='Save caption files', value=True, elem_id="clip_save_output") + clip_save_append = gr.Checkbox(label='Append caption files', value=False, elem_id="clip_save_append") with gr.Row(): - clip_model = gr.Dropdown([], value='ViT-L-14/openai', label='CLiP Batch Model') - ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models') - with gr.Row(elem_id='interrogate_buttons_batch'): - btn_interrogate_batch = gr.Button("Batch interrogate", elem_id="interrogate_btn_interrogate", variant='primary') - with gr.Row(): - save_output = gr.Checkbox(label='Save output', value=True, elem_id="extras_save_output") - with gr.Row(elem_id='interrogate_buttons_image'): - btn_interrogate_img = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary') - btn_analyze_img = gr.Button("Analyze", elem_id="interrogate_btn_analyze", variant='primary') + btn_clip_interrogate_batch = gr.Button("Batch interrogate", variant='primary', elem_id="btn_clip_interrogate_batch") + with gr.Row(): + btn_clip_interrogate_img = gr.Button("Interrogate", variant='primary', elem_id="btn_clip_interrogate_img") + btn_clip_analyze_img = gr.Button("Analyze", variant='primary', elem_id="btn_clip_analyze_img") with gr.Tab("VLM Caption"): from modules.interrogate import vqa with gr.Row(): - vqa_question = gr.Dropdown(label="Predefined question", allow_custom_value=False, choices=vqa.vlm_prompts, value=vqa.vlm_prompts[2]) + vlm_question = gr.Dropdown(label="Predefined question", allow_custom_value=False, choices=vqa.vlm_prompts, value=vqa.vlm_prompts[2], elem_id='vlm_question') with gr.Row(): - vqa_prompt = gr.Textbox(label="Prompt", placeholder="optionally enter custom prompt", lines=2) + vlm_prompt = gr.Textbox(label="Prompt", placeholder="optionally enter custom prompt", lines=2, elem_id='vlm_prompt') with gr.Row(elem_id='interrogate_buttons_query'): - vqa_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model') + vlm_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model', elem_id='vlm_model') with gr.Accordion(label='Advanced', open=False, visible=True): with gr.Row(): - vlm_max_tokens = gr.Slider(label='Max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1) - vlm_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1) - vlm_temperature = gr.Slider(label='Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01) + vlm_max_tokens = gr.Slider(label='Max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1, elem_id='vlm_max_tokens') + vlm_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1, elem_id='vlm_num_beams') + vlm_temperature = gr.Slider(label='Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01, elem_id='vlm_temperature') with gr.Row(): - vlm_do_sample = gr.Checkbox(label='Use sample', value=shared.opts.interrogate_vlm_do_sample) - vlm_max_tokens.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[]) - vlm_num_beams.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[]) - vlm_temperature.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[]) - vlm_do_sample.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[]) - with gr.Row(elem_id='interrogate_buttons_query'): - vqa_submit = gr.Button("Caption", elem_id="interrogate_btn_interrogate", variant='primary') + vlm_top_k = gr.Slider(label='Top-K', value=shared.opts.interrogate_vlm_top_k, minimum=0, maximum=99, step=1, elem_id='vlm_top_k') + vlm_top_p = gr.Slider(label='Top-P', value=shared.opts.interrogate_vlm_top_p, minimum=0.0, maximum=1.0, step=0.01, elem_id='vlm_top_p') + with gr.Row(): + vlm_do_sample = gr.Checkbox(label='Use sample', value=shared.opts.interrogate_vlm_do_sample, elem_id='vlm_do_sample') + vlm_max_tokens.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[]) + vlm_num_beams.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[]) + vlm_temperature.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[]) + vlm_do_sample.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[]) + vlm_top_k.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[]) + vlm_top_p.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[]) + with gr.Accordion(label='Batch', open=False, visible=True): + with gr.Row(): + vlm_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100, elem_id='vlm_batch_files') + with gr.Row(): + vlm_batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100, elem_id='vlm_batch_folder') + with gr.Row(): + vlm_batch_str = gr.Text(label="Folder", value="", interactive=True, elem_id='vlm_batch_str') + with gr.Row(): + vlm_save_output = gr.Checkbox(label='Save caption files', value=True, elem_id="vlm_save_output") + vlm_save_append = gr.Checkbox(label='Append caption files', value=False, elem_id="vlm_save_append") + with gr.Row(elem_id='interrogate_buttons_batch'): + btn_vlm_caption_batch = gr.Button("Batch caption", variant='primary', elem_id="btn_vlm_caption_batch") + with gr.Row(): + btn_vlm_caption = gr.Button("Caption", variant='primary', elem_id="btn_vlm_caption") with gr.Column(variant='compact'): with gr.Row(): prompt = gr.Textbox(label="Answer", lines=8, placeholder="ai generated image description") - with gr.Row(elem_id="interrogate_labels"): + with gr.Row(): medium = gr.Label(elem_id="interrogate_label_medium", label="Medium", num_top_classes=5, visible=False) artist = gr.Label(elem_id="interrogate_label_artist", label="Artist", num_top_classes=5, visible=False) movement = gr.Label(elem_id="interrogate_label_movement", label="Movement", num_top_classes=5, visible=False) @@ -88,10 +104,11 @@ def create_ui(): with gr.Row(elem_id='copy_buttons_interrogate'): copy_interrogate_buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "control", "extras"]) - btn_interrogate_img.click(openclip.interrogate_image, inputs=[image, clip_model, blip_model, mode], outputs=[prompt]) - btn_analyze_img.click(openclip.analyze_image, inputs=[image, clip_model, blip_model], outputs=[medium, artist, movement, trending, flavor]) - btn_interrogate_batch.click(fn=openclip.interrogate_batch, inputs=[batch_files, batch_folder, batch_str, clip_model, blip_model, mode, save_output], outputs=[batch]) - vqa_submit.click(vqa.interrogate, inputs=[vqa_question, vqa_prompt, image, vqa_model], outputs=[prompt]) + btn_clip_interrogate_img.click(openclip.interrogate_image, inputs=[image, clip_model, blip_model, clip_mode], outputs=[prompt]) + btn_clip_analyze_img.click(openclip.analyze_image, inputs=[image, clip_model, blip_model], outputs=[medium, artist, movement, trending, flavor]) + btn_clip_interrogate_batch.click(fn=openclip.interrogate_batch, inputs=[clip_batch_files, clip_batch_folder, clip_batch_str, clip_model, blip_model, clip_mode, clip_save_output, clip_save_append], outputs=[prompt]) + btn_vlm_caption.click(fn=vqa.interrogate, inputs=[vlm_question, vlm_prompt, image, vlm_model], outputs=[prompt]) + btn_vlm_caption_batch.click(fn=vqa.batch, inputs=[vlm_model, vlm_batch_files, vlm_batch_folder, vlm_batch_str, vlm_question, vlm_prompt, vlm_save_output, vlm_save_append], outputs=[prompt]) for tabname, button in copy_interrogate_buttons.items(): generation_parameters_copypaste.register_paste_params_button(generation_parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=prompt, source_image_component=image,)) diff --git a/requirements.txt b/requirements.txt index e15d141a2..f61890cbe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,7 +40,7 @@ compel==2.0.3 torchsde==0.2.6 antlr4-python3-runtime==4.9.3 requests==2.32.3 -tqdm==4.66.5 +tqdm==4.67.1 accelerate==1.3.0 opencv-contrib-python-headless==4.9.0.80 einops==0.4.1