diff --git a/CHANGELOG.md b/CHANGELOG.md index a20011288..515887c3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,12 @@ # Change Log for SD.Next -## Update for 2025-02-14 +## Update for 2025-02-16 ### TODO - VLM ModernUI support -- CLiP Move settings -- CLiP Batch progress bar -### Highlight for 2025-02-14 +### Highlight for 2025-02-16 We're back with another update with over 50 commits! - Starting with massive UI update with full [localization](https://vladmandic.github.io/sdnext-docs/Locale/) for 8 languages @@ -19,13 +17,13 @@ We're back with another update with over 50 commits! - And new **Mixture-of-Diffusers** regional tiling pipeline - Follow-up to last weeks **interrogate/captioning** rewrite now with redesigned captioning UI and - added **JoyTag**, **JoyCaption**, **Google PaliGemma**, **ToriiGate 0.4** to list of supported models + added **JoyTag**, **JoyCaption**, **PaliGemma**, **ToriiGate**, **Ovis2** to list of supported models - Some changes to **prompt parsing** to allow more control as well as more flexibility when mouting SDNext server to custom URL - Of course, cumulative fixes... *...and more* - see [changelog](https://github.com/vladmandic/sdnext/blob/dev/CHANGELOG.md) for full details! -### Details for 2025-02-14 +### Details for 2025-02-16 - **User Interface** - **Hints** @@ -64,19 +62,25 @@ We're back with another update with over 50 commits! - Redesigned captioning UI split from Process tab into separate tab split `clip` vs `vlm` models processing - direct *send-to* buttons on all tabs - - Add VLM advanced params: max-tokens, num-beams, temperature, top-k, top-p, do-sample - params are saved in `config.json` and used when using quick interrogate + direct *send-to* buttons on all tabs: txt/img/ctrl->process/caption, process/caption->txt/img/ctrl + - Advanced params: + VLM: *max-tokens, num-beams, temperature, top-k, top-p, do-sample* + CLiP: *min-length, max-length, chunk-size, min-flavors, max-flavors, flavor-count, num-beams* + params are auto-saved in `config.json` and used when using quick interrogate params that are set to 0 mean use model defaults - - Add VLM batch processing + - Batch processing: VLM and CLiP for example, can be used to caption your training dataset in one go add option to append to captions file, can be used to run multiple captioning models in sequence add progress bar - Add additional VLM models: [JoyTag](https://huggingface.co/fancyfeast/joytag) [JoyCaption 2](https://huggingface.co/fancyfeast/llama-joycaption-alpha-two-hf-llava) - [Google PaliGemma 2](https://huggingface.co/google/paligemma2-3b-pt-224) - [ToriiGate 0.4 7B](https://huggingface.co/Minthy/ToriiGate-v0.4-7B) + [Google PaliGemma 2](https://huggingface.co/google/paligemma2-3b-pt-224) 3B + [ToriiGate 0.4](https://huggingface.co/Minthy/ToriiGate-v0.4-7B) 7B + [AIDC Ovis2](https://huggingface.co/AIDC-AI/Ovis2-1B) 1B/2B/4B + - *Note* some models require `flash-attn` to be installed + due to binary/build dependencies, it should not be done automatically, + see [flash-attn](https://github.com/Dao-AILab/flash-attention) for installation instructions - **Docker** - updated **CUDA** receipe to `torch==2.6.0` with `cuda==12.6` and add prebuilt image - added **ROCm** receipe and prebuilt image diff --git a/modules/interrogate/openclip.py b/modules/interrogate/openclip.py index 3449dba28..68bd90b46 100644 --- a/modules/interrogate/openclip.py +++ b/modules/interrogate/openclip.py @@ -13,15 +13,6 @@ from torchvision.transforms.functional import InterpolationMode from modules import devices, paths, shared, lowvram, errors, sd_models -config = { - "caption_max_length": 74, - "chunk_size": 1024, - "flavor_intermediate_count": 1024, - "min_flavors": 2, - "max_flavors": 8, - "clip_offload": True, - "caption_offload": True, -} caption_models = { 'blip-base': 'Salesforce/blip-image-captioning-base', 'blip-large': 'Salesforce/blip-image-captioning-large', @@ -244,21 +235,13 @@ class BatchWriter: self.file.close() -def update_interrogate_params(caption_max_length:int=None, chunk_size:int=None, min_flavors:int=None, max_flavors:int=None, flavor_intermediate_count:int=None): - config["caption_max_length"] = int(caption_max_length or shared.opts.interrogate_clip_max_length) - config["clip_offload"] = shared.opts.interrogate_offload - config["caption_offload"] = shared.opts.interrogate_offload - config["min_flavors"] = int(min_flavors or shared.opts.interrogate_clip_min_flavors) - config["max_flavors"] = int(max_flavors or shared.opts.interrogate_clip_max_flavors) - if chunk_size is not None: - config["chunk_size"] = int(chunk_size) - if flavor_intermediate_count is not None: - config["flavor_intermediate_count"] = int(flavor_intermediate_count) +def update_interrogate_params(): if ci is not None: - ci.config.caption_max_length = config["caption_max_length"] - ci.config.chunk_size = config["chunk_size"] - ci.config.flavor_intermediate_count = config["flavor_intermediate_count"] - shared.log.debug(f'Interrogate: type={shared.opts.interrogate_default_type} config={config}') + ci.caption_max_length=shared.opts.interrogate_clip_max_length, + ci.chunk_size=shared.opts.interrogate_clip_chunk_size, + ci.flavor_intermediate_count=shared.opts.interrogate_clip_flavor_count, + ci.clip_offload=shared.opts.interrogate_offload, + ci.caption_offload=shared.opts.interrogate_offload, def get_clip_models(): @@ -288,11 +271,11 @@ def load_interrogator(clip_model, blip_model): clip_model_name=clip_model, caption_model_name=blip_model, quiet=True, - caption_max_length=config['caption_max_length'], - chunk_size=config['chunk_size'], - flavor_intermediate_count=config['flavor_intermediate_count'], - clip_offload=config['clip_offload'], - caption_offload=config['caption_offload'], + caption_max_length=shared.opts.interrogate_clip_max_length, + chunk_size=shared.opts.interrogate_clip_chunk_size, + flavor_intermediate_count=shared.opts.interrogate_clip_flavor_count, + clip_offload=shared.opts.interrogate_offload, + caption_offload=shared.opts.interrogate_offload, ) ci = clip_interrogator.Interrogator(interrogator_config) elif clip_model != ci.config.clip_model_name or blip_model != ci.config.caption_model_name: @@ -322,15 +305,15 @@ def interrogate(image, mode, caption=None): return '' image = image.convert("RGB") if mode == 'best': - prompt = ci.interrogate(image, caption=caption, min_flavors=config["min_flavors"], max_flavors=config["max_flavors"]) + prompt = ci.interrogate(image, caption=caption, min_flavors=shared.opts.interrogate_clip_min_flavors, max_flavors=shared.opts.interrogate_clip_max_flavors, ) elif mode == 'caption': prompt = ci.generate_caption(image) if caption is None else caption elif mode == 'classic': - prompt = ci.interrogate_classic(image, caption=caption, max_flavors=config["max_flavors"]) + prompt = ci.interrogate_classic(image, caption=caption, max_flavors=shared.opts.interrogate_clip_max_flavors) elif mode == 'fast': - prompt = ci.interrogate_fast(image, caption=caption, max_flavors=config["max_flavors"]) + prompt = ci.interrogate_fast(image, caption=caption, max_flavors=shared.opts.interrogate_clip_max_flavors) elif mode == 'negative': - prompt = ci.interrogate_negative(image, max_flavors=config["max_flavors"]) + prompt = ci.interrogate_negative(image, max_flavors=shared.opts.interrogate_clip_max_flavors) else: raise RuntimeError(f"Unknown mode {mode}") return prompt @@ -342,7 +325,7 @@ def interrogate_image(image, clip_model, blip_model, mode): if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): lowvram.send_everything_to_cpu() devices.torch_gc() - if shared.native: + if shared.native and shared.sd_loaded: sd_models.apply_balanced_offload(shared.sd_model) load_interrogator(clip_model, blip_model) image = image.convert('RGB') @@ -365,50 +348,35 @@ def interrogate_batch(batch_files, batch_folder, batch_str, clip_model, blip_mod if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str): files += [os.path.join(batch_str, f) for f in os.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))] if len(files) == 0: - shared.log.error('Interrogate batch no images') + shared.log.warning('Interrogate batch: type=clip no images') return '' - shared.state.begin('Batch interrogate') + shared.state.begin('Interrogate batch') prompts = [] - try: - if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): - lowvram.send_everything_to_cpu() - devices.torch_gc() - load_interrogator(clip_model, blip_model) - shared.log.info(f'Interrogate batch: images={len(files)} mode={mode} config={ci.config}') - captions = [] - # first pass: generate captions + + load_interrogator(clip_model, blip_model) + if write: + file_mode = 'w' if not append else 'a' + writer = BatchWriter(os.path.dirname(files[0]), mode=file_mode) + import rich.progress as rp + pbar = rp.Progress(rp.TextColumn('[cyan]Caption:'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) + with pbar: + task = pbar.add_task(total=len(files), description='starting...') for file in files: - caption = "" + pbar.update(task, advance=1, description=file) try: if shared.state.interrupted: break image = Image.open(file).convert('RGB') - caption = ci.generate_caption(image) - except Exception as e: - shared.log.error(f'Interrogate caption: {e}') - finally: - captions.append(caption) - # second pass: interrogate - if write: - mode = 'w' if not append else 'a' - writer = BatchWriter(os.path.dirname(files[0]), mode=mode) - for idx, file in enumerate(files): - try: - if shared.state.interrupted: - break - image = Image.open(file).convert('RGB') - prompt = interrogate(image, mode, caption=captions[idx]) + prompt = interrogate(image, mode) prompts.append(prompt) if write: writer.add(file, prompt) except OSError as e: shared.log.error(f'Interrogate batch: {e}') - if write: - writer.close() - ci.config.quiet = False - unload_clip_model() - except Exception as e: - shared.log.error(f'Interrogate batch: {e}') + if write: + writer.close() + ci.config.quiet = False + unload_clip_model() shared.state.end() return '\n\n'.join(prompts) diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index f98a566b4..38f3d4723 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -7,7 +7,7 @@ import torch import transformers import transformers.dynamic_module_utils from PIL import Image -from modules import shared, devices, errors +from modules import shared, devices, errors, sd_models processor = None model = None @@ -37,6 +37,9 @@ vlm_models = { "Google PaliGemma 2 3B": "google/paligemma2-3b-pt-224", "JoyCaption": "fancyfeast/llama-joycaption-alpha-two-hf-llava", # 0.7GB "JoyTag": "fancyfeast/joytag", # 17.4GB + "AIDC Ovis2 1B": "AIDC-AI/Ovis2-1B", + "AIDC Ovis2 2B": "AIDC-AI/Ovis2-2B", + "AIDC Ovis2 4B": "AIDC-AI/Ovis2-4B", # "DeepSeek VL2 Tiny": "deepseek-ai/deepseek-vl2-tiny", # broken # "nVidia Eagle 2 1B": "nvidia/Eagle2-1B", # not compatible with latest transformers } @@ -74,7 +77,7 @@ def clean(response, question): response = json.dumps(response) if isinstance(response, list): response = response[0] - question = question.replace('<', '').replace('>', '') + question = question.replace('<', '').replace('>', '').replace('_', ' ') if question in response: response = response.split(question, 1)[1] response = response.replace('\n', '').replace('\r', '').replace('\t', '').strip() @@ -110,9 +113,7 @@ def qwen(question: str, image: Image.Image, repo: str = None): processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) loaded = repo model = model.to(devices.device, devices.dtype) - if len(question) < 2: - question = "Describe the image." - question = question.replace('<', '').replace('>', '') + question = question.replace('<', '').replace('>', '').replace('_', ' ') conversation = [ { "role": "system", @@ -152,9 +153,7 @@ def paligemma(question: str, image: Image.Image, repo: str = None): model = transformers.PaliGemmaForConditionalGeneration.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype) loaded = repo model = model.to(devices.device, devices.dtype) - if len(question) < 2: - question = "Describe the image." - question = question.replace('<', '').replace('>', '') + question = question.replace('<', '').replace('>', '').replace('_', ' ') model_inputs = processor(text=question, images=image, return_tensors="pt").to(devices.device, devices.dtype) input_len = model_inputs["input_ids"].shape[-1] with devices.inference_context(): @@ -167,6 +166,45 @@ def paligemma(question: str, image: Image.Image, repo: str = None): return response +def ovis(question: str, image: Image.Image, repo: str = None): + try: + import flash_attn # pylint: disable=unused-import + except Exception: + shared.log.error(f'Interrogate: vlm="{repo}" flash-attn is not available') + return '' + global model, loaded # pylint: disable=global-statement + if model is None or loaded != repo: + shared.log.debug(f'Interrogate load: vlm="{repo}"') + model = transformers.AutoModelForCausalLM.from_pretrained(repo, torch_dtype=devices.dtype, multimodal_max_length=32768, trust_remote_code=True) + loaded = repo + model = model.to(devices.device, devices.dtype) + text_tokenizer = model.get_text_tokenizer() + visual_tokenizer = model.get_visual_tokenizer() + max_partition = 9 + question = question.replace('<', '').replace('>', '').replace('_', ' ') + question = f'\n{question}' + _prompt, input_ids, pixel_values = model.preprocess_inputs(question, [image], max_partition=max_partition) + attention_mask = torch.ne(input_ids, text_tokenizer.pad_token_id) + input_ids = input_ids.unsqueeze(0).to(device=model.device) + attention_mask = attention_mask.unsqueeze(0).to(device=model.device) + if pixel_values is not None: + pixel_values = pixel_values.to(dtype=visual_tokenizer.dtype, device=visual_tokenizer.device) + pixel_values = [pixel_values] + with devices.inference_context(): + output_ids = model.generate( + input_ids, + pixel_values=pixel_values, + attention_mask=attention_mask, + repetition_penalty=None, + eos_token_id=model.generation_config.eos_token_id, + pad_token_id=text_tokenizer.pad_token_id, + use_cache=True, + **get_kwargs()) + response = text_tokenizer.decode(output_ids[0], skip_special_tokens=True) + print(f'Output:\n{response}') + return response + + def smol(question: str, image: Image.Image, repo: str = None): global processor, model, loaded # pylint: disable=global-statement if model is None or loaded != repo: @@ -180,9 +218,7 @@ def smol(question: str, image: Image.Image, repo: str = None): processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) loaded = repo model.to(devices.device, devices.dtype) - if len(question) < 2: - question = "Describe the image." - question = question.replace('<', '').replace('>', '') + question = question.replace('<', '').replace('>', '').replace('_', ' ') conversation = [ { "role": "system", @@ -298,9 +334,7 @@ def moondream(question: str, image: Image.Image, repo: str = None): loaded = repo model.eval() model.to(devices.device, devices.dtype) - if len(question) < 2: - question = "Describe the image." - question = question.replace('<', '').replace('>', '') + question = question.replace('<', '').replace('>', '').replace('_', ' ') encoded = model.encode_image(image) with devices.inference_context(): response = model.answer_question(encoded, question, processor) @@ -331,7 +365,6 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str task = question.split('>', 1)[0] + '>' else: task = '' - # question = task + question inputs = processor(text=task, images=image, return_tensors="pt") input_ids = inputs['input_ids'].to(devices.device) pixel_values = inputs['pixel_values'].to(devices.device, devices.dtype) @@ -348,7 +381,7 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str def interrogate(question, prompt, image, model_name, quiet:bool=False): if not quiet: - shared.state.begin('Caption') + shared.state.begin('Interrogate') t0 = time.time() if isinstance(image, list): image = image[0] if len(image) > 0 else None @@ -362,6 +395,10 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False): image = image.convert('RGB') if prompt is not None and len(prompt) > 0: question = prompt + if len(question) < 2: + question = "Describe the image." + if shared.native and shared.sd_loaded: + sd_models.apply_balanced_offload(shared.sd_model) from modules import modelloader modelloader.hf_login() try: @@ -402,6 +439,8 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False): answer = deepseek.predict(question, image, vqa_model) elif 'paligemma' in vqa_model.lower(): answer = paligemma(question, image, vqa_model) + elif 'ovis' in vqa_model.lower(): + answer = ovis(question, image, vqa_model) else: answer = 'unknown model' except Exception as e: @@ -443,16 +482,16 @@ def batch(model_name, batch_files, batch_folder, batch_str, question, prompt, wr if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str): files += [os.path.join(batch_str, f) for f in os.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))] if len(files) == 0: - shared.log.error('Interrogate batch no images') + shared.log.warning('Interrogate batch: type=vlm no images') return '' - shared.state.begin('Caption batch') + shared.state.begin('Interrogate batch') prompts = [] if write: mode = 'w' if not append else 'a' writer = BatchWriter(os.path.dirname(files[0]), mode=mode) - import rich.progress as rp orig_offload = shared.opts.interrogate_offload shared.opts.interrogate_offload = False + import rich.progress as rp pbar = rp.Progress(rp.TextColumn('[cyan]Caption:'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) with pbar: task = pbar.add_task(total=len(files), description='starting...') diff --git a/modules/shared.py b/modules/shared.py index 1b36a2e83..c327d1270 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -917,11 +917,13 @@ options_templates.update(options_section(('interrogate', "Interrogate"), { "interrogate_clip_model": OptionInfo("ViT-L-14/openai", "CLiP: default model", gr.Dropdown, lambda: {"choices": get_clip_models()}, refresh=refresh_clip_models), "interrogate_clip_mode": OptionInfo(caption_types[0], "CLiP: default mode", gr.Dropdown, {"choices": caption_types}), "interrogate_blip_model": OptionInfo(list(caption_models)[0], "CLiP: default captioner", gr.Dropdown, {"choices": list(caption_models)}), - "interrogate_clip_num_beams": OptionInfo(1, "CLiP: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1}), - "interrogate_clip_min_length": OptionInfo(32, "CLiP: min length", gr.Slider, {"minimum": 1, "maximum": 128, "step": 1}), - "interrogate_clip_max_length": OptionInfo(74, "CLiP: max length", gr.Slider, {"minimum": 1, "maximum": 512, "step": 1}), - "interrogate_clip_min_flavors": OptionInfo(2, "CLiP: min flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1}), - "interrogate_clip_max_flavors": OptionInfo(8, "CLiP: max flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1}), + "interrogate_clip_num_beams": OptionInfo(1, "CLiP: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}), + "interrogate_clip_min_length": OptionInfo(32, "CLiP: min length", gr.Slider, {"minimum": 1, "maximum": 128, "step": 1, "visible": False}), + "interrogate_clip_max_length": OptionInfo(74, "CLiP: max length", gr.Slider, {"minimum": 1, "maximum": 512, "step": 1, "visible": False}), + "interrogate_clip_min_flavors": OptionInfo(2, "CLiP: min flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1, "visible": False}), + "interrogate_clip_max_flavors": OptionInfo(16, "CLiP: max flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1, "visible": False}), + "interrogate_clip_flavor_count": OptionInfo(16, "CLiP: intermediate flavors", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1, "visible": False}), + "interrogate_clip_chunk_size": OptionInfo(1024, "CLiP: chunk size", gr.Slider, {"minimum": 256, "maximum": 4096, "step": 8, "visible": False}), "interrogate_clip_skip_categories": OptionInfo(["artists", "movements", "flavors"], "CLiP: skip categories", gr.CheckboxGroup, lambda: {"choices": category_types()}, refresh=category_types), "interrogate_vlm_sep": OptionInfo("

VLM

", "", gr.HTML), diff --git a/modules/ui_caption.py b/modules/ui_caption.py index dbf00a916..a13d2f846 100644 --- a/modules/ui_caption.py +++ b/modules/ui_caption.py @@ -14,46 +14,28 @@ def update_vlm_params(*args): shared.opts.save(shared.config_filename) +def update_clip_params(*args): + """ + "interrogate_clip_num_beams": OptionInfo(1, "CLiP: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}), + """ + clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams = args + shared.opts.interrogate_clip_min_length = int(clip_min_length) + shared.opts.interrogate_clip_max_length = int(clip_max_length) + shared.opts.interrogate_clip_min_flavors = int(clip_min_flavors) + shared.opts.interrogate_clip_max_flavors = int(clip_max_flavors) + shared.opts.interrogate_clip_num_beams = int(clip_num_beams) + shared.opts.interrogate_clip_flavor_count = int(clip_flavor_count) + shared.opts.interrogate_clip_chunk_size = int(clip_chunk_size) + shared.opts.save(shared.config_filename) + openclip.update_interrogate_params() + + def create_ui(): with gr.Row(equal_height=False, variant='compact', elem_classes="caption"): with gr.Column(variant='compact'): with gr.Row(): image = gr.Image(type='pil', label="Image") with gr.Tabs(elem_id="mode_caption"): - with gr.Tab("CLiP Interrogate"): - with gr.Row(): - clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model', elem_id='clip_clip_model') - ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models') - blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model', elem_id='clip_blip_model') - clip_mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast', elem_id='clip_clip_mode') - with gr.Accordion(label='Advanced', open=False, visible=True): - with gr.Row(): - clip_caption_max_length = gr.Slider(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=512, elem_id='clip_caption_max_length') - clip_chunk_size = gr.Slider(label='Chunk size', value=1024, minimum=256, maximum=4096, elem_id='clip_chunk_size') - with gr.Row(): - clip_min_flavors = gr.Slider(label='Min flavors', value=2, minimum=1, maximum=16, elem_id='clip_min_flavors') - clip_max_flavors = gr.Slider(label='Max flavors', value=8, minimum=1, maximum=64, elem_id='clip_max_flavors') - clip_flavor_intermediate_count = gr.Slider(label='Intermediates', value=1024, minimum=256, maximum=4096, elem_id='clip_flavor_intermediate_count') - clip_caption_max_length.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[]) - clip_chunk_size.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[]) - clip_min_flavors.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[]) - clip_max_flavors.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[]) - clip_flavor_intermediate_count.change(fn=openclip.update_interrogate_params, inputs=[clip_caption_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_intermediate_count], outputs=[]) - with gr.Accordion(label='Batch', open=False, visible=True): - with gr.Row(): - clip_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_files') - with gr.Row(): - clip_batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_folder') - with gr.Row(): - clip_batch_str = gr.Text(label="Folder", value="", interactive=True, elem_id='clip_batch_str') - with gr.Row(): - clip_save_output = gr.Checkbox(label='Save caption files', value=True, elem_id="clip_save_output") - clip_save_append = gr.Checkbox(label='Append caption files', value=False, elem_id="clip_save_append") - with gr.Row(): - btn_clip_interrogate_batch = gr.Button("Batch interrogate", variant='primary', elem_id="btn_clip_interrogate_batch") - with gr.Row(): - btn_clip_interrogate_img = gr.Button("Interrogate", variant='primary', elem_id="btn_clip_interrogate_img") - btn_clip_analyze_img = gr.Button("Analyze", variant='primary', elem_id="btn_clip_analyze_img") with gr.Tab("VLM Caption"): from modules.interrogate import vqa with gr.Row(): @@ -62,7 +44,7 @@ def create_ui(): vlm_prompt = gr.Textbox(label="Prompt", placeholder="optionally enter custom prompt", lines=2, elem_id='vlm_prompt') with gr.Row(elem_id='interrogate_buttons_query'): vlm_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model', elem_id='vlm_model') - with gr.Accordion(label='Advanced', open=False, visible=True): + with gr.Accordion(label='Advanced options', open=False, visible=True): with gr.Row(): vlm_max_tokens = gr.Slider(label='Max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1, elem_id='vlm_max_tokens') vlm_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1, elem_id='vlm_num_beams') @@ -78,7 +60,7 @@ def create_ui(): vlm_do_sample.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[]) vlm_top_k.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[]) vlm_top_p.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample, vlm_top_k, vlm_top_p], outputs=[]) - with gr.Accordion(label='Batch', open=False, visible=True): + with gr.Accordion(label='Batch caption', open=False, visible=True): with gr.Row(): vlm_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100, elem_id='vlm_batch_files') with gr.Row(): @@ -92,6 +74,45 @@ def create_ui(): btn_vlm_caption_batch = gr.Button("Batch caption", variant='primary', elem_id="btn_vlm_caption_batch") with gr.Row(): btn_vlm_caption = gr.Button("Caption", variant='primary', elem_id="btn_vlm_caption") + with gr.Tab("CLiP Interrogate"): + with gr.Row(): + clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model', elem_id='clip_clip_model') + ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models') + blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model', elem_id='clip_blip_model') + clip_mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast', elem_id='clip_clip_mode') + with gr.Accordion(label='Advanced options', open=False, visible=True): + with gr.Row(): + clip_min_length = gr.Slider(label='Min length', value=shared.opts.interrogate_clip_min_length, minimum=8, maximum=75, step=1, elem_id='clip_caption_min_length') + clip_max_length = gr.Slider(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=1024, step=1, elem_id='clip_caption_max_length') + clip_chunk_size = gr.Slider(label='Chunk size', value=shared.opts.interrogate_clip_chunk_size, minimum=256, maximum=4096, step=8, elem_id='clip_chunk_size') + with gr.Row(): + clip_min_flavors = gr.Slider(label='Min flavors', value=shared.opts.interrogate_clip_min_flavors, minimum=1, maximum=16, step=1, elem_id='clip_min_flavors') + clip_max_flavors = gr.Slider(label='Max flavors', value=shared.opts.interrogate_clip_max_flavors, minimum=1, maximum=64, step=1, elem_id='clip_max_flavors') + clip_flavor_count = gr.Slider(label='Intermediates', value=shared.opts.interrogate_clip_flavor_count, minimum=256, maximum=4096, step=8, elem_id='clip_flavor_intermediate_count') + with gr.Row(): + clip_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_clip_num_beams, minimum=1, maximum=16, step=1, elem_id='clip_num_beams') + clip_min_length.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + clip_max_length.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + clip_chunk_size.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + clip_min_flavors.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + clip_max_flavors.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + clip_flavor_count.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + clip_num_beams.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) + with gr.Accordion(label='Batch interogate', open=False, visible=True): + with gr.Row(): + clip_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_files') + with gr.Row(): + clip_batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100, elem_id='clip_batch_folder') + with gr.Row(): + clip_batch_str = gr.Text(label="Folder", value="", interactive=True, elem_id='clip_batch_str') + with gr.Row(): + clip_save_output = gr.Checkbox(label='Save caption files', value=True, elem_id="clip_save_output") + clip_save_append = gr.Checkbox(label='Append caption files', value=False, elem_id="clip_save_append") + with gr.Row(): + btn_clip_interrogate_batch = gr.Button("Batch interrogate", variant='primary', elem_id="btn_clip_interrogate_batch") + with gr.Row(): + btn_clip_interrogate_img = gr.Button("Interrogate", variant='primary', elem_id="btn_clip_interrogate_img") + btn_clip_analyze_img = gr.Button("Analyze", variant='primary', elem_id="btn_clip_analyze_img") with gr.Column(variant='compact'): with gr.Row(): prompt = gr.Textbox(label="Answer", lines=8, placeholder="ai generated image description")