diff --git a/CHANGELOG.md b/CHANGELOG.md index db4ffa67f..537c24247 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,9 @@ We're back with another update with over 50 commits! - A lot of [outpainting](https://vladmandic.github.io/sdnext-docs/Outpaint/) goodies - Support for new models: [AlphaVLLM Lumina 2](https://github.com/Alpha-VLLM/Lumina-Image-2.0) and [Ostris Flex.1-Alpha](https://huggingface.co/ostris/Flex.1-alpha) - And new **Mixture-of-Diffusers** regional tiling pipeline -- Following last weeks **interrogate/captioning** rewrite, added **JoyTag** and **JoyCaption** to list of supported models +- Follow-up to last weeks **interrogate/captioning** rewrite + now with redesigned captioning UI and + added **JoyTag**, **JoyCaption**, **Google PaliGemma**, **ToriiGate 0.4** to list of supported models - Some changes to **prompt parsing** to allow more control as well as more flexibility when mouting SDNext server to custom URL - Of course, cumulative fixes... @@ -28,13 +30,13 @@ We're back with another update with over 50 commits! *english, croatian, spanish, french, italian, portuguese, chinese, japanese, korean, russian* - set in *settings -> user interface -> language* - [localization](https://vladmandic.github.io/sdnext-docs/Locale/) documentation - - **UI**: + - **UI** - force browser cache-invalidate on page load -- **Docs** +- **Docs** - New [Outpaint](https://vladmandic.github.io/sdnext-docs/Outpaint/) step-by-step guide - Updated [Docker](https://github.com/vladmandic/sdnext/wiki/Docker) guide includes build and publish and both local and cloud examples -- **Models** +- **Models** - [AlphaVLLM Lumina 2](https://github.com/Alpha-VLLM/Lumina-Image-2.0) new foundation model for image generation based o Gemma-2-2B text encoder and a flow-based diffusion transformer fully supports offloading and on-the-fly quantization @@ -44,18 +46,23 @@ We're back with another update with over 50 commits! result is model smaller than *Flux.1-Dev*, but with similar capabilities fully supports offloading and on-the-fly quantization simply select from *networks -> models -> reference* -- **Pipelines** +- **Pipelines** - [Mixture-of-Diffusers](https://huggingface.co/posts/elismasilva/251775641926329) Regional tiling type of a solution for SDXL models select from *scripts -> mixture of diffusers* - [Automatic Color Inpaint] Automatically creates mask based on selected color and triggers inpaint simply select in *scripts -> automatic color inpaint* when in img2img mode -- **Interrogate/Captioning** +- **Interrogate/Captioning** + - Redesigned captioning UI + split from Process tab into separate tab + split `clip` vs `vlm` models processing + direct *send-to* buttons on all tabs - [JoyTag](https://huggingface.co/fancyfeast/joytag) - - [JoyCaption](https://huggingface.co/fancyfeast/llama-joycaption-alpha-two-hf-llava) - *note*: this is a very large model based on LLama 3.1 -- **Docker** + - [JoyCaption 2](https://huggingface.co/fancyfeast/llama-joycaption-alpha-two-hf-llava) + - [Google PaliGemma 2](https://huggingface.co/google/paligemma2-3b-pt-224) + - [ToriiGate 0.4 7B](https://huggingface.co/Minthy/ToriiGate-v0.4-7B), +- **Docker** - updated **CUDA** receipe to `torch==2.6.0` with `cuda==12.6` and add prebuilt image - added **ROCm** receipe and prebuilt image - added **IPEX** receipe and add prebuilt image @@ -80,7 +87,7 @@ We're back with another update with over 50 commits! - **Styles** ability to save and/or restore prompts before or after parsing of wildcards set in *settings -> networks -> styles* - - **Access tokens** + - **Access tokens** persist *models -> hugginface -> token* persist *models -> civitai -> token* - global switch to lancosz method for all interal resize ops and bicubic for interpolation ops diff --git a/javascript/ui.js b/javascript/ui.js index 7193f9470..9df1a26e4 100644 --- a/javascript/ui.js +++ b/javascript/ui.js @@ -155,6 +155,11 @@ function switch_to_control(...args) { return Array.from(arguments); } +function switch_to_caption(...args) { + switchToTab('Caption'); + return Array.from(arguments); +} + function get_tab_index(tabId) { let res = 0; gradioApp().getElementById(tabId).querySelector('div').querySelectorAll('button') diff --git a/modules/api/endpoints.py b/modules/api/endpoints.py index 52c4b5301..7bdc31c6a 100644 --- a/modules/api/endpoints.py +++ b/modules/api/endpoints.py @@ -113,7 +113,7 @@ def post_vqa(req: models.ReqVQA): image = helpers.decode_base64_to_image(req.image) image = image.convert('RGB') from modules.interrogate import vqa - answer = vqa.interrogate(req.question, image, req.model) + answer = vqa.interrogate(req.question, '', image, req.model) return models.ResVQA(answer=answer) def post_unload_checkpoint(): diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py index fe65efe19..613c09ff2 100644 --- a/modules/generation_parameters_copypaste.py +++ b/modules/generation_parameters_copypaste.py @@ -101,6 +101,8 @@ def create_buttons(tabs_list): name = 'Process' elif name == 'control': name = 'Control' + elif name == 'caption': + name = 'Caption' buttons[tab] = gr.Button(f"➠ {name}", elem_id=f"{tab}_tab") return buttons diff --git a/modules/infotext.py b/modules/infotext.py index 017691681..e8788c2f5 100644 --- a/modules/infotext.py +++ b/modules/infotext.py @@ -90,6 +90,9 @@ def parse(infotext): # debug(f'Negative: {negative}') params = dict(re_param.findall(remaining)) + if len(list(params)) == 0: + params['Prompt'] = infotext + return params params['Prompt'] = prompt params['Negative prompt'] = negative for key, val in params.copy().items(): diff --git a/modules/interrogate/openclip.py b/modules/interrogate/openclip.py index 9cd658db2..fd10d09ab 100644 --- a/modules/interrogate/openclip.py +++ b/modules/interrogate/openclip.py @@ -6,6 +6,7 @@ import threading import re import torch import torch.hub # pylint: disable=ungrouped-imports +import gradio as gr from PIL import Image from torchvision import transforms from torchvision.transforms.functional import InterpolationMode @@ -423,7 +424,13 @@ def analyze_image(image, clip_model, blip_model): movement_ranks = dict(zip(top_movements, ci.similarities(image_features, top_movements))) trending_ranks = dict(zip(top_trendings, ci.similarities(image_features, top_trendings))) flavor_ranks = dict(zip(top_flavors, ci.similarities(image_features, top_flavors))) - return medium_ranks, artist_ranks, movement_ranks, trending_ranks, flavor_ranks + return [ + gr.update(value=medium_ranks, visible=True), + gr.update(value=artist_ranks, visible=True), + gr.update(value=movement_ranks, visible=True), + gr.update(value=trending_ranks, visible=True), + gr.update(value=flavor_ranks, visible=True), + ] interrogator = InterrogateModels() diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index f08169d9b..8ce6e5e77 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -31,7 +31,9 @@ vlm_models = { "Microsoft GIT VQA Base": "microsoft/git-base-vqav2", # 0.7GB "Microsoft GIT VQA Large": "microsoft/git-large-vqav2", # 1.6GB "ToriiGate 0.4 2B": "Minthy/ToriiGate-v0.4-2B", + "ToriiGate 0.4 7B": "Minthy/ToriiGate-v0.4-7B", "ViLT Base": "dandelin/vilt-b32-finetuned-vqa", # 0.5GB + "Google PaliGemma 2 3B": "google/paligemma2-3b-pt-224", "JoyCaption": "fancyfeast/llama-joycaption-alpha-two-hf-llava", # 0.7GB "JoyTag": "fancyfeast/joytag", # 17.4GB # "DeepSeek VL2 Tiny": "deepseek-ai/deepseek-vl2-tiny", # broken @@ -86,7 +88,7 @@ def qwen(question: str, image: Image.Image, repo: str = None): model = transformers.Qwen2VLForConditionalGeneration.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) loaded = repo - model.to(devices.device, devices.dtype) + model = model.to(devices.device, devices.dtype) if len(question) < 2: question = "Describe the image." question = question.replace('<', '').replace('>', '') @@ -121,6 +123,30 @@ def qwen(question: str, image: Image.Image, repo: str = None): return response +def paligemma(question: str, image: Image.Image, repo: str = None): + global processor, model, loaded # pylint: disable=global-statement + if model is None or loaded != repo: + shared.log.debug(f'Interrogate load: vlm="{repo}"') + processor = transformers.PaliGemmaProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) + model = transformers.PaliGemmaForConditionalGeneration.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype) + loaded = repo + model = model.to(devices.device, devices.dtype) + if len(question) < 2: + question = "Describe the image." + question = question.replace('<', '').replace('>', '') + model_inputs = processor(text=question, images=image, return_tensors="pt").to(devices.device, devices.dtype) + input_len = model_inputs["input_ids"].shape[-1] + with devices.inference_context(): + generation = model.generate( + **model_inputs, + max_new_tokens=shared.opts.interrogate_vlm_max_length, + do_sample=shared.opts.interrogate_vlm_do_sample, + ) + generation = generation[0][input_len:] + response = processor.decode(generation, skip_special_tokens=True) + return response + + def smol(question: str, image: Image.Image, repo: str = None): global processor, model, loaded # pylint: disable=global-statement if model is None or loaded != repo: @@ -295,14 +321,14 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str pixel_values=pixel_values, max_new_tokens=shared.opts.interrogate_vlm_max_length, num_beams=shared.opts.interrogate_vlm_num_beams, - do_sample=False + do_sample=shared.opts.interrogate_vlm_do_sample, ) generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0] response = processor.post_process_generation(generated_text, task="task", image_size=(image.width, image.height)) return response -def interrogate(question, image, model_name): +def interrogate(question, prompt, image, model_name): t0 = time.time() if isinstance(image, list): image = image[0] if len(image) > 0 else None @@ -314,6 +340,8 @@ def interrogate(question, image, model_name): image.thumbnail((768, 768), Image.Resampling.HAMMING) if image.mode != 'RGB': image = image.convert('RGB') + if prompt is not None and len(prompt) > 0: + question = prompt from modules import modelloader modelloader.hf_login() try: @@ -352,6 +380,8 @@ def interrogate(question, image, model_name): elif 'deepseek' in vqa_model.lower(): from modules.interrogate import deepseek answer = deepseek.predict(question, image, vqa_model) + elif 'paligemma' in vqa_model.lower(): + answer = paligemma(question, image, vqa_model) else: answer = 'unknown model' except Exception as e: diff --git a/modules/shared.py b/modules/shared.py index d4d6d06c4..b8f5e3fd7 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -926,8 +926,10 @@ options_templates.update(options_section(('interrogate', "Interrogate"), { "interrogate_vlm_sep": OptionInfo("

VLM

", "", gr.HTML), "interrogate_vlm_model": OptionInfo(list(vlm_models)[0], "VLM: default model", gr.Dropdown, {"choices": list(vlm_models)}), "interrogate_vlm_prompt": OptionInfo(vlm_prompts[2], "VLM: default prompt", DropdownEditable, {"choices": vlm_prompts }), - "interrogate_vlm_num_beams": OptionInfo(3, "VLM: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1}), - "interrogate_vlm_max_length": OptionInfo(512, "VLM: max length", gr.Slider, {"minimum": 1, "maximum": 4096, "step": 1}), + "interrogate_vlm_num_beams": OptionInfo(3, "VLM: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}), + "interrogate_vlm_max_length": OptionInfo(512, "VLM: max length", gr.Slider, {"minimum": 1, "maximum": 4096, "step": 1, "visible": False}), + "interrogate_vlm_do_sample": OptionInfo(False, "VLM: use sample method"), + "interrogate_vlm_temperature": OptionInfo(0.6, "VLM: num beams", gr.Slider, {"minimum": 0.1, "maximum": 1.0, "step": 0.11, "visible": False}), "deepbooru_sep": OptionInfo("

DeepBooru

", "", gr.HTML), "deepbooru_score_threshold": OptionInfo(0.65, "DeepBooru: score threshold", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}), diff --git a/modules/ui.py b/modules/ui.py index 04e813cd8..15718dbb9 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -151,6 +151,11 @@ def create_ui(startup_timer = None): ui_postprocessing.create_ui() timer.startup.record("ui-extras") + with gr.Blocks(analytics_enabled=False) as caption_interface: + from modules import ui_caption + ui_caption.create_ui() + timer.startup.record("ui-caption") + with gr.Blocks(analytics_enabled=False) as models_interface: from modules import ui_models ui_models.create_ui() @@ -389,6 +394,7 @@ def create_ui(startup_timer = None): interfaces += [(img2img_interface, "Image", "img2img")] interfaces += [(control_interface, "Control", "control")] if control_interface is not None else [] interfaces += [(extras_interface, "Process", "process")] + interfaces += [(caption_interface, "Caption", "caption")] interfaces += [(gallery_interface, "Gallery", "gallery")] interfaces += [(models_interface, "Models", "models")] interfaces += script_callbacks.ui_tabs_callback() diff --git a/modules/ui_caption.py b/modules/ui_caption.py new file mode 100644 index 000000000..a306122e0 --- /dev/null +++ b/modules/ui_caption.py @@ -0,0 +1,98 @@ +import gradio as gr +from modules import shared, ui_common, generation_parameters_copypaste +from modules.interrogate import openclip + + +def update_vlm_params(*args): + vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample = args + shared.opts.interrogate_vlm_max_length = vlm_max_tokens + shared.opts.interrogate_vlm_num_beams = vlm_num_beams + shared.opts.interrogate_vlm_temperature = vlm_temperature + shared.opts.interrogate_vlm_do_sample = vlm_do_sample + + +def create_ui(): + with gr.Row(equal_height=False, variant='compact', elem_classes="caption"): + with gr.Column(variant='compact'): + with gr.Row(): + image = gr.Image(type='pil', label="Image") + with gr.Tabs(elem_id="mode_caption"): + with gr.Tab("CLiP Interrogate"): + with gr.Row(): + clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model') + ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models') + blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model') + mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast') + with gr.Accordion(label='Advanced', open=False, visible=True): + with gr.Row(): + caption_max_length = gr.Slider(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=512, min_width=300) + chunk_size = gr.Slider(label='Chunk size', value=1024, minimum=256, maximum=4096, min_width=300) + with gr.Row(): + min_flavors = gr.Slider(label='Min flavors', value=2, minimum=1, maximum=16, min_width=300) + max_flavors = gr.Slider(label='Max flavors', value=8, minimum=1, maximum=64, min_width=300) + flavor_intermediate_count = gr.Slider(label='Intermediates', value=1024, minimum=256, maximum=4096) + caption_max_length.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) + chunk_size.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) + min_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) + max_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) + flavor_intermediate_count.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) + with gr.Accordion(label='Batch', open=False, visible=True): + with gr.Row(): + batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100) + with gr.Row(): + batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100) + with gr.Row(): + batch_str = gr.Text(label="Folder", value="", interactive=True) + with gr.Row(): + batch = gr.Text(label="Prompts", lines=10) + with gr.Row(): + clip_model = gr.Dropdown([], value='ViT-L-14/openai', label='CLiP Batch Model') + ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models') + with gr.Row(elem_id='interrogate_buttons_batch'): + btn_interrogate_batch = gr.Button("Batch interrogate", elem_id="interrogate_btn_interrogate", variant='primary') + with gr.Row(): + save_output = gr.Checkbox(label='Save output', value=True, elem_id="extras_save_output") + with gr.Row(elem_id='interrogate_buttons_image'): + btn_interrogate_img = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary') + btn_analyze_img = gr.Button("Analyze", elem_id="interrogate_btn_analyze", variant='primary') + with gr.Tab("VLM Caption"): + from modules.interrogate import vqa + with gr.Row(): + vqa_question = gr.Dropdown(label="Predefined question", allow_custom_value=False, choices=vqa.vlm_prompts, value=vqa.vlm_prompts[2]) + with gr.Row(): + vqa_prompt = gr.Textbox(label="Prompt", placeholder="optionally enter custom prompt", lines=2) + with gr.Row(elem_id='interrogate_buttons_query'): + vqa_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model') + with gr.Accordion(label='Advanced', open=False, visible=True): + with gr.Row(): + vlm_max_tokens = gr.Slider(label='Max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1) + vlm_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1) + vlm_temperature = gr.Slider(label='Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01) + with gr.Row(): + vlm_do_sample = gr.Checkbox(label='Use sample', value=shared.opts.interrogate_vlm_do_sample) + vlm_max_tokens.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[]) + vlm_num_beams.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[]) + vlm_temperature.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[]) + vlm_do_sample.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[]) + with gr.Row(elem_id='interrogate_buttons_query'): + vqa_submit = gr.Button("Caption", elem_id="interrogate_btn_interrogate", variant='primary') + with gr.Column(variant='compact'): + with gr.Row(): + prompt = gr.Textbox(label="Answer", lines=8, placeholder="ai generated image description") + with gr.Row(elem_id="interrogate_labels"): + medium = gr.Label(elem_id="interrogate_label_medium", label="Medium", num_top_classes=5, visible=False) + artist = gr.Label(elem_id="interrogate_label_artist", label="Artist", num_top_classes=5, visible=False) + movement = gr.Label(elem_id="interrogate_label_movement", label="Movement", num_top_classes=5, visible=False) + trending = gr.Label(elem_id="interrogate_label_trending", label="Trending", num_top_classes=5, visible=False) + flavor = gr.Label(elem_id="interrogate_label_flavor", label="Flavor", num_top_classes=5, visible=False) + with gr.Row(elem_id='copy_buttons_interrogate'): + copy_interrogate_buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "control", "extras"]) + + btn_interrogate_img.click(openclip.interrogate_image, inputs=[image, clip_model, blip_model, mode], outputs=[prompt]) + btn_analyze_img.click(openclip.analyze_image, inputs=[image, clip_model, blip_model], outputs=[medium, artist, movement, trending, flavor]) + btn_interrogate_batch.click(fn=openclip.interrogate_batch, inputs=[batch_files, batch_folder, batch_str, clip_model, blip_model, mode, save_output], outputs=[batch]) + vqa_submit.click(vqa.interrogate, inputs=[vqa_question, vqa_prompt, image, vqa_model], outputs=[prompt]) + + for tabname, button in copy_interrogate_buttons.items(): + generation_parameters_copypaste.register_paste_params_button(generation_parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=prompt, source_image_component=image,)) + generation_parameters_copypaste.add_paste_fields("caption", image, None) diff --git a/modules/ui_common.py b/modules/ui_common.py index efc40e1e0..d408890bb 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -275,7 +275,7 @@ def create_output_panel(tabname, preview=True, prompt=None, height=None): if not shared.native: buttons = generation_parameters_copypaste.create_buttons(["img2img", "inpaint", "extras"]) else: - buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "control", "extras"]) + buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "control", "extras", "caption"]) download_files = gr.File(None, file_count="multiple", interactive=False, show_label=False, visible=False, elem_id=f'download_files_{tabname}') with gr.Group(): diff --git a/modules/ui_postprocessing.py b/modules/ui_postprocessing.py index 0614e7569..27806e35c 100644 --- a/modules/ui_postprocessing.py +++ b/modules/ui_postprocessing.py @@ -1,8 +1,6 @@ import json import gradio as gr from modules import scripts, shared, ui_common, postprocessing, call_queue, generation_parameters_copypaste -from modules.call_queue import wrap_gradio_gpu_call, wrap_queued_call, wrap_gradio_call # pylint: disable=unused-import -from modules.interrogate import openclip def submit_info(image): @@ -25,81 +23,14 @@ def create_ui(): with gr.Tab('Process Image', id="single_image", elem_id="extras_single_tab") as tab_single: with gr.Row(): extras_image = gr.Image(label="Source", source="upload", interactive=True, type="pil", elem_id="extras_image") - with gr.Row(elem_id='copy_buttons_process'): - copy_process_buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "inpaint", "control"]) with gr.Tab('Process Batch', id="batch_process", elem_id="extras_batch_process_tab") as tab_batch: image_batch = gr.Files(label="Batch process", interactive=True, elem_id="extras_image_batch") with gr.Tab('Process Folder', id="batch_from_directory", elem_id="extras_batch_directory_tab") as tab_batch_dir: extras_batch_input_dir = gr.Textbox(label="Input directory", **shared.hide_dirs, placeholder="A directory on the same machine where the server is running.", elem_id="extras_batch_input_dir") extras_batch_output_dir = gr.Textbox(label="Output directory", **shared.hide_dirs, placeholder="Leave blank to save images to the default path.", elem_id="extras_batch_output_dir") show_extras_results = gr.Checkbox(label='Show result images', value=True, elem_id="extras_show_extras_results") - - with gr.Tab("Interrogate Image"): - with gr.Row(): - image = gr.Image(type='pil', label="Image") - with gr.Row(): - prompt = gr.Textbox(label="Prompt", lines=3) - with gr.Row(elem_id="interrogate_labels"): - medium = gr.Label(elem_id="interrogate_label_medium", label="Medium", num_top_classes=5) - artist = gr.Label(elem_id="interrogate_label_artist", label="Artist", num_top_classes=5) - movement = gr.Label(elem_id="interrogate_label_movement", label="Movement", num_top_classes=5) - trending = gr.Label(elem_id="interrogate_label_trending", label="Trending", num_top_classes=5) - flavor = gr.Label(elem_id="interrogate_label_flavor", label="Flavor", num_top_classes=5) - with gr.Row(): - clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model') - ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models') - blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model') - mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast') - with gr.Accordion(label='Advanced', open=False, visible=True): - with gr.Row(): - caption_max_length = gr.Number(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=512, min_width=300) - chunk_size = gr.Number(label='Chunk size', value=1024, minimum=256, maximum=4096, min_width=300) - min_flavors = gr.Number(label='Min flavors', value=2, minimum=1, maximum=16, min_width=300) - max_flavors = gr.Number(label='Max flavors', value=8, minimum=1, maximum=64, min_width=300) - flavor_intermediate_count = gr.Number(label='Intermediates', value=1024, minimum=256, maximum=4096) - caption_max_length.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) - chunk_size.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) - min_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) - max_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) - flavor_intermediate_count.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[]) - with gr.Row(elem_id='interrogate_buttons_image'): - btn_interrogate_img = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary') - btn_analyze_img = gr.Button("Analyze", elem_id="interrogate_btn_analyze", variant='primary') - btn_unload = gr.Button("Unload", elem_id="interrogate_btn_unload") - with gr.Row(elem_id='copy_buttons_interrogate'): - copy_interrogate_buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "extras", "control"]) - btn_interrogate_img.click(openclip.interrogate_image, inputs=[image, clip_model, blip_model, mode], outputs=prompt) - btn_analyze_img.click(openclip.analyze_image, inputs=[image, clip_model, blip_model], outputs=[medium, artist, movement, trending, flavor]) - btn_unload.click(openclip.unload_clip_model) - with gr.Tab("Interrogate Batch"): - with gr.Row(): - batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100) - with gr.Row(): - batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100) - with gr.Row(): - batch_str = gr.Text(label="Folder", value="", interactive=True) - with gr.Row(): - batch = gr.Text(label="Prompts", lines=10) - with gr.Row(): - clip_model = gr.Dropdown([], value='ViT-L-14/openai', label='CLiP Batch Model') - ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models') - with gr.Row(elem_id='interrogate_buttons_batch'): - btn_interrogate_batch = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary') - with gr.Tab("Visual Query"): - from modules.interrogate import vqa - with gr.Row(): - vqa_image = gr.Image(type='pil', label="Image") - with gr.Row(): - vqa_question = gr.Dropdown(label="Question", allow_custom_value=True, choices=vqa.vlm_prompts, value=vqa.vlm_prompts[2]) - with gr.Row(): - vqa_answer = gr.Textbox(label="Answer", lines=5) - with gr.Row(elem_id='interrogate_buttons_query'): - vqa_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model') - vqa_submit = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary') - vqa_submit.click(vqa.interrogate, inputs=[vqa_question, vqa_image, vqa_model], outputs=[vqa_answer]) - - with gr.Row(): - save_output = gr.Checkbox(label='Save output', value=True, elem_id="extras_save_output") + with gr.Row(): + save_output = gr.Checkbox(label='Save output', value=True, elem_id="extras_save_output") script_inputs = scripts.scripts_postproc.setup_ui() with gr.Column(): @@ -114,20 +45,18 @@ def create_ui(): gr.HTML('File metadata') exif_info = gr.HTML(elem_id="pnginfo_html_info") gen_info = gr.Text(elem_id="pnginfo_gen_info", visible=False) + with gr.Row(elem_id='copy_buttons_process'): + copy_process_buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "control", "caption"]) + for tabname, button in copy_process_buttons.items(): generation_parameters_copypaste.register_paste_params_button(generation_parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=gen_info, source_image_component=extras_image)) - for tabname, button in copy_interrogate_buttons.items(): - generation_parameters_copypaste.register_paste_params_button(generation_parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=prompt, source_image_component=image,)) - + generation_parameters_copypaste.add_paste_fields("extras", extras_image, None) tab_single.select(fn=lambda: 0, inputs=[], outputs=[tab_index]) tab_batch.select(fn=lambda: 1, inputs=[], outputs=[tab_index]) tab_batch_dir.select(fn=lambda: 2, inputs=[], outputs=[tab_index]) - extras_image.change( - fn=wrap_gradio_call(submit_info), - inputs=[extras_image], - outputs=[html_info_formatted, exif_info, gen_info], - ) + extras_image.change(fn=submit_info, inputs=[extras_image], outputs=[html_info_formatted, exif_info, gen_info]) + extras_image.change(fn=scripts.scripts_postproc.image_changed, inputs=[], outputs=[]) submit.click( _js="submit_postprocessing", fn=call_queue.wrap_gradio_gpu_call(submit_process, extra_outputs=[None, ''], name='Postprocess'), @@ -148,15 +77,3 @@ def create_ui(): html_log, ] ) - btn_interrogate_batch.click( - fn=openclip.interrogate_batch, - inputs=[batch_files, batch_folder, batch_str, clip_model, blip_model, mode, save_output], - outputs=[batch], - ) - - generation_parameters_copypaste.add_paste_fields("extras", extras_image, None) - - extras_image.change( - fn=scripts.scripts_postproc.image_changed, - inputs=[], outputs=[] - )