diff --git a/CHANGELOG.md b/CHANGELOG.md
index db4ffa67f..537c24247 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,7 +11,9 @@ We're back with another update with over 50 commits!
- A lot of [outpainting](https://vladmandic.github.io/sdnext-docs/Outpaint/) goodies
- Support for new models: [AlphaVLLM Lumina 2](https://github.com/Alpha-VLLM/Lumina-Image-2.0) and [Ostris Flex.1-Alpha](https://huggingface.co/ostris/Flex.1-alpha)
- And new **Mixture-of-Diffusers** regional tiling pipeline
-- Following last weeks **interrogate/captioning** rewrite, added **JoyTag** and **JoyCaption** to list of supported models
+- Follow-up to last weeks **interrogate/captioning** rewrite
+ now with redesigned captioning UI and
+ added **JoyTag**, **JoyCaption**, **Google PaliGemma**, **ToriiGate 0.4** to list of supported models
- Some changes to **prompt parsing** to allow more control as well as more flexibility when mouting SDNext server to custom URL
- Of course, cumulative fixes...
@@ -28,13 +30,13 @@ We're back with another update with over 50 commits!
*english, croatian, spanish, french, italian, portuguese, chinese, japanese, korean, russian*
- set in *settings -> user interface -> language*
- [localization](https://vladmandic.github.io/sdnext-docs/Locale/) documentation
- - **UI**:
+ - **UI**
- force browser cache-invalidate on page load
-- **Docs**
+- **Docs**
- New [Outpaint](https://vladmandic.github.io/sdnext-docs/Outpaint/) step-by-step guide
- Updated [Docker](https://github.com/vladmandic/sdnext/wiki/Docker) guide
includes build and publish and both local and cloud examples
-- **Models**
+- **Models**
- [AlphaVLLM Lumina 2](https://github.com/Alpha-VLLM/Lumina-Image-2.0)
new foundation model for image generation based o Gemma-2-2B text encoder and a flow-based diffusion transformer
fully supports offloading and on-the-fly quantization
@@ -44,18 +46,23 @@ We're back with another update with over 50 commits!
result is model smaller than *Flux.1-Dev*, but with similar capabilities
fully supports offloading and on-the-fly quantization
simply select from *networks -> models -> reference*
-- **Pipelines**
+- **Pipelines**
- [Mixture-of-Diffusers](https://huggingface.co/posts/elismasilva/251775641926329)
Regional tiling type of a solution for SDXL models
select from *scripts -> mixture of diffusers*
- [Automatic Color Inpaint]
Automatically creates mask based on selected color and triggers inpaint
simply select in *scripts -> automatic color inpaint* when in img2img mode
-- **Interrogate/Captioning**
+- **Interrogate/Captioning**
+ - Redesigned captioning UI
+ split from Process tab into separate tab
+ split `clip` vs `vlm` models processing
+ direct *send-to* buttons on all tabs
- [JoyTag](https://huggingface.co/fancyfeast/joytag)
- - [JoyCaption](https://huggingface.co/fancyfeast/llama-joycaption-alpha-two-hf-llava)
- *note*: this is a very large model based on LLama 3.1
-- **Docker**
+ - [JoyCaption 2](https://huggingface.co/fancyfeast/llama-joycaption-alpha-two-hf-llava)
+ - [Google PaliGemma 2](https://huggingface.co/google/paligemma2-3b-pt-224)
+ - [ToriiGate 0.4 7B](https://huggingface.co/Minthy/ToriiGate-v0.4-7B),
+- **Docker**
- updated **CUDA** receipe to `torch==2.6.0` with `cuda==12.6` and add prebuilt image
- added **ROCm** receipe and prebuilt image
- added **IPEX** receipe and add prebuilt image
@@ -80,7 +87,7 @@ We're back with another update with over 50 commits!
- **Styles**
ability to save and/or restore prompts before or after parsing of wildcards
set in *settings -> networks -> styles*
- - **Access tokens**
+ - **Access tokens**
persist *models -> hugginface -> token*
persist *models -> civitai -> token*
- global switch to lancosz method for all interal resize ops and bicubic for interpolation ops
diff --git a/javascript/ui.js b/javascript/ui.js
index 7193f9470..9df1a26e4 100644
--- a/javascript/ui.js
+++ b/javascript/ui.js
@@ -155,6 +155,11 @@ function switch_to_control(...args) {
return Array.from(arguments);
}
+function switch_to_caption(...args) {
+ switchToTab('Caption');
+ return Array.from(arguments);
+}
+
function get_tab_index(tabId) {
let res = 0;
gradioApp().getElementById(tabId).querySelector('div').querySelectorAll('button')
diff --git a/modules/api/endpoints.py b/modules/api/endpoints.py
index 52c4b5301..7bdc31c6a 100644
--- a/modules/api/endpoints.py
+++ b/modules/api/endpoints.py
@@ -113,7 +113,7 @@ def post_vqa(req: models.ReqVQA):
image = helpers.decode_base64_to_image(req.image)
image = image.convert('RGB')
from modules.interrogate import vqa
- answer = vqa.interrogate(req.question, image, req.model)
+ answer = vqa.interrogate(req.question, '', image, req.model)
return models.ResVQA(answer=answer)
def post_unload_checkpoint():
diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py
index fe65efe19..613c09ff2 100644
--- a/modules/generation_parameters_copypaste.py
+++ b/modules/generation_parameters_copypaste.py
@@ -101,6 +101,8 @@ def create_buttons(tabs_list):
name = 'Process'
elif name == 'control':
name = 'Control'
+ elif name == 'caption':
+ name = 'Caption'
buttons[tab] = gr.Button(f"➠ {name}", elem_id=f"{tab}_tab")
return buttons
diff --git a/modules/infotext.py b/modules/infotext.py
index 017691681..e8788c2f5 100644
--- a/modules/infotext.py
+++ b/modules/infotext.py
@@ -90,6 +90,9 @@ def parse(infotext):
# debug(f'Negative: {negative}')
params = dict(re_param.findall(remaining))
+ if len(list(params)) == 0:
+ params['Prompt'] = infotext
+ return params
params['Prompt'] = prompt
params['Negative prompt'] = negative
for key, val in params.copy().items():
diff --git a/modules/interrogate/openclip.py b/modules/interrogate/openclip.py
index 9cd658db2..fd10d09ab 100644
--- a/modules/interrogate/openclip.py
+++ b/modules/interrogate/openclip.py
@@ -6,6 +6,7 @@ import threading
import re
import torch
import torch.hub # pylint: disable=ungrouped-imports
+import gradio as gr
from PIL import Image
from torchvision import transforms
from torchvision.transforms.functional import InterpolationMode
@@ -423,7 +424,13 @@ def analyze_image(image, clip_model, blip_model):
movement_ranks = dict(zip(top_movements, ci.similarities(image_features, top_movements)))
trending_ranks = dict(zip(top_trendings, ci.similarities(image_features, top_trendings)))
flavor_ranks = dict(zip(top_flavors, ci.similarities(image_features, top_flavors)))
- return medium_ranks, artist_ranks, movement_ranks, trending_ranks, flavor_ranks
+ return [
+ gr.update(value=medium_ranks, visible=True),
+ gr.update(value=artist_ranks, visible=True),
+ gr.update(value=movement_ranks, visible=True),
+ gr.update(value=trending_ranks, visible=True),
+ gr.update(value=flavor_ranks, visible=True),
+ ]
interrogator = InterrogateModels()
diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py
index f08169d9b..8ce6e5e77 100644
--- a/modules/interrogate/vqa.py
+++ b/modules/interrogate/vqa.py
@@ -31,7 +31,9 @@ vlm_models = {
"Microsoft GIT VQA Base": "microsoft/git-base-vqav2", # 0.7GB
"Microsoft GIT VQA Large": "microsoft/git-large-vqav2", # 1.6GB
"ToriiGate 0.4 2B": "Minthy/ToriiGate-v0.4-2B",
+ "ToriiGate 0.4 7B": "Minthy/ToriiGate-v0.4-7B",
"ViLT Base": "dandelin/vilt-b32-finetuned-vqa", # 0.5GB
+ "Google PaliGemma 2 3B": "google/paligemma2-3b-pt-224",
"JoyCaption": "fancyfeast/llama-joycaption-alpha-two-hf-llava", # 0.7GB
"JoyTag": "fancyfeast/joytag", # 17.4GB
# "DeepSeek VL2 Tiny": "deepseek-ai/deepseek-vl2-tiny", # broken
@@ -86,7 +88,7 @@ def qwen(question: str, image: Image.Image, repo: str = None):
model = transformers.Qwen2VLForConditionalGeneration.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
loaded = repo
- model.to(devices.device, devices.dtype)
+ model = model.to(devices.device, devices.dtype)
if len(question) < 2:
question = "Describe the image."
question = question.replace('<', '').replace('>', '')
@@ -121,6 +123,30 @@ def qwen(question: str, image: Image.Image, repo: str = None):
return response
+def paligemma(question: str, image: Image.Image, repo: str = None):
+ global processor, model, loaded # pylint: disable=global-statement
+ if model is None or loaded != repo:
+ shared.log.debug(f'Interrogate load: vlm="{repo}"')
+ processor = transformers.PaliGemmaProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
+ model = transformers.PaliGemmaForConditionalGeneration.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype)
+ loaded = repo
+ model = model.to(devices.device, devices.dtype)
+ if len(question) < 2:
+ question = "Describe the image."
+ question = question.replace('<', '').replace('>', '')
+ model_inputs = processor(text=question, images=image, return_tensors="pt").to(devices.device, devices.dtype)
+ input_len = model_inputs["input_ids"].shape[-1]
+ with devices.inference_context():
+ generation = model.generate(
+ **model_inputs,
+ max_new_tokens=shared.opts.interrogate_vlm_max_length,
+ do_sample=shared.opts.interrogate_vlm_do_sample,
+ )
+ generation = generation[0][input_len:]
+ response = processor.decode(generation, skip_special_tokens=True)
+ return response
+
+
def smol(question: str, image: Image.Image, repo: str = None):
global processor, model, loaded # pylint: disable=global-statement
if model is None or loaded != repo:
@@ -295,14 +321,14 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str
pixel_values=pixel_values,
max_new_tokens=shared.opts.interrogate_vlm_max_length,
num_beams=shared.opts.interrogate_vlm_num_beams,
- do_sample=False
+ do_sample=shared.opts.interrogate_vlm_do_sample,
)
generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
response = processor.post_process_generation(generated_text, task="task", image_size=(image.width, image.height))
return response
-def interrogate(question, image, model_name):
+def interrogate(question, prompt, image, model_name):
t0 = time.time()
if isinstance(image, list):
image = image[0] if len(image) > 0 else None
@@ -314,6 +340,8 @@ def interrogate(question, image, model_name):
image.thumbnail((768, 768), Image.Resampling.HAMMING)
if image.mode != 'RGB':
image = image.convert('RGB')
+ if prompt is not None and len(prompt) > 0:
+ question = prompt
from modules import modelloader
modelloader.hf_login()
try:
@@ -352,6 +380,8 @@ def interrogate(question, image, model_name):
elif 'deepseek' in vqa_model.lower():
from modules.interrogate import deepseek
answer = deepseek.predict(question, image, vqa_model)
+ elif 'paligemma' in vqa_model.lower():
+ answer = paligemma(question, image, vqa_model)
else:
answer = 'unknown model'
except Exception as e:
diff --git a/modules/shared.py b/modules/shared.py
index d4d6d06c4..b8f5e3fd7 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -926,8 +926,10 @@ options_templates.update(options_section(('interrogate', "Interrogate"), {
"interrogate_vlm_sep": OptionInfo("
VLM
", "", gr.HTML),
"interrogate_vlm_model": OptionInfo(list(vlm_models)[0], "VLM: default model", gr.Dropdown, {"choices": list(vlm_models)}),
"interrogate_vlm_prompt": OptionInfo(vlm_prompts[2], "VLM: default prompt", DropdownEditable, {"choices": vlm_prompts }),
- "interrogate_vlm_num_beams": OptionInfo(3, "VLM: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1}),
- "interrogate_vlm_max_length": OptionInfo(512, "VLM: max length", gr.Slider, {"minimum": 1, "maximum": 4096, "step": 1}),
+ "interrogate_vlm_num_beams": OptionInfo(3, "VLM: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}),
+ "interrogate_vlm_max_length": OptionInfo(512, "VLM: max length", gr.Slider, {"minimum": 1, "maximum": 4096, "step": 1, "visible": False}),
+ "interrogate_vlm_do_sample": OptionInfo(False, "VLM: use sample method"),
+ "interrogate_vlm_temperature": OptionInfo(0.6, "VLM: num beams", gr.Slider, {"minimum": 0.1, "maximum": 1.0, "step": 0.11, "visible": False}),
"deepbooru_sep": OptionInfo("DeepBooru
", "", gr.HTML),
"deepbooru_score_threshold": OptionInfo(0.65, "DeepBooru: score threshold", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}),
diff --git a/modules/ui.py b/modules/ui.py
index 04e813cd8..15718dbb9 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -151,6 +151,11 @@ def create_ui(startup_timer = None):
ui_postprocessing.create_ui()
timer.startup.record("ui-extras")
+ with gr.Blocks(analytics_enabled=False) as caption_interface:
+ from modules import ui_caption
+ ui_caption.create_ui()
+ timer.startup.record("ui-caption")
+
with gr.Blocks(analytics_enabled=False) as models_interface:
from modules import ui_models
ui_models.create_ui()
@@ -389,6 +394,7 @@ def create_ui(startup_timer = None):
interfaces += [(img2img_interface, "Image", "img2img")]
interfaces += [(control_interface, "Control", "control")] if control_interface is not None else []
interfaces += [(extras_interface, "Process", "process")]
+ interfaces += [(caption_interface, "Caption", "caption")]
interfaces += [(gallery_interface, "Gallery", "gallery")]
interfaces += [(models_interface, "Models", "models")]
interfaces += script_callbacks.ui_tabs_callback()
diff --git a/modules/ui_caption.py b/modules/ui_caption.py
new file mode 100644
index 000000000..a306122e0
--- /dev/null
+++ b/modules/ui_caption.py
@@ -0,0 +1,98 @@
+import gradio as gr
+from modules import shared, ui_common, generation_parameters_copypaste
+from modules.interrogate import openclip
+
+
+def update_vlm_params(*args):
+ vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample = args
+ shared.opts.interrogate_vlm_max_length = vlm_max_tokens
+ shared.opts.interrogate_vlm_num_beams = vlm_num_beams
+ shared.opts.interrogate_vlm_temperature = vlm_temperature
+ shared.opts.interrogate_vlm_do_sample = vlm_do_sample
+
+
+def create_ui():
+ with gr.Row(equal_height=False, variant='compact', elem_classes="caption"):
+ with gr.Column(variant='compact'):
+ with gr.Row():
+ image = gr.Image(type='pil', label="Image")
+ with gr.Tabs(elem_id="mode_caption"):
+ with gr.Tab("CLiP Interrogate"):
+ with gr.Row():
+ clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model')
+ ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models')
+ blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model')
+ mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast')
+ with gr.Accordion(label='Advanced', open=False, visible=True):
+ with gr.Row():
+ caption_max_length = gr.Slider(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=512, min_width=300)
+ chunk_size = gr.Slider(label='Chunk size', value=1024, minimum=256, maximum=4096, min_width=300)
+ with gr.Row():
+ min_flavors = gr.Slider(label='Min flavors', value=2, minimum=1, maximum=16, min_width=300)
+ max_flavors = gr.Slider(label='Max flavors', value=8, minimum=1, maximum=64, min_width=300)
+ flavor_intermediate_count = gr.Slider(label='Intermediates', value=1024, minimum=256, maximum=4096)
+ caption_max_length.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
+ chunk_size.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
+ min_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
+ max_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
+ flavor_intermediate_count.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
+ with gr.Accordion(label='Batch', open=False, visible=True):
+ with gr.Row():
+ batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100)
+ with gr.Row():
+ batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100)
+ with gr.Row():
+ batch_str = gr.Text(label="Folder", value="", interactive=True)
+ with gr.Row():
+ batch = gr.Text(label="Prompts", lines=10)
+ with gr.Row():
+ clip_model = gr.Dropdown([], value='ViT-L-14/openai', label='CLiP Batch Model')
+ ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models')
+ with gr.Row(elem_id='interrogate_buttons_batch'):
+ btn_interrogate_batch = gr.Button("Batch interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
+ with gr.Row():
+ save_output = gr.Checkbox(label='Save output', value=True, elem_id="extras_save_output")
+ with gr.Row(elem_id='interrogate_buttons_image'):
+ btn_interrogate_img = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
+ btn_analyze_img = gr.Button("Analyze", elem_id="interrogate_btn_analyze", variant='primary')
+ with gr.Tab("VLM Caption"):
+ from modules.interrogate import vqa
+ with gr.Row():
+ vqa_question = gr.Dropdown(label="Predefined question", allow_custom_value=False, choices=vqa.vlm_prompts, value=vqa.vlm_prompts[2])
+ with gr.Row():
+ vqa_prompt = gr.Textbox(label="Prompt", placeholder="optionally enter custom prompt", lines=2)
+ with gr.Row(elem_id='interrogate_buttons_query'):
+ vqa_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model')
+ with gr.Accordion(label='Advanced', open=False, visible=True):
+ with gr.Row():
+ vlm_max_tokens = gr.Slider(label='Max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1)
+ vlm_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1)
+ vlm_temperature = gr.Slider(label='Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01)
+ with gr.Row():
+ vlm_do_sample = gr.Checkbox(label='Use sample', value=shared.opts.interrogate_vlm_do_sample)
+ vlm_max_tokens.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[])
+ vlm_num_beams.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[])
+ vlm_temperature.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[])
+ vlm_do_sample.change(fn=update_vlm_params, inputs=[vlm_max_tokens, vlm_num_beams, vlm_temperature, vlm_do_sample], outputs=[])
+ with gr.Row(elem_id='interrogate_buttons_query'):
+ vqa_submit = gr.Button("Caption", elem_id="interrogate_btn_interrogate", variant='primary')
+ with gr.Column(variant='compact'):
+ with gr.Row():
+ prompt = gr.Textbox(label="Answer", lines=8, placeholder="ai generated image description")
+ with gr.Row(elem_id="interrogate_labels"):
+ medium = gr.Label(elem_id="interrogate_label_medium", label="Medium", num_top_classes=5, visible=False)
+ artist = gr.Label(elem_id="interrogate_label_artist", label="Artist", num_top_classes=5, visible=False)
+ movement = gr.Label(elem_id="interrogate_label_movement", label="Movement", num_top_classes=5, visible=False)
+ trending = gr.Label(elem_id="interrogate_label_trending", label="Trending", num_top_classes=5, visible=False)
+ flavor = gr.Label(elem_id="interrogate_label_flavor", label="Flavor", num_top_classes=5, visible=False)
+ with gr.Row(elem_id='copy_buttons_interrogate'):
+ copy_interrogate_buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "control", "extras"])
+
+ btn_interrogate_img.click(openclip.interrogate_image, inputs=[image, clip_model, blip_model, mode], outputs=[prompt])
+ btn_analyze_img.click(openclip.analyze_image, inputs=[image, clip_model, blip_model], outputs=[medium, artist, movement, trending, flavor])
+ btn_interrogate_batch.click(fn=openclip.interrogate_batch, inputs=[batch_files, batch_folder, batch_str, clip_model, blip_model, mode, save_output], outputs=[batch])
+ vqa_submit.click(vqa.interrogate, inputs=[vqa_question, vqa_prompt, image, vqa_model], outputs=[prompt])
+
+ for tabname, button in copy_interrogate_buttons.items():
+ generation_parameters_copypaste.register_paste_params_button(generation_parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=prompt, source_image_component=image,))
+ generation_parameters_copypaste.add_paste_fields("caption", image, None)
diff --git a/modules/ui_common.py b/modules/ui_common.py
index efc40e1e0..d408890bb 100644
--- a/modules/ui_common.py
+++ b/modules/ui_common.py
@@ -275,7 +275,7 @@ def create_output_panel(tabname, preview=True, prompt=None, height=None):
if not shared.native:
buttons = generation_parameters_copypaste.create_buttons(["img2img", "inpaint", "extras"])
else:
- buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "control", "extras"])
+ buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "control", "extras", "caption"])
download_files = gr.File(None, file_count="multiple", interactive=False, show_label=False, visible=False, elem_id=f'download_files_{tabname}')
with gr.Group():
diff --git a/modules/ui_postprocessing.py b/modules/ui_postprocessing.py
index 0614e7569..27806e35c 100644
--- a/modules/ui_postprocessing.py
+++ b/modules/ui_postprocessing.py
@@ -1,8 +1,6 @@
import json
import gradio as gr
from modules import scripts, shared, ui_common, postprocessing, call_queue, generation_parameters_copypaste
-from modules.call_queue import wrap_gradio_gpu_call, wrap_queued_call, wrap_gradio_call # pylint: disable=unused-import
-from modules.interrogate import openclip
def submit_info(image):
@@ -25,81 +23,14 @@ def create_ui():
with gr.Tab('Process Image', id="single_image", elem_id="extras_single_tab") as tab_single:
with gr.Row():
extras_image = gr.Image(label="Source", source="upload", interactive=True, type="pil", elem_id="extras_image")
- with gr.Row(elem_id='copy_buttons_process'):
- copy_process_buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "inpaint", "control"])
with gr.Tab('Process Batch', id="batch_process", elem_id="extras_batch_process_tab") as tab_batch:
image_batch = gr.Files(label="Batch process", interactive=True, elem_id="extras_image_batch")
with gr.Tab('Process Folder', id="batch_from_directory", elem_id="extras_batch_directory_tab") as tab_batch_dir:
extras_batch_input_dir = gr.Textbox(label="Input directory", **shared.hide_dirs, placeholder="A directory on the same machine where the server is running.", elem_id="extras_batch_input_dir")
extras_batch_output_dir = gr.Textbox(label="Output directory", **shared.hide_dirs, placeholder="Leave blank to save images to the default path.", elem_id="extras_batch_output_dir")
show_extras_results = gr.Checkbox(label='Show result images', value=True, elem_id="extras_show_extras_results")
-
- with gr.Tab("Interrogate Image"):
- with gr.Row():
- image = gr.Image(type='pil', label="Image")
- with gr.Row():
- prompt = gr.Textbox(label="Prompt", lines=3)
- with gr.Row(elem_id="interrogate_labels"):
- medium = gr.Label(elem_id="interrogate_label_medium", label="Medium", num_top_classes=5)
- artist = gr.Label(elem_id="interrogate_label_artist", label="Artist", num_top_classes=5)
- movement = gr.Label(elem_id="interrogate_label_movement", label="Movement", num_top_classes=5)
- trending = gr.Label(elem_id="interrogate_label_trending", label="Trending", num_top_classes=5)
- flavor = gr.Label(elem_id="interrogate_label_flavor", label="Flavor", num_top_classes=5)
- with gr.Row():
- clip_model = gr.Dropdown([], value=shared.opts.interrogate_clip_model, label='CLiP model')
- ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models')
- blip_model = gr.Dropdown(list(openclip.caption_models), value=shared.opts.interrogate_blip_model, label='Caption model')
- mode = gr.Dropdown(openclip.caption_types, label='Mode', value='fast')
- with gr.Accordion(label='Advanced', open=False, visible=True):
- with gr.Row():
- caption_max_length = gr.Number(label='Max length', value=shared.opts.interrogate_clip_max_length, minimum=16, maximum=512, min_width=300)
- chunk_size = gr.Number(label='Chunk size', value=1024, minimum=256, maximum=4096, min_width=300)
- min_flavors = gr.Number(label='Min flavors', value=2, minimum=1, maximum=16, min_width=300)
- max_flavors = gr.Number(label='Max flavors', value=8, minimum=1, maximum=64, min_width=300)
- flavor_intermediate_count = gr.Number(label='Intermediates', value=1024, minimum=256, maximum=4096)
- caption_max_length.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
- chunk_size.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
- min_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
- max_flavors.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
- flavor_intermediate_count.change(fn=openclip.update_interrogate_params, inputs=[caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count], outputs=[])
- with gr.Row(elem_id='interrogate_buttons_image'):
- btn_interrogate_img = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
- btn_analyze_img = gr.Button("Analyze", elem_id="interrogate_btn_analyze", variant='primary')
- btn_unload = gr.Button("Unload", elem_id="interrogate_btn_unload")
- with gr.Row(elem_id='copy_buttons_interrogate'):
- copy_interrogate_buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "extras", "control"])
- btn_interrogate_img.click(openclip.interrogate_image, inputs=[image, clip_model, blip_model, mode], outputs=prompt)
- btn_analyze_img.click(openclip.analyze_image, inputs=[image, clip_model, blip_model], outputs=[medium, artist, movement, trending, flavor])
- btn_unload.click(openclip.unload_clip_model)
- with gr.Tab("Interrogate Batch"):
- with gr.Row():
- batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100)
- with gr.Row():
- batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100)
- with gr.Row():
- batch_str = gr.Text(label="Folder", value="", interactive=True)
- with gr.Row():
- batch = gr.Text(label="Prompts", lines=10)
- with gr.Row():
- clip_model = gr.Dropdown([], value='ViT-L-14/openai', label='CLiP Batch Model')
- ui_common.create_refresh_button(clip_model, openclip.refresh_clip_models, lambda: {"choices": openclip.refresh_clip_models()}, 'refresh_interrogate_models')
- with gr.Row(elem_id='interrogate_buttons_batch'):
- btn_interrogate_batch = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
- with gr.Tab("Visual Query"):
- from modules.interrogate import vqa
- with gr.Row():
- vqa_image = gr.Image(type='pil', label="Image")
- with gr.Row():
- vqa_question = gr.Dropdown(label="Question", allow_custom_value=True, choices=vqa.vlm_prompts, value=vqa.vlm_prompts[2])
- with gr.Row():
- vqa_answer = gr.Textbox(label="Answer", lines=5)
- with gr.Row(elem_id='interrogate_buttons_query'):
- vqa_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model')
- vqa_submit = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
- vqa_submit.click(vqa.interrogate, inputs=[vqa_question, vqa_image, vqa_model], outputs=[vqa_answer])
-
- with gr.Row():
- save_output = gr.Checkbox(label='Save output', value=True, elem_id="extras_save_output")
+ with gr.Row():
+ save_output = gr.Checkbox(label='Save output', value=True, elem_id="extras_save_output")
script_inputs = scripts.scripts_postproc.setup_ui()
with gr.Column():
@@ -114,20 +45,18 @@ def create_ui():
gr.HTML('File metadata')
exif_info = gr.HTML(elem_id="pnginfo_html_info")
gen_info = gr.Text(elem_id="pnginfo_gen_info", visible=False)
+ with gr.Row(elem_id='copy_buttons_process'):
+ copy_process_buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "control", "caption"])
+
for tabname, button in copy_process_buttons.items():
generation_parameters_copypaste.register_paste_params_button(generation_parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=gen_info, source_image_component=extras_image))
- for tabname, button in copy_interrogate_buttons.items():
- generation_parameters_copypaste.register_paste_params_button(generation_parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=prompt, source_image_component=image,))
-
+ generation_parameters_copypaste.add_paste_fields("extras", extras_image, None)
tab_single.select(fn=lambda: 0, inputs=[], outputs=[tab_index])
tab_batch.select(fn=lambda: 1, inputs=[], outputs=[tab_index])
tab_batch_dir.select(fn=lambda: 2, inputs=[], outputs=[tab_index])
- extras_image.change(
- fn=wrap_gradio_call(submit_info),
- inputs=[extras_image],
- outputs=[html_info_formatted, exif_info, gen_info],
- )
+ extras_image.change(fn=submit_info, inputs=[extras_image], outputs=[html_info_formatted, exif_info, gen_info])
+ extras_image.change(fn=scripts.scripts_postproc.image_changed, inputs=[], outputs=[])
submit.click(
_js="submit_postprocessing",
fn=call_queue.wrap_gradio_gpu_call(submit_process, extra_outputs=[None, ''], name='Postprocess'),
@@ -148,15 +77,3 @@ def create_ui():
html_log,
]
)
- btn_interrogate_batch.click(
- fn=openclip.interrogate_batch,
- inputs=[batch_files, batch_folder, batch_str, clip_model, blip_model, mode, save_output],
- outputs=[batch],
- )
-
- generation_parameters_copypaste.add_paste_fields("extras", extras_image, None)
-
- extras_image.change(
- fn=scripts.scripts_postproc.image_changed,
- inputs=[], outputs=[]
- )