From 766cb499287e93bb37e6ac97aad10b38b1dfea7e Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Tue, 2 Dec 2025 21:43:13 +0000 Subject: [PATCH 1/4] feat(ui): add vision and reasoning symbols, fix dropdown fonts Add new Font Awesome symbols for model capability indicators: - vision symbol (eye icon) for vision-capable VLM models - reasoning symbol (lightbulb icon) for thinking/reasoning models Also fix dropdown font styling by adding NotoSans font-family. --- javascript/sdnext.css | 6 ++++++ modules/ui_symbols.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 2821fb164..6c780713c 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -289,6 +289,12 @@ input::-webkit-outer-spin-button, input::-webkit-inner-spin-button { .gradio-dropdown .token { overflow-x: hidden; padding: var(--spacing-xs) !important; + font-family: 'NotoSans', var(--font); +} + +.gradio-dropdown .wrap input, +.gradio-dropdown input { + font-family: 'NotoSans', var(--font); } .gradio-html { diff --git a/modules/ui_symbols.py b/modules/ui_symbols.py index 6a77c0e40..099ec03b8 100644 --- a/modules/ui_symbols.py +++ b/modules/ui_symbols.py @@ -28,6 +28,8 @@ image = '🖌️' resize = '⁜' interrogate = '♻' bullet = '⃝' +vision = '\uf06e' # Font Awesome eye icon (more minimalistic) +reasoning = '\uf0eb' # Font Awesome lightbulb icon (represents thinking/reasoning) sort_alpha_asc = '\uf15d' sort_alpha_dsc = '\uf15e' sort_size_asc = '\uf160' From eb832a4850f931532a0d3366e11c513e966451e1 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Tue, 2 Dec 2025 21:46:09 +0000 Subject: [PATCH 2/4] fix(vqa): respect offload setting in JoyCaption, add max_pixels Two fixes for the JoyCaption handler: - Only offload model if shared.opts.interrogate_offload is True - Add max_pixels=1024*1024 to AutoProcessor for consistent image handling --- modules/interrogate/joycaption.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/interrogate/joycaption.py b/modules/interrogate/joycaption.py index dc5e07213..114888f4e 100644 --- a/modules/interrogate/joycaption.py +++ b/modules/interrogate/joycaption.py @@ -67,7 +67,7 @@ def predict(question: str, image, vqa_model: str = None) -> str: if llava_model is None: shared.log.info(f'Interrogate: type=vlm model="JoyCaption" {str(opts)}') - processor = AutoProcessor.from_pretrained(opts.repo) + processor = AutoProcessor.from_pretrained(opts.repo, max_pixels=1024*1024) quant_args = model_quant.create_config(module='LLM') llava_model = LlavaForConditionalGeneration.from_pretrained( opts.repo, @@ -105,6 +105,7 @@ def predict(question: str, image, vqa_model: str = None) -> str: )[0] generate_ids = generate_ids[inputs['input_ids'].shape[1]:] # Trim off the prompt caption = processor.tokenizer.decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False) # Decode the caption - sd_models.move_model(llava_model, devices.cpu, force=True) + if shared.opts.interrogate_offload: + sd_models.move_model(llava_model, devices.cpu, force=True) caption = caption.replace('\n\n', '\n').strip() return caption From 85cd222793de4226a2075b5a61cb637be45fee12 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Tue, 2 Dec 2025 21:48:09 +0000 Subject: [PATCH 3/4] fix(vqa): sort CLiP analysis results and add text output Improvements to the OpenCLIP interrogation: - Sort all ranking dicts by similarity score (descending) - Add format_category() helper for text formatting - Add formatted text output for CLIP labels textbox - Return additional text update in analyze_image() --- modules/interrogate/openclip.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/modules/interrogate/openclip.py b/modules/interrogate/openclip.py index 508a4d897..2350dc440 100644 --- a/modules/interrogate/openclip.py +++ b/modules/interrogate/openclip.py @@ -204,15 +204,32 @@ def analyze_image(image, clip_model, blip_model): top_movements = ci.movements.rank(image_features, 5) top_trendings = ci.trendings.rank(image_features, 5) top_flavors = ci.flavors.rank(image_features, 5) - medium_ranks = dict(zip(top_mediums, ci.similarities(image_features, top_mediums))) - artist_ranks = dict(zip(top_artists, ci.similarities(image_features, top_artists))) - movement_ranks = dict(zip(top_movements, ci.similarities(image_features, top_movements))) - trending_ranks = dict(zip(top_trendings, ci.similarities(image_features, top_trendings))) - flavor_ranks = dict(zip(top_flavors, ci.similarities(image_features, top_flavors))) + medium_ranks = dict(sorted(zip(top_mediums, ci.similarities(image_features, top_mediums)), key=lambda x: x[1], reverse=True)) + artist_ranks = dict(sorted(zip(top_artists, ci.similarities(image_features, top_artists)), key=lambda x: x[1], reverse=True)) + movement_ranks = dict(sorted(zip(top_movements, ci.similarities(image_features, top_movements)), key=lambda x: x[1], reverse=True)) + trending_ranks = dict(sorted(zip(top_trendings, ci.similarities(image_features, top_trendings)), key=lambda x: x[1], reverse=True)) + flavor_ranks = dict(sorted(zip(top_flavors, ci.similarities(image_features, top_flavors)), key=lambda x: x[1], reverse=True)) + + # Format labels as text + def format_category(name, ranks): + lines = [f"{name}:"] + for item, score in ranks.items(): + lines.append(f" • {item} - {score*100:.1f}%") + return '\n'.join(lines) + + formatted_text = '\n\n'.join([ + format_category("Medium", medium_ranks), + format_category("Artist", artist_ranks), + format_category("Movement", movement_ranks), + format_category("Trending", trending_ranks), + format_category("Flavor", flavor_ranks), + ]) + return [ gr.update(value=medium_ranks, visible=True), gr.update(value=artist_ranks, visible=True), gr.update(value=movement_ranks, visible=True), gr.update(value=trending_ranks, visible=True), gr.update(value=flavor_ranks, visible=True), + gr.update(value=formatted_text, visible=True), # New text output for the textbox ] From 9505a674eafcf6fd6cd42f884b20ad1716bf04c7 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Tue, 2 Dec 2025 23:06:46 +0000 Subject: [PATCH 4/4] docs(i18n): add detailed VLM/CLiP tooltips and improve labels Add comprehensive tooltips explaining VLM and CLiP parameters: - New labels for VLM: Prompt, Task, Prefill, Thinking mode - New labels for CLiP: min/max length, chunk size, flavors - Detailed hints for sampling parameters (temperature, top-k, top-p) - Improved system prompt and sampling method descriptions - Standardized label capitalization --- html/locale_en.json | 47 ++++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/html/locale_en.json b/html/locale_en.json index 54f20deb1..5686e5a54 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -33,10 +33,17 @@ ], "main": [ {"id":"","label":"Prompt","localized":"","reload":"","hint":"Describe image you want to generate"}, + {"id":"","label":"VLM: Prompt","localized":"Prompt","reload":"","hint":"Enter your prompt/question here."}, + {"id":"","label":"VLM: Advanced Options","localized":"Advanced Options","reload":"","hint":"Advanced configuration options for the VLM model."}, + {"id":"","label":"VLM: Batch Caption","localized":"Batch Caption","reload":"","hint":"Process multiple images in a batch using VLM."}, + {"id":"","label":"CLiP: Advanced Options","localized":"Advanced Options","reload":"","hint":"Advanced configuration options for CLiP interrogation."}, + {"id":"","label":"CLiP: Batch Interrogate","localized":"Batch Interrogate","reload":"","hint":"Process multiple images in a batch using CLiP."}, + {"id":"","label":"Task","localized":"","reload":"","hint":"Changes which task the model will perform. Regular text prompts can be used when the default option Use Prompt is selected.
When other options are selected, see the hint text inside an empty Prompt field for guidance."}, + {"id":"","label":"Prefill text","localized":"","reload":"","hint":"Pre-fills the start of the model's response to guide its output format or content by forcing it to continue the prefill text.
Prefill is filtered out and does not appear in the final response.

Leave empty to let the model generate its own response from scratch."}, {"id":"","label":"Start","localized":"","reload":"","hint":"Start"}, {"id":"","label":"End","localized":"","reload":"","hint":"End"}, {"id":"","label":"Core","localized":"","reload":"","hint":"Core settings"}, - {"id":"","label":"System prompt","localized":"","reload":"","hint":"System prompt controls behavior of LLM"}, + {"id":"","label":"System prompt","localized":"","reload":"","hint":"System prompt controls behavior of the LLM. Processed first and persists throughout conversation. Has highest priority weighting and is always appended at the beginning of the sequence.

Use for: Response formatting rules, role definition, style."}, {"id":"","label":"Negative prompt","localized":"","reload":"","hint":"Describe what you don't want to see in generated image"}, {"id":"","label":"Text","localized":"","reload":"","hint":"Create image from text"}, {"id":"","label":"Image","localized":"","reload":"","hint":"Create image from image"}, @@ -46,6 +53,13 @@ {"id":"","label":"I2I","localized":"","reload":"","hint":"Create image from image
Legacy interface that mimics original image-to-image interface and behavior"}, {"id":"","label":"Process","localized":"","reload":"","hint":"Process existing image
Can be used to upscale images, remove backgrounds, obfuscate NSFW content, apply various filters and effects"}, {"id":"","label":"Caption","localized":"","reload":"","hint":"Analyze existing images and create text descriptions"}, + {"id":"","label":"clip: min length","localized":"Min Length","reload":"","hint":"Minimum number of tokens in the generated caption."}, + {"id":"","label":"clip: max length","localized":"Max Length","reload":"","hint":"Maximum number of tokens in the generated caption."}, + {"id":"","label":"clip: chunk size","localized":"Chunk Size","reload":"","hint":"Batch size for processing description candidates (flavors). Higher values speed up interrogation but increase VRAM usage."}, + {"id":"","label":"clip: min flavors","localized":"Min Flavors","reload":"","hint":"Minimum number of descriptive tags (flavors) to keep in the final prompt."}, + {"id":"","label":"clip: max flavors","localized":"Max Flavors","reload":"","hint":"Maximum number of descriptive tags (flavors) to keep in the final prompt."}, + {"id":"","label":"clip: intermediates","localized":"Intermediates","reload":"","hint":"Size of the intermediate candidate pool when matching image features to descriptive tags (flavours). From this pool, the final tags are selected based on Min/Max Flavors. Higher values may improve quality but are slower."}, + {"id":"","label":"clip: num beams","localized":"CLiP Num Beams","reload":"","hint":"Number of beams for beam search during caption generation. Higher values search more possibilities but are slower."}, {"id":"","label":"Interrogate","localized":"","reload":"","hint":"Run interrogate to get description of your image"}, {"id":"","label":"Models","localized":"","reload":"","hint":"Download, convert or merge your models and manage models metadata"}, {"id":"","label":"Sampler","localized":"","reload":"","hint":"Settings related to sampler and seed selection and configuration. Samplers guide the process of turning noise into an image over multiple steps."}, @@ -834,6 +848,8 @@ {"id":"","label":"kdpm2","localized":"","reload":"","hint":"kdpm2"}, {"id":"","label":"kdpm2 a","localized":"","reload":"","hint":"kdpm2 a"}, {"id":"","label":"keep incomplete images","localized":"","reload":"","hint":"keep incomplete images"}, + {"id":"","label":"Keep Thinking Trace","localized":"","reload":"","hint":"Include the model's reasoning process in the final output.
Useful for understanding how the model arrived at its answer.
Only works with models that support thinking mode."}, + {"id":"","label":"Keep Prefill","localized":"","reload":"","hint":"Include the prefill text at the beginning of the final output.
If disabled, the prefill text used to guide the model is removed from the result."}, {"id":"","label":"large","localized":"","reload":"","hint":"large"}, {"id":"","label":"latent history size","localized":"","reload":"","hint":"latent history size"}, {"id":"","label":"latent mode","localized":"","reload":"","hint":"latent mode"}, @@ -882,7 +898,7 @@ {"id":"","label":"max length","localized":"","reload":"","hint":"max length"}, {"id":"","label":"max object size","localized":"","reload":"","hint":"max object size"}, {"id":"","label":"max range","localized":"","reload":"","hint":"max range"}, - {"id":"","label":"max tokens","localized":"","reload":"","hint":"max tokens"}, + {"id":"","label":"Max tokens","localized":"","reload":"","hint":"Maximum number of tokens the model can generate in its response.
The model is not aware of this limit during generation and it won't make the model try to generate more detailed or more concise responses, it simply sets the hard limit for the length, and will forcefully cut off the response when the limit is reached."}, {"id":"","label":"max words","localized":"","reload":"","hint":"max words"}, {"id":"","label":"max-autotune","localized":"","reload":"","hint":"max-autotune"}, {"id":"","label":"max-autotune-no-cudagraphs","localized":"","reload":"","hint":"max-autotune-no-cudagraphs"}, @@ -954,7 +970,7 @@ {"id":"","label":"none","localized":"","reload":"","hint":"none"}, {"id":"","label":"note","localized":"","reload":"","hint":"note"}, {"id":"","label":"nothing","localized":"","reload":"","hint":"nothing"}, - {"id":"","label":"num beams","localized":"","reload":"","hint":"num beams"}, + {"id":"","label":"num beams","localized":"","reload":"","hint":"Maintains multiple candidate paths simultaneously and selects the overall best sequence.
Like exploring several drafts at once to find the best one. More thorough but much slower and less creative than random sampling.
Generally not recommended, most modern VLMs perform better with sampling methods.
Set to 1 to disable."}, {"id":"","label":"number","localized":"","reload":"","hint":"number"}, {"id":"","label":"numbered filenames","localized":"","reload":"","hint":"numbered filenames"}, {"id":"","label":"offload","localized":"","reload":"","hint":"offload"}, @@ -1012,7 +1028,6 @@ {"id":"","label":"postprocess upscale","localized":"","reload":"","hint":"postprocess upscale"}, {"id":"","label":"postprocessing operation order","localized":"","reload":"","hint":"postprocessing operation order"}, {"id":"","label":"power","localized":"","reload":"","hint":"power"}, - {"id":"","label":"predefined question","localized":"","reload":"","hint":"predefined question"}, {"id":"","label":"preset","localized":"","reload":"","hint":"preset"}, {"id":"","label":"preset block merge","localized":"","reload":"","hint":"preset block merge"}, {"id":"","label":"preview","localized":"","reload":"","hint":"preview"}, @@ -1193,7 +1208,11 @@ {"id":"","label":"tcd","localized":"","reload":"","hint":"tcd"}, {"id":"","label":"tdd","localized":"","reload":"","hint":"tdd"}, {"id":"","label":"te","localized":"","reload":"","hint":"te"}, - {"id":"","label":"temperature","localized":"","reload":"","hint":"temperature"}, + {"id":"","label":"temperature","localized":"","reload":"","hint":"Controls randomness in token selection by reshaping the probability distribution.
Like adjusting a dial between cautious predictability (low values ~0.4) and creative exploration (higher values ~1). Higher temperatures increase willingness to choose less obvious options, but makes outputs more unpredictable.

Set to 0 to disable, resulting in silent switch to greedy decoding, disabling sampling."}, + {"id":"","label":"Thinking mode","localized":"","reload":"","hint":"Enables thinking/reasoning, allowing the model to take more time to generate responses.
This can lead to more thoughtful and detailed answers, but will increase response time.
This setting affects both hybrid and thinking-only models, and in some may result in lower overall quality than expected. For thinking-only models like Qwen3-VL this setting might have to be combined with prefill to guarantee preventing thinking.

Models supporting this feature are marked with an \uf0eb icon."}, + {"id":"","label":"Repetition penalty","localized":"","reload":"","hint":"Discourages reusing tokens that already appear in the prompt or output by penalizing their probabilities.
Like adding friction to revisiting previous choices. Helps break repetitive loops but may reduce coherence at aggressive values.

Set to 1 to disable."}, + {"id":"","label":"text guidance scale","localized":"","reload":"","hint":"text guidance scale"}, + {"id":"","label":"template","localized":"","reload":"","hint":"template"}, {"id":"","label":"temporal frequency","localized":"","reload":"","hint":"temporal frequency"}, {"id":"","label":"tertiary model","localized":"","reload":"","hint":"tertiary model"}, {"id":"","label":"text encoder cache size","localized":"","reload":"","hint":"text encoder cache size"}, @@ -1234,8 +1253,8 @@ {"id":"","label":"todo","localized":"","reload":"","hint":"todo"}, {"id":"","label":"tome","localized":"","reload":"","hint":"tome"}, {"id":"","label":"tool","localized":"","reload":"","hint":"tool"}, - {"id":"","label":"top-k","localized":"","reload":"","hint":"top-k"}, - {"id":"","label":"top-p","localized":"","reload":"","hint":"top-p"}, + {"id":"","label":"top-k","localized":"","reload":"","hint":"Limits token selection to the K most likely candidates at each step.
Lower values (e.g., 40) make outputs more focused and predictable, while higher values allow more diverse choices.

Set to 0 to disable."}, + {"id":"","label":"top-p","localized":"","reload":"","hint":"Selects tokens from the smallest set whose cumulative probability exceeds P (e.g., 0.9).
Dynamically adapts the number of candidates based on model confidence; fewer options when certain, more when uncertain.

Set to 1 to disable."}, {"id":"","label":"torch","localized":"","reload":"","hint":"torch"}, {"id":"","label":"transformer","localized":"","reload":"","hint":"transformer"}, {"id":"","label":"trigger word","localized":"","reload":"","hint":"trigger word"}, @@ -1279,7 +1298,7 @@ {"id":"","label":"use random seeds","localized":"","reload":"","hint":"use random seeds"}, {"id":"","label":"use reference values when available","localized":"","reload":"","hint":"use reference values when available"}, {"id":"","label":"use same seed","localized":"","reload":"","hint":"use same seed"}, - {"id":"","label":"use sample","localized":"","reload":"","hint":"use sample"}, + {"id":"","label":"use samplers","localized":"","reload":"","hint":"Enable to use sampling (randomly selecting tokens based on sampling methods like Top-k or Top-p) or disable to use greedy decoding (selecting the most probable token at each step).
Enabling makes outputs more diverse and more creative but less deterministic."}, {"id":"","label":"use separate base dict","localized":"","reload":"","hint":"use separate base dict"}, {"id":"","label":"use simplified solvers in final steps","localized":"","reload":"","hint":"use simplified solvers in final steps"}, {"id":"","label":"use text inputs","localized":"","reload":"","hint":"use text inputs"}, @@ -1299,14 +1318,16 @@ {"id":"","label":"video file","localized":"","reload":"","hint":"video file"}, {"id":"","label":"video type","localized":"","reload":"","hint":"video type"}, {"id":"","label":"vlm","localized":"","reload":"","hint":"vlm"}, - {"id":"","label":"vlm model","localized":"","reload":"","hint":"vlm model"}, + {"id":"","label":"vlm model","localized":"","reload":"","hint":"Select which model to use for Visual Language tasks.

Models which support thinking mode are marked with an \uf0eb icon."}, {"id":"","label":"vlm: default model","localized":"","reload":"","hint":"vlm: default model"}, {"id":"","label":"vlm: default prompt","localized":"","reload":"","hint":"vlm: default prompt"}, {"id":"","label":"vlm: max length","localized":"","reload":"","hint":"vlm: max length"}, - {"id":"","label":"vlm: num beams","localized":"","reload":"","hint":"vlm: num beams"}, - {"id":"","label":"vlm: top-k","localized":"","reload":"","hint":"vlm: top-k"}, - {"id":"","label":"vlm: top-p","localized":"","reload":"","hint":"vlm: top-p"}, - {"id":"","label":"vlm: use sample method","localized":"","reload":"","hint":"vlm: use sample method"}, + {"id":"","label":"VLM Num Beams","localized":"","reload":"","hint":"Maintains multiple candidate paths simultaneously and selects the overall best sequence.
Like exploring several drafts at once to find the best one. More thorough but much slower and less creative than random sampling.
Generally not recommended, most modern VLMs perform better with sampling methods.
Set to 1 to disable."}, + {"id":"","label":"vlm: top-k","localized":"","reload":"","hint":"Limits token selection to the K most likely candidates at each step.
Lower values (e.g., 40) make outputs more focused and predictable, while higher values allow more diverse choices.
Set to 0 to disable."}, + {"id":"","label":"vlm: top-p","localized":"","reload":"","hint":"Selects tokens from the smallest set whose cumulative probability exceeds P (e.g., 0.9).
Dynamically adapts the number of candidates based on model confidence; fewer options when certain, more when uncertain.
Set to 1 to disable."}, + {"id":"","label":"vlm: use sample method","localized":"","reload":"","hint":"Enable to use sampling (randomly selecting tokens based on sampling methods like Top-k or Top-p) or disable to use greedy decoding (selecting the most probable token at each step).
Enabling makes outputs more diverse and creative but less deterministic."}, + {"id":"","label":"VLM Max tokens","localized":"","reload":"","hint":"Maximum number of tokens the model can generate in its response.
The model is not aware of this limit during generation and it won't make the model try to generate more detailed or more concise responses, it simply sets the hard limit for the length, and will forcefully cut off the response when the limit is reached."}, + {"id":"","label":"VLM Temperature","localized":"","reload":"","hint":"Controls randomness in token selection. Lower values (e.g., 0.1) make outputs more focused and deterministic, always choosing high-probability tokens.
Higher values (e.g., 0.9) increase creativity and diversity by allowing less probable tokens.

Set to 0 for fully deterministic output (always picks the most likely token)."}, {"id":"","label":"warmth","localized":"","reload":"","hint":"warmth"}, {"id":"","label":"webp lossless compression","localized":"","reload":"","hint":"webp lossless compression"}, {"id":"","label":"weight","localized":"","reload":"","hint":"weight"},