diff --git a/CHANGELOG.md b/CHANGELOG.md index f92934cc4..5507bc1f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,12 @@ # Change Log for SD.Next -## Update for 2025-12-09 +## Update for 2025-12-10 ### TBD Merge commit: `f903a36d9` -### Highlights for 2025-12-09 +### Highlights for 2025-12-10 New native [kanvas](https://vladmandic.github.io/sdnext-docs/Kanvas/) module for image manipulation that fully replaces *img2img*, *inpaint* and *outpaint* controls, massive update to **Captioning/VQA** models and features New generation of **Flux.2** large image model, new **Z-Image** model that is creating a lot of buzz, new **Kandinsky 5 Lite** image model and new **Photoroom PRX** model @@ -18,8 +18,8 @@ Plus a lot of internal improvements and fixes [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic) -### Details for 2025-12-09 - +### Details for 2025-12-10 + - **Models** - [Black Forest Labs FLUX.2 Dev](https://bfl.ai/blog/flux-2) and prequantized variation [SDNQ-SVD-Uint4](https://huggingface.co/Disty0/FLUX.2-dev-SDNQ-uint4-svd-r32) **FLUX.2-Dev** is a brand new model from BFL and uses large 32B DiT together with Mistral 24B as text encoder @@ -50,23 +50,23 @@ Plus a lot of internal improvements and fixes kanvas is a full replacement for *img2img, inpaint and outpaint* controls see [docs](https://vladmandic.github.io/sdnext-docs/Kanvas/) for details *experimental*: report any feedback in master [issue](https://github.com/vladmandic/sdnext/issues/4358) -- **Captioning** and **VQA: visual query analyzer** +- **Captioning** and **VQA: Visual Question & Answer** massive update to both features and supported models, thanks @CalamitousFelicitousness - models: + models: - additional `mooondream-2` features - support for `moondream-3-preview` - support for `qwen3-vl` with thinking - additional `gemma-3-vl` finetunes - support for `XiaomiMiMo` - features: + ui: - ability to annotate actual image, not just generate captions/answers e.g. actualy mark detected regions/points + features: - ui indicator of model capabilities - support for *prefill* style of prompting/answering - support for *reasoning* mode for supported models with option to output answer-only or reasoning-process - additional debug logging - - **Other Features** - **wildcards**: allow recursive inline wildcards using curly braces syntax - **sdnq**: simplify pre-quantization saved config @@ -99,6 +99,7 @@ Plus a lot of internal improvements and fixes - python: start work on future-proofing for modern python versions, thanks @awsr - nunchaku: update to `1.0.2` - lint: add rules for run-on-windows + - gallery: setting to enable/disable client-side caching, thanks @awsr - gallery: faster thumbnail generation, thanks @awsr - gallery: purge old thumbnails, thanks @awsr - **Docs** diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 9fb5bde56..e92ee8ee9 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 9fb5bde56ada40e0bc3d1888f4793ab10ce1e8a6 +Subproject commit e92ee8ee9a783bea78b90e23eadfbdf82ba05f2c diff --git a/html/locale_en.json b/html/locale_en.json index 5686e5a54..970ab3eb8 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -38,7 +38,7 @@ {"id":"","label":"VLM: Batch Caption","localized":"Batch Caption","reload":"","hint":"Process multiple images in a batch using VLM."}, {"id":"","label":"CLiP: Advanced Options","localized":"Advanced Options","reload":"","hint":"Advanced configuration options for CLiP interrogation."}, {"id":"","label":"CLiP: Batch Interrogate","localized":"Batch Interrogate","reload":"","hint":"Process multiple images in a batch using CLiP."}, - {"id":"","label":"Task","localized":"","reload":"","hint":"Changes which task the model will perform. Regular text prompts can be used when the default option Use Prompt is selected.
When other options are selected, see the hint text inside an empty Prompt field for guidance."}, + {"id":"","label":"Task","localized":"","reload":"","hint":"Changes which task the model will perform. Regular text prompts can be used when the task is set to Use Prompt.
When other options are selected, see the hint text inside an empty Prompt field for guidance."}, {"id":"","label":"Prefill text","localized":"","reload":"","hint":"Pre-fills the start of the model's response to guide its output format or content by forcing it to continue the prefill text.
Prefill is filtered out and does not appear in the final response.

Leave empty to let the model generate its own response from scratch."}, {"id":"","label":"Start","localized":"","reload":"","hint":"Start"}, {"id":"","label":"End","localized":"","reload":"","hint":"End"}, diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index a5ac64766..ded9ad9c4 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -359,7 +359,6 @@ class VQA: self.processor = None self.model = None self.loaded: str = None - self.quant_args = None self.last_annotated_image = None self.last_detection_data = None @@ -386,7 +385,6 @@ class VQA: shared.log.error(f'VQA load: unknown model="{model_name}"') return - self.quant_args = model_quant.create_config(module='LLM') shared.log.debug(f'VQA load: pre-loading model="{model_name}" repo="{repo}"') # Dispatch to appropriate loader (same logic as interrogate) @@ -448,6 +446,7 @@ class VQA: """Load FastVLM model and tokenizer.""" if self.model is None or self.loaded != repo: shared.log.debug(f'Interrogate load: vlm="{repo}"') + quant_args = model_quant.create_config(module='LLM') self.model = None self.processor = transformers.AutoTokenizer.from_pretrained(repo, trust_remote_code=True, cache_dir=shared.opts.hfcache_dir) self.model = transformers.AutoModelForCausalLM.from_pretrained( @@ -455,7 +454,7 @@ class VQA: torch_dtype=devices.dtype, trust_remote_code=True, cache_dir=shared.opts.hfcache_dir, - **self.quant_args, + **quant_args, ) self.loaded = repo devices.torch_gc() @@ -502,11 +501,12 @@ class VQA: cls_name = transformers.Qwen2VLForConditionalGeneration else: cls_name = transformers.AutoModelForCausalLM + quant_args = model_quant.create_config(module='LLM') self.model = cls_name.from_pretrained( repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, - **self.quant_args, + **quant_args, ) self.processor = transformers.AutoProcessor.from_pretrained(repo, max_pixels=1024*1024, cache_dir=shared.opts.hfcache_dir) if 'LLM' in shared.opts.cuda_compile: @@ -638,11 +638,12 @@ class VQA: cls = transformers.Gemma3nForConditionalGeneration # pylint: disable=no-member else: cls = transformers.Gemma3ForConditionalGeneration + quant_args = model_quant.create_config(module='LLM') self.model = cls.from_pretrained( repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, - **self.quant_args, + **quant_args, ) if 'LLM' in shared.opts.cuda_compile: self.model = sd_models_compile.compile_torch(self.model) @@ -839,11 +840,12 @@ class VQA: if self.model is None or self.loaded != repo: shared.log.debug(f'Interrogate load: vlm="{repo}"') self.model = None + quant_args = model_quant.create_config(module='LLM') self.model = transformers.AutoModelForVision2Seq.from_pretrained( repo, cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, - **self.quant_args, + **quant_args, ) self.processor = transformers.AutoProcessor.from_pretrained(repo, max_pixels=1024*1024, cache_dir=shared.opts.hfcache_dir) if 'LLM' in shared.opts.cuda_compile: @@ -1150,13 +1152,14 @@ class VQA: shared.log.debug(f'Interrogate load: vlm="{repo_name}" revision="{effective_revision}" path="{shared.opts.hfcache_dir}"') transformers.dynamic_module_utils.get_imports = get_imports self.model = None + quant_args = model_quant.create_config(module='LLM') self.model = transformers.Florence2ForConditionalGeneration.from_pretrained( repo_name, dtype=torch.bfloat16, revision=effective_revision, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, - **self.quant_args, + **quant_args, ) self.processor = transformers.AutoProcessor.from_pretrained(repo_name, max_pixels=1024*1024, trust_remote_code=True, revision=effective_revision, cache_dir=shared.opts.hfcache_dir) transformers.dynamic_module_utils.get_imports = _get_imports @@ -1230,7 +1233,6 @@ class VQA: self.last_detection_data = None jobid = shared.state.begin('Interrogate LLM') t0 = time.time() - self.quant_args = model_quant.create_config(module='LLM') model_name = model_name or shared.opts.interrogate_vlm_model prefill = vlm_prefill if prefill is None else prefill # Use provided prefill when specified if isinstance(image, list): diff --git a/modules/shared.py b/modules/shared.py index 36ed578f7..4f834b320 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -502,7 +502,7 @@ options_templates.update(options_section(('saving-images', "Image Options"), { "font_color": OptionInfo("#FFFFFF", "Font color", gr.ColorPicker, {}), "image_sep_browser": OptionInfo("

Image Gallery

", "", gr.HTML), - "browser_cache": OptionInfo(True, "Use image gallery cache [STRONGLY recommended]"), + "browser_cache": OptionInfo(True, "Use image gallery cache"), "browser_folders": OptionInfo("", "Additional image browser folders"), "browser_fixed_width": OptionInfo(False, "Use fixed width thumbnails"), "viewer_show_metadata": OptionInfo(True, "Show metadata in full screen image browser"), @@ -643,7 +643,7 @@ options_templates.update(options_section(('postprocessing', "Postprocessing"), { })) options_templates.update(options_section(('interrogate', "Interrogate"), { - "interrogate_default_type": OptionInfo("OpenCLiP", "Default type", gr.Radio, {"choices": ["OpenCLiP", "VLM", "DeepBooru"]}), + "interrogate_default_type": OptionInfo("VLM", "Default caption type", gr.Radio, {"choices": ["OpenCLiP", "VLM", "DeepBooru"]}), "interrogate_offload": OptionInfo(True, "Offload models "), "interrogate_score": OptionInfo(False, "Include scores in results when available"), diff --git a/modules/ui_caption.py b/modules/ui_caption.py index 95c71065b..aad6deddb 100644 --- a/modules/ui_caption.py +++ b/modules/ui_caption.py @@ -3,6 +3,8 @@ from modules import shared, ui_common, generation_parameters_copypaste from modules.interrogate import openclip +default_task = "Short Caption" + def vlm_caption_wrapper(question, system_prompt, prompt, image, model_name, prefill, thinking_mode): """Wrapper for vqa.interrogate that handles annotated image display.""" from modules.interrogate import vqa @@ -17,7 +19,7 @@ def update_vlm_prompts_for_model(model_name): """Update the task dropdown choices based on selected model.""" from modules.interrogate import vqa prompts = vqa.get_prompts_for_model(model_name) - return gr.update(choices=prompts, value=prompts[0] if prompts else "Use Prompt") + return gr.update(choices=prompts, value=prompts[0] if prompts else default_task) def update_vlm_prompt_placeholder(question): @@ -68,9 +70,9 @@ def create_ui(): with gr.Row(): vlm_system = gr.Textbox(label="System Prompt", value=vqa.vlm_system, lines=1, elem_id='vlm_system') with gr.Row(): - vlm_question = gr.Dropdown(label="Task", allow_custom_value=False, choices=initial_prompts, value=initial_prompts[0] if initial_prompts else "Use Prompt", elem_id='vlm_question') + vlm_question = gr.Dropdown(label="Task", allow_custom_value=False, choices=initial_prompts, value=default_task, elem_id='vlm_question') with gr.Row(): - vlm_prompt = gr.Textbox(label="Prompt", placeholder=vqa.get_prompt_placeholder(initial_prompts[0] if initial_prompts else "Use Prompt"), lines=2, elem_id='vlm_prompt') + vlm_prompt = gr.Textbox(label="Prompt", placeholder=vqa.get_prompt_placeholder(initial_prompts[0]), lines=2, elem_id='vlm_prompt') with gr.Row(elem_id='interrogate_buttons_query'): vlm_model = gr.Dropdown(list(vqa.vlm_models), value=current_vlm_model, label='VLM Model', elem_id='vlm_model') with gr.Row(): diff --git a/wiki b/wiki index 2fb950abd..83c4488c4 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 2fb950abdeaad2d2a7976857f04646fc8c6963e1 +Subproject commit 83c4488c4a20c5540c7a8cbaa8cfa4cd33148880