mirror of
https://github.com/vladmandic/automatic
synced 2026-09-19 01:04:32 +02:00
+9
-8
@@ -1,12 +1,12 @@
|
||||
# Change Log for SD.Next
|
||||
|
||||
## Update for 2025-12-09
|
||||
## Update for 2025-12-10
|
||||
|
||||
### TBD
|
||||
|
||||
Merge commit: `f903a36d9`
|
||||
|
||||
### Highlights for 2025-12-09
|
||||
### Highlights for 2025-12-10
|
||||
|
||||
New native [kanvas](https://vladmandic.github.io/sdnext-docs/Kanvas/) module for image manipulation that fully replaces *img2img*, *inpaint* and *outpaint* controls, massive update to **Captioning/VQA** models and features
|
||||
New generation of **Flux.2** large image model, new **Z-Image** model that is creating a lot of buzz, new **Kandinsky 5 Lite** image model and new **Photoroom PRX** model
|
||||
@@ -18,8 +18,8 @@ Plus a lot of internal improvements and fixes
|
||||
|
||||
[ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic)
|
||||
|
||||
### Details for 2025-12-09
|
||||
|
||||
### Details for 2025-12-10
|
||||
|
||||
- **Models**
|
||||
- [Black Forest Labs FLUX.2 Dev](https://bfl.ai/blog/flux-2) and prequantized variation [SDNQ-SVD-Uint4](https://huggingface.co/Disty0/FLUX.2-dev-SDNQ-uint4-svd-r32)
|
||||
**FLUX.2-Dev** is a brand new model from BFL and uses large 32B DiT together with Mistral 24B as text encoder
|
||||
@@ -50,23 +50,23 @@ Plus a lot of internal improvements and fixes
|
||||
kanvas is a full replacement for *img2img, inpaint and outpaint* controls
|
||||
see [docs](https://vladmandic.github.io/sdnext-docs/Kanvas/) for details
|
||||
*experimental*: report any feedback in master [issue](https://github.com/vladmandic/sdnext/issues/4358)
|
||||
- **Captioning** and **VQA: visual query analyzer**
|
||||
- **Captioning** and **VQA: Visual Question & Answer**
|
||||
massive update to both features and supported models, thanks @CalamitousFelicitousness
|
||||
models:
|
||||
models:
|
||||
- additional `mooondream-2` features
|
||||
- support for `moondream-3-preview`
|
||||
- support for `qwen3-vl` with thinking
|
||||
- additional `gemma-3-vl` finetunes
|
||||
- support for `XiaomiMiMo`
|
||||
features:
|
||||
ui:
|
||||
- ability to annotate actual image, not just generate captions/answers
|
||||
e.g. actualy mark detected regions/points
|
||||
features:
|
||||
- ui indicator of model capabilities
|
||||
- support for *prefill* style of prompting/answering
|
||||
- support for *reasoning* mode for supported models
|
||||
with option to output answer-only or reasoning-process
|
||||
- additional debug logging
|
||||
|
||||
- **Other Features**
|
||||
- **wildcards**: allow recursive inline wildcards using curly braces syntax
|
||||
- **sdnq**: simplify pre-quantization saved config
|
||||
@@ -99,6 +99,7 @@ Plus a lot of internal improvements and fixes
|
||||
- python: start work on future-proofing for modern python versions, thanks @awsr
|
||||
- nunchaku: update to `1.0.2`
|
||||
- lint: add rules for run-on-windows
|
||||
- gallery: setting to enable/disable client-side caching, thanks @awsr
|
||||
- gallery: faster thumbnail generation, thanks @awsr
|
||||
- gallery: purge old thumbnails, thanks @awsr
|
||||
- **Docs**
|
||||
|
||||
Submodule extensions-builtin/sdnext-modernui updated: 9fb5bde56a...e92ee8ee9a
+1
-1
@@ -38,7 +38,7 @@
|
||||
{"id":"","label":"VLM: Batch Caption","localized":"Batch Caption","reload":"","hint":"Process multiple images in a batch using VLM."},
|
||||
{"id":"","label":"CLiP: Advanced Options","localized":"Advanced Options","reload":"","hint":"Advanced configuration options for CLiP interrogation."},
|
||||
{"id":"","label":"CLiP: Batch Interrogate","localized":"Batch Interrogate","reload":"","hint":"Process multiple images in a batch using CLiP."},
|
||||
{"id":"","label":"Task","localized":"","reload":"","hint":"Changes which task the model will perform. Regular text prompts can be used when the default option <b>Use Prompt</b> is selected.<br>When other options are selected, see the hint text inside an empty <b>Prompt</b> field for guidance."},
|
||||
{"id":"","label":"Task","localized":"","reload":"","hint":"Changes which task the model will perform. Regular text prompts can be used when the task is set to <b>Use Prompt</b>.<br>When other options are selected, see the hint text inside an empty <b>Prompt</b> field for guidance."},
|
||||
{"id":"","label":"Prefill text","localized":"","reload":"","hint":"Pre-fills the start of the model's response to guide its output format or content by forcing it to continue the prefill text.<br>Prefill is filtered out and does not appear in the final response.<br><br>Leave empty to let the model generate its own response from scratch."},
|
||||
{"id":"","label":"Start","localized":"","reload":"","hint":"Start"},
|
||||
{"id":"","label":"End","localized":"","reload":"","hint":"End"},
|
||||
|
||||
@@ -359,7 +359,6 @@ class VQA:
|
||||
self.processor = None
|
||||
self.model = None
|
||||
self.loaded: str = None
|
||||
self.quant_args = None
|
||||
self.last_annotated_image = None
|
||||
self.last_detection_data = None
|
||||
|
||||
@@ -386,7 +385,6 @@ class VQA:
|
||||
shared.log.error(f'VQA load: unknown model="{model_name}"')
|
||||
return
|
||||
|
||||
self.quant_args = model_quant.create_config(module='LLM')
|
||||
shared.log.debug(f'VQA load: pre-loading model="{model_name}" repo="{repo}"')
|
||||
|
||||
# Dispatch to appropriate loader (same logic as interrogate)
|
||||
@@ -448,6 +446,7 @@ class VQA:
|
||||
"""Load FastVLM model and tokenizer."""
|
||||
if self.model is None or self.loaded != repo:
|
||||
shared.log.debug(f'Interrogate load: vlm="{repo}"')
|
||||
quant_args = model_quant.create_config(module='LLM')
|
||||
self.model = None
|
||||
self.processor = transformers.AutoTokenizer.from_pretrained(repo, trust_remote_code=True, cache_dir=shared.opts.hfcache_dir)
|
||||
self.model = transformers.AutoModelForCausalLM.from_pretrained(
|
||||
@@ -455,7 +454,7 @@ class VQA:
|
||||
torch_dtype=devices.dtype,
|
||||
trust_remote_code=True,
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
**self.quant_args,
|
||||
**quant_args,
|
||||
)
|
||||
self.loaded = repo
|
||||
devices.torch_gc()
|
||||
@@ -502,11 +501,12 @@ class VQA:
|
||||
cls_name = transformers.Qwen2VLForConditionalGeneration
|
||||
else:
|
||||
cls_name = transformers.AutoModelForCausalLM
|
||||
quant_args = model_quant.create_config(module='LLM')
|
||||
self.model = cls_name.from_pretrained(
|
||||
repo,
|
||||
torch_dtype=devices.dtype,
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
**self.quant_args,
|
||||
**quant_args,
|
||||
)
|
||||
self.processor = transformers.AutoProcessor.from_pretrained(repo, max_pixels=1024*1024, cache_dir=shared.opts.hfcache_dir)
|
||||
if 'LLM' in shared.opts.cuda_compile:
|
||||
@@ -638,11 +638,12 @@ class VQA:
|
||||
cls = transformers.Gemma3nForConditionalGeneration # pylint: disable=no-member
|
||||
else:
|
||||
cls = transformers.Gemma3ForConditionalGeneration
|
||||
quant_args = model_quant.create_config(module='LLM')
|
||||
self.model = cls.from_pretrained(
|
||||
repo,
|
||||
torch_dtype=devices.dtype,
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
**self.quant_args,
|
||||
**quant_args,
|
||||
)
|
||||
if 'LLM' in shared.opts.cuda_compile:
|
||||
self.model = sd_models_compile.compile_torch(self.model)
|
||||
@@ -839,11 +840,12 @@ class VQA:
|
||||
if self.model is None or self.loaded != repo:
|
||||
shared.log.debug(f'Interrogate load: vlm="{repo}"')
|
||||
self.model = None
|
||||
quant_args = model_quant.create_config(module='LLM')
|
||||
self.model = transformers.AutoModelForVision2Seq.from_pretrained(
|
||||
repo,
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
torch_dtype=devices.dtype,
|
||||
**self.quant_args,
|
||||
**quant_args,
|
||||
)
|
||||
self.processor = transformers.AutoProcessor.from_pretrained(repo, max_pixels=1024*1024, cache_dir=shared.opts.hfcache_dir)
|
||||
if 'LLM' in shared.opts.cuda_compile:
|
||||
@@ -1150,13 +1152,14 @@ class VQA:
|
||||
shared.log.debug(f'Interrogate load: vlm="{repo_name}" revision="{effective_revision}" path="{shared.opts.hfcache_dir}"')
|
||||
transformers.dynamic_module_utils.get_imports = get_imports
|
||||
self.model = None
|
||||
quant_args = model_quant.create_config(module='LLM')
|
||||
self.model = transformers.Florence2ForConditionalGeneration.from_pretrained(
|
||||
repo_name,
|
||||
dtype=torch.bfloat16,
|
||||
revision=effective_revision,
|
||||
torch_dtype=devices.dtype,
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
**self.quant_args,
|
||||
**quant_args,
|
||||
)
|
||||
self.processor = transformers.AutoProcessor.from_pretrained(repo_name, max_pixels=1024*1024, trust_remote_code=True, revision=effective_revision, cache_dir=shared.opts.hfcache_dir)
|
||||
transformers.dynamic_module_utils.get_imports = _get_imports
|
||||
@@ -1230,7 +1233,6 @@ class VQA:
|
||||
self.last_detection_data = None
|
||||
jobid = shared.state.begin('Interrogate LLM')
|
||||
t0 = time.time()
|
||||
self.quant_args = model_quant.create_config(module='LLM')
|
||||
model_name = model_name or shared.opts.interrogate_vlm_model
|
||||
prefill = vlm_prefill if prefill is None else prefill # Use provided prefill when specified
|
||||
if isinstance(image, list):
|
||||
|
||||
+2
-2
@@ -502,7 +502,7 @@ options_templates.update(options_section(('saving-images', "Image Options"), {
|
||||
"font_color": OptionInfo("#FFFFFF", "Font color", gr.ColorPicker, {}),
|
||||
|
||||
"image_sep_browser": OptionInfo("<h2>Image Gallery</h2>", "", gr.HTML),
|
||||
"browser_cache": OptionInfo(True, "Use image gallery cache [STRONGLY recommended]"),
|
||||
"browser_cache": OptionInfo(True, "Use image gallery cache"),
|
||||
"browser_folders": OptionInfo("", "Additional image browser folders"),
|
||||
"browser_fixed_width": OptionInfo(False, "Use fixed width thumbnails"),
|
||||
"viewer_show_metadata": OptionInfo(True, "Show metadata in full screen image browser"),
|
||||
@@ -643,7 +643,7 @@ options_templates.update(options_section(('postprocessing', "Postprocessing"), {
|
||||
}))
|
||||
|
||||
options_templates.update(options_section(('interrogate', "Interrogate"), {
|
||||
"interrogate_default_type": OptionInfo("OpenCLiP", "Default type", gr.Radio, {"choices": ["OpenCLiP", "VLM", "DeepBooru"]}),
|
||||
"interrogate_default_type": OptionInfo("VLM", "Default caption type", gr.Radio, {"choices": ["OpenCLiP", "VLM", "DeepBooru"]}),
|
||||
"interrogate_offload": OptionInfo(True, "Offload models "),
|
||||
"interrogate_score": OptionInfo(False, "Include scores in results when available"),
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ from modules import shared, ui_common, generation_parameters_copypaste
|
||||
from modules.interrogate import openclip
|
||||
|
||||
|
||||
default_task = "Short Caption"
|
||||
|
||||
def vlm_caption_wrapper(question, system_prompt, prompt, image, model_name, prefill, thinking_mode):
|
||||
"""Wrapper for vqa.interrogate that handles annotated image display."""
|
||||
from modules.interrogate import vqa
|
||||
@@ -17,7 +19,7 @@ def update_vlm_prompts_for_model(model_name):
|
||||
"""Update the task dropdown choices based on selected model."""
|
||||
from modules.interrogate import vqa
|
||||
prompts = vqa.get_prompts_for_model(model_name)
|
||||
return gr.update(choices=prompts, value=prompts[0] if prompts else "Use Prompt")
|
||||
return gr.update(choices=prompts, value=prompts[0] if prompts else default_task)
|
||||
|
||||
|
||||
def update_vlm_prompt_placeholder(question):
|
||||
@@ -68,9 +70,9 @@ def create_ui():
|
||||
with gr.Row():
|
||||
vlm_system = gr.Textbox(label="System Prompt", value=vqa.vlm_system, lines=1, elem_id='vlm_system')
|
||||
with gr.Row():
|
||||
vlm_question = gr.Dropdown(label="Task", allow_custom_value=False, choices=initial_prompts, value=initial_prompts[0] if initial_prompts else "Use Prompt", elem_id='vlm_question')
|
||||
vlm_question = gr.Dropdown(label="Task", allow_custom_value=False, choices=initial_prompts, value=default_task, elem_id='vlm_question')
|
||||
with gr.Row():
|
||||
vlm_prompt = gr.Textbox(label="Prompt", placeholder=vqa.get_prompt_placeholder(initial_prompts[0] if initial_prompts else "Use Prompt"), lines=2, elem_id='vlm_prompt')
|
||||
vlm_prompt = gr.Textbox(label="Prompt", placeholder=vqa.get_prompt_placeholder(initial_prompts[0]), lines=2, elem_id='vlm_prompt')
|
||||
with gr.Row(elem_id='interrogate_buttons_query'):
|
||||
vlm_model = gr.Dropdown(list(vqa.vlm_models), value=current_vlm_model, label='VLM Model', elem_id='vlm_model')
|
||||
with gr.Row():
|
||||
|
||||
+1
-1
Submodule wiki updated: 2fb950abde...83c4488c4a
Reference in New Issue
Block a user