Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 09:14:35 +02:00
Code Issues Packages Projects Releases Wiki Activity

add moondream2, sdnq xyzgrid timing info

Browse Source 
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
2025-06-27 09:41:32 -04:00
parent f9480a1047
commit 78330142ae
8 changed files with 38 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+4 -6
View File
@@ -19,8 +19,9 @@
- available via *networks -> models -> reference*
- *note*: model is still in training so future updates will trigger re-download
- [JoyCaption Beta](https://huggingface.co/fancyfeast/llama-joycaption-beta-one-hf-llava) support (in addition to existing JoyCaption Alpha)
- available via *caption -> vlm caption*
- available via *caption -> vlm caption*
- [MoonDream 2](https://huggingface.co/vikhyatk/moondream2) support (updated)
- available via *caption -> vlm caption*
- **Changes**
- Support Remote VAE with *Omnigen, Lumina 2 and PixArt*
- Add `--trace` command line param that enables trace logging
@@ -28,20 +29,17 @@
- Control move global settings to control elements -> control settings tab
- Control add setting to run hires with or without control
- Update OpenVINO to 2025.2.0
- **SDNQ Quantization**
- Add modules_to_not_convert support for post mode
- Add `modules_to_not_convert` support for post mode
- Fix Qwen 2.5 with int8 matmul
- Fix Dora loading
- Remove per layer GC
- Improve offload compatibility
- Add support for XYZ grid to test quantization modes
*note*: you need to enable quantization and choose what it applies on, then xyz grid can change quantization mode
- **API**
- Add `/sdapi/v1/lora?lora=<lora_name>` endpoint that returns full lora info and metadata
- Add `/sdapi/v1/controlnets?model_type=<model_type|all|None>` endpoints that returns list of available controlnets for specific model type
- **Fixes**
- IPEX with DPM2++ FlowMatch samplers
- Invalid attention processor with ControlNet
TODO.md
-1
View File
@@ -27,7 +27,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
### Models
- [Moondream vlm](https://github.com/vikhyat/moondream)
- [AniSora t2v](https://github.com/bilibili/Index-anisora)
- [Ming t2i](https://github.com/inclusionAI/Ming)
- [Magi t2v](https://github.com/SandAI-org/MAGI-1)
modules/interrogate/vqa.py
+12 -2
View File
@@ -429,7 +429,7 @@ def moondream(question: str, image: Image.Image, repo: str = None):
model = None
model = transformers.AutoModelForCausalLM.from_pretrained(
repo,
revision="2024-08-26",
revision="2025-06-21",
trust_remote_code=True,
cache_dir=shared.opts.hfcache_dir
)
@@ -442,7 +442,17 @@ def moondream(question: str, image: Image.Image, repo: str = None):
question = question.replace('<', '').replace('>', '').replace('_', ' ')
encoded = model.encode_image(image)
with devices.inference_context():
response = model.answer_question(encoded, question, processor)
if question == 'CAPTION':
response = model.caption(image, length="short")['caption']
elif question == 'DETAILED CAPTION':
response = model.caption(image, length="normal")['caption']
elif question == 'MORE DETAILED CAPTION':
response = model.caption(image, length="long")['caption']
else:
response = model.answer_question(encoded, question, processor)['answer']
# model.detect(image, "face")
# model.point(image, "person")
# model.detect_gaze(image)
return response
scripts/prompt_matrix.py
+1 -1
View File
@@ -21,7 +21,7 @@ def draw_xy_grid(xs, ys, x_label, y_label, cell):
for ix, x in enumerate(xs):
state.job = f"{ix + iy * len(xs) + 1} out of {len(xs) * len(ys)}"
processed = cell(x, y)
processed, t = cell(x, y)
if first_processed is None:
first_processed = processed
scripts/xyz_grid.py
+6 -2
View File
@@ -1,6 +1,7 @@
# xyz grid that shows as selectable script
import os
import csv
import time
import random
from collections import namedtuple
from copy import copy
@@ -303,7 +304,7 @@ class Script(scripts.Script):
def cell(x, y, z, ix, iy, iz):
if shared.state.interrupted:
return processing.Processed(p, [], p.seed, "")
return processing.Processed(p, [], p.seed, ""), 0
p.xyz = True
pc = copy(p)
pc.override_settings_restore_afterwards = False
@@ -311,6 +312,8 @@ class Script(scripts.Script):
x_opt.apply(pc, x, xs)
y_opt.apply(pc, y, ys)
z_opt.apply(pc, z, zs)
t0 = time.time()
try:
processed = processing.process_images(pc)
except Exception as e:
@@ -341,7 +344,8 @@ class Script(scripts.Script):
pc.extra_generation_params["Fixed Z Values"] = ", ".join([str(z) for z in zs])
grid_text = f'{len(zs)}x{len(xs)}x{len(ys)}' if len(zs) > 0 else f'{len(xs)}x{len(ys)}'
grid_infotext[0] = processing.create_infotext(pc, pc.all_prompts, pc.all_seeds, pc.all_subseeds, grid=grid_text)
return processed
t1 = time.time()
return processed, t1-t0
with SharedSettingsStackHelper():
processed = draw_xyz_grid(
scripts/xyz_grid_draw.py
+5 -5
View File
@@ -22,9 +22,9 @@ def draw_xyz_grid(p, xs, ys, zs, x_labels, y_labels, z_labels, cell, draw_legend
def index(ix, iy, iz):
return ix + iy * len(xs) + iz * len(xs) * len(ys)
p0 = time.time()
processed: processing.Processed = cell(x, y, z, ix, iy, iz)
p1 = time.time()
res = cell(x, y, z, ix, iy, iz)
processed: processing.Processed = res[0] if isinstance(res, tuple) else res
elapsed = res[1] if isinstance(res, tuple) else 0
if processed_result is None:
processed_result = copy(processed)
if processed_result is None:
@@ -48,13 +48,13 @@ def draw_xyz_grid(p, xs, ys, zs, x_labels, y_labels, z_labels, cell, draw_legend
if len(z_labels[iz]) > 0:
overlay_text += f'{z_labels[iz]}\n'
if include_time:
overlay_text += f'Time: {p1 - p0:.2f}'
overlay_text += f'Time: {elapsed:.2f}'
if len(overlay_text) > 0:
processed_result.images[idx] = images.draw_overlay(processed_result.images[idx], overlay_text)
processed_result.all_prompts[idx] = processed.prompt
processed_result.all_seeds[idx] = processed.seed
processed_result.infotexts[idx] = processed.infotexts[0]
processed_result.time[idx] = round(p1 - p0, 2)
processed_result.time[idx] = round(elapsed, 2)
else:
cell_mode = "P"
cell_size = (processed_result.width, processed_result.height)
scripts/xyz_grid_on.py
+6 -2
View File
@@ -1,6 +1,7 @@
# xyz grid that shows up as alwayson script
import os
import csv
import time
import random
from collections import namedtuple
from copy import copy
@@ -322,7 +323,7 @@ class Script(scripts.Script):
def cell(x, y, z, ix, iy, iz):
if shared.state.interrupted:
return processing.Processed(p, [], p.seed, "")
return processing.Processed(p, [], p.seed, ""), 0
p.xyz = True
pc = copy(p)
pc.override_settings_restore_afterwards = False
@@ -331,6 +332,8 @@ class Script(scripts.Script):
y_opt.apply(pc, y, ys)
z_opt.apply(pc, z, zs)
print('HERE')
t0 = time.time()
try:
processed = processing.process_images(pc)
except Exception as e:
@@ -362,7 +365,8 @@ class Script(scripts.Script):
pc.extra_generation_params["Fixed Z Values"] = ", ".join([str(z) for z in zs])
info = processing.create_infotext(pc, pc.all_prompts, pc.all_seeds, pc.all_subseeds, grid=f'{len(zs)}x{len(xs)}x{len(ys)}')
grid_infotext.insert(0, info)
return processed
t1 = time.time()
return processed, t1-t0
with SharedSettingsStackHelper():
processed: processing.Processed = draw_xyz_grid(
scripts/xyz_grid_shared.py
+4 -2
View File
@@ -149,13 +149,15 @@ def confirm_samplers(p, xs):
def apply_sdnq_quant(p, x, xs):
shared.opts.sdnq_quantize_weights_mode = x
sd_models.unload_model_weights(op='model') # reload will happen on-demand
sd_models.unload_model_weights(op='model')
sd_models.reload_model_weights()
shared.log.debug(f'XYZ grid apply sdnq quant: mode="{x}"')
def apply_sdnq_quant_te(p, x, xs):
shared.opts.sdnq_quantize_weights_mode_te = x
sd_models.unload_model_weights(op='model') # reload will happen on-demand
sd_models.unload_model_weights(op='model')
sd_models.reload_model_weights()
shared.log.debug(f'XYZ grid apply sdnq quant te: mode="{x}"')