From 78330142aee73db142fce40caec240d9d5519866 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 27 Jun 2025 09:41:32 -0400 Subject: [PATCH] add moondream2, sdnq xyzgrid timing info Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 10 ++++------ TODO.md | 1 - modules/interrogate/vqa.py | 14 ++++++++++++-- scripts/prompt_matrix.py | 2 +- scripts/xyz_grid.py | 8 ++++++-- scripts/xyz_grid_draw.py | 10 +++++----- scripts/xyz_grid_on.py | 8 ++++++-- scripts/xyz_grid_shared.py | 6 ++++-- 8 files changed, 38 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 849ed50af..4334ac18b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,9 @@ - available via *networks -> models -> reference* - *note*: model is still in training so future updates will trigger re-download - [JoyCaption Beta](https://huggingface.co/fancyfeast/llama-joycaption-beta-one-hf-llava) support (in addition to existing JoyCaption Alpha) - - available via *caption -> vlm caption* - + - available via *caption -> vlm caption* + - [MoonDream 2](https://huggingface.co/vikhyatk/moondream2) support (updated) + - available via *caption -> vlm caption* - **Changes** - Support Remote VAE with *Omnigen, Lumina 2 and PixArt* - Add `--trace` command line param that enables trace logging @@ -28,20 +29,17 @@ - Control move global settings to control elements -> control settings tab - Control add setting to run hires with or without control - Update OpenVINO to 2025.2.0 - - **SDNQ Quantization** - - Add modules_to_not_convert support for post mode + - Add `modules_to_not_convert` support for post mode - Fix Qwen 2.5 with int8 matmul - Fix Dora loading - Remove per layer GC - Improve offload compatibility - Add support for XYZ grid to test quantization modes *note*: you need to enable quantization and choose what it applies on, then xyz grid can change quantization mode - - **API** - Add `/sdapi/v1/lora?lora=` endpoint that returns full lora info and metadata - Add `/sdapi/v1/controlnets?model_type=` endpoints that returns list of available controlnets for specific model type - - **Fixes** - IPEX with DPM2++ FlowMatch samplers - Invalid attention processor with ControlNet diff --git a/TODO.md b/TODO.md index b35b5ea0e..8bc429532 100644 --- a/TODO.md +++ b/TODO.md @@ -27,7 +27,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ### Models -- [Moondream vlm](https://github.com/vikhyat/moondream) - [AniSora t2v](https://github.com/bilibili/Index-anisora) - [Ming t2i](https://github.com/inclusionAI/Ming) - [Magi t2v](https://github.com/SandAI-org/MAGI-1) diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 5d74a8b9c..f11065617 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -429,7 +429,7 @@ def moondream(question: str, image: Image.Image, repo: str = None): model = None model = transformers.AutoModelForCausalLM.from_pretrained( repo, - revision="2024-08-26", + revision="2025-06-21", trust_remote_code=True, cache_dir=shared.opts.hfcache_dir ) @@ -442,7 +442,17 @@ def moondream(question: str, image: Image.Image, repo: str = None): question = question.replace('<', '').replace('>', '').replace('_', ' ') encoded = model.encode_image(image) with devices.inference_context(): - response = model.answer_question(encoded, question, processor) + if question == 'CAPTION': + response = model.caption(image, length="short")['caption'] + elif question == 'DETAILED CAPTION': + response = model.caption(image, length="normal")['caption'] + elif question == 'MORE DETAILED CAPTION': + response = model.caption(image, length="long")['caption'] + else: + response = model.answer_question(encoded, question, processor)['answer'] + # model.detect(image, "face") + # model.point(image, "person") + # model.detect_gaze(image) return response diff --git a/scripts/prompt_matrix.py b/scripts/prompt_matrix.py index 5babf20e7..37cc2795c 100644 --- a/scripts/prompt_matrix.py +++ b/scripts/prompt_matrix.py @@ -21,7 +21,7 @@ def draw_xy_grid(xs, ys, x_label, y_label, cell): for ix, x in enumerate(xs): state.job = f"{ix + iy * len(xs) + 1} out of {len(xs) * len(ys)}" - processed = cell(x, y) + processed, t = cell(x, y) if first_processed is None: first_processed = processed diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index 240376b3c..c79ce6909 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -1,6 +1,7 @@ # xyz grid that shows as selectable script import os import csv +import time import random from collections import namedtuple from copy import copy @@ -303,7 +304,7 @@ class Script(scripts.Script): def cell(x, y, z, ix, iy, iz): if shared.state.interrupted: - return processing.Processed(p, [], p.seed, "") + return processing.Processed(p, [], p.seed, ""), 0 p.xyz = True pc = copy(p) pc.override_settings_restore_afterwards = False @@ -311,6 +312,8 @@ class Script(scripts.Script): x_opt.apply(pc, x, xs) y_opt.apply(pc, y, ys) z_opt.apply(pc, z, zs) + + t0 = time.time() try: processed = processing.process_images(pc) except Exception as e: @@ -341,7 +344,8 @@ class Script(scripts.Script): pc.extra_generation_params["Fixed Z Values"] = ", ".join([str(z) for z in zs]) grid_text = f'{len(zs)}x{len(xs)}x{len(ys)}' if len(zs) > 0 else f'{len(xs)}x{len(ys)}' grid_infotext[0] = processing.create_infotext(pc, pc.all_prompts, pc.all_seeds, pc.all_subseeds, grid=grid_text) - return processed + t1 = time.time() + return processed, t1-t0 with SharedSettingsStackHelper(): processed = draw_xyz_grid( diff --git a/scripts/xyz_grid_draw.py b/scripts/xyz_grid_draw.py index d654a2c7f..646a3bd52 100644 --- a/scripts/xyz_grid_draw.py +++ b/scripts/xyz_grid_draw.py @@ -22,9 +22,9 @@ def draw_xyz_grid(p, xs, ys, zs, x_labels, y_labels, z_labels, cell, draw_legend def index(ix, iy, iz): return ix + iy * len(xs) + iz * len(xs) * len(ys) - p0 = time.time() - processed: processing.Processed = cell(x, y, z, ix, iy, iz) - p1 = time.time() + res = cell(x, y, z, ix, iy, iz) + processed: processing.Processed = res[0] if isinstance(res, tuple) else res + elapsed = res[1] if isinstance(res, tuple) else 0 if processed_result is None: processed_result = copy(processed) if processed_result is None: @@ -48,13 +48,13 @@ def draw_xyz_grid(p, xs, ys, zs, x_labels, y_labels, z_labels, cell, draw_legend if len(z_labels[iz]) > 0: overlay_text += f'{z_labels[iz]}\n' if include_time: - overlay_text += f'Time: {p1 - p0:.2f}' + overlay_text += f'Time: {elapsed:.2f}' if len(overlay_text) > 0: processed_result.images[idx] = images.draw_overlay(processed_result.images[idx], overlay_text) processed_result.all_prompts[idx] = processed.prompt processed_result.all_seeds[idx] = processed.seed processed_result.infotexts[idx] = processed.infotexts[0] - processed_result.time[idx] = round(p1 - p0, 2) + processed_result.time[idx] = round(elapsed, 2) else: cell_mode = "P" cell_size = (processed_result.width, processed_result.height) diff --git a/scripts/xyz_grid_on.py b/scripts/xyz_grid_on.py index f40a1731b..db454b2b7 100644 --- a/scripts/xyz_grid_on.py +++ b/scripts/xyz_grid_on.py @@ -1,6 +1,7 @@ # xyz grid that shows up as alwayson script import os import csv +import time import random from collections import namedtuple from copy import copy @@ -322,7 +323,7 @@ class Script(scripts.Script): def cell(x, y, z, ix, iy, iz): if shared.state.interrupted: - return processing.Processed(p, [], p.seed, "") + return processing.Processed(p, [], p.seed, ""), 0 p.xyz = True pc = copy(p) pc.override_settings_restore_afterwards = False @@ -331,6 +332,8 @@ class Script(scripts.Script): y_opt.apply(pc, y, ys) z_opt.apply(pc, z, zs) + print('HERE') + t0 = time.time() try: processed = processing.process_images(pc) except Exception as e: @@ -362,7 +365,8 @@ class Script(scripts.Script): pc.extra_generation_params["Fixed Z Values"] = ", ".join([str(z) for z in zs]) info = processing.create_infotext(pc, pc.all_prompts, pc.all_seeds, pc.all_subseeds, grid=f'{len(zs)}x{len(xs)}x{len(ys)}') grid_infotext.insert(0, info) - return processed + t1 = time.time() + return processed, t1-t0 with SharedSettingsStackHelper(): processed: processing.Processed = draw_xyz_grid( diff --git a/scripts/xyz_grid_shared.py b/scripts/xyz_grid_shared.py index 265af6862..64f179075 100644 --- a/scripts/xyz_grid_shared.py +++ b/scripts/xyz_grid_shared.py @@ -149,13 +149,15 @@ def confirm_samplers(p, xs): def apply_sdnq_quant(p, x, xs): shared.opts.sdnq_quantize_weights_mode = x - sd_models.unload_model_weights(op='model') # reload will happen on-demand + sd_models.unload_model_weights(op='model') + sd_models.reload_model_weights() shared.log.debug(f'XYZ grid apply sdnq quant: mode="{x}"') def apply_sdnq_quant_te(p, x, xs): shared.opts.sdnq_quantize_weights_mode_te = x - sd_models.unload_model_weights(op='model') # reload will happen on-demand + sd_models.unload_model_weights(op='model') + sd_models.reload_model_weights() shared.log.debug(f'XYZ grid apply sdnq quant te: mode="{x}"')