From a51e1501d60a4740ed4b2932e5ef505d52c1caa0 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sat, 6 Dec 2025 02:26:34 +0000 Subject: [PATCH] fix(vqa): no moondream3 compile during explicit load - Initialize KV caches before moving model to device - Disable flex_attention decoding to avoid torch.compile hang - Remove unused compile step (controlled by cuda_compile setting) The flex_attention's create_block_mask triggers torch compilation which can hang the system when called during model preload. --- modules/interrogate/moondream3.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/interrogate/moondream3.py b/modules/interrogate/moondream3.py index 4983cfcd3..c1e10e4a9 100644 --- a/modules/interrogate/moondream3.py +++ b/modules/interrogate/moondream3.py @@ -41,7 +41,7 @@ def get_settings(): def load_model(repo: str): - """Load and compile Moondream 3 model.""" + """Load Moondream 3 model.""" global moondream3_model, loaded # pylint: disable=global-statement if moondream3_model is None or loaded != repo: @@ -56,9 +56,15 @@ def load_model(repo: str): ) moondream3_model.eval() - if 'LLM' in shared.opts.cuda_compile: - debug('VQA interrogate: handler=moondream3 compiling model for fast decoding') - moondream3_model.compile() # Critical for fast decoding per moondream3 docs + + # Initialize KV caches before moving to device (they're lazy by default) + if hasattr(moondream3_model, '_setup_caches'): + moondream3_model._setup_caches() + + # Disable flex_attention decoding (can cause hangs due to torch.compile) + if hasattr(moondream3_model, 'model') and hasattr(moondream3_model.model, 'use_flex_decoding'): + moondream3_model.model.use_flex_decoding = False + loaded = repo devices.torch_gc()