prompt enhance and caption use llm-context

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -11,7 +11,7 @@
     Mage-Flow is a 4B-scale generative stack for efficient text-to-image generation and instruction-based image editing  
     *note*: Microsoft released and then unpublished the model, but we still have a mirror available for download  
 - **Features**
-  - storage analyzer: new feature that analyzes your used storage by sdnext per type and location  
+  - storage analyzer: new feature that analyzes your storage used by sdnext per type and location  
     *system -> storage*  
   - video: support for scripts/extensions  
     video processing now supports scripts and extensions (if they support video processing)  
@@ -22,11 +22,14 @@
   - remove background: new [lucida](https://huggingface.co/egeorcun/lucida) model
 - **API**
   - add `/sdapi/v1/storage` endpoint to return storage usage info  
+- **Internal**
+  - update core requirements
 - **Fixes**
   - seedvr quality
   - skip-all do not skip env init
   - sdnq check contiguous
   - torch reset compile cache on reload
+  - bypass sdna for caption/prompt-enhance calls
 
 ## Update for 2026-07-23
 

installer.py

@@ -554,7 +554,7 @@ def check_diffusers():
     t_start = time.time()
     if args.skip_all:
         return
-    target_commit = "8b33b473324423a9773d121c7caccb13601493a1" # diffusers commit hash == 0.40.0.dev0 == 07-29-2026
+    target_commit = "6f2010e8bbe61fd2a81a659b858e298edcba8fab" # diffusers commit hash == 0.40.0.dev0 == 08-04-2026
     # if args.use_rocm or args.use_zluda or args.use_directml:
     #     sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now
     pkg = package_spec('diffusers')
@@ -583,7 +583,7 @@ def check_transformers():
     pkg_transformers = package_spec('transformers')
     pkg_tokenizers = package_spec('tokenizers')
     # target_commit = '753d61104116eefc8ffc977327b441ee0c8d599f' # transformers commit hash == 4.57.6
-    # target_commit = "380e3cc5d59912a48508cb6d4959a31cd460e12e" # transformers commit hash == 5.5.0.dev-0409
+    # target_commit = "cf8572d34e39818e42dbf220701fbd3eb5b5a82a" # transformers commit hash == 5.14.0.dev0 == 08-04-2026
     target_commit = "b70d02fc724d04c916832ca4ead03ff05e8fb1ee" # transformers commit hash == 5.13.0.dev0 == 07-03-2026
     if args.use_directml:
         target_transformers = '4.52.4'

modules/caption/deepbooru.py

@@ -93,7 +93,7 @@ class DeepDanbooru:
             return ''
         pic = pil_image.resize((512, 512), resample=Image.Resampling.LANCZOS).convert("RGB")
         a = np.expand_dims(np.array(pic, dtype=np.float32), 0) / 255
-        with devices.inference_context():
+        with devices.llm_context():
             x = torch.from_numpy(a).to(device=devices.device, dtype=devices.dtype)
             y = self.model(x)[0].detach().float().cpu().numpy()
         probability_dict = {}

modules/caption/joycaption.py

@@ -108,7 +108,7 @@ def predict(question: str, image, vqa_model: str | None = None) -> str:
     inputs = processor(text=[convo_string], images=[image], return_tensors="pt").to(devices.device)
     inputs['pixel_values'] = inputs['pixel_values'].to(devices.dtype)
     try:
-        with devices.inference_context():
+        with devices.llm_context():
             generate_ids = llava_model.generate( # Generate the captions
                 **inputs,
                 # input_ids=inputs['input_ids'],

modules/caption/joytag.py

@@ -1073,7 +1073,7 @@ def predict(image: Image.Image):
     load()
     image_tensor = prepare_image(image, model.image_size).unsqueeze(0).to(device=devices.device, dtype=devices.dtype)
     try:
-        with devices.inference_context():
+        with devices.llm_context():
             preds = model({'image': image_tensor})
             tag_preds = preds['tags'].sigmoid().cpu()
     finally:

modules/caption/moondream3.py

@@ -105,7 +105,7 @@ def encode_image(image: Image.Image, cache_key: str | None = None):
 
     model = load_model(loaded)
 
-    with devices.inference_context():
+    with devices.llm_context():
         encoded = model.encode_image(image)
 
     if cache_key:
@@ -157,7 +157,7 @@ def query(image: Image.Image, question: str, repo: str, stream: bool = False,
     else:
         image_input = image
 
-    with devices.inference_context():
+    with devices.llm_context():
         response = model.query(
             image=image_input,
             question=question,
@@ -212,7 +212,7 @@ def caption(image: Image.Image, repo: str, length: str = 'normal', stream: bool
 
     debug(f'LLM: handler=moondream3 method=caption length={length} stream={stream} settings={settings}')
 
-    with devices.inference_context():
+    with devices.llm_context():
         response = model.caption(
             image,
             length=length,
@@ -244,7 +244,7 @@ def point(image: Image.Image, object_name: str, repo: str):
 
     debug(f'LLM: handler=moondream3 method=point object_name="{object_name}"')
 
-    with devices.inference_context():
+    with devices.llm_context():
         result = model.point(image, object_name)
 
     debug(f'LLM: handler=moondream3 point_raw_result="{result}" type={type(result)}')
@@ -281,7 +281,7 @@ def detect(image: Image.Image, object_name: str, repo: str, max_objects: int = 1
 
     debug(f'LLM: handler=moondream3 method=detect object_name="{object_name}" max_objects={max_objects}')
 
-    with devices.inference_context():
+    with devices.llm_context():
         result = model.detect(image, object_name)
 
     debug(f'LLM: handler=moondream3 detect_raw_result="{result}" type={type(result)}')

modules/caption/vqa.py

@@ -519,7 +519,7 @@ class VQA:
         attention_mask = torch.ones_like(input_ids, device=devices.device)
         px = self.model.get_vision_tower().image_processor(images=image, return_tensors="pt")
         px = px["pixel_values"].to(self.model.device, dtype=self.model.dtype)
-        with devices.inference_context():
+        with devices.llm_context():
             outputs = self.model.generate(
                 inputs=input_ids,
                 attention_mask=attention_mask,
@@ -673,7 +673,7 @@ class VQA:
         defaults = {k: v for k, v in helpers.get_default_args(self.model).items() if k not in gen_kwargs}
         log.debug(f'LLM: defaults={defaults}')
 
-        with devices.inference_context():
+        with devices.llm_context():
             output_ids = self.model.generate(
                 **inputs,
                 **gen_kwargs,
@@ -812,7 +812,7 @@ class VQA:
         defaults = {k: v for k, v in helpers.get_default_args(self.model).items() if k not in gen_kwargs}
         log.debug(f'LLM: defaults={defaults}')
 
-        with devices.inference_context():
+        with devices.llm_context():
             generation = self.model.generate(
                 **inputs,
                 **gen_kwargs,
@@ -899,7 +899,7 @@ class VQA:
         defaults = {k: v for k, v in helpers.get_default_args(self.model).items() if k not in gen_kwargs}
         log.debug(f'LLM: defaults={defaults}')
 
-        with devices.inference_context():
+        with devices.llm_context():
             generation = self.model.generate(**inputs, **gen_kwargs)
         generation = generation[0][input_len:]
         response = self.processor.decode(generation, skip_special_tokens=True)
@@ -932,7 +932,7 @@ class VQA:
         question = question.replace('<', '').replace('>', '').replace('_', ' ')
         model_inputs = self.processor(text=question, images=image, return_tensors="pt").to(devices.device, devices.dtype)
         input_len = model_inputs["input_ids"].shape[-1]
-        with devices.inference_context():
+        with devices.llm_context():
             generation = self.model.generate(
                 **model_inputs,
                 **get_kwargs(self.model),
@@ -989,7 +989,7 @@ class VQA:
         if pixel_values is not None:
             pixel_values = pixel_values.to(dtype=visual_tokenizer.dtype, device=visual_tokenizer.device)
         pixel_values = [pixel_values]
-        with devices.inference_context():
+        with devices.llm_context():
             output_ids = self.model.generate(
                 input_ids,
                 pixel_values=pixel_values,
@@ -1091,7 +1091,7 @@ class VQA:
         defaults = {k: v for k, v in helpers.get_default_args(self.model).items() if k not in gen_kwargs}
         log.debug(f'LLM: defaults={defaults}')
 
-        with devices.inference_context():
+        with devices.llm_context():
             output_ids = self.model.generate(
                 **inputs,
                 **gen_kwargs,
@@ -1135,7 +1135,7 @@ class VQA:
             input_ids = [self.processor.tokenizer.cls_token_id] + input_ids
             input_ids = torch.tensor(input_ids).unsqueeze(0)
             git_dict['input_ids'] = input_ids.to(devices.device)
-        with devices.inference_context():
+        with devices.llm_context():
             generated_ids = self.model.generate(**git_dict)
         response = self.processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
         return response
@@ -1164,7 +1164,7 @@ class VQA:
         move_aux_to_gpu('vqa')
         inputs = self.processor(image, question, return_tensors="pt")
         inputs = inputs.to(devices.device, devices.dtype)
-        with devices.inference_context():
+        with devices.llm_context():
             outputs = self.model.generate(**inputs)
         response = self.processor.decode(outputs[0], skip_special_tokens=True)
         return response
@@ -1193,7 +1193,7 @@ class VQA:
         move_aux_to_gpu('vqa')
         inputs = self.processor(image, question, return_tensors="pt")
         inputs = inputs.to(devices.device)
-        with devices.inference_context():
+        with devices.llm_context():
             outputs = self.model(**inputs)
         logits = outputs.logits
         idx = logits.argmax(-1).item()
@@ -1227,7 +1227,7 @@ class VQA:
         else:
             inputs = self.processor(images=image, return_tensors="pt")
         inputs = {k: v.to(devices.device, devices.dtype) if v.is_floating_point() else v.to(devices.device) for k, v in inputs.items()}
-        with devices.inference_context():
+        with devices.llm_context():
             outputs = self.model.generate(**inputs)
         response = self.processor.decode(outputs[0], skip_special_tokens=True)
         return response
@@ -1258,7 +1258,7 @@ class VQA:
         self._load_moondream(repo)
         move_aux_to_gpu('vqa')
         question = question.replace('<', '').replace('>', '').replace('_', ' ')
-        with devices.inference_context():
+        with devices.llm_context():
             if question == 'CAPTION':
                 response = self.model.caption(image, length="short")['caption']
             elif question == 'DETAILED CAPTION':
@@ -1379,7 +1379,7 @@ class VQA:
             gen_kwargs['decoder_start_token_id'] = bos_token_id
             debug(f'LLM: handler=florence setting decoder_start_token_id={bos_token_id}')
         debug(f'LLM: handler=florence generation_kwargs={gen_kwargs}')
-        with devices.inference_context(), devices.bypass_sdpa_hijacks():
+        with devices.llm_context():
             generated_ids = self.model.generate(
                 input_ids=input_ids,
                 pixel_values=pixel_values,
@@ -1431,7 +1431,7 @@ class VQA:
             'mask_prompts': None,
             'tokenizer': self.processor,
         }
-        with devices.inference_context():
+        with devices.llm_context():
             return_dict = self.model.predict_forward(**input_dict)
         response = return_dict["prediction"]  # the text format answer
         return response

modules/devices.py

@@ -765,6 +765,15 @@ def bypass_sdpa_hijacks():
             log.debug('SDPA bypass: restored hijacked attention')
 
 
+@contextlib.contextmanager
+def llm_context():
+    """
+    Combined context manager that applies both inference_context and bypass_sdpa_hijacks.
+    """
+    with inference_context(), bypass_sdpa_hijacks():
+        yield
+
+
 def torch_reset() -> bool:
     """
     Resets PyTorch execution graph, flushes VRAM caches, and syncs streams.

modules/model_quant.py

@@ -94,7 +94,14 @@ def get_sdnq_devices(mode="pre"):
     return quantization_device, return_device
 
 
-def create_sdnq_config(kwargs = None, allow: bool = True, module: str = 'Model', weights_dtype: str | None = None, quantized_matmul_dtype: str | None = None, modules_to_not_convert: list | None = None, modules_dtype_dict: dict | None = None):
+def create_sdnq_config(kwargs = None,
+                       allow: bool = True,
+                       module: str = 'Model',
+                       weights_dtype: str | None = None,
+                       quantized_matmul_dtype: str | None = None,
+                       modules_to_not_convert: list | None = None,
+                       modules_dtype_dict: dict | None = None,
+                      ):
     from modules import shared
     if allow and (shared.opts.sdnq_quantize_mode in {'pre', 'auto'}) and (module == 'any' or module in shared.opts.sdnq_quantize_weights):
         from modules.sdnq import SDNQConfig
@@ -216,7 +223,12 @@ def create_config(kwargs = None, allow: bool = True, module: str = 'Model', modu
         kwargs = {}
     if module == 'Model' and dont_quant():
         return kwargs
-    kwargs = create_sdnq_config(kwargs, allow=allow, module=module, modules_to_not_convert=modules_to_not_convert, modules_dtype_dict=modules_dtype_dict)
+    kwargs = create_sdnq_config(kwargs,
+                                allow=allow,
+                                module=module,
+                                modules_to_not_convert=modules_to_not_convert,
+                                modules_dtype_dict=modules_dtype_dict
+                               )
     if kwargs is not None and 'quantization_config' in kwargs:
         if debug:
             log.trace(f'Quantization: type=sdnq config={kwargs.get("quantization_config", None)}')

pipelines/sefi/pipeline_sefi.py

@@ -252,7 +252,7 @@ class SeFiPipeline(DiffusionPipeline):
         """
         x_list = []
         for data, pos in zip(x, x_ids):
-            _, ch = data.shape  # noqa: F841
+            _, ch = data.shape
             h_ids = pos[:, 1].to(torch.int64)
             w_ids = pos[:, 2].to(torch.int64)
 

requirements.txt

@@ -30,8 +30,8 @@ requests==2.34.2
 tqdm==4.68.3
 accelerate==1.14.0
 einops==0.8.2
-huggingface_hub==1.25.1
-hf_xet==1.5.2
+huggingface_hub==1.26.0
+hf_xet==1.6.0
 numpy==2.1.2
 pandas==2.3.1
 protobuf==7.35.1
@@ -44,6 +44,7 @@ typing-extensions==4.15.0
 sentencepiece==0.2.1
 
 # lint
+ty
 ruff
 pylint
 pre-commit

scripts/prompt_enhance_ext.py

@@ -80,7 +80,7 @@ class PromptEnhanceScript(scripts_manager.Script):
             gguf_args['model_type'] = model_type
             gguf_args['gguf_file'] = model_file
 
-        quant_args = model_quant.create_config(module='LLM') if not gguf_args else {}
+        quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d', 'linear_attn.conv1d']) if not gguf_args else {}
 
         try:
             t0 = time.time()
@@ -127,6 +127,7 @@ class PromptEnhanceScript(scripts_manager.Script):
                 )
             finally:
                 sd_models.set_huggingface_options(quiet=True)
+
             self.llm.eval()
             register_aux('prompt_enhance', self.llm)
             tokenizer_args = { 'pretrained_model_name_or_path': model_repo }
@@ -489,7 +490,7 @@ class PromptEnhanceScript(scripts_manager.Script):
             return prompt_text # Return original text part on error
 
         try:
-            with devices.inference_context():
+            with devices.llm_context():
                 move_aux_to_gpu('prompt_enhance')
                 gen_kwargs = {
                     'do_sample': sample,
@@ -535,8 +536,7 @@ class PromptEnhanceScript(scripts_manager.Script):
         except Exception as e:
             outputs = None
             log.error(f'Prompt enhance generate: {e}')
-            if debug_enabled:
-                errors.display(e, 'Prompt enhance')
+            errors.display(e, 'Prompt enhance')
             self.busy = False
             response = f'Error: {str(e)}'
         finally: