From a83795273e20724f21639e798520dd4762b64c4e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Aug 2026 14:21:53 +0200 Subject: [PATCH] prompt enhance and caption use llm-context Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 5 ++++- installer.py | 4 ++-- modules/caption/deepbooru.py | 2 +- modules/caption/joycaption.py | 2 +- modules/caption/joytag.py | 2 +- modules/caption/moondream3.py | 10 +++++----- modules/caption/vqa.py | 28 ++++++++++++++-------------- modules/devices.py | 9 +++++++++ modules/model_quant.py | 16 ++++++++++++++-- pipelines/sefi/pipeline_sefi.py | 2 +- requirements.txt | 5 +++-- scripts/prompt_enhance_ext.py | 8 ++++---- 12 files changed, 59 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3ea72705..68e663c9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Mage-Flow is a 4B-scale generative stack for efficient text-to-image generation and instruction-based image editing *note*: Microsoft released and then unpublished the model, but we still have a mirror available for download - **Features** - - storage analyzer: new feature that analyzes your used storage by sdnext per type and location + - storage analyzer: new feature that analyzes your storage used by sdnext per type and location *system -> storage* - video: support for scripts/extensions video processing now supports scripts and extensions (if they support video processing) @@ -22,11 +22,14 @@ - remove background: new [lucida](https://huggingface.co/egeorcun/lucida) model - **API** - add `/sdapi/v1/storage` endpoint to return storage usage info +- **Internal** + - update core requirements - **Fixes** - seedvr quality - skip-all do not skip env init - sdnq check contiguous - torch reset compile cache on reload + - bypass sdna for caption/prompt-enhance calls ## Update for 2026-07-23 diff --git a/installer.py b/installer.py index 75db42840..b021f3050 100644 --- a/installer.py +++ b/installer.py @@ -554,7 +554,7 @@ def check_diffusers(): t_start = time.time() if args.skip_all: return - target_commit = "8b33b473324423a9773d121c7caccb13601493a1" # diffusers commit hash == 0.40.0.dev0 == 07-29-2026 + target_commit = "6f2010e8bbe61fd2a81a659b858e298edcba8fab" # diffusers commit hash == 0.40.0.dev0 == 08-04-2026 # if args.use_rocm or args.use_zluda or args.use_directml: # sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now pkg = package_spec('diffusers') @@ -583,7 +583,7 @@ def check_transformers(): pkg_transformers = package_spec('transformers') pkg_tokenizers = package_spec('tokenizers') # target_commit = '753d61104116eefc8ffc977327b441ee0c8d599f' # transformers commit hash == 4.57.6 - # target_commit = "380e3cc5d59912a48508cb6d4959a31cd460e12e" # transformers commit hash == 5.5.0.dev-0409 + # target_commit = "cf8572d34e39818e42dbf220701fbd3eb5b5a82a" # transformers commit hash == 5.14.0.dev0 == 08-04-2026 target_commit = "b70d02fc724d04c916832ca4ead03ff05e8fb1ee" # transformers commit hash == 5.13.0.dev0 == 07-03-2026 if args.use_directml: target_transformers = '4.52.4' diff --git a/modules/caption/deepbooru.py b/modules/caption/deepbooru.py index 88095ed01..facd30fd0 100644 --- a/modules/caption/deepbooru.py +++ b/modules/caption/deepbooru.py @@ -93,7 +93,7 @@ class DeepDanbooru: return '' pic = pil_image.resize((512, 512), resample=Image.Resampling.LANCZOS).convert("RGB") a = np.expand_dims(np.array(pic, dtype=np.float32), 0) / 255 - with devices.inference_context(): + with devices.llm_context(): x = torch.from_numpy(a).to(device=devices.device, dtype=devices.dtype) y = self.model(x)[0].detach().float().cpu().numpy() probability_dict = {} diff --git a/modules/caption/joycaption.py b/modules/caption/joycaption.py index ee527bc70..a96226f73 100644 --- a/modules/caption/joycaption.py +++ b/modules/caption/joycaption.py @@ -108,7 +108,7 @@ def predict(question: str, image, vqa_model: str | None = None) -> str: inputs = processor(text=[convo_string], images=[image], return_tensors="pt").to(devices.device) inputs['pixel_values'] = inputs['pixel_values'].to(devices.dtype) try: - with devices.inference_context(): + with devices.llm_context(): generate_ids = llava_model.generate( # Generate the captions **inputs, # input_ids=inputs['input_ids'], diff --git a/modules/caption/joytag.py b/modules/caption/joytag.py index e105eb2fe..d4bc98a0c 100644 --- a/modules/caption/joytag.py +++ b/modules/caption/joytag.py @@ -1073,7 +1073,7 @@ def predict(image: Image.Image): load() image_tensor = prepare_image(image, model.image_size).unsqueeze(0).to(device=devices.device, dtype=devices.dtype) try: - with devices.inference_context(): + with devices.llm_context(): preds = model({'image': image_tensor}) tag_preds = preds['tags'].sigmoid().cpu() finally: diff --git a/modules/caption/moondream3.py b/modules/caption/moondream3.py index 07866adba..eacbec621 100644 --- a/modules/caption/moondream3.py +++ b/modules/caption/moondream3.py @@ -105,7 +105,7 @@ def encode_image(image: Image.Image, cache_key: str | None = None): model = load_model(loaded) - with devices.inference_context(): + with devices.llm_context(): encoded = model.encode_image(image) if cache_key: @@ -157,7 +157,7 @@ def query(image: Image.Image, question: str, repo: str, stream: bool = False, else: image_input = image - with devices.inference_context(): + with devices.llm_context(): response = model.query( image=image_input, question=question, @@ -212,7 +212,7 @@ def caption(image: Image.Image, repo: str, length: str = 'normal', stream: bool debug(f'LLM: handler=moondream3 method=caption length={length} stream={stream} settings={settings}') - with devices.inference_context(): + with devices.llm_context(): response = model.caption( image, length=length, @@ -244,7 +244,7 @@ def point(image: Image.Image, object_name: str, repo: str): debug(f'LLM: handler=moondream3 method=point object_name="{object_name}"') - with devices.inference_context(): + with devices.llm_context(): result = model.point(image, object_name) debug(f'LLM: handler=moondream3 point_raw_result="{result}" type={type(result)}') @@ -281,7 +281,7 @@ def detect(image: Image.Image, object_name: str, repo: str, max_objects: int = 1 debug(f'LLM: handler=moondream3 method=detect object_name="{object_name}" max_objects={max_objects}') - with devices.inference_context(): + with devices.llm_context(): result = model.detect(image, object_name) debug(f'LLM: handler=moondream3 detect_raw_result="{result}" type={type(result)}') diff --git a/modules/caption/vqa.py b/modules/caption/vqa.py index a2c4bee58..784f6d941 100644 --- a/modules/caption/vqa.py +++ b/modules/caption/vqa.py @@ -519,7 +519,7 @@ class VQA: attention_mask = torch.ones_like(input_ids, device=devices.device) px = self.model.get_vision_tower().image_processor(images=image, return_tensors="pt") px = px["pixel_values"].to(self.model.device, dtype=self.model.dtype) - with devices.inference_context(): + with devices.llm_context(): outputs = self.model.generate( inputs=input_ids, attention_mask=attention_mask, @@ -673,7 +673,7 @@ class VQA: defaults = {k: v for k, v in helpers.get_default_args(self.model).items() if k not in gen_kwargs} log.debug(f'LLM: defaults={defaults}') - with devices.inference_context(): + with devices.llm_context(): output_ids = self.model.generate( **inputs, **gen_kwargs, @@ -812,7 +812,7 @@ class VQA: defaults = {k: v for k, v in helpers.get_default_args(self.model).items() if k not in gen_kwargs} log.debug(f'LLM: defaults={defaults}') - with devices.inference_context(): + with devices.llm_context(): generation = self.model.generate( **inputs, **gen_kwargs, @@ -899,7 +899,7 @@ class VQA: defaults = {k: v for k, v in helpers.get_default_args(self.model).items() if k not in gen_kwargs} log.debug(f'LLM: defaults={defaults}') - with devices.inference_context(): + with devices.llm_context(): generation = self.model.generate(**inputs, **gen_kwargs) generation = generation[0][input_len:] response = self.processor.decode(generation, skip_special_tokens=True) @@ -932,7 +932,7 @@ class VQA: question = question.replace('<', '').replace('>', '').replace('_', ' ') model_inputs = self.processor(text=question, images=image, return_tensors="pt").to(devices.device, devices.dtype) input_len = model_inputs["input_ids"].shape[-1] - with devices.inference_context(): + with devices.llm_context(): generation = self.model.generate( **model_inputs, **get_kwargs(self.model), @@ -989,7 +989,7 @@ class VQA: if pixel_values is not None: pixel_values = pixel_values.to(dtype=visual_tokenizer.dtype, device=visual_tokenizer.device) pixel_values = [pixel_values] - with devices.inference_context(): + with devices.llm_context(): output_ids = self.model.generate( input_ids, pixel_values=pixel_values, @@ -1091,7 +1091,7 @@ class VQA: defaults = {k: v for k, v in helpers.get_default_args(self.model).items() if k not in gen_kwargs} log.debug(f'LLM: defaults={defaults}') - with devices.inference_context(): + with devices.llm_context(): output_ids = self.model.generate( **inputs, **gen_kwargs, @@ -1135,7 +1135,7 @@ class VQA: input_ids = [self.processor.tokenizer.cls_token_id] + input_ids input_ids = torch.tensor(input_ids).unsqueeze(0) git_dict['input_ids'] = input_ids.to(devices.device) - with devices.inference_context(): + with devices.llm_context(): generated_ids = self.model.generate(**git_dict) response = self.processor.batch_decode(generated_ids, skip_special_tokens=True)[0] return response @@ -1164,7 +1164,7 @@ class VQA: move_aux_to_gpu('vqa') inputs = self.processor(image, question, return_tensors="pt") inputs = inputs.to(devices.device, devices.dtype) - with devices.inference_context(): + with devices.llm_context(): outputs = self.model.generate(**inputs) response = self.processor.decode(outputs[0], skip_special_tokens=True) return response @@ -1193,7 +1193,7 @@ class VQA: move_aux_to_gpu('vqa') inputs = self.processor(image, question, return_tensors="pt") inputs = inputs.to(devices.device) - with devices.inference_context(): + with devices.llm_context(): outputs = self.model(**inputs) logits = outputs.logits idx = logits.argmax(-1).item() @@ -1227,7 +1227,7 @@ class VQA: else: inputs = self.processor(images=image, return_tensors="pt") inputs = {k: v.to(devices.device, devices.dtype) if v.is_floating_point() else v.to(devices.device) for k, v in inputs.items()} - with devices.inference_context(): + with devices.llm_context(): outputs = self.model.generate(**inputs) response = self.processor.decode(outputs[0], skip_special_tokens=True) return response @@ -1258,7 +1258,7 @@ class VQA: self._load_moondream(repo) move_aux_to_gpu('vqa') question = question.replace('<', '').replace('>', '').replace('_', ' ') - with devices.inference_context(): + with devices.llm_context(): if question == 'CAPTION': response = self.model.caption(image, length="short")['caption'] elif question == 'DETAILED CAPTION': @@ -1379,7 +1379,7 @@ class VQA: gen_kwargs['decoder_start_token_id'] = bos_token_id debug(f'LLM: handler=florence setting decoder_start_token_id={bos_token_id}') debug(f'LLM: handler=florence generation_kwargs={gen_kwargs}') - with devices.inference_context(), devices.bypass_sdpa_hijacks(): + with devices.llm_context(): generated_ids = self.model.generate( input_ids=input_ids, pixel_values=pixel_values, @@ -1431,7 +1431,7 @@ class VQA: 'mask_prompts': None, 'tokenizer': self.processor, } - with devices.inference_context(): + with devices.llm_context(): return_dict = self.model.predict_forward(**input_dict) response = return_dict["prediction"] # the text format answer return response diff --git a/modules/devices.py b/modules/devices.py index 20593bdcc..34e96f700 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -765,6 +765,15 @@ def bypass_sdpa_hijacks(): log.debug('SDPA bypass: restored hijacked attention') +@contextlib.contextmanager +def llm_context(): + """ + Combined context manager that applies both inference_context and bypass_sdpa_hijacks. + """ + with inference_context(), bypass_sdpa_hijacks(): + yield + + def torch_reset() -> bool: """ Resets PyTorch execution graph, flushes VRAM caches, and syncs streams. diff --git a/modules/model_quant.py b/modules/model_quant.py index e4e528ec4..c9733ed56 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -94,7 +94,14 @@ def get_sdnq_devices(mode="pre"): return quantization_device, return_device -def create_sdnq_config(kwargs = None, allow: bool = True, module: str = 'Model', weights_dtype: str | None = None, quantized_matmul_dtype: str | None = None, modules_to_not_convert: list | None = None, modules_dtype_dict: dict | None = None): +def create_sdnq_config(kwargs = None, + allow: bool = True, + module: str = 'Model', + weights_dtype: str | None = None, + quantized_matmul_dtype: str | None = None, + modules_to_not_convert: list | None = None, + modules_dtype_dict: dict | None = None, + ): from modules import shared if allow and (shared.opts.sdnq_quantize_mode in {'pre', 'auto'}) and (module == 'any' or module in shared.opts.sdnq_quantize_weights): from modules.sdnq import SDNQConfig @@ -216,7 +223,12 @@ def create_config(kwargs = None, allow: bool = True, module: str = 'Model', modu kwargs = {} if module == 'Model' and dont_quant(): return kwargs - kwargs = create_sdnq_config(kwargs, allow=allow, module=module, modules_to_not_convert=modules_to_not_convert, modules_dtype_dict=modules_dtype_dict) + kwargs = create_sdnq_config(kwargs, + allow=allow, + module=module, + modules_to_not_convert=modules_to_not_convert, + modules_dtype_dict=modules_dtype_dict + ) if kwargs is not None and 'quantization_config' in kwargs: if debug: log.trace(f'Quantization: type=sdnq config={kwargs.get("quantization_config", None)}') diff --git a/pipelines/sefi/pipeline_sefi.py b/pipelines/sefi/pipeline_sefi.py index 3dc18d1cc..1711981ce 100644 --- a/pipelines/sefi/pipeline_sefi.py +++ b/pipelines/sefi/pipeline_sefi.py @@ -252,7 +252,7 @@ class SeFiPipeline(DiffusionPipeline): """ x_list = [] for data, pos in zip(x, x_ids): - _, ch = data.shape # noqa: F841 + _, ch = data.shape h_ids = pos[:, 1].to(torch.int64) w_ids = pos[:, 2].to(torch.int64) diff --git a/requirements.txt b/requirements.txt index bfd38ad96..9e72d22cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,8 +30,8 @@ requests==2.34.2 tqdm==4.68.3 accelerate==1.14.0 einops==0.8.2 -huggingface_hub==1.25.1 -hf_xet==1.5.2 +huggingface_hub==1.26.0 +hf_xet==1.6.0 numpy==2.1.2 pandas==2.3.1 protobuf==7.35.1 @@ -44,6 +44,7 @@ typing-extensions==4.15.0 sentencepiece==0.2.1 # lint +ty ruff pylint pre-commit diff --git a/scripts/prompt_enhance_ext.py b/scripts/prompt_enhance_ext.py index 248efa4ca..14d801359 100644 --- a/scripts/prompt_enhance_ext.py +++ b/scripts/prompt_enhance_ext.py @@ -80,7 +80,7 @@ class PromptEnhanceScript(scripts_manager.Script): gguf_args['model_type'] = model_type gguf_args['gguf_file'] = model_file - quant_args = model_quant.create_config(module='LLM') if not gguf_args else {} + quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d', 'linear_attn.conv1d']) if not gguf_args else {} try: t0 = time.time() @@ -127,6 +127,7 @@ class PromptEnhanceScript(scripts_manager.Script): ) finally: sd_models.set_huggingface_options(quiet=True) + self.llm.eval() register_aux('prompt_enhance', self.llm) tokenizer_args = { 'pretrained_model_name_or_path': model_repo } @@ -489,7 +490,7 @@ class PromptEnhanceScript(scripts_manager.Script): return prompt_text # Return original text part on error try: - with devices.inference_context(): + with devices.llm_context(): move_aux_to_gpu('prompt_enhance') gen_kwargs = { 'do_sample': sample, @@ -535,8 +536,7 @@ class PromptEnhanceScript(scripts_manager.Script): except Exception as e: outputs = None log.error(f'Prompt enhance generate: {e}') - if debug_enabled: - errors.display(e, 'Prompt enhance') + errors.display(e, 'Prompt enhance') self.busy = False response = f'Error: {str(e)}' finally: