diff --git a/CHANGELOG.md b/CHANGELOG.md index 537a566ee..c08b89aba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Change Log for SD.Next +## Update for 2026-08-08 + +- **Detailer** support for VL models in addition to standard YOLO models + select any of Qwen3-VL models in model selection + enter the human-readable ask as detailer instructions + for example: *glasses, optional hat, left hand, largest tree in the background* + *note*: VL models are much larger, so use with caution + *note*: LLM processes data sequentially, so if it cannot find one item, it may skip the rest of the items in the prompt + ## Update for 2026-08-07 ### Highlights for 2026-08-07 diff --git a/modules/detailer/detailer.py b/modules/detailer/detailer.py index 7470e00e1..23225448e 100644 --- a/modules/detailer/detailer.py +++ b/modules/detailer/detailer.py @@ -38,7 +38,10 @@ class Detailer(): ) -> list[DetailerResult]: if 'LocateAnything' in name: from modules.detailer import locateanything - return locateanything.predict(self, model, image, device=device, mask=mask, offload=offload, p=p) + return locateanything.predict(self, name, image, device=device, mask=mask, offload=offload, p=p) + if 'Qwen3-VL' in name: + from modules.detailer import qwen + return qwen.predict(self, name, image, device=device, mask=mask, offload=offload, p=p) from modules.detailer import yolo return yolo.predict(self, model, image, imgsz=imgsz, half=half, device=device, agnostic=agnostic, retina=retina, mask=mask, augment=augment, offload=offload, p=p) @@ -50,7 +53,10 @@ class Detailer(): def load(self, model_name: str | None = None): if 'LocateAnything' in model_name: from modules.detailer import locateanything - return locateanything.load(model_name=model_name) + return locateanything.load(self, model_name=model_name) + if 'Qwen3-VL' in model_name: + from modules.detailer import qwen + return qwen.load(self, model_name=model_name) from modules.detailer import yolo return yolo.load(self, model_name=model_name) @@ -356,7 +362,7 @@ class Detailer(): return gr.update(visible=False), gr.update(visible=True, value=value), gr.update(visible=False) def ui(self, tab: str): - def ui_settings_change(merge, detailers, text, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps, renoise_value, renoise_end, resolution, save, sort, seg): + def ui_settings_change(merge, detailers, text, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps, renoise_value, renoise_end, resolution, save, sort, seg): # pylint: disable=unused-argument shared.opts.detailer_merge = merge shared.opts.detailer_models = detailers shared.opts.detailer_args = text if not self.ui_mode else '' @@ -375,7 +381,7 @@ class Detailer(): shared.opts.detailer_segmentation = seg # shared.opts.detailer_resolution = resolution shared.opts.save(silent=True) - log.debug(f'Detailer settings: models={detailers} classes={classes} strength={strength} conf={min_confidence} max={max_detected} iou={iou} size={min_size}-{max_size} padding={padding} steps={steps} resolution={resolution} save={save} sort={sort} seg={seg}') + # log.debug(f'Detailer settings: models={detailers} classes={classes} strength={strength} conf={min_confidence} max={max_detected} iou={iou} size={min_size}-{max_size} padding={padding} steps={steps} resolution={resolution} save={save} sort={sort} seg={seg}') if not self.ui_mode: log.debug(f'Detailer expert: {text}') @@ -395,7 +401,7 @@ class Detailer(): ui_mode = ui_components.ToolButton(value=ui_symbols.view, elem_id=f'{tab}_yolo_models_list') ui_mode.click(fn=self.change_mode, inputs=[detailers, detailers_text], outputs=[detailers, detailers_text, refresh_btn]) with gr.Row(): - classes = gr.Textbox(label="Detailer classes", placeholder="Classes", elem_id=f"{tab}_detailer_classes") + classes = gr.Textbox(label="Detailer classes or instructions", placeholder="List of classes or human instructions", elem_id=f"{tab}_detailer_classes") if tab == 'extras': # Process tab is standalone, there is no base prompt to fall back to prompt_placeholder = 'detailer prompt, leave empty for none' negative_placeholder = 'detailer negative prompt, leave empty for none' diff --git a/modules/detailer/helper.py b/modules/detailer/helper.py index 70bfae236..8ac00c7dd 100644 --- a/modules/detailer/helper.py +++ b/modules/detailer/helper.py @@ -44,9 +44,9 @@ class DetailerResult: self.box = box self.mask = mask self.item = item - self.width = width - self.height = height + self.width = width if width > 0 else box[2] - box[0] + self.height = height if height > 0 else box[3] - box[1] self.args = args def __str__(self): - return f'DetailerResult(cls={self.cls} label={self.label} score={self.score:.2f} box={self.box} size={self.width}x{self.height} args={self.args})' + return f'DetailerResult(cls={self.cls} label="{self.label}" score={self.score:.2f} box={self.box} size={self.width}x{self.height} args={self.args})' diff --git a/modules/detailer/locateanything.py b/modules/detailer/locateanything.py index 6fb911197..70090436e 100644 --- a/modules/detailer/locateanything.py +++ b/modules/detailer/locateanything.py @@ -14,7 +14,7 @@ def dependencies(): install('decord') -def load(model_name: str | None = None): +def load(self, model_name: str | None = None): import transformers global tokenizer, processor # pylint: disable=global-statement load_kwargs = { @@ -30,6 +30,7 @@ def load(model_name: str | None = None): if shared.opts.detailer_unload: model.to(devices.cpu) log.info(f'Detailer model="{model_name}" cls={model.__class__.__name__} loaded') + self.models[model_name] = model return model_name, model @@ -44,16 +45,16 @@ def predict( ) -> list[DetailerResult]: if offload is None: offload = shared.opts.detailer_unload - log.info(f'Detailer cls="{model.__class__.__name__}" image={image} device={device} mask={mask} offload={offload}') result = [] if isinstance(model, str): cached = self.models.get(model, None) if cached is None: - _, model = self.load(model) + _, model = load(self, model) else: model = cached if model is None: return result + log.info(f'Detailer cls="{model.__class__.__name__}" image={image} device={device} mask={mask} offload={offload}') model = model.to(device) prompt = detailer_opt(p, 'detailer_classes') or '' log.debug(f'Detailer prompt="{prompt}"') diff --git a/modules/detailer/models.py b/modules/detailer/models.py index 678ee77ee..3237f15c3 100644 --- a/modules/detailer/models.py +++ b/modules/detailer/models.py @@ -9,5 +9,8 @@ detailer_models = [ # list[dict]: + def confidence() -> str: + conf = max(0.0, min(1.0, float(min_confidence))) + if conf >= 0.85: + instruction = "Detect only obvious, fully visible target objects." + elif conf >= 0.65: + instruction = "Detect clear targets, ignoring faint or ambiguous cases." + elif conf >= 0.45: + instruction = "Detect all distinct targets, including partially covered ones." + elif conf >= 0.25: + instruction = "Detect candidate targets, including small, blurry, or occluded ones." + else: + instruction = "Detect all possible target candidates, background objects, or fragments." + return instruction + + instructions = ( + "You are an expert vision assistant for object detection.\n" + f"INSTRUCTIONS:\n" + f"1. Scan the image carefully for EACH requested class.\n" + f"2. Add a BoundingBoxItem to 'objects' for EVERY instance found.\n" + f"3. In the 'response' field, explicitly list which target classes were found and which were missing.\n" + f"4. {confidence()}\n" + f"5. You MUST respond strictly with a valid JSON object matching this schema: \n```json\n{schema}\n```\n\n" + "6. Do not include any Markdown text outside of the JSON string. Add any text explanations or clarifications inside the 'response' field of the JSON object.\n" + "7. Bounding box coordinates must be in 0-1000 normalized format: [ymin, xmin, ymax, xmax]." + ) + return [ + { "role": "system", "content": instructions }, + { "role": "user", + "content": [ + {"type": "image"}, # injected later using processor + {"type": "text", "text": prompt}, + ], + }, + ] + + +def load(self, model_name: str | None = None) -> tuple[str, transformers.Qwen3VLForConditionalGeneration]: # pylint: disable=unused-argument + cached = sd_offload_aux.get_aux_model(model_name) + if cached is not None: + return model_name, cached + repo_id = 'Qwen/' + model_name if not model_name.startswith('Qwen/') else model_name + load_kwargs = { + 'pretrained_model_name_or_path': repo_id, + 'cache_dir': shared.opts.hfcache_dir, + 'torch_dtype': devices.dtype, + } + quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d', 'linear_attn.conv1d']) + model = transformers.Qwen3VLForConditionalGeneration.from_pretrained(**load_kwargs, **quant_args, attn_implementation="sdpa") + model = model.eval() + model.processor: transformers.Qwen3VLProcessor = transformers.Qwen3VLProcessor.from_pretrained(**load_kwargs) + sd_offload_aux.register_aux(model_name, model) + if shared.opts.detailer_unload: + sd_offload_aux.offload_aux(model_name) + log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"') + return model_name, model + + +def parse(data: str, image: Image.Image) -> tuple[str, list[DetailerResult]]: + results = [] + response = '' + w, h = image.size + try: + clean = data.strip() + if clean.startswith("```json"): + clean = clean[7:] + if clean.endswith("```"): + clean = clean[:-3] + parsed = json.loads(clean.strip()) + response = parsed.get("response", "") + objects = parsed.get("objects", []) + for item in objects: + box_2d = item.get("box_2d", []) + label = item.get("label", "") + confidence = float(item.get("confidence", 1.0)) + if len(box_2d) == 4: + xmin, ymin, xmax, ymax = map(int, box_2d) + xmin, ymin = int((xmin / 1000.0) * w), int((ymin / 1000.0) * h) + xmax, ymax = int((xmax / 1000.0) * w), int((ymax / 1000.0) * h) + box = (xmin, ymin, xmax, ymax) + mask = Image.new('L', image.size, 0) + draw_mask = ImageDraw.Draw(mask) + draw_mask.rectangle(box, fill="white", outline=None, width=0) + result = DetailerResult(box=box, label=label, score=confidence, cls=-1, mask=mask) + log.trace(f'Detailer box: {result}') + results.append(result) + except Exception as err: + log.error(f'Detailer: failed to parse object detection output: {err}') + log.error(f'Detailer: raw output: {data}') + return response, results + + +def predict( + self, + name: str, + image: Image.Image, + device = devices.device, + mask: bool = True, + offload: bool | None = None, + p = None, +) -> list[DetailerResult]: + if offload is None: + offload = shared.opts.detailer_unload + if image is None: + return [] + cached = sd_offload_aux.get_aux_model(name) + if cached is None: + name, model = load(self, name) + else: + model = cached + if model is None: + return [] + + prompt = detailer_opt(p, 'detailer_classes') or '' + if not prompt: + prompt = 'Detect and locate all objects' + + log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{prompt}" image={image.size} device={device} mask={mask} offload={offload}') + sd_offload_aux.move_aux_to_gpu(name) + + t0 = time.time() + schema = json.dumps(ObjectDetectionOutput.model_json_schema(), indent=2) + messages = template(prompt=prompt, schema=schema, min_confidence=shared.opts.detailer_conf) + text = model.processor.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + ) + inputs = model.processor( + text=[text], + images=[image], + padding=True, + return_tensors="pt", + min_pixels=128 * 28 * 28, + max_pixels=1280 * 28 * 28, # Force 1 MP cap + ) + inputs.pop("token_type_ids", None) + inputs = inputs.to(model.device) + eos_id = model.processor.tokenizer.convert_tokens_to_ids("<|im_end|>") + if eos_id is None or isinstance(eos_id, list): + eos_id = model.processor.tokenizer.eos_token_id + pad_id = model.processor.tokenizer.pad_token_id if model.processor.tokenizer.pad_token_id is not None else eos_id + + with devices.llm_context(): + generated_ids = model.generate( + **inputs, + max_new_tokens=2048, + do_sample=False, # Deterministic decoding keeps bbox integer tokens strict + temperature=None, # Forces argmax token selection + repetition_penalty=1.01, # Breaks coordinate repetition loops without distorting valid coordinates + no_repeat_ngram_size=0, # MUST be 0/None—setting this > 0 corrupts valid repeated bbox coordinates + eos_token_id=eos_id, # Prevent premature EOS token stopping + pad_token_id=pad_id, # Prevent premature EOS token stopping + ) + + prompt_len = inputs["input_ids"].shape[1] + output_tokens = generated_ids[0][prompt_len:] + output_text = model.processor.tokenizer.decode(output_tokens, skip_special_tokens=True) + t1 = time.time() + response, results = parse(output_text, image) + + log.debug(f'Detailer: name="{name}" tokens={output_tokens.shape[0]} response="{response}" items={len(results)} time={t1-t0:.3f}') + + sd_offload_aux.offload_aux(name) + return results diff --git a/modules/sd_offload_aux.py b/modules/sd_offload_aux.py index e8cf54d50..32e0c0d68 100644 --- a/modules/sd_offload_aux.py +++ b/modules/sd_offload_aux.py @@ -91,3 +91,10 @@ def offload_aux(name: str) -> None: if hasattr(entry.model, 'device') and devices.same_device(entry.model.device, devices.cpu): return _do_move_to_cpu(entry.model, f'post:{name}', entry.size) + + +def get_aux_model(name: str) -> torch.nn.Module | None: + entry = aux_models.get(name) + if entry is None: + return None + return entry.model diff --git a/ui/locale/locale_en.json b/ui/locale/locale_en.json index 22f9d0b83..91537762d 100644 --- a/ui/locale/locale_en.json +++ b/ui/locale/locale_en.json @@ -366,7 +366,7 @@ {"id":"","label":"dynamic","localized":"","hint":"Dynamic shifting automatically adjusts the denoising schedule based on your image resolution.

The scheduler interpolates between base_shift and max_shift based on actual image resolution.

Enabling disables static Flow shift.","ui":"txt2img"}, {"id":"","label":"Detailer models","localized":"","hint":"YOLO detection models used to find regions to re-render. Multiple models can be selected and they run in sequence.
Models live in models/yolo. Filename hints at target: face-* detects faces, eyes-* detects eyes, hand-* detects hands, person-* detects whole subjects, and so on.
Models with -seg in the name produce a precise segmentation outline (used when Use segmentation is on); the rest produce only bounding boxes.

Per-model overrides can be appended with colon syntax, for example face-yolo8n:conf=0.5:strength=0.4.","ui":"txt2img"}, {"id":"","label":"Detailer list","localized":"","hint":"","ui":"txt2img"}, - {"id":"","label":"Detailer classes","localized":"","hint":"Comma-separated list of class names to keep when the selected detailer model is multi-class (e.g., a YOLO model that detects faces, eyes, and hands all in one file).
Only detections matching these labels are processed; everything else is dropped. Leave empty to accept all classes.

Names must match the model's class names exactly (case-insensitive). Single-class models like a face-only detector ignore this field.","ui":"txt2img"}, + {"id":"","label":"Detailer classes or instructions","localized":"","hint":"When using standard single-mode model, this field is ignored
When using multi-class model such as YOLO, this field should include comma-separated list of class names to keep or leave blank to detect all known classes
When using VL model such as Qwen, this field should contain human readable instructions on what to detect","ui":"txt2img"}, {"id":"","label":"Detailer prompt","localized":"","hint":"Optional dedicated prompt for the detailer pass.
Leave empty to inherit the main prompt. Useful for steering the inpaint differently from the rest of the image: a face detailer can use just portrait, sharp eyes, detailed skin while the main prompt covers the full scene.

The placeholder [PROMPT] (or [prompt]) is replaced with the original main prompt, so you can append to it: [PROMPT], detailed face.","ui":"txt2img"}, {"id":"","label":"Detailer negative prompt","localized":"","hint":"Optional dedicated negative prompt for the detailer pass.
Leave empty to inherit the main negative prompt. Same [PROMPT] / [prompt] placeholder behavior as the positive detailer prompt: it expands to the original main negative prompt.","ui":"txt2img"}, {"id":"","label":"Detailer steps","localized":"","hint":"Number of sampling steps used for each detailer inpaint pass.
Independent of the main generation steps. Higher values give cleaner detail but cost more time per detected region.

Set to 0 to inherit the main generation step count.
Default 10.","ui":"txt2img"},