detailer.next

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -1,14 +1,21 @@
 # Change Log for SD.Next
 
-## Update for 2026-08-08
+## Update for 2026-08-09
 
-- **Detailer** support for VL models in addition to standard YOLO models  
-  select any of Qwen3-VL models in model selection  
-  enter the human-readable ask as detailer instructions  
+- **Detailer**: Pretty much *detailer.next* :)  
+  Detailer detection models were traditionally *YOLO* models, but now we also use:  
+  - [Facebook-SAM3](https://huggingface.co/facebook/sam3) hybrid promptable concept segmentation and detection network
+  - [Qwen3-VL](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct) vision-language autoregressive foundation models, in *2B, 4B, and 8B* variants
+  - [Florence-2](https://huggingface.co/microsoft/Florence-2-large) lightweight multi-task vision sequence-to-sequence models, in *base and large* variants
+  - [Grounding-DINO](https://huggingface.co/IDEA-Research/grounding-dino-base) open-vocabulary object detection models, in *tiny and base* variants
+
+  select any of the above models in the detailer model selection dropdown  
+  and enter your human-readable target descriptions as *detailer instructions*  
   for example: *glasses, optional hat, left hand, largest tree in the background*  
-  *note*: VL models are much larger, so use with caution  
-  *note*: LLM processes data sequentially, so if it cannot find one item, it may skip the rest of the items in the prompt  
-
+  *note*: VL models are much larger, so use them with caution  
+  *note*: LLMs process data sequentially, meaning if a model cannot find one specific item, it may skip subsequent items in the prompt  
+  *note*: SAM3 is a [gated model](https://vladmandic.github.io/sdnext-docs/Gated/)  
+  
 ## Update for 2026-08-07
 
 ### Highlights for 2026-08-07

installer.py

@@ -478,8 +478,8 @@ def get_platform():
             release = platform.release()
         return {
             'arch': platform.machine(),
-            'cpu': f'{platform.processor()}',
-            'system': platform.system(),
+            'cpu': f'"{platform.processor()}"',
+            'system': f'"{platform.system()}"',
             'release': release,
             'python': platform.python_version(),
             'locale': locale.getlocale(),

modules/api/middleware.py

@@ -144,7 +144,7 @@ def setup_middleware(app: FastAPI, cmd_opts):
 
         if not isinstance(e, HTTPException) and err['error'] != 'TypeError': # do not print backtrace on known httpexceptions
             errors.display(e, 'HTTP API', [anyio, fastapi, uvicorn, starlette])
-        elif err['code'] in [404, 401, 400]:
+        elif err['code'] in [404, 401, 400, 403]:
             pass
         else:
             log.debug(e, exc_info=True) # print stack trace

modules/api/process.py

@@ -190,6 +190,7 @@ class APIProcess:
                 strength=req.detailer_strength if req.detailer_strength is not None else 0.3,
                 resolution=req.detailer_resolution if req.detailer_resolution is not None else 1024,
                 seed=req.seed if req.seed is not None else -1,
+                classes=req.detailer_classes if req.detailer_classes is not None else None,
                 overrides=overrides,
             )
 

modules/control/run.py

@@ -346,7 +346,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
                 guidance_name: str = 'Default', guidance_scale: float = 6.0, guidance_rescale: float = 0.0, guidance_start: float = 0.0, guidance_stop: float = 1.0,
                 cfg_scale: float = 6.0, clip_skip: float = 1.0, cfg_image: float = 6.0, cfg_rescale: float = 0.7, cfg_true: float = 0.0, cfg_adaptive: float = 0.5, cfg_end: float = 1.0,
                 vae_type: str = 'Full', tiling: bool = False, hidiffusion: bool = False,
-                detailer_enabled: bool = False, detailer_prompt: str = '', detailer_negative: str = '', detailer_steps: int = 10, detailer_strength: float = 0.3, detailer_resolution: int = 1024,
+                detailer_enabled: bool = False, detailer_prompt: str = '', detailer_negative: str = '', detailer_steps: int = 10, detailer_strength: float = 0.3, detailer_resolution: int = 1024,  detailer_classes: str = '',
                 hdr_mode: int = 0, hdr_brightness: float = 0, hdr_color: float = 0, hdr_sharpen: float = 0, hdr_clamp: bool = False, hdr_boundary: float = 4.0, hdr_threshold: float = 0.95,
                 hdr_maximize: bool = False, hdr_max_center: float = 0.6, hdr_max_boundary: float = 1.0, hdr_color_picker: str | None = None, hdr_tint_ratio: float = 0, hdr_apply_hires: bool = True,
                 grading_brightness: float = 0.0, grading_contrast: float = 0.0, grading_saturation: float = 0.0, grading_hue: float = 0.0,
@@ -368,7 +368,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
                 override_script_name: str | None = None, override_script_args = None, extra: dict | None = None,
                 *input_script_args,
                 # API-only params (keyword-only, not wired to Gradio)
-                detailer_segmentation: bool | None = None, detailer_include_detections: bool | None = None, detailer_merge: bool | None = None, detailer_sort: bool | None = None, detailer_classes: str | None = None,
+                detailer_segmentation: bool | None = None, detailer_include_detections: bool | None = None, detailer_merge: bool | None = None, detailer_sort: bool | None = None,
                 detailer_conf: float | None = None, detailer_iou: float | None = None, detailer_max: int | None = None,
                 detailer_min_size: float | None = None, detailer_max_size: float | None = None,
                 detailer_blur: int | None = None, detailer_padding: int | None = None,

modules/detailer/__init__.py

@@ -1,5 +1,5 @@
 from .models import detailer_models
-from .helper import detailer_opt, DetailerResult, list_models
+from .helper import DetailerResult, detailer_opt, list_models, get_mask
 from .detailer import Detailer
 
 

modules/detailer/detailer.py

@@ -36,34 +36,64 @@ class Detailer():
             offload: bool | None = None,
             p = None,
         ) -> list[DetailerResult]:
+        jobid = shared.state.begin('Detect')
         if 'LocateAnything' in name:
             from modules.detailer import locateanything
-            return locateanything.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
-        if 'Qwen3-VL' in name:
+            results = locateanything.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
+        elif 'Qwen3-VL' in name:
             from modules.detailer import qwen
-            return qwen.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
-
-        from modules.detailer import yolo
-        return yolo.predict(self, model, image, imgsz=imgsz, half=half, device=device, agnostic=agnostic, retina=retina, mask=mask, augment=augment, offload=offload, p=p)
+            results = qwen.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
+        elif 'Florence-2' in name:
+            from modules.detailer import florence
+            results = florence.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
+        elif 'Grounding-DINO' in name:
+            from modules.detailer import dino
+            results = dino.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
+        elif 'Rex-Omni' in name:
+            from modules.detailer import rexomni
+            results = rexomni.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
+        elif 'Facebook-SAM3' in name:
+            from modules.detailer import sam
+            results = sam.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
+        else:
+            from modules.detailer import yolo
+            results = yolo.predict(self, model, image, imgsz=imgsz, half=half, device=device, agnostic=agnostic, retina=retina, mask=mask, augment=augment, offload=offload, p=p)
+        shared.state.end(jobid)
+        return results
 
     def enumerate(self):
         from modules.detailer import list_models
         return list_models(self)
 
     def load(self, model_name: str | None = None):
+        jobid = shared.state.begin('Load detailer')
         if 'LocateAnything' in model_name:
             from modules.detailer import locateanything
-            return locateanything.load(self, model_name=model_name)
-        if 'Qwen3-VL' in model_name:
+            model_name, model = locateanything.load(self, model_name=model_name)
+        elif 'Qwen3-VL' in model_name:
             from modules.detailer import qwen
-            return qwen.load(self, model_name=model_name)
-
-        from modules.detailer import yolo
-        return yolo.load(self, model_name=model_name)
+            model_name, model = qwen.load(self, model_name=model_name)
+        elif 'Florence-2' in model_name:
+            from modules.detailer import florence
+            model_name, model = florence.load(self, model_name=model_name)
+        elif 'Grounding-DINO' in model_name:
+            from modules.detailer import dino
+            model_name, model = dino.load(self, model_name=model_name)
+        elif 'Rex-Omni' in model_name:
+            from modules.detailer import rexomni
+            model_name, model = rexomni.load(self, model_name=model_name)
+        elif 'Facebook-SAM3' in model_name:
+            from modules.detailer import sam
+            model_name, model = sam.load(self, model_name=model_name)
+        else:
+            from modules.detailer import yolo
+            model_name, model = yolo.load(self, model_name=model_name)
+        shared.state.end(jobid)
+        return model_name, model
 
     def merge(self, items: list[DetailerResult]) -> list[DetailerResult]:
         if items is None or len(items) == 0:
-            return None
+            return []
         box=[min(item.box[0] for item in items), min(item.box[1] for item in items), max(item.box[2] for item in items), max(item.box[3] for item in items)]
         mask = Image.new('L', items[0].mask.size, 0)
         for item in items:
@@ -80,6 +110,30 @@ class Detailer():
         )
         return [merged]
 
+    def filter(self, items: list[DetailerResult], image: Image.Image, p: processing.StableDiffusionProcessing = None) -> list[DetailerResult]:
+        if items is None or len(items) == 0:
+            return []
+        if p is not None:
+            min_conf = detailer_opt(p, 'detailer_conf')
+            max_detected = detailer_opt(p, 'detailer_max')
+            filtered = [item for item in items if item.score >= min_conf]
+            opt_min = detailer_opt(p, 'detailer_min_size') or 0
+            opt_max = detailer_opt(p, 'detailer_max_size') or 1
+            for item in filtered.copy():
+                w, h = item.box[2] - item.box[0], item.box[3] - item.box[1]
+                x_size, y_size = w/image.width, h/image.height
+                min_size = opt_min if 0 <= opt_min <= 1 else 0
+                max_size = opt_max if 0 < opt_max <= 1 else 1
+                if not ((x_size >= min_size) and (y_size >= min_size) and (x_size <= max_size) and (y_size <= max_size)):
+                    filtered.remove(item)
+            filtered = sorted(filtered, key=lambda x: x.score, reverse=True)
+            filtered = filtered[:max_detected]
+        else:
+            filtered = items
+        if len(filtered) != len(items):
+            log.debug(f'Detailer: items={len(items)} filtered={len(filtered)}')
+        return filtered
+
     def draw_masks(self, image: Image.Image, items: list[DetailerResult], p=None) -> Image.Image:
         if not isinstance(image, Image.Image):
             image = Image.fromarray(image)
@@ -87,7 +141,7 @@ class Detailer():
         size = min(image.width, image.height) // 32
         font = images.get_font(size)
         color = (0, 190, 190)
-        log.debug(f'Detailer: draw={items}')
+        # log.debug(f'Detailer: draw={items}')
         for i, item in enumerate(items):
             if detailer_opt(p, 'detailer_segmentation') and item.mask is not None:
                 mask = item.mask.convert('L')
@@ -157,6 +211,7 @@ class Detailer():
             if image is None:
                 image = Image.fromarray(np_image)
             items = self.predict(name, model, image, p=p)
+            items = self.filter(items, image, p=p)
 
             if len(items) == 0:
                 log.info(f'Detailer: model="{name}" no items detected')
@@ -305,7 +360,7 @@ class Detailer():
             np_images.append(annotated) # save debug image with boxes
         return np_images
 
-    def make_processing(self, image, prompt='', negative='', steps=10, strength=0.3, resolution=1024, seed=-1, overrides=None):
+    def make_processing(self, image, prompt='', negative='', steps=10, strength=0.3, resolution=1024, seed=-1, overrides=None, classes=''):
         """Build a synthetic Img2Img processing object to run restore() standalone, with no base generation pass.
 
         The primary params map to the detailer_* fields restore() reads directly. overrides is an optional
@@ -337,6 +392,7 @@ class Detailer():
             detailer_steps=steps,
             detailer_strength=strength,
             detailer_resolution=resolution,
+            detailer_classes=classes,
         )
         for attr, val in (overrides or {}).items():
             if val is not None:
@@ -366,7 +422,7 @@ class Detailer():
             shared.opts.detailer_merge = merge
             shared.opts.detailer_models = detailers
             shared.opts.detailer_args = text if not self.ui_mode else ''
-            shared.opts.detailer_classes = classes
+            # shared.opts.detailer_classes = classes
             shared.opts.detailer_padding = padding
             shared.opts.detailer_blur = blur
             shared.opts.detailer_conf = min_confidence
@@ -466,5 +522,5 @@ class Detailer():
             sort.change(fn=ui_settings_change, inputs=[merge, detailers, detailers_text, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps, renoise_value, renoise_end, resolution, save, sort, seg], outputs=[])
             seg.change(fn=ui_settings_change, inputs=[merge, detailers, detailers_text, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps, renoise_value, renoise_end, resolution, save, sort, seg], outputs=[])
             if tab == 'extras':
-                return enabled, prompt, negative, steps, strength, resolution, sampler_block
-            return enabled, prompt, negative, steps, strength, resolution
+                return enabled, prompt, negative, steps, strength, resolution, classes, sampler_block
+            return enabled, prompt, negative, steps, strength, resolution, classes

modules/detailer/dino.py

@@ -0,0 +1,124 @@
+import time
+import re
+import torch
+import transformers
+from PIL import Image
+from modules import shared, devices, sd_offload_aux, model_quant
+from modules.detailer import DetailerResult, detailer_opt, get_mask
+from modules.logger import log
+
+
+def format_grounding_dino_prompt(prompt: str) -> str:
+    # Formats user input into Grounding DINO query format. Grounding DINO requires lowercase text separated by periods and ending with a period.
+    clean_prompt = prompt.strip().lower()
+    if not clean_prompt or clean_prompt == "detect and locate all objects":
+        return "object."
+    items = [i.strip() for i in re.split(r'[,.]', clean_prompt) if i.strip()]
+    if not items:
+        return "object."
+    return ". ".join(items) + "."
+
+
+def load(self, model_name: str | None = None) -> tuple[str, transformers.AutoModelForZeroShotObjectDetection]: # pylint: disable=unused-argument
+    cached = sd_offload_aux.get_aux_model(model_name)
+    if cached is not None:
+        return model_name, cached
+    repo_id = 'IDEA-Research/' + model_name.lower() if '/' not in model_name else model_name
+    load_kwargs = {
+        'pretrained_model_name_or_path': repo_id,
+        'cache_dir': shared.opts.hfcache_dir,
+        'torch_dtype': devices.dtype,
+    }
+    quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d'])
+    model = transformers.AutoModelForZeroShotObjectDetection.from_pretrained(**load_kwargs, **quant_args)
+    model = model.eval()
+    model.processor = transformers.AutoProcessor.from_pretrained(**load_kwargs)
+
+    sd_offload_aux.register_aux(model_name, model)
+    if shared.opts.detailer_unload:
+        sd_offload_aux.offload_aux(model_name)
+
+    log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"')
+    return model_name, model
+
+
+def parse(data: dict, image: Image.Image, include_mask: bool = True) -> tuple[str, list[DetailerResult]]:
+    results = []
+    w, h = image.size
+    boxes = data.get("boxes", [])
+    scores = data.get("scores", [])
+    labels = data.get("labels", [])
+    for box, score, label in zip(boxes, scores, labels):
+        if len(box) == 4:
+            if isinstance(box, torch.Tensor):
+                box = box.tolist()
+            xmin, ymin, xmax, ymax = map(int, box)
+            box = (max(0, xmin), max(0, ymin), min(w, xmax), min(h, ymax))
+            mask, cropped = get_mask(box, image, include_mask=include_mask)
+            result = DetailerResult(box=box,
+                                    label=str(label),
+                                    score=float(score),
+                                    cls=-1,
+                                    mask=mask,
+                                    item=cropped
+                                   )
+            # log.trace(f'Detailer box: {result}')
+            results.append(result)
+    response = f"Grounding DINO detected {len(results)} objects."
+    return response, results
+
+
+def predict(
+    self,
+    name: str,
+    image: Image.Image,
+    device = devices.device,
+    mask: bool = True,
+    offload: bool | None = None,
+    p = None,
+) -> list[DetailerResult]:
+    if offload is None:
+        offload = shared.opts.detailer_unload
+    if image is None:
+        return []
+
+    cached = sd_offload_aux.get_aux_model(name)
+    if cached is None:
+        name, model = load(self, name)
+    else:
+        model = cached
+    if model is None:
+        return []
+
+    prompt = detailer_opt(p, 'detailer_classes') or ''
+    text_input = format_grounding_dino_prompt(prompt)
+    threshold = detailer_opt(p, 'detailer_conf')
+
+    log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{text_input}" image={image.size} device={device} mask={mask} offload={offload} threshold={threshold}')
+    sd_offload_aux.move_aux_to_gpu(name)
+
+    t0 = time.time()
+    with devices.llm_context():
+        inputs = model.processor(
+            images=image,
+            text=text_input,
+            return_tensors="pt"
+        ).to(model.device, dtype=devices.dtype)
+        with torch.autocast(device_type=model.device.type, dtype=devices.dtype):
+            outputs = model(**inputs)
+
+    parsed_output = model.processor.post_process_grounded_object_detection(
+        outputs=outputs,
+        input_ids=inputs.input_ids,
+        threshold=threshold,
+        text_threshold=threshold,
+        target_sizes=[(image.height, image.width)]
+    )[0]
+
+    t1 = time.time()
+    response, results = parse(parsed_output, image, include_mask=mask)
+
+    log.debug(f'Detailer: name="{name}" response="{response}" items={len(results)} time={t1-t0:.3f}')
+
+    sd_offload_aux.offload_aux(name)
+    return results

modules/detailer/florence.py

@@ -0,0 +1,171 @@
+import time
+import json
+import transformers
+from PIL import Image
+from modules import shared, devices, sd_offload_aux, model_quant
+from modules.detailer import DetailerResult, detailer_opt, get_mask
+from modules.logger import log
+
+
+def select_florence_task(prompt: str) -> tuple[str, str]:
+    # Analyzes a user prompt and automatically determines whether to use <OPEN_VOCABULARY_DETECTION>, <CAPTION_TO_PHRASE_GROUNDING>, or default <OD>
+    clean_prompt = prompt.strip()
+    if not clean_prompt or clean_prompt == "Detect and locate all objects":
+        return "<OD>", "<OD>"
+    """
+    import re
+    items = [i.strip() for i in clean_prompt.split(",") if i.strip()]
+    is_class_list = len(items) > 1 or all(len(item.split()) <= 2 for item in items)
+    descriptive_keywords = re.search(r'\b(a|an|the|with|wearing|in|on|next to|holding|under|near)\b', clean_prompt, re.IGNORECASE)
+    if is_class_list and not descriptive_keywords:
+        task = "<OPEN_VOCABULARY_DETECTION>"
+        # formatted = f"{task}{', '.join(items)}"
+        formatted = f"{task}{'. '.join(items)}"
+    else:
+        task = "<CAPTION_TO_PHRASE_GROUNDING>"
+        formatted = f"{task}{clean_prompt}"
+    """
+    task = "<CAPTION_TO_PHRASE_GROUNDING>"
+    formatted = f"{task}{clean_prompt}"
+    return task, formatted
+
+
+def load(self, model_name: str | None = None) -> tuple[str, transformers.AutoModelForCausalLM]: # pylint: disable=unused-argument
+    cached = sd_offload_aux.get_aux_model(model_name)
+    if cached is not None:
+        return model_name, cached
+    repo_id = 'florence-community/' + model_name if '/' not in model_name else model_name
+
+    orig_get_imports = transformers.dynamic_module_utils.get_imports
+
+    def hijack_get_imports(f):
+        R = orig_get_imports(f)
+        if "flash_attn" in R:
+            R.remove("flash_attn")  # flash_attn is optional
+        return R
+
+    transformers.dynamic_module_utils.get_imports = hijack_get_imports
+    load_kwargs = {
+        'pretrained_model_name_or_path': repo_id,
+        'cache_dir': shared.opts.hfcache_dir,
+        'torch_dtype': devices.dtype,
+        'trust_remote_code': True,
+    }
+    quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d'])
+    model = transformers.Florence2ForConditionalGeneration.from_pretrained(
+        **load_kwargs,
+        **quant_args,
+        attn_implementation="sdpa"
+    )
+    model = model.eval()
+    model.processor = transformers.AutoProcessor.from_pretrained(**load_kwargs, max_pixels=1024*1024)
+
+    transformers.dynamic_module_utils.get_imports = orig_get_imports
+    sd_offload_aux.register_aux(model_name, model)
+    if shared.opts.detailer_unload:
+        sd_offload_aux.offload_aux(model_name)
+
+    log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"')
+    return model_name, model
+
+
+def parse(data: dict | str, image: Image.Image, include_mask: bool = True) -> tuple[str, list[DetailerResult]]:
+    results = []
+    response = ""
+    w, h = image.size
+    try:
+        parsed_data = {}
+        if isinstance(data, str):
+            clean = data.strip()
+            if clean.startswith("```json"):
+                clean = clean[7:]
+            if clean.endswith("```"):
+                clean = clean[:-3]
+            parsed_data = json.loads(clean.strip())
+        elif isinstance(data, dict):
+            parsed_data = data
+        detection_data = None
+        for key in ["<OPEN_VOCABULARY_DETECTION>", "<CAPTION_TO_PHRASE_GROUNDING>", "<OD>"]:
+            if key in parsed_data:
+                detection_data = parsed_data[key]
+                response = f"task '{key}' executed successfully."
+                break
+        if detection_data:
+            bboxes = detection_data.get("bboxes", [])
+            # Fix key mismatch: Florence-2 returns 'bboxes_labels' for OPEN_VOCABULARY_DETECTION and CAPTION_TO_PHRASE_GROUNDING
+            labels = detection_data.get("labels") or detection_data.get("bboxes_labels") or ["object"] * len(bboxes)
+
+            for box, label in zip(bboxes, labels):
+                if len(box) == 4:
+                    xmin, ymin, xmax, ymax = map(int, box)
+                    box = (max(0, xmin), max(0, ymin), min(w, xmax), min(h, ymax))
+                    mask, cropped = get_mask(box, image, include_mask=include_mask)
+                    result = DetailerResult(box=box, label=label, score=1.0, cls=-1, mask=mask, item=cropped)
+                    log.trace(f'Detailer box: {result}')
+                    results.append(result)
+    except Exception as err:
+        log.error(f'Detailer: failed to parse detection output: {err}')
+        log.error(f'Detailer: raw output: {data}')
+
+    return response, results
+
+
+def predict(
+    self,
+    name: str,
+    image: Image.Image,
+    device = devices.device,
+    mask: bool = True,
+    offload: bool | None = None,
+    p = None,
+) -> list[DetailerResult]:
+    if offload is None:
+        offload = shared.opts.detailer_unload
+    if image is None:
+        return []
+
+    cached = sd_offload_aux.get_aux_model(name)
+    if cached is None:
+        name, model = load(self, name)
+    else:
+        model = cached
+    if model is None:
+        return []
+
+    prompt = detailer_opt(p, 'detailer_classes') or ''
+
+    # Dynamic task routing (<OD>, <OPEN_VOCABULARY_DETECTION>, or <CAPTION_TO_PHRASE_GROUNDING>)
+    task_prompt, text_input = select_florence_task(prompt)
+
+    log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{text_input}" image={image.size} device={device} mask={mask} offload={offload}')
+    sd_offload_aux.move_aux_to_gpu(name)
+
+    t0 = time.time()
+    with devices.llm_context():
+        inputs = model.processor(
+            text=text_input,
+            images=image,
+            return_tensors="pt"
+        ).to(model.device, dtype=devices.dtype)
+        generated_ids = model.generate(
+            input_ids=inputs["input_ids"],
+            pixel_values=inputs["pixel_values"],
+            max_new_tokens=1024,
+            do_sample=False,
+            num_beams=3, # beams=3 improves grounding recall on small objects
+            early_stopping=False,
+        )
+    generated_text = model.processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
+    parsed_output = model.processor.post_process_generation(
+        generated_text,
+        task=task_prompt,
+        image_size=(image.width, image.height)
+    )
+    t1 = time.time()
+    response, results = parse(parsed_output, image, include_mask=mask)
+
+    token_count = generated_ids.shape[1] if hasattr(generated_ids, 'shape') else 0
+    log.debug(f'Detailer: name="{name}" tokens={token_count} response="{response}" items={len(results)} time={t1-t0:.3f}')
+
+    sd_offload_aux.offload_aux(name)
+    return results

modules/detailer/helper.py

@@ -1,5 +1,5 @@
 import os
-from PIL import Image
+from PIL import Image, ImageDraw
 from modules.logger import log
 
 
@@ -34,6 +34,16 @@ def detailer_opt(p, attr, opts_attr=None):
     return getattr(shared.opts, opts_attr or attr, None)
 
 
+def get_mask(box: list[int], image: Image.Image, include_mask: bool = True) -> tuple[Image.Image | None, Image.Image]:
+    cropped = image.crop(box)
+    if not include_mask:
+        return None, cropped
+    mask = Image.new('L', image.size, 0)
+    draw_mask = ImageDraw.Draw(mask)
+    draw_mask.rectangle(box, fill="white", outline=None, width=0)
+    return mask, cropped
+
+
 class DetailerResult:
     def __init__(self, cls: int, label: str, score: float, box: list[int], mask: Image.Image = None, item: Image.Image = None, width = 0, height = 0, args = None):
         if args is None:

modules/detailer/models.py

@@ -12,5 +12,11 @@ detailer_models = [ # <https://huggingface.co/vladmandic/yolo-detailers/tree/mai
     'Qwen3-VL-2B-Instruct',
     'Qwen3-VL-4B-Instruct',
     'Qwen3-VL-8B-Instruct',
-    # 'nvidia-LocateAnything-3B',
+    'Florence-2-base-ft',
+    'Florence-2-large-ft',
+    'Grounding-DINO-tiny',
+    'Grounding-DINO-base',
+    'Facebook-SAM3',
+    # 'Rex-Omni', # not compatible with transformers==5
+    # 'nvidia-LocateAnything-3B', # not compatible with transformers==5
 ]

modules/detailer/qwen.py

@@ -2,9 +2,9 @@ import time
 import json
 import transformers
 from pydantic import BaseModel, Field
-from PIL import Image, ImageDraw
+from PIL import Image
 from modules import shared, devices, sd_offload_aux, model_quant
-from modules.detailer import DetailerResult, detailer_opt
+from modules.detailer import DetailerResult, detailer_opt, get_mask
 from modules.logger import log
 
 
@@ -77,7 +77,7 @@ def load(self, model_name: str | None = None) -> tuple[str, transformers.Qwen3VL
     return model_name, model
 
 
-def parse(data: str, image: Image.Image) -> tuple[str, list[DetailerResult]]:
+def parse(data: str, image: Image.Image, include_mask: bool = True) -> tuple[str, list[DetailerResult]]:
     results = []
     response = ''
     w, h = image.size
@@ -99,11 +99,15 @@ def parse(data: str, image: Image.Image) -> tuple[str, list[DetailerResult]]:
                 xmin, ymin = int((xmin / 1000.0) * w), int((ymin / 1000.0) * h)
                 xmax, ymax = int((xmax / 1000.0) * w), int((ymax / 1000.0) * h)
                 box = (xmin, ymin, xmax, ymax)
-                mask = Image.new('L', image.size, 0)
-                draw_mask = ImageDraw.Draw(mask)
-                draw_mask.rectangle(box, fill="white", outline=None, width=0)
-                result = DetailerResult(box=box, label=label, score=confidence, cls=-1, mask=mask)
-                log.trace(f'Detailer box: {result}')
+                mask, cropped = get_mask(box, image, include_mask)
+                result = DetailerResult(box=box,
+                                        label=label,
+                                        score=confidence,
+                                        cls=-1,
+                                        mask=mask,
+                                        item=cropped
+                                       )
+                # log.trace(f'Detailer box: {result}')
                 results.append(result)
     except Exception as err:
         log.error(f'Detailer: failed to parse object detection output: {err}')
@@ -141,34 +145,34 @@ def predict(
 
     t0 = time.time()
     schema = json.dumps(ObjectDetectionOutput.model_json_schema(), indent=2)
-    messages = template(prompt=prompt, schema=schema, min_confidence=shared.opts.detailer_conf)
-    text = model.processor.apply_chat_template(
-        messages,
-        tokenize=False,
-        add_generation_prompt=True,
-    )
-    inputs = model.processor(
-        text=[text],
-        images=[image],
-        padding=True,
-        return_tensors="pt",
-        min_pixels=128 * 28 * 28,
-        max_pixels=1280 * 28 * 28,  # Force 1 MP cap
-    )
-    inputs.pop("token_type_ids", None)
-    inputs = inputs.to(model.device)
-    eos_id = model.processor.tokenizer.convert_tokens_to_ids("<|im_end|>")
-    if eos_id is None or isinstance(eos_id, list):
-        eos_id = model.processor.tokenizer.eos_token_id
-    pad_id = model.processor.tokenizer.pad_token_id if model.processor.tokenizer.pad_token_id is not None else eos_id
+    messages = template(prompt=prompt, schema=schema, min_confidence=detailer_opt(p, 'detailer_conf'))
 
     with devices.llm_context():
+        text = model.processor.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True,
+        )
+        inputs = model.processor(
+            text=[text],
+            images=[image],
+            padding=True,
+            return_tensors="pt",
+            min_pixels=128 * 28 * 28,
+            max_pixels=1280 * 28 * 28,  # Force 1 MP cap
+        )
+        inputs.pop("token_type_ids", None)
+        inputs = inputs.to(model.device)
+        eos_id = model.processor.tokenizer.convert_tokens_to_ids("<|im_end|>")
+        if eos_id is None or isinstance(eos_id, list):
+            eos_id = model.processor.tokenizer.eos_token_id
+        pad_id = model.processor.tokenizer.pad_token_id if model.processor.tokenizer.pad_token_id is not None else eos_id
         generated_ids = model.generate(
             **inputs,
             max_new_tokens=2048,
             do_sample=False,         # Deterministic decoding keeps bbox integer tokens strict
             temperature=None,        # Forces argmax token selection
-            repetition_penalty=1.01, # Breaks coordinate repetition loops without distorting valid coordinates
+            repetition_penalty=1.03, # Breaks coordinate repetition loops without distorting valid coordinates
             no_repeat_ngram_size=0,  # MUST be 0/None—setting this > 0 corrupts valid repeated bbox coordinates
             eos_token_id=eos_id,     # Prevent premature EOS token stopping
             pad_token_id=pad_id,     # Prevent premature EOS token stopping
@@ -178,7 +182,7 @@ def predict(
     output_tokens = generated_ids[0][prompt_len:]
     output_text = model.processor.tokenizer.decode(output_tokens, skip_special_tokens=True)
     t1 = time.time()
-    response, results = parse(output_text, image)
+    response, results = parse(output_text, image, include_mask=mask)
 
     log.debug(f'Detailer: name="{name}" tokens={output_tokens.shape[0]} response="{response}" items={len(results)} time={t1-t0:.3f}')
 

modules/detailer/rexomni.py

@@ -0,0 +1,157 @@
+import time
+import re
+import torch
+import transformers
+from PIL import Image
+from modules import shared, devices, sd_offload_aux, model_quant
+from modules.detailer import DetailerResult, detailer_opt, get_mask
+from modules.logger import log
+
+
+def format_rex_prompt(prompt: str) -> str:
+    clean_prompt = prompt.strip()
+    if not clean_prompt or clean_prompt.lower() == "detect and locate all objects":
+        return "<|grounding|>Locate all objects in the image."
+    if clean_prompt.startswith("<|") and "|>" in clean_prompt:
+        return clean_prompt
+    return f"<|grounding|>{clean_prompt}"
+
+
+def load(self, model_name: str | None = None) -> tuple[str, transformers.AutoModelForCausalLM]: # pylint: disable=unused-argument
+    cached = sd_offload_aux.get_aux_model(model_name)
+    if cached is not None:
+        return model_name, cached
+
+    repo_id = 'IDEA-Research/' + model_name if '/' not in model_name else model_name
+    load_kwargs = {
+        'pretrained_model_name_or_path': repo_id,
+        'cache_dir': shared.opts.hfcache_dir,
+        'torch_dtype': devices.dtype,
+        'trust_remote_code': True,
+    }
+    quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d', 'linear_attn.conv1d', 'embed_tokens', 'lm_head'])
+    model = transformers.Qwen2_5_VLForConditionalGeneration.from_pretrained(
+        **load_kwargs,
+        **quant_args,
+        attn_implementation="sdpa"
+    )
+    model = model.eval()
+    model.processor = transformers.AutoProcessor.from_pretrained(
+        repo_id,
+        cache_dir=shared.opts.hfcache_dir,
+        trust_remote_code=True
+    )
+
+    sd_offload_aux.register_aux(model_name, model)
+    if shared.opts.detailer_unload:
+        sd_offload_aux.offload_aux(model_name)
+
+    log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"')
+    return model_name, model
+
+
+def parse(raw_output: str, image: Image.Image, include_mask: bool = True) -> tuple[str, list[DetailerResult]]:
+    results = []
+    w, h = image.size
+    bboxes = []
+    labels = []
+    box_pattern = re.compile(r'(?:<box>|\(|\[)\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*(?:</box>|\)|\])\s*([^<\n,]+)?')
+    matches = box_pattern.findall(raw_output)
+    for match in matches:
+        if len(match) >= 4:
+            coords = [int(match[i]) for i in range(4)]
+            label = match[4].strip() if len(match) > 4 and match[4].strip() else "object"
+            ymin, xmin, ymax, xmax = coords
+            if max(coords) <= 1000:
+                xmin = int((xmin / 1000.0) * w)
+                ymin = int((ymin / 1000.0) * h)
+                xmax = int((xmax / 1000.0) * w)
+                ymax = int((ymax / 1000.0) * h)
+            bboxes.append((xmin, ymin, xmax, ymax))
+            labels.append(label)
+    for box, label in zip(bboxes, labels):
+        if len(box) == 4:
+            xmin, ymin, xmax, ymax = map(int, box)
+            box = (max(0, xmin), max(0, ymin), min(w, xmax), min(h, ymax))
+            if box[2] > box[0] and box[3] > box[1]:
+                mask, cropped = get_mask(box, image, include_mask=include_mask)
+                result = DetailerResult(box=box, label=label, score=1.0, cls=-1, mask=mask, item=cropped)
+                log.trace(f'Detailer box: {result}')
+                results.append(result)
+    return raw_output, results
+
+
+def predict(
+    self,
+    name: str,
+    image: Image.Image,
+    device = devices.device,
+    mask: bool = True,
+    offload: bool | None = None,
+    p = None,
+) -> list[DetailerResult]:
+    if offload is None:
+        offload = shared.opts.detailer_unload
+    if image is None:
+        return []
+
+    cached = sd_offload_aux.get_aux_model(name)
+    if cached is None:
+        name, model = load(self, name)
+    else:
+        model = cached
+
+    prompt = detailer_opt(p, 'detailer_classes') or ''
+    text_input = format_rex_prompt(prompt)
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "image", "image": image},
+                {"type": "text", "text": text_input},
+            ],
+        }
+    ]
+
+    log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{text_input}" image={image.size} device={device} mask={mask} offload={offload}')
+    sd_offload_aux.move_aux_to_gpu(name)
+
+    t0 = time.time()
+    with devices.llm_context():
+        # Use processor directly with images and text to automatically build image tokens and grid thw
+        text = model.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+        inputs = model.processor(
+            text=[text],
+            images=[image],
+            padding=True,
+            return_tensors="pt",
+        )
+        target_device = model.device
+        prepared_inputs = {}
+        for k, v in inputs.items():
+            if isinstance(v, torch.Tensor):
+                v = v.to(device=target_device)
+                if torch.is_floating_point(v):
+                    v = v.to(dtype=devices.dtype)
+                prepared_inputs[k] = v
+            else:
+                prepared_inputs[k] = v
+        generated_ids = model.generate(
+            **prepared_inputs,
+            max_new_tokens=1024,
+            do_sample=False,
+            use_cache=True,
+        )
+
+    input_len = prepared_inputs["input_ids"].shape[1]
+    generated_ids_trimmed = generated_ids[:, input_len:]
+    generated_text = model.processor.batch_decode(generated_ids_trimmed, skip_special_tokens=False)[0]
+
+    t1 = time.time()
+    response, results = parse(generated_text, image, include_mask=mask)
+
+    token_count = generated_ids.shape[1]
+    log.debug(f'Detailer: name="{name}" tokens={token_count} response="{response}" items={len(results)} time={t1-t0:.3f}')
+
+    sd_offload_aux.offload_aux(name)
+    return results

modules/detailer/sam.py

@@ -0,0 +1,102 @@
+import time
+import transformers
+from PIL import Image
+from modules import shared, devices, sd_offload_aux
+from modules.detailer import DetailerResult, detailer_opt, get_mask
+from modules.logger import log
+
+
+def load(self, model_name: str | None = None) -> tuple[str, transformers.Sam3Model]: # pylint: disable=unused-argument
+    cached = sd_offload_aux.get_aux_model(model_name)
+    if cached is not None:
+        return model_name, cached
+    repo_id = model_name.lower().replace('-', '/')
+    load_kwargs = {
+        'pretrained_model_name_or_path': repo_id,
+        'cache_dir': shared.opts.hfcache_dir,
+        'torch_dtype': devices.dtype,
+    }
+    model = transformers.Sam3Model.from_pretrained(**load_kwargs)
+    model = model.eval()
+    model.processor = transformers.Sam3Processor.from_pretrained(**load_kwargs)
+    sd_offload_aux.register_aux(model_name, model)
+    if shared.opts.detailer_unload:
+        sd_offload_aux.offload_aux(model_name)
+    log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"')
+    return model_name, model
+
+
+def predict(
+    self,
+    name: str,
+    image: Image.Image,
+    device = devices.device,
+    mask: bool = True,
+    offload: bool | None = None,
+    p = None,
+) -> list[DetailerResult]:
+    if offload is None:
+        offload = shared.opts.detailer_unload
+    if image is None:
+        return []
+    cached = sd_offload_aux.get_aux_model(name)
+    if cached is None:
+        name, model = load(self, name)
+    else:
+        model = cached
+    if model is None:
+        return []
+    prompt = detailer_opt(p, 'detailer_classes') or ''
+    if not prompt:
+        prompt = 'object'
+
+    log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{prompt}" image={image.size} device={device} mask={mask} offload={offload}')
+    sd_offload_aux.move_aux_to_gpu(name)
+
+    t0 = time.time()
+    results = []
+
+    with devices.llm_context():
+        inputs = model.processor(images=image, text=prompt, return_tensors="pt")
+        inputs = inputs.to(model.device)
+        outputs = model(**inputs)
+        target_sizes = inputs.get("original_sizes").tolist() if "original_sizes" in inputs else [image.size[::-1]]
+        results_list = model.processor.post_process_instance_segmentation(
+            outputs,
+            threshold=detailer_opt(p, 'detailer_conf') or 0.3,
+            mask_threshold=0.5,
+            target_sizes=target_sizes,
+        )
+
+    w, h = image.size
+    if results_list and len(results_list) > 0:
+        res = results_list[0]
+        boxes = res.get("boxes", [])
+        scores = res.get("scores", [])
+        labels = res.get("labels", [])
+        masks = res.get("masks", []) if mask else [None] * len(boxes)
+        for i, box_tensor in enumerate(boxes):
+            box_coords = box_tensor.tolist()
+            xmin, ymin, xmax, ymax = map(int, box_coords)
+            box = (max(0, xmin), max(0, ymin), min(w, xmax), min(h, ymax))
+            score = float(scores[i].item()) if i < len(scores) else 1.0
+            label = str(labels[i].item()) if i < len(labels) else prompt
+            masked, cropped = get_mask(box, image, include_mask=mask)
+            if mask and detailer_opt(p, 'detailer_segmentation') and (i < len(masks) and masks[i] is not None):
+                masked = Image.fromarray(masks[i].detach().cpu().numpy().astype('uint8') * 255)
+            cropped = image.crop(box)
+            result = DetailerResult(
+                box=box,
+                label=label,
+                score=score,
+                cls=-1,
+                mask=masked,
+                item=cropped
+            )
+            results.append(result)
+
+    t1 = time.time()
+    log.debug(f'Detailer: name="{name}" items={len(results)} time={t1-t0:.3f}')
+
+    sd_offload_aux.offload_aux(name)
+    return results

modules/detailer/yolo.py

@@ -2,8 +2,8 @@ from typing import TYPE_CHECKING
 import os
 import threading
 import numpy as np
-from PIL import Image, ImageDraw
-from modules.detailer import DetailerResult, detailer_opt
+from PIL import Image
+from modules.detailer import DetailerResult, detailer_opt, get_mask
 from modules.logger import log
 from modules import shared, devices
 
@@ -138,33 +138,19 @@ def predict(
             if len(desired) > 0 and label.lower() not in desired:
                 continue
             box = box.tolist()
-            w, h = box[2] - box[0], box[3] - box[1]
-            x_size, y_size = w/image.width, h/image.height
-            opt_min = detailer_opt(p, 'detailer_min_size') or 0
-            opt_max = detailer_opt(p, 'detailer_max_size') or 1
-            min_size = opt_min if 0 <= opt_min <= 1 else 0
-            max_size = opt_max if 0 < opt_max <= 1 else 1
-            if x_size >= min_size and y_size >=min_size and x_size <= max_size and y_size <= max_size:
-                if mask:
-                    if detailer_opt(p, 'detailer_segmentation') and seg is not None:
-                        masked = seg
-                    else:
-                        masked = Image.new('L', image.size, 0)
-                        draw = ImageDraw.Draw(masked)
-                        draw.rectangle(box, fill="white", outline=None, width=0)
-                    cropped = image.crop(box)
-                    res = DetailerResult(
-                        cls=cls,
-                        label=label,
-                        score=round(score, 2),
-                        box=box,
-                        mask=masked,
-                        item=cropped,
-                        width=w,
-                        height=h,
-                        args=args,
-                    )
-                    result.append(res)
+            masked, cropped = get_mask(box, image, include_mask=mask)
+            if detailer_opt(p, 'detailer_segmentation') and seg is not None:
+                masked = seg
+            res = DetailerResult(
+                cls=cls,
+                label=label,
+                score=round(score, 2),
+                box=box,
+                mask=masked,
+                item=cropped,
+                args=args,
+            )
+            result.append(res)
             if len(result) >= (detailer_opt(p, 'detailer_max') or 2):
                 break
     return result

modules/img2img.py

@@ -161,7 +161,7 @@ def img2img(id_task: str, state: str, mode: int,
             sampler_index,
             mask_blur, mask_alpha,
             vae_type, tiling, hidiffusion,
-            detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution,
+            detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes,
             n_iter, batch_size,
             guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop,
             cfg_scale, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end,
@@ -280,6 +280,7 @@ def img2img(id_task: str, state: str, mode: int,
         detailer_steps=detailer_steps,
         detailer_strength=detailer_strength,
         detailer_resolution=detailer_resolution,
+        detailer_classes=detailer_classes,
         init_images=[image],
         mask=mask,
         mask_blur=mask_blur,

modules/sd_offload_aux.py

@@ -94,7 +94,10 @@ def offload_aux(name: str) -> None:
 
 
 def get_aux_model(name: str) -> torch.nn.Module | None:
-    entry = aux_models.get(name)
+    entry = aux_models.get(name, None)
     if entry is None:
+        entry = aux_models.get(name.lower(), None)
+    if entry is None:
+        # log.warning(f'Model not found: requested="{name}" available={list(aux_models.keys())}')
         return None
     return entry.model

modules/txt2img.py

@@ -14,7 +14,7 @@ def txt2img(id_task, state,
             prompt, negative_prompt, prompt_styles,
             steps, sampler_index, hr_sampler_index,
             vae_type, tiling, hidiffusion,
-            detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution,
+            detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes,
             n_iter, batch_size,
             guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop,
             cfg_scale, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end,
@@ -83,6 +83,7 @@ def txt2img(id_task, state,
         detailer_steps=detailer_steps,
         detailer_strength=detailer_strength,
         detailer_resolution=detailer_resolution,
+        detailer_classes=detailer_classes,
         tiling=tiling,
         hidiffusion=hidiffusion,
         enable_hr=enable_hr,

modules/ui_control.py

@@ -207,7 +207,7 @@ def create_ui(_blocks: gr.Blocks=None):
                         video_type, video_duration, video_loop, video_pad, video_interpolate = create_video_inputs(tab='control')
 
                 enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('control')
-                detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution = shared.detailer.ui('control')
+                detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes = shared.detailer.ui('control')
 
             with gr.Row():
                 override_script_name = gr.State(value='', visible=False, elem_id='control_override_script_name')
@@ -311,7 +311,7 @@ def create_ui(_blocks: gr.Blocks=None):
                 seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w,
                 guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop,
                 cfg_scale, clip_skip, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end, vae_type, tiling, hidiffusion,
-                detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution,
+                detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes,
                 hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio, hdr_apply_hires,
                 grading_brightness, grading_contrast, grading_saturation, grading_hue, grading_gamma, grading_sharpness, grading_color_temp,
                 grading_shadows, grading_midtones, grading_highlights, grading_clahe_clip, grading_clahe_grid,

modules/ui_img2img.py

@@ -142,7 +142,7 @@ def create_ui():
                     grading_brightness, grading_contrast, grading_saturation, grading_hue, grading_gamma, grading_sharpness, grading_color_temp, grading_shadows, grading_midtones, grading_highlights, grading_clahe_clip, grading_clahe_grid, grading_shadows_tint, grading_highlights_tint, grading_split_tone_balance, grading_vignette, grading_grain, grading_lut_file, grading_lut_strength = ui_sections.create_color_inputs('img2img')
                     hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio, hdr_apply_hires = ui_sections.create_latent_inputs('img2img')
                     enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, hr_refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('img2img')
-                    detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution = shared.detailer.ui('img2img')
+                    detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes = shared.detailer.ui('img2img')
 
                     # with gr.Group(elem_id="inpaint_controls", visible=False) as inpaint_controls:
                     with gr.Accordion(open=False, label="Mask", elem_classes=["small-accordion"], elem_id="img2img_mask_group") as inpaint_controls:
@@ -181,7 +181,7 @@ def create_ui():
                 sampler_index,
                 mask_blur, mask_alpha,
                 vae_type, tiling, hidiffusion,
-                detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution,
+                detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes,
                 batch_count, batch_size,
                 guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop,
                 cfg_scale, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end,

modules/ui_txt2img.py

@@ -38,7 +38,7 @@ def create_ui():
                     grading_brightness, grading_contrast, grading_saturation, grading_hue, grading_gamma, grading_sharpness, grading_color_temp, grading_shadows, grading_midtones, grading_highlights, grading_clahe_clip, grading_clahe_grid, grading_shadows_tint, grading_highlights_tint, grading_split_tone_balance, grading_vignette, grading_grain, grading_lut_file, grading_lut_strength = ui_sections.create_color_inputs('txt2img')
                     hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio, hdr_apply_hires = ui_sections.create_latent_inputs('txt2img')
                     enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img')
-                    detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution  = shared.detailer.ui('txt2img')
+                    detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes  = shared.detailer.ui('txt2img')
                     override_settings = ui_common.create_override_inputs('txt2img')
                     state = gr.Textbox(value='', visible=False)
 
@@ -56,7 +56,7 @@ def create_ui():
                 txt2img_prompt, txt2img_negative_prompt, txt2img_prompt_styles,
                 steps, sampler_index, hr_sampler_index,
                 vae_type, tiling, hidiffusion,
-                detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution,
+                detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes,
                 batch_count, batch_size,
                 guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop,
                 cfg_scale, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end,

scripts/postprocessing_detailer.py

@@ -12,7 +12,7 @@ class ScriptPostprocessingDetailer(scripts_postprocessing.ScriptPostprocessing):
         # The detailer accordion (built by yolo.ui) now contains the Sampler sub-accordion too, so for 'extras'
         # it returns a 7th element: a dict of the sampler-block controls. Spread it into the control map; their
         # values are stamped onto the synthetic p in process()/make_processing(), applying to this pass only.
-        enabled, prompt, negative, steps, strength, resolution, sampler_block = shared.detailer.ui('extras')
+        enabled, prompt, negative, steps, strength, resolution, classes, sampler_block = shared.detailer.ui('extras')
         return {
             "enabled": enabled,
             "prompt": prompt,
@@ -20,11 +20,12 @@ class ScriptPostprocessingDetailer(scripts_postprocessing.ScriptPostprocessing):
             "steps": steps,
             "strength": strength,
             "resolution": resolution,
+            "classes": classes,
             **sampler_block,
         }
 
-    def process(self, pp: scripts_postprocessing.PostprocessedImage,  # pylint: disable=arguments-differ
-                enabled=False, prompt='', negative='', steps=10, strength=0.3, resolution=1024,
+    def process(self, pp: scripts_postprocessing.PostprocessedImage, # pylint: disable=arguments-differ
+                enabled=False, prompt='', negative='', steps=10, strength=0.3, resolution=1024, classes='',
                 sampler='Default', prediction='default', shift=3.0, cfg_scale=6.0, options=None, seed=-1):
         if not enabled:
             return pp
@@ -52,7 +53,16 @@ class ScriptPostprocessingDetailer(scripts_postprocessing.ScriptPostprocessing):
             'schedulers_rescale_betas': 'rescale' in options,
         }
         log.info(f'Detailer postprocess: strength={strength} steps={steps} resolution={resolution} sampler={sampler} cfg={cfg_scale}')
-        p = shared.detailer.make_processing(pp.image, prompt=prompt, negative=negative, steps=steps, strength=strength, resolution=resolution, seed=int(seed) if seed is not None else -1, overrides=overrides)
+        p = shared.detailer.make_processing(pp.image,
+                                            prompt=prompt,
+                                            negative=negative,
+                                            steps=steps,
+                                            strength=strength,
+                                            resolution=resolution,
+                                            classes=classes,
+                                            seed=int(seed) if seed is not None else -1,
+                                            overrides=overrides,
+                                           )
 
         try:
             result = shared.detailer.restore(np.array(pp.image), p)