diff --git a/CHANGELOG.md b/CHANGELOG.md index c08b89aba..4ab63178d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,21 @@ # Change Log for SD.Next -## Update for 2026-08-08 +## Update for 2026-08-09 -- **Detailer** support for VL models in addition to standard YOLO models - select any of Qwen3-VL models in model selection - enter the human-readable ask as detailer instructions +- **Detailer**: Pretty much *detailer.next* :) + Detailer detection models were traditionally *YOLO* models, but now we also use: + - [Facebook-SAM3](https://huggingface.co/facebook/sam3) hybrid promptable concept segmentation and detection network + - [Qwen3-VL](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct) vision-language autoregressive foundation models, in *2B, 4B, and 8B* variants + - [Florence-2](https://huggingface.co/microsoft/Florence-2-large) lightweight multi-task vision sequence-to-sequence models, in *base and large* variants + - [Grounding-DINO](https://huggingface.co/IDEA-Research/grounding-dino-base) open-vocabulary object detection models, in *tiny and base* variants + + select any of the above models in the detailer model selection dropdown + and enter your human-readable target descriptions as *detailer instructions* for example: *glasses, optional hat, left hand, largest tree in the background* - *note*: VL models are much larger, so use with caution - *note*: LLM processes data sequentially, so if it cannot find one item, it may skip the rest of the items in the prompt - + *note*: VL models are much larger, so use them with caution + *note*: LLMs process data sequentially, meaning if a model cannot find one specific item, it may skip subsequent items in the prompt + *note*: SAM3 is a [gated model](https://vladmandic.github.io/sdnext-docs/Gated/) + ## Update for 2026-08-07 ### Highlights for 2026-08-07 diff --git a/installer.py b/installer.py index 833ace926..f90945bcd 100644 --- a/installer.py +++ b/installer.py @@ -478,8 +478,8 @@ def get_platform(): release = platform.release() return { 'arch': platform.machine(), - 'cpu': f'{platform.processor()}', - 'system': platform.system(), + 'cpu': f'"{platform.processor()}"', + 'system': f'"{platform.system()}"', 'release': release, 'python': platform.python_version(), 'locale': locale.getlocale(), diff --git a/modules/api/middleware.py b/modules/api/middleware.py index 6775873a3..8ef55e525 100644 --- a/modules/api/middleware.py +++ b/modules/api/middleware.py @@ -144,7 +144,7 @@ def setup_middleware(app: FastAPI, cmd_opts): if not isinstance(e, HTTPException) and err['error'] != 'TypeError': # do not print backtrace on known httpexceptions errors.display(e, 'HTTP API', [anyio, fastapi, uvicorn, starlette]) - elif err['code'] in [404, 401, 400]: + elif err['code'] in [404, 401, 400, 403]: pass else: log.debug(e, exc_info=True) # print stack trace diff --git a/modules/api/process.py b/modules/api/process.py index 6264fb741..ef75f902e 100644 --- a/modules/api/process.py +++ b/modules/api/process.py @@ -190,6 +190,7 @@ class APIProcess: strength=req.detailer_strength if req.detailer_strength is not None else 0.3, resolution=req.detailer_resolution if req.detailer_resolution is not None else 1024, seed=req.seed if req.seed is not None else -1, + classes=req.detailer_classes if req.detailer_classes is not None else None, overrides=overrides, ) diff --git a/modules/control/run.py b/modules/control/run.py index 4c9834aa5..e84bb5fa0 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -346,7 +346,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg guidance_name: str = 'Default', guidance_scale: float = 6.0, guidance_rescale: float = 0.0, guidance_start: float = 0.0, guidance_stop: float = 1.0, cfg_scale: float = 6.0, clip_skip: float = 1.0, cfg_image: float = 6.0, cfg_rescale: float = 0.7, cfg_true: float = 0.0, cfg_adaptive: float = 0.5, cfg_end: float = 1.0, vae_type: str = 'Full', tiling: bool = False, hidiffusion: bool = False, - detailer_enabled: bool = False, detailer_prompt: str = '', detailer_negative: str = '', detailer_steps: int = 10, detailer_strength: float = 0.3, detailer_resolution: int = 1024, + detailer_enabled: bool = False, detailer_prompt: str = '', detailer_negative: str = '', detailer_steps: int = 10, detailer_strength: float = 0.3, detailer_resolution: int = 1024, detailer_classes: str = '', hdr_mode: int = 0, hdr_brightness: float = 0, hdr_color: float = 0, hdr_sharpen: float = 0, hdr_clamp: bool = False, hdr_boundary: float = 4.0, hdr_threshold: float = 0.95, hdr_maximize: bool = False, hdr_max_center: float = 0.6, hdr_max_boundary: float = 1.0, hdr_color_picker: str | None = None, hdr_tint_ratio: float = 0, hdr_apply_hires: bool = True, grading_brightness: float = 0.0, grading_contrast: float = 0.0, grading_saturation: float = 0.0, grading_hue: float = 0.0, @@ -368,7 +368,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg override_script_name: str | None = None, override_script_args = None, extra: dict | None = None, *input_script_args, # API-only params (keyword-only, not wired to Gradio) - detailer_segmentation: bool | None = None, detailer_include_detections: bool | None = None, detailer_merge: bool | None = None, detailer_sort: bool | None = None, detailer_classes: str | None = None, + detailer_segmentation: bool | None = None, detailer_include_detections: bool | None = None, detailer_merge: bool | None = None, detailer_sort: bool | None = None, detailer_conf: float | None = None, detailer_iou: float | None = None, detailer_max: int | None = None, detailer_min_size: float | None = None, detailer_max_size: float | None = None, detailer_blur: int | None = None, detailer_padding: int | None = None, diff --git a/modules/detailer/__init__.py b/modules/detailer/__init__.py index 2b6901196..f0bf2e64d 100644 --- a/modules/detailer/__init__.py +++ b/modules/detailer/__init__.py @@ -1,5 +1,5 @@ from .models import detailer_models -from .helper import detailer_opt, DetailerResult, list_models +from .helper import DetailerResult, detailer_opt, list_models, get_mask from .detailer import Detailer diff --git a/modules/detailer/detailer.py b/modules/detailer/detailer.py index 23225448e..b7f316c45 100644 --- a/modules/detailer/detailer.py +++ b/modules/detailer/detailer.py @@ -36,34 +36,64 @@ class Detailer(): offload: bool | None = None, p = None, ) -> list[DetailerResult]: + jobid = shared.state.begin('Detect') if 'LocateAnything' in name: from modules.detailer import locateanything - return locateanything.predict(self, name, image, device=device, mask=mask, offload=offload, p=p) - if 'Qwen3-VL' in name: + results = locateanything.predict(self, name, image, device=device, mask=mask, offload=offload, p=p) + elif 'Qwen3-VL' in name: from modules.detailer import qwen - return qwen.predict(self, name, image, device=device, mask=mask, offload=offload, p=p) - - from modules.detailer import yolo - return yolo.predict(self, model, image, imgsz=imgsz, half=half, device=device, agnostic=agnostic, retina=retina, mask=mask, augment=augment, offload=offload, p=p) + results = qwen.predict(self, name, image, device=device, mask=mask, offload=offload, p=p) + elif 'Florence-2' in name: + from modules.detailer import florence + results = florence.predict(self, name, image, device=device, mask=mask, offload=offload, p=p) + elif 'Grounding-DINO' in name: + from modules.detailer import dino + results = dino.predict(self, name, image, device=device, mask=mask, offload=offload, p=p) + elif 'Rex-Omni' in name: + from modules.detailer import rexomni + results = rexomni.predict(self, name, image, device=device, mask=mask, offload=offload, p=p) + elif 'Facebook-SAM3' in name: + from modules.detailer import sam + results = sam.predict(self, name, image, device=device, mask=mask, offload=offload, p=p) + else: + from modules.detailer import yolo + results = yolo.predict(self, model, image, imgsz=imgsz, half=half, device=device, agnostic=agnostic, retina=retina, mask=mask, augment=augment, offload=offload, p=p) + shared.state.end(jobid) + return results def enumerate(self): from modules.detailer import list_models return list_models(self) def load(self, model_name: str | None = None): + jobid = shared.state.begin('Load detailer') if 'LocateAnything' in model_name: from modules.detailer import locateanything - return locateanything.load(self, model_name=model_name) - if 'Qwen3-VL' in model_name: + model_name, model = locateanything.load(self, model_name=model_name) + elif 'Qwen3-VL' in model_name: from modules.detailer import qwen - return qwen.load(self, model_name=model_name) - - from modules.detailer import yolo - return yolo.load(self, model_name=model_name) + model_name, model = qwen.load(self, model_name=model_name) + elif 'Florence-2' in model_name: + from modules.detailer import florence + model_name, model = florence.load(self, model_name=model_name) + elif 'Grounding-DINO' in model_name: + from modules.detailer import dino + model_name, model = dino.load(self, model_name=model_name) + elif 'Rex-Omni' in model_name: + from modules.detailer import rexomni + model_name, model = rexomni.load(self, model_name=model_name) + elif 'Facebook-SAM3' in model_name: + from modules.detailer import sam + model_name, model = sam.load(self, model_name=model_name) + else: + from modules.detailer import yolo + model_name, model = yolo.load(self, model_name=model_name) + shared.state.end(jobid) + return model_name, model def merge(self, items: list[DetailerResult]) -> list[DetailerResult]: if items is None or len(items) == 0: - return None + return [] box=[min(item.box[0] for item in items), min(item.box[1] for item in items), max(item.box[2] for item in items), max(item.box[3] for item in items)] mask = Image.new('L', items[0].mask.size, 0) for item in items: @@ -80,6 +110,30 @@ class Detailer(): ) return [merged] + def filter(self, items: list[DetailerResult], image: Image.Image, p: processing.StableDiffusionProcessing = None) -> list[DetailerResult]: + if items is None or len(items) == 0: + return [] + if p is not None: + min_conf = detailer_opt(p, 'detailer_conf') + max_detected = detailer_opt(p, 'detailer_max') + filtered = [item for item in items if item.score >= min_conf] + opt_min = detailer_opt(p, 'detailer_min_size') or 0 + opt_max = detailer_opt(p, 'detailer_max_size') or 1 + for item in filtered.copy(): + w, h = item.box[2] - item.box[0], item.box[3] - item.box[1] + x_size, y_size = w/image.width, h/image.height + min_size = opt_min if 0 <= opt_min <= 1 else 0 + max_size = opt_max if 0 < opt_max <= 1 else 1 + if not ((x_size >= min_size) and (y_size >= min_size) and (x_size <= max_size) and (y_size <= max_size)): + filtered.remove(item) + filtered = sorted(filtered, key=lambda x: x.score, reverse=True) + filtered = filtered[:max_detected] + else: + filtered = items + if len(filtered) != len(items): + log.debug(f'Detailer: items={len(items)} filtered={len(filtered)}') + return filtered + def draw_masks(self, image: Image.Image, items: list[DetailerResult], p=None) -> Image.Image: if not isinstance(image, Image.Image): image = Image.fromarray(image) @@ -87,7 +141,7 @@ class Detailer(): size = min(image.width, image.height) // 32 font = images.get_font(size) color = (0, 190, 190) - log.debug(f'Detailer: draw={items}') + # log.debug(f'Detailer: draw={items}') for i, item in enumerate(items): if detailer_opt(p, 'detailer_segmentation') and item.mask is not None: mask = item.mask.convert('L') @@ -157,6 +211,7 @@ class Detailer(): if image is None: image = Image.fromarray(np_image) items = self.predict(name, model, image, p=p) + items = self.filter(items, image, p=p) if len(items) == 0: log.info(f'Detailer: model="{name}" no items detected') @@ -305,7 +360,7 @@ class Detailer(): np_images.append(annotated) # save debug image with boxes return np_images - def make_processing(self, image, prompt='', negative='', steps=10, strength=0.3, resolution=1024, seed=-1, overrides=None): + def make_processing(self, image, prompt='', negative='', steps=10, strength=0.3, resolution=1024, seed=-1, overrides=None, classes=''): """Build a synthetic Img2Img processing object to run restore() standalone, with no base generation pass. The primary params map to the detailer_* fields restore() reads directly. overrides is an optional @@ -337,6 +392,7 @@ class Detailer(): detailer_steps=steps, detailer_strength=strength, detailer_resolution=resolution, + detailer_classes=classes, ) for attr, val in (overrides or {}).items(): if val is not None: @@ -366,7 +422,7 @@ class Detailer(): shared.opts.detailer_merge = merge shared.opts.detailer_models = detailers shared.opts.detailer_args = text if not self.ui_mode else '' - shared.opts.detailer_classes = classes + # shared.opts.detailer_classes = classes shared.opts.detailer_padding = padding shared.opts.detailer_blur = blur shared.opts.detailer_conf = min_confidence @@ -466,5 +522,5 @@ class Detailer(): sort.change(fn=ui_settings_change, inputs=[merge, detailers, detailers_text, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps, renoise_value, renoise_end, resolution, save, sort, seg], outputs=[]) seg.change(fn=ui_settings_change, inputs=[merge, detailers, detailers_text, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps, renoise_value, renoise_end, resolution, save, sort, seg], outputs=[]) if tab == 'extras': - return enabled, prompt, negative, steps, strength, resolution, sampler_block - return enabled, prompt, negative, steps, strength, resolution + return enabled, prompt, negative, steps, strength, resolution, classes, sampler_block + return enabled, prompt, negative, steps, strength, resolution, classes diff --git a/modules/detailer/dino.py b/modules/detailer/dino.py new file mode 100644 index 000000000..6eb3416a6 --- /dev/null +++ b/modules/detailer/dino.py @@ -0,0 +1,124 @@ +import time +import re +import torch +import transformers +from PIL import Image +from modules import shared, devices, sd_offload_aux, model_quant +from modules.detailer import DetailerResult, detailer_opt, get_mask +from modules.logger import log + + +def format_grounding_dino_prompt(prompt: str) -> str: + # Formats user input into Grounding DINO query format. Grounding DINO requires lowercase text separated by periods and ending with a period. + clean_prompt = prompt.strip().lower() + if not clean_prompt or clean_prompt == "detect and locate all objects": + return "object." + items = [i.strip() for i in re.split(r'[,.]', clean_prompt) if i.strip()] + if not items: + return "object." + return ". ".join(items) + "." + + +def load(self, model_name: str | None = None) -> tuple[str, transformers.AutoModelForZeroShotObjectDetection]: # pylint: disable=unused-argument + cached = sd_offload_aux.get_aux_model(model_name) + if cached is not None: + return model_name, cached + repo_id = 'IDEA-Research/' + model_name.lower() if '/' not in model_name else model_name + load_kwargs = { + 'pretrained_model_name_or_path': repo_id, + 'cache_dir': shared.opts.hfcache_dir, + 'torch_dtype': devices.dtype, + } + quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d']) + model = transformers.AutoModelForZeroShotObjectDetection.from_pretrained(**load_kwargs, **quant_args) + model = model.eval() + model.processor = transformers.AutoProcessor.from_pretrained(**load_kwargs) + + sd_offload_aux.register_aux(model_name, model) + if shared.opts.detailer_unload: + sd_offload_aux.offload_aux(model_name) + + log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"') + return model_name, model + + +def parse(data: dict, image: Image.Image, include_mask: bool = True) -> tuple[str, list[DetailerResult]]: + results = [] + w, h = image.size + boxes = data.get("boxes", []) + scores = data.get("scores", []) + labels = data.get("labels", []) + for box, score, label in zip(boxes, scores, labels): + if len(box) == 4: + if isinstance(box, torch.Tensor): + box = box.tolist() + xmin, ymin, xmax, ymax = map(int, box) + box = (max(0, xmin), max(0, ymin), min(w, xmax), min(h, ymax)) + mask, cropped = get_mask(box, image, include_mask=include_mask) + result = DetailerResult(box=box, + label=str(label), + score=float(score), + cls=-1, + mask=mask, + item=cropped + ) + # log.trace(f'Detailer box: {result}') + results.append(result) + response = f"Grounding DINO detected {len(results)} objects." + return response, results + + +def predict( + self, + name: str, + image: Image.Image, + device = devices.device, + mask: bool = True, + offload: bool | None = None, + p = None, +) -> list[DetailerResult]: + if offload is None: + offload = shared.opts.detailer_unload + if image is None: + return [] + + cached = sd_offload_aux.get_aux_model(name) + if cached is None: + name, model = load(self, name) + else: + model = cached + if model is None: + return [] + + prompt = detailer_opt(p, 'detailer_classes') or '' + text_input = format_grounding_dino_prompt(prompt) + threshold = detailer_opt(p, 'detailer_conf') + + log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{text_input}" image={image.size} device={device} mask={mask} offload={offload} threshold={threshold}') + sd_offload_aux.move_aux_to_gpu(name) + + t0 = time.time() + with devices.llm_context(): + inputs = model.processor( + images=image, + text=text_input, + return_tensors="pt" + ).to(model.device, dtype=devices.dtype) + with torch.autocast(device_type=model.device.type, dtype=devices.dtype): + outputs = model(**inputs) + + parsed_output = model.processor.post_process_grounded_object_detection( + outputs=outputs, + input_ids=inputs.input_ids, + threshold=threshold, + text_threshold=threshold, + target_sizes=[(image.height, image.width)] + )[0] + + t1 = time.time() + response, results = parse(parsed_output, image, include_mask=mask) + + log.debug(f'Detailer: name="{name}" response="{response}" items={len(results)} time={t1-t0:.3f}') + + sd_offload_aux.offload_aux(name) + return results diff --git a/modules/detailer/florence.py b/modules/detailer/florence.py new file mode 100644 index 000000000..a568168c6 --- /dev/null +++ b/modules/detailer/florence.py @@ -0,0 +1,171 @@ +import time +import json +import transformers +from PIL import Image +from modules import shared, devices, sd_offload_aux, model_quant +from modules.detailer import DetailerResult, detailer_opt, get_mask +from modules.logger import log + + +def select_florence_task(prompt: str) -> tuple[str, str]: + # Analyzes a user prompt and automatically determines whether to use , , or default + clean_prompt = prompt.strip() + if not clean_prompt or clean_prompt == "Detect and locate all objects": + return "", "" + """ + import re + items = [i.strip() for i in clean_prompt.split(",") if i.strip()] + is_class_list = len(items) > 1 or all(len(item.split()) <= 2 for item in items) + descriptive_keywords = re.search(r'\b(a|an|the|with|wearing|in|on|next to|holding|under|near)\b', clean_prompt, re.IGNORECASE) + if is_class_list and not descriptive_keywords: + task = "" + # formatted = f"{task}{', '.join(items)}" + formatted = f"{task}{'. '.join(items)}" + else: + task = "" + formatted = f"{task}{clean_prompt}" + """ + task = "" + formatted = f"{task}{clean_prompt}" + return task, formatted + + +def load(self, model_name: str | None = None) -> tuple[str, transformers.AutoModelForCausalLM]: # pylint: disable=unused-argument + cached = sd_offload_aux.get_aux_model(model_name) + if cached is not None: + return model_name, cached + repo_id = 'florence-community/' + model_name if '/' not in model_name else model_name + + orig_get_imports = transformers.dynamic_module_utils.get_imports + + def hijack_get_imports(f): + R = orig_get_imports(f) + if "flash_attn" in R: + R.remove("flash_attn") # flash_attn is optional + return R + + transformers.dynamic_module_utils.get_imports = hijack_get_imports + load_kwargs = { + 'pretrained_model_name_or_path': repo_id, + 'cache_dir': shared.opts.hfcache_dir, + 'torch_dtype': devices.dtype, + 'trust_remote_code': True, + } + quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d']) + model = transformers.Florence2ForConditionalGeneration.from_pretrained( + **load_kwargs, + **quant_args, + attn_implementation="sdpa" + ) + model = model.eval() + model.processor = transformers.AutoProcessor.from_pretrained(**load_kwargs, max_pixels=1024*1024) + + transformers.dynamic_module_utils.get_imports = orig_get_imports + sd_offload_aux.register_aux(model_name, model) + if shared.opts.detailer_unload: + sd_offload_aux.offload_aux(model_name) + + log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"') + return model_name, model + + +def parse(data: dict | str, image: Image.Image, include_mask: bool = True) -> tuple[str, list[DetailerResult]]: + results = [] + response = "" + w, h = image.size + try: + parsed_data = {} + if isinstance(data, str): + clean = data.strip() + if clean.startswith("```json"): + clean = clean[7:] + if clean.endswith("```"): + clean = clean[:-3] + parsed_data = json.loads(clean.strip()) + elif isinstance(data, dict): + parsed_data = data + detection_data = None + for key in ["", "", ""]: + if key in parsed_data: + detection_data = parsed_data[key] + response = f"task '{key}' executed successfully." + break + if detection_data: + bboxes = detection_data.get("bboxes", []) + # Fix key mismatch: Florence-2 returns 'bboxes_labels' for OPEN_VOCABULARY_DETECTION and CAPTION_TO_PHRASE_GROUNDING + labels = detection_data.get("labels") or detection_data.get("bboxes_labels") or ["object"] * len(bboxes) + + for box, label in zip(bboxes, labels): + if len(box) == 4: + xmin, ymin, xmax, ymax = map(int, box) + box = (max(0, xmin), max(0, ymin), min(w, xmax), min(h, ymax)) + mask, cropped = get_mask(box, image, include_mask=include_mask) + result = DetailerResult(box=box, label=label, score=1.0, cls=-1, mask=mask, item=cropped) + log.trace(f'Detailer box: {result}') + results.append(result) + except Exception as err: + log.error(f'Detailer: failed to parse detection output: {err}') + log.error(f'Detailer: raw output: {data}') + + return response, results + + +def predict( + self, + name: str, + image: Image.Image, + device = devices.device, + mask: bool = True, + offload: bool | None = None, + p = None, +) -> list[DetailerResult]: + if offload is None: + offload = shared.opts.detailer_unload + if image is None: + return [] + + cached = sd_offload_aux.get_aux_model(name) + if cached is None: + name, model = load(self, name) + else: + model = cached + if model is None: + return [] + + prompt = detailer_opt(p, 'detailer_classes') or '' + + # Dynamic task routing (, , or ) + task_prompt, text_input = select_florence_task(prompt) + + log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{text_input}" image={image.size} device={device} mask={mask} offload={offload}') + sd_offload_aux.move_aux_to_gpu(name) + + t0 = time.time() + with devices.llm_context(): + inputs = model.processor( + text=text_input, + images=image, + return_tensors="pt" + ).to(model.device, dtype=devices.dtype) + generated_ids = model.generate( + input_ids=inputs["input_ids"], + pixel_values=inputs["pixel_values"], + max_new_tokens=1024, + do_sample=False, + num_beams=3, # beams=3 improves grounding recall on small objects + early_stopping=False, + ) + generated_text = model.processor.batch_decode(generated_ids, skip_special_tokens=False)[0] + parsed_output = model.processor.post_process_generation( + generated_text, + task=task_prompt, + image_size=(image.width, image.height) + ) + t1 = time.time() + response, results = parse(parsed_output, image, include_mask=mask) + + token_count = generated_ids.shape[1] if hasattr(generated_ids, 'shape') else 0 + log.debug(f'Detailer: name="{name}" tokens={token_count} response="{response}" items={len(results)} time={t1-t0:.3f}') + + sd_offload_aux.offload_aux(name) + return results diff --git a/modules/detailer/helper.py b/modules/detailer/helper.py index 8ac00c7dd..fa6af9f86 100644 --- a/modules/detailer/helper.py +++ b/modules/detailer/helper.py @@ -1,5 +1,5 @@ import os -from PIL import Image +from PIL import Image, ImageDraw from modules.logger import log @@ -34,6 +34,16 @@ def detailer_opt(p, attr, opts_attr=None): return getattr(shared.opts, opts_attr or attr, None) +def get_mask(box: list[int], image: Image.Image, include_mask: bool = True) -> tuple[Image.Image | None, Image.Image]: + cropped = image.crop(box) + if not include_mask: + return None, cropped + mask = Image.new('L', image.size, 0) + draw_mask = ImageDraw.Draw(mask) + draw_mask.rectangle(box, fill="white", outline=None, width=0) + return mask, cropped + + class DetailerResult: def __init__(self, cls: int, label: str, score: float, box: list[int], mask: Image.Image = None, item: Image.Image = None, width = 0, height = 0, args = None): if args is None: diff --git a/modules/detailer/models.py b/modules/detailer/models.py index 3237f15c3..5bfd47ca0 100644 --- a/modules/detailer/models.py +++ b/modules/detailer/models.py @@ -12,5 +12,11 @@ detailer_models = [ # tuple[str, transformers.Qwen3VL return model_name, model -def parse(data: str, image: Image.Image) -> tuple[str, list[DetailerResult]]: +def parse(data: str, image: Image.Image, include_mask: bool = True) -> tuple[str, list[DetailerResult]]: results = [] response = '' w, h = image.size @@ -99,11 +99,15 @@ def parse(data: str, image: Image.Image) -> tuple[str, list[DetailerResult]]: xmin, ymin = int((xmin / 1000.0) * w), int((ymin / 1000.0) * h) xmax, ymax = int((xmax / 1000.0) * w), int((ymax / 1000.0) * h) box = (xmin, ymin, xmax, ymax) - mask = Image.new('L', image.size, 0) - draw_mask = ImageDraw.Draw(mask) - draw_mask.rectangle(box, fill="white", outline=None, width=0) - result = DetailerResult(box=box, label=label, score=confidence, cls=-1, mask=mask) - log.trace(f'Detailer box: {result}') + mask, cropped = get_mask(box, image, include_mask) + result = DetailerResult(box=box, + label=label, + score=confidence, + cls=-1, + mask=mask, + item=cropped + ) + # log.trace(f'Detailer box: {result}') results.append(result) except Exception as err: log.error(f'Detailer: failed to parse object detection output: {err}') @@ -141,34 +145,34 @@ def predict( t0 = time.time() schema = json.dumps(ObjectDetectionOutput.model_json_schema(), indent=2) - messages = template(prompt=prompt, schema=schema, min_confidence=shared.opts.detailer_conf) - text = model.processor.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True, - ) - inputs = model.processor( - text=[text], - images=[image], - padding=True, - return_tensors="pt", - min_pixels=128 * 28 * 28, - max_pixels=1280 * 28 * 28, # Force 1 MP cap - ) - inputs.pop("token_type_ids", None) - inputs = inputs.to(model.device) - eos_id = model.processor.tokenizer.convert_tokens_to_ids("<|im_end|>") - if eos_id is None or isinstance(eos_id, list): - eos_id = model.processor.tokenizer.eos_token_id - pad_id = model.processor.tokenizer.pad_token_id if model.processor.tokenizer.pad_token_id is not None else eos_id + messages = template(prompt=prompt, schema=schema, min_confidence=detailer_opt(p, 'detailer_conf')) with devices.llm_context(): + text = model.processor.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + ) + inputs = model.processor( + text=[text], + images=[image], + padding=True, + return_tensors="pt", + min_pixels=128 * 28 * 28, + max_pixels=1280 * 28 * 28, # Force 1 MP cap + ) + inputs.pop("token_type_ids", None) + inputs = inputs.to(model.device) + eos_id = model.processor.tokenizer.convert_tokens_to_ids("<|im_end|>") + if eos_id is None or isinstance(eos_id, list): + eos_id = model.processor.tokenizer.eos_token_id + pad_id = model.processor.tokenizer.pad_token_id if model.processor.tokenizer.pad_token_id is not None else eos_id generated_ids = model.generate( **inputs, max_new_tokens=2048, do_sample=False, # Deterministic decoding keeps bbox integer tokens strict temperature=None, # Forces argmax token selection - repetition_penalty=1.01, # Breaks coordinate repetition loops without distorting valid coordinates + repetition_penalty=1.03, # Breaks coordinate repetition loops without distorting valid coordinates no_repeat_ngram_size=0, # MUST be 0/None—setting this > 0 corrupts valid repeated bbox coordinates eos_token_id=eos_id, # Prevent premature EOS token stopping pad_token_id=pad_id, # Prevent premature EOS token stopping @@ -178,7 +182,7 @@ def predict( output_tokens = generated_ids[0][prompt_len:] output_text = model.processor.tokenizer.decode(output_tokens, skip_special_tokens=True) t1 = time.time() - response, results = parse(output_text, image) + response, results = parse(output_text, image, include_mask=mask) log.debug(f'Detailer: name="{name}" tokens={output_tokens.shape[0]} response="{response}" items={len(results)} time={t1-t0:.3f}') diff --git a/modules/detailer/rexomni.py b/modules/detailer/rexomni.py new file mode 100644 index 000000000..ba4288ba2 --- /dev/null +++ b/modules/detailer/rexomni.py @@ -0,0 +1,157 @@ +import time +import re +import torch +import transformers +from PIL import Image +from modules import shared, devices, sd_offload_aux, model_quant +from modules.detailer import DetailerResult, detailer_opt, get_mask +from modules.logger import log + + +def format_rex_prompt(prompt: str) -> str: + clean_prompt = prompt.strip() + if not clean_prompt or clean_prompt.lower() == "detect and locate all objects": + return "<|grounding|>Locate all objects in the image." + if clean_prompt.startswith("<|") and "|>" in clean_prompt: + return clean_prompt + return f"<|grounding|>{clean_prompt}" + + +def load(self, model_name: str | None = None) -> tuple[str, transformers.AutoModelForCausalLM]: # pylint: disable=unused-argument + cached = sd_offload_aux.get_aux_model(model_name) + if cached is not None: + return model_name, cached + + repo_id = 'IDEA-Research/' + model_name if '/' not in model_name else model_name + load_kwargs = { + 'pretrained_model_name_or_path': repo_id, + 'cache_dir': shared.opts.hfcache_dir, + 'torch_dtype': devices.dtype, + 'trust_remote_code': True, + } + quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d', 'linear_attn.conv1d', 'embed_tokens', 'lm_head']) + model = transformers.Qwen2_5_VLForConditionalGeneration.from_pretrained( + **load_kwargs, + **quant_args, + attn_implementation="sdpa" + ) + model = model.eval() + model.processor = transformers.AutoProcessor.from_pretrained( + repo_id, + cache_dir=shared.opts.hfcache_dir, + trust_remote_code=True + ) + + sd_offload_aux.register_aux(model_name, model) + if shared.opts.detailer_unload: + sd_offload_aux.offload_aux(model_name) + + log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"') + return model_name, model + + +def parse(raw_output: str, image: Image.Image, include_mask: bool = True) -> tuple[str, list[DetailerResult]]: + results = [] + w, h = image.size + bboxes = [] + labels = [] + box_pattern = re.compile(r'(?:|\(|\[)\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*(?:|\)|\])\s*([^<\n,]+)?') + matches = box_pattern.findall(raw_output) + for match in matches: + if len(match) >= 4: + coords = [int(match[i]) for i in range(4)] + label = match[4].strip() if len(match) > 4 and match[4].strip() else "object" + ymin, xmin, ymax, xmax = coords + if max(coords) <= 1000: + xmin = int((xmin / 1000.0) * w) + ymin = int((ymin / 1000.0) * h) + xmax = int((xmax / 1000.0) * w) + ymax = int((ymax / 1000.0) * h) + bboxes.append((xmin, ymin, xmax, ymax)) + labels.append(label) + for box, label in zip(bboxes, labels): + if len(box) == 4: + xmin, ymin, xmax, ymax = map(int, box) + box = (max(0, xmin), max(0, ymin), min(w, xmax), min(h, ymax)) + if box[2] > box[0] and box[3] > box[1]: + mask, cropped = get_mask(box, image, include_mask=include_mask) + result = DetailerResult(box=box, label=label, score=1.0, cls=-1, mask=mask, item=cropped) + log.trace(f'Detailer box: {result}') + results.append(result) + return raw_output, results + + +def predict( + self, + name: str, + image: Image.Image, + device = devices.device, + mask: bool = True, + offload: bool | None = None, + p = None, +) -> list[DetailerResult]: + if offload is None: + offload = shared.opts.detailer_unload + if image is None: + return [] + + cached = sd_offload_aux.get_aux_model(name) + if cached is None: + name, model = load(self, name) + else: + model = cached + + prompt = detailer_opt(p, 'detailer_classes') or '' + text_input = format_rex_prompt(prompt) + messages = [ + { + "role": "user", + "content": [ + {"type": "image", "image": image}, + {"type": "text", "text": text_input}, + ], + } + ] + + log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{text_input}" image={image.size} device={device} mask={mask} offload={offload}') + sd_offload_aux.move_aux_to_gpu(name) + + t0 = time.time() + with devices.llm_context(): + # Use processor directly with images and text to automatically build image tokens and grid thw + text = model.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + inputs = model.processor( + text=[text], + images=[image], + padding=True, + return_tensors="pt", + ) + target_device = model.device + prepared_inputs = {} + for k, v in inputs.items(): + if isinstance(v, torch.Tensor): + v = v.to(device=target_device) + if torch.is_floating_point(v): + v = v.to(dtype=devices.dtype) + prepared_inputs[k] = v + else: + prepared_inputs[k] = v + generated_ids = model.generate( + **prepared_inputs, + max_new_tokens=1024, + do_sample=False, + use_cache=True, + ) + + input_len = prepared_inputs["input_ids"].shape[1] + generated_ids_trimmed = generated_ids[:, input_len:] + generated_text = model.processor.batch_decode(generated_ids_trimmed, skip_special_tokens=False)[0] + + t1 = time.time() + response, results = parse(generated_text, image, include_mask=mask) + + token_count = generated_ids.shape[1] + log.debug(f'Detailer: name="{name}" tokens={token_count} response="{response}" items={len(results)} time={t1-t0:.3f}') + + sd_offload_aux.offload_aux(name) + return results diff --git a/modules/detailer/sam.py b/modules/detailer/sam.py new file mode 100644 index 000000000..9fd662516 --- /dev/null +++ b/modules/detailer/sam.py @@ -0,0 +1,102 @@ +import time +import transformers +from PIL import Image +from modules import shared, devices, sd_offload_aux +from modules.detailer import DetailerResult, detailer_opt, get_mask +from modules.logger import log + + +def load(self, model_name: str | None = None) -> tuple[str, transformers.Sam3Model]: # pylint: disable=unused-argument + cached = sd_offload_aux.get_aux_model(model_name) + if cached is not None: + return model_name, cached + repo_id = model_name.lower().replace('-', '/') + load_kwargs = { + 'pretrained_model_name_or_path': repo_id, + 'cache_dir': shared.opts.hfcache_dir, + 'torch_dtype': devices.dtype, + } + model = transformers.Sam3Model.from_pretrained(**load_kwargs) + model = model.eval() + model.processor = transformers.Sam3Processor.from_pretrained(**load_kwargs) + sd_offload_aux.register_aux(model_name, model) + if shared.opts.detailer_unload: + sd_offload_aux.offload_aux(model_name) + log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"') + return model_name, model + + +def predict( + self, + name: str, + image: Image.Image, + device = devices.device, + mask: bool = True, + offload: bool | None = None, + p = None, +) -> list[DetailerResult]: + if offload is None: + offload = shared.opts.detailer_unload + if image is None: + return [] + cached = sd_offload_aux.get_aux_model(name) + if cached is None: + name, model = load(self, name) + else: + model = cached + if model is None: + return [] + prompt = detailer_opt(p, 'detailer_classes') or '' + if not prompt: + prompt = 'object' + + log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{prompt}" image={image.size} device={device} mask={mask} offload={offload}') + sd_offload_aux.move_aux_to_gpu(name) + + t0 = time.time() + results = [] + + with devices.llm_context(): + inputs = model.processor(images=image, text=prompt, return_tensors="pt") + inputs = inputs.to(model.device) + outputs = model(**inputs) + target_sizes = inputs.get("original_sizes").tolist() if "original_sizes" in inputs else [image.size[::-1]] + results_list = model.processor.post_process_instance_segmentation( + outputs, + threshold=detailer_opt(p, 'detailer_conf') or 0.3, + mask_threshold=0.5, + target_sizes=target_sizes, + ) + + w, h = image.size + if results_list and len(results_list) > 0: + res = results_list[0] + boxes = res.get("boxes", []) + scores = res.get("scores", []) + labels = res.get("labels", []) + masks = res.get("masks", []) if mask else [None] * len(boxes) + for i, box_tensor in enumerate(boxes): + box_coords = box_tensor.tolist() + xmin, ymin, xmax, ymax = map(int, box_coords) + box = (max(0, xmin), max(0, ymin), min(w, xmax), min(h, ymax)) + score = float(scores[i].item()) if i < len(scores) else 1.0 + label = str(labels[i].item()) if i < len(labels) else prompt + masked, cropped = get_mask(box, image, include_mask=mask) + if mask and detailer_opt(p, 'detailer_segmentation') and (i < len(masks) and masks[i] is not None): + masked = Image.fromarray(masks[i].detach().cpu().numpy().astype('uint8') * 255) + cropped = image.crop(box) + result = DetailerResult( + box=box, + label=label, + score=score, + cls=-1, + mask=masked, + item=cropped + ) + results.append(result) + + t1 = time.time() + log.debug(f'Detailer: name="{name}" items={len(results)} time={t1-t0:.3f}') + + sd_offload_aux.offload_aux(name) + return results diff --git a/modules/detailer/yolo.py b/modules/detailer/yolo.py index 45fa26fb4..aedf121a7 100644 --- a/modules/detailer/yolo.py +++ b/modules/detailer/yolo.py @@ -2,8 +2,8 @@ from typing import TYPE_CHECKING import os import threading import numpy as np -from PIL import Image, ImageDraw -from modules.detailer import DetailerResult, detailer_opt +from PIL import Image +from modules.detailer import DetailerResult, detailer_opt, get_mask from modules.logger import log from modules import shared, devices @@ -138,33 +138,19 @@ def predict( if len(desired) > 0 and label.lower() not in desired: continue box = box.tolist() - w, h = box[2] - box[0], box[3] - box[1] - x_size, y_size = w/image.width, h/image.height - opt_min = detailer_opt(p, 'detailer_min_size') or 0 - opt_max = detailer_opt(p, 'detailer_max_size') or 1 - min_size = opt_min if 0 <= opt_min <= 1 else 0 - max_size = opt_max if 0 < opt_max <= 1 else 1 - if x_size >= min_size and y_size >=min_size and x_size <= max_size and y_size <= max_size: - if mask: - if detailer_opt(p, 'detailer_segmentation') and seg is not None: - masked = seg - else: - masked = Image.new('L', image.size, 0) - draw = ImageDraw.Draw(masked) - draw.rectangle(box, fill="white", outline=None, width=0) - cropped = image.crop(box) - res = DetailerResult( - cls=cls, - label=label, - score=round(score, 2), - box=box, - mask=masked, - item=cropped, - width=w, - height=h, - args=args, - ) - result.append(res) + masked, cropped = get_mask(box, image, include_mask=mask) + if detailer_opt(p, 'detailer_segmentation') and seg is not None: + masked = seg + res = DetailerResult( + cls=cls, + label=label, + score=round(score, 2), + box=box, + mask=masked, + item=cropped, + args=args, + ) + result.append(res) if len(result) >= (detailer_opt(p, 'detailer_max') or 2): break return result diff --git a/modules/img2img.py b/modules/img2img.py index c739a1501..c6210cd16 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -161,7 +161,7 @@ def img2img(id_task: str, state: str, mode: int, sampler_index, mask_blur, mask_alpha, vae_type, tiling, hidiffusion, - detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes, n_iter, batch_size, guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end, @@ -280,6 +280,7 @@ def img2img(id_task: str, state: str, mode: int, detailer_steps=detailer_steps, detailer_strength=detailer_strength, detailer_resolution=detailer_resolution, + detailer_classes=detailer_classes, init_images=[image], mask=mask, mask_blur=mask_blur, diff --git a/modules/sd_offload_aux.py b/modules/sd_offload_aux.py index 32e0c0d68..f77f58755 100644 --- a/modules/sd_offload_aux.py +++ b/modules/sd_offload_aux.py @@ -94,7 +94,10 @@ def offload_aux(name: str) -> None: def get_aux_model(name: str) -> torch.nn.Module | None: - entry = aux_models.get(name) + entry = aux_models.get(name, None) if entry is None: + entry = aux_models.get(name.lower(), None) + if entry is None: + # log.warning(f'Model not found: requested="{name}" available={list(aux_models.keys())}') return None return entry.model diff --git a/modules/txt2img.py b/modules/txt2img.py index 6edf03982..c79433443 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -14,7 +14,7 @@ def txt2img(id_task, state, prompt, negative_prompt, prompt_styles, steps, sampler_index, hr_sampler_index, vae_type, tiling, hidiffusion, - detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes, n_iter, batch_size, guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end, @@ -83,6 +83,7 @@ def txt2img(id_task, state, detailer_steps=detailer_steps, detailer_strength=detailer_strength, detailer_resolution=detailer_resolution, + detailer_classes=detailer_classes, tiling=tiling, hidiffusion=hidiffusion, enable_hr=enable_hr, diff --git a/modules/ui_control.py b/modules/ui_control.py index 951e0ba4b..dd07aaa03 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -207,7 +207,7 @@ def create_ui(_blocks: gr.Blocks=None): video_type, video_duration, video_loop, video_pad, video_interpolate = create_video_inputs(tab='control') enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('control') - detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution = shared.detailer.ui('control') + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes = shared.detailer.ui('control') with gr.Row(): override_script_name = gr.State(value='', visible=False, elem_id='control_override_script_name') @@ -311,7 +311,7 @@ def create_ui(_blocks: gr.Blocks=None): seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, clip_skip, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end, vae_type, tiling, hidiffusion, - detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes, hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio, hdr_apply_hires, grading_brightness, grading_contrast, grading_saturation, grading_hue, grading_gamma, grading_sharpness, grading_color_temp, grading_shadows, grading_midtones, grading_highlights, grading_clahe_clip, grading_clahe_grid, diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index b944ce3cb..20890ed18 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -142,7 +142,7 @@ def create_ui(): grading_brightness, grading_contrast, grading_saturation, grading_hue, grading_gamma, grading_sharpness, grading_color_temp, grading_shadows, grading_midtones, grading_highlights, grading_clahe_clip, grading_clahe_grid, grading_shadows_tint, grading_highlights_tint, grading_split_tone_balance, grading_vignette, grading_grain, grading_lut_file, grading_lut_strength = ui_sections.create_color_inputs('img2img') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio, hdr_apply_hires = ui_sections.create_latent_inputs('img2img') enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, hr_refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('img2img') - detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution = shared.detailer.ui('img2img') + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes = shared.detailer.ui('img2img') # with gr.Group(elem_id="inpaint_controls", visible=False) as inpaint_controls: with gr.Accordion(open=False, label="Mask", elem_classes=["small-accordion"], elem_id="img2img_mask_group") as inpaint_controls: @@ -181,7 +181,7 @@ def create_ui(): sampler_index, mask_blur, mask_alpha, vae_type, tiling, hidiffusion, - detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes, batch_count, batch_size, guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end, diff --git a/modules/ui_txt2img.py b/modules/ui_txt2img.py index 141f16847..d639eb356 100644 --- a/modules/ui_txt2img.py +++ b/modules/ui_txt2img.py @@ -38,7 +38,7 @@ def create_ui(): grading_brightness, grading_contrast, grading_saturation, grading_hue, grading_gamma, grading_sharpness, grading_color_temp, grading_shadows, grading_midtones, grading_highlights, grading_clahe_clip, grading_clahe_grid, grading_shadows_tint, grading_highlights_tint, grading_split_tone_balance, grading_vignette, grading_grain, grading_lut_file, grading_lut_strength = ui_sections.create_color_inputs('txt2img') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio, hdr_apply_hires = ui_sections.create_latent_inputs('txt2img') enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') - detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution = shared.detailer.ui('txt2img') + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes = shared.detailer.ui('txt2img') override_settings = ui_common.create_override_inputs('txt2img') state = gr.Textbox(value='', visible=False) @@ -56,7 +56,7 @@ def create_ui(): txt2img_prompt, txt2img_negative_prompt, txt2img_prompt_styles, steps, sampler_index, hr_sampler_index, vae_type, tiling, hidiffusion, - detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, detailer_classes, batch_count, batch_size, guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end, diff --git a/scripts/postprocessing_detailer.py b/scripts/postprocessing_detailer.py index 771f2bd7f..b267c959c 100644 --- a/scripts/postprocessing_detailer.py +++ b/scripts/postprocessing_detailer.py @@ -12,7 +12,7 @@ class ScriptPostprocessingDetailer(scripts_postprocessing.ScriptPostprocessing): # The detailer accordion (built by yolo.ui) now contains the Sampler sub-accordion too, so for 'extras' # it returns a 7th element: a dict of the sampler-block controls. Spread it into the control map; their # values are stamped onto the synthetic p in process()/make_processing(), applying to this pass only. - enabled, prompt, negative, steps, strength, resolution, sampler_block = shared.detailer.ui('extras') + enabled, prompt, negative, steps, strength, resolution, classes, sampler_block = shared.detailer.ui('extras') return { "enabled": enabled, "prompt": prompt, @@ -20,11 +20,12 @@ class ScriptPostprocessingDetailer(scripts_postprocessing.ScriptPostprocessing): "steps": steps, "strength": strength, "resolution": resolution, + "classes": classes, **sampler_block, } - def process(self, pp: scripts_postprocessing.PostprocessedImage, # pylint: disable=arguments-differ - enabled=False, prompt='', negative='', steps=10, strength=0.3, resolution=1024, + def process(self, pp: scripts_postprocessing.PostprocessedImage, # pylint: disable=arguments-differ + enabled=False, prompt='', negative='', steps=10, strength=0.3, resolution=1024, classes='', sampler='Default', prediction='default', shift=3.0, cfg_scale=6.0, options=None, seed=-1): if not enabled: return pp @@ -52,7 +53,16 @@ class ScriptPostprocessingDetailer(scripts_postprocessing.ScriptPostprocessing): 'schedulers_rescale_betas': 'rescale' in options, } log.info(f'Detailer postprocess: strength={strength} steps={steps} resolution={resolution} sampler={sampler} cfg={cfg_scale}') - p = shared.detailer.make_processing(pp.image, prompt=prompt, negative=negative, steps=steps, strength=strength, resolution=resolution, seed=int(seed) if seed is not None else -1, overrides=overrides) + p = shared.detailer.make_processing(pp.image, + prompt=prompt, + negative=negative, + steps=steps, + strength=strength, + resolution=resolution, + classes=classes, + seed=int(seed) if seed is not None else -1, + overrides=overrides, + ) try: result = shared.detailer.restore(np.array(pp.image), p)