refactor detailer

Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
2026-07-04 11:17:56 +02:00
parent d62a0690a2
commit 2c632bb562
11 changed files with 339 additions and 215 deletions
+7 -3
View File
@@ -1,8 +1,8 @@
# Change Log for SD.Next
## Update for 2026-07-03
## Update for 2026-07-04
### Highlights for 2026-07-03
### Highlights for 2026-07-04
Service-pack update with number of fixes and quality-of-life improvements
Plus few new models: **Krea 2**, **Photoroom PRXPixel**, **FLUX.2 Klein 9B KV** and some new community models
@@ -11,7 +11,7 @@ Also couple of *experimental* features: see below for details...
[Home](https://vladmandic.github.io/sdnext/) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic)
### Details for 2026-07-03
### Details for 2026-07-04
- **Models**
- [Krea 2](https://www.krea.ai/blog/krea-2-image-model) in *base* and *turbo* (distilled) variants
@@ -38,6 +38,10 @@ Also couple of *experimental* features: see below for details...
- prompt encode caching for pass-through text-encoders
- reference models: validate and update info for all reference models
add size preview before download for all reference models
- **CivitAI** download improvements
auto-select download path
improved search
validate downloads
- **UI**
- dynamic visibility of image controls
- improve main panel positioning: *portrait/landscape*
+8
View File
@@ -0,0 +1,8 @@
from .models import detailer_models
from .helper import detailer_opt, DetailerResult, list_models
from .detailer import Detailer
def initialize():
from modules import shared
shared.detailer = Detailer()
@@ -1,60 +1,17 @@
from typing import TYPE_CHECKING
import os
import re
import threading
from copy import copy
import numpy as np
import gradio as gr
from PIL import Image, ImageDraw
from modules.logger import log
from modules import shared, processing, devices, processing_class, ui_common, ui_components, ui_symbols, images, extra_networks, sd_models
def detailer_opt(p, attr, opts_attr=None):
"""Read detailer param from processing object if set, otherwise fall back to shared.opts."""
if p is not None:
val = getattr(p, attr, None)
if val is not None:
return val
return getattr(shared.opts, opts_attr or attr, None)
predefined = [ # <https://huggingface.co/vladmandic/yolo-detailers/tree/main>
'https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11m.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/face-yolo8n.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/face-yolo8m.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/hand_yolov8n.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/person_yolov8n-seg.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/eyes-v1.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/eyes-full-v1.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/anzhc-eyes-seg.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/anzhc-face-1024-seg-8n.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/anzhc-head-seg-8n.pt',
]
load_lock = threading.Lock()
class DetailerResult:
def __init__(self, cls: int, label: str, score: float, box: list[int], mask: Image.Image = None, item: Image.Image = None, width = 0, height = 0, args = None):
if args is None:
args = {}
self.cls = cls
self.label = label
self.score = score
self.box = box
self.mask = mask
self.item = item
self.width = width
self.height = height
self.args = args
def __repl__(self):
return f'cls={self.cls} label={self.label} score={self.score} box={self.box} mask={self.mask} item={self.item} size={self.width}x{self.height} args={self.args}'
from modules.detailer import DetailerResult, detailer_opt
class Detailer():
def __init__(self):
super().__init__()
self.model_name = None
self.models = {} # cache loaded models
self.list = {}
self.ui_mode = True
@@ -64,32 +21,9 @@ class Detailer():
def name(self):
return "Detailer"
def enumerate(self):
self.list.clear()
files = []
downloaded = 0
for m in predefined:
name = os.path.splitext(os.path.basename(m))[0]
self.list[name] = m
files.append(name)
if os.path.exists(shared.opts.yolo_dir):
for f in os.listdir(shared.opts.yolo_dir):
if f.endswith('.pt'):
downloaded += 1
name = os.path.splitext(os.path.basename(f))[0]
if name not in files:
self.list[name] = os.path.join(shared.opts.yolo_dir, f)
log.info(f'Available Detailer: path="{shared.opts.yolo_dir}" items={len(list(self.list))} downloaded={downloaded}')
return list(self.list)
def dependencies(self):
from installer import install
install('ultralytics==8.4.67', ignore=True, quiet=True)
install('omegaconf')
install('antlr4-python3-runtime')
def predict(
self,
name: str,
model,
image: Image.Image,
imgsz: int = 640,
@@ -102,141 +36,24 @@ class Detailer():
offload: bool | None = None,
p = None,
) -> list[DetailerResult]:
if augment is None:
augment = detailer_opt(p, 'detailer_augment')
if offload is None:
offload = shared.opts.detailer_unload
if 'LocateAnything' in name:
from modules.detailer import locateanything
return locateanything.predict(self, model, image, device=device, mask=mask, offload=offload, p=p)
if model is None or (isinstance(model, str) and len(model) == 0):
model = 'yolo11m'
result = []
if isinstance(model, str):
cached = self.models.get(model, None)
if cached is None:
_, model = self.load(model)
else:
model = cached
if model is None:
return result
args = {
'conf': detailer_opt(p, 'detailer_conf'),
'iou': detailer_opt(p, 'detailer_iou'),
# 'max_det': detailer_opt(p, 'detailer_max'),
}
try:
if TYPE_CHECKING:
from ultralytics import YOLO # pylint: disable=import-outside-toplevel, unused-import
model: YOLO = model.to(device)
predictions = model.predict(
source=[image],
stream=False,
verbose=False,
imgsz=imgsz,
half=half,
device=device,
augment=augment,
agnostic_nms=agnostic,
retina_masks=retina,
**args
)
if offload:
model.to('cpu')
except Exception as e:
log.error(f'Detailer predict: {e}')
return result
from modules.detailer import yolo
return yolo.predict(self, model, image, imgsz=imgsz, half=half, device=device, agnostic=agnostic, retina=retina, mask=mask, augment=augment, offload=offload, p=p)
classes = detailer_opt(p, 'detailer_classes') or ''
desired = classes.split(',')
desired = [d.lower().strip() for d in desired]
desired = [d for d in desired if len(d) > 0]
for prediction in predictions:
boxes = prediction.boxes.xyxy.detach().int().cpu().numpy() if prediction.boxes is not None else []
scores = prediction.boxes.conf.detach().float().cpu().numpy() if prediction.boxes is not None else []
classes = prediction.boxes.cls.detach().float().cpu().numpy() if prediction.boxes is not None else []
masks = prediction.masks.data.cpu().float().numpy() if prediction.masks is not None else []
if len(masks) < len(classes):
masks = len(classes) * [None]
for score, box, cls, seg in zip(scores, boxes, classes, masks, strict=False):
if seg is not None:
try:
seg = (255 * seg).astype(np.uint8)
seg = Image.fromarray(seg).resize(image.size).convert('L')
except Exception:
seg = None
cls = int(cls)
label = prediction.names[cls] if cls < len(prediction.names) else f'cls{cls}'
if len(desired) > 0 and label.lower() not in desired:
continue
box = box.tolist()
w, h = box[2] - box[0], box[3] - box[1]
x_size, y_size = w/image.width, h/image.height
opt_min = detailer_opt(p, 'detailer_min_size') or 0
opt_max = detailer_opt(p, 'detailer_max_size') or 1
min_size = opt_min if 0 <= opt_min <= 1 else 0
max_size = opt_max if 0 < opt_max <= 1 else 1
if x_size >= min_size and y_size >=min_size and x_size <= max_size and y_size <= max_size:
if mask:
if detailer_opt(p, 'detailer_segmentation') and seg is not None:
masked = seg
else:
masked = Image.new('L', image.size, 0)
draw = ImageDraw.Draw(masked)
draw.rectangle(box, fill="white", outline=None, width=0)
cropped = image.crop(box)
res = DetailerResult(
cls=cls,
label=label,
score=round(score, 2),
box=box,
mask=masked,
item=cropped,
width=w,
height=h,
args=args,
)
result.append(res)
if len(result) >= (detailer_opt(p, 'detailer_max') or 2):
break
return result
def enumerate(self):
from modules.detailer import list_models
return list_models(self)
def load(self, model_name: str | None = None):
with load_lock:
from modules import modelloader
model = None
if model_name is None:
model_name = list(self.list)[0]
if model_name in self.models:
return model_name, self.models[model_name]
else:
model_url = self.list.get(model_name, None)
if model_url is None:
log.error(f'Load: type=Detailer name="{model_name}" error="model not found"')
return None, None
file_name = os.path.basename(model_url)
model_file = None
try:
model_file = modelloader.load_file_from_url(url=model_url, model_dir=shared.opts.yolo_dir, file_name=file_name)
if model_file is None:
log.error(f'Load: type=Detailer name="{model_name}" url="{model_url}" error="failed to fetch model"')
elif model_file.endswith('.onnx'):
import onnxruntime as ort
options = ort.SessionOptions()
# options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
session = ort.InferenceSession(model_file, sess_options=options, providers=devices.onnx)
self.models[model_name] = session
return model_name, session
else:
self.dependencies()
import ultralytics
model = ultralytics.YOLO(model_file)
classes = list(model.names.values())
log.info(f'Load: type=Detailer name="{model_name}" model="{model_file}" ultralytics={ultralytics.__version__} classes={classes}')
self.models[model_name] = model
return model_name, model
except Exception as e:
log.error(f'Load: type=Detailer name="{model_name}" error="{e}"')
return None, None
if 'LocateAnything' in model_name:
from modules.detailer import locateanything
return locateanything.load(model_name=model_name)
from modules.detailer import yolo
return yolo.load(self, model_name=model_name)
def merge(self, items: list[DetailerResult]) -> list[DetailerResult]:
if items is None or len(items) == 0:
@@ -333,7 +150,7 @@ class Detailer():
if image is None:
image = Image.fromarray(np_image)
items = self.predict(model, image, p=p)
items = self.predict(name, model, image, p=p)
if len(items) == 0:
log.info(f'Detailer: model="{name}" no items detected')
@@ -643,8 +460,3 @@ class Detailer():
if tab == 'extras':
return enabled, prompt, negative, steps, strength, resolution, sampler_block
return enabled, prompt, negative, steps, strength, resolution
def initialize():
shared.detailer = Detailer()
# shared.detailers.append(shared.detailer)
+52
View File
@@ -0,0 +1,52 @@
import os
from PIL import Image
from modules.logger import log
def list_models(self):
from modules.detailer import detailer_models
from modules import shared
self.list.clear()
files = []
downloaded = 0
for m in detailer_models:
name = os.path.splitext(os.path.basename(m))[0]
self.list[name] = m
files.append(name)
if os.path.exists(shared.opts.yolo_dir):
for f in os.listdir(shared.opts.yolo_dir):
if f.endswith('.pt'):
downloaded += 1
name = os.path.splitext(os.path.basename(f))[0]
if name not in files:
self.list[name] = os.path.join(shared.opts.yolo_dir, f)
log.info(f'Available Detailer: path="{shared.opts.yolo_dir}" items={len(list(self.list))} downloaded={downloaded}')
return list(self.list)
def detailer_opt(p, attr, opts_attr=None):
"""Read detailer param from processing object if set, otherwise fall back to shared.opts."""
from modules import shared
if p is not None:
val = getattr(p, attr, None)
if val is not None:
return val
return getattr(shared.opts, opts_attr or attr, None)
class DetailerResult:
def __init__(self, cls: int, label: str, score: float, box: list[int], mask: Image.Image = None, item: Image.Image = None, width = 0, height = 0, args = None):
if args is None:
args = {}
self.cls = cls
self.label = label
self.score = score
self.box = box
self.mask = mask
self.item = item
self.width = width
self.height = height
self.args = args
def __str__(self):
return f'DetailerResult(cls={self.cls} label={self.label} score={self.score:.2f} box={self.box} size={self.width}x{self.height} args={self.args})'
+65
View File
@@ -0,0 +1,65 @@
from PIL import Image
from modules import shared, devices
from modules.detailer import DetailerResult, detailer_opt
from modules.logger import log
repo_id = 'nvidia/LocateAnything-3B'
processor = None
tokenizer = None
def dependencies():
from installer import install
install('decord')
def load(model_name: str | None = None):
import transformers
global tokenizer, processor # pylint: disable=global-statement
load_kwargs = {
'pretrained_model_name_or_path': repo_id,
'cache_dir': shared.opts.hfcache_dir,
'trust_remote_code': True,
}
dependencies()
tokenizer = transformers.AutoTokenizer.from_pretrained(**load_kwargs)
processor = transformers.AutoProcessor.from_pretrained(**load_kwargs)
model = transformers.AutoModel.from_pretrained(**load_kwargs, torch_dtype=devices.dtype)
model = model.to(devices.device).eval()
if shared.opts.detailer_unload:
model.to(devices.cpu)
log.info(f'Detailer model="{model_name}" cls={model.__class__.__name__} loaded')
return model_name, model
def predict(
self,
model,
image: Image.Image,
device = devices.device,
mask: bool = True,
offload: bool | None = None,
p = None,
) -> list[DetailerResult]:
if offload is None:
offload = shared.opts.detailer_unload
log.info(f'Detailer cls="{model.__class__.__name__}" image={image} device={device} mask={mask} offload={offload}')
result = []
if isinstance(model, str):
cached = self.models.get(model, None)
if cached is None:
_, model = self.load(model)
else:
model = cached
if model is None:
return result
model = model.to(device)
prompt = detailer_opt(p, 'detailer_classes') or ''
log.debug(f'Detailer prompt="{prompt}"')
# TODO locateanything: implement when compatible with transformers=5
if offload:
model.to(devices.cpu)
return result
+13
View File
@@ -0,0 +1,13 @@
detailer_models = [ # <https://huggingface.co/vladmandic/yolo-detailers/tree/main>
'https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11m.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/face-yolo8n.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/face-yolo8m.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/hand_yolov8n.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/person_yolov8n-seg.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/eyes-v1.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/eyes-full-v1.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/anzhc-eyes-seg.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/anzhc-face-1024-seg-8n.pt',
'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/anzhc-head-seg-8n.pt',
# 'nvidia-LocateAnything-3B',
]
+170
View File
@@ -0,0 +1,170 @@
from typing import TYPE_CHECKING
import os
import threading
import numpy as np
from PIL import Image, ImageDraw
from modules.detailer import DetailerResult, detailer_opt
from modules.logger import log
from modules import shared, devices
load_lock = threading.Lock()
def dependencies():
from installer import install
install('ultralytics==8.4.67', ignore=True, quiet=True)
install('omegaconf')
install('antlr4-python3-runtime')
def load(self, model_name: str | None = None):
with load_lock:
from modules import modelloader
model = None
if model_name is None:
model_name = list(self.list)[0]
if model_name in self.models:
return model_name, self.models[model_name]
else:
model_url = self.list.get(model_name, None)
if model_url is None:
log.error(f'Load: type=Detailer name="{model_name}" error="model not found"')
return None, None
file_name = os.path.basename(model_url)
model_file = None
try:
model_file = modelloader.load_file_from_url(url=model_url, model_dir=shared.opts.yolo_dir, file_name=file_name)
if model_file is None:
log.error(f'Load: type=Detailer name="{model_name}" url="{model_url}" error="failed to fetch model"')
elif model_file.endswith('.onnx'):
import onnxruntime as ort
options = ort.SessionOptions()
# options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
session = ort.InferenceSession(model_file, sess_options=options, providers=devices.onnx)
self.models[model_name] = session
return model_name, session
else:
dependencies()
import ultralytics
model = ultralytics.YOLO(model_file)
classes = list(model.names.values())
log.info(f'Load: type=Detailer name="{model_name}" model="{model_file}" ultralytics={ultralytics.__version__} classes={classes}')
self.models[model_name] = model
return model_name, model
except Exception as e:
log.error(f'Load: type=Detailer name="{model_name}" error="{e}"')
return None, None
def predict(
self,
model,
image: Image.Image,
imgsz: int = 640,
half: bool = True,
device = devices.device,
agnostic: bool = False,
retina: bool = False,
mask: bool = True,
augment: bool | None = None,
offload: bool | None = None,
p = None,
) -> list[DetailerResult]:
if augment is None:
augment = detailer_opt(p, 'detailer_augment')
if offload is None:
offload = shared.opts.detailer_unload
if model is None or (isinstance(model, str) and len(model) == 0):
model = 'yolo11m'
result = []
if isinstance(model, str):
cached = self.models.get(model, None)
if cached is None:
_, model = self.load(model)
else:
model = cached
if model is None:
return result
args = {
'conf': detailer_opt(p, 'detailer_conf'),
'iou': detailer_opt(p, 'detailer_iou'),
# 'max_det': detailer_opt(p, 'detailer_max'),
}
try:
if TYPE_CHECKING:
from ultralytics import YOLO # pylint: disable=import-outside-toplevel, unused-import
model: YOLO = model.to(device)
predictions = model.predict(
source=[image],
stream=False,
verbose=False,
imgsz=imgsz,
half=half,
device=device,
augment=augment,
agnostic_nms=agnostic,
retina_masks=retina,
**args
)
if offload:
model.to('cpu')
except Exception as e:
log.error(f'Detailer predict: {e}')
return result
classes = detailer_opt(p, 'detailer_classes') or ''
desired = classes.split(',')
desired = [d.lower().strip() for d in desired]
desired = [d for d in desired if len(d) > 0]
for prediction in predictions:
boxes = prediction.boxes.xyxy.detach().int().cpu().numpy() if prediction.boxes is not None else []
scores = prediction.boxes.conf.detach().float().cpu().numpy() if prediction.boxes is not None else []
classes = prediction.boxes.cls.detach().float().cpu().numpy() if prediction.boxes is not None else []
masks = prediction.masks.data.cpu().float().numpy() if prediction.masks is not None else []
if len(masks) < len(classes):
masks = len(classes) * [None]
for score, box, cls, seg in zip(scores, boxes, classes, masks, strict=False):
if seg is not None:
try:
seg = (255 * seg).astype(np.uint8)
seg = Image.fromarray(seg).resize(image.size).convert('L')
except Exception:
seg = None
cls = int(cls)
label = prediction.names[cls] if cls < len(prediction.names) else f'cls{cls}'
if len(desired) > 0 and label.lower() not in desired:
continue
box = box.tolist()
w, h = box[2] - box[0], box[3] - box[1]
x_size, y_size = w/image.width, h/image.height
opt_min = detailer_opt(p, 'detailer_min_size') or 0
opt_max = detailer_opt(p, 'detailer_max_size') or 1
min_size = opt_min if 0 <= opt_min <= 1 else 0
max_size = opt_max if 0 < opt_max <= 1 else 1
if x_size >= min_size and y_size >=min_size and x_size <= max_size and y_size <= max_size:
if mask:
if detailer_opt(p, 'detailer_segmentation') and seg is not None:
masked = seg
else:
masked = Image.new('L', image.size, 0)
draw = ImageDraw.Draw(masked)
draw.rectangle(box, fill="white", outline=None, width=0)
cropped = image.crop(box)
res = DetailerResult(
cls=cls,
label=label,
score=round(score, 2),
box=box,
mask=masked,
item=cropped,
width=w,
height=h,
args=args,
)
result.append(res)
if len(result) >= (detailer_opt(p, 'detailer_max') or 2):
break
return result
+2 -1
View File
@@ -127,9 +127,10 @@ def crop_images(images, crops):
if crops[i]:
cropped = []
for image in images[i]:
faces = shared.detailer.predict('face-yolo8n', image)
faces = shared.detailer.predict('face-yolo8n', 'face-yolo8n', image)
if len(faces) > 0:
cropped.append(faces[0].item)
log.debug(f'IP adapter crop: {faces[0]}')
if len(cropped) == len(images[i]):
images[i] = cropped
else:
+2 -2
View File
@@ -320,13 +320,13 @@ def process_samples(p: StableDiffusionProcessing, samples):
sample = shared.detailer.restore(sample, p)
if isinstance(sample, list):
if len(sample) > 0:
image = Image.fromarray(sample[0])
image = sample[0] if isinstance(sample[0], Image.Image) else Image.fromarray(sample[0])
if len(sample) > 1:
annotated = sample[1] if isinstance(sample[1], Image.Image) else Image.fromarray(sample[1])
out_images.append(annotated)
out_infotexts.append("Detailer annotations")
elif sample is not None:
image = Image.fromarray(sample)
image = sample if isinstance(sample, Image.Image) else Image.fromarray(sample)
if p.color_corrections is not None and i < len(p.color_corrections):
p.ops.append('color')
+2 -2
View File
@@ -30,6 +30,7 @@ if TYPE_CHECKING:
from diffusers import DiffusionPipeline
from modules.shared_legacy import LegacyOption
from modules.ui_extra_networks import ExtraNetworksPage
from modules.detailer import Detailer
class Backend(Enum):
@@ -48,8 +49,7 @@ listfiles = listdir
xformers_available = False
compiled_model_state = None
sd_upscalers = []
detailers = []
detailer = None
detailer: Detailer | None = None
tab_names = []
extra_networks: list[ExtraNetworksPage] = []
hypernetworks = {}
-1
View File
@@ -800,7 +800,6 @@ def create_settings(cmd_opts):
"uni_pc_variant": OptionInfo("bh2", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"], "visible": False}),
"uni_pc_skip_type": OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"], "visible": False}),
# detailer settings are handled separately
# "detailer_model": OptionInfo("Detailer", "Detailer model", gr.Radio, lambda: {"choices": [x.name() for x in shared.detailers], "visible": False}),
"detailer_classes": OptionInfo("", "Detailer classes", gr.Textbox, {"visible": False}),
"detailer_conf": OptionInfo(0.6, "Min confidence", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.05, "visible": False}),
"detailer_max": OptionInfo(2, "Max detected", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1, "visible": False}),