diff --git a/.pylintrc b/.pylintrc index 2e737148f..7bf2523ce 100644 --- a/.pylintrc +++ b/.pylintrc @@ -24,6 +24,7 @@ ignore-paths=/usr/lib/.*$, modules/unipc, modules/xadapter, modules/dcsolver, + modules/ctrlx, modules/vdm, modules/apg, repositories, diff --git a/.ruff.toml b/.ruff.toml index a4f37328a..5731fde69 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -67,6 +67,7 @@ ignore = [ "E731", # Do not assign a `lambda` expression, use a `def` "E741", # Ambiguous variable name "F401", # Imported by unused + "NPY002", # replace legacy random "RUF005", # Consider iterable unpacking "RUF010", # Use explicit conversion flag "RUF012", # Mutable class attributes diff --git a/extensions-builtin/Lora/lora_convert.py b/extensions-builtin/Lora/lora_convert.py index ae44bc458..be2ef356c 100644 --- a/extensions-builtin/Lora/lora_convert.py +++ b/extensions-builtin/Lora/lora_convert.py @@ -338,10 +338,10 @@ def _convert_kohya_flux_lora_to_diffusers(state_dict): ait_sd.update({k: down_weight for k in ait_down_keys}) # up_weight is split to each split - ait_sd.update({k: v for k, v in zip(ait_up_keys, torch.split(up_weight, dims, dim=0))}) # noqa: C416 + ait_sd.update({k: v for k, v in zip(ait_up_keys, torch.split(up_weight, dims, dim=0))}) # noqa: C416 # pylint: disable=unnecessary-comprehension else: # down_weight is chunked to each split - ait_sd.update({k: v for k, v in zip(ait_down_keys, torch.chunk(down_weight, num_splits, dim=0))}) # noqa: C416 + ait_sd.update({k: v for k, v in zip(ait_down_keys, torch.chunk(down_weight, num_splits, dim=0))}) # noqa: C416 # pylint: disable=unnecessary-comprehension # up_weight is sparse: only non-zero values are copied to each split i = 0 diff --git a/html/locale_en.json b/html/locale_en.json index 800ee6e58..3c1ac70a4 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -119,11 +119,11 @@ {"id":"","label":"Steps","localized":"","hint":"How many times to improve the generated image iteratively; higher values take longer; very low values can produce bad results"}, {"id":"","label":"Tiling","localized":"","hint":"Produce an image that can be tiled"}, {"id":"","label":"full quality","localized":"","hint":"Use full quality VAE to decode latent samples"}, - {"id":"","label":"face restore","localized":"","hint":"Run processed image through additional face restoration model"}, + {"id":"","label":"detailer","localized":"","hint":"Run processed image through additional detailer model"}, {"id":"","label":"hidiffusion","localized":"","hint":"HiDiffusion allows creation of high-resolution images using your standard models without duplicates/distortions and improved performance"}, {"id":"","label":"HDR Clamp","localized":"","hint":"Adjusts the level of nonsensical details by pruning values that deviate significantly from the distribution mean. It is particularly useful for enhancing generation at higher guidance scales, identifying outliers early in the process and applying mathematical adjustments based on the Range (Boundary) and Threshold settings. Think of it as setting the range within which you want your image values to be, and adjusting the threshold determines which values should be brought back into that range"}, {"id":"","label":"HDR Maximize","localized":"","hint":"Calculates a 'normalization factor' by dividing the maximum tensor value by the specified range multiplied by 4. This factor is then used to shift the channels within the given boundary, ensuring maximum dynamic range for subsequent processing. The objective is to optimize dynamic range for external applications like Photoshop, particularly for adjusting levels, contrast, and brightness"}, - {"id":"","label":"Enable second pass","localized":"","hint":"Use a similar process as image to image to upscale and/or add detail to the final image. Optionally uses refiner model to enhance image details."}, + {"id":"","label":"Enable refine pass","localized":"","hint":"Use a similar process as image to image to upscale and/or add detail to the final image. Optionally uses refiner model to enhance image details."}, {"id":"","label":"Denoising strength","localized":"","hint":"Determines how little respect the algorithm should have for image's content. At 0, nothing will change, and at 1 you'll get an unrelated image. With values below 1.0, processing will take less steps than the Sampling Steps slider specifies"}, {"id":"","label":"Denoise start","localized":"","hint":"Override denoise strength by stating how early base model should finish and when refiner should start. Only applicable to refiner usage. If set to 0 or 1, denoising strength will be used"}, {"id":"","label":"Hires steps","localized":"","hint":"Number of sampling steps for upscaled picture. If 0, uses same as for original"}, @@ -132,13 +132,13 @@ {"id":"","label":"Force Hires","localized":"","hint":"Hires runs automatically when Latent upscale is selected, but its skipped when using non-latent upscalers. Enable force hires to run hires with non-latent upscalers"}, {"id":"","label":"Resize width to","localized":"","hint":"Resizes image to this width. If 0, width is inferred from either of two nearby sliders"}, {"id":"","label":"Resize height to","localized":"","hint":"Resizes image to this height. If 0, height is inferred from either of two nearby sliders"}, - {"id":"","label":"Secondary sampler","localized":"","hint":"Use specific sampler as fallback sampler if primary is not supported for specific operation"}, + {"id":"","label":"Refine sampler","localized":"","hint":"Use specific sampler as fallback sampler if primary is not supported for specific operation"}, {"id":"","label":"Refiner start","localized":"","hint":"Refiner pass will start when base model is this much complete (set to larger than 0 and smaller than 1 to run after full base model run)"}, {"id":"","label":"Refiner steps","localized":"","hint":"Number of steps to use for refiner pass"}, - {"id":"","label":"Secondary CFG Scale","localized":"","hint":"CFG scale used for refiner pass"}, + {"id":"","label":"Refine CFG Scale","localized":"","hint":"CFG scale used for refiner pass"}, {"id":"","label":"Rescale guidance","localized":"","hint":"Rescale CFG generated noise to avoid overexposed images"}, - {"id":"","label":"Secondary Prompt","localized":"","hint":"Prompt used for both second encoder in base model (if it exists) and for refiner pass (if enabled)"}, - {"id":"","label":"Secondary negative prompt","localized":"","hint":"Negative prompt used for both second encoder in base model (if it exists) and for refiner pass (if enabled)"}, + {"id":"","label":"Refine Prompt","localized":"","hint":"Prompt used for both second encoder in base model (if it exists) and for refiner pass (if enabled)"}, + {"id":"","label":"Refine negative prompt","localized":"","hint":"Negative prompt used for both second encoder in base model (if it exists) and for refiner pass (if enabled)"}, {"id":"","label":"Width","localized":"","hint":"Image width"}, {"id":"","label":"Height","localized":"","hint":"Image height"}, {"id":"","label":"Batch count","localized":"","hint":"How many batches of images to create (has no impact on generation performance or VRAM usage)"}, @@ -179,7 +179,7 @@ {"id":"","label":"Show result images","localized":"","hint":"Enable to show the processed images in the image pane"}, {"id":"","label":"Resize","localized":"","hint":"Resizing details. Higher resolutions require additional processing memory."}, {"id":"","label":"Crop to fit","localized":"","hint":"If the dimensions of your source image (e.g. 512x510) deviate from your target dimensions (e.g. 1024x768) this function will fit your upscaled image into your target size image. Excess will be cropped"}, - {"id":"","label":"Secondary Upscaler","localized":"","hint":"Select secondary upscaler to run after initial upscaler"}, + {"id":"","label":"Refine Upscaler","localized":"","hint":"Select secondary upscaler to run after initial upscaler"}, {"id":"","label":"Upscaler 2 visibility","localized":"","hint":"Strength of the secondary upscaler"} ], "models tabs": [ diff --git a/javascript/imageViewer.js b/javascript/imageViewer.js index 4cebde21c..c8a2dfd65 100644 --- a/javascript/imageViewer.js +++ b/javascript/imageViewer.js @@ -252,10 +252,10 @@ async function initImageViewer() { modalToggleParamsBtn.title = 'Toggle Parameters'; modalToggleParamsBtn.addEventListener('click', modalToggleParams, true); -// exif -const modalExif = document.createElement('div'); -modalExif.id = 'modalExif'; -modalExif.style = 'position: absolute; bottom: 0px; width: 100%; background-color: rgba(0, 0, 0, 0.5); color: var(--neutral-300); padding: 1em; font-size: small; line-height: 1.2em; z-index: 1; display: none;'; + // exif + const modalExif = document.createElement('div'); + modalExif.id = 'modalExif'; + modalExif.style = 'position: absolute; bottom: 0px; width: 100%; background-color: rgba(0, 0, 0, 0.5); color: var(--neutral-300); padding: 1em; font-size: small; line-height: 1.2em; z-index: 1; display: none;'; // handlers modalPreviewZone.addEventListener('mousedown', () => { previewDrag = false; }); diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 0ce893e2f..879999549 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -1,6 +1,6 @@ @font-face { font-family: 'NotoSans'; font-display: swap; font-style: normal; font-weight: 100; src: local('NotoSans'), url('notosans-nerdfont-regular.ttf') } :root { - --left-column: 520px; + --left-column: 530px; --color-trace: #666666; --color-debug: #7F7F7F; --color-info: #D4D4D4; @@ -83,6 +83,7 @@ button.custom-button { border-radius: var(--button-large-radius); padding: var(- .performance .time { margin-right: 0; } .thumbnails { background: var(--body-background-fill); } .control-image { height: calc(100vw/3) !important; } +.prompt textarea { resize: vertical; } #control_results { margin: 0; padding: 0; } #control_gallery { height: calc(100vw/3 + 60px); } #txt2img_gallery, #img2img_gallery { height: 50vh; } diff --git a/modules/api/endpoints.py b/modules/api/endpoints.py index 310abed1e..08b54c9df 100644 --- a/modules/api/endpoints.py +++ b/modules/api/endpoints.py @@ -91,13 +91,13 @@ def post_interrogate(req: models.ReqInterrogate): if req.model not in get_clip_models(): raise HTTPException(status_code=404, detail="Model not found") try: - caption = interrogate_image(image, model=req.model, mode=req.mode) + caption = interrogate_image(image, clip_model=req.clip_model, blip_model=req.blip_model, mode=req.mode) except Exception as e: caption = str(e) if not req.analyze: return models.ResInterrogate(caption=caption) else: - medium, artist, movement, trending, flavor = analyze_image(image, model=req.model) + medium, artist, movement, trending, flavor = analyze_image(image, clip_model=req.clip_model, blip_model=req.blip_model) return models.ResInterrogate(caption=caption, medium=medium, artist=artist, movement=movement, trending=trending, flavor=flavor) def post_vqa(req: models.ReqVQA): diff --git a/modules/api/models.py b/modules/api/models.py index 6e658b484..cd67522d1 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -259,8 +259,8 @@ class ReqProcess(BaseModel): upscaling_resize_h: int = Field(default=512, title="Target Height", ge=1, description="Target height for the upscaler to hit. Only used when resize_mode=1.") upscaling_crop: bool = Field(default=True, title="Crop to fit", description="Should the upscaler crop the image to fit in the chosen size?") upscaler_1: str = Field(default="None", title="Main upscaler", description=f"The name of the main upscaler to use, it has to be one of this list: {' , '.join([x.name for x in shared.sd_upscalers])}") - upscaler_2: str = Field(default="None", title="Secondary upscaler", description=f"The name of the secondary upscaler to use, it has to be one of this list: {' , '.join([x.name for x in shared.sd_upscalers])}") - extras_upscaler_2_visibility: float = Field(default=0, title="Secondary upscaler visibility", ge=0, le=1, allow_inf_nan=False, description="Sets the visibility of secondary upscaler, values should be between 0 and 1.") + upscaler_2: str = Field(default="None", title="Refine upscaler", description=f"The name of the secondary upscaler to use, it has to be one of this list: {' , '.join([x.name for x in shared.sd_upscalers])}") + extras_upscaler_2_visibility: float = Field(default=0, title="Refine upscaler visibility", ge=0, le=1, allow_inf_nan=False, description="Sets the visibility of secondary upscaler, values should be between 0 and 1.") upscale_first: bool = Field(default=False, title="Upscale first", description="Should the upscaler run before restoring faces?") class ResProcess(BaseModel): @@ -302,7 +302,8 @@ class ResProgress(BaseModel): class ReqInterrogate(BaseModel): image: str = Field(default="", title="Image", description="Image to work on, must be a Base64 string containing the image's data.") - model: str = Field(default="clip", title="Model", description="The interrogate model used.") + clip_model: str = Field(default="", title="CLiP Model", description="The interrogate model used.") + blip_model: str = Field(default="", title="BLiP Model", description="The interrogate model used.") class ResInterrogate(BaseModel): caption: Optional[str] = Field(default=None, title="Caption", description="The generated caption for the image.") diff --git a/modules/api/process.py b/modules/api/process.py index 505ba1374..dcb3b6487 100644 --- a/modules/api/process.py +++ b/modules/api/process.py @@ -107,7 +107,7 @@ class APIProcess(): return ResMask(mask=image) def post_face(self, req: ReqFace): - from scripts.detailer import yolo # pylint: disable=no-name-in-module + from shared import yolo # pylint: disable=no-name-in-module image = decode_base64_to_image(req.image) shared.state.begin('API-FACE', api=True) images = [] diff --git a/modules/control/util.py b/modules/control/util.py index b33062d5e..31741c6a1 100644 --- a/modules/control/util.py +++ b/modules/control/util.py @@ -35,7 +35,7 @@ def HWC3(x): def make_noise_disk(H, W, C, F): - noise = np.random.uniform(low=0, high=1, size=((H // F) + 2, (W // F) + 2, C)) # noqa + noise = np.random.uniform(low=0, high=1, size=((H // F) + 2, (W // F) + 2, C)) noise = cv2.resize(noise, (W + 2 * F, H + 2 * F), interpolation=cv2.INTER_CUBIC) noise = noise[F: F + H, F: F + W] noise -= np.min(noise) diff --git a/modules/ctrlx/__init__.py b/modules/ctrlx/__init__.py index caa0e44a1..87ff52f6c 100644 --- a/modules/ctrlx/__init__.py +++ b/modules/ctrlx/__init__.py @@ -412,10 +412,7 @@ class CtrlXStableDiffusionXLPipeline(StableDiffusionXLPipeline): # diffusers==0 and denoising_value_valid(self.denoising_start) and self.denoising_start >= self.denoising_end ): - raise ValueError( - f"`denoising_start`: {self.denoising_start} cannot be larger than or equal to `denoising_end`: " - + f" {self.denoising_end} when using type float." - ) + raise ValueError(f"`denoising_start`: {self.denoising_start} cannot be larger than or equal to `denoising_end`: {self.denoising_end} when using type float.") elif self.denoising_end is not None and denoising_value_valid(self.denoising_end): discrete_timestep_cutoff = int( round( diff --git a/modules/ctrlx/sdxl.py b/modules/ctrlx/sdxl.py index 4b964e7e8..700e7df7f 100644 --- a/modules/ctrlx/sdxl.py +++ b/modules/ctrlx/sdxl.py @@ -38,8 +38,8 @@ def convolution_forward( # From self, input_tensor: torch.Tensor, temb: torch.Tensor, - *args, - **kwargs, + *args, # pylint: disable=unused-argument + **kwargs, # pylint: disable=unused-argument ) -> torch.Tensor: do_structure_control = self.do_control and self.t in self.structure_schedule @@ -108,7 +108,7 @@ class AttnProcessor2_0: # From 1 else 0 global previous_oom # pylint: disable=global-statement threshold = 0 if (cmd_opts.lowvram and not cmd_opts.use_zluda) else opts.torch_gc_threshold + collected = 0 if force or threshold == 0 or used_gpu >= threshold or used_ram >= threshold: force = True if oom > previous_oom: diff --git a/modules/ipadapter.py b/modules/ipadapter.py index a92a30459..80c1a9b7f 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -83,7 +83,7 @@ def crop_images(images, crops): try: for i in range(len(images)): if crops[i]: - from scripts.detailer import yolo # pylint: disable=no-name-in-module + from shared import yolo # pylint: disable=no-name-in-module yolo.load() cropped = [] for image in images[i]: diff --git a/modules/postprocess/yolo.py b/modules/postprocess/yolo.py new file mode 100644 index 000000000..82626fdd6 --- /dev/null +++ b/modules/postprocess/yolo.py @@ -0,0 +1,312 @@ +import os +import numpy as np +import gradio as gr +from PIL import Image, ImageDraw +from modules import shared, processing, devices, processing_class, ui_common +from modules.detailer import Detailer + + +PREDEFINED = [ # + 'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/face-yolo8n.pt', + 'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/hand_yolov8n.pt', + 'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/person_yolov8n-seg.pt', + 'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/eyes-v1.pt', + 'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/eyes-full-v1.pt', +] + + +class YoloResult: + def __init__(self, score: float, box: list[int], mask: Image.Image = None, item: Image.Image = None, size: float = 0, width = 0, height = 0, args = {}): + self.score = score + self.box = box + self.mask = mask + self.item = item + self.size = size + self.width = width + self.height = height + self.args = args + + +class YoloRestorer(Detailer): + def __init__(self): + super().__init__() + self.models = {} # cache loaded models + self.list = {} + self.enumerate() + + def name(self): + return "Detailer" + + def enumerate(self): + self.list.clear() + files = [] + downloaded = 0 + for m in PREDEFINED: + name = os.path.splitext(os.path.basename(m))[0] + self.list[name] = m + files.append(name) + if os.path.exists(shared.opts.yolo_dir): + for f in os.listdir(shared.opts.yolo_dir): + if f.endswith('.pt'): + downloaded += 1 + name = os.path.splitext(os.path.basename(f))[0] + if name not in files: + self.list[name] = os.path.join(shared.opts.yolo_dir, f) + shared.log.info(f'Available Yolo: path="{shared.opts.yolo_dir} items={len(list(self.list))} downloaded={downloaded}') + return self.list + + def dependencies(self): + import installer + installer.install('ultralytics', ignore=True, quiet=True) + + def predict( + self, + model, + image: Image.Image, + imgsz: int = 640, + half: bool = True, + device = devices.device, + augment: bool = True, + agnostic: bool = False, + retina: bool = False, + mask: bool = True, + offload: bool = shared.opts.detailer_unload, + ) -> list[YoloResult]: + + result = [] + if model is None: + return result + args = { + 'conf': shared.opts.detailer_conf, + 'iou': shared.opts.detailer_iou, + 'max_det': shared.opts.detailer_max, + } + try: + model.to(device) + predictions = model.predict( + source=[image], + stream=False, + verbose=False, + imgsz=imgsz, + half=half, + device=device, + augment=augment, + agnostic_nms=agnostic, + retina_masks=retina, + **args + ) + if offload: + model.to('cpu') + except Exception as e: + shared.log.error(f'Detailer predict: {e}') + return result + + for prediction in predictions: + boxes = prediction.boxes.xyxy.detach().int().cpu().numpy() if prediction.boxes is not None else [] + scores = prediction.boxes.conf.detach().float().cpu().numpy() if prediction.boxes is not None else [] + for score, box in zip(scores, boxes): + box = box.tolist() + mask_image = None + w, h = box[2] - box[0], box[3] - box[1] + size = w * h / (image.width * image.height) + if (min(w, h) > shared.opts.detailer_min_size if shared.opts.detailer_min_size > 0 else True) and (max(w, h) < shared.opts.detailer_max_size if shared.opts.detailer_max_size > 0 else True): + if mask: + mask_image = image.copy() + mask_image = Image.new('L', image.size, 0) + draw = ImageDraw.Draw(mask_image) + draw.rectangle(box, fill="white", outline=None, width=0) + cropped = image.crop(box) + result.append(YoloResult(score=round(score, 2), box=box, mask=mask_image, item=cropped, size=size, width=w, height=h, args=args)) + return result + + def load(self, model_name: str = None): + from modules import modelloader + self.dependencies() + if model_name is None: + model_name = list(self.list)[0] + if model_name in self.models: + return model_name, self.models[model_name] + else: + model_url = self.list.get(model_name) + file_name = os.path.basename(model_url) + model_file = None + try: + model_file = modelloader.load_file_from_url(url=model_url, model_dir=shared.opts.yolo_dir, file_name=file_name) + if model_file is not None: + shared.log.info(f'Load: type=Detailer name="{model_name}" model="{model_file}"') + from ultralytics import YOLO # pylint: disable=import-outside-toplevel + model = YOLO(model_file) + self.models[model_name] = model + return model_name, model + except Exception as e: + shared.log.error(f'Load: type=Detailer name="{model_name}" error="{e}"') + return None + + def restore(self, np_image, p: processing.StableDiffusionProcessing = None): + if hasattr(p, 'recursion'): + return np_image + if not hasattr(p, 'detailer_active'): + p.detailer_active = 0 + if np_image is None or p.detailer_active >= p.batch_size * p.n_iter: + return np_image + if len(shared.opts.detailer_models) == 0: + shared.log.warning('Detailer: model=None') + return np_image + models_used = [] + # create backups + orig_apply_overlay = shared.opts.mask_apply_overlay + orig_p = p.__dict__.copy() + orig_cls = p.__class__ + + for i, model_name in enumerate(shared.opts.detailer_models): + name, model = self.load(model_name) + if model is None: + shared.log.warning(f'Detailer: model="{name}" not loaded') + continue + + image = Image.fromarray(np_image) + items = self.predict(model, image) + if len(items) == 0: + shared.log.info(f'Detailer: model="{name}" no items detected') + continue + + pp = None + shared.opts.data['mask_apply_overlay'] = True + resolution = 512 if shared.sd_model_type in ['none', 'sd', 'lcm', 'unknown'] else 1024 + prompt: str = orig_p.get('refiner_prompt', '') + negative: str = orig_p.get('refiner_negative', '') + if len(prompt) == 0: + prompt = orig_p.get('all_prompts', [''])[0] + if len(negative) == 0: + negative = orig_p.get('all_negative_prompts', [''])[0] + prompt_lines = prompt.split('\n') + negative_lines = negative.split('\n') + prompt = prompt_lines[i % len(prompt_lines)] + negative = negative_lines[i % len(negative_lines)] + + args = { + 'detailer': True, + 'batch_size': 1, + 'n_iter': 1, + 'prompt': prompt, + 'negative_prompt': negative, + 'denoising_strength': shared.opts.detailer_strength, + 'sampler_name': orig_p.get('hr_sampler_name', 'default'), + 'steps': orig_p.get('refiner_steps', 0), + 'styles': [], + 'inpaint_full_res': True, + 'inpainting_mask_invert': 0, + 'inpainting_fill': 1, # no fill + 'mask_blur': shared.opts.detailer_blur, + 'inpaint_full_res_padding': shared.opts.detailer_padding, + 'width': resolution, + 'height': resolution, + } + if args['denoising_strength'] == 0: + shared.log.debug(f'Detailer: model="{name}" strength=0 skip') + return np_image + control_pipeline = None + orig_class = shared.sd_model.__class__ + if getattr(p, 'is_control', False): + from modules.control import run + control_pipeline = shared.sd_model + run.restore_pipeline() + + p = processing_class.switch_class(p, processing.StableDiffusionProcessingImg2Img, args) + p.detailer_active += 1 # set flag to avoid recursion + + if p.steps < 1: + p.steps = orig_p.get('steps', 0) + + report = [{'score': i.score, 'size': f'{i.width}x{i.height}' } for i in items] + shared.log.info(f'Detailer: model="{name}" items={report} args={items[0].args} denoise={p.denoising_strength} blur={p.mask_blur} width={p.width} height={p.height} padding={p.inpaint_full_res_padding}') + shared.log.debug(f'Detailer: prompt="{prompt}" negative="{negative}"') + models_used.append(name) + + mask_all = [] + p.state = '' + for item in items: + if item.mask is None: + continue + p.init_images = [image] + p.image_mask = [item.mask] + # mask_all.append(item.mask) + p.recursion = True + pp = processing.process_images_inner(p) + del p.recursion + p.overlay_images = None # skip applying overlay twice + if pp is not None and pp.images is not None and len(pp.images) > 0: + image = pp.images[0] # update image to be reused for next item + if len(pp.images) > 1: + mask_all.append(pp.images[1]) + + # restore pipeline + if control_pipeline is not None: + shared.sd_model = control_pipeline + else: + shared.sd_model.__class__ = orig_class + p = processing_class.switch_class(p, orig_cls, orig_p) + p.init_images = orig_p.get('init_images', None) + p.image_mask = orig_p.get('image_mask', None) + p.state = orig_p.get('state', None) + p.ops = orig_p.get('ops', []) + shared.opts.data['mask_apply_overlay'] = orig_apply_overlay + np_image = np.array(image) + + if len(mask_all) > 0 and shared.opts.include_mask: + from modules.control.util import blend + p.image_mask = blend([np.array(m) for m in mask_all]) + # combined = blend([np_image, p.image_mask]) + # combined = Image.fromarray(combined) + # combined.save('/tmp/item.png') + p.image_mask = Image.fromarray(p.image_mask) + + shared.log.debug(f'Detailer processed: models={models_used}') + return np_image + + def ui(self, tab: str): + def ui_settings_change(detailers, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou): + shared.opts.detailer_models = detailers + shared.opts.detailer_strength = strength + shared.opts.detailer_padding = padding + shared.opts.detailer_blur = blur + shared.opts.detailer_conf = min_confidence + shared.opts.detailer_max = max_detected + shared.opts.detailer_min_size = min_size + shared.opts.detailer_max_size = max_size + shared.opts.detailer_iou = iou + shared.log.debug(f'Detailer settings: models={shared.opts.detailer_models} strength={shared.opts.detailer_strength} conf={shared.opts.detailer_conf} max={shared.opts.detailer_max} iou={shared.opts.detailer_iou} size={shared.opts.detailer_min_size}-{shared.opts.detailer_max_size} padding={shared.opts.detailer_padding}') + + with gr.Accordion(open=False, label="Detailer", elem_id=f"{tab}_detailer", elem_classes=["small-accordion"], visible=shared.native): + with gr.Row(): + enabled = gr.Checkbox(label="Enable detailer pass", elem_id=f"{tab}_detailer", value=False) + with gr.Row(): + detailers = gr.Dropdown(label="Detailers", elem_id=f"{tab}_detailers", choices=self.list, value=shared.opts.detailer_models, multiselect=True) + ui_common.create_refresh_button(detailers, self.enumerate, {}, elem_id=f"{tab}_detailers_refresh") + with gr.Row(): + strength = gr.Slider(label="Detailer strength", elem_id=f"{tab}_detailer_strength", value=shared.opts.detailer_strength, minimum=0, maximum=1, step=0.01) + max_detected = gr.Slider(label="Max detected", elem_id=f"{tab}_detailer_max", value=shared.opts.detailer_max, min=1, maximum=10, step=1) + with gr.Row(): + padding = gr.Slider(label="Edge padding", elem_id=f"{tab}_detailer_padding", value=shared.opts.detailer_padding, minimum=0, maximum=100, step=1) + blur = gr.Slider(label="Edge blur", elem_id=f"{tab}_detailer_blur", value=shared.opts.detailer_blur, minimum=0, maximum=100, step=1) + with gr.Row(): + min_confidence = gr.Slider(label="Min confidence", elem_id=f"{tab}_detailer_conf", value=shared.opts.detailer_conf, minimum=0.0, maximum=1.0, step=0.05) + iou = gr.Slider(label="Max overlap", elem_id=f"{tab}_detailer_iou", value=shared.opts.detailer_iou, minimum=0, maximum=1.0, step=0.05) + with gr.Row(): + min_size = gr.Slider(label="Min size", elem_id=f"{tab}_detailer_min_size", value=shared.opts.detailer_min_size, minimum=0, maximum=1024, step=1) + max_size = gr.Slider(label="Max size", elem_id=f"{tab}_detailer_max_size", value=shared.opts.detailer_max_size, minimum=0, maximum=1024, step=1) + detailers.change(fn=ui_settings_change, inputs=[detailers, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) + strength.change(fn=ui_settings_change, inputs=[detailers, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) + padding.change(fn=ui_settings_change, inputs=[detailers, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) + blur.change(fn=ui_settings_change, inputs=[detailers, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) + min_confidence.change(fn=ui_settings_change, inputs=[detailers, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) + max_detected.change(fn=ui_settings_change, inputs=[detailers, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) + min_size.change(fn=ui_settings_change, inputs=[detailers, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) + max_size.change(fn=ui_settings_change, inputs=[detailers, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) + iou.change(fn=ui_settings_change, inputs=[detailers, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) + return enabled + + +def initialize(): + shared.yolo = YoloRestorer() + shared.detailers.append(shared.yolo) diff --git a/modules/processing.py b/modules/processing.py index e0b89ef47..d582806cb 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -348,11 +348,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: for i, sample in enumerate(samples): debug(f'Processing result: index={i+1}/{len(samples)} iteration={n+1}/{p.n_iter}') p.batch_index = i - if len(infotexts) > i: - info = infotexts[i] - else: - info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i, all_negative_prompts=p.negative_prompts) - infotexts.append(info) if type(sample) == Image.Image: image = sample sample = np.array(sample) @@ -361,8 +356,9 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: image = Image.fromarray(sample) if p.detailer: if not p.do_not_save_samples and shared.opts.save_images_before_detailer: - images.save_image(Image.fromarray(sample), path=p.outpath_samples, basename="", seed=p.seeds[i], prompt=p.prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix="-before-face-restore") - p.ops.append('face') + info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i) + images.save_image(Image.fromarray(sample), path=p.outpath_samples, basename="", seed=p.seeds[i], prompt=p.prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix="-before-detailer") + p.ops.append('detailer') sample = detailer.detail(sample, p) if sample is not None: image = Image.fromarray(sample) @@ -377,14 +373,22 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: p.color_corrections = None p.color_corrections = orig image_without_cc = apply_overlay(image, p.paste_to, i, p.overlay_images) + info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i) images.save_image(image_without_cc, path=p.outpath_samples, basename="", seed=p.seeds[i], prompt=p.prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix="-before-color-correct") p.ops.append('color') image = apply_color_correction(p.color_corrections[i], image) if shared.opts.mask_apply_overlay: image = apply_overlay(image, p.paste_to, i, p.overlay_images) + + if len(infotexts) > i: + info = infotexts[i] + else: + info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i, all_negative_prompts=p.negative_prompts) + infotexts.append(info) image.info["parameters"] = info output_images.append(image) if shared.opts.samples_save and not p.do_not_save_samples and p.outpath_samples is not None: + info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i) images.save_image(image, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image if hasattr(p, 'mask_for_overlay') and p.mask_for_overlay and any([shared.opts.save_mask, shared.opts.save_mask_composite, shared.opts.return_mask, shared.opts.return_mask_composite]): image_mask = p.mask_for_overlay.convert('RGB') @@ -400,6 +404,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: output_images.append(image_mask) if shared.opts.return_mask_composite: output_images.append(image_mask_composite) + timer.process.record('post') del samples devices.torch_gc() diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index 6da4511f5..dcd016913 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -569,4 +569,5 @@ def update_sampler(p, sd_model, second_pass=False): sampler_options.append('dynamic') if 'lower_order_final' in sampler.config and shared.opts.schedulers_use_loworder != shared.opts.data_labels.get('schedulers_use_loworder').default: sampler_options.append('low order') - p.extra_generation_params['Sampler options'] = '/'.join(sampler_options) + if len(sampler_options) > 0: + p.extra_generation_params['Sampler options'] = '/'.join(sampler_options) diff --git a/modules/processing_info.py b/modules/processing_info.py index b0f4f20d8..7337b0fe4 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -123,8 +123,8 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No args['Size mode mask'] = p.resize_mode_mask args['Size scale mask'] = p.scale_by_mask args['Size name mask'] = p.resize_name_mask - if 'face' in p.ops: - args["Face restoration"] = shared.opts.detailer_model + if 'detailer' in p.ops: + args["Detailer"] = ', '.join(shared.opts.detailer_models) if 'color' in p.ops: args["Color correction"] = True # embeddings diff --git a/modules/shared.py b/modules/shared.py index aba93bbaf..94635f6ed 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -484,7 +484,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "ipex_optimize": OptionInfo([], "IPEX Optimize for Intel GPUs", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"], "visible": devices.backend == "ipex"}), "openvino_sep": OptionInfo("

OpenVINO

", "", gr.HTML, {"visible": cmd_opts.use_openvino}), - "openvino_devices": OptionInfo([], "OpenVINO devices to use", gr.CheckboxGroup, {"choices": get_openvino_device_list() if cmd_opts.use_openvino else [], "visible": cmd_opts.use_openvino}), # pylint disable:possibly-used-before-assignment + "openvino_devices": OptionInfo([], "OpenVINO devices to use", gr.CheckboxGroup, {"choices": get_openvino_device_list() if cmd_opts.use_openvino else [], "visible": cmd_opts.use_openvino}), # pylint disable:E0606 "nncf_quantize": OptionInfo([], "OpenVINO Quantize Models with NNCF", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": cmd_opts.use_openvino}), "nncf_quant_mode": OptionInfo("INT8", "OpenVINO quantization mode for NNCF", gr.Radio, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}), "nncf_compress_weights_mode": OptionInfo("INT8", "OpenVINO compress mode for NNCF", gr.Radio, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'], "visible": cmd_opts.use_openvino}), @@ -784,18 +784,18 @@ options_templates.update(options_section(('postprocessing', "Postprocessing"), { "img2img_extra_noise": OptionInfo(0.0, "Extra noise multiplier for img2img", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), "CLIP_stop_at_last_layers": OptionInfo(1, "Clip skip", gr.Slider, {"minimum": 1, "maximum": 8, "step": 1, "visible": False}), - "postprocessing_sep_detailer": OptionInfo("

Detailer

", "", gr.HTML), - "detailer_model": OptionInfo("Detailer", "Detailer model", gr.Radio, lambda: {"choices": [x.name() for x in detailers]}), - "detailer_sep": OptionInfo("

Detailer

", "", gr.HTML), - "detailer_conf": OptionInfo(0.6, "Min confidence", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.05}), - "detailer_max": OptionInfo(5, "Max detected", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), - "detailer_iou": OptionInfo(0.5, "Max overlap", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.05}), - "detailer_min_size": OptionInfo(0, "Min object size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 1}), - "detailer_max_size": OptionInfo(0, "Max object size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 1}), - "detailer_padding": OptionInfo(20, "Object padding", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), - "detailer_strength": OptionInfo(0.0, "Detailer strength", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}), - "detailer_models": OptionInfo(['Face yolo-8n'], "Detailer models", gr.Dropdown, lambda: {"multiselect":True, "choices": list(yolo.list)}), - "code_former_weight": OptionInfo(0.2, "CodeFormer weight parameter", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}), + # "postprocessing_sep_detailer": OptionInfo("

Detailer

", "", gr.HTML), + "detailer_model": OptionInfo("Detailer", "Detailer model", gr.Radio, lambda: {"choices": [x.name() for x in detailers], "visible": False}), + "detailer_conf": OptionInfo(0.6, "Min confidence", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.05, "visible": False}), + "detailer_max": OptionInfo(2, "Max detected", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1, "visible": False}), + "detailer_iou": OptionInfo(0.5, "Max overlap", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.05, "visible": False}), + "detailer_min_size": OptionInfo(0, "Min object size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 1, "visible": False}), + "detailer_max_size": OptionInfo(0, "Max object size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 1, "visible": False}), + "detailer_padding": OptionInfo(20, "Item padding", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1, "visible": False}), + "detailer_blur": OptionInfo(10, "Item edge blur", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1, "visible": False}), + "detailer_strength": OptionInfo(0.5, "Detailer strength", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False}), + "detailer_models": OptionInfo(['face-yolo8n'], "Detailer models", gr.Dropdown, lambda: {"multiselect":True, "choices": list(yolo.list), "visible": False}), + "code_former_weight": OptionInfo(0.2, "CodeFormer weight parameter", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False}), "detailer_unload": OptionInfo(False, "Move detailer model to CPU when complete"), "postprocessing_sep_upscalers": OptionInfo("

Upscaling

", "", gr.HTML), diff --git a/modules/textual_inversion/image_embedding.py b/modules/textual_inversion/image_embedding.py index c9fe40767..74eb88fd2 100644 --- a/modules/textual_inversion/image_embedding.py +++ b/modules/textual_inversion/image_embedding.py @@ -182,7 +182,7 @@ if __name__ == '__main__': new_image = Image.new('RGBA', (512, 512), (255, 255, 200, 255)) cap_image = caption_image_overlay(new_image, 'title', 'footerLeft', 'footerMid', 'footerRight') - test_embed = {'string_to_param': {'*': torch.from_numpy(np.random.random((2, 4096)))}} # noqa: NPY002 + test_embed = {'string_to_param': {'*': torch.from_numpy(np.random.random((2, 4096)))}} embedded_image = insert_image_data_embed(cap_image, test_embed) diff --git a/modules/ui_control.py b/modules/ui_control.py index cb7ae2921..d7cfcdf0a 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -102,7 +102,6 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Row(elem_id='control_settings'): - full_quality, detailer, tiling, hidiffusion = ui_sections.create_options('control') state = gr.Textbox(value='', visible=False) with gr.Accordion(open=False, label="Input", elem_id="control_input", elem_classes=["small-accordion"]): @@ -132,7 +131,7 @@ def create_ui(_blocks: gr.Blocks=None): mask_controls = masking.create_segment_ui() - cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_sections.create_advanced_inputs('control') + full_quality, tiling, hidiffusion, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_sections.create_advanced_inputs('control') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('control') with gr.Accordion(open=False, label="Video", elem_id="control_video", elem_classes=["small-accordion"]): @@ -148,6 +147,7 @@ def create_ui(_blocks: gr.Blocks=None): video_type.change(fn=helpers.video_type_change, inputs=[video_type], outputs=[video_duration, video_loop, video_pad, video_interpolate]) enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('control') + detailer = shared.yolo.ui('control') with gr.Row(): override_settings = ui_common.create_override_inputs('control') diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index ec37f5e23..d46ea4dd3 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -129,9 +129,9 @@ def create_ui(): denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Denoising strength', value=0.50, elem_id="img2img_denoising_strength") refiner_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.0, elem_id="img2img_refiner_start") - cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_sections.create_advanced_inputs('img2img') - full_quality, detailer, tiling, hidiffusion = ui_sections.create_options('img2img') + full_quality, tiling, hidiffusion, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_sections.create_advanced_inputs('img2img') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('img2img') + detailer = shared.yolo.ui('img2img') # with gr.Group(elem_id="inpaint_controls", visible=False) as inpaint_controls: with gr.Accordion(open=False, label="Mask", elem_classes=["small-accordion"], elem_id="img2img_mask_group") as inpaint_controls: diff --git a/modules/ui_sections.py b/modules/ui_sections.py index aa076b1ea..27ea2d328 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -146,15 +146,6 @@ def create_seed_inputs(tab, reuse_visible=True): return seed, reuse_seed, subseed, reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w -def create_options(tab): - with gr.Row(elem_id=f"{tab}_advanced_options"): - full_quality = gr.Checkbox(label='Full quality', value=True, elem_id=f"{tab}_full_quality") - detailer = gr.Checkbox(label='Detailer', value=False, elem_id=f"{tab}_detailer") - tiling = gr.Checkbox(label='Tiling', value=False, elem_id=f"{tab}_tiling") - hidiffusion = gr.Checkbox(label='HiDiffusion', value=False, elem_id=f"{tab}_hidiffusion") - return full_quality, detailer, tiling, hidiffusion - - def create_cfg_inputs(tab): with gr.Row(): cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id=f"{tab}_cfg_scale") @@ -165,19 +156,23 @@ def create_cfg_inputs(tab): def create_advanced_inputs(tab, base=True): with gr.Accordion(open=False, label="Advanced", elem_id=f"{tab}_advanced", elem_classes=["small-accordion"]): with gr.Group(): + with gr.Row(elem_id=f"{tab}_advanced_options"): + full_quality = gr.Checkbox(label='Full quality', value=True, elem_id=f"{tab}_full_quality") + tiling = gr.Checkbox(label='Tiling', value=False, elem_id=f"{tab}_tiling") + hidiffusion = gr.Checkbox(label='HiDiffusion', value=False, elem_id=f"{tab}_hidiffusion") if base: cfg_scale, cfg_end = create_cfg_inputs(tab) else: cfg_scale, cfg_end = None, None with gr.Row(): - image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Secondary guidance', value=6.0, elem_id=f"{tab}_image_cfg_scale") + image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Refine guidance', value=6.0, elem_id=f"{tab}_image_cfg_scale") diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Rescale guidance', value=0.7, elem_id=f"{tab}_image_cfg_rescale", visible=shared.native) with gr.Row(): diffusers_pag_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.05, label='Attention guidance', value=0.0, elem_id=f"{tab}_pag_scale", visible=shared.native) diffusers_pag_adaptive = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Adaptive scaling', value=0.5, elem_id=f"{tab}_pag_adaptive", visible=shared.native) with gr.Row(): clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=0, maximum=12, step=0.1, elem_id=f"{tab}_clip_skip", interactive=True) - return cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, diffusers_pag_scale, diffusers_pag_adaptive, cfg_end + return full_quality, tiling, hidiffusion, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, diffusers_pag_scale, diffusers_pag_adaptive, cfg_end def create_correction_inputs(tab): @@ -312,7 +307,7 @@ def create_hires_inputs(tab): with gr.Accordion(open=False, label="Refine", elem_id=f"{tab}_second_pass", elem_classes=["small-accordion"]): with gr.Group(): with gr.Row(elem_id=f"{tab}_hires_row1"): - enable_hr = gr.Checkbox(label='Enable second pass', value=False, elem_id=f"{tab}_enable_hr") + enable_hr = gr.Checkbox(label='Enable refine pass', value=False, elem_id=f"{tab}_enable_hr") """ with gr.Row(elem_id=f"{tab}_hires_fix_row1", variant="compact"): hr_upscaler = gr.Dropdown(label="Upscaler", elem_id=f"{tab}_hr_upscaler", choices=[*shared.latent_upscale_modes, *[x.name for x in shared.sd_upscalers]], value=shared.latent_upscale_default_mode) @@ -324,18 +319,18 @@ def create_hires_inputs(tab): hr_resize_mode, hr_upscaler, hr_resize_context, hr_resize_x, hr_resize_y, hr_scale, _selected_scale_tab = create_resize_inputs(tab, None, accordion=False, latent=True, non_zero=False) with gr.Row(elem_id=f"{tab}_hires_fix_row2", variant="compact"): hr_force = gr.Checkbox(label='Force HiRes', value=False, elem_id=f"{tab}_hr_force") - hr_sampler_index = gr.Dropdown(label='Secondary sampler', elem_id=f"{tab}_sampling_alt", choices=[x.name for x in sd_samplers.samplers], value='Same as primary', type="index") + hr_sampler_index = gr.Dropdown(label='Refine sampler', elem_id=f"{tab}_sampling_alt", choices=[x.name for x in sd_samplers.samplers], value='Same as primary', type="index") with gr.Row(elem_id=f"{tab}_hires_row2"): hr_second_pass_steps = gr.Slider(minimum=0, maximum=99, step=1, label='HiRes steps', elem_id=f"{tab}_steps_alt", value=20) denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Strength', value=0.3, elem_id=f"{tab}_denoising_strength") with gr.Group(visible=shared.native): with gr.Row(elem_id=f"{tab}_refiner_row1", variant="compact"): refiner_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Refiner start', value=0.0, elem_id=f"{tab}_refiner_start") - refiner_steps = gr.Slider(minimum=0, maximum=99, step=1, label="Refiner steps", elem_id=f"{tab}_refiner_steps", value=10) + refiner_steps = gr.Slider(minimum=0, maximum=99, step=1, label="Refiner steps", elem_id=f"{tab}_refiner_steps", value=20) with gr.Row(elem_id=f"{tab}_refiner_row3", variant="compact"): - refiner_prompt = gr.Textbox(value='', label='Secondary prompt', elem_id=f"{tab}_refiner_prompt") + refiner_prompt = gr.Textbox(value='', lines=2, label='Refine prompt', elem_id=f"{tab}_refiner_prompt", elem_classes=["prompt"]) with gr.Row(elem_id="txt2img_refiner_row4", variant="compact"): - refiner_negative = gr.Textbox(value='', label='Secondary negative prompt', elem_id=f"{tab}_refiner_neg_prompt") + refiner_negative = gr.Textbox(value='', lines=2, label='Refine negative prompt', elem_id=f"{tab}_refiner_neg_prompt", elem_classes=["prompt"]) return enable_hr, hr_sampler_index, denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative diff --git a/modules/ui_txt2img.py b/modules/ui_txt2img.py index 171bc9ac1..f61196997 100644 --- a/modules/ui_txt2img.py +++ b/modules/ui_txt2img.py @@ -39,15 +39,15 @@ def create_ui(): batch_count, batch_size = ui_sections.create_batch_inputs('txt2img', accordion=False) cfg_scale, cfg_end = ui_sections.create_cfg_inputs('txt2img') steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "txt2img") - full_quality, detailer, tiling, hidiffusion = ui_sections.create_options('txt2img') with gr.Group(elem_classes="settings-accordion"): with gr.Accordion(open=False, label="Samplers", elem_classes=["small-accordion"], elem_id="txt2img_sampler_group"): ui_sections.create_sampler_options('txt2img') seed, reuse_seed, subseed, reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w = ui_sections.create_seed_inputs('txt2img') - _cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, _cfg_end = ui_sections.create_advanced_inputs('txt2img', base=False) + full_quality, tiling, hidiffusion, _cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, _cfg_end = ui_sections.create_advanced_inputs('txt2img', base=False) hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('txt2img') enable_hr, hr_sampler_index, denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') + detailer = shared.yolo.ui('txt2img') override_settings = ui_common.create_override_inputs('txt2img') state = gr.Textbox(value='', visible=False) diff --git a/scripts/detailer.py b/scripts/detailer.py deleted file mode 100644 index e9f94667d..000000000 --- a/scripts/detailer.py +++ /dev/null @@ -1,233 +0,0 @@ -import os -import numpy as np -from PIL import Image, ImageDraw -from modules import shared, processing -from modules.detailer import Detailer -from modules import devices, processing_class - - -PREDEFINED = { # - 'Face yolo-8n': 'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/face-yolo8n.pt', - 'Eyefull paired v2': 'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/eyeful-paired-v2.pt', -} - - -class YoloResult: - def __init__(self, score: float, box: list[int], mask: Image.Image = None, item: Image.Image = None, size: float = 0, width = 0, height = 0, args = {}): - self.score = score - self.box = box - self.mask = mask - self.item = item - self.size = size - self.width = width - self.height = height - self.args = args - - -class YoloRestorer(Detailer): - def __init__(self): - super().__init__() - self.models = {} - self.list = {} - self.enumerate() - - def name(self): - return "Detailer" - - def enumerate(self): - self.list.clear() - files = [] - for k, v in PREDEFINED.items(): - self.list[k] = v - files.append(os.path.basename(v)) - for f in os.listdir(shared.opts.yolo_dir): - if f not in files: - name = os.path.basename(f) - self.list[name] = os.path.join(shared.opts.yolo_dir, f) - shared.log.info(f'Available Yolo: path="{shared.opts.yolo_dir} items={len(list(self.list))}') - - def dependencies(self): - import installer - installer.install('ultralytics', ignore=True, quiet=True) - - def predict( - self, - model, - image: Image.Image, - imgsz: int = 640, - half: bool = True, - device = devices.device, - augment: bool = True, - agnostic: bool = False, - retina: bool = False, - mask: bool = True, - offload: bool = shared.opts.detailer_unload, - ) -> list[YoloResult]: - - args = { - 'conf': shared.opts.detailer_conf, - 'iou': shared.opts.detailer_iou, - 'max_det': shared.opts.detailer_max, - } - model.to(device) - predictions = model.predict( - source=[image], - stream=False, - verbose=False, - imgsz=imgsz, - half=half, - device=device, - augment=augment, - agnostic_nms=agnostic, - retina_masks=retina, - **args - ) - if offload: - model.to('cpu') - - result = [] - for prediction in predictions: - boxes = prediction.boxes.xyxy.detach().int().cpu().numpy() if prediction.boxes is not None else [] - scores = prediction.boxes.conf.detach().float().cpu().numpy() if prediction.boxes is not None else [] - for score, box in zip(scores, boxes): - box = box.tolist() - mask_image = None - w, h = box[2] - box[0], box[3] - box[1] - size = w * h / (image.width * image.height) - if (min(w, h) > shared.opts.detailer_min_size if shared.opts.detailer_min_size > 0 else True) and (max(w, h) < shared.opts.detailer_max_size if shared.opts.detailer_max_size > 0 else True): - if mask: - mask_image = image.copy() - mask_image = Image.new('L', image.size, 0) - draw = ImageDraw.Draw(mask_image) - draw.rectangle(box, fill="white", outline=None, width=0) - cropped = image.crop(box) - result.append(YoloResult(score=round(score, 2), box=box, mask=mask_image, item=cropped, size=size, width=w, height=h, args=args)) - return result - - def load(self, model_name: str = None): - from modules import modelloader - self.dependencies() - if model_name is None: - model_name = list(self.list)[0] - if model_name in self.models: - return model_name, self.models[model_name] - else: - model_url = self.list.get(model_name) - file_name = os.path.basename(model_url) - model_file = modelloader.load_file_from_url(url=model_url, model_dir=shared.opts.yolo_dir, file_name=file_name) - if model_file is not None: - shared.log.info(f'Load: type=Detailer name="{model_name}" model="{model_file}"') - from ultralytics import YOLO # pylint: disable=import-outside-toplevel - model = YOLO(model_file) - self.models[model_name] = model - return model_name, model - return None - - def restore(self, np_image, p: processing.StableDiffusionProcessing = None): - if hasattr(p, 'recursion'): - return - if not hasattr(p, 'detailer_active'): - p.detailer_active = 0 - if np_image is None or p.detailer_active >= p.batch_size * p.n_iter: - return np_image - name, model = self.load() - if model is None: - shared.log.warning(f'Detailer: model="{name}" not loaded') - return np_image - image = Image.fromarray(np_image) - items = self.predict(model, image) - if len(items) == 0: - shared.log.info(f'Detailer: model="{name}" no items detected') - return np_image - - # create backups - orig_apply_overlay = shared.opts.mask_apply_overlay - orig_p = p.__dict__.copy() - orig_cls = p.__class__ - - pp = None - shared.opts.data['mask_apply_overlay'] = True - resolution = 512 if shared.sd_model_type in ['none', 'sd', 'lcm', 'unknown'] else 1024 - args = { - 'batch_size': 1, - 'n_iter': 1, - 'inpaint_full_res': True, - 'inpainting_mask_invert': 0, - 'inpainting_fill': 1, # no fill - 'sampler_name': orig_p.get('hr_sampler_name', 'default'), - 'steps': orig_p.get('hr_second_pass_steps', 0), - 'negative_prompt': orig_p.get('refiner_negative', ''), - 'denoising_strength': shared.opts.detailer_strength if shared.opts.detailer_strength > 0 else orig_p.get('denoising_strength', 0.3), - 'styles': [], - 'prompt': orig_p.get('refiner_prompt', ''), - 'mask_blur': 10, - 'inpaint_full_res_padding': shared.opts.detailer_padding, - 'detailer': True, - 'width': resolution, - 'height': resolution, - } - if args['denoising_strength'] == 0: - shared.log.debug(f'Detailer: model="{name}" strength=0 skip') - return np_image - control_pipeline = None - orig_class = shared.sd_model.__class__ - if getattr(p, 'is_control', False): - from modules.control import run - control_pipeline = shared.sd_model - run.restore_pipeline() - - p = processing_class.switch_class(p, processing.StableDiffusionProcessingImg2Img, args) - p.detailer_active += 1 # set flag to avoid recursion - - if p.steps < 1: - p.steps = orig_p.get('steps', 0) - if len(p.prompt) == 0: - p.prompt = orig_p.get('all_prompts', [''])[0] - if len(p.negative_prompt) == 0: - p.negative_prompt = orig_p.get('all_negative_prompts', [''])[0] - - report = [{'score': i.score, 'size': f'{i.width}x{i.height}' } for i in items] - shared.log.info(f'Detailer: model="{name}" items={report} args={items[0].args} denoise={p.denoising_strength} blur={p.mask_blur} width={p.width} height={p.height} padding={p.inpaint_full_res_padding}') - - mask_all = [] - p.state = '' - for item in items: - if item.mask is None: - continue - p.init_images = [image] - p.image_mask = [item.mask] - # mask_all.append(item.mask) - p.recursion = True - pp = processing.process_images_inner(p) - del p.recursion - p.overlay_images = None # skip applying overlay twice - if pp is not None and pp.images is not None and len(pp.images) > 0: - image = pp.images[0] # update image to be reused for next item - if len(pp.images) > 1: - mask_all.append(pp.images[1]) - - # restore pipeline - if control_pipeline is not None: - shared.sd_model = control_pipeline - else: - shared.sd_model.__class__ = orig_class - p = processing_class.switch_class(p, orig_cls, orig_p) - p.init_images = getattr(orig_p, 'init_images', None) - p.image_mask = getattr(orig_p, 'image_mask', None) - p.state = getattr(orig_p, 'state', None) - shared.opts.data['mask_apply_overlay'] = orig_apply_overlay - np_image = np.array(image) - - if len(mask_all) > 0 and shared.opts.include_mask: - from modules.control.util import blend - p.image_mask = blend([np.array(m) for m in mask_all]) - # combined = blend([np_image, p.image_mask]) - # combined = Image.fromarray(combined) - # combined.save('/tmp/item.png') - p.image_mask = Image.fromarray(p.image_mask) - return np_image - - -yolo = YoloRestorer() -shared.detailers.append(yolo) -shared.yolo = yolo diff --git a/scripts/postprocessing_upscale.py b/scripts/postprocessing_upscale.py index e19d577ee..0e07bc847 100644 --- a/scripts/postprocessing_upscale.py +++ b/scripts/postprocessing_upscale.py @@ -30,7 +30,7 @@ class ScriptPostprocessingUpscale(scripts_postprocessing.ScriptPostprocessing): extras_upscaler_1 = gr.Dropdown(label='Upscaler', elem_id="extras_upscaler_1", choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name) with gr.Row(): - extras_upscaler_2 = gr.Dropdown(label='Secondary Upscaler', elem_id="extras_upscaler_2", choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name) + extras_upscaler_2 = gr.Dropdown(label='Refine Upscaler', elem_id="extras_upscaler_2", choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name) extras_upscaler_2_visibility = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label="Upscaler 2 visibility", value=0.0, elem_id="extras_upscaler_2_visibility") upscaling_res_switch_btn.click(lambda w, h: (h, w), inputs=[upscaling_resize_w, upscaling_resize_h], outputs=[upscaling_resize_w, upscaling_resize_h], show_progress=False) diff --git a/webui.py b/webui.py index e4ecd9f5d..9a980e641 100644 --- a/webui.py +++ b/webui.py @@ -106,7 +106,9 @@ def initialize(): sys.modules["modules.codeformer_model"] = codeformer import modules.postprocess.gfpgan_model as gfpgan gfpgan.setup_model(shared.opts.gfpgan_models_path) - timer.startup.record("face-restore") + import modules.postprocess.yolo as yolo + yolo.initialize() + timer.startup.record("detailer") log.debug('Load extensions') t_timer, t_total = modules.scripts.load_scripts()