diff --git a/CHANGELOG.md b/CHANGELOG.md index 1def2d984..02d931243 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-02-15 +## Update for 2024-02-16 - **improvements**: - **IP Adapter** major refactor @@ -26,10 +26,15 @@ - **Outpaint** control outpaint now uses new alghorithm: noised-edge-extend new method allows for much larger outpaint areas in a single pass, even outpaint 512->1024 works well note that denoise strength should be increased for larger the outpaint areas, for example outpainting 512->1024 works well with denoise 0.75 + outpaint can run in *img2img* mode (default) and *inpaint* mode where original image is masked (if inpaint masked only is selected) - [DeepCache](https://github.com/horseee/DeepCache) model acceleration it can produce massive speedups (2x-5x) with no overhead, but with some loss of quality - *settings -> compute -> model compile -> deep-cache* and *settings -> compute -> model compile -> cache interval* - - **Control** units now have extra option to re-use current preview image as processor input + *settings -> compute -> model compile -> deep-cache* and *settings -> compute -> model compile -> cache interval* + - [ZLUDA](https://github.com/vosen/ZLUDA) CUDA wrapper experimental support, thanks @lshqqytiger + - best use case is *AMD GPUs on Windows*, see [wiki](https://github.com/vladmandic/automatic/wiki/ZLUDA) for details + - **Control** + - when performing inpainting, you can specify processing resolution using **size->mask** + - units now have extra option to re-use current preview image as processor input - **Cross-attention** refactored cross-attention methods, thanks @Disty0 - for backend:original, its unchanged: SDP, xFormers, Doggettxs, InvokeAI, Sub-quadratic, Split attention - for backend:diffuers, list is now: SDP, xFormers, Batch matrix-matrix, Split attention, Dynamic Attention BMM, Dynamic Attention SDP @@ -54,7 +59,8 @@ - add `--theme` cli param to force theme on startup - add `--allow-paths` cli param to add additional paths that are allowed to be accessed via web, thanks @OuticNZ - **wiki**: - - updated benchmark notes for IPEX and Olive + - added benchmark notes for IPEX, OpenVINO and Olive + - added ZLUDA wiki page - **fixes**: - handle extensions that install conflicting versions of packages `onnxruntime`, `opencv2-python` diff --git a/README.md b/README.md index 40261e6d4..9fd68aec9 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ Below is partial list of all available parameters, run `webui --help` for the fu SD.Next comes with several extensions pre-installed: -- [ControlNet](https://github.com/Mikubill/sd-webui-controlnet) +- [ControlNet](https://github.com/Mikubill/sd-webui-controlnet) (*active in backend: original only*) - [Agent Scheduler](https://github.com/ArtVentureX/sd-webui-agent-scheduler) - [Image Browser](https://github.com/AlUlkesh/stable-diffusion-webui-images-browser) @@ -214,9 +214,9 @@ This should be fully cross-platform, but we'd really love to have additional con ### **Credits** -- Main credit goes to [Automatic1111 WebUI](https://github.com/AUTOMATIC1111/stable-diffusion-webui) -- Additional credits are listed in [Credits](https://github.com/AUTOMATIC1111/stable-diffusion-webui/#credits) -- Licenses for modules are listed in [Licenses](html/licenses.html) +- Main credit goes to [Automatic1111 WebUI](https://github.com/AUTOMATIC1111/stable-diffusion-webui) for original codebase +- Additional credits are listed in [Credits](https://github.com/AUTOMATIC1111/stable-diffusion-webui/#credits) +- Licenses for modules are listed in [Licenses](html/licenses.html) ### **Evolution** diff --git a/TODO.md b/TODO.md index f01276e38..c82e89768 100644 --- a/TODO.md +++ b/TODO.md @@ -16,8 +16,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Control missing features - second pass: -- inpaint masking explicit processing size: -- outpaint return mask and optional override future run_masking - control api - masking api - inpaint pan/zoom diff --git a/modules/control/run.py b/modules/control/run.py index c7f67ff6b..e49707cac 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -38,6 +38,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio, resize_mode_before, resize_name_before, width_before, height_before, scale_by_before, selected_scale_tab_before, resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after, + resize_mode_mask, resize_name_mask, width_mask, height_mask, scale_by_mask, selected_scale_tab_mask, denoising_strength, batch_count, batch_size, video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, *input_script_args # pylint: disable=unused-argument @@ -462,6 +463,13 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ if hasattr(p, 'init_images') and p.init_images is None: # delete as its set via task_args del p.init_images + # resize mask + if mask is not None and resize_mode_mask != 0 and resize_name_mask != 'None': + if selected_scale_tab_mask == 1: + width_mask, height_mask = int(input_image.width * scale_by_before), int(input_image.height * scale_by_before) + p.width, p.height = width_mask, height_mask + debug(f'Control resize: op=mask image={mask} width={width_mask} height={height_mask} mode={resize_mode_mask} name={resize_name_mask}') + # pipeline output = None if pipe is not None: # run new pipeline diff --git a/modules/masking.py b/modules/masking.py index 88bc32347..ed0fdb7ee 100644 --- a/modules/masking.py +++ b/modules/masking.py @@ -335,16 +335,30 @@ def get_mask(input_image: gr.Image, input_mask: gr.Image): def outpaint(input_image: Image.Image, outpaint_type: str = 'Edge'): image = cv2.cvtColor(np.array(input_image), cv2.COLOR_RGB2BGR) - h, w = image.shape[:2] + h0, w0 = image.shape[:2] empty = (image == 0).all(axis=2) y0, x0 = np.where(~empty) # non empty x1, x2 = min(x0), max(x0) y1, y2 = min(y0), max(y0) cropped = image[y1:y2, x1:x2] + h1, w1 = cropped.shape[:2] + mask = None + + if opts.mask_only: + mask = cv2.copyMakeBorder(cropped, y1, h0-y2, x1, w0-x2, cv2.BORDER_CONSTANT, value=(0, 0, 0)) + mask = cv2.resize(mask, (w0, h0)) + mask = cv2.cvtColor(np.array(mask), cv2.COLOR_BGR2GRAY) + mask = cv2.threshold(mask, 0, 255, cv2.THRESH_BINARY)[1] + sigmaX, sigmaY = int((h0-h1)/3), int((w0-w1)/3) + kernel = np.ones((5, 5), np.uint8) + mask = cv2.erode(mask, kernel, iterations=max(sigmaX, sigmaY) // 3) # increase overlap area + mask = cv2.GaussianBlur(mask, (0, 0), sigmaX=sigmaX, sigmaY=sigmaY) # blur mask + mask = Image.fromarray(mask) + mask.save('/tmp/mask2.png') if outpaint_type == 'Edge': - bordered = cv2.copyMakeBorder(cropped, y1, h-y2, x1, w-x2, cv2.BORDER_REPLICATE) - bordered = cv2.resize(bordered, (w, h)) + bordered = cv2.copyMakeBorder(cropped, y1, h0-y2, x1, w0-x2, cv2.BORDER_REPLICATE) + bordered = cv2.resize(bordered, (w0, h0)) image = bordered # noise = np.random.normal(1, variation, bordered.shape) # noised = (noise * bordered).astype(np.uint8) @@ -354,7 +368,7 @@ def outpaint(input_image: Image.Image, outpaint_type: str = 'Edge'): image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) image = Image.fromarray(image) - return image + return image, mask def run_mask(input_image: Image.Image, input_mask: Image.Image = None, return_type: str = None, mask_blur: int = None, mask_padding: int = None, segment_enable=True, invert=None): @@ -428,6 +442,9 @@ def run_mask(input_image: Image.Image, input_mask: Image.Image = None, return_ty shared.log.debug(f'Mask: size={input_image.width}x{input_image.height} masked={mask_size}px area={area_size/total_size:.2f} auto={opts.auto_mask} blur={opts.mask_blur} erode={opts.mask_erode} dilate={opts.mask_dilate} type={return_type} time={t1-t0:.2f}') if return_type == 'None': return input_mask + elif return_type == 'Opaque': + binary_mask = cv2.threshold(mask, 0, 255, cv2.THRESH_BINARY)[1] + return Image.fromarray(binary_mask) elif return_type == 'Binary': binary_mask = cv2.threshold(mask, 127, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1] # otsu uses mean instead of threshold return Image.fromarray(binary_mask) diff --git a/modules/ui_control.py b/modules/ui_control.py index ff6f883a1..c20585ef1 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -96,6 +96,8 @@ def create_ui(_blocks: gr.Blocks=None): resize_mode_before, resize_name_before, width_before, height_before, scale_by_before, selected_scale_tab_before = ui_sections.create_resize_inputs('control', [], scale_visible=False, mode='Fixed', accordion=False, latent=True) with gr.Tab('After'): resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after = ui_sections.create_resize_inputs('control', [], scale_visible=False, mode='Fixed', accordion=False, latent=False) + with gr.Tab('Mask'): + resize_mode_mask, resize_name_mask, width_mask, height_mask, scale_by_mask, selected_scale_tab_mask = ui_sections.create_resize_inputs('control', [], scale_visible=False, mode='Fixed', accordion=False, latent=False) with gr.Accordion(open=False, label="Sampler", elem_id="control_sampler", elem_classes=["small-accordion"]): sd_samplers.set_samplers() @@ -497,6 +499,7 @@ def create_ui(_blocks: gr.Blocks=None): hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio, resize_mode_before, resize_name_before, width_before, height_before, scale_by_before, selected_scale_tab_before, resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after, + resize_mode_mask, resize_name_mask, width_mask, height_mask, scale_by_mask, selected_scale_tab_mask, denoising_strength, batch_count, batch_size, video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, ] diff --git a/modules/ui_control_helpers.py b/modules/ui_control_helpers.py index 75a8e17f1..6c0840c36 100644 --- a/modules/ui_control_helpers.py +++ b/modules/ui_control_helpers.py @@ -126,7 +126,9 @@ def select_input(input_mode, input_image, init_image, init_type, input_resize, i if isinstance(selected_input, Image.Image): # image via upload -> image if input_mode == 'Outpaint': masking.opts.invert = True - selected_input = masking.outpaint(input_image=selected_input) + selected_input, outpaint_mask = masking.outpaint(input_image=selected_input) + if outpaint_mask is not None: + input_mask = outpaint_mask input_source = [selected_input] input_type = 'PIL.Image' status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}' diff --git a/wiki b/wiki index 4d6f56592..eaa8ca8f4 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 4d6f56592ee2328c93f8e39a76fae67e5a4c9b6c +Subproject commit eaa8ca8f42d8031fdb796ea01845201fa2b75e12