From 935d7ef0ad317cb21505b40430856ba127be772c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 13 Feb 2024 20:57:30 -0500 Subject: [PATCH] new control outpaint --- CHANGELOG.md | 4 +++ modules/masking.py | 26 +++++++++++++- modules/processing_class.py | 13 ------- modules/theme.py | 66 ++++++++++++++--------------------- modules/ui_control_helpers.py | 4 +-- 5 files changed, 58 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0c5903e1..0d8f7ad3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ it can produce massive speedups (2x-5x) with no overhead, but with some loss of quality *settings -> compute -> model compile -> deep-cache* and *settings -> compute -> model compile -> cache interval* - **Control** units now have extra option to re-use current preview image as processor input + - **Outpaint** control outpaint now uses new alghorithm: noised-edge-extend + new method allows for much larger outpaint areas in a single pass, even outpaint 512->1024 works well + note that denoise strength should be increased for larger the outpaint areas, for example outpainting 512->1024 works well with denoise 0.75 - **Clip-skip** reworked completely, thanks @AI-Casanova & @Disty0 now clip-skip range is 0-12 where previously lowest value was 1 (default is still 1) values can also be decimal to interpolate between different layers, for example `clip-skip: 1.5`, thanks @AI-Casanova @@ -66,6 +69,7 @@ - handle pipelines that return dict instead of object - fix vae dtype mismatch, thanks @Disty0 - fix controlnet inpaint mask + - fix theme list refresh - fix extensions update information in ui - bind controlnet extension to last known working commit, thanks @Aptronymist - prompts-from-file fix resizable prompt area diff --git a/modules/masking.py b/modules/masking.py index 8bb2206db..18104759d 100644 --- a/modules/masking.py +++ b/modules/masking.py @@ -333,7 +333,31 @@ def get_mask(input_image: gr.Image, input_mask: gr.Image): return output_mask -def run_mask(input_image: gr.Image, input_mask: gr.Image = None, return_type: str = None, mask_blur: int = None, mask_padding: int = None, segment_enable=True, invert=None): +def outpaint(input_image: Image.Image, outpaint_type: str = 'Edge'): + image = cv2.cvtColor(np.array(input_image), cv2.COLOR_RGB2BGR) + h, w = image.shape[:2] + empty = (image == 0).all(axis=2) + y0, x0 = np.where(~empty) # non empty + x1, x2 = min(x0), max(x0) + y1, y2 = min(y0), max(y0) + cropped = image[y1:y2, x1:x2] + + if outpaint_type == 'Edge': + bordered = cv2.copyMakeBorder(cropped, y1, h-y2, x1, w-x2, cv2.BORDER_REPLICATE) + bordered = cv2.resize(bordered, (w, h)) + image = bordered + # noise = np.random.normal(1, variation, bordered.shape) + # noised = (noise * bordered).astype(np.uint8) + # h, w = cropped.shape[:2] + # noised[y1:y1 + h, x1:x1 + w] = cropped # overlay original over initialized + # image = noised + + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + image = Image.fromarray(image) + return image + + +def run_mask(input_image: Image.Image, input_mask: Image.Image = None, return_type: str = None, mask_blur: int = None, mask_padding: int = None, segment_enable=True, invert=None): debug(f'Run mask: function={sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access if input_image is None: diff --git a/modules/processing_class.py b/modules/processing_class.py index 5cfb50cb2..57f586521 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -341,19 +341,6 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): np_mask = cv2.GaussianBlur(np_mask, (kernel_size, 1), self.mask_blur) np_mask = cv2.GaussianBlur(np_mask, (1, kernel_size), self.mask_blur) self.image_mask = Image.fromarray(np_mask) - """ - else: # handled in processing_diffusers - if hasattr(self, 'init_images'): - self.image_mask = masking.run_mask( - input_image=self.init_images, - input_mask=self.image_mask, - return_type='Grayscale', - mask_blur=self.mask_blur, - mask_padding=self.inpaint_full_res_padding, - segment_enable=False, - invert=self.inpainting_mask_invert==1, - ) - """ if self.inpaint_full_res: # mask only inpaint self.mask_for_overlay = self.image_mask mask = self.image_mask.convert('L') diff --git a/modules/theme.py b/modules/theme.py index 134822b0e..50e9b9dc3 100644 --- a/modules/theme.py +++ b/modules/theme.py @@ -14,16 +14,30 @@ def list_builtin_themes(): return files -def list_themes(): +def refresh_themes(no_update=False): fn = os.path.join('html', 'themes.json') - if not os.path.exists(fn): - refresh_themes() + res = [] if os.path.exists(fn): - with open(fn, mode='r', encoding='utf=8') as f: - res = json.loads(f.read()) - else: - res = [] + try: + with open(fn, 'r', encoding='utf8') as f: + res = json.load(f) + except Exception: + modules.shared.log.error('Exception loading UI themes') + if not no_update: + try: + modules.shared.log.info('Refreshing UI themes') + r = modules.shared.req('https://huggingface.co/datasets/freddyaboulton/gradio-theme-subdomains/resolve/main/subdomains.json') + if r.status_code == 200: + res = r.json() + modules.shared.writefile(res, fn) + else: + modules.shared.log.error('Error refreshing UI themes') + except Exception: + modules.shared.log.error('Exception refreshing UI themes') + return res + +def list_themes(): builtin = list_builtin_themes() extensions = [e.name for e in modules.extensions.extensions if e.enabled] engines = [] @@ -32,45 +46,20 @@ def list_themes(): if 'sd-webui-lobe-theme' in extensions: engines.append('lobe') gradio = ["gradio/default", "gradio/base", "gradio/glass", "gradio/monochrome", "gradio/soft"] - huggingface = {x['id'] for x in res if x['status'] == 'RUNNING' and 'test' not in x['id'].lower()} + huggingface = refresh_themes(no_update=True) + huggingface = {x['id'] for x in huggingface if x['status'] == 'RUNNING' and 'test' not in x['id'].lower()} huggingface = [f'huggingface/{x}' for x in huggingface] modules.shared.log.debug(f'Themes: builtin={len(builtin)} gradio={len(gradio)} huggingface={len(huggingface)}') themes = sorted(engines) + sorted(builtin) + sorted(gradio) + sorted(huggingface, key=str.casefold) return themes -def refresh_themes(): - try: - r = modules.shared.req('https://huggingface.co/datasets/freddyaboulton/gradio-theme-subdomains/resolve/main/subdomains.json') - if r.status_code == 200: - res = r.json() - fn = os.path.join('html', 'themes.json') - modules.shared.writefile(res, fn) - list_themes() - else: - modules.shared.log.error('Error refreshing UI themes') - except Exception: - modules.shared.log.error('Exception refreshing UI themes') - - def reload_gradio_theme(theme_name=None): global gradio_theme # pylint: disable=global-statement theme_name = theme_name or modules.shared.cmd_opts.theme or modules.shared.opts.gradio_theme if theme_name == 'default': theme_name = 'black-teal' modules.shared.opts.data['gradio_theme'] = theme_name - default_font_params = {} - """ - res = 0 - try: - import urllib.request - request = urllib.request.Request("https://fonts.googleapis.com/css2?family=IBM+Plex+Mono", method="HEAD") - res = urllib.request.urlopen(request, timeout=3.0).status # pylint: disable=consider-using-with - except Exception: - res = 0 - if res != 200: - modules.shared.log.info('No internet access detected, using default fonts') - """ default_font_params = { 'font':['Helvetica', 'ui-sans-serif', 'system-ui', 'sans-serif'], 'font_mono':['IBM Plex Mono', 'ui-monospace', 'Consolas', 'monospace'] @@ -78,16 +67,15 @@ def reload_gradio_theme(theme_name=None): is_builtin = theme_name.lower() in list_builtin_themes() is_external = theme_name.lower() in ['lobe', 'modern'] base = 'base.css' - if is_external: - gradio_theme = gr.themes.Base(**default_font_params) - modules.shared.log.info(f'UI theme: name="{theme_name}" style={modules.shared.opts.theme_style} base={base}') - return False if is_builtin: base = 'sdnext.css' gradio_theme = gr.themes.Base(**default_font_params) modules.shared.log.info(f'UI theme: name="{theme_name}" style={modules.shared.opts.theme_style} base={base}') return True - if theme_name.startswith("gradio/"): + elif is_external: + gradio_theme = gr.themes.Base(**default_font_params) + modules.shared.log.info(f'UI theme: name="{theme_name}" style={modules.shared.opts.theme_style} base={base}') + elif theme_name.startswith("gradio/"): modules.shared.log.info(f'UI theme: name="{theme_name}" style={modules.shared.opts.theme_style} base={base}') modules.shared.log.warning('UI theme: using Gradio default theme which is not optimized for SD.Next') if theme_name == "gradio/default": diff --git a/modules/ui_control_helpers.py b/modules/ui_control_helpers.py index 416607913..75a8e17f1 100644 --- a/modules/ui_control_helpers.py +++ b/modules/ui_control_helpers.py @@ -125,13 +125,13 @@ def select_input(input_mode, input_image, init_image, init_type, input_resize, i # control inputs if isinstance(selected_input, Image.Image): # image via upload -> image if input_mode == 'Outpaint': - input_mask = masking.run_mask(input_image=selected_input, input_mask=None, return_type='Grayscale') + masking.opts.invert = True + selected_input = masking.outpaint(input_image=selected_input) input_source = [selected_input] input_type = 'PIL.Image' status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}' res = [gr.Tabs.update(selected='out-gallery'), status] elif isinstance(selected_input, dict): # inpaint -> dict image+mask - # input_mask = masking.run_mask(input_image=selected_input['image'], input_mask=selected_input['mask'], return_type='Grayscale') input_mask = selected_input['mask'] selected_input = selected_input['image'] input_source = [selected_input]