new control outpaint

CHANGELOG.md

@@ -20,6 +20,9 @@
     it can produce massive speedups (2x-5x) with no overhead, but with some loss of quality  
       *settings -> compute -> model compile -> deep-cache* and *settings -> compute -> model compile -> cache interval*  
   - **Control** units now have extra option to re-use current preview image as processor input  
+  - **Outpaint** control outpaint now uses new alghorithm: noised-edge-extend  
+    new method allows for much larger outpaint areas in a single pass, even outpaint 512->1024 works well  
+    note that denoise strength should be increased for larger the outpaint areas, for example outpainting 512->1024 works well with denoise 0.75  
   - **Clip-skip** reworked completely, thanks @AI-Casanova & @Disty0  
     now clip-skip range is 0-12 where previously lowest value was 1 (default is still 1)  
     values can also be decimal to interpolate between different layers, for example `clip-skip: 1.5`, thanks @AI-Casanova  
@@ -66,6 +69,7 @@
   - handle pipelines that return dict instead of object  
   - fix vae dtype mismatch, thanks @Disty0  
   - fix controlnet inpaint mask  
+  - fix theme list refresh  
   - fix extensions update information in ui  
   - bind controlnet extension to last known working commit, thanks @Aptronymist  
   - prompts-from-file fix resizable prompt area  

modules/masking.py

@@ -333,7 +333,31 @@ def get_mask(input_image: gr.Image, input_mask: gr.Image):
         return output_mask
 
 
-def run_mask(input_image: gr.Image, input_mask: gr.Image = None, return_type: str = None, mask_blur: int = None, mask_padding: int = None, segment_enable=True, invert=None):
+def outpaint(input_image: Image.Image, outpaint_type: str = 'Edge'):
+    image = cv2.cvtColor(np.array(input_image), cv2.COLOR_RGB2BGR)
+    h, w = image.shape[:2]
+    empty = (image == 0).all(axis=2)
+    y0, x0 = np.where(~empty) # non empty
+    x1, x2 = min(x0), max(x0)
+    y1, y2 = min(y0), max(y0)
+    cropped = image[y1:y2, x1:x2]
+
+    if outpaint_type == 'Edge':
+        bordered = cv2.copyMakeBorder(cropped, y1, h-y2, x1, w-x2, cv2.BORDER_REPLICATE)
+        bordered = cv2.resize(bordered, (w, h))
+        image = bordered
+        # noise = np.random.normal(1, variation, bordered.shape)
+        # noised = (noise * bordered).astype(np.uint8)
+        # h, w = cropped.shape[:2]
+        # noised[y1:y1 + h, x1:x1 + w] = cropped # overlay original over initialized
+        # image = noised
+
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    image = Image.fromarray(image)
+    return image
+
+
+def run_mask(input_image: Image.Image, input_mask: Image.Image = None, return_type: str = None, mask_blur: int = None, mask_padding: int = None, segment_enable=True, invert=None):
     debug(f'Run mask: function={sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access
 
     if input_image is None:

modules/processing_class.py

@@ -341,19 +341,6 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
                     np_mask = cv2.GaussianBlur(np_mask, (kernel_size, 1), self.mask_blur)
                     np_mask = cv2.GaussianBlur(np_mask, (1, kernel_size), self.mask_blur)
                     self.image_mask = Image.fromarray(np_mask)
-            """
-            else: # handled in processing_diffusers
-                if hasattr(self, 'init_images'):
-                    self.image_mask = masking.run_mask(
-                        input_image=self.init_images,
-                        input_mask=self.image_mask,
-                        return_type='Grayscale',
-                        mask_blur=self.mask_blur,
-                        mask_padding=self.inpaint_full_res_padding,
-                        segment_enable=False,
-                        invert=self.inpainting_mask_invert==1,
-                    )
-            """
             if self.inpaint_full_res: # mask only inpaint
                 self.mask_for_overlay = self.image_mask
                 mask = self.image_mask.convert('L')

modules/theme.py

@@ -14,16 +14,30 @@ def list_builtin_themes():
     return files
 
 
-def list_themes():
+def refresh_themes(no_update=False):
     fn = os.path.join('html', 'themes.json')
-    if not os.path.exists(fn):
-        refresh_themes()
+    res = []
     if os.path.exists(fn):
-        with open(fn, mode='r', encoding='utf=8') as f:
-            res = json.loads(f.read())
-    else:
-        res = []
+        try:
+            with open(fn, 'r', encoding='utf8') as f:
+                res = json.load(f)
+        except Exception:
+            modules.shared.log.error('Exception loading UI themes')
+    if not no_update:
+        try:
+            modules.shared.log.info('Refreshing UI themes')
+            r = modules.shared.req('https://huggingface.co/datasets/freddyaboulton/gradio-theme-subdomains/resolve/main/subdomains.json')
+            if r.status_code == 200:
+                res = r.json()
+                modules.shared.writefile(res, fn)
+            else:
+                modules.shared.log.error('Error refreshing UI themes')
+        except Exception:
+            modules.shared.log.error('Exception refreshing UI themes')
+    return res
 
+
+def list_themes():
     builtin = list_builtin_themes()
     extensions = [e.name for e in modules.extensions.extensions if e.enabled]
     engines = []
@@ -32,45 +46,20 @@ def list_themes():
     if 'sd-webui-lobe-theme' in extensions:
         engines.append('lobe')
     gradio = ["gradio/default", "gradio/base", "gradio/glass", "gradio/monochrome", "gradio/soft"]
-    huggingface = {x['id'] for x in res if x['status'] == 'RUNNING' and 'test' not in x['id'].lower()}
+    huggingface = refresh_themes(no_update=True)
+    huggingface = {x['id'] for x in huggingface if x['status'] == 'RUNNING' and 'test' not in x['id'].lower()}
     huggingface = [f'huggingface/{x}' for x in huggingface]
     modules.shared.log.debug(f'Themes: builtin={len(builtin)} gradio={len(gradio)} huggingface={len(huggingface)}')
     themes = sorted(engines) + sorted(builtin) + sorted(gradio) + sorted(huggingface, key=str.casefold)
     return themes
 
 
-def refresh_themes():
-    try:
-        r = modules.shared.req('https://huggingface.co/datasets/freddyaboulton/gradio-theme-subdomains/resolve/main/subdomains.json')
-        if r.status_code == 200:
-            res = r.json()
-            fn = os.path.join('html', 'themes.json')
-            modules.shared.writefile(res, fn)
-            list_themes()
-        else:
-            modules.shared.log.error('Error refreshing UI themes')
-    except Exception:
-        modules.shared.log.error('Exception refreshing UI themes')
-
-
 def reload_gradio_theme(theme_name=None):
     global gradio_theme # pylint: disable=global-statement
     theme_name = theme_name or modules.shared.cmd_opts.theme or modules.shared.opts.gradio_theme
     if theme_name == 'default':
         theme_name = 'black-teal'
     modules.shared.opts.data['gradio_theme'] = theme_name
-    default_font_params = {}
-    """
-    res = 0
-    try:
-        import urllib.request
-        request = urllib.request.Request("https://fonts.googleapis.com/css2?family=IBM+Plex+Mono", method="HEAD")
-        res = urllib.request.urlopen(request, timeout=3.0).status # pylint: disable=consider-using-with
-    except Exception:
-        res = 0
-    if res != 200:
-        modules.shared.log.info('No internet access detected, using default fonts')
-    """
     default_font_params = {
         'font':['Helvetica', 'ui-sans-serif', 'system-ui', 'sans-serif'],
         'font_mono':['IBM Plex Mono', 'ui-monospace', 'Consolas', 'monospace']
@@ -78,16 +67,15 @@ def reload_gradio_theme(theme_name=None):
     is_builtin = theme_name.lower() in list_builtin_themes()
     is_external = theme_name.lower() in ['lobe', 'modern']
     base = 'base.css'
-    if is_external:
-        gradio_theme = gr.themes.Base(**default_font_params)
-        modules.shared.log.info(f'UI theme: name="{theme_name}" style={modules.shared.opts.theme_style} base={base}')
-        return False
     if is_builtin:
         base = 'sdnext.css'
         gradio_theme = gr.themes.Base(**default_font_params)
         modules.shared.log.info(f'UI theme: name="{theme_name}" style={modules.shared.opts.theme_style} base={base}')
         return True
-    if theme_name.startswith("gradio/"):
+    elif is_external:
+        gradio_theme = gr.themes.Base(**default_font_params)
+        modules.shared.log.info(f'UI theme: name="{theme_name}" style={modules.shared.opts.theme_style} base={base}')
+    elif theme_name.startswith("gradio/"):
         modules.shared.log.info(f'UI theme: name="{theme_name}" style={modules.shared.opts.theme_style} base={base}')
         modules.shared.log.warning('UI theme: using Gradio default theme which is not optimized for SD.Next')
         if theme_name == "gradio/default":

modules/ui_control_helpers.py

@@ -125,13 +125,13 @@ def select_input(input_mode, input_image, init_image, init_type, input_resize, i
     # control inputs
     if isinstance(selected_input, Image.Image): # image via upload -> image
         if input_mode == 'Outpaint':
-            input_mask = masking.run_mask(input_image=selected_input, input_mask=None, return_type='Grayscale')
+            masking.opts.invert = True
+            selected_input = masking.outpaint(input_image=selected_input)
         input_source = [selected_input]
         input_type = 'PIL.Image'
         status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}'
         res = [gr.Tabs.update(selected='out-gallery'), status]
     elif isinstance(selected_input, dict): # inpaint -> dict image+mask
-        # input_mask = masking.run_mask(input_image=selected_input['image'], input_mask=selected_input['mask'], return_type='Grayscale')
         input_mask = selected_input['mask']
         selected_input = selected_input['image']
         input_source = [selected_input]