add mixture tiling

CHANGELOG.md

@@ -119,6 +119,10 @@ As of this release, default backend is set to **diffusers** as its more feature
     **SD15**: Base, Base ViT-G, Light, Plus, Plus Face, Full Face  
     **SDXL**: Base SXDL, Base ViT-H SXDL, Plus ViT-H SXDL, Plus Face ViT-H SXDL  
   - enable use via api, thanks @trojaner  
+- [Mixture Tiling](https://arxiv.org/abs/2302.02412)  
+  - uses multiple prompts to guide different parts of the grid during diffusion process  
+  - can be used ot create complex scenes with multiple subjects  
+  - simply select from scripts  
 - [Self-attention guidance](https://github.com/SusungHong/Self-Attention-Guidance)  
   - simply select scale in advanced menu  
   - can drastically improve image coherence as well as reduce artifacts  

modules/scripts.py

@@ -5,7 +5,7 @@ import time
 from collections import namedtuple
 import gradio as gr
 from modules import paths, script_callbacks, extensions, script_loading, scripts_postprocessing, errors, timer
-from installer import log
+from installer import log, args as cmd_opts
 
 
 AlwaysVisible = object()
@@ -200,6 +200,8 @@ def list_scripts(scriptdirname, extension):
     base = os.path.join(paths.script_path, scriptdirname)
     if os.path.exists(base):
         for filename in sorted(os.listdir(base)):
+            if filename.startswith('_') and not cmd_opts.experimental:
+                continue
             tmp_list.append(ScriptFile(paths.script_path, filename, os.path.join(base, filename), '50'))
     for ext in extensions.active():
         tmp_list += ext.list_files(scriptdirname, extension)

scripts/mixture_tiling.py

@@ -0,0 +1,93 @@
+import gradio as gr
+import torch
+from modules import shared, devices, scripts, processing, sd_models
+
+
+checked_ok = False
+
+
+def check_dependencies():
+    global checked_ok # pylint: disable=global-statement
+    from installer import installed, install
+    packages = [
+        ('ligo-segments', 'ligo-segments'),
+    ]
+    for pkg in packages:
+        if not installed(pkg[1], reload=True, quiet=True):
+            install(pkg[0], pkg[1], ignore=False)
+    try:
+        from ligo.segments import segment # pylint: disable=unused-import
+        checked_ok = True
+        return True
+    except Exception as e:
+        shared.log.error(f'Mixture tiling: {e}')
+        return False
+
+
+class Script(scripts.Script):
+    def title(self):
+        return 'Mixture tiling'
+
+    def show(self, is_img2img):
+        return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False
+
+    def ui(self, _is_img2img):
+        with gr.Row():
+            gr.HTML('<a href="https://arxiv.org/abs/2302.02412">&nbsp Mixture tiling</a>')
+        with gr.Row():
+            gr.HTML('<span>&nbsp Separated prompts using new lines<br>&nbsp Number of prompts must matcxh X*Y</span>')
+        with gr.Row():
+            x_size = gr.Slider(label='X components', minimum=1, maximum=5, step=1, value=1)
+            y_size = gr.Slider(label='Y components', minimum=1, maximum=5, step=1, value=1)
+        with gr.Row():
+            x_overlap = gr.Slider(label='X overlap', minimum=0, maximum=1, step=0.01, value=0.5)
+            y_overlap = gr.Slider(label='Y overlap', minimum=0, maximum=1, step=0.01, value=0.5)
+        return x_size, y_size, x_overlap, y_overlap
+
+    def run(self, p: processing.StableDiffusionProcessing, x_size, y_size, x_overlap, y_overlap): # pylint: disable=arguments-differ
+        if not checked_ok:
+            if not check_dependencies():
+                return
+        prompts = p.prompt.splitlines()
+        if len(prompts) != x_size * y_size:
+            shared.log.error(f'Mixture tiling prompt count mismatch: prompts={len(prompts)} required={x_size * y_size}')
+            return
+        # backup pipeline and params
+        orig_pipeline = shared.sd_model
+        orig_dtype = devices.dtype
+        orig_prompt_attention = shared.opts.prompt_attention
+        # create pipeline
+        shared.sd_model = sd_models.switch_pipe('mixture_tiling', shared.sd_model)
+        sd_models.set_diffuser_options(shared.sd_model)
+        shared.opts.data['prompt_attention'] = 'Fixed attention' # this pipeline is not compatible with embeds
+        shared.sd_model.to(torch.float32) # this pipeline unet is not compatible with fp16
+        processing.fix_seed(p)
+        # set pipeline specific params, note that standard params are applied when applicable
+        y_prompts = []
+        for y in range(y_size):
+            x_prompts = []
+            for x in range(x_size):
+                x_prompts.append(prompts[y * x_size + x])
+            y_prompts.append(x_prompts)
+        p.task_args['prompt'] = y_prompts
+        """
+        p.task_args['prompt'] = [[
+                "old house in the countryside during sunset",
+                "windy dirt road during sunset with mountains in the background",
+                "old rusty giant robot in the countryside during sunset"
+            ]]
+        """
+        p.task_args['seed'] = p.seed
+        p.task_args['tile_width'] = p.height
+        p.task_args['tile_height'] = p.width
+        p.task_args['tile_col_overlap'] = int(p.height * x_overlap)
+        p.task_args['tile_row_overlap'] = int(p.width * y_overlap)
+        p.task_args['output_type'] = 'np'
+        # run pipeline
+        shared.log.debug(f'Tiling: args={p.task_args}')
+        processed: processing.Processed = processing.process_images(p) # runs processing using main loop
+        # restore pipeline and params
+        shared.opts.data['prompt_attention'] = orig_prompt_attention
+        shared.sd_model = orig_pipeline
+        shared.sd_model.to(orig_dtype)
+        return processed