diff --git a/CHANGELOG.md b/CHANGELOG.md index ef66629c1..51b9ebe96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,6 +119,10 @@ As of this release, default backend is set to **diffusers** as its more feature **SD15**: Base, Base ViT-G, Light, Plus, Plus Face, Full Face **SDXL**: Base SXDL, Base ViT-H SXDL, Plus ViT-H SXDL, Plus Face ViT-H SXDL - enable use via api, thanks @trojaner +- [Mixture Tiling](https://arxiv.org/abs/2302.02412) + - uses multiple prompts to guide different parts of the grid during diffusion process + - can be used ot create complex scenes with multiple subjects + - simply select from scripts - [Self-attention guidance](https://github.com/SusungHong/Self-Attention-Guidance) - simply select scale in advanced menu - can drastically improve image coherence as well as reduce artifacts diff --git a/modules/scripts.py b/modules/scripts.py index a7951f484..561bb3e1c 100644 --- a/modules/scripts.py +++ b/modules/scripts.py @@ -5,7 +5,7 @@ import time from collections import namedtuple import gradio as gr from modules import paths, script_callbacks, extensions, script_loading, scripts_postprocessing, errors, timer -from installer import log +from installer import log, args as cmd_opts AlwaysVisible = object() @@ -200,6 +200,8 @@ def list_scripts(scriptdirname, extension): base = os.path.join(paths.script_path, scriptdirname) if os.path.exists(base): for filename in sorted(os.listdir(base)): + if filename.startswith('_') and not cmd_opts.experimental: + continue tmp_list.append(ScriptFile(paths.script_path, filename, os.path.join(base, filename), '50')) for ext in extensions.active(): tmp_list += ext.list_files(scriptdirname, extension) diff --git a/scripts/mixture_tiling.py b/scripts/mixture_tiling.py new file mode 100644 index 000000000..59feb8491 --- /dev/null +++ b/scripts/mixture_tiling.py @@ -0,0 +1,93 @@ +import gradio as gr +import torch +from modules import shared, devices, scripts, processing, sd_models + + +checked_ok = False + + +def check_dependencies(): + global checked_ok # pylint: disable=global-statement + from installer import installed, install + packages = [ + ('ligo-segments', 'ligo-segments'), + ] + for pkg in packages: + if not installed(pkg[1], reload=True, quiet=True): + install(pkg[0], pkg[1], ignore=False) + try: + from ligo.segments import segment # pylint: disable=unused-import + checked_ok = True + return True + except Exception as e: + shared.log.error(f'Mixture tiling: {e}') + return False + + +class Script(scripts.Script): + def title(self): + return 'Mixture tiling' + + def show(self, is_img2img): + return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + + def ui(self, _is_img2img): + with gr.Row(): + gr.HTML('  Mixture tiling') + with gr.Row(): + gr.HTML('  Separated prompts using new lines
  Number of prompts must matcxh X*Y
') + with gr.Row(): + x_size = gr.Slider(label='X components', minimum=1, maximum=5, step=1, value=1) + y_size = gr.Slider(label='Y components', minimum=1, maximum=5, step=1, value=1) + with gr.Row(): + x_overlap = gr.Slider(label='X overlap', minimum=0, maximum=1, step=0.01, value=0.5) + y_overlap = gr.Slider(label='Y overlap', minimum=0, maximum=1, step=0.01, value=0.5) + return x_size, y_size, x_overlap, y_overlap + + def run(self, p: processing.StableDiffusionProcessing, x_size, y_size, x_overlap, y_overlap): # pylint: disable=arguments-differ + if not checked_ok: + if not check_dependencies(): + return + prompts = p.prompt.splitlines() + if len(prompts) != x_size * y_size: + shared.log.error(f'Mixture tiling prompt count mismatch: prompts={len(prompts)} required={x_size * y_size}') + return + # backup pipeline and params + orig_pipeline = shared.sd_model + orig_dtype = devices.dtype + orig_prompt_attention = shared.opts.prompt_attention + # create pipeline + shared.sd_model = sd_models.switch_pipe('mixture_tiling', shared.sd_model) + sd_models.set_diffuser_options(shared.sd_model) + shared.opts.data['prompt_attention'] = 'Fixed attention' # this pipeline is not compatible with embeds + shared.sd_model.to(torch.float32) # this pipeline unet is not compatible with fp16 + processing.fix_seed(p) + # set pipeline specific params, note that standard params are applied when applicable + y_prompts = [] + for y in range(y_size): + x_prompts = [] + for x in range(x_size): + x_prompts.append(prompts[y * x_size + x]) + y_prompts.append(x_prompts) + p.task_args['prompt'] = y_prompts + """ + p.task_args['prompt'] = [[ + "old house in the countryside during sunset", + "windy dirt road during sunset with mountains in the background", + "old rusty giant robot in the countryside during sunset" + ]] + """ + p.task_args['seed'] = p.seed + p.task_args['tile_width'] = p.height + p.task_args['tile_height'] = p.width + p.task_args['tile_col_overlap'] = int(p.height * x_overlap) + p.task_args['tile_row_overlap'] = int(p.width * y_overlap) + p.task_args['output_type'] = 'np' + # run pipeline + shared.log.debug(f'Tiling: args={p.task_args}') + processed: processing.Processed = processing.process_images(p) # runs processing using main loop + # restore pipeline and params + shared.opts.data['prompt_attention'] = orig_prompt_attention + shared.sd_model = orig_pipeline + shared.sd_model.to(orig_dtype) + return processed