From 761d04dd87fda5231ae0af325463cd5dd9e3a307 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 12 Feb 2025 11:15:19 -0500 Subject: [PATCH] add mod regional tiling Signed-off-by: Vladimir Mandic --- .pylintrc | 1 + .ruff.toml | 1 + CHANGELOG.md | 6 ++ modules/extra_networks.py | 5 +- modules/processing.py | 12 ++-- modules/processing_helpers.py | 2 + modules/styles.py | 3 +- scripts/mixture_of_diffusers.py | 107 +++++++++++++++++++++++++------- 8 files changed, 106 insertions(+), 31 deletions(-) diff --git a/.pylintrc b/.pylintrc index 78361f7c4..6395e3422 100644 --- a/.pylintrc +++ b/.pylintrc @@ -36,6 +36,7 @@ ignore-paths=/usr/lib/.*$, modules/todo, modules/unipc, modules/xadapter, + modules/mod, repositories, extensions-builtin/Lora, extensions-builtin/sd-webui-agent-scheduler, diff --git a/.ruff.toml b/.ruff.toml index 89bd1586d..1cca9cebb 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -31,6 +31,7 @@ exclude = [ "modules/todo", "modules/unipc", "modules/xadapter", + "modules/mod", "repositories", "extensions-builtin/Lora", "extensions-builtin/sd-extension-chainner/nodes", diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e65fb85b..b3123004d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,17 @@ - **Models** - [AlphaVLLM Lumina 2](https://github.com/Alpha-VLLM/Lumina-Image-2.0) new foundation model for image generation based o Gemma-2-2B text encoder and a flow-based diffusion transformer + fully supports offloading and on-the-fly quantization simply select from *networks -> models -> reference* - [Ostris Flex.1-Alpha](https://huggingface.co/ostris/Flex.1-alpha) originally based on *Flux.1-Schnell*, but retrained and with different architecture result is model smaller than *Flux.1-Dev*, but with similar capabilities + fully supports offloading and on-the-fly quantization simply select from *networks -> models -> reference* +- **Pipelines** + - [Mixture-of-Diffusers](https://huggingface.co/posts/elismasilva/251775641926329) + Regional tiling type of a solution for SDXL models + select from *scripts -> mixture of diffusers* - **Docs** - New [Outpaint](https://github.com/vladmandic/sdnext/wiki/Outpaint) step-by-step guide - Updated [Docker](https://github.com/vladmandic/sdnext/wiki/Docker) guide diff --git a/modules/extra_networks.py b/modules/extra_networks.py index 25b366e05..420f3beda 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -162,7 +162,10 @@ def parse_prompt(prompt): args = m.group(2) res[name].append(ExtraNetworkParams(items=args.split(":"))) return "" - prompt = re.sub(re_extra_net, found, prompt) + if isinstance(prompt, list): + prompt = [re.sub(re_extra_net, found, p) for p in prompt] + else: + prompt = re.sub(re_extra_net, found, prompt) return prompt, res diff --git a/modules/processing.py b/modules/processing.py index 549fead3e..de7bd93de 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -260,9 +260,10 @@ def process_init(p: StableDiffusionProcessing): else: p.all_subseeds = [int(subseed) + x for x in range(len(p.all_prompts))] if reset_prompts: - p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts(p.all_prompts, p.all_negative_prompts, p.styles, p.all_seeds) - p.prompts = p.all_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size] - p.negative_prompts = p.all_negative_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size] + if not hasattr(p, 'keep_prompts'): + p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts(p.all_prompts, p.all_negative_prompts, p.styles, p.all_seeds) + p.prompts = p.all_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size] + p.negative_prompts = p.all_negative_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size] p.prompts, _ = extra_networks.parse_prompts(p.prompts) @@ -312,8 +313,9 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if shared.native: from modules import ipadapter ipadapter.apply(shared.sd_model, p) - p.prompts = p.all_prompts[n * p.batch_size:(n+1) * p.batch_size] - p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n+1) * p.batch_size] + if not hasattr(p, 'keep_prompts'): + p.prompts = p.all_prompts[n * p.batch_size:(n+1) * p.batch_size] + p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n+1) * p.batch_size] p.seeds = p.all_seeds[n * p.batch_size:(n+1) * p.batch_size] p.subseeds = p.all_subseeds[n * p.batch_size:(n+1) * p.batch_size] if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner): diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index 083d3489a..cd8c1bb9a 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -428,6 +428,8 @@ def resize_hires(p, latents): # input=latents output=pil if not latent_upscaler def fix_prompts(p, prompts, negative_prompts, prompts_2, negative_prompts_2): + if hasattr(p, 'keep_prompts'): + return prompts, negative_prompts, prompts_2, negative_prompts_2 if type(prompts) is str: prompts = [prompts] if type(negative_prompts) is str: diff --git a/modules/styles.py b/modules/styles.py index 8fc1b543d..0a1812789 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -69,7 +69,7 @@ def apply_file_wildcards(prompt, replaced = [], not_found = [], recursion=0, see return check_wildcard_files(prompt, wildcard, files, file_only=False) recursion += 1 - if not shared.opts.wildcards_enabled or recursion >= 10: + if not shared.opts.wildcards_enabled or recursion >= 10 or not isinstance(prompt, str) or len(prompt) == 0: return prompt, replaced, not_found matches = re.findall(r'__(.*?)__', prompt, re.DOTALL) matches = [m for m in matches if m not in not_found] @@ -303,6 +303,7 @@ class StyleDatabase: prompt = apply_styles_to_prompt(prompt, [self.find_style(x).prompt for x in styles]) prompt = apply_wildcards_to_prompt(prompt, [self.find_style(x).wildcards for x in styles], seeds[i]) parsed_positive.append(prompt) + for i in range(len(negatives)): prompt = negatives[i] prompt = apply_styles_to_prompt(prompt, [self.find_style(x).negative_prompt for x in styles]) prompt = apply_wildcards_to_prompt(prompt, [self.find_style(x).wildcards for x in styles], seeds[i]) diff --git a/scripts/mixture_of_diffusers.py b/scripts/mixture_of_diffusers.py index 21139e6ce..4e0b15d27 100644 --- a/scripts/mixture_of_diffusers.py +++ b/scripts/mixture_of_diffusers.py @@ -1,64 +1,123 @@ import gradio as gr from modules import scripts, processing, shared, sd_models + +supported_models = ['sdxl'] max_xtiles = 4 max_ytiles = 4 + class Script(scripts.Script): def __init__(self): super().__init__() self.orig_pipe = None + self.orig_attn = None def title(self): - return 'Mixture-of-Diffusers' + return 'Mixture-of-Diffusers: Tile Control' def show(self, is_img2img): return shared.native - def update_ui(self, x, y): + def update_ui(self, x_tiles, y_tiles): updates = [] - for i in range(max_xtiles): - for j in range(max_ytiles): - updates.append(gr.update(visible=(i < x) and (j < y))) + for x in range(max_xtiles): + for y in range(max_ytiles): + updates.append(gr.update(visible=(x < x_tiles) and (y < y_tiles))) return updates def ui(self, _is_img2img): # ui elements with gr.Row(): - gr.HTML('  Mixture-of-Diffusers
') + gr.HTML('  Mixture-of-Diffusers
') with gr.Row(): - x_tiles = gr.Slider(minimum=1, maximum=max_xtiles, default=1, label="X-axis tiles") - y_tiles = gr.Slider(minimum=1, maximum=max_ytiles, default=1, label="Y-axis tiles") + gr.HTML('  Use base prompt to define image background and common elements
  Set image width and height to final image size') with gr.Row(): - tile_width = gr.Number(minimum=1, maximum=2048, value=1024, label="Tile width") - tile_height = gr.Number(minimum=1, maximum=2048, value=1024, label="Tile height") + x_tiles = gr.Slider(minimum=1, maximum=max_xtiles, step=1, value=1, label="X-axis tiles") + y_tiles = gr.Slider(minimum=1, maximum=max_ytiles, step=1, value=1, label="Y-axis tiles") with gr.Row(): - overlap_width = gr.Number(minimum=1, maximum=512, value=128, label="Overlap width") - overlap_height = gr.Number(minimum=1, maximum=512, value=128, label="Overlap height") - with gr.Row(): - prompts = [] - for i in range(max_xtiles*max_ytiles): - prompts.append(gr.Textbox('', label=f"Tile prompt: x={i%max_xtiles} y={i//max_ytiles}", placeholder='Prompt for tile', visible=False)) + x_overlap = gr.Slider(minimum=0, maximum=512, value=128, label="X-axis tile overlap") + y_overlap = gr.Slider(minimum=0, maximum=512, value=128, label="Y-axis tile overlap") + prompts = [] + for x in range(max_xtiles): + for y in range(max_ytiles): + with gr.Row(): + prompts.append(gr.Textbox('', label=f"Tile prompt: x={x+1} y={y+1}", placeholder='Prompt for tile', visible=False, lines=2)) x_tiles.change(fn=self.update_ui, inputs=[x_tiles, y_tiles], outputs=prompts) y_tiles.change(fn=self.update_ui, inputs=[x_tiles, y_tiles], outputs=prompts) - return [] + return [x_tiles, y_tiles, x_overlap, y_overlap] + prompts - def run(self, p: processing.StableDiffusionProcessing): # pylint: disable=arguments-differ, unused-argument - supported_model_list = ['sdxl'] - if shared.sd_model_type not in supported_model_list: - shared.log.warning(f'MoD: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={supported_model_list}') + def calc_size(self, size, tiles, overlap): + tile_size = (size / tiles) + (overlap / 2) if tiles > 1 else size + return 8 * int(tile_size // 8) + + def get_prompts(self, x_tiles, y_tiles, prompts, base_prompt, guidance): + y_prompts = [] + y_guidance = [] + for y in range(max_ytiles): + x_prompts = [] + x_guidance = [] + for x in range(max_xtiles): + if (x < x_tiles) and (y < y_tiles): + prompt = prompts[x * max_xtiles + y] + ' ' + base_prompt + x_prompts.append(prompt.strip()) + x_guidance.append(guidance) + if len(x_prompts) > 0: + y_prompts.append(x_prompts) + y_guidance.append(x_guidance) + return y_prompts, y_guidance + + def check_dependencies(self): + from installer import install + install('ligo-segments') + try: + from ligo.segments import segment # pylint: disable=unused-import + return True + except Exception as e: + shared.log.error(f'MoD: {e}') + return False + + def run(self, p: processing.StableDiffusionProcessing, *args): # pylint: disable=arguments-differ, unused-argument + if shared.sd_model_type not in supported_models: + shared.log.warning(f'MoD: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={supported_models}') + return None + if not self.check_dependencies(): return None - self.orig_pipe = shared.sd_model from modules.mod import StableDiffusionXLTilingPipeline + self.orig_pipe = shared.sd_model + self.orig_attn = shared.opts.prompt_attention + + [x_tiles, y_tiles, x_overlap, y_overlap], prompts = args[:4], args[4:] + p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) + p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) + p.prompts, guidance = self.get_prompts(x_tiles, y_tiles, prompts, p.prompt, p.cfg_scale) + p.all_prompts = p.prompts + p.task_args['prompts'] = p.prompts + p.task_args['negative_prompt'] = p.negative_prompt + p.task_args['tile_col_overlap'] = x_overlap if x_tiles > 1 else 0 + p.task_args['tile_row_overlap'] = y_overlap if y_tiles > 1 else 0 + p.task_args['tile_width'] = self.calc_size(p.width, x_tiles, x_overlap) + p.task_args['tile_height'] = self.calc_size(p.height, y_tiles, y_overlap) + p.task_args['guidance_scale_tiles'] = guidance + p.task_args['width'] = p.width + p.task_args['height'] = p.height + p.extra_generation_params["MoD X"] = f'{x_tiles}/{p.task_args["tile_width"]}/{p.task_args['tile_col_overlap']}' + p.extra_generation_params["MoD Y"] = f'{y_tiles}/{p.task_args["tile_height"]}/{p.task_args['tile_row_overlap']}' + p.keep_prompts = True + shared.opts.prompt_attention = 'fixed' + shared.log.info(f'MoD: xtiles={x_tiles} ytiles={y_tiles} xoverlap={p.task_args['tile_col_overlap']} yoverlap={p.task_args['tile_row_overlap']} xsize={p.task_args["tile_width"]} ysize={p.task_args["tile_height"]}') + shared.sd_model = sd_models.switch_pipe(StableDiffusionXLTilingPipeline, shared.sd_model) sd_models.set_diffuser_options(shared.sd_model) sd_models.apply_balanced_offload(shared.sd_model) - shared.log.info(f'MoD: ') - def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed): # pylint: disable=arguments-differ, unused-argument + def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=arguments-differ, unused-argument if self.orig_pipe is None: return processed if shared.sd_model_type == "sdxl": shared.sd_model = self.orig_pipe + if self.orig_attn is not None: + shared.opts.prompt_attention = self.orig_attn self.orig_pipe = None + self.orig_attn = None return processed