diff --git a/CHANGELOG.md b/CHANGELOG.md index 0970a09a5..c0c5903e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ - handle extensions that install conflicting versions of packages `onnxruntime`, `opencv2-python` - installer refresh package cache on any install + - fix embeddings registration on server startup, thanks @AI-Casanova - ipex handle dependencies, thanks @Disty0 - insightface handle dependencies - img2img mask blur and padding @@ -62,6 +63,7 @@ - fix interrogate api endpoint - control fix resize causing runtime errors - control fix processor override image after processor change + - handle pipelines that return dict instead of object - fix vae dtype mismatch, thanks @Disty0 - fix controlnet inpaint mask - fix extensions update information in ui diff --git a/TODO.md b/TODO.md index 4fcf2c936..8edee25c0 100644 --- a/TODO.md +++ b/TODO.md @@ -6,6 +6,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - control second pass: - onediff: +- regional prompting pipeline: - diffusers public callbacks - image2video: pia and vgen pipelines - video2video diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 051ca13a1..2703c37c5 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -1,3 +1,4 @@ +from types import SimpleNamespace import os import time import math @@ -437,6 +438,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing): try: t0 = time.time() output = shared.sd_model(**base_args) # pylint: disable=not-callable + if isinstance(output, dict): + output = SimpleNamespace(**output) openvino_post_compile(op="base") # only executes on compiled vino models if shared.cmd_opts.profile: t1 = time.time() @@ -446,9 +449,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing): shared.log.debug(f'Generated: frames={output.frames[0].shape[1]}') else: shared.log.debug(f'Generated: frames={len(output.frames[0])}') - if isinstance(output, dict): - from types import SimpleNamespace - output = SimpleNamespace(**output) output.images = output.frames[0] if isinstance(output.images, np.ndarray): output.images = torch.from_numpy(output.images) @@ -512,6 +512,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing): shared.state.sampling_steps = hires_args['num_inference_steps'] try: output = shared.sd_model(**hires_args) # pylint: disable=not-callable + if isinstance(output, dict): + output = SimpleNamespace(**output) openvino_post_compile(op="base") except AssertionError as e: shared.log.info(e) @@ -570,6 +572,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing): if 'requires_aesthetics_score' in shared.sd_refiner.config: shared.sd_refiner.register_to_config(requires_aesthetics_score=shared.opts.diffusers_aesthetics_score) refiner_output = shared.sd_refiner(**refiner_args) # pylint: disable=not-callable + if isinstance(refiner_output, dict): + refiner_output = SimpleNamespace(**refiner_output) openvino_post_compile(op="refiner") except AssertionError as e: shared.log.info(e) @@ -589,9 +593,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing): # final decode since there is no refiner if not is_refiner_enabled(): if output is not None: - if isinstance(output, dict): - from types import SimpleNamespace - output = SimpleNamespace(**output) if not hasattr(output, 'images') and hasattr(output, 'frames'): shared.log.debug(f'Generated: frames={len(output.frames[0])}') output.images = output.frames[0] diff --git a/scripts/regional_prompting.py b/scripts/regional_prompting.py new file mode 100644 index 000000000..677d712a1 --- /dev/null +++ b/scripts/regional_prompting.py @@ -0,0 +1,67 @@ +# https://github.com/huggingface/diffusers/blob/main/examples/community/README.md#regional-prompting-pipeline +# https://github.com/huggingface/diffusers/blob/main/examples/community/regional_prompting_stable_diffusion.py + +import gradio as gr +from modules import shared, devices, scripts, processing, sd_models + + +class Script(scripts.Script): + def title(self): + return 'Regional prompting' + + def show(self, is_img2img): + return False + return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + + def change(self, mode): + return [gr.update(visible='Col' in mode or 'Row' in mode), gr.update(visible='Prompt' in mode)] + + def ui(self, _is_img2img): + with gr.Row(): + gr.HTML('  Regional prompting') + with gr.Row(): + mode = gr.Radio(label='Mode', choices=['None', 'Prompt', 'Prompt EX', 'Columns', 'Rows'], value='None') + with gr.Row(): + power = gr.Slider(label='Power', minimum=0, maximum=1, value=1.0, step=0.01) + threshold = gr.Textbox('', label='Prompt thresholds:', default='', visible=False) + grid = gr.Text('', label='Grid sections:', default='', visible=False) + mode.change(fn=self.change, inputs=[mode], outputs=[grid, threshold]) + return mode, grid, power, threshold + + def run(self, p: processing.StableDiffusionProcessing, mode, grid, power, threshold): # pylint: disable=arguments-differ + if mode is None or mode == 'None': + return + # backup pipeline and params + orig_pipeline = shared.sd_model + orig_dtype = devices.dtype + orig_prompt_attention = shared.opts.prompt_attention + # create pipeline + if shared.sd_model_type != 'sd': + shared.log.error(f'Regional prompting: incorrect base model: {shared.sd_model.__class__.__name__}') + return + shared.sd_model = sd_models.switch_pipe('regional_prompting_stable_diffusion', shared.sd_model) + if shared.sd_model.__class__.__name__ != 'RegionalPromptingStableDiffusionPipeline': # switch failed + shared.log.error(f'Regional prompting: not a tiling pipeline: {shared.sd_model.__class__.__name__}') + shared.sd_model = orig_pipeline + return + sd_models.set_diffuser_options(shared.sd_model) + shared.opts.data['prompt_attention'] = 'Fixed attention' # this pipeline is not compatible with embeds + processing.fix_seed(p) + # set pipeline specific params, note that standard params are applied when applicable + rp_args = { + 'mode': mode.lower(), + 'power': power, + } + if 'prompt' in mode.lower(): + rp_args['th'] = threshold + else: + rp_args['div'] = grid + p.task_args = { **p.task_args, 'rp_args': rp_args } + # run pipeline + shared.log.debug(f'Regional: args={p.task_args}') + processed: processing.Processed = processing.process_images(p) # runs processing using main loop + # restore pipeline and params + shared.opts.data['prompt_attention'] = orig_prompt_attention + shared.sd_model = orig_pipeline + shared.sd_model.to(orig_dtype) + return processed