handle dict as pipeline result

CHANGELOG.md

@@ -53,6 +53,7 @@
   - handle extensions that install conflicting versions of packages  
     `onnxruntime`, `opencv2-python`  
   - installer refresh package cache on any install  
+  - fix embeddings registration on server startup, thanks @AI-Casanova
   - ipex handle dependencies, thanks @Disty0  
   - insightface handle dependencies  
   - img2img mask blur and padding  
@@ -62,6 +63,7 @@
   - fix interrogate api endpoint  
   - control fix resize causing runtime errors  
   - control fix processor override image after processor change  
+  - handle pipelines that return dict instead of object  
   - fix vae dtype mismatch, thanks @Disty0  
   - fix controlnet inpaint mask  
   - fix extensions update information in ui  

TODO.md

@@ -6,6 +6,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
 
 - control second pass: <https://github.com/vladmandic/automatic/issues/2783>  
 - onediff: <https://github.com/siliconflow/onediff>
+- regional prompting pipeline: <https://github.com/huggingface/diffusers/blob/main/examples/community/README.md#regional-prompting-pipeline>
 - diffusers public callbacks  
 - image2video: pia and vgen pipelines  
 - video2video

modules/processing_diffusers.py

@@ -1,3 +1,4 @@
+from types import SimpleNamespace
 import os
 import time
 import math
@@ -437,6 +438,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
     try:
         t0 = time.time()
         output = shared.sd_model(**base_args) # pylint: disable=not-callable
+        if isinstance(output, dict):
+            output = SimpleNamespace(**output)
         openvino_post_compile(op="base") # only executes on compiled vino models
         if shared.cmd_opts.profile:
             t1 = time.time()
@@ -446,9 +449,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
                 shared.log.debug(f'Generated: frames={output.frames[0].shape[1]}')
             else:
                 shared.log.debug(f'Generated: frames={len(output.frames[0])}')
-            if isinstance(output, dict):
-                from types import SimpleNamespace
-                output = SimpleNamespace(**output)
             output.images = output.frames[0]
         if isinstance(output.images, np.ndarray):
             output.images = torch.from_numpy(output.images)
@@ -512,6 +512,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
                 shared.state.sampling_steps = hires_args['num_inference_steps']
                 try:
                     output = shared.sd_model(**hires_args) # pylint: disable=not-callable
+                    if isinstance(output, dict):
+                        output = SimpleNamespace(**output)
                     openvino_post_compile(op="base")
                 except AssertionError as e:
                     shared.log.info(e)
@@ -570,6 +572,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
                 if 'requires_aesthetics_score' in shared.sd_refiner.config:
                     shared.sd_refiner.register_to_config(requires_aesthetics_score=shared.opts.diffusers_aesthetics_score)
                 refiner_output = shared.sd_refiner(**refiner_args) # pylint: disable=not-callable
+                if isinstance(refiner_output, dict):
+                    refiner_output = SimpleNamespace(**refiner_output)
                 openvino_post_compile(op="refiner")
             except AssertionError as e:
                 shared.log.info(e)
@@ -589,9 +593,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
     # final decode since there is no refiner
     if not is_refiner_enabled():
         if output is not None:
-            if isinstance(output, dict):
-                from types import SimpleNamespace
-                output = SimpleNamespace(**output)
             if not hasattr(output, 'images') and hasattr(output, 'frames'):
                 shared.log.debug(f'Generated: frames={len(output.frames[0])}')
                 output.images = output.frames[0]

scripts/regional_prompting.py

@@ -0,0 +1,67 @@
+# https://github.com/huggingface/diffusers/blob/main/examples/community/README.md#regional-prompting-pipeline
+# https://github.com/huggingface/diffusers/blob/main/examples/community/regional_prompting_stable_diffusion.py
+
+import gradio as gr
+from modules import shared, devices, scripts, processing, sd_models
+
+
+class Script(scripts.Script):
+    def title(self):
+        return 'Regional prompting'
+
+    def show(self, is_img2img):
+        return False
+        return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False
+
+    def change(self, mode):
+        return [gr.update(visible='Col' in mode or 'Row' in mode), gr.update(visible='Prompt' in mode)]
+
+    def ui(self, _is_img2img):
+        with gr.Row():
+            gr.HTML('<a href="https://github.com/huggingface/diffusers/blob/main/examples/community/README.md#regional-prompting-pipeline">&nbsp Regional prompting</a>')
+        with gr.Row():
+            mode = gr.Radio(label='Mode', choices=['None', 'Prompt', 'Prompt EX', 'Columns', 'Rows'], value='None')
+        with gr.Row():
+            power = gr.Slider(label='Power', minimum=0, maximum=1, value=1.0, step=0.01)
+            threshold = gr.Textbox('', label='Prompt thresholds:', default='', visible=False)
+            grid = gr.Text('', label='Grid sections:', default='', visible=False)
+        mode.change(fn=self.change, inputs=[mode], outputs=[grid, threshold])
+        return mode, grid, power, threshold
+
+    def run(self, p: processing.StableDiffusionProcessing, mode, grid, power, threshold): # pylint: disable=arguments-differ
+        if mode is None or mode == 'None':
+            return
+        # backup pipeline and params
+        orig_pipeline = shared.sd_model
+        orig_dtype = devices.dtype
+        orig_prompt_attention = shared.opts.prompt_attention
+        # create pipeline
+        if shared.sd_model_type != 'sd':
+            shared.log.error(f'Regional prompting: incorrect base model: {shared.sd_model.__class__.__name__}')
+            return
+        shared.sd_model = sd_models.switch_pipe('regional_prompting_stable_diffusion', shared.sd_model)
+        if shared.sd_model.__class__.__name__ != 'RegionalPromptingStableDiffusionPipeline': # switch failed
+            shared.log.error(f'Regional prompting: not a tiling pipeline: {shared.sd_model.__class__.__name__}')
+            shared.sd_model = orig_pipeline
+            return
+        sd_models.set_diffuser_options(shared.sd_model)
+        shared.opts.data['prompt_attention'] = 'Fixed attention' # this pipeline is not compatible with embeds
+        processing.fix_seed(p)
+        # set pipeline specific params, note that standard params are applied when applicable
+        rp_args = {
+            'mode': mode.lower(),
+            'power': power,
+        }
+        if 'prompt' in mode.lower():
+            rp_args['th'] = threshold
+        else:
+            rp_args['div'] = grid
+        p.task_args = { **p.task_args, 'rp_args': rp_args }
+        # run pipeline
+        shared.log.debug(f'Regional: args={p.task_args}')
+        processed: processing.Processed = processing.process_images(p) # runs processing using main loop
+        # restore pipeline and params
+        shared.opts.data['prompt_attention'] = orig_prompt_attention
+        shared.sd_model = orig_pipeline
+        shared.sd_model.to(orig_dtype)
+        return processed