add blip diffusion

CHANGELOG.md

@@ -27,6 +27,10 @@
     - select from *networks -> reference*  
   - [Playground v1](https://huggingface.co/playgroundai/playground-v1), [Playground v2 256](https://huggingface.co/playgroundai/playground-v2-256px-base), [Playground v2 512](https://huggingface.co/playgroundai/playground-v2-512px-base), [Playground v2 1024](https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic) model support  
     - simply select from *networks -> reference* and use as usual  
+  - [BLIP-Diffusion](https://dxli94.github.io/BLIP-Diffusion-website/)  
+    - img2img model that can replace subjects in images using prompt keywords  
+    - download and load by selecting from *networks -> reference -> blip diffusion*
+    - in image tab, select `blip diffusion` script
   - [DemoFusion](https://github.com/PRIS-CV/DemoFusion) run your SDXL generations at any resolution!  
     - in **Text** tab select *script* -> *demofusion*  
     - *note*: GPU VRAM limits do not automatically go away so be careful when using it with large resolutions  

html/reference.json

@@ -113,5 +113,10 @@
     "path": "thu-ml/unidiffuser-v1",
     "desc": "UniDiffuser is a unified diffusion framework to fit all distributions relevant to a set of multi-modal data in one transformer. UniDiffuser is able to perform image, text, text-to-image, image-to-text, and image-text pair generation by setting proper timesteps without additional overhead.\nSpecifically, UniDiffuser employs a variation of transformer, called U-ViT, which parameterizes the joint noise prediction network. Other components perform as encoders and decoders of different modalities, including a pretrained image autoencoder from Stable Diffusion, a pretrained image ViT-B/32 CLIP encoder, a pretrained text ViT-L CLIP encoder, and a GPT-2 text decoder finetuned by ourselves.",
     "preview": "thu-ml--unidiffuser-v1.jpg"
+  },
+  "SalesForce BLIP-Diffusion": {
+    "path": "salesforce/blipdiffusion",
+    "desc": "BLIP-Diffusion, a new subject-driven image generation model that supports multimodal control which consumes inputs of subject images and text prompts. Unlike other subject-driven generation models, BLIP-Diffusion introduces a new multimodal encoder which is pre-trained to provide subject representation.",
+    "preview": "salesforce--blipdiffusion.jpg"
   }
 }

modules/ui.py

@@ -484,7 +484,7 @@ def create_hires_inputs(tab):
                 refiner_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Refiner start', value=0.8, elem_id=f"{tab}_refiner_start")
                 refiner_steps = gr.Slider(minimum=0, maximum=99, step=1, label="Refiner steps", elem_id=f"{tab}_refiner_steps", value=5)
             with FormRow(elem_id=f"{tab}_refiner_row3", variant="compact"):
-                refiner_prompt = gr.Textbox(value='', label='Secondary Prompt', elem_id=f"{tab}_refiner_prompt")
+                refiner_prompt = gr.Textbox(value='', label='Secondary prompt', elem_id=f"{tab}_refiner_prompt")
             with FormRow(elem_id="txt2img_refiner_row4", variant="compact"):
                 refiner_negative = gr.Textbox(value='', label='Secondary negative prompt', elem_id=f"{tab}_refiner_neg_prompt")
     return enable_hr, latent_index, denoising_strength, hr_final_resolution, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative

scripts/blipdiffusion.py

@@ -0,0 +1,43 @@
+import gradio as gr
+from modules import scripts, processing, shared, sd_models
+
+
+title = 'BLIP Diffusion'
+
+
+class Script(scripts.Script):
+    def title(self):
+        return title
+
+    def show(self, is_img2img):
+        return is_img2img if shared.backend == shared.Backend.DIFFUSERS else False
+
+    def ui(self, _is_img2img):
+        with gr.Row():
+            source_subject = gr.Textbox(value='', label='Source subject')
+        with gr.Row():
+            target_subject = gr.Textbox(value='', label='Target subject')
+        with gr.Row():
+            prompt_strength = gr.Slider(label='Prompt strength', minimum=0.0, maximum=1.0, step=0.01, value=0.5)
+        return [source_subject, target_subject, prompt_strength]
+
+    def run(self, p: processing.StableDiffusionProcessing, source_subject, target_subject, prompt_strength): # pylint: disable=arguments-differ, unused-argument
+        c = shared.sd_model.__class__.__name__ if shared.sd_model is not None else ''
+        if c != 'BlipDiffusionPipeline':
+            shared.log.error(f'{title}: model selected={c} required=BLIPDiffusion')
+            return None
+        if hasattr(p, 'init_images') and len(p.init_images) > 0:
+            p.task_args['reference_image'] = p.init_images[0]
+            p.task_args['prompt'] = [p.prompt]
+            p.task_args['neg_prompt'] = p.negative_prompt
+            p.task_args['prompt_strength'] = prompt_strength
+            p.task_args['source_subject_category'] = [source_subject]
+            p.task_args['target_subject_category'] = [target_subject]
+            p.task_args['output_type'] = 'pil'
+            shared.log.debug(f'BLIP Diffusion: args={p.task_args}')
+            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
+            processed = processing.process_images(p)
+            return processed
+        else:
+            shared.log.error(f'{title}: no init_images')
+            return None

scripts/stablevideodiffusion.py

@@ -47,6 +47,7 @@ class Script(scripts.Script):
     def run(self, p: processing.StableDiffusionProcessing, num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
         c = shared.sd_model.__class__.__name__ if shared.sd_model is not None else ''
         if c != 'StableVideoDiffusionPipeline' and c != 'TextToVideoSDPipeline':
+            shared.log.error(f'StableVideo: model selected={c} required=StableVideoDiffusion')
             return None
         if hasattr(p, 'init_images') and len(p.init_images) > 0:
             if override_resolution: