diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cd934a21..c3453f8d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,10 @@ - select from *networks -> reference* - [Playground v1](https://huggingface.co/playgroundai/playground-v1), [Playground v2 256](https://huggingface.co/playgroundai/playground-v2-256px-base), [Playground v2 512](https://huggingface.co/playgroundai/playground-v2-512px-base), [Playground v2 1024](https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic) model support - simply select from *networks -> reference* and use as usual + - [BLIP-Diffusion](https://dxli94.github.io/BLIP-Diffusion-website/) + - img2img model that can replace subjects in images using prompt keywords + - download and load by selecting from *networks -> reference -> blip diffusion* + - in image tab, select `blip diffusion` script - [DemoFusion](https://github.com/PRIS-CV/DemoFusion) run your SDXL generations at any resolution! - in **Text** tab select *script* -> *demofusion* - *note*: GPU VRAM limits do not automatically go away so be careful when using it with large resolutions diff --git a/html/reference.json b/html/reference.json index 4043fe2da..5c8d26378 100644 --- a/html/reference.json +++ b/html/reference.json @@ -113,5 +113,10 @@ "path": "thu-ml/unidiffuser-v1", "desc": "UniDiffuser is a unified diffusion framework to fit all distributions relevant to a set of multi-modal data in one transformer. UniDiffuser is able to perform image, text, text-to-image, image-to-text, and image-text pair generation by setting proper timesteps without additional overhead.\nSpecifically, UniDiffuser employs a variation of transformer, called U-ViT, which parameterizes the joint noise prediction network. Other components perform as encoders and decoders of different modalities, including a pretrained image autoencoder from Stable Diffusion, a pretrained image ViT-B/32 CLIP encoder, a pretrained text ViT-L CLIP encoder, and a GPT-2 text decoder finetuned by ourselves.", "preview": "thu-ml--unidiffuser-v1.jpg" + }, + "SalesForce BLIP-Diffusion": { + "path": "salesforce/blipdiffusion", + "desc": "BLIP-Diffusion, a new subject-driven image generation model that supports multimodal control which consumes inputs of subject images and text prompts. Unlike other subject-driven generation models, BLIP-Diffusion introduces a new multimodal encoder which is pre-trained to provide subject representation.", + "preview": "salesforce--blipdiffusion.jpg" } } \ No newline at end of file diff --git a/models/Reference/salesforce--blipdiffusion.jpg b/models/Reference/salesforce--blipdiffusion.jpg new file mode 100644 index 000000000..79155aba9 Binary files /dev/null and b/models/Reference/salesforce--blipdiffusion.jpg differ diff --git a/modules/ui.py b/modules/ui.py index 3a7f4f52e..0ad53a98e 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -484,7 +484,7 @@ def create_hires_inputs(tab): refiner_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Refiner start', value=0.8, elem_id=f"{tab}_refiner_start") refiner_steps = gr.Slider(minimum=0, maximum=99, step=1, label="Refiner steps", elem_id=f"{tab}_refiner_steps", value=5) with FormRow(elem_id=f"{tab}_refiner_row3", variant="compact"): - refiner_prompt = gr.Textbox(value='', label='Secondary Prompt', elem_id=f"{tab}_refiner_prompt") + refiner_prompt = gr.Textbox(value='', label='Secondary prompt', elem_id=f"{tab}_refiner_prompt") with FormRow(elem_id="txt2img_refiner_row4", variant="compact"): refiner_negative = gr.Textbox(value='', label='Secondary negative prompt', elem_id=f"{tab}_refiner_neg_prompt") return enable_hr, latent_index, denoising_strength, hr_final_resolution, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative diff --git a/scripts/blipdiffusion.py b/scripts/blipdiffusion.py new file mode 100644 index 000000000..50ae01021 --- /dev/null +++ b/scripts/blipdiffusion.py @@ -0,0 +1,43 @@ +import gradio as gr +from modules import scripts, processing, shared, sd_models + + +title = 'BLIP Diffusion' + + +class Script(scripts.Script): + def title(self): + return title + + def show(self, is_img2img): + return is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + + def ui(self, _is_img2img): + with gr.Row(): + source_subject = gr.Textbox(value='', label='Source subject') + with gr.Row(): + target_subject = gr.Textbox(value='', label='Target subject') + with gr.Row(): + prompt_strength = gr.Slider(label='Prompt strength', minimum=0.0, maximum=1.0, step=0.01, value=0.5) + return [source_subject, target_subject, prompt_strength] + + def run(self, p: processing.StableDiffusionProcessing, source_subject, target_subject, prompt_strength): # pylint: disable=arguments-differ, unused-argument + c = shared.sd_model.__class__.__name__ if shared.sd_model is not None else '' + if c != 'BlipDiffusionPipeline': + shared.log.error(f'{title}: model selected={c} required=BLIPDiffusion') + return None + if hasattr(p, 'init_images') and len(p.init_images) > 0: + p.task_args['reference_image'] = p.init_images[0] + p.task_args['prompt'] = [p.prompt] + p.task_args['neg_prompt'] = p.negative_prompt + p.task_args['prompt_strength'] = prompt_strength + p.task_args['source_subject_category'] = [source_subject] + p.task_args['target_subject_category'] = [target_subject] + p.task_args['output_type'] = 'pil' + shared.log.debug(f'BLIP Diffusion: args={p.task_args}') + shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) + processed = processing.process_images(p) + return processed + else: + shared.log.error(f'{title}: no init_images') + return None diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index 078c0a054..3e20c4373 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -47,6 +47,7 @@ class Script(scripts.Script): def run(self, p: processing.StableDiffusionProcessing, num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument c = shared.sd_model.__class__.__name__ if shared.sd_model is not None else '' if c != 'StableVideoDiffusionPipeline' and c != 'TextToVideoSDPipeline': + shared.log.error(f'StableVideo: model selected={c} required=StableVideoDiffusion') return None if hasattr(p, 'init_images') and len(p.init_images) > 0: if override_resolution: