diff --git a/CHANGELOG.md b/CHANGELOG.md
index a79f2d3a5..6a2da8074 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,6 +28,14 @@
- download using networks -> reference
- see [wiki](https://github.com/vladmandic/automatic/wiki/Stable-Cascade) for details
- currently requires 10GB VRAM, lighter version is in development
+- [LEdit++](https://leditsplusplus-project.static.hf.space/index.html)
+ - context aware img2img method with image analysis and positive/negative prompt handling
+ - enable via img2img -> scripts -> ledit
+ - uses following params from standard img2img: cfg scale (recommended ~3), steps (recommended ~50), denoise strength (recommended ~0.7)
+ - can use postive and/or negative prompt to guide editing process
+ - positive prompt: what to enhance, strength and threshold for auto-masking
+ - negative prompt: what to remove, strength and threshold for auto-masking
+ - *note*: not compatible with model offloading
- **Visual Query** visual query & answer in process tab
- go to process -> visual query
- ask your questions, e.g. "describe the image", "what is behind the subject", "what are predominant colors of the image?"
diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py
index 0c785cbb1..8638bd83e 100644
--- a/modules/processing_diffusers.py
+++ b/modules/processing_diffusers.py
@@ -89,7 +89,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
ip_adapter_scales[i] *= float(step <= pipe.num_timesteps * ip_adapter_ends[i])
debug(f"Callback: IP Adapter scales={ip_adapter_scales}")
pipe.set_ip_adapter_scale(ip_adapter_scales)
- if step != pipe.num_timesteps:
+ if step != getattr(pipe, 'num_timesteps', 0):
kwargs = processing_correction.correction_callback(p, timestep, kwargs)
if p.scheduled_prompt and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs:
try:
@@ -99,7 +99,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
kwargs["negative_prompt_embeds"] = p.negative_embeds[j][0:1].expand(kwargs["negative_prompt_embeds"].shape)
except Exception as e:
shared.log.debug(f"Callback: {e}")
- if step == int(pipe.num_timesteps * p.cfg_end) and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs:
+ if step == int(getattr(pipe, 'num_timesteps', 100) * p.cfg_end) and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs:
pipe._guidance_scale = 0.0 # pylint: disable=protected-access
for key in {"prompt_embeds", "negative_prompt_embeds", "add_text_embeds", "add_time_ids"} & set(kwargs):
kwargs[key] = kwargs[key].chunk(2)[-1]
@@ -316,8 +316,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
def update_sampler(sd_model, second_pass=False):
sampler_selection = p.hr_sampler_name if second_pass else p.sampler_name
- if sd_model.__class__.__name__ in ['AmusedPipeline']:
- return # models with their own schedulers
if hasattr(sd_model, 'scheduler') and sampler_selection != 'Default':
sampler = sd_samplers.all_samplers_map.get(sampler_selection, None)
if sampler is None:
diff --git a/modules/sd_models.py b/modules/sd_models.py
index 381ec37bf..91fc06168 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -1136,9 +1136,9 @@ class DiffusersTaskType(Enum):
def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType:
- if pipe.__class__.__name__ == "StableVideoDiffusionPipeline":
+ if pipe.__class__.__name__ in ["StableVideoDiffusionPipeline", "LEditsPPPipelineStableDiffusion", "LEditsPPPipelineStableDiffusionXL"]:
return DiffusersTaskType.IMAGE_2_IMAGE
- if pipe.__class__.__name__ == "StableDiffusionXLInstructPix2PixPipeline":
+ elif pipe.__class__.__name__ == "StableDiffusionXLInstructPix2PixPipeline":
return DiffusersTaskType.INSTRUCT
elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING.values():
return DiffusersTaskType.IMAGE_2_IMAGE
diff --git a/scripts/ledits.py b/scripts/ledits.py
new file mode 100644
index 000000000..b87bb0fc1
--- /dev/null
+++ b/scripts/ledits.py
@@ -0,0 +1,101 @@
+import diffusers
+import gradio as gr
+from modules import scripts, processing, shared, devices, sd_models
+
+
+class Script(scripts.Script):
+ def title(self):
+ return 'LEdits++'
+
+ def show(self, is_img2img):
+ return is_img2img if shared.backend == shared.Backend.DIFFUSERS else False
+
+ # return signature is array of gradio components
+ def ui(self, _is_img2img):
+ with gr.Row():
+ gr.HTML('  LEdits++
')
+ with gr.Row():
+ edit_start = gr.Slider(label='Edit start', minimum=0.0, maximum=1.0, step=0.01, value=0.1)
+ edit_stop = gr.Slider(label='Edit stop', minimum=0.0, maximum=1.0, step=0.01, value=1.0)
+ intersect_mask = gr.Checkbox(label='Smooth mask', value=True)
+ with gr.Row():
+ prompt1 = gr.Textbox(show_label=False, placeholder='Positive prompt')
+ scale1 = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=0.5)
+ threshold1 = gr.Slider(label='Threshold', minimum=0.0, maximum=1.0, step=0.01, value=0.9)
+ with gr.Row():
+ prompt2 = gr.Textbox(show_label=False, placeholder='Negative prompt')
+ scale2 = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=0.5)
+ threshold2 = gr.Slider(label='Threshold', minimum=0.0, maximum=1.0, step=0.01, value=0.9)
+ return [edit_start, edit_stop, intersect_mask, prompt1, scale1, threshold1, prompt2, scale2, threshold2]
+
+ def run(self, p: processing.StableDiffusionProcessing, edit_start, edit_stop, intersect_mask, prompt1, scale1, threshold1, prompt2, scale2, threshold2): # pylint: disable=arguments-differ, unused-argument
+ image = getattr(p, 'init_images', None)
+ if len(prompt1) == 0 and len(prompt2) == 0:
+ shared.log.error('LEdits: no prompts')
+ return None
+ if image is None or len(image) == 0:
+ shared.log.error('LEdits: no init_images')
+ return None
+ else:
+ image = image[0]
+ if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
+ shared.log.error(f'LEdits: invalid model type: {shared.sd_model_type}')
+ return None
+
+ orig_pipeline = shared.sd_model
+ orig_offload = shared.opts.diffusers_model_cpu_offload
+ orig_prompt_attention = shared.opts.prompt_attention
+ shared.opts.data['diffusers_model_cpu_offload'] = False
+ shared.opts.data['prompt_attention'] = 'Fixed attention'
+ # shared.sd_model.maybe_free_model_hooks() # ledits is not compatible with offloading
+ # shared.sd_model.has_accelerate = False
+ sd_models.move_model(shared.sd_model, devices.device, force=True)
+ if shared.sd_model_type == 'sd':
+ shared.sd_model = sd_models.switch_pipe(diffusers.LEditsPPPipelineStableDiffusion, shared.sd_model)
+ elif shared.sd_model_type == 'sdxl':
+ shared.sd_model = sd_models.switch_pipe(diffusers.LEditsPPPipelineStableDiffusionXL, shared.sd_model)
+ if str(devices.dtype) == 'torch.float16':
+ shared.sd_model.vae.config.force_upcast = False # not compatible
+
+ shared.sd_model.scheduler = diffusers.DPMSolverMultistepScheduler.from_config(shared.sd_model.scheduler.config, algorithm_type="sde-dpmsolver++", solver_order=2) # ledits is very picky
+ p.sampler_name = 'Default'
+ invert_args = {
+ 'image': image,
+ 'source_prompt': p.prompt,
+ 'source_guidance_scale': p.cfg_scale,
+ 'num_inversion_steps': p.steps,
+ 'skip': 1.0 - p.denoising_strength, # invert start
+ 'generator': None, # not supported
+ }
+ shared.log.info(f'LEdits invert: {invert_args}')
+ _output = shared.sd_model.invert(**invert_args)
+ p.task_args = {
+ 'editing_prompt': [],
+ 'reverse_editing_direction': [],
+ 'edit_guidance_scale': [],
+ 'edit_threshold': [],
+ 'edit_warmup_steps': int(edit_start * p.steps),
+ 'edit_cooldown_steps': int((1.0 - edit_stop) * p.steps) if edit_stop < 1.0 else None,
+ 'use_intersect_mask': intersect_mask, # smoothing?
+ 'generator': None,
+ 'guidance_rescale': 0.0, # bug in pipeline if guidance rescale is enabled
+ }
+ if len(prompt1) > 0:
+ p.task_args['editing_prompt'].append(prompt1)
+ p.task_args['reverse_editing_direction'].append(False)
+ p.task_args['edit_guidance_scale'].append(10.0 * scale1)
+ p.task_args['edit_threshold'].append(threshold1)
+ if len(prompt2) > 0:
+ p.task_args['editing_prompt'].append(prompt2)
+ p.task_args['reverse_editing_direction'].append(True)
+ p.task_args['edit_guidance_scale'].append(10.0 * scale2)
+ p.task_args['edit_threshold'].append(threshold2)
+
+ shared.log.info(f'LEdits: {p.task_args}')
+ processed = processing.process_images(p)
+
+ # restore pipeline
+ shared.sd_model = orig_pipeline
+ shared.opts.data['prompt_attention'] = orig_prompt_attention
+ shared.opts.data['diffusers_model_cpu_offload'] = orig_offload
+ return processed