diff --git a/CHANGELOG.md b/CHANGELOG.md index a79f2d3a5..6a2da8074 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,14 @@ - download using networks -> reference - see [wiki](https://github.com/vladmandic/automatic/wiki/Stable-Cascade) for details - currently requires 10GB VRAM, lighter version is in development +- [LEdit++](https://leditsplusplus-project.static.hf.space/index.html) + - context aware img2img method with image analysis and positive/negative prompt handling + - enable via img2img -> scripts -> ledit + - uses following params from standard img2img: cfg scale (recommended ~3), steps (recommended ~50), denoise strength (recommended ~0.7) + - can use postive and/or negative prompt to guide editing process + - positive prompt: what to enhance, strength and threshold for auto-masking + - negative prompt: what to remove, strength and threshold for auto-masking + - *note*: not compatible with model offloading - **Visual Query** visual query & answer in process tab - go to process -> visual query - ask your questions, e.g. "describe the image", "what is behind the subject", "what are predominant colors of the image?" diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 0c785cbb1..8638bd83e 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -89,7 +89,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): ip_adapter_scales[i] *= float(step <= pipe.num_timesteps * ip_adapter_ends[i]) debug(f"Callback: IP Adapter scales={ip_adapter_scales}") pipe.set_ip_adapter_scale(ip_adapter_scales) - if step != pipe.num_timesteps: + if step != getattr(pipe, 'num_timesteps', 0): kwargs = processing_correction.correction_callback(p, timestep, kwargs) if p.scheduled_prompt and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs: try: @@ -99,7 +99,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): kwargs["negative_prompt_embeds"] = p.negative_embeds[j][0:1].expand(kwargs["negative_prompt_embeds"].shape) except Exception as e: shared.log.debug(f"Callback: {e}") - if step == int(pipe.num_timesteps * p.cfg_end) and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs: + if step == int(getattr(pipe, 'num_timesteps', 100) * p.cfg_end) and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs: pipe._guidance_scale = 0.0 # pylint: disable=protected-access for key in {"prompt_embeds", "negative_prompt_embeds", "add_text_embeds", "add_time_ids"} & set(kwargs): kwargs[key] = kwargs[key].chunk(2)[-1] @@ -316,8 +316,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing): def update_sampler(sd_model, second_pass=False): sampler_selection = p.hr_sampler_name if second_pass else p.sampler_name - if sd_model.__class__.__name__ in ['AmusedPipeline']: - return # models with their own schedulers if hasattr(sd_model, 'scheduler') and sampler_selection != 'Default': sampler = sd_samplers.all_samplers_map.get(sampler_selection, None) if sampler is None: diff --git a/modules/sd_models.py b/modules/sd_models.py index 381ec37bf..91fc06168 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1136,9 +1136,9 @@ class DiffusersTaskType(Enum): def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType: - if pipe.__class__.__name__ == "StableVideoDiffusionPipeline": + if pipe.__class__.__name__ in ["StableVideoDiffusionPipeline", "LEditsPPPipelineStableDiffusion", "LEditsPPPipelineStableDiffusionXL"]: return DiffusersTaskType.IMAGE_2_IMAGE - if pipe.__class__.__name__ == "StableDiffusionXLInstructPix2PixPipeline": + elif pipe.__class__.__name__ == "StableDiffusionXLInstructPix2PixPipeline": return DiffusersTaskType.INSTRUCT elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING.values(): return DiffusersTaskType.IMAGE_2_IMAGE diff --git a/scripts/ledits.py b/scripts/ledits.py new file mode 100644 index 000000000..b87bb0fc1 --- /dev/null +++ b/scripts/ledits.py @@ -0,0 +1,101 @@ +import diffusers +import gradio as gr +from modules import scripts, processing, shared, devices, sd_models + + +class Script(scripts.Script): + def title(self): + return 'LEdits++' + + def show(self, is_img2img): + return is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + + # return signature is array of gradio components + def ui(self, _is_img2img): + with gr.Row(): + gr.HTML('  LEdits++
') + with gr.Row(): + edit_start = gr.Slider(label='Edit start', minimum=0.0, maximum=1.0, step=0.01, value=0.1) + edit_stop = gr.Slider(label='Edit stop', minimum=0.0, maximum=1.0, step=0.01, value=1.0) + intersect_mask = gr.Checkbox(label='Smooth mask', value=True) + with gr.Row(): + prompt1 = gr.Textbox(show_label=False, placeholder='Positive prompt') + scale1 = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=0.5) + threshold1 = gr.Slider(label='Threshold', minimum=0.0, maximum=1.0, step=0.01, value=0.9) + with gr.Row(): + prompt2 = gr.Textbox(show_label=False, placeholder='Negative prompt') + scale2 = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=0.5) + threshold2 = gr.Slider(label='Threshold', minimum=0.0, maximum=1.0, step=0.01, value=0.9) + return [edit_start, edit_stop, intersect_mask, prompt1, scale1, threshold1, prompt2, scale2, threshold2] + + def run(self, p: processing.StableDiffusionProcessing, edit_start, edit_stop, intersect_mask, prompt1, scale1, threshold1, prompt2, scale2, threshold2): # pylint: disable=arguments-differ, unused-argument + image = getattr(p, 'init_images', None) + if len(prompt1) == 0 and len(prompt2) == 0: + shared.log.error('LEdits: no prompts') + return None + if image is None or len(image) == 0: + shared.log.error('LEdits: no init_images') + return None + else: + image = image[0] + if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl': + shared.log.error(f'LEdits: invalid model type: {shared.sd_model_type}') + return None + + orig_pipeline = shared.sd_model + orig_offload = shared.opts.diffusers_model_cpu_offload + orig_prompt_attention = shared.opts.prompt_attention + shared.opts.data['diffusers_model_cpu_offload'] = False + shared.opts.data['prompt_attention'] = 'Fixed attention' + # shared.sd_model.maybe_free_model_hooks() # ledits is not compatible with offloading + # shared.sd_model.has_accelerate = False + sd_models.move_model(shared.sd_model, devices.device, force=True) + if shared.sd_model_type == 'sd': + shared.sd_model = sd_models.switch_pipe(diffusers.LEditsPPPipelineStableDiffusion, shared.sd_model) + elif shared.sd_model_type == 'sdxl': + shared.sd_model = sd_models.switch_pipe(diffusers.LEditsPPPipelineStableDiffusionXL, shared.sd_model) + if str(devices.dtype) == 'torch.float16': + shared.sd_model.vae.config.force_upcast = False # not compatible + + shared.sd_model.scheduler = diffusers.DPMSolverMultistepScheduler.from_config(shared.sd_model.scheduler.config, algorithm_type="sde-dpmsolver++", solver_order=2) # ledits is very picky + p.sampler_name = 'Default' + invert_args = { + 'image': image, + 'source_prompt': p.prompt, + 'source_guidance_scale': p.cfg_scale, + 'num_inversion_steps': p.steps, + 'skip': 1.0 - p.denoising_strength, # invert start + 'generator': None, # not supported + } + shared.log.info(f'LEdits invert: {invert_args}') + _output = shared.sd_model.invert(**invert_args) + p.task_args = { + 'editing_prompt': [], + 'reverse_editing_direction': [], + 'edit_guidance_scale': [], + 'edit_threshold': [], + 'edit_warmup_steps': int(edit_start * p.steps), + 'edit_cooldown_steps': int((1.0 - edit_stop) * p.steps) if edit_stop < 1.0 else None, + 'use_intersect_mask': intersect_mask, # smoothing? + 'generator': None, + 'guidance_rescale': 0.0, # bug in pipeline if guidance rescale is enabled + } + if len(prompt1) > 0: + p.task_args['editing_prompt'].append(prompt1) + p.task_args['reverse_editing_direction'].append(False) + p.task_args['edit_guidance_scale'].append(10.0 * scale1) + p.task_args['edit_threshold'].append(threshold1) + if len(prompt2) > 0: + p.task_args['editing_prompt'].append(prompt2) + p.task_args['reverse_editing_direction'].append(True) + p.task_args['edit_guidance_scale'].append(10.0 * scale2) + p.task_args['edit_threshold'].append(threshold2) + + shared.log.info(f'LEdits: {p.task_args}') + processed = processing.process_images(p) + + # restore pipeline + shared.sd_model = orig_pipeline + shared.opts.data['prompt_attention'] = orig_prompt_attention + shared.opts.data['diffusers_model_cpu_offload'] = orig_offload + return processed