From 4033f2b63f58bc17d5d8eae035c2768213512962 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 14 Nov 2024 08:16:09 -0500 Subject: [PATCH] pulid enable inpaint mask only Signed-off-by: Vladimir Mandic --- TODO.md | 5 ----- modules/pulid/pulid_sdxl.py | 19 +++++++++++++++---- modules/sd_models.py | 22 +++++++++++++--------- scripts/pulid_ext.py | 4 ---- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/TODO.md b/TODO.md index ed3ebfbdb..d5cc19cf7 100644 --- a/TODO.md +++ b/TODO.md @@ -7,11 +7,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - sd35 ip-adapter - flux.1 ip-adapter - flow-match scheudlers: -- async lowvram: -- fp8: - ipadapter-negative: - include reference styles - -### Missing - - control api scripts compatibility diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index 7ee9a138e..01ca660a7 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -1,3 +1,4 @@ +from typing import Union import os import cv2 import insightface @@ -5,7 +6,7 @@ import numpy as np import torch import torch.nn as nn from PIL import Image -from diffusers import StableDiffusionXLPipeline +from diffusers import StableDiffusionXLPipeline, StableDiffusionXLImg2ImgPipeline, StableDiffusionXLInpaintPipeline from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput from huggingface_hub import hf_hub_download, snapshot_download @@ -28,7 +29,17 @@ debug = log.trace if os.environ.get('SD_PULID_DEBUG', None) is not None else lam class StableDiffusionXLPuLIDPipeline: - def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, dtype: torch.dtype=None, providers: list=None, offload: bool=True, sampler=None, cache_dir=None, sdp: bool=True, version: str='v1.1'): + def __init__(self, + pipe: Union[StableDiffusionXLPipeline, StableDiffusionXLImg2ImgPipeline, StableDiffusionXLInpaintPipeline], + device: torch.device, + dtype: torch.dtype=None, + providers: list=None, + offload: bool=True, + sampler=None, + cache_dir=None, + sdp: bool=True, + version: str='v1.1', + ): super().__init__() self.device = device self.dtype = dtype or torch.float16 @@ -282,7 +293,7 @@ class StableDiffusionXLPuLIDPipeline: t = self.timestep(sigma) x_ddim_space = x / (sigma[:, None, None, None] ** 2 + self.sigma_data**2) ** 0.5 cfg_scale = extra_args['cfg_scale'] - debug(f'PulID sample start: step={self.step+1} x={x.shape} dtype={x.dtype} timestep={t.item()} sigma={sigma.shape} cfg={cfg_scale} args={extra_args.keys()}') + # debug(f'PulID sample start: step={self.step+1} x={x.shape} dtype={x.dtype} timestep={t.item()} sigma={sigma.shape} cfg={cfg_scale} args={extra_args.keys()}') eps_positive = self.pipe.unet(x_ddim_space, t, return_dict=False, **extra_args['positive'])[0] eps_negative = self.pipe.unet(x_ddim_space, t, return_dict=False, **extra_args['negative'])[0] noise_pred = eps_negative + cfg_scale * (eps_positive - eps_negative) @@ -290,7 +301,7 @@ class StableDiffusionXLPuLIDPipeline: if self.callback_on_step_end is not None: self.step += 1 self.callback_on_step_end(self.pipe, step=self.step, timestep=t, kwargs={ 'latents': latent }) - debug(f'PulID sample end: step={self.step} x={latent.shape} dtype={x.dtype} min={torch.amin(latent)} max={torch.amax(latent)}') + # debug(f'PulID sample end: step={self.step} x={latent.shape} dtype={x.dtype} min={torch.amin(latent)} max={torch.amax(latent)}') return latent def init_latent(self, seed, size, image, mask_image, strength, width, height): # pylint: disable=unused-argument diff --git a/modules/sd_models.py b/modules/sd_models.py index e139895ea..be543cb49 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1081,9 +1081,10 @@ def set_diffuser_pipe(pipe, new_pipe_type): return pipe # skip specific pipelines + cls = pipe.__class__.__name__ if n in exclude: return pipe - if 'Onnx' in pipe.__class__.__name__: + if 'Onnx' in cls: return pipe new_pipe = None @@ -1114,27 +1115,27 @@ def set_diffuser_pipe(pipe, new_pipe_type): elif new_pipe_type == DiffusersTaskType.INPAINTING: new_pipe = diffusers.AutoPipelineForInpainting.from_pipe(pipe) else: - shared.log.error(f'Pipeline class change failed: type={new_pipe_type} pipeline={pipe.__class__.__name__}') + shared.log.error(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls}') return pipe except Exception as e: # pylint: disable=unused-variable - shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}') + shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls} {e}') return pipe else: try: # maybe a wrapper pipeline so just change the class if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: - pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING, pipe.__class__.__name__) # pylint: disable=protected-access + pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access new_pipe = pipe elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: - pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, pipe.__class__.__name__) # pylint: disable=protected-access + pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access new_pipe = pipe elif new_pipe_type == DiffusersTaskType.INPAINTING: - pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING, pipe.__class__.__name__) # pylint: disable=protected-access + pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING, cls) # pylint: disable=protected-access new_pipe = pipe else: - shared.log.error(f'Pipeline class change failed: type={new_pipe_type} pipeline={pipe.__class__.__name__}') + shared.log.error(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls}') return pipe except Exception as e: # pylint: disable=unused-variable - shared.log.warning(f'Pipeline class set failed: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}') + shared.log.warning(f'Pipeline class set failed: type={new_pipe_type} pipeline={cls} {e}') return pipe # if pipe.__class__ == new_pipe.__class__: @@ -1158,7 +1159,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): new_pipe.pipe = set_diffuser_pipe(new_pipe.pipe, new_pipe_type) fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access - shared.log.debug(f"Pipeline class change: original={pipe.__class__.__name__} target={new_pipe.__class__.__name__} device={pipe.device} fn={fn}") # pylint: disable=protected-access + shared.log.debug(f"Pipeline class change: original={cls} target={new_pipe.__class__.__name__} device={pipe.device} fn={fn}") # pylint: disable=protected-access pipe = new_pipe return pipe @@ -1187,6 +1188,9 @@ def set_diffusers_attention(pipe): else: module.set_attn_processor(attention) + if hasattr(pipe, 'pipe'): + set_diffusers_attention(pipe.pipe) + if 'ControlNet' in pipe.__class__.__name__: # do not replace attention in ControlNet pipelines return shared.log.debug(f'Setting model: attention="{shared.opts.cross_attention_optimization}"') diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 43039d73a..d01ca2847 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -18,7 +18,6 @@ class Script(scripts.Script): def __init__(self): self.pulid = None self.cache = None - self.mask_apply_overlay = shared.opts.mask_apply_overlay self.preprocess = 0 super().__init__() self.register() # pulid is script with processing override so xyz doesnt execute @@ -151,8 +150,6 @@ class Script(scripts.Script): shared.log.warning('PuLID: batch size not supported') p.batch_size = 1 - self.mask_apply_overlay = shared.opts.mask_apply_overlay - shared.opts.data['mask_apply_overlay'] = False sdp = shared.opts.cross_attention_optimization == "Scaled-Dot-Product" strength = getattr(p, 'pulid_strength', strength) zero = getattr(p, 'pulid_zero', zero) @@ -242,7 +239,6 @@ class Script(scripts.Script): def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=unused-argument _strength, _zero, _sampler, _ortho, _gallery, restore, _offload, _version = args if hasattr(shared.sd_model, 'pipe') and shared.sd_model_type == "sdxl": - shared.opts.data['mask_apply_overlay'] = self.mask_apply_overlay restore = getattr(p, 'pulid_restore', restore) if restore: if hasattr(shared.sd_model, 'app'):