Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 01:04:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

pulid enable inpaint mask only

Browse Source 
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
2024-11-14 08:16:09 -05:00
parent b59a21f924
commit 4033f2b63f
4 changed files with 28 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
TODO.md
-5
View File
@@ -7,11 +7,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
- sd35 ip-adapter
- flux.1 ip-adapter
- flow-match scheudlers: <https://github.com/huggingface/diffusers/issues/9607>
- async lowvram: <https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/14855>
- fp8: <https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/14031>
- ipadapter-negative: <https://github.com/huggingface/diffusers/discussions/7167>
- include reference styles
### Missing
- control api scripts compatibility
modules/pulid/pulid_sdxl.py
+15 -4
View File
@@ -1,3 +1,4 @@
from typing import Union
import os
import cv2
import insightface
@@ -5,7 +6,7 @@ import numpy as np
import torch
import torch.nn as nn
from PIL import Image
from diffusers import StableDiffusionXLPipeline
from diffusers import StableDiffusionXLPipeline, StableDiffusionXLImg2ImgPipeline, StableDiffusionXLInpaintPipeline
from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput
from huggingface_hub import hf_hub_download, snapshot_download
@@ -28,7 +29,17 @@ debug = log.trace if os.environ.get('SD_PULID_DEBUG', None) is not None else lam
class StableDiffusionXLPuLIDPipeline:
def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, dtype: torch.dtype=None, providers: list=None, offload: bool=True, sampler=None, cache_dir=None, sdp: bool=True, version: str='v1.1'):
def __init__(self,
pipe: Union[StableDiffusionXLPipeline, StableDiffusionXLImg2ImgPipeline, StableDiffusionXLInpaintPipeline],
device: torch.device,
dtype: torch.dtype=None,
providers: list=None,
offload: bool=True,
sampler=None,
cache_dir=None,
sdp: bool=True,
version: str='v1.1',
):
super().__init__()
self.device = device
self.dtype = dtype or torch.float16
@@ -282,7 +293,7 @@ class StableDiffusionXLPuLIDPipeline:
t = self.timestep(sigma)
x_ddim_space = x / (sigma[:, None, None, None] ** 2 + self.sigma_data**2) ** 0.5
cfg_scale = extra_args['cfg_scale']
debug(f'PulID sample start: step={self.step+1} x={x.shape} dtype={x.dtype} timestep={t.item()} sigma={sigma.shape} cfg={cfg_scale} args={extra_args.keys()}')
# debug(f'PulID sample start: step={self.step+1} x={x.shape} dtype={x.dtype} timestep={t.item()} sigma={sigma.shape} cfg={cfg_scale} args={extra_args.keys()}')
eps_positive = self.pipe.unet(x_ddim_space, t, return_dict=False, **extra_args['positive'])[0]
eps_negative = self.pipe.unet(x_ddim_space, t, return_dict=False, **extra_args['negative'])[0]
noise_pred = eps_negative + cfg_scale * (eps_positive - eps_negative)
@@ -290,7 +301,7 @@ class StableDiffusionXLPuLIDPipeline:
if self.callback_on_step_end is not None:
self.step += 1
self.callback_on_step_end(self.pipe, step=self.step, timestep=t, kwargs={ 'latents': latent })
debug(f'PulID sample end: step={self.step} x={latent.shape} dtype={x.dtype} min={torch.amin(latent)} max={torch.amax(latent)}')
# debug(f'PulID sample end: step={self.step} x={latent.shape} dtype={x.dtype} min={torch.amin(latent)} max={torch.amax(latent)}')
return latent
def init_latent(self, seed, size, image, mask_image, strength, width, height): # pylint: disable=unused-argument
modules/sd_models.py
+13 -9
View File
@@ -1081,9 +1081,10 @@ def set_diffuser_pipe(pipe, new_pipe_type):
return pipe
# skip specific pipelines
cls = pipe.__class__.__name__
if n in exclude:
return pipe
if 'Onnx' in pipe.__class__.__name__:
if 'Onnx' in cls:
return pipe
new_pipe = None
@@ -1114,27 +1115,27 @@ def set_diffuser_pipe(pipe, new_pipe_type):
elif new_pipe_type == DiffusersTaskType.INPAINTING:
new_pipe = diffusers.AutoPipelineForInpainting.from_pipe(pipe)
else:
shared.log.error(f'Pipeline class change failed: type={new_pipe_type} pipeline={pipe.__class__.__name__}')
shared.log.error(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls}')
return pipe
except Exception as e: # pylint: disable=unused-variable
shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}')
shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls} {e}')
return pipe
else:
try: # maybe a wrapper pipeline so just change the class
if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE:
pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING, pipe.__class__.__name__) # pylint: disable=protected-access
pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access
new_pipe = pipe
elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE:
pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, pipe.__class__.__name__) # pylint: disable=protected-access
pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access
new_pipe = pipe
elif new_pipe_type == DiffusersTaskType.INPAINTING:
pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING, pipe.__class__.__name__) # pylint: disable=protected-access
pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING, cls) # pylint: disable=protected-access
new_pipe = pipe
else:
shared.log.error(f'Pipeline class change failed: type={new_pipe_type} pipeline={pipe.__class__.__name__}')
shared.log.error(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls}')
return pipe
except Exception as e: # pylint: disable=unused-variable
shared.log.warning(f'Pipeline class set failed: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}')
shared.log.warning(f'Pipeline class set failed: type={new_pipe_type} pipeline={cls} {e}')
return pipe
# if pipe.__class__ == new_pipe.__class__:
@@ -1158,7 +1159,7 @@ def set_diffuser_pipe(pipe, new_pipe_type):
new_pipe.pipe = set_diffuser_pipe(new_pipe.pipe, new_pipe_type)
fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
shared.log.debug(f"Pipeline class change: original={pipe.__class__.__name__} target={new_pipe.__class__.__name__} device={pipe.device} fn={fn}") # pylint: disable=protected-access
shared.log.debug(f"Pipeline class change: original={cls} target={new_pipe.__class__.__name__} device={pipe.device} fn={fn}") # pylint: disable=protected-access
pipe = new_pipe
return pipe
@@ -1187,6 +1188,9 @@ def set_diffusers_attention(pipe):
else:
module.set_attn_processor(attention)
if hasattr(pipe, 'pipe'):
set_diffusers_attention(pipe.pipe)
if 'ControlNet' in pipe.__class__.__name__: # do not replace attention in ControlNet pipelines
return
shared.log.debug(f'Setting model: attention="{shared.opts.cross_attention_optimization}"')
scripts/pulid_ext.py
-4
View File
@@ -18,7 +18,6 @@ class Script(scripts.Script):
def __init__(self):
self.pulid = None
self.cache = None
self.mask_apply_overlay = shared.opts.mask_apply_overlay
self.preprocess = 0
super().__init__()
self.register() # pulid is script with processing override so xyz doesnt execute
@@ -151,8 +150,6 @@ class Script(scripts.Script):
shared.log.warning('PuLID: batch size not supported')
p.batch_size = 1
self.mask_apply_overlay = shared.opts.mask_apply_overlay
shared.opts.data['mask_apply_overlay'] = False
sdp = shared.opts.cross_attention_optimization == "Scaled-Dot-Product"
strength = getattr(p, 'pulid_strength', strength)
zero = getattr(p, 'pulid_zero', zero)
@@ -242,7 +239,6 @@ class Script(scripts.Script):
def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=unused-argument
_strength, _zero, _sampler, _ortho, _gallery, restore, _offload, _version = args
if hasattr(shared.sd_model, 'pipe') and shared.sd_model_type == "sdxl":
shared.opts.data['mask_apply_overlay'] = self.mask_apply_overlay
restore = getattr(p, 'pulid_restore', restore)
if restore:
if hasattr(shared.sd_model, 'app'):