diff --git a/modules/processing_args.py b/modules/processing_args.py index 002655cd5..0e84f2a4a 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -27,7 +27,7 @@ def task_specific_kwargs(p, model): 'height': 8 * math.ceil(p.height / 8), } elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or is_img2img_model) and len(getattr(p, 'init_images', [])) > 0: - if shared.sd_model_type == 'sdxl': + if shared.sd_model_type == 'sdxl' and hasattr(model, 'register_to_config'): model.register_to_config(requires_aesthetics_score = False) p.ops.append('img2img') task_args = { @@ -55,7 +55,7 @@ def task_specific_kwargs(p, model): 'strength': p.denoising_strength, } elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INPAINTING or is_img2img_model) and len(getattr(p, 'init_images', [])) > 0: - if shared.sd_model_type == 'sdxl': + if shared.sd_model_type == 'sdxl' and hasattr(model, 'register_to_config'): model.register_to_config(requires_aesthetics_score = False) if p.detailer: p.ops.append('detailer') diff --git a/modules/pulid/__init__.py b/modules/pulid/__init__.py index 785b849c2..dcee2d7b9 100644 --- a/modules/pulid/__init__.py +++ b/modules/pulid/__init__.py @@ -5,7 +5,7 @@ Credit and original implementation: import os import sys sys.path.append(os.path.dirname(__file__)) -from pulid_sdxl import StableDiffusionXLPuLIDPipeline +from pulid_sdxl import StableDiffusionXLPuLIDPipeline, StableDiffusionXLPuLIDPipelineImage, StableDiffusionXLPuLIDPipelineInpaint from pulid_utils import resize_numpy_image_long as resize import attention_processor as attention import pulid_sampling as sampling diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index de650b839..af7b8e443 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -307,6 +307,7 @@ class StableDiffusionXLPuLIDPipeline: num_inference_steps: int=50, seed: int=-1, image: np.ndarray=None, + mask_image: np.ndarray=None, strength: float=0.3, id_embedding=None, uncond_id_embedding=None, @@ -356,4 +357,22 @@ class StableDiffusionXLPuLIDPipeline: images = self.pipe.vae.decode(latents).sample images = self.pipe.image_processor.postprocess(images, output_type='pil') + if mask_image is not None: + # TODO: pulid inpaint + # easiest inpaint is to use normal img2img and then combine output with input using mask + # note that mask can be binary or grayscale (soft mask) + raise NotImplementedError('pulid: inpaint') + return images + + +class StableDiffusionXLPuLIDPipelineImage(StableDiffusionXLPuLIDPipeline): + def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, sampler=None, cache_dir=None): # pylint: disable=useless-parent-delegation + super().__init__(pipe, device, sampler, cache_dir) + # we dont do anything special here, just having different class so task-type can be detected/assigned + + +class StableDiffusionXLPuLIDPipelineInpaint(StableDiffusionXLPuLIDPipeline): + def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, sampler=None, cache_dir=None): # pylint: disable=useless-parent-delegation + super().__init__(pipe, device, sampler, cache_dir) + # we dont do anything special here, just having different class so task-type can be detected/assigned diff --git a/modules/sd_models.py b/modules/sd_models.py index 0d1e91f91..279959fde 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1061,7 +1061,6 @@ def set_diffuser_pipe(pipe, new_pipe_type): 'AnimateDiffSDXLPipeline', 'OmniGenPipeline', 'StableDiffusion3ControlNetPipeline', - 'StableDiffusionXLPuLIDPipeline', 'InstantIRPipeline', ] @@ -1083,6 +1082,13 @@ def set_diffuser_pipe(pipe, new_pipe_type): pipe = switch_pipe(diffusers.StableDiffusionPipeline, pipe) if n == 'StableDiffusionXLPAGPipeline': pipe = switch_pipe(diffusers.StableDiffusionXLPipeline, pipe) + if n == 'StableDiffusionXLPuLIDPipeline': + from modules import pulid + if new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: + pipe.__class__ = pulid.StableDiffusionXLPuLIDPipelineImage + else: + pipe.__class__ = pulid.StableDiffusionXLPuLIDPipelineInpaint + return pipe sd_checkpoint_info = getattr(pipe, "sd_checkpoint_info", None) sd_model_checkpoint = getattr(pipe, "sd_model_checkpoint", None) diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 54189cc05..3a157d0fe 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -106,9 +106,10 @@ class Script(scripts.Script): try: from modules import pulid # pylint: disable=redefined-outer-name self.pulid = pulid - # from diffusers import pipelines - # pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["pilid"] = pulid.StableDiffusionXLPuLIDPipeline - # pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["omnigen"] = pulid.StableDiffusionXLPuLIDPipelineImg2Img + from diffusers import pipelines + pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["pulid"] = pulid.StableDiffusionXLPuLIDPipeline + pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["pulid"] = pulid.StableDiffusionXLPuLIDPipelineImage + pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["pulid"] = pulid.StableDiffusionXLPuLIDPipelineInpaint except Exception as e: shared.log.error(f'PuLID: failed to import library: {e}') return None @@ -124,6 +125,8 @@ class Script(scripts.Script): ortho = getattr(p, 'pulid_ortho', ortho) sampler = getattr(p, 'pulid_sampler', sampler) sampler_fn = getattr(self.pulid.sampling, f'sample_{sampler}', None) + if sampler_fn is None: + sampler_fn = self.pulid.sampling.sample_dpmpp_2m_sde if shared.sd_model_type == 'sdxl' and not hasattr(shared.sd_model, 'pipe'): try: @@ -146,7 +149,7 @@ class Script(scripts.Script): return None shared.sd_model.sampler = sampler_fn - shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} strength={strength} zero={zero} ortho={ortho} sampler={sampler} images={[i.shape for i in images]}') + shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} strength={strength} zero={zero} ortho={ortho} sampler={sampler_fn} images={[i.shape for i in images]}') self.pulid.attention.NUM_ZERO = zero self.pulid.attention.ORTHO = ortho == 'v1' self.pulid.attention.ORTHO_v2 = ortho == 'v2' @@ -184,6 +187,7 @@ class Script(scripts.Script): p.task_args['image'] = p.init_images[0] p.task_args['strength'] = p.denoising_strength p.extra_generation_params["PuLID"] = f'Strength={strength} Zero={zero} Ortho={ortho}' + p.extra_generation_params["Sampler"] = sampler if getattr(p, 'xyz', False): # xyz will run its own processing return None processed: processing.Processed = processing.process_images(p) # runs processing using main loop