diff --git a/CHANGELOG.md b/CHANGELOG.md index cf496f81c..8e91157bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,9 @@ This release can be considered an LTS release before we kick off the next round go to system -> changelog and search/highligh/navigate directly in UI! - [PuLID](https://github.com/ToTheBeginning/PuLID): Pure and Lightning ID Customization via Contrastive Alignment - advanced method of face transfer with better quality as well as control over identity and appearance - - compatible with *sdxl* - select in *scripts -> pulid* + - compatible with *sdxl* + - can be used in xyz grid - SD3: ControlNets: - *InstantX Canny, Pose, Depth, Tile* - *Alimama Inpainting, SoftEdge* diff --git a/modules/processing_args.py b/modules/processing_args.py index 1ea91fb08..002655cd5 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -100,10 +100,11 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 if hasattr(model, "set_progress_bar_config"): model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba') args = {} - if hasattr(model, 'pipe'): # recurse + if hasattr(model, 'pipe') and not hasattr(model, 'no_recurse'): # recurse model = model.pipe signature = inspect.signature(type(model).__call__, follow_wrapped=True) possible = list(signature.parameters) + debug(f'Diffusers pipeline possible: {possible}') prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2) parser = 'Fixed attention' @@ -128,7 +129,7 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 if 'prompt' in possible: if 'OmniGen' in model.__class__.__name__: prompts = [p.replace('|image|', '<|image_1|>') for p in prompts] - if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and len(p.prompt_embeds) > 0 and p.prompt_embeds[0] is not None: + if hasattr(model, 'text_encoder') and hasattr(model, 'tokenizer') and 'prompt_embeds' in possible and len(p.prompt_embeds) > 0 and p.prompt_embeds[0] is not None: args['prompt_embeds'] = p.prompt_embeds[0] if 'StableCascade' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0: args['prompt_embeds_pooled'] = p.positive_pooleds[0].unsqueeze(0) @@ -141,7 +142,7 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 else: args['prompt'] = prompts if 'negative_prompt' in possible: - if hasattr(model, 'text_encoder') and 'negative_prompt_embeds' in possible and len(p.negative_embeds) > 0 and p.negative_embeds[0] is not None: + if hasattr(model, 'text_encoder') and hasattr(model, 'tokenizer') and 'negative_prompt_embeds' in possible and len(p.negative_embeds) > 0 and p.negative_embeds[0] is not None: args['negative_prompt_embeds'] = p.negative_embeds[0] if 'StableCascade' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0: args['negative_prompt_embeds_pooled'] = p.negative_pooleds[0].unsqueeze(0) diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 47c8e8827..45a0724fb 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -33,7 +33,7 @@ def diffusers_callback_legacy(step: int, timestep: int, latents: typing.Union[to time.sleep(0.1) -def diffusers_callback(pipe, step: int, timestep: int, kwargs: dict): +def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {}): t0 = time.time() if p is None: return kwargs diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index cc814f379..2994d6ef5 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -158,6 +158,8 @@ def get_tokens(msg, prompt): def encode_prompts(pipe, p, prompts: list, negative_prompts: list, steps: int, clip_skip: typing.Optional[int] = None): + if not hasattr(pipe, 'text_encoder') or not hasattr(pipe, 'tokenizer'): + return params_match = prompts == cache.get('prompts', None) and negative_prompts == cache.get('negative_prompts', None) and clip_skip == cache.get('clip_skip', None) and steps == cache.get('steps', None) if ( 'StableDiffusion' not in pipe.__class__.__name__ and diff --git a/modules/pulid/__init__.py b/modules/pulid/__init__.py index 77a1bae8f..000f45293 100644 --- a/modules/pulid/__init__.py +++ b/modules/pulid/__init__.py @@ -5,6 +5,6 @@ Credit and original implementation: import os import sys sys.path.append(os.path.dirname(__file__)) -from pipe_sdxl import PuLIDPipeline as PuLIDPipelineXL +from pulid_sdxl import StableDiffusionXLPuLIDPipeline from pulid_utils import resize_numpy_image_long as resize import attention_processor as attention diff --git a/modules/pulid/pipe_sdxl.py b/modules/pulid/pulid_sdxl.py similarity index 83% rename from modules/pulid/pipe_sdxl.py rename to modules/pulid/pulid_sdxl.py index 247a2067e..a4c3bf06c 100644 --- a/modules/pulid/pipe_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -24,7 +24,7 @@ from attention_processor import AttnProcessor2_0 as AttnProcessor from attention_processor import IDAttnProcessor2_0 as IDAttnProcessor -class PuLIDPipeline: +class StableDiffusionXLPuLIDPipeline: def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, sampler='dpmpp_sde', cache_dir=None): super().__init__() self.device = device @@ -73,7 +73,6 @@ class PuLIDPipeline: self.handler_ante = insightface.model_zoo.get_model(os.path.join(local_dir, 'glintr100.onnx')) self.handler_ante.prepare(ctx_id=0) - torch.cuda.empty_cache() self.load_pretrain() # other configs @@ -249,34 +248,75 @@ class PuLIDPipeline: # return id_embedding return uncond_id_embedding, id_embedding - def __call__(self, x, sigma, **extra_args): + def set_progress_bar_config(self, bar_format: str = None, ncols: int = 80, colour: str = None): + import functools + from tqdm.auto import trange as trange_orig + import pulid_utils + pulid_utils.trange = functools.partial(trange_orig, bar_format=bar_format, ncols=ncols, colour=colour) + + def sample(self, x, sigma, **extra_args): x_ddim_space = x / (sigma[:, None, None, None] ** 2 + self.sigma_data**2) ** 0.5 t = self.timestep(sigma) cfg_scale = extra_args['cfg_scale'] eps_positive = self.pipe.unet(x_ddim_space, t, return_dict=False, **extra_args['positive'])[0] eps_negative = self.pipe.unet(x_ddim_space, t, return_dict=False, **extra_args['negative'])[0] noise_pred = eps_negative + cfg_scale * (eps_positive - eps_negative) - return x - noise_pred * sigma[:, None, None, None] + latent = x - noise_pred * sigma[:, None, None, None] + if self.callback_on_step_end is not None: + self.step += 1 + self.callback_on_step_end(self.pipe, step=self.step, timestep=t, kwargs={ 'latents': latent }) + return latent - def inference( + def init_latent(self, seed, size, image, strength): # pylint: disable=unused-argument + if image is not None and strength > 0: + # TODO pulid img2img + # input can be PIL.Image or np.ndarray so it needs to be converted to rgb tensor + # image must be resized, encoded and noised according to denoising strength + # see below for example from StableDiffusionXLImg2ImgPipeline + latents = None + """ + image = self.image_processor.preprocess(image) + latents = self.prepare_latents( + image, + latent_timestep, + batch_size, + num_images_per_prompt, + prompt_embeds.dtype, + device, + generator, + add_noise, + ) + """ + raise NotImplementedError('pulid: img2img') + else: + latents = torch.randn((size[0], 4, size[1] // 8, size[2] // 8), device="cpu", generator=torch.manual_seed(seed)) + latents = latents.to(dtype=self.pipe.unet.dtype, device=self.device) + return latents + + def __call__( self, - prompt, - size, - prompt_n='', + prompt: str='', + negative_prompt: str='', + width: int=1024, + height: int=1024, + guidance_scale: float=7.0, + num_inference_steps: int=50, + seed: int=-1, + image: np.ndarray=None, + strength: float=0.3, id_embedding=None, uncond_id_embedding=None, - id_scale=1.0, - guidance_scale=1.2, - steps=4, - seed=-1, + id_scale: float=1.0, + callback_on_step_end=None, ): - + self.step = 0 # pylint: disable=attribute-defined-outside-init + self.callback_on_step_end = callback_on_step_end # pylint: disable=attribute-defined-outside-init + size = (1, height, width) # sigmas - sigmas = self.get_sigmas_karras(steps).to(self.device) + sigmas = self.get_sigmas_karras(num_inference_steps).to(self.device) # latents - noise = torch.randn((size[0], 4, size[1] // 8, size[2] // 8), device="cpu", generator=torch.manual_seed(seed)) - noise = noise.to(dtype=self.pipe.unet.dtype, device=self.device) + noise = self.init_latent(seed, size, image, strength) latents = noise * sigmas[0].to(noise) ( @@ -286,7 +326,7 @@ class PuLIDPipeline: negative_pooled_prompt_embeds, ) = self.pipe.encode_prompt( prompt=prompt, - negative_prompt=prompt_n, + negative_prompt=negative_prompt, ) add_time_ids = list((size[1], size[2]) + (0, 0) + (size[1], size[2])) @@ -307,7 +347,7 @@ class PuLIDPipeline: ), ) - latents = self.sampler(self, latents, sigmas, extra_args=sampler_kwargs, disable=False) + latents = self.sampler(self.sample, latents, sigmas, extra_args=sampler_kwargs, disable=False) latents = latents.to(dtype=self.pipe.vae.dtype, device=self.device) / self.pipe.vae.config.scaling_factor images = self.pipe.vae.decode(latents).sample images = self.pipe.image_processor.postprocess(images, output_type='pil') diff --git a/modules/sd_models.py b/modules/sd_models.py index 610f5a248..575e9318a 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1052,6 +1052,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): 'AnimateDiffSDXLPipeline', 'OmniGenPipeline', 'StableDiffusion3ControlNetPipeline', + 'StableDiffusionXLPuLIDPipeline', ] n = getattr(pipe.__class__, '__name__', '') diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 4ee696538..d31c18164 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -1,33 +1,37 @@ -import time +import io +import os +import contextlib import gradio as gr import numpy as np from PIL import Image -from modules import shared, devices, errors, sd_models, scripts, processing, processing_helpers +from modules import shared, devices, errors, scripts, processing, processing_helpers, sd_models -pulid = None +debug = os.environ.get('SD_PULID_DEBUG', None) is not None class Script(scripts.Script): def __init__(self): self.images = [] + self.pulid = None + self.cache = None super().__init__() - # self.register() # pulid is script with processing override so xyz doesnt execute + self.register() # pulid is script with processing override so xyz doesnt execute def title(self): return 'PuLID' def show(self, _is_img2img): - return not _is_img2img + return shared.native def dependencies(self): from installer import install, installed - # if not installed('apex', reload=False, quiet=True): - # install('apex', 'apex', ignore=False) if not installed('insightface', reload=False, quiet=True): install('insightface', 'insightface', ignore=False) install('albumentations==1.4.3', 'albumentations', ignore=False, reinstall=True) install('pydantic==1.10.15', 'pydantic', ignore=False, reinstall=True) + # if not installed('apex', reload=False, quiet=True): + # install('apex', 'apex', ignore=False) def register(self): # register xyz grid elements def apply_field(field): @@ -39,7 +43,8 @@ class Script(scripts.Script): import sys xyz_classes = [v for k, v in sys.modules.items() if 'xyz_grid_classes' in k][0] xyz_classes.axis_options.append(xyz_classes.AxisOption("[PuLID] Strength", float, apply_field("pulid_strength"))) - xyz_classes.axis_options.append(xyz_classes.AxisOption("[PuLID] Zero", float, apply_field("pulid_zero"))) + xyz_classes.axis_options.append(xyz_classes.AxisOption("[PuLID] Zero", int, apply_field("pulid_zero"))) + xyz_classes.axis_options.append(xyz_classes.AxisOption("[PuLID] Ortho", str, apply_field("pulid_ortho"), choices=lambda: ['off', 'v1', 'v2'])) def load_images(self, files): self.images = [] @@ -79,117 +84,123 @@ class Script(scripts.Script): return [strength, zero, sampler, ortho, gallery] def run(self, p: processing.StableDiffusionProcessing, strength: float = 0.8, zero: int = 20, sampler: str = 'dpmpp_sde', ortho: str = 'v2', gallery: list = []): # pylint: disable=arguments-differ - global pulid # pylint: disable=global-statement images = [] try: if len(gallery) == 0: - gallery = self.images - images = [Image.open(f['name']) for f in gallery if isinstance(f, dict)] + from modules.api.api import decode_base64_to_image + images = getattr(p, 'pulid_images', self.images) + images = [decode_base64_to_image(image) if isinstance(image, str) else image for image in images] + else: + images = [Image.open(f['name']) if isinstance(f, dict) else f for f in gallery] images = [np.array(image) for image in images] except Exception as e: shared.log.error(f'PuLID: failed to load images: {e}') return None if len(images) == 0: - shared.log.error('PuLID: no images loaded') + shared.log.error('PuLID: no images') return None supported_model_list = ['sdxl'] if shared.sd_model_type not in supported_model_list: shared.log.error(f'PuLID: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={supported_model_list}') return None - if pulid is None: + if self.pulid is None: self.dependencies() try: from modules import pulid # pylint: disable=redefined-outer-name + self.pulid = pulid + # from diffusers import pipelines + # pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["pilid"] = pulid.StableDiffusionXLPuLIDPipeline + # pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["omnigen"] = pulid.StableDiffusionXLPuLIDPipelineImg2Img except Exception as e: shared.log.error(f'PuLID: failed to import library: {e}') return None - # import os - # import importlib - # module_path = os.path.join(os.path.dirname(__file__), '..', 'pulid', '__init__.py') - # module_spec = importlib.util.spec_from_file_location('pulid', module_path) - # pulid = importlib.util.module_from_spec(module_spec) - # module_spec.loader.exec_module(pulid) - if pulid is None: - shared.log.error('PuLID: failed to load PuLID library') - return None + if self.pulid is None: + shared.log.error('PuLID: failed to load PuLID library') + return None if p.batch_size > 1: shared.log.warning('PuLID: batch size not supported') p.batch_size = 1 + strength = getattr(p, 'pulid_strength', strength) zero = getattr(p, 'pulid_zero', zero) + ortho = getattr(p, 'pulid_ortho', ortho) - processing.fix_seed(p) - pipe = None - if shared.sd_model_type == 'sdxl': - # TODO pulid has monolithic inference so not really working with offloading - sd_models.move_model(shared.sd_model, devices.device) - sd_models.move_model(shared.sd_model.vae, devices.device) - sd_models.move_model(shared.sd_model.unet, devices.device) - sd_models.move_model(shared.sd_model.text_encoder, devices.device) - sd_models.move_model(shared.sd_model.text_encoder_2, devices.device) + if shared.sd_model_type == 'sdxl' and not hasattr(shared.sd_model, 'pipe'): try: - pipe = pulid.PuLIDPipelineXL( - pipe =shared.sd_model, - device=devices.device, - sampler=sampler, - cache_dir=shared.opts.hfcache_dir, - ) + stdout = io.StringIO() + ctx = contextlib.nullcontext if debug else contextlib.redirect_stdout(stdout) + with ctx: + shared.sd_model = self.pulid.StableDiffusionXLPuLIDPipeline( + pipe =shared.sd_model, + device=devices.device, + sampler=sampler, + cache_dir=shared.opts.hfcache_dir, + ) + shared.sd_model.no_recurse = True + sd_models.copy_diffuser_options(shared.sd_model, shared.sd_model.pipe) + sd_models.move_model(shared.sd_model, devices.device) # move pipeline to device + sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model') + devices.torch_gc() except Exception as e: shared.log.error(f'PuLID: failed to create pipeline: {e}') errors.display(e, 'PuLID') return None - if pipe is None: - return None - shared.state.begin('PuLID') - shared.log.info(f'PuLID: class={pipe.__class__.__name__} strength={strength} zero={zero} ortho={ortho} sampler={sampler} images={[i.shape for i in images]}') - pipe.debug_img_list = [] - pulid.attention.NUM_ZERO = zero - if ortho == 'v2': - pulid.attention.ORTHO = False - pulid.attention.ORTHO_v2 = True - elif ortho == 'v1': - pulid.attention.ORTHO = True - pulid.attention.ORTHO_v2 = False - else: - pulid.attention.ORTHO = False - pulid.attention.ORTHO_v2 = False + shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} strength={strength} zero={zero} ortho={ortho} sampler={sampler} images={[i.shape for i in images]}') + self.pulid.attention.NUM_ZERO = zero + self.pulid.attention.ORTHO = ortho == 'v1' + self.pulid.attention.ORTHO_v2 = ortho == 'v2' + images = [self.pulid.resize(image, 1024) for image in images] + shared.sd_model.debug_img_list = [] + uncond_id_embedding, id_embedding = shared.sd_model.get_id_embedding(images) - t0 = time.time() - images = [pulid.resize(image, 1024) for image in images] - outputs = [] - infotexts = [] - seeds = [] - prompts = [] - negative_prompts = [] - - for _n in range(p.n_iter): - seed = processing_helpers.get_fixed_seed(p.seed) - prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) - negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) + if debug: # run pipeline directly + shared.state.begin('PuLID') + processing.fix_seed(p) + p.seed = processing_helpers.get_fixed_seed(p.seed) + p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) + p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) with devices.inference_context(): - uncond_id_embedding, id_embedding = pipe.get_id_embedding(images) - output = pipe.inference(prompt, (1, p.height, p.width), negative_prompt, id_embedding, uncond_id_embedding, strength, p.cfg_scale, p.steps, seed)[0] - if output is not None: - outputs.append(output) - infotexts.append(processing.create_infotext(p)) - seeds.append(seed) - prompts.append(prompt) - negative_prompts.append(negative_prompt) - - interim = [Image.fromarray(face) for face in pipe.debug_img_list] - t1 = time.time() - shared.log.debug(f'PuLID: output={output} interim={interim} time={t1-t0:.2f}') - - if len(outputs) > 0: - p.prompt = prompts[0] - p.negative_prompt = negative_prompts[0] - p.seed = seeds[0] - p.all_prompts = prompts - p.all_negative_prompts = negative_prompts - p.all_seeds = seeds + output = shared.sd_model( + prompt=p.prompt, + negative_prompt=p.negative_prompt, + width=p.width, + height=p.height, + seed=p.seed, + num_inference_steps=p.steps, + guidance_scale=p.cfg_scale, + id_embedding=id_embedding, + uncond_id_embedding=uncond_id_embedding, + id_scale=strength, + )[0] + info = processing.create_infotext(p) + processed = processing.Processed(p, [output], info=info) + shared.state.end('PuLID') + else: # let processing run the pipeline + p.task_args['id_embedding'] = id_embedding + p.task_args['uncond_id_embedding'] = uncond_id_embedding + p.task_args['id_scale'] = strength + if len(getattr(p, 'init_images', [])) > 0: + p.task_args['image'] = p.init_images[0] + p.task_args['strength'] = p.denoising_strength p.extra_generation_params["PuLID"] = f'Strength={strength} Zero={zero} Ortho={ortho}' - processed = processing.Processed(p, outputs, infotexts=infotexts) + if getattr(p, 'xyz', False): # xyz will run its own processing + return None + processed: processing.Processed = processing.process_images(p) # runs processing using main loop - shared.state.end('PuLID') + # interim = [Image.fromarray(img) for img in shared.sd_model.debug_img_list] + # shared.log.debug(f'PuLID: time={t1-t0:.2f}') + return processed + + def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=unused-argument + if hasattr(shared.sd_model, 'pipe') and shared.sd_model_type == "sdxl": + if hasattr(shared.sd_model, 'app'): + shared.sd_model.app = None + shared.sd_model.ip_adapter = None + shared.sd_model.face_helper = None + shared.sd_model.clip_vision_model = None + shared.sd_model.handler_ante = None + devices.torch_gc(force=True) + shared.sd_model = shared.sd_model.pipe + # shared.log.debug(f'PuLID restore: class={shared.sd_model.__class__.__name__}') return processed diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index e8ecb5bd4..62a722bb3 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -258,6 +258,7 @@ class Script(scripts.Script): def cell(x, y, z, ix, iy, iz): if shared.state.interrupted: return processing.Processed(p, [], p.seed, "") + p.xyz = True pc = copy(p) pc.override_settings_restore_afterwards = False pc.styles = pc.styles[:] diff --git a/scripts/xyz_grid_classes.py b/scripts/xyz_grid_classes.py index 202482157..4898c6b73 100644 --- a/scripts/xyz_grid_classes.py +++ b/scripts/xyz_grid_classes.py @@ -1,4 +1,4 @@ -from scripts.xyz_grid_shared import apply_field, apply_task_args, apply_setting, apply_prompt, apply_order, apply_sampler, apply_hr_sampler_name, confirm_samplers, apply_checkpoint, apply_refiner, apply_unet, apply_dict, apply_clip_skip, apply_vae, list_lora, apply_lora, apply_te, apply_styles, apply_upscaler, apply_context, apply_detailer, apply_override, apply_processing, apply_options, apply_seed, format_value_add_label, format_value, format_value_join_list, do_nothing, format_nothing, str_permutations # pylint: disable=no-name-in-module +from scripts.xyz_grid_shared import apply_field, apply_task_args, apply_setting, apply_prompt, apply_order, apply_sampler, apply_hr_sampler_name, confirm_samplers, apply_checkpoint, apply_refiner, apply_unet, apply_dict, apply_clip_skip, apply_vae, list_lora, apply_lora, apply_te, apply_styles, apply_upscaler, apply_context, apply_detailer, apply_override, apply_processing, apply_options, apply_seed, format_value_add_label, format_value, format_value_join_list, do_nothing, format_nothing, str_permutations # pylint: disable=no-name-in-module, unused-import from modules import shared, shared_items, sd_samplers, ipadapter, sd_models, sd_vae, sd_unet diff --git a/scripts/xyz_grid_on.py b/scripts/xyz_grid_on.py index cdef01e60..6affde005 100644 --- a/scripts/xyz_grid_on.py +++ b/scripts/xyz_grid_on.py @@ -273,6 +273,7 @@ class Script(scripts.Script): def cell(x, y, z, ix, iy, iz): if shared.state.interrupted: return processing.Processed(p, [], p.seed, "") + p.xyz = True pc = copy(p) pc.override_settings_restore_afterwards = False pc.styles = pc.styles[:]