From dab9087069b635d45e32bce56afd33b7a7d2f4be Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 6 Jan 2024 11:52:52 -0500 Subject: [PATCH] full faceid and updated ipadapter --- CHANGELOG.md | 13 ++- extensions-builtin/sd-webui-controlnet | 2 +- modules/cmd_args.py | 10 +- scripts/faceid.py | 139 ++++++++++++++++++++----- scripts/ipadapter.py | 67 +++++++----- 5 files changed, 172 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ef93e289..a26efb7be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,16 @@ And it also includes fixes for all reported issues so far - fix correct image mode - fix batch/folder/video modes - fix pipeline switching between different modes +- [FaceID](https://huggingface.co/h94/IP-Adapter-FaceID) + full implementation for *SD15* and *SD-XL*, to use simply select from *Scripts* + - **Base** (93MB) uses *InsightFace* to generate face embeds and *OpenCLIP-ViT-H-14* (2.5GB) as image encoder + - **SXDL** (1022MB) uses *InsightFace* to generate face embeds and *OpenCLIP-ViT-bigG-14* (3.7GB) as image encoder + - **Plus** (150MB) uses *InsightFace* to generate face embeds and *CLIP-ViT-H-14-laion2B* (3.8GB) as image encoder + *note*: all models are downloaded on first use +- [IPAdapter](https://huggingface.co/h94/IP-Adapter) + additional models for *SD15* and *SD-XL*, to use simply select from *Scripts*: + - **SD15**: Base, Base ViT-G, Light, Plus, Plus Face, Full Face + - **SDXL**: Base SXDL, Base ViT-H SXDL, Plus ViT-H SXDL, Plus Face ViT-H SXDL - **Improvements** - **server startup**: performance - faster extension load @@ -44,7 +54,7 @@ And it also includes fixes for all reported issues so far - enable vae tiling - add autodetect optimial value set tile size to 0 to use autodetected value - - **cli**: + - **cli** - `sdapi.py` allow manual api invoke example: `python cli/sdapi.py /sdapi/v1/sd-models` - `image-exif.py` improve metadata parsing @@ -94,6 +104,7 @@ And it also includes fixes for all reported issues so far - processing: correct display metadata - live preview: fix when using `bfloat16` - upscale: fix ldsr + - cli: fix cmd args parsing ## Update for 2023-12-29 diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 82e406928..9d1f0a07f 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 82e40692870218b8b1f3842ec11d0b5d7acfdb6e +Subproject commit 9d1f0a07fa2754338a6b260fad2abaf9d3f42ade diff --git a/modules/cmd_args.py b/modules/cmd_args.py index 0fc387268..40aeeebce 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -80,12 +80,12 @@ def compatibility_args(opts, args): group.add_argument("--swinir-models-path", help=argparse.SUPPRESS, default=opts.swinir_models_path) group.add_argument("--ldsr-models-path", help=argparse.SUPPRESS, default=opts.ldsr_models_path) group.add_argument("--clip-models-path", type=str, help=argparse.SUPPRESS, default=opts.clip_models_path) - group.add_argument("--opt-channelslast", help=argparse.SUPPRESS, default=opts.opt_channelslast) + group.add_argument("--opt-channelslast", help=argparse.SUPPRESS, action='store_true', default=opts.opt_channelslast) group.add_argument("--xformers", default=(opts.cross_attention_optimization == "xFormers"), action='store_true', help=argparse.SUPPRESS) - group.add_argument("--disable-nan-check", help=argparse.SUPPRESS, default=opts.disable_nan_check) + group.add_argument("--disable-nan-check", help=argparse.SUPPRESS, action='store_true', default=opts.disable_nan_check) group.add_argument("--rollback-vae", help=argparse.SUPPRESS, default=opts.rollback_vae) - group.add_argument("--no-half", help=argparse.SUPPRESS, default=opts.no_half) - group.add_argument("--no-half-vae", help=argparse.SUPPRESS, default=opts.no_half_vae) + group.add_argument("--no-half", help=argparse.SUPPRESS, action='store_true', default=opts.no_half) + group.add_argument("--no-half-vae", help=argparse.SUPPRESS, action='store_true', default=opts.no_half_vae) group.add_argument("--precision", help=argparse.SUPPRESS, default=opts.precision) group.add_argument("--sub-quad-q-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_q_chunk_size) group.add_argument("--sub-quad-kv-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_kv_chunk_size) @@ -94,7 +94,7 @@ def compatibility_args(opts, args): group.add_argument("--lyco-dir", help=argparse.SUPPRESS, default=opts.lyco_dir) group.add_argument("--embeddings-dir", help=argparse.SUPPRESS, default=opts.embeddings_dir) group.add_argument("--hypernetwork-dir", help=argparse.SUPPRESS, default=opts.hypernetwork_dir) - group.add_argument("--lyco-patch-lora", help=argparse.SUPPRESS, default=False) + group.add_argument("--lyco-patch-lora", help=argparse.SUPPRESS, action='store_true', default=False) group.add_argument("--lyco-debug", help=argparse.SUPPRESS, action='store_true', default=False) group.add_argument("--enable-console-prompts", help=argparse.SUPPRESS, action='store_true', default=False) group.add_argument("--safe", help=argparse.SUPPRESS, action='store_true', default=False) diff --git a/scripts/faceid.py b/scripts/faceid.py index deb20f3c0..a4eabebe8 100644 --- a/scripts/faceid.py +++ b/scripts/faceid.py @@ -6,11 +6,30 @@ import gradio as gr import diffusers import huggingface_hub as hf from modules import scripts, processing, shared, devices -from installer import installed +MODELS = { + 'FaceID Base': 'h94/IP-Adapter-FaceID/ip-adapter-faceid_sd15.bin', + 'FaceID Plus': 'h94/IP-Adapter-FaceID/ip-adapter-faceid-plus_sd15.bin', + 'FaceID Plus v2': 'h94/IP-Adapter-FaceID/ip-adapter-faceid-plusv2_sd15.bin', + 'FaceID XL': 'h94/IP-Adapter-FaceID/ip-adapter-faceid_sdxl.bin' +} app = None -ok = installed('insightface', reload=False, quiet=True) and installed('ip_adapter', reload=False, quiet=True) +ip_model = None +ip_model_name = None +ip_model_tokens = None +ip_model_rank = None + + +def dependencies(): + from installer import installed, install + packages = [ + ('insightface', 'insightface'), + ('git+https://github.com/tencent-ailab/IP-Adapter.git', 'ip_adapter'), + ] + for pkg in packages: + if not installed(pkg[1], reload=False, quiet=True): + install(pkg[0], pkg[1], ignore=True) class Script(scripts.Script): @@ -18,32 +37,43 @@ class Script(scripts.Script): return 'FaceID' def show(self, is_img2img): - return ok if shared.backend == shared.Backend.DIFFUSERS else False + return True if shared.backend == shared.Backend.DIFFUSERS else False # return signature is array of gradio components def ui(self, _is_img2img): with gr.Row(): - scale = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=1.0) + model = gr.Dropdown(choices=list(MODELS), label='Model', value='FaceID Base') + with gr.Row(visible=True): + override = gr.Checkbox(label='Override sampler', value=True) + cache = gr.Checkbox(label='Cache model', value=True) + with gr.Row(visible=True): + scale = gr.Slider(label='Strength', minimum=0.0, maximum=1.0, step=0.01, value=1.0) + structure = gr.Slider(label='Structure', minimum=0.0, maximum=1.0, step=0.01, value=1.0) + with gr.Row(visible=False): + rank = gr.Slider(label='Rank', minimum=4, maximum=256, step=4, value=128) + tokens = gr.Slider(label='Tokens', minimum=1, maximum=16, step=1, value=4) with gr.Row(): image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512) - return [scale, image] + return [model, scale, image, override, rank, tokens, structure, cache] - def run(self, p: processing.StableDiffusionProcessing, scale, image): # pylint: disable=arguments-differ, unused-argument + def run(self, p: processing.StableDiffusionProcessing, model, scale, image, override, rank, tokens, structure, cache): # pylint: disable=arguments-differ, unused-argument + dependencies() try: import onnxruntime from insightface.app import FaceAnalysis - from ip_adapter.ip_adapter_faceid import IPAdapterFaceID + from insightface.utils import face_align + from ip_adapter.ip_adapter_faceid import IPAdapterFaceID, IPAdapterFaceIDPlus, IPAdapterFaceIDXL except Exception as e: shared.log.error(f'FaceID: {e}') return None if image is None: shared.log.error('FaceID: no init_images') return None - if shared.sd_model_type != 'sd': + if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl': shared.log.error('FaceID: base model not supported') return None - global app # pylint: disable=global-statement + global app, ip_model, ip_model_name, ip_model_tokens, ip_model_rank # pylint: disable=global-statement if app is None: shared.log.debug(f"ONNX: device={onnxruntime.get_device()} providers={onnxruntime.get_available_providers()}") app = FaceAnalysis(name="buffalo_l", providers=['CUDAExecutionProvider', 'CPUExecutionProvider']) @@ -57,28 +87,68 @@ class Script(scripts.Script): return None for face in faces: shared.log.debug(f'FaceID face: score={face.det_score:.2f} gender={"female" if face.gender==0 else "male"} age={face.age} bbox={face.bbox}') - embeds = torch.from_numpy(faces[0].normed_embedding).unsqueeze(0) + face_embeds = torch.from_numpy(faces[0].normed_embedding).unsqueeze(0) + face_image = face_align.norm_crop(image, landmark=faces[0].kps, image_size=224) # you can also segment the face - ip_ckpt = "h94/IP-Adapter-FaceID/ip-adapter-faceid_sd15.bin" - shared.log.debug(f'FaceID model load: {ip_ckpt}') + ip_ckpt = MODELS[model] folder, filename = os.path.split(ip_ckpt) basename, _ext = os.path.splitext(filename) model_path = hf.hf_hub_download(repo_id=folder, filename=filename, cache_dir=shared.opts.diffusers_dir) if model_path is None: - shared.log.error(f'FaceID: model download failed: {ip_ckpt}') + shared.log.error(f'FaceID download failed: model={model} file={ip_ckpt}') return None processing.process_init(p) - shared.sd_model.scheduler = diffusers.DDIMScheduler( - num_train_timesteps=1000, - beta_start=0.00085, - beta_end=0.012, - beta_schedule="scaled_linear", - clip_sample=False, - set_alpha_to_one=False, - steps_offset=1, - ) - ip_model = IPAdapterFaceID(shared.sd_model, model_path, devices.device) + if override: + shared.sd_model.scheduler = diffusers.DDIMScheduler( + num_train_timesteps=1000, + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + steps_offset=1, + ) + shortcut = None + if ip_model is None or ip_model_name != model or ip_model_tokens != tokens or ip_model_rank != rank or not cache: + shared.log.debug(f'FaceID load: model={model} file={ip_ckpt} tokens={tokens} rank={rank}') + if 'Plus' in model: + image_encoder_path = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K" + ip_model = IPAdapterFaceIDPlus( + sd_pipe=shared.sd_model, + image_encoder_path=image_encoder_path, + ip_ckpt=model_path, + lora_rank=rank, + num_tokens=tokens, + device=devices.device, + torch_dtype=devices.dtype, + ) + shortcut = 'v2' in model + elif 'XL' in model: + ip_model = IPAdapterFaceIDXL( + sd_pipe=shared.sd_model, + ip_ckpt=model_path, + lora_rank=rank, + num_tokens=tokens, + device=devices.device, + torch_dtype=devices.dtype, + ) + else: + ip_model = IPAdapterFaceID( + sd_pipe=shared.sd_model, + ip_ckpt=model_path, + lora_rank=rank, + num_tokens=tokens, + device=devices.device, + torch_dtype=devices.dtype, + ) + ip_model_name = model + ip_model_tokens = tokens + ip_model_rank = rank + else: + shared.log.debug(f'FaceID cached: model={model} file={ip_ckpt} tokens={tokens} rank={rank}') + + # main generate dict ip_model_dict = { 'prompt': p.all_prompts[0], 'negative_prompt': p.all_negative_prompts[0], @@ -89,18 +159,34 @@ class Script(scripts.Script): 'scale': scale, 'guidance_scale': p.cfg_scale, 'seed': int(p.all_seeds[0]), - 'faceid_embeds': None, + 'faceid_embeds': face_embeds.shape, } + + # optional generate dict + if shortcut is not None: + ip_model_dict['shortcut'] = shortcut + if 'Plus' in model: + ip_model_dict['s_scale'] = structure + ip_model_dict['face_image'] = face_image.shape shared.log.debug(f'FaceID args: {ip_model_dict}') - ip_model_dict['faceid_embeds'] = embeds + if 'Plus' in model: + ip_model_dict['face_image'] = face_image + ip_model_dict['faceid_embeds'] = face_embeds + + # run generate images = ip_model.generate(**ip_model_dict) - ip_model = None + if not cache: + ip_model = None + ip_model_name = None + devices.torch_gc() + p.extra_generation_params["IP Adapter"] = f'{basename}:{scale}' for i, face in enumerate(faces): p.extra_generation_params[f"FaceID {i} score"] = f'{face.det_score:.2f}' p.extra_generation_params[f"FaceID {i} gender"] = "female" if face.gender==0 else "male" p.extra_generation_params[f"FaceID {i} age"] = face.age + processed = processing.Processed( p, images_list=images, @@ -110,5 +196,4 @@ class Script(scripts.Script): ) processed.info = processed.infotext(p, 0) processed.infotexts = [processed.info] - devices.torch_gc() return processed diff --git a/scripts/ipadapter.py b/scripts/ipadapter.py index edb326667..6fb77adaa 100644 --- a/scripts/ipadapter.py +++ b/scripts/ipadapter.py @@ -15,20 +15,22 @@ from modules import scripts, processing, shared, devices image_encoder = None image_encoder_type = None +image_encoder_name = None loaded = None checkpoint = None +base_repo = "h94/IP-Adapter" ADAPTERS = { 'None': 'none', - 'Base': 'ip-adapter_sd15', - 'Light': 'ip-adapter_sd15_light', - 'Plus': 'ip-adapter-plus_sd15', - 'Plus Face': 'ip-adapter-plus-face_sd15', - 'Full face': 'ip-adapter-full-face_sd15', - 'Base SXDL': 'ip-adapter_sdxl', - # 'models/ip-adapter_sd15_vit-G', # RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x1024 and 1280x3072) - # 'sdxl_models/ip-adapter_sdxl_vit-h', - # 'sdxl_models/ip-adapter-plus_sdxl_vit-h', - # 'sdxl_models/ip-adapter-plus-face_sdxl_vit-h', + 'Base': 'ip-adapter_sd15.safetensors', + 'Base ViT-G': 'ip-adapter_sd15_vit-G.safetensors', + 'Light': 'ip-adapter_sd15_light.safetensors', + 'Plus': 'ip-adapter-plus_sd15.safetensors', + 'Plus Face': 'ip-adapter-plus-face_sd15.safetensors', + 'Full Face': 'ip-adapter-full-face_sd15.safetensors', + 'Base SXDL': 'ip-adapter_sdxl.safetensors', + 'Base ViT-H SXDL': 'ip-adapter_sdxl_vit-h.safetensors', + 'Plus ViT-H SXDL': 'ip-adapter-plus_sdxl_vit-h.safetensors', + 'Plus Face ViT-H SXDL': 'ip-adapter-plus-face_sdxl_vit-h.safetensors', } @@ -48,9 +50,9 @@ class Script(scripts.Script): image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512) return [adapter, scale, image] - def process(self, p: processing.StableDiffusionProcessing, adapter, scale, image): # pylint: disable=arguments-differ + def process(self, p: processing.StableDiffusionProcessing, adapter_name, scale, image): # pylint: disable=arguments-differ # overrides - adapter = ADAPTERS.get(adapter, None) + adapter = ADAPTERS.get(adapter_name, None) if hasattr(p, 'ip_adapter_name'): adapter = p.ip_adapter_name if hasattr(p, 'ip_adapter_scale'): @@ -60,7 +62,7 @@ class Script(scripts.Script): if adapter is None: return # init code - global loaded, checkpoint, image_encoder, image_encoder_type # pylint: disable=global-statement + global loaded, checkpoint, image_encoder, image_encoder_type, image_encoder_name # pylint: disable=global-statement if shared.sd_model is None: return if shared.backend != shared.Backend.DIFFUSERS: @@ -80,25 +82,39 @@ class Script(scripts.Script): if not hasattr(shared.sd_model, 'load_ip_adapter'): shared.log.error(f'IP adapter: pipeline not supported: {shared.sd_model.__class__.__name__}') return - if getattr(shared.sd_model, 'image_encoder', None) is None: - if shared.sd_model_type == 'sd': - subfolder = 'models/image_encoder' - elif shared.sd_model_type == 'sdxl': - subfolder = 'sdxl_models/image_encoder' - else: - shared.log.error(f'IP adapter: unsupported model type: {shared.sd_model_type}') - return - if image_encoder is None or image_encoder_type != shared.sd_model_type or checkpoint != shared.opts.sd_model_checkpoint: + + # which clip to use + if 'ViT' not in adapter_name: + clip_repo = base_repo + subfolder = 'models/image_encoder' if shared.sd_model_type == 'sd' else 'sdxl_models/image_encoder' # defaults per model + elif 'ViT-H' in adapter_name: + clip_repo = base_repo + subfolder = 'models/image_encoder' # this is vit-h + elif 'ViT-G' in adapter_name: + clip_repo = base_repo + subfolder = 'sdxl_models/image_encoder' # this is vit-g + else: + shared.log.error(f'IP adapter: unknown model type: {adapter_name}') + return + + # load image encoder used by ip adapter + if getattr(shared.sd_model, 'image_encoder', None) is None or image_encoder_name != clip_repo + '/' + subfolder: + if image_encoder is None or image_encoder_type != shared.sd_model_type or checkpoint != shared.opts.sd_model_checkpoint or image_encoder_name != clip_repo + '/' + subfolder: + if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl': + shared.log.error(f'IP adapter: unsupported model type: {shared.sd_model_type}') + return try: from transformers import CLIPVisionModelWithProjection - image_encoder = CLIPVisionModelWithProjection.from_pretrained("h94/IP-Adapter", subfolder=subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True).to(devices.device) + shared.log.debug(f'IP adapter: load image encoder: {clip_repo}/{subfolder}') + image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True).to(devices.device) image_encoder_type = shared.sd_model_type + image_encoder_name = clip_repo + '/' + subfolder except Exception as e: shared.log.error(f'IP adapter: failed to load image encoder: {e}') return # main code - subfolder = 'models' if 'sd15' in adapter else 'sdxl_models' + # subfolder = 'models' if 'sd15' in adapter else 'sdxl_models' if adapter != loaded or getattr(shared.sd_model.unet.config, 'encoder_hid_dim_type', None) is None or checkpoint != shared.opts.sd_model_checkpoint: t0 = time.time() if loaded is not None: @@ -107,7 +123,8 @@ class Script(scripts.Script): else: shared.log.debug('IP adapter: load attention processor') shared.sd_model.image_encoder = image_encoder - shared.sd_model.load_ip_adapter("h94/IP-Adapter", subfolder=subfolder, weight_name=f'{adapter}.safetensors') + subfolder = 'models' if shared.sd_model_type == 'sd' else 'sdxl_models' + shared.sd_model.load_ip_adapter(base_repo, subfolder=subfolder, weight_name=adapter) t1 = time.time() shared.log.info(f'IP adapter load: adapter="{adapter}" scale={scale} image={image} time={t1-t0:.2f}') loaded = adapter