full faceid and updated ipadapter

2026-09-19 01:04:32 +02:00 · 2024-01-06 11:52:52 -05:00
parent 746fd1dfaa
commit dab9087069
5 changed files with 172 additions and 59 deletions
@@ -30,6 +30,16 @@ And it also includes fixes for all reported issues so far
  - fix correct image mode  
  - fix batch/folder/video modes  
  - fix pipeline switching between different modes  
+- [FaceID](https://huggingface.co/h94/IP-Adapter-FaceID)  
+  full implementation for *SD15* and *SD-XL*, to use simply select from *Scripts*  
+  - **Base** (93MB) uses *InsightFace* to generate face embeds and *OpenCLIP-ViT-H-14* (2.5GB) as image encoder  
+  - **SXDL** (1022MB) uses *InsightFace* to generate face embeds and *OpenCLIP-ViT-bigG-14* (3.7GB) as image encoder  
+  - **Plus** (150MB) uses *InsightFace* to generate face embeds and *CLIP-ViT-H-14-laion2B* (3.8GB) as image encoder  
+  *note*: all models are downloaded on first use  
+- [IPAdapter](https://huggingface.co/h94/IP-Adapter)  
+  additional models for *SD15* and *SD-XL*, to use simply select from *Scripts*:  
+  - **SD15**: Base, Base ViT-G, Light, Plus, Plus Face, Full Face  
+  - **SDXL**: Base SXDL, Base ViT-H SXDL, Plus ViT-H SXDL, Plus Face ViT-H SXDL  
 - **Improvements**  
  - **server startup**: performance  
    - faster extension load
@@ -44,7 +54,7 @@ And it also includes fixes for all reported issues so far
    - enable vae tiling  
    - add autodetect optimial value  
      set tile size to 0 to use autodetected value  
-  - **cli**: 
+  - **cli**  
    - `sdapi.py` allow manual api invoke  
      example: `python cli/sdapi.py /sdapi/v1/sd-models`  
    - `image-exif.py` improve metadata parsing  
@@ -94,6 +104,7 @@ And it also includes fixes for all reported issues so far
  - processing: correct display metadata  
  - live preview: fix when using `bfloat16`
  - upscale: fix ldsr
+  - cli: fix cmd args parsing  

 ## Update for 2023-12-29

@@ -80,12 +80,12 @@ def compatibility_args(opts, args):
    group.add_argument("--swinir-models-path", help=argparse.SUPPRESS, default=opts.swinir_models_path)
    group.add_argument("--ldsr-models-path", help=argparse.SUPPRESS, default=opts.ldsr_models_path)
    group.add_argument("--clip-models-path", type=str, help=argparse.SUPPRESS, default=opts.clip_models_path)
-    group.add_argument("--opt-channelslast", help=argparse.SUPPRESS, default=opts.opt_channelslast)
+    group.add_argument("--opt-channelslast", help=argparse.SUPPRESS, action='store_true', default=opts.opt_channelslast)
    group.add_argument("--xformers", default=(opts.cross_attention_optimization == "xFormers"), action='store_true', help=argparse.SUPPRESS)
-    group.add_argument("--disable-nan-check", help=argparse.SUPPRESS, default=opts.disable_nan_check)
+    group.add_argument("--disable-nan-check", help=argparse.SUPPRESS, action='store_true', default=opts.disable_nan_check)
    group.add_argument("--rollback-vae", help=argparse.SUPPRESS, default=opts.rollback_vae)
-    group.add_argument("--no-half", help=argparse.SUPPRESS, default=opts.no_half)
-    group.add_argument("--no-half-vae", help=argparse.SUPPRESS, default=opts.no_half_vae)
+    group.add_argument("--no-half", help=argparse.SUPPRESS, action='store_true', default=opts.no_half)
+    group.add_argument("--no-half-vae", help=argparse.SUPPRESS, action='store_true', default=opts.no_half_vae)
    group.add_argument("--precision", help=argparse.SUPPRESS, default=opts.precision)
    group.add_argument("--sub-quad-q-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_q_chunk_size)
    group.add_argument("--sub-quad-kv-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_kv_chunk_size)
@@ -94,7 +94,7 @@ def compatibility_args(opts, args):
    group.add_argument("--lyco-dir", help=argparse.SUPPRESS, default=opts.lyco_dir)
    group.add_argument("--embeddings-dir", help=argparse.SUPPRESS, default=opts.embeddings_dir)
    group.add_argument("--hypernetwork-dir", help=argparse.SUPPRESS, default=opts.hypernetwork_dir)
-    group.add_argument("--lyco-patch-lora", help=argparse.SUPPRESS, default=False)
+    group.add_argument("--lyco-patch-lora", help=argparse.SUPPRESS, action='store_true', default=False)
    group.add_argument("--lyco-debug", help=argparse.SUPPRESS, action='store_true', default=False)
    group.add_argument("--enable-console-prompts", help=argparse.SUPPRESS, action='store_true', default=False)
    group.add_argument("--safe", help=argparse.SUPPRESS, action='store_true', default=False)
@@ -6,11 +6,30 @@ import gradio as gr
 import diffusers
 import huggingface_hub as hf
 from modules import scripts, processing, shared, devices
-from installer import installed


+MODELS = {
+    'FaceID Base': 'h94/IP-Adapter-FaceID/ip-adapter-faceid_sd15.bin',
+    'FaceID Plus': 'h94/IP-Adapter-FaceID/ip-adapter-faceid-plus_sd15.bin',
+    'FaceID Plus v2': 'h94/IP-Adapter-FaceID/ip-adapter-faceid-plusv2_sd15.bin',
+    'FaceID XL': 'h94/IP-Adapter-FaceID/ip-adapter-faceid_sdxl.bin'
+}
 app = None
-ok = installed('insightface', reload=False, quiet=True) and installed('ip_adapter', reload=False, quiet=True)
+ip_model = None
+ip_model_name = None
+ip_model_tokens = None
+ip_model_rank = None
+
+
+def dependencies():
+    from installer import installed, install
+    packages = [
+        ('insightface', 'insightface'),
+        ('git+https://github.com/tencent-ailab/IP-Adapter.git', 'ip_adapter'),
+    ]
+    for pkg in packages:
+        if not installed(pkg[1], reload=False, quiet=True):
+            install(pkg[0], pkg[1], ignore=True)


 class Script(scripts.Script):
@@ -18,32 +37,43 @@ class Script(scripts.Script):
        return 'FaceID'

    def show(self, is_img2img):
-        return ok if shared.backend == shared.Backend.DIFFUSERS else False
+        return True if shared.backend == shared.Backend.DIFFUSERS else False

    # return signature is array of gradio components
    def ui(self, _is_img2img):
        with gr.Row():
-            scale = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=1.0)
+            model = gr.Dropdown(choices=list(MODELS), label='Model', value='FaceID Base')
+        with gr.Row(visible=True):
+            override = gr.Checkbox(label='Override sampler', value=True)
+            cache = gr.Checkbox(label='Cache model', value=True)
+        with gr.Row(visible=True):
+            scale = gr.Slider(label='Strength', minimum=0.0, maximum=1.0, step=0.01, value=1.0)
+            structure = gr.Slider(label='Structure', minimum=0.0, maximum=1.0, step=0.01, value=1.0)
+        with gr.Row(visible=False):
+            rank = gr.Slider(label='Rank', minimum=4, maximum=256, step=4, value=128)
+            tokens = gr.Slider(label='Tokens', minimum=1, maximum=16, step=1, value=4)
        with gr.Row():
            image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512)
-        return [scale, image]
+        return [model, scale, image, override, rank, tokens, structure, cache]

-    def run(self, p: processing.StableDiffusionProcessing, scale, image): # pylint: disable=arguments-differ, unused-argument
+    def run(self, p: processing.StableDiffusionProcessing, model, scale, image, override, rank, tokens, structure, cache): # pylint: disable=arguments-differ, unused-argument
+        dependencies()
        try:
            import onnxruntime
            from insightface.app import FaceAnalysis
-            from ip_adapter.ip_adapter_faceid import IPAdapterFaceID
+            from insightface.utils import face_align
+            from ip_adapter.ip_adapter_faceid import IPAdapterFaceID, IPAdapterFaceIDPlus, IPAdapterFaceIDXL
        except Exception as e:
            shared.log.error(f'FaceID: {e}')
            return None
        if image is None:
            shared.log.error('FaceID: no init_images')
            return None
-        if shared.sd_model_type != 'sd':
+        if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
            shared.log.error('FaceID: base model not supported')
            return None

-        global app # pylint: disable=global-statement
+        global app, ip_model, ip_model_name, ip_model_tokens, ip_model_rank # pylint: disable=global-statement
        if app is None:
            shared.log.debug(f"ONNX: device={onnxruntime.get_device()} providers={onnxruntime.get_available_providers()}")
            app = FaceAnalysis(name="buffalo_l", providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
@@ -57,28 +87,68 @@ class Script(scripts.Script):
            return None
        for face in faces:
            shared.log.debug(f'FaceID face: score={face.det_score:.2f} gender={"female" if face.gender==0 else "male"} age={face.age} bbox={face.bbox}')
-        embeds = torch.from_numpy(faces[0].normed_embedding).unsqueeze(0)
+        face_embeds = torch.from_numpy(faces[0].normed_embedding).unsqueeze(0)
+        face_image = face_align.norm_crop(image, landmark=faces[0].kps, image_size=224) # you can also segment the face

-        ip_ckpt = "h94/IP-Adapter-FaceID/ip-adapter-faceid_sd15.bin"
-        shared.log.debug(f'FaceID model load: {ip_ckpt}')
+        ip_ckpt = MODELS[model]
        folder, filename = os.path.split(ip_ckpt)
        basename, _ext = os.path.splitext(filename)
        model_path = hf.hf_hub_download(repo_id=folder, filename=filename, cache_dir=shared.opts.diffusers_dir)
        if model_path is None:
-            shared.log.error(f'FaceID: model download failed: {ip_ckpt}')
+            shared.log.error(f'FaceID download failed: model={model} file={ip_ckpt}')
            return None

        processing.process_init(p)
-        shared.sd_model.scheduler = diffusers.DDIMScheduler(
-            num_train_timesteps=1000,
-            beta_start=0.00085,
-            beta_end=0.012,
-            beta_schedule="scaled_linear",
-            clip_sample=False,
-            set_alpha_to_one=False,
-            steps_offset=1,
-        )
-        ip_model = IPAdapterFaceID(shared.sd_model, model_path, devices.device)
+        if override:
+            shared.sd_model.scheduler = diffusers.DDIMScheduler(
+                num_train_timesteps=1000,
+                beta_start=0.00085,
+                beta_end=0.012,
+                beta_schedule="scaled_linear",
+                clip_sample=False,
+                set_alpha_to_one=False,
+                steps_offset=1,
+            )
+        shortcut = None
+        if ip_model is None or ip_model_name != model or ip_model_tokens != tokens or ip_model_rank != rank or not cache:
+            shared.log.debug(f'FaceID load: model={model} file={ip_ckpt} tokens={tokens} rank={rank}')
+            if 'Plus' in model:
+                image_encoder_path = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K"
+                ip_model = IPAdapterFaceIDPlus(
+                    sd_pipe=shared.sd_model,
+                    image_encoder_path=image_encoder_path,
+                    ip_ckpt=model_path,
+                    lora_rank=rank,
+                    num_tokens=tokens,
+                    device=devices.device,
+                    torch_dtype=devices.dtype,
+                )
+                shortcut = 'v2' in model
+            elif 'XL' in model:
+                ip_model = IPAdapterFaceIDXL(
+                    sd_pipe=shared.sd_model,
+                    ip_ckpt=model_path,
+                    lora_rank=rank,
+                    num_tokens=tokens,
+                    device=devices.device,
+                    torch_dtype=devices.dtype,
+                )
+            else:
+                ip_model = IPAdapterFaceID(
+                    sd_pipe=shared.sd_model,
+                    ip_ckpt=model_path,
+                    lora_rank=rank,
+                    num_tokens=tokens,
+                    device=devices.device,
+                    torch_dtype=devices.dtype,
+                )
+            ip_model_name = model
+            ip_model_tokens = tokens
+            ip_model_rank = rank
+        else:
+            shared.log.debug(f'FaceID cached: model={model} file={ip_ckpt} tokens={tokens} rank={rank}')
+
+        # main generate dict
        ip_model_dict = {
            'prompt': p.all_prompts[0],
            'negative_prompt': p.all_negative_prompts[0],
@@ -89,18 +159,34 @@ class Script(scripts.Script):
            'scale': scale,
            'guidance_scale': p.cfg_scale,
            'seed': int(p.all_seeds[0]),
-            'faceid_embeds': None,
+            'faceid_embeds': face_embeds.shape,
        }
+
+        # optional generate dict
+        if shortcut is not None:
+            ip_model_dict['shortcut'] = shortcut
+        if 'Plus' in model:
+            ip_model_dict['s_scale'] = structure
+            ip_model_dict['face_image'] = face_image.shape
        shared.log.debug(f'FaceID args: {ip_model_dict}')
-        ip_model_dict['faceid_embeds'] = embeds
+        if 'Plus' in model:
+            ip_model_dict['face_image'] = face_image
+        ip_model_dict['faceid_embeds'] = face_embeds
+
+        # run generate
        images = ip_model.generate(**ip_model_dict)

-        ip_model = None
+        if not cache:
+            ip_model = None
+            ip_model_name = None
+        devices.torch_gc()
+
        p.extra_generation_params["IP Adapter"] = f'{basename}:{scale}'
        for i, face in enumerate(faces):
            p.extra_generation_params[f"FaceID {i} score"] = f'{face.det_score:.2f}'
            p.extra_generation_params[f"FaceID {i} gender"] = "female" if face.gender==0 else "male"
            p.extra_generation_params[f"FaceID {i} age"] = face.age
+
        processed = processing.Processed(
            p,
            images_list=images,
@@ -110,5 +196,4 @@ class Script(scripts.Script):
        )
        processed.info = processed.infotext(p, 0)
        processed.infotexts = [processed.info]
-        devices.torch_gc()
        return processed
@@ -15,20 +15,22 @@ from modules import scripts, processing, shared, devices

 image_encoder = None
 image_encoder_type = None
+image_encoder_name = None
 loaded = None
 checkpoint = None
+base_repo = "h94/IP-Adapter"
 ADAPTERS = {
    'None': 'none',
-    'Base': 'ip-adapter_sd15',
-    'Light': 'ip-adapter_sd15_light',
-    'Plus': 'ip-adapter-plus_sd15',
-    'Plus Face': 'ip-adapter-plus-face_sd15',
-    'Full face': 'ip-adapter-full-face_sd15',
-    'Base SXDL': 'ip-adapter_sdxl',
-    # 'models/ip-adapter_sd15_vit-G', # RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x1024 and 1280x3072)
-    # 'sdxl_models/ip-adapter_sdxl_vit-h',
-    # 'sdxl_models/ip-adapter-plus_sdxl_vit-h',
-    # 'sdxl_models/ip-adapter-plus-face_sdxl_vit-h',
+    'Base': 'ip-adapter_sd15.safetensors',
+    'Base ViT-G': 'ip-adapter_sd15_vit-G.safetensors',
+    'Light': 'ip-adapter_sd15_light.safetensors',
+    'Plus': 'ip-adapter-plus_sd15.safetensors',
+    'Plus Face': 'ip-adapter-plus-face_sd15.safetensors',
+    'Full Face': 'ip-adapter-full-face_sd15.safetensors',
+    'Base SXDL': 'ip-adapter_sdxl.safetensors',
+    'Base ViT-H SXDL': 'ip-adapter_sdxl_vit-h.safetensors',
+    'Plus ViT-H SXDL': 'ip-adapter-plus_sdxl_vit-h.safetensors',
+    'Plus Face ViT-H SXDL': 'ip-adapter-plus-face_sdxl_vit-h.safetensors',
 }


@@ -48,9 +50,9 @@ class Script(scripts.Script):
                image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512)
        return [adapter, scale, image]

-    def process(self, p: processing.StableDiffusionProcessing, adapter, scale, image): # pylint: disable=arguments-differ
+    def process(self, p: processing.StableDiffusionProcessing, adapter_name, scale, image): # pylint: disable=arguments-differ
        # overrides
-        adapter = ADAPTERS.get(adapter, None)
+        adapter = ADAPTERS.get(adapter_name, None)
        if hasattr(p, 'ip_adapter_name'):
            adapter = p.ip_adapter_name
        if hasattr(p, 'ip_adapter_scale'):
@@ -60,7 +62,7 @@ class Script(scripts.Script):
        if adapter is None:
            return
        # init code
-        global loaded, checkpoint, image_encoder, image_encoder_type # pylint: disable=global-statement
+        global loaded, checkpoint, image_encoder, image_encoder_type, image_encoder_name # pylint: disable=global-statement
        if shared.sd_model is None:
            return
        if shared.backend != shared.Backend.DIFFUSERS:
@@ -80,25 +82,39 @@ class Script(scripts.Script):
        if not hasattr(shared.sd_model, 'load_ip_adapter'):
            shared.log.error(f'IP adapter: pipeline not supported: {shared.sd_model.__class__.__name__}')
            return
-        if getattr(shared.sd_model, 'image_encoder', None) is None:
-            if shared.sd_model_type == 'sd':
-                subfolder = 'models/image_encoder'
-            elif shared.sd_model_type == 'sdxl':
-                subfolder = 'sdxl_models/image_encoder'
-            else:
-                shared.log.error(f'IP adapter: unsupported model type: {shared.sd_model_type}')
-                return
-            if image_encoder is None or image_encoder_type != shared.sd_model_type or checkpoint != shared.opts.sd_model_checkpoint:
+
+        # which clip to use
+        if 'ViT' not in adapter_name:
+            clip_repo = base_repo
+            subfolder = 'models/image_encoder' if shared.sd_model_type == 'sd' else 'sdxl_models/image_encoder' # defaults per model
+        elif 'ViT-H' in adapter_name:
+            clip_repo = base_repo
+            subfolder = 'models/image_encoder' # this is vit-h
+        elif 'ViT-G' in adapter_name:
+            clip_repo = base_repo
+            subfolder = 'sdxl_models/image_encoder' # this is vit-g
+        else:
+            shared.log.error(f'IP adapter: unknown model type: {adapter_name}')
+            return
+
+        # load image encoder used by ip adapter
+        if getattr(shared.sd_model, 'image_encoder', None) is None or image_encoder_name != clip_repo + '/' + subfolder:
+            if image_encoder is None or image_encoder_type != shared.sd_model_type or checkpoint != shared.opts.sd_model_checkpoint or image_encoder_name != clip_repo + '/' + subfolder:
+                if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
+                    shared.log.error(f'IP adapter: unsupported model type: {shared.sd_model_type}')
+                    return
                try:
                    from transformers import CLIPVisionModelWithProjection
-                    image_encoder = CLIPVisionModelWithProjection.from_pretrained("h94/IP-Adapter", subfolder=subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True).to(devices.device)
+                    shared.log.debug(f'IP adapter: load image encoder: {clip_repo}/{subfolder}')
+                    image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True).to(devices.device)
                    image_encoder_type = shared.sd_model_type
+                    image_encoder_name = clip_repo + '/' + subfolder
                except Exception as e:
                    shared.log.error(f'IP adapter: failed to load image encoder: {e}')
                    return

        # main code
-        subfolder = 'models' if 'sd15' in adapter else 'sdxl_models'
+        # subfolder = 'models' if 'sd15' in adapter else 'sdxl_models'
        if adapter != loaded or getattr(shared.sd_model.unet.config, 'encoder_hid_dim_type', None) is None or checkpoint != shared.opts.sd_model_checkpoint:
            t0 = time.time()
            if loaded is not None:
@@ -107,7 +123,8 @@ class Script(scripts.Script):
            else:
                shared.log.debug('IP adapter: load attention processor')
            shared.sd_model.image_encoder = image_encoder
-            shared.sd_model.load_ip_adapter("h94/IP-Adapter", subfolder=subfolder, weight_name=f'{adapter}.safetensors')
+            subfolder = 'models' if shared.sd_model_type == 'sd' else 'sdxl_models'
+            shared.sd_model.load_ip_adapter(base_repo, subfolder=subfolder, weight_name=adapter)
            t1 = time.time()
            shared.log.info(f'IP adapter load: adapter="{adapter}" scale={scale} image={image} time={t1-t0:.2f}')
            loaded = adapter