mirror of
https://github.com/vladmandic/automatic
synced 2026-08-28 16:11:02 +02:00
Vladimir Mandic
91 lines
3.9 KiB
Python
91 lines
3.9 KiB
Python
import time
|
|
from PIL import Image
|
|
from modules import shared, processing, devices
|
|
|
|
|
|
image_encoder = None
|
|
image_encoder_type = None
|
|
loaded = None
|
|
ADAPTERS = [
|
|
'none',
|
|
'ip-adapter_sd15',
|
|
'ip-adapter_sd15_light',
|
|
'ip-adapter-plus_sd15',
|
|
'ip-adapter-plus-face_sd15',
|
|
'ip-adapter-full-face_sd15',
|
|
# 'models/ip-adapter_sd15_vit-G', # RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x1024 and 1280x3072)
|
|
'ip-adapter_sdxl',
|
|
# 'sdxl_models/ip-adapter_sdxl_vit-h',
|
|
# 'sdxl_models/ip-adapter-plus_sdxl_vit-h',
|
|
# 'sdxl_models/ip-adapter-plus-face_sdxl_vit-h',
|
|
]
|
|
|
|
|
|
def apply_ip_adapter(pipe, p: processing.StableDiffusionProcessing, adapter, scale, image, reset=False): # pylint: disable=arguments-differ
|
|
from transformers import CLIPVisionModelWithProjection
|
|
# overrides
|
|
if hasattr(p, 'ip_adapter_name'):
|
|
adapter = p.ip_adapter_name
|
|
if hasattr(p, 'ip_adapter_scale'):
|
|
scale = p.ip_adapter_scale
|
|
if hasattr(p, 'ip_adapter_image'):
|
|
image = p.ip_adapter_image
|
|
# init code
|
|
global loaded, image_encoder, image_encoder_type # pylint: disable=global-statement
|
|
if pipe is None:
|
|
return
|
|
if shared.backend != shared.Backend.DIFFUSERS:
|
|
shared.log.warning('IP adapter: not in diffusers mode')
|
|
return False
|
|
if adapter == 'none':
|
|
if hasattr(pipe, 'set_ip_adapter_scale'):
|
|
pipe.set_ip_adapter_scale(0)
|
|
if loaded is not None:
|
|
shared.log.debug('IP adapter: unload attention processor')
|
|
pipe.unet.set_default_attn_processor()
|
|
pipe.unet.config.encoder_hid_dim_type = None
|
|
loaded = None
|
|
return False
|
|
if image is None:
|
|
image = Image.new('RGB', (512, 512), (0, 0, 0))
|
|
if not hasattr(pipe, 'load_ip_adapter'):
|
|
shared.log.error(f'IP adapter: pipeline not supported: {pipe.__class__.__name__}')
|
|
return False
|
|
if getattr(pipe, 'image_encoder', None) is None or getattr(pipe, 'image_encoder', None) == (None, None):
|
|
if shared.sd_model_type == 'sd':
|
|
subfolder = 'models/image_encoder'
|
|
elif shared.sd_model_type == 'sdxl':
|
|
subfolder = 'sdxl_models/image_encoder'
|
|
else:
|
|
shared.log.error(f'IP adapter: unsupported model type: {shared.sd_model_type}')
|
|
return False
|
|
if image_encoder is None or image_encoder_type != shared.sd_model_type:
|
|
try:
|
|
image_encoder = CLIPVisionModelWithProjection.from_pretrained("h94/IP-Adapter", subfolder=subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True).to(devices.device)
|
|
image_encoder_type = shared.sd_model_type
|
|
except Exception as e:
|
|
shared.log.error(f'IP adapter: failed to load image encoder: {e}')
|
|
return False
|
|
pipe.image_encoder = image_encoder
|
|
|
|
# main code
|
|
subfolder = 'models' if 'sd15' in adapter else 'sdxl_models'
|
|
if adapter != loaded or getattr(pipe.unet.config, 'encoder_hid_dim_type', None) is None or reset:
|
|
t0 = time.time()
|
|
if loaded is not None:
|
|
# shared.log.debug('IP adapter: reset attention processor')
|
|
pipe.unet.set_default_attn_processor()
|
|
loaded = None
|
|
else:
|
|
shared.log.debug('IP adapter: load attention processor')
|
|
pipe.load_ip_adapter("h94/IP-Adapter", subfolder=subfolder, weight_name=f'{adapter}.safetensors')
|
|
t1 = time.time()
|
|
shared.log.info(f'IP adapter load: adapter="{adapter}" scale={scale} image={image} time={t1-t0:.2f}')
|
|
loaded = adapter
|
|
else:
|
|
shared.log.debug(f'IP adapter cache: adapter="{adapter}" scale={scale} image={image}')
|
|
pipe.set_ip_adapter_scale(scale)
|
|
p.task_args['ip_adapter_image'] = p.batch_size * [image]
|
|
p.extra_generation_params["IP Adapter"] = f'{adapter}:{scale}'
|
|
return True
|