From 2410012812aa929f40f22dab3405e03c55bc3b2b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 27 Oct 2024 11:33:40 -0400 Subject: [PATCH] update ipadapters Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 29 ++++++----- installer.py | 7 +-- modules/ipadapter.py | 103 ++++++++++++++++++++++++--------------- modules/sd_vae_approx.py | 2 +- modules/sd_vae_taesd.py | 6 +-- modules/ui_common.py | 11 +++-- scripts/ipadapter.py | 11 +++-- 7 files changed, 104 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5381b08b5..8040b1772 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,9 @@ # Change Log for SD.Next -## Update for 2024-10-26 +## Update for 2024-10-27 -Improvements: -- Torch CUDA set device memory limit - in *settings -> compute settings -> torch memory limit* - default=0 meaning no limit, if set torch will limit memory usage to specified fraction - *note*: this is not a hard limit, torch will try to stay under this value -- Model selector: +improvements: +- model selector: - change-in-behavior - when typing, it will auto-load model as soon as exactly one match is found - allows entering model that are not on the list which triggers huggingface search @@ -18,17 +14,26 @@ Improvements: e.g. `https://civitai.com/api/download/models/72396?type=Model&format=SafeTensor&size=full&fp=fp16` - auto-search-and-download can be disabled in settings -> models -> auto-download this also disables reference models as they are auto-downloaded on first use as well -- SD3 loader enhancements +- sd3 loader enhancements - report when loading incomplete model - handle missing model components - handle component preloading - native lora handler - gguf transformer loader (prototype) -- OpenVINO: add accuracy option -- ZLUDA: guess GPU arch -- Major model load refactor +- ipadapter: + - list available adapters based on loaded model type + - add adapter `ostris consistency` for sd15/sdxl +- torch + - CUDA set device memory limit + in *settings -> compute settings -> torch memory limit* + default=0 meaning no limit, if set torch will limit memory usage to specified fraction + *note*: this is not a hard limit, torch will try to stay under this value +- compute backends: + - OpenVINO: add accuracy option + - ZLUDA: guess GPU arch +- major model load refactor -Fixes: +fixes: - fix send-to-control - fix k-diffusion - fix sd3 img2img and hires diff --git a/installer.py b/installer.py index 2cc1c74a7..207dde388 100644 --- a/installer.py +++ b/installer.py @@ -254,11 +254,12 @@ def uninstall(package, quiet = False): @lru_cache() def pip(arg: str, ignore: bool = False, quiet: bool = False, uv = True): originalArg = arg - uv = uv and args.uv - pipCmd = "uv pip" if uv else "pip" arg = arg.replace('>=', '==') + package = arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip() + uv = uv and args.uv and not package.startswith('git+') + pipCmd = "uv pip" if uv else "pip" if not quiet and '-r ' not in arg: - log.info(f'Install: package="{arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip()}" mode={"uv" if uv else "pip"}') + log.info(f'Install: package="{package}" mode={"uv" if uv else "pip"}') env_args = os.environ.get("PIP_EXTRA_ARGS", "") all_args = f'{pip_log}{arg} {env_args}'.strip() if not quiet: diff --git a/modules/ipadapter.py b/modules/ipadapter.py index 0ab27f03c..0a010c41c 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -3,8 +3,6 @@ Lightweight IP-Adapter applied to existing pipeline in Diffusers - Downloads image_encoder or first usage (2.5GB) - Introduced via: https://github.com/huggingface/diffusers/pull/5713 - IP adapters: https://huggingface.co/h94/IP-Adapter -TODO ipadapter items: -- SD/SDXL autodetect """ import os @@ -14,21 +12,41 @@ from PIL import Image from modules import processing, shared, devices, sd_models -base_repo = "h94/IP-Adapter" +clip_repo = "h94/IP-Adapter" clip_loaded = None -ADAPTERS = { - 'None': 'none', - 'Base': 'ip-adapter_sd15.safetensors', - 'Base ViT-G': 'ip-adapter_sd15_vit-G.safetensors', - 'Light': 'ip-adapter_sd15_light.safetensors', - 'Plus': 'ip-adapter-plus_sd15.safetensors', - 'Plus Face': 'ip-adapter-plus-face_sd15.safetensors', - 'Full Face': 'ip-adapter-full-face_sd15.safetensors', - 'Base SDXL': 'ip-adapter_sdxl.safetensors', - 'Base ViT-H SDXL': 'ip-adapter_sdxl_vit-h.safetensors', - 'Plus ViT-H SDXL': 'ip-adapter-plus_sdxl_vit-h.safetensors', - 'Plus Face ViT-H SDXL': 'ip-adapter-plus-face_sdxl_vit-h.safetensors', +ADAPTERS_NONE = { + 'None': { 'name': 'none', 'repo': 'none', 'subfolder': 'none' }, } +ADAPTERS_SD15 = { + 'None': { 'name': 'none', 'repo': 'none', 'subfolder': 'none' }, + 'Base': { 'name': 'ip-adapter_sd15.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'models' }, + 'Base ViT-G': { 'name': 'ip-adapter_sd15_vit-G.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'models' }, + 'Light': { 'name': 'ip-adapter_sd15_light.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'models' }, + 'Plus': { 'name': 'ip-adapter-plus_sd15.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'models' }, + 'Plus Face': { 'name': 'ip-adapter-plus-face_sd15.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'models' }, + 'Full Face': { 'name': 'ip-adapter-full-face_sd15.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'models' }, + 'Ostris Composition ViT-H': { 'name': 'ip_plus_composition_sd15.safetensors', 'repo': 'ostris/ip-composition-adapter', 'subfolder': '' }, +} +ADAPTERS_SDXL = { + 'None': { 'name': 'none', 'repo': 'none', 'subfolder': 'none' }, + 'Base SDXL': { 'name': 'ip-adapter_sdxl.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'sdxl_models' }, + 'Base ViT-H SDXL': { 'name': 'ip-adapter_sdxl_vit-h.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'sdxl_models' }, + 'Plus ViT-H SDXL': { 'name': 'ip-adapter-plus_sdxl_vit-h.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'sdxl_models' }, + 'Plus Face ViT-H SDXL': { 'name': 'ip-adapter-plus-face_sdxl_vit-h.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'sdxl_models' }, + 'Ostris Composition ViT-H SDXL': { 'name': 'ip_plus_composition_sdxl.safetensors', 'repo': 'ostris/ip-composition-adapter', 'subfolder': '' }, +} +ADAPTERS = { **ADAPTERS_SD15, **ADAPTERS_SDXL } + + +def get_adapters(): + global ADAPTERS # pylint: disable=global-statement + if shared.sd_model_type == 'sd': + ADAPTERS = ADAPTERS_SD15 + elif shared.sd_model_type == 'sdxl': + ADAPTERS = ADAPTERS_SDXL + else: + ADAPTERS = ADAPTERS_NONE + return list(ADAPTERS) def get_images(input_images): @@ -117,13 +135,13 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt if hasattr(p, 'ip_adapter_names'): if isinstance(p.ip_adapter_names, str): p.ip_adapter_names = [p.ip_adapter_names] - adapters = [ADAPTERS.get(adapter, None) for adapter in p.ip_adapter_names if adapter is not None and adapter.lower() != 'none'] + adapters = [ADAPTERS.get(adapter_name, None) for adapter_name in p.ip_adapter_names if adapter_name is not None and adapter_name.lower() != 'none'] adapter_names = p.ip_adapter_names else: if isinstance(adapter_names, str): adapter_names = [adapter_names] adapters = [ADAPTERS.get(adapter, None) for adapter in adapter_names] - adapters = [adapter for adapter in adapters if adapter is not None and adapter.lower() != 'none'] + adapters = [adapter for adapter in adapters if adapter is not None and adapter['name'].lower() != 'none'] if len(adapters) == 0: unapply(pipe) if hasattr(p, 'ip_adapter_images'): @@ -189,41 +207,48 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt for adapter_name in adapter_names: # which clip to use - if 'ViT' not in adapter_name: - clip_repo = base_repo - clip_subfolder = 'models/image_encoder' if shared.sd_model_type == 'sd' else 'sdxl_models/image_encoder' # defaults per model + if 'ViT' not in adapter_name: # defaults per model + if shared.sd_model_type == 'sd': + clip_subfolder = 'models/image_encoder' + else: + clip_subfolder = 'sdxl_models/image_encoder' elif 'ViT-H' in adapter_name: - clip_repo = base_repo clip_subfolder = 'models/image_encoder' # this is vit-h elif 'ViT-G' in adapter_name: - clip_repo = base_repo clip_subfolder = 'sdxl_models/image_encoder' # this is vit-g else: shared.log.error(f'IP adapter: unknown model type: {adapter_name}') return False - # load feature extractor used by ip adapter - if pipe.feature_extractor is None: + # load feature extractor used by ip adapter + if pipe.feature_extractor is None: + try: from transformers import CLIPImageProcessor shared.log.debug('IP adapter load: feature extractor') pipe.feature_extractor = CLIPImageProcessor() - # load image encoder used by ip adapter - if pipe.image_encoder is None or clip_loaded != f'{clip_repo}/{clip_subfolder}': - try: - from transformers import CLIPVisionModelWithProjection - shared.log.debug(f'IP adapter load: image encoder="{clip_repo}/{clip_subfolder}"') - pipe.image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True) - clip_loaded = f'{clip_repo}/{clip_subfolder}' - except Exception as e: - shared.log.error(f'IP adapter: failed to load image encoder: {e}') - return False - sd_models.move_model(pipe.image_encoder, devices.device) + except Exception as e: + shared.log.error(f'IP adapter load: feature extractor {e}') + return False + + # load image encoder used by ip adapter + if pipe.image_encoder is None or clip_loaded != f'{clip_repo}/{clip_subfolder}': + try: + from transformers import CLIPVisionModelWithProjection + shared.log.debug(f'IP adapter load: image encoder="{clip_repo}/{clip_subfolder}"') + pipe.image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True) + clip_loaded = f'{clip_repo}/{clip_subfolder}' + except Exception as e: + shared.log.error(f'IP adapter load: image encoder="{clip_repo}/{clip_subfolder}" {e}') + return False + sd_models.move_model(pipe.image_encoder, devices.device) # main code - t0 = time.time() - ip_subfolder = 'models' if shared.sd_model_type == 'sd' else 'sdxl_models' try: - pipe.load_ip_adapter([base_repo], subfolder=[ip_subfolder], weight_name=adapters) + t0 = time.time() + repos = [adapter['repo'] for adapter in adapters] + subfolders = [adapter['subfolder'] for adapter in adapters] + names = [adapter['name'] for adapter in adapters] + pipe.load_ip_adapter(repos, subfolder=subfolders, weight_name=names) if hasattr(p, 'ip_adapter_layers'): pipe.set_ip_adapter_scale(p.ip_adapter_layers) ip_str = ';'.join(adapter_names) + ':' + json.dumps(p.ip_adapter_layers) @@ -240,5 +265,5 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt t1 = time.time() shared.log.info(f'IP adapter: {ip_str} image={adapter_images} mask={adapter_masks is not None} time={t1-t0:.2f}') except Exception as e: - shared.log.error(f'IP adapter failed to load: repo="{base_repo}" folder="{ip_subfolder}" weights={adapters} names={adapter_names} {e}') + shared.log.error(f'IP adapter load: adapters={adapter_names} repo={repos} folders={subfolders} names={names} {e}') return True diff --git a/modules/sd_vae_approx.py b/modules/sd_vae_approx.py index 2b4399edb..78fe8f08b 100644 --- a/modules/sd_vae_approx.py +++ b/modules/sd_vae_approx.py @@ -46,7 +46,7 @@ def nn_approximation(sample): # Approximate NN sd_vae_approx_model.load_state_dict(approx_weights) sd_vae_approx_model.eval() sd_vae_approx_model.to(device, dtype) - shared.log.debug(f'VAE load: type=approximate model={model_path}') + shared.log.debug(f'VAE load: type=approximate model="{model_path}"') try: in_sample = sample.to(device, dtype).unsqueeze(0) sd_vae_approx_model.to(device, dtype) diff --git a/modules/sd_vae_taesd.py b/modules/sd_vae_taesd.py index f99f296a8..4d213ad48 100644 --- a/modules/sd_vae_taesd.py +++ b/modules/sd_vae_taesd.py @@ -160,11 +160,11 @@ def decode(latents): download_model(model_path) if os.path.exists(model_path): taesd_models[f'{model_class}-decoder'] = TAESD(decoder_path=model_path, encoder_path=None) - shared.log.debug(f'VAE load: type=taesd model={model_path}') + shared.log.debug(f'VAE load: type=taesd model="{model_path}"') vae = taesd_models[f'{model_class}-decoder'] vae.decoder.to(devices.device, dtype) else: - shared.log.error(f'VAE load: type=taesd model={model_path} not found') + shared.log.error(f'VAE load: type=taesd model="{model_path}" not found') return latents if vae is None: return latents @@ -208,7 +208,7 @@ def encode(image): model_path = os.path.join(paths.models_path, "TAESD", f"tae{model_class}_encoder.pth") download_model(model_path) if os.path.exists(model_path): - shared.log.debug(f'VAE load: type=taesd model={model_path}') + shared.log.debug(f'VAE load: type=taesd model="{model_path}"') taesd_models[f'{model_class}-encoder'] = TAESD(encoder_path=model_path, decoder_path=None) vae = taesd_models[f'{model_class}-encoder'] vae.encoder.to(devices.device, devices.dtype_vae) diff --git a/modules/ui_common.py b/modules/ui_common.py index 5e873355b..9ad87c17f 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -319,13 +319,18 @@ def create_output_panel(tabname, preview=True, prompt=None, height=None): return result_gallery, generation_info, html_info, html_info_formatted, html_log -def create_refresh_button(refresh_component, refresh_method, refreshed_args, elem_id, visible: bool = True): +def create_refresh_button(refresh_component, refresh_method, refreshed_args = None, elem_id = None, visible: bool = True): def refresh(): refresh_method() - args = refreshed_args() if callable(refreshed_args) else refreshed_args + if refreshed_args is None: + args = {"choices": refresh_method()} # pylint: disable=unnecessary-lambda-assignment + elif callable(refreshed_args): + args = refreshed_args() + else: + args = refreshed_args for k, v in args.items(): setattr(refresh_component, k, v) - return gr.update(**(args or {})) + return gr.update(**args) refresh_button = ui_components.ToolButton(value=ui_symbols.refresh, elem_id=elem_id, visible=visible) refresh_button.click(fn=refresh, inputs=[], outputs=[refresh_component]) diff --git a/scripts/ipadapter.py b/scripts/ipadapter.py index c7fcb3053..60c70b9dc 100644 --- a/scripts/ipadapter.py +++ b/scripts/ipadapter.py @@ -1,7 +1,7 @@ import json from PIL import Image import gradio as gr -from modules import scripts, processing, shared, ipadapter +from modules import scripts, processing, shared, ipadapter, ui_common MAX_ADAPTERS = 4 @@ -60,9 +60,12 @@ class Script(scripts.Script): for i in range(MAX_ADAPTERS): with gr.Accordion(f'Adapter {i+1}', visible=i==0) as unit: with gr.Row(): - adapters.append(gr.Dropdown(label='Adapter', choices=list(ipadapter.ADAPTERS), value='None')) - scales.append(gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=0.5)) - crops.append(gr.Checkbox(label='Crop', default=False, interactive=True)) + adapter = gr.Dropdown(label='Adapter', choices=list(ipadapter.get_adapters()), value='None') + adapters.append(adapter) + ui_common.create_refresh_button(adapter, ipadapter.get_adapters) + with gr.Row(): + scales.append(gr.Slider(label='Strength', minimum=0.0, maximum=1.0, step=0.01, value=0.5)) + crops.append(gr.Checkbox(label='Crop to portrait', default=False, interactive=True)) with gr.Row(): starts.append(gr.Slider(label='Start', minimum=0.0, maximum=1.0, step=0.1, value=0)) ends.append(gr.Slider(label='End', minimum=0.0, maximum=1.0, step=0.1, value=1))