Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-08-26 15:16:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

update ipadapters

Browse Source 
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
2024-10-27 11:33:40 -04:00
parent 706852e7c5
commit 2410012812
7 changed files with 104 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+17 -12
View File
@@ -1,13 +1,9 @@
# Change Log for SD.Next
## Update for 2024-10-26
## Update for 2024-10-27
Improvements:
- Torch CUDA set device memory limit
in *settings -> compute settings -> torch memory limit*
default=0 meaning no limit, if set torch will limit memory usage to specified fraction
*note*: this is not a hard limit, torch will try to stay under this value
- Model selector:
improvements:
- model selector:
- change-in-behavior
- when typing, it will auto-load model as soon as exactly one match is found
- allows entering model that are not on the list which triggers huggingface search
@@ -18,17 +14,26 @@ Improvements:
e.g. `https://civitai.com/api/download/models/72396?type=Model&format=SafeTensor&size=full&fp=fp16`
- auto-search-and-download can be disabled in settings -> models -> auto-download
this also disables reference models as they are auto-downloaded on first use as well
- SD3 loader enhancements
- sd3 loader enhancements
- report when loading incomplete model
- handle missing model components
- handle component preloading
- native lora handler
- gguf transformer loader (prototype)
- OpenVINO: add accuracy option
- ZLUDA: guess GPU arch
- Major model load refactor
- ipadapter:
- list available adapters based on loaded model type
- add adapter `ostris consistency` for sd15/sdxl
- torch
- CUDA set device memory limit
in *settings -> compute settings -> torch memory limit*
default=0 meaning no limit, if set torch will limit memory usage to specified fraction
*note*: this is not a hard limit, torch will try to stay under this value
- compute backends:
- OpenVINO: add accuracy option
- ZLUDA: guess GPU arch
- major model load refactor
Fixes:
fixes:
- fix send-to-control
- fix k-diffusion
- fix sd3 img2img and hires
installer.py
+4 -3
View File
@@ -254,11 +254,12 @@ def uninstall(package, quiet = False):
@lru_cache()
def pip(arg: str, ignore: bool = False, quiet: bool = False, uv = True):
originalArg = arg
uv = uv and args.uv
pipCmd = "uv pip" if uv else "pip"
arg = arg.replace('>=', '==')
package = arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip()
uv = uv and args.uv and not package.startswith('git+')
pipCmd = "uv pip" if uv else "pip"
if not quiet and '-r ' not in arg:
log.info(f'Install: package="{arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip()}" mode={"uv" if uv else "pip"}')
log.info(f'Install: package="{package}" mode={"uv" if uv else "pip"}')
env_args = os.environ.get("PIP_EXTRA_ARGS", "")
all_args = f'{pip_log}{arg} {env_args}'.strip()
if not quiet:
modules/ipadapter.py
+64 -39
View File
@@ -3,8 +3,6 @@ Lightweight IP-Adapter applied to existing pipeline in Diffusers
- Downloads image_encoder or first usage (2.5GB)
- Introduced via: https://github.com/huggingface/diffusers/pull/5713
- IP adapters: https://huggingface.co/h94/IP-Adapter
TODO ipadapter items:
- SD/SDXL autodetect
"""
import os
@@ -14,21 +12,41 @@ from PIL import Image
from modules import processing, shared, devices, sd_models
base_repo = "h94/IP-Adapter"
clip_repo = "h94/IP-Adapter"
clip_loaded = None
ADAPTERS = {
'None': 'none',
'Base': 'ip-adapter_sd15.safetensors',
'Base ViT-G': 'ip-adapter_sd15_vit-G.safetensors',
'Light': 'ip-adapter_sd15_light.safetensors',
'Plus': 'ip-adapter-plus_sd15.safetensors',
'Plus Face': 'ip-adapter-plus-face_sd15.safetensors',
'Full Face': 'ip-adapter-full-face_sd15.safetensors',
'Base SDXL': 'ip-adapter_sdxl.safetensors',
'Base ViT-H SDXL': 'ip-adapter_sdxl_vit-h.safetensors',
'Plus ViT-H SDXL': 'ip-adapter-plus_sdxl_vit-h.safetensors',
'Plus Face ViT-H SDXL': 'ip-adapter-plus-face_sdxl_vit-h.safetensors',
ADAPTERS_NONE = {
'None': { 'name': 'none', 'repo': 'none', 'subfolder': 'none' },
}
ADAPTERS_SD15 = {
'None': { 'name': 'none', 'repo': 'none', 'subfolder': 'none' },
'Base': { 'name': 'ip-adapter_sd15.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'models' },
'Base ViT-G': { 'name': 'ip-adapter_sd15_vit-G.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'models' },
'Light': { 'name': 'ip-adapter_sd15_light.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'models' },
'Plus': { 'name': 'ip-adapter-plus_sd15.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'models' },
'Plus Face': { 'name': 'ip-adapter-plus-face_sd15.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'models' },
'Full Face': { 'name': 'ip-adapter-full-face_sd15.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'models' },
'Ostris Composition ViT-H': { 'name': 'ip_plus_composition_sd15.safetensors', 'repo': 'ostris/ip-composition-adapter', 'subfolder': '' },
}
ADAPTERS_SDXL = {
'None': { 'name': 'none', 'repo': 'none', 'subfolder': 'none' },
'Base SDXL': { 'name': 'ip-adapter_sdxl.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'sdxl_models' },
'Base ViT-H SDXL': { 'name': 'ip-adapter_sdxl_vit-h.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'sdxl_models' },
'Plus ViT-H SDXL': { 'name': 'ip-adapter-plus_sdxl_vit-h.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'sdxl_models' },
'Plus Face ViT-H SDXL': { 'name': 'ip-adapter-plus-face_sdxl_vit-h.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'sdxl_models' },
'Ostris Composition ViT-H SDXL': { 'name': 'ip_plus_composition_sdxl.safetensors', 'repo': 'ostris/ip-composition-adapter', 'subfolder': '' },
}
ADAPTERS = { **ADAPTERS_SD15, **ADAPTERS_SDXL }
def get_adapters():
global ADAPTERS # pylint: disable=global-statement
if shared.sd_model_type == 'sd':
ADAPTERS = ADAPTERS_SD15
elif shared.sd_model_type == 'sdxl':
ADAPTERS = ADAPTERS_SDXL
else:
ADAPTERS = ADAPTERS_NONE
return list(ADAPTERS)
def get_images(input_images):
@@ -117,13 +135,13 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt
if hasattr(p, 'ip_adapter_names'):
if isinstance(p.ip_adapter_names, str):
p.ip_adapter_names = [p.ip_adapter_names]
adapters = [ADAPTERS.get(adapter, None) for adapter in p.ip_adapter_names if adapter is not None and adapter.lower() != 'none']
adapters = [ADAPTERS.get(adapter_name, None) for adapter_name in p.ip_adapter_names if adapter_name is not None and adapter_name.lower() != 'none']
adapter_names = p.ip_adapter_names
else:
if isinstance(adapter_names, str):
adapter_names = [adapter_names]
adapters = [ADAPTERS.get(adapter, None) for adapter in adapter_names]
adapters = [adapter for adapter in adapters if adapter is not None and adapter.lower() != 'none']
adapters = [adapter for adapter in adapters if adapter is not None and adapter['name'].lower() != 'none']
if len(adapters) == 0:
unapply(pipe)
if hasattr(p, 'ip_adapter_images'):
@@ -189,41 +207,48 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt
for adapter_name in adapter_names:
# which clip to use
if 'ViT' not in adapter_name:
clip_repo = base_repo
clip_subfolder = 'models/image_encoder' if shared.sd_model_type == 'sd' else 'sdxl_models/image_encoder' # defaults per model
if 'ViT' not in adapter_name: # defaults per model
if shared.sd_model_type == 'sd':
clip_subfolder = 'models/image_encoder'
else:
clip_subfolder = 'sdxl_models/image_encoder'
elif 'ViT-H' in adapter_name:
clip_repo = base_repo
clip_subfolder = 'models/image_encoder' # this is vit-h
elif 'ViT-G' in adapter_name:
clip_repo = base_repo
clip_subfolder = 'sdxl_models/image_encoder' # this is vit-g
else:
shared.log.error(f'IP adapter: unknown model type: {adapter_name}')
return False
# load feature extractor used by ip adapter
if pipe.feature_extractor is None:
# load feature extractor used by ip adapter
if pipe.feature_extractor is None:
try:
from transformers import CLIPImageProcessor
shared.log.debug('IP adapter load: feature extractor')
pipe.feature_extractor = CLIPImageProcessor()
# load image encoder used by ip adapter
if pipe.image_encoder is None or clip_loaded != f'{clip_repo}/{clip_subfolder}':
try:
from transformers import CLIPVisionModelWithProjection
shared.log.debug(f'IP adapter load: image encoder="{clip_repo}/{clip_subfolder}"')
pipe.image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True)
clip_loaded = f'{clip_repo}/{clip_subfolder}'
except Exception as e:
shared.log.error(f'IP adapter: failed to load image encoder: {e}')
return False
sd_models.move_model(pipe.image_encoder, devices.device)
except Exception as e:
shared.log.error(f'IP adapter load: feature extractor {e}')
return False
# load image encoder used by ip adapter
if pipe.image_encoder is None or clip_loaded != f'{clip_repo}/{clip_subfolder}':
try:
from transformers import CLIPVisionModelWithProjection
shared.log.debug(f'IP adapter load: image encoder="{clip_repo}/{clip_subfolder}"')
pipe.image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True)
clip_loaded = f'{clip_repo}/{clip_subfolder}'
except Exception as e:
shared.log.error(f'IP adapter load: image encoder="{clip_repo}/{clip_subfolder}" {e}')
return False
sd_models.move_model(pipe.image_encoder, devices.device)
# main code
t0 = time.time()
ip_subfolder = 'models' if shared.sd_model_type == 'sd' else 'sdxl_models'
try:
pipe.load_ip_adapter([base_repo], subfolder=[ip_subfolder], weight_name=adapters)
t0 = time.time()
repos = [adapter['repo'] for adapter in adapters]
subfolders = [adapter['subfolder'] for adapter in adapters]
names = [adapter['name'] for adapter in adapters]
pipe.load_ip_adapter(repos, subfolder=subfolders, weight_name=names)
if hasattr(p, 'ip_adapter_layers'):
pipe.set_ip_adapter_scale(p.ip_adapter_layers)
ip_str = ';'.join(adapter_names) + ':' + json.dumps(p.ip_adapter_layers)
@@ -240,5 +265,5 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt
t1 = time.time()
shared.log.info(f'IP adapter: {ip_str} image={adapter_images} mask={adapter_masks is not None} time={t1-t0:.2f}')
except Exception as e:
shared.log.error(f'IP adapter failed to load: repo="{base_repo}" folder="{ip_subfolder}" weights={adapters} names={adapter_names} {e}')
shared.log.error(f'IP adapter load: adapters={adapter_names} repo={repos} folders={subfolders} names={names} {e}')
return True
modules/sd_vae_approx.py
+1 -1
View File
@@ -46,7 +46,7 @@ def nn_approximation(sample): # Approximate NN
sd_vae_approx_model.load_state_dict(approx_weights)
sd_vae_approx_model.eval()
sd_vae_approx_model.to(device, dtype)
shared.log.debug(f'VAE load: type=approximate model={model_path}')
shared.log.debug(f'VAE load: type=approximate model="{model_path}"')
try:
in_sample = sample.to(device, dtype).unsqueeze(0)
sd_vae_approx_model.to(device, dtype)
modules/sd_vae_taesd.py
+3 -3
View File
@@ -160,11 +160,11 @@ def decode(latents):
download_model(model_path)
if os.path.exists(model_path):
taesd_models[f'{model_class}-decoder'] = TAESD(decoder_path=model_path, encoder_path=None)
shared.log.debug(f'VAE load: type=taesd model={model_path}')
shared.log.debug(f'VAE load: type=taesd model="{model_path}"')
vae = taesd_models[f'{model_class}-decoder']
vae.decoder.to(devices.device, dtype)
else:
shared.log.error(f'VAE load: type=taesd model={model_path} not found')
shared.log.error(f'VAE load: type=taesd model="{model_path}" not found')
return latents
if vae is None:
return latents
@@ -208,7 +208,7 @@ def encode(image):
model_path = os.path.join(paths.models_path, "TAESD", f"tae{model_class}_encoder.pth")
download_model(model_path)
if os.path.exists(model_path):
shared.log.debug(f'VAE load: type=taesd model={model_path}')
shared.log.debug(f'VAE load: type=taesd model="{model_path}"')
taesd_models[f'{model_class}-encoder'] = TAESD(encoder_path=model_path, decoder_path=None)
vae = taesd_models[f'{model_class}-encoder']
vae.encoder.to(devices.device, devices.dtype_vae)
modules/ui_common.py
+8 -3
View File
@@ -319,13 +319,18 @@ def create_output_panel(tabname, preview=True, prompt=None, height=None):
return result_gallery, generation_info, html_info, html_info_formatted, html_log
def create_refresh_button(refresh_component, refresh_method, refreshed_args, elem_id, visible: bool = True):
def create_refresh_button(refresh_component, refresh_method, refreshed_args = None, elem_id = None, visible: bool = True):
def refresh():
refresh_method()
args = refreshed_args() if callable(refreshed_args) else refreshed_args
if refreshed_args is None:
args = {"choices": refresh_method()} # pylint: disable=unnecessary-lambda-assignment
elif callable(refreshed_args):
args = refreshed_args()
else:
args = refreshed_args
for k, v in args.items():
setattr(refresh_component, k, v)
return gr.update(**(args or {}))
return gr.update(**args)
refresh_button = ui_components.ToolButton(value=ui_symbols.refresh, elem_id=elem_id, visible=visible)
refresh_button.click(fn=refresh, inputs=[], outputs=[refresh_component])
scripts/ipadapter.py
+7 -4
View File
@@ -1,7 +1,7 @@
import json
from PIL import Image
import gradio as gr
from modules import scripts, processing, shared, ipadapter
from modules import scripts, processing, shared, ipadapter, ui_common
MAX_ADAPTERS = 4
@@ -60,9 +60,12 @@ class Script(scripts.Script):
for i in range(MAX_ADAPTERS):
with gr.Accordion(f'Adapter {i+1}', visible=i==0) as unit:
with gr.Row():
adapters.append(gr.Dropdown(label='Adapter', choices=list(ipadapter.ADAPTERS), value='None'))
scales.append(gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=0.5))
crops.append(gr.Checkbox(label='Crop', default=False, interactive=True))
adapter = gr.Dropdown(label='Adapter', choices=list(ipadapter.get_adapters()), value='None')
adapters.append(adapter)
ui_common.create_refresh_button(adapter, ipadapter.get_adapters)
with gr.Row():
scales.append(gr.Slider(label='Strength', minimum=0.0, maximum=1.0, step=0.01, value=0.5))
crops.append(gr.Checkbox(label='Crop to portrait', default=False, interactive=True))
with gr.Row():
starts.append(gr.Slider(label='Start', minimum=0.0, maximum=1.0, step=0.1, value=0))
ends.append(gr.Slider(label='End', minimum=0.0, maximum=1.0, step=0.1, value=1))