From d5a4f43f437114a008313a823f7e520841f20e77 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 8 Feb 2024 12:10:32 -0500 Subject: [PATCH] post release jumbo update --- CHANGELOG.md | 27 +- TODO.md | 1 + .../stable-diffusion-webui-rembg | 2 +- installer.py | 38 +- launch.py | 1 + modules/control/run.py | 4 +- modules/control/util.py | 2 +- modules/devices.py | 1 + modules/face/__init__.py | 7 +- modules/face/faceid.py | 160 +++++--- modules/face/insightface.py | 7 +- modules/face/instantid.py | 32 +- modules/loader.py | 4 + modules/onnx_impl/__init__.py | 25 +- modules/onnx_impl/execution_providers.py | 46 +-- modules/onnx_impl/ui.py | 377 +++++++++--------- modules/processing_class.py | 3 +- modules/processing_diffusers.py | 9 +- modules/shared.py | 11 +- modules/ui.py | 11 +- wiki | 2 +- 21 files changed, 405 insertions(+), 365 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7d8ff52e..d1cb6e0c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,25 @@ # Change Log for SD.Next +## Update for 2024-02-08 + +TODO: controlnet, adetailer, img2img mask blur and padding + +- **FaceID** now works with multiple input images +- **ONNX**: + - allow specify onnx default provider and cpu fallback + *settings -> diffusers* + - allow manual install of specific onnx flavor + *settings -> onnx* +- **fixes**: + - `ipex` handle dependencies, thanks @Disty0 + - `insightface` handle dependencies + - `img2img` mask blur and padding + ## Update for 2024-02-07 Another big release just hit the shelves! -### Highlights +### Highlights 2024-02-07 - A lot more functionality in the **Control** module: - Inpaint and outpaint support, flexible resizing options, optional hires @@ -36,7 +51,7 @@ Further details: - For more details on all new features see full [CHANGELOG](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) - For documentation, see [WiKi](https://github.com/vladmandic/automatic/wiki) -### Full changelog +### Full changelog 2024-02-07 - Heavily updated [Wiki](https://github.com/vladmandic/automatic/wiki) - **Control**: @@ -239,8 +254,8 @@ Further details: best used together with torch compile: *inductor* this feature is highly experimental and will evolve over time requires nightly versions of `torch` and `torchao` - > pip install -U --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121 - > pip install -U git+https://github.com/pytorch-labs/ao + > `pip install -U --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121` + > `pip install -U git+https://github.com/pytorch-labs/ao` - new option: **compile text encoder** (experimental) - **correction** - new section in generate, allows for image corrections during generataion directly in latent space @@ -336,7 +351,7 @@ Further details: To wrap up this amazing year, were releasing a new version of [SD.Next](https://github.com/vladmandic/automatic), this one is absolutely massive! -### Highlights +### Highlights 2023-12-29 - Brand new Control module for *text, image, batch and video* processing Native implementation of all control methods for both *SD15* and *SD-XL* @@ -360,7 +375,7 @@ And others improvements in areas such as: Upscaling (up to 8x now with 40+ avail Plus some nifty new modules such as **FaceID** automatic face guidance using embeds during generation and **Depth 3D** image to 3D scene -### Full changelog +### Full changelog 2023-12-29 - **Control** - native implementation of all image control methods: diff --git a/TODO.md b/TODO.md index f728924d1..34edd0de5 100644 --- a/TODO.md +++ b/TODO.md @@ -14,3 +14,4 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - preprocess api - remove kohya from submodules - bind panZoom to control input +- deep-cache: diff --git a/extensions-builtin/stable-diffusion-webui-rembg b/extensions-builtin/stable-diffusion-webui-rembg index 4d6b4fd70..7fd9904d9 160000 --- a/extensions-builtin/stable-diffusion-webui-rembg +++ b/extensions-builtin/stable-diffusion-webui-rembg @@ -1 +1 @@ -Subproject commit 4d6b4fd70b00f0ffb4a66a381c85e74e88e04752 +Subproject commit 7fd9904d9b01bdc8e2029f908ca5005aff184f61 diff --git a/installer.py b/installer.py index aabcc6020..280345935 100644 --- a/installer.py +++ b/installer.py @@ -201,10 +201,15 @@ def installed(package, friendly: str = None, reload = False, quiet = False): return False -def uninstall(package): - if installed(package, package, quiet=True): - log.warning(f'Uninstalling: {package}') - pip(f"uninstall {package} --yes --quiet", ignore=True, quiet=True) +def uninstall(package, quiet = False): + packages = package if isinstance(package, list) else [package] + res = '' + for p in packages: + if installed(p, p, quiet=True): + if not quiet: + log.warning(f'Uninstalling: {p}') + res += pip(f"uninstall {p} --yes --quiet", ignore=True, quiet=True) + return res def pip(arg: str, ignore: bool = False, quiet: bool = False): @@ -229,11 +234,15 @@ def pip(arg: str, ignore: bool = False, quiet: bool = False): # install package using pip if not already installed def install(package, friendly: str = None, ignore: bool = False): + res = '' if args.reinstall or args.upgrade: global quick_allowed # pylint: disable=global-statement quick_allowed = False if args.reinstall or not installed(package, friendly): - pip(f"install --upgrade {package}", ignore=ignore) + res = pip(f"install --upgrade {package}", ignore=ignore) + import imp # pylint: disable=deprecated-module + imp.reload(pkg_resources) + return res # execute git command @@ -362,6 +371,12 @@ def check_python(): log.debug(f'Git {git_version.replace("git version", "").strip()}') +# check onnx version +def check_onnx(): + if not installed('onnxruntime', quiet=True) and not installed('onnxruntime-gpu', quiet=True): # allow either + install('onnxruntime', 'onnxruntime', ignore=True) + + # check torch version def check_torch(): if args.skip_torch: @@ -379,8 +394,6 @@ def check_torch(): log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml} openvino={allow_openvino}') torch_command = os.environ.get('TORCH_COMMAND', '') xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') - if not installed('onnxruntime', quiet=True) and not installed('onnxruntime-gpu', quiet=True): # allow either - install('onnxruntime', 'onnxruntime', ignore=True) if torch_command != '': pass elif allow_cuda and (shutil.which('nvidia-smi') is not None or args.use_xformers or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))): @@ -843,20 +856,11 @@ def get_version(): def get_onnxruntime_source_for_rocm(rocm_ver): - ort_version = "1.16.3" - - try: - import onnxruntime - ort_version = onnxruntime.__version__ - except ImportError: - pass - + ort_version = "1.16.3" # hardcoded cp_str = f"{sys.version_info.major}{sys.version_info.minor}" - if rocm_ver is None: command = subprocess.run('hipconfig --version', shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) rocm_ver = command.stdout.decode(encoding="utf8", errors="ignore").split('.') - return f"https://download.onnxruntime.ai/onnxruntime_training-{ort_version}%2Brocm{rocm_ver[0]}{rocm_ver[1]}-cp{cp_str}-cp{cp_str}-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" diff --git a/launch.py b/launch.py index 4617e5f60..7080b0fa6 100755 --- a/launch.py +++ b/launch.py @@ -197,6 +197,7 @@ if __name__ == "__main__": installer.log.info(f'Platform: {installer.print_dict(installer.get_platform())}') installer.set_environment() installer.check_torch() + installer.check_onnx() installer.check_modified_files() if args.reinstall: installer.log.info('Forcing reinstall of all packages') diff --git a/modules/control/run.py b/modules/control/run.py index 65e8074d3..00337a56d 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -371,7 +371,9 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ debug(f'Control processed: {len(processed_images)}') if len(processed_images) > 0: - p.extra_generation_params["Control process"] = [p.processor_id for p in active_process] + p.extra_generation_params["Control process"] = [p.processor_id for p in active_process if p.processor_id is not None] + if len(p.extra_generation_params["Control process"]) == 0: + p.extra_generation_params["Control process"] = None if any(img is None for img in processed_images): msg = 'Control: attempting process but output is none' shared.log.error(f'{msg}: {processed_images}') diff --git a/modules/control/util.py b/modules/control/util.py index f19175c43..8a6a1ac12 100644 --- a/modules/control/util.py +++ b/modules/control/util.py @@ -10,7 +10,7 @@ annotator_ckpts_path = os.path.join(os.path.dirname(__file__), 'ckpts') def dict2str(d: dict): - arr = [f'{name}: {d[name]}' for i, name in enumerate(d)] + arr = [f'{name} {d[name]}' for i, name in enumerate(d) if d[name] is not None and d[name] != ''] return ' | '.join(arr) diff --git a/modules/devices.py b/modules/devices.py index a3463eb1c..d8b7fa268 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -302,6 +302,7 @@ dtype = torch.float16 dtype_vae = torch.float16 dtype_unet = torch.float16 unet_needs_upcast = False +onnx = None if args.profile: log.info(f'Torch build config: {torch.__config__.show()}') # set_cuda_sync_mode('block') # none/auto/spin/yield/block diff --git a/modules/face/__init__.py b/modules/face/__init__.py index 57ba09ecc..e1f4a2583 100644 --- a/modules/face/__init__.py +++ b/modules/face/__init__.py @@ -105,14 +105,13 @@ class Script(scripts.Script): for i, image in enumerate(input_images): if not isinstance(image, Image.Image): input_images[i] = Image.open(image['name']) - source_image = input_images[0] processing.process_init(p) if mode == 'FaceID': # faceid runs as ipadapter in its own pipeline from modules.face.insightface import get_app app = get_app('buffalo_l') from modules.face.faceid import face_id - processed_images = face_id(p, app=app, source_image=source_image, model=ip_model, override=ip_override, cache=ip_cache, scale=ip_strength, structure=ip_structure) # run faceid pipeline + processed_images = face_id(p, app=app, source_images=input_images, model=ip_model, override=ip_override, cache=ip_cache, scale=ip_strength, structure=ip_structure) # run faceid pipeline processed = processing.Processed(p, images_list=processed_images, seed=p.seed, subseed=p.subseed, index_of_first_image=0) # manually created processed object elif mode == 'PhotoMaker': # photomaker creates pipeline and triggers original process_images from modules.face.photomaker import photo_maker @@ -121,7 +120,7 @@ class Script(scripts.Script): from modules.face.insightface import get_app app=get_app('antelopev2') from modules.face.instantid import instant_id # instantid creates pipeline and triggers original process_images - processed = instant_id(p, app=app, source_image=source_image, strength=id_strength, conditioning=id_conditioning, cache=id_cache) + processed = instant_id(p, app=app, source_images=input_images, strength=id_strength, conditioning=id_conditioning, cache=id_cache) if processed is None: # run normal pipeline processed = processing.process_images(p) @@ -134,7 +133,7 @@ class Script(scripts.Script): for i, image in enumerate(processed.images): info = processing.create_infotext(p, index=i) images.save_image(image, path=p.outpath_samples, seed=p.all_seeds[i], prompt=p.all_prompts[i], info=info, p=p, suffix="-before-faceswap") - processed.images = face_swap(p, app=app, input_images=processed.images, source_image=source_image, cache=fs_cache) + processed.images = face_swap(p, app=app, input_images=processed.images, source_image=input_images[0], cache=fs_cache) processed.info = processed.infotext(p, 0) processed.infotexts = [processed.info] diff --git a/modules/face/faceid.py b/modules/face/faceid.py index aff59477a..7dd2396c7 100644 --- a/modules/face/faceid.py +++ b/modules/face/faceid.py @@ -1,3 +1,4 @@ +from typing import List import os import cv2 import torch @@ -9,29 +10,56 @@ from modules import processing, shared, devices FACEID_MODELS = { - 'FaceID Base': 'h94/IP-Adapter-FaceID/ip-adapter-faceid_sd15.bin', - 'FaceID Plus v1': 'h94/IP-Adapter-FaceID/ip-adapter-faceid-plus_sd15.bin', - 'FaceID Plus v2': 'h94/IP-Adapter-FaceID/ip-adapter-faceid-plusv2_sd15.bin', - 'FaceID XL': 'h94/IP-Adapter-FaceID/ip-adapter-faceid_sdxl.bin' + "FaceID Base": "h94/IP-Adapter-FaceID/ip-adapter-faceid_sd15.bin", + "FaceID Plus v1": "h94/IP-Adapter-FaceID/ip-adapter-faceid-plus_sd15.bin", + "FaceID Plus v2": "h94/IP-Adapter-FaceID/ip-adapter-faceid-plusv2_sd15.bin", + "FaceID XL": "h94/IP-Adapter-FaceID/ip-adapter-faceid_sdxl.bin", + # "FaceID Portrait v10": "h94/IP-Adapter-FaceID/ip-adapter-faceid-portrait_sd15.bin", + # "FaceID Portrait v11": "h94/IP-Adapter-FaceID/ip-adapter-faceid-portrait-v11_sd15.bin", + # "FaceID XL Plus v2": "h94/IP-Adapter-FaceID/ip-adapter-faceid_sdxl.bin", } faceid_model = None faceid_model_name = None -debug = shared.log.trace if os.environ.get('SD_FACE_DEBUG', None) is not None else lambda *args, **kwargs: None +debug = shared.log.trace if os.environ.get("SD_FACE_DEBUG", None) is not None else lambda *args, **kwargs: None -def face_id(p: processing.StableDiffusionProcessing, app, source_image: Image.Image, model: str, override: bool, cache: bool, scale: float, structure: float): - global faceid_model, faceid_model_name # pylint: disable=global-statement +def face_id( + p: processing.StableDiffusionProcessing, + app, + source_images: List[Image.Image], + model: str, + override: bool, + cache: bool, + scale: float, + structure: float, +): + global faceid_model, faceid_model_name # pylint: disable=global-statement + if source_images is None or len(source_images) == 0: + shared.log.warning('FaceID: no input images') + return None + from insightface.utils import face_align - from ip_adapter.ip_adapter_faceid import IPAdapterFaceID, IPAdapterFaceIDPlus, IPAdapterFaceIDXL + try: + from ip_adapter.ip_adapter_faceid import ( + IPAdapterFaceID, + IPAdapterFaceIDPlus, + IPAdapterFaceIDXL, + IPAdapterFaceIDPlusXL, + ) + from ip_adapter.ip_adapter_faceid_separate import ( + IPAdapterFaceID as IPAdapterFaceIDPortrait, + ) + except Exception as e: + shared.log.error(f"FaceID incorrect version of ip_adapter: {e}") + return None ip_ckpt = FACEID_MODELS[model] folder, filename = os.path.split(ip_ckpt) basename, _ext = os.path.splitext(filename) model_path = hf.hf_hub_download(repo_id=folder, filename=filename, cache_dir=shared.opts.diffusers_dir) if model_path is None: - shared.log.error(f'FaceID download failed: model={model} file={ip_ckpt}') + shared.log.error(f"FaceID download failed: model={model} file={ip_ckpt}") return None - if override: shared.sd_model.scheduler = diffusers.DDIMScheduler( num_train_timesteps=1000, @@ -44,72 +72,106 @@ def face_id(p: processing.StableDiffusionProcessing, app, source_image: Image.Im ) shortcut = None if faceid_model is None or faceid_model_name != model or not cache: - shared.log.debug(f'FaceID load: model={model} file={ip_ckpt}') - if 'Plus' in model: + shared.log.debug(f"FaceID load: model={model} file={ip_ckpt}") + if "XL Plus" in model: + image_encoder_path = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K" + faceid_model = IPAdapterFaceIDPlusXL( + sd_pipe=shared.sd_model, + image_encoder_path=image_encoder_path, + ip_ckpt=model_path, + lora_rank=128, + num_tokens=4, + device=devices.device, + torch_dtype=devices.dtype, + ) + elif "XL" in model: + faceid_model = IPAdapterFaceIDXL( + sd_pipe=shared.sd_model, + ip_ckpt=model_path, + lora_rank=128, + num_tokens=4, + device=devices.device, + torch_dtype=devices.dtype, + ) + elif "Plus" in model: image_encoder_path = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K" faceid_model = IPAdapterFaceIDPlus( sd_pipe=shared.sd_model, image_encoder_path=image_encoder_path, ip_ckpt=model_path, - lora_rank=128, num_tokens=4, device=devices.device, torch_dtype=devices.dtype, + lora_rank=128, + num_tokens=4, + device=devices.device, + torch_dtype=devices.dtype, ) - shortcut = 'v2' in model - elif 'XL' in model: - faceid_model = IPAdapterFaceIDXL( + elif "Portrait" in model: + faceid_model = IPAdapterFaceIDPortrait( sd_pipe=shared.sd_model, ip_ckpt=model_path, - lora_rank=128, num_tokens=4, device=devices.device, torch_dtype=devices.dtype, + num_tokens=16, + n_cond=5, + device=devices.device, + torch_dtype=devices.dtype, ) else: faceid_model = IPAdapterFaceID( sd_pipe=shared.sd_model, ip_ckpt=model_path, - lora_rank=128, num_tokens=4, device=devices.device, torch_dtype=devices.dtype, + lora_rank=128, + num_tokens=4, + device=devices.device, + torch_dtype=devices.dtype, ) + shortcut = "v2" in model faceid_model_name = model else: - shared.log.debug(f'FaceID cached: model={model} file={ip_ckpt}') + shared.log.debug(f"FaceID cached: model={model} file={ip_ckpt}") processed_images = [] - np_image = cv2.cvtColor(np.array(source_image), cv2.COLOR_RGB2BGR) - faces = app.get(np_image) - if len(faces) == 0: - shared.log.error('FaceID: no faces found') + face_embeds = [] + face_images = [] + for i, source_image in enumerate(source_images): + np_image = cv2.cvtColor(np.array(source_image), cv2.COLOR_RGB2BGR) + faces = app.get(np_image) + if len(faces) == 0: + shared.log.error("FaceID: no faces found") + break + face_embeds.append(torch.from_numpy(faces[0].normed_embedding).unsqueeze(0)) + face_images.append(face_align.norm_crop(np_image, landmark=faces[0].kps, image_size=224)) + shared.log.debug(f'FaceID face: i={i+1} score={faces[0].det_score:.2f} gender={"female" if faces[0].gender==0 else "male"} age={faces[0].age} bbox={faces[0].bbox}') + p.extra_generation_params[f"FaceID {i+1}"] = f'{faces[0].det_score:.2f} {"female" if faces[0].gender==0 else "male"} {faces[0].age}y' + if len(face_embeds) == 0: + shared.log.error("FaceID: no faces found") return None - face_embeds = torch.from_numpy(faces[0].normed_embedding).unsqueeze(0) - face_image = face_align.norm_crop(np_image, landmark=faces[0].kps, image_size=224) # you can also segment the face + face_embeds = torch.cat(face_embeds, dim=0) - for i, face in enumerate(faces): - shared.log.debug(f'FaceID face: i={i+1} score={face.det_score:.2f} gender={"female" if face.gender==0 else "male"} age={face.age} bbox={face.bbox}') - p.extra_generation_params[f"FaceID {i+1}"] = f'{face.det_score:.2f} {"female" if face.gender==0 else "male"} {face.age}y' - ip_model_dict = { # main generate dict - 'num_samples': p.batch_size, - 'width': p.width, - 'height': p.height, - 'num_inference_steps': p.steps, - 'scale': scale, - 'guidance_scale': p.cfg_scale, - 'faceid_embeds': face_embeds.shape, + ip_model_dict = { # main generate dict + "num_samples": p.batch_size, + "width": p.width, + "height": p.height, + "num_inference_steps": p.steps, + "scale": scale, + "guidance_scale": p.cfg_scale, + "faceid_embeds": face_embeds.shape, # placeholder } # optional generate dict if shortcut is not None: - ip_model_dict['shortcut'] = shortcut - if 'Plus' in model: - ip_model_dict['s_scale'] = structure - ip_model_dict['face_image'] = face_image.shape - shared.log.debug(f'FaceID args: {ip_model_dict}') - if 'Plus' in model: - ip_model_dict['face_image'] = face_image - ip_model_dict['faceid_embeds'] = face_embeds + ip_model_dict["shortcut"] = shortcut + if "Plus" in model: + ip_model_dict["s_scale"] = structure + shared.log.debug(f"FaceID args: {ip_model_dict}") + if "Plus" in model: + ip_model_dict["face_image"] = face_images + ip_model_dict["faceid_embeds"] = face_embeds # overwrite placeholder # run generate faceid_model.set_scale(scale) for i in range(p.n_iter): ip_model_dict.update({ - 'prompt': p.all_prompts[i], - 'negative_prompt': p.all_negative_prompts[i], - 'seed': int(p.all_seeds[i]), + "prompt": p.all_prompts[i], + "negative_prompt": p.all_negative_prompts[i], + "seed": int(p.all_seeds[i]), }) - debug(f'FaceID: {ip_model_dict}') + debug(f"FaceID: {ip_model_dict}") res = faceid_model.generate(**ip_model_dict) if isinstance(res, list): processed_images += res @@ -120,5 +182,5 @@ def face_id(p: processing.StableDiffusionProcessing, app, source_image: Image.Im faceid_model_name = None devices.torch_gc() - p.extra_generation_params["IP Adapter"] = f'{basename}:{scale}' + p.extra_generation_params["IP Adapter"] = f"{basename}:{scale}" return processed_images diff --git a/modules/face/insightface.py b/modules/face/insightface.py index 5bf1cdcd8..5a5978fe2 100644 --- a/modules/face/insightface.py +++ b/modules/face/insightface.py @@ -1,5 +1,6 @@ import os from modules.shared import log, opts +from modules import devices insightface_app = None @@ -17,11 +18,10 @@ def get_app(mp_name): install(pkg[0], pkg[1], ignore=True) global insightface_app, instightface_mp # pylint: disable=global-statement if insightface_app is None or mp_name != instightface_mp: - import onnxruntime from insightface.app import FaceAnalysis import huggingface_hub as hf import zipfile - log.debug(f"InsightFace: mp={mp_name} device={onnxruntime.get_device()} providers={onnxruntime.get_available_providers()}") + log.debug(f"InsightFace: mp={mp_name} provider={devices.onnx}") root_dir = os.path.join(opts.diffusers_dir, 'models--vladmandic--insightface-faceanalysis') local_dir = os.path.join(root_dir, 'models') extract_dir = os.path.join(local_dir, mp_name) @@ -44,8 +44,7 @@ def get_app(mp_name): 'download': False, 'download_zip': False, } - insightface_app = FaceAnalysis(name=mp_name, providers=['CUDAExecutionProvider', 'CPUExecutionProvider'], **kwargs) + insightface_app = FaceAnalysis(name=mp_name, providers=devices.onnx, **kwargs) instightface_mp = mp_name - onnxruntime.set_default_logger_severity(3) insightface_app.prepare(ctx_id=0, det_thresh=0.5, det_size=(640, 640)) return insightface_app diff --git a/modules/face/instantid.py b/modules/face/instantid.py index 4cefbc62d..74141da3d 100644 --- a/modules/face/instantid.py +++ b/modules/face/instantid.py @@ -1,5 +1,6 @@ import os import cv2 +import torch import numpy as np import huggingface_hub as hf from modules import shared, processing, sd_models, devices @@ -10,13 +11,13 @@ controlnet_model = None debug = shared.log.trace if os.environ.get('SD_FACE_DEBUG', None) is not None else lambda *args, **kwargs: None -def instant_id(p: processing.StableDiffusionProcessing, app, source_image, strength=1.0, conditioning=0.5, cache=True): # pylint: disable=arguments-differ +def instant_id(p: processing.StableDiffusionProcessing, app, source_images, strength=1.0, conditioning=0.5, cache=True): # pylint: disable=arguments-differ from modules.face.instantid_model import StableDiffusionXLInstantIDPipeline, draw_kps from diffusers.models import ControlNetModel global controlnet_model # pylint: disable=global-statement # prepare pipeline - if source_image is None: + if source_images is None or len(source_images) == 0: shared.log.warning('InstantID: no input images') return None @@ -26,12 +27,16 @@ def instant_id(p: processing.StableDiffusionProcessing, app, source_image, stren return None # prepare face emb - faces = app.get(cv2.cvtColor(np.array(source_image), cv2.COLOR_RGB2BGR)) - face = sorted(faces, key=lambda x:(x['bbox'][2]-x['bbox'][0])*x['bbox'][3]-x['bbox'][1])[-1] # only use the maximum face - face_emb = face['embedding'] - face_kps = draw_kps(source_image, face['kps']) + face_embeds = [] + face_images = [] + for i, source_image in enumerate(source_images): + faces = app.get(cv2.cvtColor(np.array(source_image), cv2.COLOR_RGB2BGR)) + face = sorted(faces, key=lambda x:(x['bbox'][2]-x['bbox'][0])*x['bbox'][3]-x['bbox'][1])[-1] # only use the maximum face + face_embeds.append(torch.from_numpy(face['embedding'])) + face_images.append(draw_kps(source_image, face['kps'])) + p.extra_generation_params[f"InstantID {i+1}"] = f'{faces[0].det_score:.2f} {"female" if faces[0].gender==0 else "male"} {faces[0].age}y' + shared.log.debug(f'InstantID face: score={face.det_score:.2f} gender={"female" if face.gender==0 else "male"} age={face.age} bbox={face.bbox}') - shared.log.debug(f'InstantID face: score={face.det_score:.2f} gender={"female" if face.gender==0 else "male"} age={face.age} bbox={face.bbox}') shared.log.debug(f'InstantID loading: model={REPO_ID}') face_adapter = hf.hf_hub_download(repo_id=REPO_ID, filename="ip-adapter.bin") if controlnet_model is None or not cache: @@ -63,20 +68,19 @@ def instant_id(p: processing.StableDiffusionProcessing, app, source_image, stren # pipeline specific args orig_prompt_attention = shared.opts.prompt_attention shared.opts.data['prompt_attention'] = 'Fixed attention' # otherwise need to deal with class_tokens_mask - p.task_args['prompt'] = p.all_prompts[0] # override all logic - p.task_args['negative_prompt'] = p.all_negative_prompts[0] - p.task_args['image_embeds'] = face_emb - p.task_args['image'] = face_kps + p.task_args['image_embeds'] = face_embeds[0].shape # placeholder + p.task_args['image'] = face_images[0] p.task_args['controlnet_conditioning_scale'] = float(conditioning) p.task_args['ip_adapter_scale'] = float(strength) - debug(f'InstantID: args={p.task_args}') + shared.log.debug(f"InstantID args: {p.task_args}") + p.task_args['prompt'] = p.all_prompts[0] # override all logic + p.task_args['negative_prompt'] = p.all_negative_prompts[0] + p.task_args['image_embeds'] = face_embeds[0] # overwrite placeholder # run processing - shared.log.debug(f'InstantID: strength={strength} conditioning={conditioning} image={source_image}') processed: processing.Processed = processing.process_images(p) shared.sd_model.set_ip_adapter_scale(0) p.extra_generation_params['InstantID'] = f'{strength}/{conditioning}' - p.extra_generation_params["Face"] = f'{face.det_score:.2f} {"female" if face.gender==0 else "male"} {face.age}y' if not cache: controlnet_model = None diff --git a/modules/loader.py b/modules/loader.py index 2b502c9af..e96c194de 100644 --- a/modules/loader.py +++ b/modules/loader.py @@ -37,6 +37,10 @@ if ".dev" in torch.__version__ or "+git" in torch.__version__: torch.__version__ = re.search(r'[\d.]+[\d]', torch.__version__).group(0) timer.startup.record("torch") +import onnxruntime +onnxruntime.set_default_logger_severity(3) +timer.startup.record("onnx") + from modules.onnx_impl import initialize_olive # pylint: disable=ungrouped-imports initialize_olive() timer.startup.record("olive") diff --git a/modules/onnx_impl/__init__.py b/modules/onnx_impl/__init__.py index abdea8e59..d1785da9b 100644 --- a/modules/onnx_impl/__init__.py +++ b/modules/onnx_impl/__init__.py @@ -16,7 +16,6 @@ class DynamicSessionOptions(ort.SessionOptions): def __init__(self): super().__init__() - self.enable_mem_pattern = False @classmethod @@ -77,7 +76,6 @@ class TemporalModule(TorchCompatibleModule): device = extract_device(args, kwargs) if device is not None and device.type != "cpu": from .execution_providers import TORCH_DEVICE_TO_EP - provider = TORCH_DEVICE_TO_EP[device.type] if device.type in TORCH_DEVICE_TO_EP else self.provider return OnnxRuntimeModel.load_model(self.path, provider, DynamicSessionOptions.from_sess_options(self.sess_options)) return self @@ -100,10 +98,7 @@ class OnnxRuntimeModel(TorchCompatibleModule, diffusers.OnnxRuntimeModel): class VAEConfig: - DEFAULTS = { - "scaling_factor": 0.18215, - } - + DEFAULTS = { "scaling_factor": 0.18215 } config: Dict def __init__(self, config: Dict): @@ -151,10 +146,8 @@ class VAE(TorchCompatibleModule): def check_parameters_changed(p, refiner_enabled: bool): from modules import shared, sd_models - if shared.sd_model.__class__.__name__ == "OnnxRawPipeline" or not shared.sd_model.__class__.__name__.startswith("Onnx"): return shared.sd_model - compile_height = p.height compile_width = p.width if (shared.compiled_model_state is None or @@ -172,17 +165,14 @@ def check_parameters_changed(p, refiner_enabled: bool): shared.compiled_model_state.height = compile_height shared.compiled_model_state.width = compile_width shared.compiled_model_state.batch_size = p.batch_size - return shared.sd_model def preprocess_pipeline(p): from modules import shared, sd_models - if "ONNX" not in shared.opts.diffusers_pipeline: shared.log.warning(f"Unsupported pipeline for 'olive-ai' compile backend: {shared.opts.diffusers_pipeline}. You should select one of the ONNX pipelines.") return shared.sd_model - if hasattr(shared.sd_model, "preprocess"): shared.sd_model = shared.sd_model.preprocess(p) if hasattr(shared.sd_refiner, "preprocess"): @@ -192,7 +182,6 @@ def preprocess_pipeline(p): if shared.opts.onnx_unload_base: sd_models.reload_model_weights(op='model') shared.sd_model = shared.sd_model.preprocess(p) - return shared.sd_model @@ -201,25 +190,20 @@ def ORTDiffusionModelPart_to(self, *args, **kwargs): return self -def initialize(): +def initialize_onnx(): global initialized # pylint: disable=global-statement - if initialized: return - from installer import log from modules import devices from modules.paths import models_path from modules.shared import opts from .execution_providers import ExecutionProvider, TORCH_DEVICE_TO_EP, available_execution_providers - onnx_dir = os.path.join(models_path, "ONNX") if not os.path.isdir(onnx_dir): os.mkdir(onnx_dir) - if devices.backend == "rocm": TORCH_DEVICE_TO_EP["cuda"] = ExecutionProvider.ROCm - from .pipelines.onnx_stable_diffusion_pipeline import OnnxStableDiffusionPipeline from .pipelines.onnx_stable_diffusion_img2img_pipeline import OnnxStableDiffusionImg2ImgPipeline from .pipelines.onnx_stable_diffusion_inpaint_pipeline import OnnxStableDiffusionInpaintPipeline @@ -254,8 +238,7 @@ def initialize(): optimum.onnxruntime.modeling_diffusion._ORTDiffusionModelPart.to = ORTDiffusionModelPart_to # pylint: disable=protected-access - log.info(f'ONNX: selected={opts.onnx_execution_provider}, available={available_execution_providers}') - + log.debug(f'ONNX: version={ort.__version__} provider={opts.onnx_execution_provider}, available={available_execution_providers}') initialized = True @@ -283,10 +266,8 @@ def initialize_olive(): def install_olive(): from installer import installed, install, log - if installed("olive-ai"): return - try: log.info('Installing Olive') install('olive-ai', 'olive-ai', ignore=True) diff --git a/modules/onnx_impl/execution_providers.py b/modules/onnx_impl/execution_providers.py index 0ae94b67b..954fe09bc 100644 --- a/modules/onnx_impl/execution_providers.py +++ b/modules/onnx_impl/execution_providers.py @@ -33,7 +33,6 @@ TORCH_DEVICE_TO_EP = { def get_default_execution_provider() -> ExecutionProvider: from modules import devices - if devices.backend == "cpu": return ExecutionProvider.CPU elif devices.backend == "directml": @@ -41,10 +40,7 @@ def get_default_execution_provider() -> ExecutionProvider: elif devices.backend == "cuda": return ExecutionProvider.CUDA elif devices.backend == "rocm": - if ExecutionProvider.ROCm in available_execution_providers: - return ExecutionProvider.ROCm - else: - return ExecutionProvider.CPU + return ExecutionProvider.ROCm elif devices.backend == "ipex" or devices.backend == "openvino": return ExecutionProvider.OpenVINO return ExecutionProvider.CPU @@ -52,11 +48,7 @@ def get_default_execution_provider() -> ExecutionProvider: def get_execution_provider_options(): from modules.shared import cmd_opts, opts - - execution_provider_options = { - "device_id": int(cmd_opts.device_id or 0), - } - + execution_provider_options = { "device_id": int(cmd_opts.device_id or 0) } if opts.onnx_execution_provider == ExecutionProvider.ROCm: if ExecutionProvider.ROCm in available_execution_providers: execution_provider_options["tunable_op_enable"] = 1 @@ -68,32 +60,21 @@ def get_execution_provider_options(): raw_openvino_device = f"{raw_openvino_device}_FP16" execution_provider_options["device_type"] = raw_openvino_device del execution_provider_options["device_id"] - return execution_provider_options def get_provider() -> Tuple: from modules.shared import opts - return (opts.onnx_execution_provider, get_execution_provider_options(),) def install_execution_provider(ep: ExecutionProvider): - from installer import pip, uninstall, installed, get_onnxruntime_source_for_rocm - - if installed("onnxruntime"): - uninstall("onnxruntime") - if installed("onnxruntime-directml"): - uninstall("onnxruntime-directml") - if installed("onnxruntime-gpu"): - uninstall("onnxruntime-gpu") - if installed("onnxruntime-training"): - uninstall("onnxruntime-training") - if installed("onnxruntime-openvino"): - uninstall("onnxruntime-openvino") - + import imp # pylint: disable=deprecated-module + from installer import installed, install, uninstall, get_onnxruntime_source_for_rocm + res = "
"
+    res += uninstall(["onnxruntime", "onnxruntime-directml", "onnxruntime-gpu", "onnxruntime-training", "onnxruntime-openvino"], quiet=True)
+    installed("onnxruntime", reload=True)
     packages = ["onnxruntime"] # Failed to load olive: cannot import name '__version__' from 'onnxruntime'
-
     if ep == ExecutionProvider.DirectML:
         packages.append("onnxruntime-directml")
     elif ep == ExecutionProvider.CUDA:
@@ -102,13 +83,14 @@ def install_execution_provider(ep: ExecutionProvider):
         if "linux" not in sys.platform:
             log.warning("ROCMExecutionProvider is not supported on Windows.")
             return
-
         packages.append(get_onnxruntime_source_for_rocm(None))
     elif ep == ExecutionProvider.OpenVINO:
-        if installed("openvino"):
-            uninstall("openvino")
         packages.append("openvino")
         packages.append("onnxruntime-openvino")
-
-    pip(f"install --upgrade {' '.join(packages)}")
-    log.info("Please restart SD.Next.")
+    for package in packages:
+        res += install(package)
+    res += '

' + res += 'Server restart required' + log.info("Server restart required") + imp.reload(ort) + return res diff --git a/modules/onnx_impl/ui.py b/modules/onnx_impl/ui.py index bcddf1b80..19b3add23 100644 --- a/modules/onnx_impl/ui.py +++ b/modules/onnx_impl/ui.py @@ -21,246 +21,227 @@ def create_ui(): from .utils import check_diffusers_cache with gr.Blocks(analytics_enabled=False) as ui: - with gr.Row(): - with gr.Tabs(elem_id="tabs_onnx"): - with gr.TabItem("Manage execution providers", id="onnxep"): - gr.Markdown("Uninstall existing execution provider and install another one.") - - choices = [] - - for ep in ExecutionProvider: - choices.append(ep) - - ep_default = None - if cmd_opts.use_directml: - ep_default = ExecutionProvider.DirectML - elif cmd_opts.use_cuda: - ep_default = ExecutionProvider.CUDA - elif cmd_opts.use_rocm: - ep_default = ExecutionProvider.ROCm - elif cmd_opts.use_openvino: - ep_default = ExecutionProvider.OpenVINO - - ep_checkbox = gr.Radio(label="Execution provider", value=ep_default, choices=choices) - ep_install = gr.Button(value="Install") - gr.Markdown("**Warning! If you are trying to reinstall, it may not work due to permission issue.**") - - ep_install.click(fn=install_execution_provider, inputs=ep_checkbox) + with gr.Tabs(elem_id="tabs_onnx"): + with gr.TabItem("Provider", id="onnxep"): + gr.Markdown("Install ONNX execution provider") + ep_default = None + if cmd_opts.use_directml: + ep_default = ExecutionProvider.DirectML + elif cmd_opts.use_cuda: + ep_default = ExecutionProvider.CUDA + elif cmd_opts.use_rocm: + ep_default = ExecutionProvider.ROCm + elif cmd_opts.use_openvino: + ep_default = ExecutionProvider.OpenVINO + ep_checkbox = gr.Radio(label="Execution provider", value=ep_default, choices=ExecutionProvider) + ep_install = gr.Button(value="Reinstall") + ep_log = gr.HTML("") + ep_install.click(fn=install_execution_provider, inputs=[ep_checkbox], outputs=[ep_log]) if opts.cuda_compile_backend == "olive-ai": import olive.passes as olive_passes from olive.hardware.accelerator import AcceleratorSpec, Device - accelerator = AcceleratorSpec(accelerator_type=Device.GPU, execution_provider=opts.onnx_execution_provider) - with gr.Tabs(elem_id="tabs_olive"): - with gr.TabItem("Manage cache", id="manage_cache"): - cache_state_dirname = gr.Textbox(value=None, visible=False) - - with gr.Row(): - model_dropdown = gr.Dropdown(label="Model", value="Please select model", choices=checkpoint_tiles()) - create_refresh_button(model_dropdown, refresh_checkpoints, {}, "onnx_cache_refresh_diffusers_model") - - with gr.Row(): - def remove_cache_onnx_converted(dirname: str): - shutil.rmtree(os.path.join(opts.onnx_cached_models_path, dirname)) - log.info(f"ONNX converted cache of '{dirname}' is removed.") - - cache_onnx_converted = gr.Markdown("Please select model") - cache_remove_onnx_converted = gr.Button(value="Remove cache", visible=False) - cache_remove_onnx_converted.click(fn=remove_cache_onnx_converted, inputs=[cache_state_dirname,]) - - with gr.Column(): - cache_optimized_selected = gr.Textbox(value=None, visible=False) - - def select_cache_optimized(evt: gr.SelectData, data): - return ",".join(data[evt.index[0]]) - - def remove_cache_optimized(dirname: str, s: str): - if s == "": - return - size = s.split(",") - shutil.rmtree(os.path.join(opts.onnx_cached_models_path, f"{dirname}-{size[0]}w-{size[1]}h")) - log.info(f"Olive processed cache of '{dirname}' is removed: width={size[0]}, height={size[1]}") - - with gr.Row(): - cache_list_optimized_headers = ["height", "width"] - cache_list_optimized_types = ["str", "str"] - cache_list_optimized = gr.Dataframe(None, label="Optimized caches", show_label=True, overflow_row_behaviour='paginate', interactive=False, max_rows=10, headers=cache_list_optimized_headers, datatype=cache_list_optimized_types, type="array") - cache_list_optimized.select(fn=select_cache_optimized, inputs=[cache_list_optimized,], outputs=[cache_optimized_selected,]) - - cache_remove_optimized = gr.Button(value="Remove selected cache", visible=False) - cache_remove_optimized.click(fn=remove_cache_optimized, inputs=[cache_state_dirname, cache_optimized_selected,]) - - def cache_update_menus(query: str): - checkpoint_info = get_closet_checkpoint_match(query) - if checkpoint_info is None: - log.error(f"Could not find checkpoint object for '{query}'.") + with gr.TabItem("Manage cache", id="manage_cache"): + cache_state_dirname = gr.Textbox(value=None, visible=False) + with gr.Row(): + model_dropdown = gr.Dropdown(label="Model", value="Please select model", choices=checkpoint_tiles()) + create_refresh_button(model_dropdown, refresh_checkpoints, {}, "onnx_cache_refresh_diffusers_model") + with gr.Row(): + def remove_cache_onnx_converted(dirname: str): + shutil.rmtree(os.path.join(opts.onnx_cached_models_path, dirname)) + log.info(f"ONNX converted cache of '{dirname}' is removed.") + cache_onnx_converted = gr.Markdown("Please select model") + cache_remove_onnx_converted = gr.Button(value="Remove cache", visible=False) + cache_remove_onnx_converted.click(fn=remove_cache_onnx_converted, inputs=[cache_state_dirname,]) + with gr.Column(): + cache_optimized_selected = gr.Textbox(value=None, visible=False) + def select_cache_optimized(evt: gr.SelectData, data): + return ",".join(data[evt.index[0]]) + def remove_cache_optimized(dirname: str, s: str): + if s == "": return - model_name = os.path.basename(os.path.dirname(os.path.dirname(checkpoint_info.path)) if check_diffusers_cache(checkpoint_info.path) else checkpoint_info.path) - caches = os.listdir(opts.onnx_cached_models_path) - onnx_converted = False - optimized_sizes = [] - for cache in caches: - if cache == model_name: - onnx_converted = True - elif model_name in cache: - try: - splitted = cache.split("-") - height = splitted[-1][:-1] - width = splitted[-2][:-1] - optimized_sizes.append((width, height,)) - except Exception: - pass - return ( - model_name, - cache_onnx_converted.update(value="ONNX model cache of this model exists." if onnx_converted else "ONNX model cache of this model does not exist."), - cache_remove_onnx_converted.update(visible=onnx_converted), - None if len(optimized_sizes) == 0 else optimized_sizes, - cache_remove_optimized.update(visible=True), - ) + size = s.split(",") + shutil.rmtree(os.path.join(opts.onnx_cached_models_path, f"{dirname}-{size[0]}w-{size[1]}h")) + log.info(f"Olive processed cache of '{dirname}' is removed: width={size[0]}, height={size[1]}") + with gr.Row(): + cache_list_optimized_headers = ["height", "width"] + cache_list_optimized_types = ["str", "str"] + cache_list_optimized = gr.Dataframe(None, label="Optimized caches", show_label=True, overflow_row_behaviour='paginate', interactive=False, max_rows=10, headers=cache_list_optimized_headers, datatype=cache_list_optimized_types, type="array") + cache_list_optimized.select(fn=select_cache_optimized, inputs=[cache_list_optimized,], outputs=[cache_optimized_selected,]) + cache_remove_optimized = gr.Button(value="Remove selected cache", visible=False) + cache_remove_optimized.click(fn=remove_cache_optimized, inputs=[cache_state_dirname, cache_optimized_selected,]) - model_dropdown.change(fn=cache_update_menus, inputs=[model_dropdown,], outputs=[ - cache_state_dirname, - cache_onnx_converted, cache_remove_onnx_converted, - cache_list_optimized, cache_remove_optimized, - ]) + def cache_update_menus(query: str): + checkpoint_info = get_closet_checkpoint_match(query) + if checkpoint_info is None: + log.error(f"Could not find checkpoint object for '{query}'.") + return + model_name = os.path.basename(os.path.dirname(os.path.dirname(checkpoint_info.path)) if check_diffusers_cache(checkpoint_info.path) else checkpoint_info.path) + caches = os.listdir(opts.onnx_cached_models_path) + onnx_converted = False + optimized_sizes = [] + for cache in caches: + if cache == model_name: + onnx_converted = True + elif model_name in cache: + try: + splitted = cache.split("-") + height = splitted[-1][:-1] + width = splitted[-2][:-1] + optimized_sizes.append((width, height,)) + except Exception: + pass + return ( + model_name, + cache_onnx_converted.update(value="ONNX model cache of this model exists." if onnx_converted else "ONNX model cache of this model does not exist."), + cache_remove_onnx_converted.update(visible=onnx_converted), + None if len(optimized_sizes) == 0 else optimized_sizes, + cache_remove_optimized.update(visible=True), + ) - with gr.TabItem("Customize pass flow", id="pass_flow"): - with gr.Tabs(elem_id="tabs_model_type"): - with gr.TabItem("Stable Diffusion", id="sd"): - sd_config_path = os.path.join(sd_configs_path, "olive", "sd") - sd_submodels = os.listdir(sd_config_path) - sd_configs: Dict[str, Dict[str, Dict[str, Dict]]] = {} - sd_pass_config_components: Dict[str, Dict[str, Dict]] = {} + model_dropdown.change(fn=cache_update_menus, inputs=[model_dropdown,], outputs=[ + cache_state_dirname, + cache_onnx_converted, cache_remove_onnx_converted, + cache_list_optimized, cache_remove_optimized, + ]) - with gr.Tabs(elem_id="tabs_sd_submodel"): - def sd_create_change_listener(*args): - def listener(v: Dict): - get_recursively(sd_configs, *args[:-1])[args[-1]] = v - return listener + with gr.TabItem("Customize pass flow", id="pass_flow"): + with gr.Tabs(elem_id="tabs_model_type"): + with gr.TabItem("Stable Diffusion", id="sd"): + sd_config_path = os.path.join(sd_configs_path, "olive", "sd") + sd_submodels = os.listdir(sd_config_path) + sd_configs: Dict[str, Dict[str, Dict[str, Dict]]] = {} + sd_pass_config_components: Dict[str, Dict[str, Dict]] = {} - for submodel in sd_submodels: - config: Dict = None + with gr.Tabs(elem_id="tabs_sd_submodel"): + def sd_create_change_listener(*args): + def listener(v: Dict): + get_recursively(sd_configs, *args[:-1])[args[-1]] = v + return listener - sd_pass_config_components[submodel] = {} + for submodel in sd_submodels: + config: Dict = None - with open(os.path.join(sd_config_path, submodel), "r", encoding="utf-8") as file: - config = json.load(file) - sd_configs[submodel] = config + sd_pass_config_components[submodel] = {} - submodel_name = submodel[:-5] - with gr.TabItem(submodel_name, id=f"sd_{submodel_name}"): - pass_flows = DropdownMulti(label="Pass flow", value=sd_configs[submodel]["pass_flows"][0], choices=sd_configs[submodel]["passes"].keys()) - pass_flows.change(fn=sd_create_change_listener(submodel, "pass_flows", 0), inputs=pass_flows) + with open(os.path.join(sd_config_path, submodel), "r", encoding="utf-8") as file: + config = json.load(file) + sd_configs[submodel] = config - with gr.Tabs(elem_id=f"tabs_sd_{submodel_name}_pass"): - for pass_name in sd_configs[submodel]["passes"]: - sd_pass_config_components[submodel][pass_name] = {} + submodel_name = submodel[:-5] + with gr.TabItem(submodel_name, id=f"sd_{submodel_name}"): + pass_flows = DropdownMulti(label="Pass flow", value=sd_configs[submodel]["pass_flows"][0], choices=sd_configs[submodel]["passes"].keys()) + pass_flows.change(fn=sd_create_change_listener(submodel, "pass_flows", 0), inputs=pass_flows) - with gr.TabItem(pass_name, id=f"sd_{submodel_name}_pass_{pass_name}"): - config_dict = sd_configs[submodel]["passes"][pass_name] + with gr.Tabs(elem_id=f"tabs_sd_{submodel_name}_pass"): + for pass_name in sd_configs[submodel]["passes"]: + sd_pass_config_components[submodel][pass_name] = {} - pass_type = gr.Dropdown(label="Type", value=config_dict["type"], choices=(x.__name__ for x in tuple(olive_passes.REGISTRY.values()))) + with gr.TabItem(pass_name, id=f"sd_{submodel_name}_pass_{pass_name}"): + config_dict = sd_configs[submodel]["passes"][pass_name] + + pass_type = gr.Dropdown(label="Type", value=config_dict["type"], choices=(x.__name__ for x in tuple(olive_passes.REGISTRY.values()))) - def create_pass_config_change_listener(submodel, pass_name, config_key): - def listener(value): - sd_configs[submodel]["passes"][pass_name]["config"][config_key] = value - return listener + def create_pass_config_change_listener(submodel, pass_name, config_key): + def listener(value): + sd_configs[submodel]["passes"][pass_name]["config"][config_key] = value + return listener - for config_key, v in getattr(olive_passes, config_dict["type"], olive_passes.Pass)._default_config(accelerator).items(): # pylint: disable=protected-access - component = None + for config_key, v in getattr(olive_passes, config_dict["type"], olive_passes.Pass)._default_config(accelerator).items(): # pylint: disable=protected-access + component = None - if v.type_ == bool: - component = gr.Checkbox - elif v.type_ == str: - component = gr.Textbox - elif v.type_ == int: - component = gr.Number + if v.type_ == bool: + component = gr.Checkbox + elif v.type_ == str: + component = gr.Textbox + elif v.type_ == int: + component = gr.Number - if component is not None: - component = component(value=config_dict["config"][config_key] if config_key in config_dict["config"] else v.default_value, label=config_key) - sd_pass_config_components[submodel][pass_name][config_key] = component - component.change(fn=create_pass_config_change_listener(submodel, pass_name, config_key), inputs=component) + if component is not None: + component = component(value=config_dict["config"][config_key] if config_key in config_dict["config"] else v.default_value, label=config_key) + sd_pass_config_components[submodel][pass_name][config_key] = component + component.change(fn=create_pass_config_change_listener(submodel, pass_name, config_key), inputs=component) - pass_type.change(fn=sd_create_change_listener(submodel, "passes", config_key, "type"), inputs=pass_type) # pylint: disable=undefined-loop-variable + pass_type.change(fn=sd_create_change_listener(submodel, "passes", config_key, "type"), inputs=pass_type) # pylint: disable=undefined-loop-variable - def sd_save(): - for k, v in sd_configs.items(): - with open(os.path.join(sd_config_path, k), "w", encoding="utf-8") as file: - json.dump(v, file) - log.info("Olive: config for SD was saved.") + def sd_save(): + for k, v in sd_configs.items(): + with open(os.path.join(sd_config_path, k), "w", encoding="utf-8") as file: + json.dump(v, file) + log.info("Olive: config for SD was saved.") - sd_save_button = gr.Button(value="Save") - sd_save_button.click(fn=sd_save) + sd_save_button = gr.Button(value="Save") + sd_save_button.click(fn=sd_save) - with gr.TabItem("Stable Diffusion XL", id="sdxl"): - sdxl_config_path = os.path.join(sd_configs_path, "olive", "sdxl") - sdxl_submodels = os.listdir(sdxl_config_path) - sdxl_configs: Dict[str, Dict[str, Dict[str, Dict]]] = {} - sdxl_pass_config_components: Dict[str, Dict[str, Dict]] = {} + with gr.TabItem("Stable Diffusion XL", id="sdxl"): + sdxl_config_path = os.path.join(sd_configs_path, "olive", "sdxl") + sdxl_submodels = os.listdir(sdxl_config_path) + sdxl_configs: Dict[str, Dict[str, Dict[str, Dict]]] = {} + sdxl_pass_config_components: Dict[str, Dict[str, Dict]] = {} - with gr.Tabs(elem_id="tabs_sdxl_submodel"): - def sdxl_create_change_listener(*args): - def listener(v: Dict): - get_recursively(sdxl_configs, *args[:-1])[args[-1]] = v - return listener + with gr.Tabs(elem_id="tabs_sdxl_submodel"): + def sdxl_create_change_listener(*args): + def listener(v: Dict): + get_recursively(sdxl_configs, *args[:-1])[args[-1]] = v + return listener - for submodel in sdxl_submodels: - config: Dict = None + for submodel in sdxl_submodels: + config: Dict = None - sdxl_pass_config_components[submodel] = {} + sdxl_pass_config_components[submodel] = {} - with open(os.path.join(sdxl_config_path, submodel), "r", encoding="utf-8") as file: - config = json.load(file) - sdxl_configs[submodel] = config + with open(os.path.join(sdxl_config_path, submodel), "r", encoding="utf-8") as file: + config = json.load(file) + sdxl_configs[submodel] = config - submodel_name = submodel[:-5] - with gr.TabItem(submodel_name, id=f"sdxl_{submodel_name}"): - pass_flows = DropdownMulti(label="Pass flow", value=sdxl_configs[submodel]["pass_flows"][0], choices=sdxl_configs[submodel]["passes"].keys()) - pass_flows.change(fn=sdxl_create_change_listener(submodel, "pass_flows", 0), inputs=pass_flows) + submodel_name = submodel[:-5] + with gr.TabItem(submodel_name, id=f"sdxl_{submodel_name}"): + pass_flows = DropdownMulti(label="Pass flow", value=sdxl_configs[submodel]["pass_flows"][0], choices=sdxl_configs[submodel]["passes"].keys()) + pass_flows.change(fn=sdxl_create_change_listener(submodel, "pass_flows", 0), inputs=pass_flows) - with gr.Tabs(elem_id=f"tabs_sdxl_{submodel_name}_pass"): - for pass_name in sdxl_configs[submodel]["passes"]: - sdxl_pass_config_components[submodel][pass_name] = {} + with gr.Tabs(elem_id=f"tabs_sdxl_{submodel_name}_pass"): + for pass_name in sdxl_configs[submodel]["passes"]: + sdxl_pass_config_components[submodel][pass_name] = {} - with gr.TabItem(pass_name, id=f"sdxl_{submodel_name}_pass_{pass_name}"): - config_dict = sdxl_configs[submodel]["passes"][pass_name] + with gr.TabItem(pass_name, id=f"sdxl_{submodel_name}_pass_{pass_name}"): + config_dict = sdxl_configs[submodel]["passes"][pass_name] - pass_type = gr.Dropdown(label="Type", value=sdxl_configs[submodel]["passes"][pass_name]["type"], choices=(x.__name__ for x in tuple(olive_passes.REGISTRY.values()))) + pass_type = gr.Dropdown(label="Type", value=sdxl_configs[submodel]["passes"][pass_name]["type"], choices=(x.__name__ for x in tuple(olive_passes.REGISTRY.values()))) - def create_pass_config_change_listener(submodel, pass_name, config_key): # pylint: disable=function-redefined - def listener(value): - sdxl_configs[submodel]["passes"][pass_name]["config"][config_key] = value - return listener + def create_pass_config_change_listener(submodel, pass_name, config_key): # pylint: disable=function-redefined + def listener(value): + sdxl_configs[submodel]["passes"][pass_name]["config"][config_key] = value + return listener - for config_key, v in getattr(olive_passes, config_dict["type"], olive_passes.Pass)._default_config(accelerator).items(): # pylint: disable=protected-access - component = None + for config_key, v in getattr(olive_passes, config_dict["type"], olive_passes.Pass)._default_config(accelerator).items(): # pylint: disable=protected-access + component = None - if v.type_ == bool: - component = gr.Checkbox - elif v.type_ == str: - component = gr.Textbox - elif v.type_ == int: - component = gr.Number + if v.type_ == bool: + component = gr.Checkbox + elif v.type_ == str: + component = gr.Textbox + elif v.type_ == int: + component = gr.Number - if component is not None: - component = component(value=config_dict["config"][config_key] if config_key in config_dict["config"] else v.default_value, label=config_key) - sdxl_pass_config_components[submodel][pass_name][config_key] = component - component.change(fn=create_pass_config_change_listener(submodel, pass_name, config_key), inputs=component) + if component is not None: + component = component(value=config_dict["config"][config_key] if config_key in config_dict["config"] else v.default_value, label=config_key) + sdxl_pass_config_components[submodel][pass_name][config_key] = component + component.change(fn=create_pass_config_change_listener(submodel, pass_name, config_key), inputs=component) - pass_type.change(fn=sdxl_create_change_listener(submodel, "passes", pass_name, "type"), inputs=pass_type) + pass_type.change(fn=sdxl_create_change_listener(submodel, "passes", pass_name, "type"), inputs=pass_type) - def sdxl_save(): - for k, v in sdxl_configs.items(): - with open(os.path.join(sdxl_config_path, k), "w", encoding="utf-8") as file: - json.dump(v, file) - log.info("Olive: config for SDXL was saved.") + def sdxl_save(): + for k, v in sdxl_configs.items(): + with open(os.path.join(sdxl_config_path, k), "w", encoding="utf-8") as file: + json.dump(v, file) + log.info("Olive: config for SDXL was saved.") - sdxl_save_button = gr.Button(value="Save") - sdxl_save_button.click(fn=sdxl_save) + sdxl_save_button = gr.Button(value="Save") + sdxl_save_button.click(fn=sdxl_save) return ui diff --git a/modules/processing_class.py b/modules/processing_class.py index f6465b3ca..9fb90a2d4 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -376,7 +376,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): self.init_images = [self.init_images] for img in self.init_images: if img is None: - shared.log.warning(f"Skipping empty image: images={self.init_images}") + # shared.log.warning(f"Skipping empty image: images={self.init_images}") continue self.init_img_hash = hashlib.sha256(img.tobytes()).hexdigest()[0:8] # pylint: disable=attribute-defined-outside-init self.init_img_width = img.width # pylint: disable=attribute-defined-outside-init @@ -472,6 +472,7 @@ class StableDiffusionProcessingControl(StableDiffusionProcessingImg2Img): self.adapter_conditioning_factor = 1.0 self.attention = 'Attention' self.fidelity = 0.5 + self.mask_image = None self.override = None self.ip_adapter_name = None self.ip_adapter_scale = 1.0 diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 722ed0313..046cfa50b 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -113,10 +113,13 @@ def process_diffusers(p: processing.StableDiffusionProcessing): } elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INPAINTING or is_img2img_model) and len(getattr(p, 'init_images' ,[])) > 0: p.ops.append('inpaint') - if p.task_args.get('mask_image', None) is not None: # provided as override by a module + if p.task_args.get('mask_image', None) is not None: # provided as override by a control module p.mask = masking.run_mask(input_image=p.init_images, input_mask=p.task_args['mask_image'], return_type='Grayscale', invert=p.inpainting_mask_invert==1) - elif getattr(p, 'image_mask', None) is not None: # standard - p.mask = masking.run_mask(input_image=p.init_images, input_mask=p.image_mask, return_type='Grayscale', invert=p.inpainting_mask_invert==1) + elif getattr(p, 'image_mask', None) is not None: # standard img2img + if 'control' in p.ops: + p.mask = masking.run_mask(input_image=p.init_images, input_mask=p.image_mask, return_type='Grayscale', invert=p.inpainting_mask_invert==1) # blur/padding are handled in masking module + else: + p.mask = masking.run_mask(input_image=p.init_images, input_mask=p.image_mask, return_type='Grayscale', invert=p.inpainting_mask_invert==1, mask_blur=p.mask_blur, mask_padding=p.inpaint_full_res_padding) # old img2img elif getattr(p, 'mask', None) is not None: # backward compatibility pass else: # fallback diff --git a/modules/shared.py b/modules/shared.py index bd30f75c1..d27e04822 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -16,8 +16,7 @@ from rich.console import Console from modules import errors, shared_items, shared_state, cmd_args, theme from modules.paths import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611 from modules.dml import memory_providers, default_memory_provider, directml_do_hijack -from modules.onnx_impl import initialize as initialize_onnx -from modules.onnx_impl.execution_providers import available_execution_providers, get_default_execution_provider +from modules.onnx_impl import initialize_onnx, execution_providers import modules.interrogate import modules.memmon import modules.styles @@ -445,8 +444,8 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), { "huggingface_token": OptionInfo('', 'HuggingFace token'), "onnx_sep": OptionInfo("

ONNX Runtime

", "", gr.HTML), - "onnx_execution_provider": OptionInfo(get_default_execution_provider().value, 'Execution Provider', gr.Dropdown, lambda: {"choices": available_execution_providers }), - "onnx_show_menu": OptionInfo(False, 'ONNX show onnx-specific menu'), + "onnx_execution_provider": OptionInfo(execution_providers.get_default_execution_provider().value, 'Execution Provider', gr.Dropdown, lambda: {"choices": execution_providers.available_execution_providers }), + "onnx_cpu_fallback": OptionInfo(True, 'ONNX allow fallback to CPU'), "onnx_cache_converted": OptionInfo(True, 'ONNX cache converted models'), "onnx_unload_base": OptionInfo(False, 'ONNX unload base model when processing refiner'), })) @@ -902,6 +901,10 @@ log.info(f'Device: {print_dict(devices.get_gpu_info())}') prompt_styles = modules.styles.StyleDatabase(opts) cmd_opts.disable_extension_access = (cmd_opts.share or cmd_opts.listen or (cmd_opts.server_name or False)) and not cmd_opts.insecure devices.device, devices.device_interrogate, devices.device_gfpgan, devices.device_esrgan, devices.device_codeformer = (devices.cpu if any(y in cmd_opts.use_cpu for y in [x, 'all']) else devices.get_optimal_device() for x in ['sd', 'interrogate', 'gfpgan', 'esrgan', 'codeformer']) +devices.onnx = [opts.onnx_execution_provider] +if opts.onnx_cpu_fallback and 'CPUExecutionProvider' not in devices.onnx: + devices.onnx.append('CPUExecutionProvider') +print("HERE1", opts.onnx_cpu_fallback, devices.onnx) device = devices.device batch_cond_uncond = opts.always_batch_cond_uncond or not (cmd_opts.lowvram or cmd_opts.medvram) parallel_processing_allowed = not cmd_opts.lowvram diff --git a/modules/ui.py b/modules/ui.py index 178d82edf..343b59d6b 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -342,6 +342,10 @@ def create_ui(startup_timer = None): loadsave.create_ui() create_dirty_indicator("tab_defaults", [], interactive=False) + with gr.TabItem("ONNX", id="onnx_config", elem_id="tab_onnx"): + from modules.onnx_impl import ui as ui_onnx + ui_onnx.create_ui() + with gr.TabItem("Change log", id="change_log", elem_id="system_tab_changelog"): with open('CHANGELOG.md', 'r', encoding='utf-8') as f: md = f.read() @@ -373,13 +377,6 @@ def create_ui(startup_timer = None): interfaces += [(interrogate_interface, "Interrogate", "interrogate")] interfaces += [(train_interface, "Train", "train")] interfaces += [(models_interface, "Models", "models")] - if shared.opts.onnx_show_menu: - with gr.Blocks(analytics_enabled=False) as onnx_interface: - if shared.backend == shared.Backend.DIFFUSERS: - from modules.onnx_impl import ui as ui_onnx - ui_onnx.create_ui() - timer.startup.record("ui-onnx") - interfaces += [(onnx_interface, "ONNX", "onnx")] interfaces += script_callbacks.ui_tabs_callback() interfaces += [(settings_interface, "System", "system")] diff --git a/wiki b/wiki index eaca1886b..4335042b8 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit eaca1886bc918d22f8d9fdbea3d409ed4461cf83 +Subproject commit 4335042b81949976b92c4358e02640d57cdf93b2