From edf1dc68f4fce2b06c4813343cbb2ea1bbd15bc8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 23 Jan 2024 14:15:07 -0500 Subject: [PATCH] add depth-anything controlnet --- CHANGELOG.md | 6 ++++-- modules/control/units/controlnet.py | 25 +++++++++++++++++-------- modules/control/units/xs_model.py | 19 +++++++++---------- modules/lama.py | 6 ++++++ modules/masking.py | 2 ++ modules/ui_control.py | 2 +- 6 files changed, 39 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b64ece8ad..a9a50f095 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ OPTIONAL: - style aligned [pr](https://github.com/huggingface/diffusers/pull/6489) - mixture tiling [pr](https://github.com/huggingface/diffusers/tree/main/examples/community#stable-diffusion-mixture-tiling) - depth anything [repo](https://depth-anything.github.io/) +- instaflow [pr](https://github.com/huggingface/diffusers/pull/6057)[repo](https://github.com/gnobitab/RectifiedFlow) - control api - photomaker api - interrogate api @@ -20,7 +21,7 @@ OPTIONAL: - masking api - preprocess api -## Update for 2023-01-22 +## Update for 2023-01-23 Another big release, highlights being: - A lot more functionality in the **Control** module: @@ -70,6 +71,7 @@ As of this release, default backend is set to **diffusers** as its more feature - add support for **scripts** and **extensions** you can now combine control workflow with your favorite script or extension *note* extensions that are hard-coded for txt2img or img2img tabs may not work until they are updated + - add **depth-anything** depth map processor and trained controlnet - add **marigold** depth map processor this is state-of-the-art depth estimation model, but its quite heavy on resources - add **openpose xl** controlnet @@ -104,7 +106,7 @@ As of this release, default backend is set to **diffusers** as its more feature - additional models for *SD15* and *SD-XL*, to use simply select from *Scripts*: **SD15**: Base, Base ViT-G, Light, Plus, Plus Face, Full Face **SDXL**: Base SXDL, Base ViT-H SXDL, Plus ViT-H SXDL, Plus Face ViT-H SXDL - - enable use via api, thanks @trojaner + - enable use via api, thanks @trojaner - [PhotoMaker](https://github.com/TencentARC/PhotoMaker) - for *SD-XL* only - simply select from *scripts* diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index 59198a9da..bac2196d1 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -25,6 +25,7 @@ predefined_sd15 = { 'Shuffle': "lllyasviel/control_v11e_sd15_shuffle", 'SoftEdge': "lllyasviel/control_v11p_sd15_softedge", 'Tile': "lllyasviel/control_v11f1e_sd15_tile", + 'Depth Anything': 'vladmandic/depth-anything', 'Canny FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_canny.safetensors', 'Inpaint FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_inpaint.safetensors', 'LineArt Anime FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_animeline.safetensors', @@ -116,21 +117,29 @@ class ControlNet(): def load_safetensors(self, model_path): name = os.path.splitext(model_path)[0] - yaml_path = None + config_path = None if not os.path.exists(model_path): import huggingface_hub as hf parts = model_path.split('/') repo_id = f'{parts[0]}/{parts[1]}' filename = os.path.splitext('/'.join(parts[2:]))[0] model_path = hf.hf_hub_download(repo_id=repo_id, filename=f'{filename}.safetensors', cache_dir=cache_dir) - try: - yaml_path = hf.hf_hub_download(repo_id=repo_id, filename=f'{filename}.yaml', cache_dir=cache_dir) - except Exception: - pass # no yaml file + if config_path is None: + try: + config_path = hf.hf_hub_download(repo_id=repo_id, filename=f'{filename}.yaml', cache_dir=cache_dir) + except Exception: + pass # no yaml file + if config_path is None: + try: + config_path = hf.hf_hub_download(repo_id=repo_id, filename=f'{filename}.json', cache_dir=cache_dir) + except Exception: + pass # no yaml file elif os.path.exists(name + '.yaml'): - yaml_path = f'{name}.yaml' - if yaml_path is not None: - self.load_config['original_config_file '] = yaml_path + config_path = f'{name}.yaml' + elif os.path.exists(name + '.json'): + config_path = f'{name}.json' + if config_path is not None: + self.load_config['original_config_file '] = config_path self.model = ControlNetModel.from_single_file(model_path, **self.load_config) def load(self, model_id: str = None) -> str: diff --git a/modules/control/units/xs_model.py b/modules/control/units/xs_model.py index 440756f53..3a6721766 100644 --- a/modules/control/units/xs_model.py +++ b/modules/control/units/xs_model.py @@ -26,16 +26,15 @@ from diffusers.models.attention_processor import USE_PEFT_BACKEND, AttentionProc from diffusers.models.autoencoders import AutoencoderKL from diffusers.models.lora import LoRACompatibleConv from diffusers.models.modeling_utils import ModelMixin -from diffusers.models.unet_2d_blocks import ( - CrossAttnDownBlock2D, - CrossAttnUpBlock2D, - DownBlock2D, - Downsample2D, - ResnetBlock2D, - Transformer2DModel, - UpBlock2D, - Upsample2D, -) +try: + from diffusers.models.unet_2d_blocks import CrossAttnDownBlock2D, CrossAttnUpBlock2D, DownBlock2D, Downsample2D, ResnetBlock2D, Transformer2DModel, UpBlock2D, Upsample2D +except Exception: + pass +try: + from diffusers.models.unets.unet_2d_blocks import CrossAttnDownBlock2D, CrossAttnUpBlock2D, DownBlock2D, Downsample2D, ResnetBlock2D, Transformer2DModel, UpBlock2D, Upsample2D +except Exception: + pass + from diffusers.models.unet_2d_condition import UNet2DConditionModel from diffusers.utils import BaseOutput, logging diff --git a/modules/lama.py b/modules/lama.py index 69f5e8133..06e360caa 100644 --- a/modules/lama.py +++ b/modules/lama.py @@ -87,6 +87,12 @@ class SimpleLama: self.model.to(self.device) def __call__(self, image: Image.Image | np.ndarray, mask: Image.Image | np.ndarray): + if image is None: + log.warning('LaMa: image is none') + return None + if mask is None: + mask = Image.new('L', image.size, 0) + return None image, mask = prepare_img_and_mask(image, mask, self.device) with devices.inference_context(): inpainted = self.model(image, mask) diff --git a/modules/masking.py b/modules/masking.py index 6226abe6c..ad19f5106 100644 --- a/modules/masking.py +++ b/modules/masking.py @@ -217,6 +217,8 @@ def run_segment(input_image: gr.Image, input_mask: np.ndarray): continue overlap = 0 if input_mask_size > 0: + if mask.shape != input_mask.shape: + mask = cv2.resize(mask, (input_mask.shape[1], input_mask.shape[0]), interpolation=cv2.INTER_CUBIC) overlap = cv2.bitwise_and(mask, input_mask) overlap = np.count_nonzero(overlap) if overlap == 0: diff --git a/modules/ui_control.py b/modules/ui_control.py index eab240d8f..ca55a266e 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -629,7 +629,7 @@ def create_ui(_blocks: gr.Blocks=None): settings.append(gr.Slider(label="Denoising steps", minimum=1, maximum=99, step=1, value=10)) settings.append(gr.Slider(label="Ensemble size", minimum=1, maximum=99, step=1, value=10)) with gr.Accordion('Depth Anything', open=True, elem_classes=['processor-settings']): - settings.append(gr.Dropdown(label="Color map", choices=['inferno'] + masking.COLORMAP, value='inferno')) + settings.append(gr.Dropdown(label="Color map", choices=['none'] + masking.COLORMAP, value='inferno')) for setting in settings: setting.change(fn=processors.update_settings, inputs=settings, outputs=[])