add additional controlnets

CHANGELOG.md

@@ -16,16 +16,17 @@ OPTIONAL:
 - masking api
 - preprocess api
 
-## Update for 2023-01-19
+## Update for 2023-01-20
 
 Another big release, highlights being:  
 - A lot more functionality in the **Control** module:
   - Inpaint and outpaint support, flexible resizing options, optional hires  
-  - More processors and models  
+  - Built-in support for many new processors and models which are auto-downloaded on first use  
   - Full support for scripts and extensions  
 - Fully baked-in **FaceID**, **FaceSwap** and **PhotoMaker** modules
 - Much enhanced **IPAdapter** modules  
-- Brand new **intelligent masking**, manual or automatic using ML models and with live previews  
+- Brand new **intelligent masking**, manual or automatic  
+  Using ML models (object removal, background removal, segmentation, etc.) and with live previews  
 
 Plus welcome additions to **UI performance, usability and accessibility** and flexibility of deployment  
 And it also includes fixes for all reported issues so far  
@@ -47,6 +48,8 @@ And it also includes fixes for all reported issues so far
       this is especially useful if you want to use advanced masking on batch or video inputs and dont want to manually mask each image  
       *note*: such auto-created mask is also subject to all other selected settings such as auto-segmentation, blur, erode and dilate  
     - masking can be combined with control processors in which case mask is applied before processor  
+  - support for many additional controlnet models  
+    now built-in models include 30+ SD15 models and 15+ SDXL models  
   - allow **resize** both *before* and *after* generate operation  
     this allows for workflows such as: *image -> upscale or downscale -> generate -> upscale or downscale -> output*  
     providing more flexibility and than standard hires workflow  
@@ -71,6 +74,7 @@ And it also includes fixes for all reported issues so far
   - move ip-adapter implementation to control tabs  
   - resize by now applies to input image or frame individually  
     allows for processing where input images are of different sizes  
+  - support controlnets with non-default yaml config files
   - fix input image size  
   - fix video color mode  
   - fix correct image mode  
@@ -207,6 +211,7 @@ And it also includes fixes for all reported issues so far
   - api: return current image in progress api if requested  
   - api: sanitize response object  
   - api: cleanup error logging  
+  - api: fix api-only errors  
   - sampler: add sampler options info to metadata
   - sampler: guard against invalid sampler index  
   - sampler: add img2img_extra_noise option
@@ -217,6 +222,7 @@ And it also includes fixes for all reported issues so far
   - upscale: fix ldsr
   - cli: fix cmd args parsing  
   - global crlf->lf switch  
+  - model type switch if there is loaded submodels  
 - **other**
   - updated core requirements  
   - major internal ui module refactoring  

modules/api/api.py

@@ -173,7 +173,7 @@ class Api:
 
     def session_start(self, req: Request, agent: Optional[str] = None):
         token = req.cookies.get("access-token") or req.cookies.get("access-token-unsecure")
-        user = self.app.tokens.get(token)
+        user = self.app.tokens.get(token) if hasattr(self.app, 'tokens') else None
         shared.log.info(f'Browser session: user={user} client={req.client.host} agent={agent}')
         return {}
 
@@ -261,7 +261,7 @@ class Api:
                     p.per_script_args[alwayson_script.title()] = request.alwayson_scripts[alwayson_script_name]["args"]
         return script_args
 
-    def prepare_img_gen_request(self, request, img_gen_type: str):
+    def prepare_img_gen_request(self, request, img_gen_type: str): # pylint: disable=unused-argument
         if hasattr(request, "face_id") and request.face_id and not request.script_name and (not request.alwayson_scripts or "FaceID" not in request.alwayson_scripts.keys()):
             request.script_name = "FaceID"
             request.script_args = [

modules/control/units/controlnet.py

@@ -11,22 +11,36 @@ what = 'ControlNet'
 debug = log.trace if os.environ.get('SD_CONTROL_DEBUG', None) is not None else lambda *args, **kwargs: None
 debug('Trace: CONTROL')
 predefined_sd15 = {
-    'Canny FP32': "lllyasviel/control_v11p_sd15_canny",
-    'Depth FP32': "lllyasviel/control_v11f1p_sd15_depth",
-    'HED FP32': "lllyasviel/sd-controlnet-hed",
-    'IP2P FP32': "lllyasviel/control_v11e_sd15_ip2p",
-    'LineArt FP32': "lllyasviel/control_v11p_sd15_lineart",
-    'LineArt Anime FP32': "lllyasviel/control_v11p_sd15s2_lineart_anime",
-    'MLDS FP32': "lllyasviel/control_v11p_sd15_mlsd",
-    'NormalBae FP32': "lllyasviel/control_v11p_sd15_normalbae",
-    'OpenPose FP32': "lllyasviel/control_v11p_sd15_openpose",
-    'Scribble FP32': "lllyasviel/control_v11p_sd15_scribble",
-    'Segment FP32': "lllyasviel/control_v11p_sd15_seg",
-    'Shuffle FP32': "lllyasviel/control_v11e_sd15_shuffle",
-    'SoftEdge FP32': "lllyasviel/control_v11p_sd15_softedge",
-    'TemporalNet FP32': "CiaraRowles/TemporalNet",
-    'Tile FP32': "lllyasviel/control_v11f1e_sd15_tile",
-    'Canny FP16': 'kamaltdin/controlnet1-1_safetensors_with_yaml/controlnet11Models_canny.safetensors',
+    'Canny': "lllyasviel/control_v11p_sd15_canny",
+    'Depth': "lllyasviel/control_v11f1p_sd15_depth",
+    'HED': "lllyasviel/sd-controlnet-hed",
+    'IP2P': "lllyasviel/control_v11e_sd15_ip2p",
+    'LineArt': "lllyasviel/control_v11p_sd15_lineart",
+    'LineArt Anime': "lllyasviel/control_v11p_sd15s2_lineart_anime",
+    'MLDS': "lllyasviel/control_v11p_sd15_mlsd",
+    'NormalBae': "lllyasviel/control_v11p_sd15_normalbae",
+    'OpenPose': "lllyasviel/control_v11p_sd15_openpose",
+    'Scribble': "lllyasviel/control_v11p_sd15_scribble",
+    'Segment': "lllyasviel/control_v11p_sd15_seg",
+    'Shuffle': "lllyasviel/control_v11e_sd15_shuffle",
+    'SoftEdge': "lllyasviel/control_v11p_sd15_softedge",
+    'Tile': "lllyasviel/control_v11f1e_sd15_tile",
+    'Canny FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_canny.safetensors',
+    'Inpaint FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_inpaint.safetensors',
+    'LineArt Anime FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_animeline.safetensors',
+    'LineArt FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_lineart.safetensors',
+    'MLSD FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_mlsd.safetensors',
+    'NormalBae FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_normal.safetensors',
+    'OpenPose FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_openpose.safetensors',
+    'Pix2Pix FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_pix2pix.safetensors',
+    'Scribble FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_scribble.safetensors',
+    'Segment FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_seg.safetensors',
+    'Shuffle FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_shuffle.safetensors',
+    'SoftEdge FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_softedge.safetensors',
+    'Tile FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_tileE.safetensors',
+    'CiaraRowles TemporalNet': "CiaraRowles/TemporalNet",
+    'Ciaochaos Recolor': 'ioclab/control_v1p_sd15_brightness',
+    'Ciaochaos Illumination': 'ioclab/control_v1u_sd15_illumination/illumination20000.safetensors',
 }
 predefined_sdxl = {
     'Canny Small XL': 'diffusers/controlnet-canny-sdxl-1.0-small',
@@ -35,6 +49,14 @@ predefined_sdxl = {
     'Depth Zoe XL': 'diffusers/controlnet-zoe-depth-sdxl-1.0',
     'Depth Mid XL': 'diffusers/controlnet-depth-sdxl-1.0-mid',
     'OpenPose XL': 'thibaud/controlnet-openpose-sdxl-1.0',
+    # 'StabilityAI Canny R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-canny-rank128.safetensors',
+    # 'StabilityAI Depth R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-depth-rank128.safetensors',
+    # 'StabilityAI Recolor R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-recolor-rank128.safetensors',
+    # 'StabilityAI Sketch R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-sketch-rank128-metadata.safetensors',
+    # 'StabilityAI Canny R256': 'stabilityai/control-lora/control-LoRAs-rank256/control-lora-canny-rank256.safetensors',
+    # 'StabilityAI Depth R256': 'stabilityai/control-lora/control-LoRAs-rank256/control-lora-depth-rank256.safetensors',
+    # 'StabilityAI Recolor R256': 'stabilityai/control-lora/control-LoRAs-rank256/control-lora-recolor-rank256.safetensors',
+    # 'StabilityAI Sketch R256': 'stabilityai/control-lora/control-LoRAs-rank256/control-lora-sketch-rank256.safetensors',
 }
 models = {}
 all_models = {}
@@ -64,9 +86,9 @@ def list_models(refresh=False):
     if modules.shared.sd_model_type == 'none':
         models = ['None']
     elif modules.shared.sd_model_type == 'sdxl':
-        models = ['None'] + sorted(predefined_sdxl) + sorted(find_models())
+        models = ['None'] + list(predefined_sdxl) + sorted(find_models())
     elif modules.shared.sd_model_type == 'sd':
-        models = ['None'] + sorted(predefined_sd15) + sorted(find_models())
+        models = ['None'] + list(predefined_sd15) + sorted(find_models())
     else:
         log.warning(f'Control {what} model list failed: unknown model type')
         models = ['None'] + sorted(predefined_sd15) + sorted(predefined_sdxl) + sorted(find_models())
@@ -97,7 +119,9 @@ class ControlNet():
         yaml_path = None
         if not os.path.exists(model_path):
             import huggingface_hub as hf
-            repo_id, filename = os.path.dirname(model_path), os.path.basename(name)
+            parts = model_path.split('/')
+            repo_id = f'{parts[0]}/{parts[1]}'
+            filename = os.path.splitext('/'.join(parts[2:]))[0]
             model_path = hf.hf_hub_download(repo_id=repo_id, filename=f'{filename}.safetensors', cache_dir=cache_dir)
             try:
                 yaml_path = hf.hf_hub_download(repo_id=repo_id, filename=f'{filename}.yaml', cache_dir=cache_dir)
@@ -107,7 +131,6 @@ class ControlNet():
             yaml_path = f'{name}.yaml'
         if yaml_path is not None:
             self.load_config['original_config_file '] = yaml_path
-        print('HERE', self.load_config)
         self.model = ControlNetModel.from_single_file(model_path, **self.load_config)
 
     def load(self, model_id: str = None) -> str:

modules/middleware.py

@@ -47,7 +47,7 @@ def setup_middleware(app: FastAPI, cmd_opts):
                 if '/sdapi/v1/log' in endpoint:
                     return res
                 log.info('API {user} {code} {prot}/{ver} {method} {endpoint} {cli} {duration}'.format( # pylint: disable=consider-using-f-string, logging-format-interpolation
-                    user = app.tokens.get(token),
+                    user = app.tokens.get(token) if hasattr(app, 'tokens') else None,
                     code = res.status_code,
                     ver = req.scope.get('http_version', '0.0'),
                     cli = req.scope.get('client', ('0:0.0.0', 0))[0],

modules/sd_models.py

@@ -1292,8 +1292,12 @@ def unload_model_weights(op='model'):
                 sd_hijack.model_hijack.undo_hijack(model_data.sd_model)
             elif not (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"):
                 disable_offload(model_data.sd_model)
-                model_data.sd_model.to('meta')
+                try:
+                    model_data.sd_model.to('meta')
+                except Exception:
+                    pass
             model_data.sd_model = None
+            devices.torch_gc(force=True)
             shared.log.debug(f'Unload weights {op}: {memory_stats()}')
     else:
         if model_data.sd_refiner:
@@ -1305,8 +1309,8 @@ def unload_model_weights(op='model'):
                 disable_offload(model_data.sd_model)
                 model_data.sd_refiner.to('meta')
             model_data.sd_refiner = None
+            devices.torch_gc(force=True)
             shared.log.debug(f'Unload weights {op}: {memory_stats()}')
-    devices.torch_gc(force=True)
 
 
 def apply_token_merging(sd_model, token_merging_ratio=0):

modules/shared.py

@@ -13,7 +13,7 @@ import fasteners
 import orjson
 import diffusers
 from rich.console import Console
-from modules import errors, shared_items, shared_state, cmd_args, ui_components, theme
+from modules import errors, shared_items, shared_state, cmd_args, theme
 from modules.paths import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611
 from modules.dml import memory_providers, default_memory_provider, directml_do_hijack
 import modules.interrogate