diff --git a/CHANGELOG.md b/CHANGELOG.md index e254edd21..9a865c2a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,16 +16,17 @@ OPTIONAL: - masking api - preprocess api -## Update for 2023-01-19 +## Update for 2023-01-20 Another big release, highlights being: - A lot more functionality in the **Control** module: - Inpaint and outpaint support, flexible resizing options, optional hires - - More processors and models + - Built-in support for many new processors and models which are auto-downloaded on first use - Full support for scripts and extensions - Fully baked-in **FaceID**, **FaceSwap** and **PhotoMaker** modules - Much enhanced **IPAdapter** modules -- Brand new **intelligent masking**, manual or automatic using ML models and with live previews +- Brand new **intelligent masking**, manual or automatic + Using ML models (object removal, background removal, segmentation, etc.) and with live previews Plus welcome additions to **UI performance, usability and accessibility** and flexibility of deployment And it also includes fixes for all reported issues so far @@ -47,6 +48,8 @@ And it also includes fixes for all reported issues so far this is especially useful if you want to use advanced masking on batch or video inputs and dont want to manually mask each image *note*: such auto-created mask is also subject to all other selected settings such as auto-segmentation, blur, erode and dilate - masking can be combined with control processors in which case mask is applied before processor + - support for many additional controlnet models + now built-in models include 30+ SD15 models and 15+ SDXL models - allow **resize** both *before* and *after* generate operation this allows for workflows such as: *image -> upscale or downscale -> generate -> upscale or downscale -> output* providing more flexibility and than standard hires workflow @@ -71,6 +74,7 @@ And it also includes fixes for all reported issues so far - move ip-adapter implementation to control tabs - resize by now applies to input image or frame individually allows for processing where input images are of different sizes + - support controlnets with non-default yaml config files - fix input image size - fix video color mode - fix correct image mode @@ -207,6 +211,7 @@ And it also includes fixes for all reported issues so far - api: return current image in progress api if requested - api: sanitize response object - api: cleanup error logging + - api: fix api-only errors - sampler: add sampler options info to metadata - sampler: guard against invalid sampler index - sampler: add img2img_extra_noise option @@ -217,6 +222,7 @@ And it also includes fixes for all reported issues so far - upscale: fix ldsr - cli: fix cmd args parsing - global crlf->lf switch + - model type switch if there is loaded submodels - **other** - updated core requirements - major internal ui module refactoring diff --git a/modules/api/api.py b/modules/api/api.py index 3b530d85b..c1cf45b46 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -173,7 +173,7 @@ class Api: def session_start(self, req: Request, agent: Optional[str] = None): token = req.cookies.get("access-token") or req.cookies.get("access-token-unsecure") - user = self.app.tokens.get(token) + user = self.app.tokens.get(token) if hasattr(self.app, 'tokens') else None shared.log.info(f'Browser session: user={user} client={req.client.host} agent={agent}') return {} @@ -261,7 +261,7 @@ class Api: p.per_script_args[alwayson_script.title()] = request.alwayson_scripts[alwayson_script_name]["args"] return script_args - def prepare_img_gen_request(self, request, img_gen_type: str): + def prepare_img_gen_request(self, request, img_gen_type: str): # pylint: disable=unused-argument if hasattr(request, "face_id") and request.face_id and not request.script_name and (not request.alwayson_scripts or "FaceID" not in request.alwayson_scripts.keys()): request.script_name = "FaceID" request.script_args = [ diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index c6a6f6e65..59198a9da 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -11,22 +11,36 @@ what = 'ControlNet' debug = log.trace if os.environ.get('SD_CONTROL_DEBUG', None) is not None else lambda *args, **kwargs: None debug('Trace: CONTROL') predefined_sd15 = { - 'Canny FP32': "lllyasviel/control_v11p_sd15_canny", - 'Depth FP32': "lllyasviel/control_v11f1p_sd15_depth", - 'HED FP32': "lllyasviel/sd-controlnet-hed", - 'IP2P FP32': "lllyasviel/control_v11e_sd15_ip2p", - 'LineArt FP32': "lllyasviel/control_v11p_sd15_lineart", - 'LineArt Anime FP32': "lllyasviel/control_v11p_sd15s2_lineart_anime", - 'MLDS FP32': "lllyasviel/control_v11p_sd15_mlsd", - 'NormalBae FP32': "lllyasviel/control_v11p_sd15_normalbae", - 'OpenPose FP32': "lllyasviel/control_v11p_sd15_openpose", - 'Scribble FP32': "lllyasviel/control_v11p_sd15_scribble", - 'Segment FP32': "lllyasviel/control_v11p_sd15_seg", - 'Shuffle FP32': "lllyasviel/control_v11e_sd15_shuffle", - 'SoftEdge FP32': "lllyasviel/control_v11p_sd15_softedge", - 'TemporalNet FP32': "CiaraRowles/TemporalNet", - 'Tile FP32': "lllyasviel/control_v11f1e_sd15_tile", - 'Canny FP16': 'kamaltdin/controlnet1-1_safetensors_with_yaml/controlnet11Models_canny.safetensors', + 'Canny': "lllyasviel/control_v11p_sd15_canny", + 'Depth': "lllyasviel/control_v11f1p_sd15_depth", + 'HED': "lllyasviel/sd-controlnet-hed", + 'IP2P': "lllyasviel/control_v11e_sd15_ip2p", + 'LineArt': "lllyasviel/control_v11p_sd15_lineart", + 'LineArt Anime': "lllyasviel/control_v11p_sd15s2_lineart_anime", + 'MLDS': "lllyasviel/control_v11p_sd15_mlsd", + 'NormalBae': "lllyasviel/control_v11p_sd15_normalbae", + 'OpenPose': "lllyasviel/control_v11p_sd15_openpose", + 'Scribble': "lllyasviel/control_v11p_sd15_scribble", + 'Segment': "lllyasviel/control_v11p_sd15_seg", + 'Shuffle': "lllyasviel/control_v11e_sd15_shuffle", + 'SoftEdge': "lllyasviel/control_v11p_sd15_softedge", + 'Tile': "lllyasviel/control_v11f1e_sd15_tile", + 'Canny FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_canny.safetensors', + 'Inpaint FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_inpaint.safetensors', + 'LineArt Anime FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_animeline.safetensors', + 'LineArt FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_lineart.safetensors', + 'MLSD FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_mlsd.safetensors', + 'NormalBae FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_normal.safetensors', + 'OpenPose FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_openpose.safetensors', + 'Pix2Pix FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_pix2pix.safetensors', + 'Scribble FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_scribble.safetensors', + 'Segment FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_seg.safetensors', + 'Shuffle FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_shuffle.safetensors', + 'SoftEdge FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_softedge.safetensors', + 'Tile FP16': 'Aptronym/SDNext/ControlNet11/controlnet11Models_tileE.safetensors', + 'CiaraRowles TemporalNet': "CiaraRowles/TemporalNet", + 'Ciaochaos Recolor': 'ioclab/control_v1p_sd15_brightness', + 'Ciaochaos Illumination': 'ioclab/control_v1u_sd15_illumination/illumination20000.safetensors', } predefined_sdxl = { 'Canny Small XL': 'diffusers/controlnet-canny-sdxl-1.0-small', @@ -35,6 +49,14 @@ predefined_sdxl = { 'Depth Zoe XL': 'diffusers/controlnet-zoe-depth-sdxl-1.0', 'Depth Mid XL': 'diffusers/controlnet-depth-sdxl-1.0-mid', 'OpenPose XL': 'thibaud/controlnet-openpose-sdxl-1.0', + # 'StabilityAI Canny R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-canny-rank128.safetensors', + # 'StabilityAI Depth R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-depth-rank128.safetensors', + # 'StabilityAI Recolor R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-recolor-rank128.safetensors', + # 'StabilityAI Sketch R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-sketch-rank128-metadata.safetensors', + # 'StabilityAI Canny R256': 'stabilityai/control-lora/control-LoRAs-rank256/control-lora-canny-rank256.safetensors', + # 'StabilityAI Depth R256': 'stabilityai/control-lora/control-LoRAs-rank256/control-lora-depth-rank256.safetensors', + # 'StabilityAI Recolor R256': 'stabilityai/control-lora/control-LoRAs-rank256/control-lora-recolor-rank256.safetensors', + # 'StabilityAI Sketch R256': 'stabilityai/control-lora/control-LoRAs-rank256/control-lora-sketch-rank256.safetensors', } models = {} all_models = {} @@ -64,9 +86,9 @@ def list_models(refresh=False): if modules.shared.sd_model_type == 'none': models = ['None'] elif modules.shared.sd_model_type == 'sdxl': - models = ['None'] + sorted(predefined_sdxl) + sorted(find_models()) + models = ['None'] + list(predefined_sdxl) + sorted(find_models()) elif modules.shared.sd_model_type == 'sd': - models = ['None'] + sorted(predefined_sd15) + sorted(find_models()) + models = ['None'] + list(predefined_sd15) + sorted(find_models()) else: log.warning(f'Control {what} model list failed: unknown model type') models = ['None'] + sorted(predefined_sd15) + sorted(predefined_sdxl) + sorted(find_models()) @@ -97,7 +119,9 @@ class ControlNet(): yaml_path = None if not os.path.exists(model_path): import huggingface_hub as hf - repo_id, filename = os.path.dirname(model_path), os.path.basename(name) + parts = model_path.split('/') + repo_id = f'{parts[0]}/{parts[1]}' + filename = os.path.splitext('/'.join(parts[2:]))[0] model_path = hf.hf_hub_download(repo_id=repo_id, filename=f'{filename}.safetensors', cache_dir=cache_dir) try: yaml_path = hf.hf_hub_download(repo_id=repo_id, filename=f'{filename}.yaml', cache_dir=cache_dir) @@ -107,7 +131,6 @@ class ControlNet(): yaml_path = f'{name}.yaml' if yaml_path is not None: self.load_config['original_config_file '] = yaml_path - print('HERE', self.load_config) self.model = ControlNetModel.from_single_file(model_path, **self.load_config) def load(self, model_id: str = None) -> str: diff --git a/modules/middleware.py b/modules/middleware.py index 6bebe0e69..978d14b11 100644 --- a/modules/middleware.py +++ b/modules/middleware.py @@ -47,7 +47,7 @@ def setup_middleware(app: FastAPI, cmd_opts): if '/sdapi/v1/log' in endpoint: return res log.info('API {user} {code} {prot}/{ver} {method} {endpoint} {cli} {duration}'.format( # pylint: disable=consider-using-f-string, logging-format-interpolation - user = app.tokens.get(token), + user = app.tokens.get(token) if hasattr(app, 'tokens') else None, code = res.status_code, ver = req.scope.get('http_version', '0.0'), cli = req.scope.get('client', ('0:0.0.0', 0))[0], diff --git a/modules/sd_models.py b/modules/sd_models.py index 07ab30570..9a06f1245 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1292,8 +1292,12 @@ def unload_model_weights(op='model'): sd_hijack.model_hijack.undo_hijack(model_data.sd_model) elif not (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"): disable_offload(model_data.sd_model) - model_data.sd_model.to('meta') + try: + model_data.sd_model.to('meta') + except Exception: + pass model_data.sd_model = None + devices.torch_gc(force=True) shared.log.debug(f'Unload weights {op}: {memory_stats()}') else: if model_data.sd_refiner: @@ -1305,8 +1309,8 @@ def unload_model_weights(op='model'): disable_offload(model_data.sd_model) model_data.sd_refiner.to('meta') model_data.sd_refiner = None + devices.torch_gc(force=True) shared.log.debug(f'Unload weights {op}: {memory_stats()}') - devices.torch_gc(force=True) def apply_token_merging(sd_model, token_merging_ratio=0): diff --git a/modules/shared.py b/modules/shared.py index fe5d68116..5b3392be7 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -13,7 +13,7 @@ import fasteners import orjson import diffusers from rich.console import Console -from modules import errors, shared_items, shared_state, cmd_args, ui_components, theme +from modules import errors, shared_items, shared_state, cmd_args, theme from modules.paths import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611 from modules.dml import memory_providers, default_memory_provider, directml_do_hijack import modules.interrogate