From 6a4d4ea5b7a6ecfb0d1c15813ec18f49a36195a9 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 22 Aug 2023 18:28:09 +0000 Subject: [PATCH] update logging and model hashinh --- CHANGELOG.md | 15 ++++++++--- extensions-builtin/a1111-sd-webui-lycoris | 2 +- extensions-builtin/sd-webui-controlnet | 2 +- .../stable-diffusion-webui-images-browser | 2 +- installer.py | 3 +++ launch.py | 7 ++++- modules/devices.py | 1 + modules/hashes.py | 7 ++--- modules/images.py | 2 +- modules/modelloader.py | 2 +- modules/sd_models.py | 27 ++++++++++++------- modules/sd_vae.py | 2 +- modules/shared.py | 10 ++++--- modules/styles.py | 2 +- modules/ui.py | 8 +++--- modules/ui_extra_networks.py | 2 +- 16 files changed, 59 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5212f7b5a..54c37fc1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,18 @@ # Change Log for SD.Next -## Update for 2023-08-21 +## Update for 2023-08-22 - general: - - all system and image paths are now relative by default - - fix extra networks previews - - add settings validation when performing load/save + - all system and image paths are now relative by default + - fix extra networks previews + - add settings validation when performing load/save + - optimize model hashing + - add cli param `--skip-all` that skips all installer checks + use at personal discretion, but it can be useful for bulk deployments + - add **openvino** (experimental) cpu optimized model compile and inference + enable with `--use-openvino` + thanks @disty0 + - add model **precompile** option (when model compile is enbled) ## Update for 2023-08-20 diff --git a/extensions-builtin/a1111-sd-webui-lycoris b/extensions-builtin/a1111-sd-webui-lycoris index 912576970..df9c0093b 160000 --- a/extensions-builtin/a1111-sd-webui-lycoris +++ b/extensions-builtin/a1111-sd-webui-lycoris @@ -1 +1 @@ -Subproject commit 912576970a9fe55537853e595e7ed4c27a645bc7 +Subproject commit df9c0093bc9e079487302d3ed3f5db4337d6f19a diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 0cfc88b6a..943197e1b 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 0cfc88b6a892076d199e68c02e9c306ac6ab2ead +Subproject commit 943197e1be49d8ce1bf964e15dd19e386f5ad5b8 diff --git a/extensions-builtin/stable-diffusion-webui-images-browser b/extensions-builtin/stable-diffusion-webui-images-browser index 65618f3e0..15c337914 160000 --- a/extensions-builtin/stable-diffusion-webui-images-browser +++ b/extensions-builtin/stable-diffusion-webui-images-browser @@ -1 +1 @@ -Subproject commit 65618f3e077d4014a6f0569be2bbb76fc09cbfec +Subproject commit 15c337914d86040d50864a37488f447cd233b2bc diff --git a/installer.py b/installer.py index a14908499..a5b67f9e4 100644 --- a/installer.py +++ b/installer.py @@ -736,6 +736,8 @@ def check_extensions(): # check version of the main repo and optionally upgrade it def check_version(offline=False, reset=True): # pylint: disable=unused-argument + if args.skip_all: + return if not os.path.exists('.git'): log.error('Not a git repository') if not args.ignore: @@ -843,6 +845,7 @@ def add_args(parser): group.add_argument('--skip-extensions', default = False, action='store_true', help = "Skips running individual extension installers, default: %(default)s") group.add_argument('--skip-git', default = False, action='store_true', help = "Skips running all GIT operations, default: %(default)s") group.add_argument('--skip-torch', default = False, action='store_true', help = "Skips running Torch checks, default: %(default)s") + group.add_argument('--skip-all', default = False, action='store_true', help = "Skips running all checks, default: %(default)s") group.add_argument('--experimental', default = False, action='store_true', help = "Allow unsupported versions of libraries, default: %(default)s") group.add_argument('--reinstall', default = False, action='store_true', help = "Force reinstallation of all requirements, default: %(default)s") group.add_argument('--test', default = False, action='store_true', help = "Run test only and exit") diff --git a/launch.py b/launch.py index 7ffd5f72a..8a03274e7 100644 --- a/launch.py +++ b/launch.py @@ -154,6 +154,8 @@ if __name__ == "__main__": installer.setup_logging() installer.log.info('Starting SD.Next') installer.read_options() + if args.skip_all: + args.quick = True installer.check_python() if args.reset: installer.git_reset() @@ -166,7 +168,10 @@ if __name__ == "__main__": if args.reinstall: installer.log.info('Forcing reinstall of all packages') installer.quick_allowed = False - if installer.check_timestamp(): + if args.skip_all: + installer.log.info('Skipping all checks') + installer.quick_allowed = True + elif installer.check_timestamp(): installer.log.info('No changes detected: Quick launch active') installer.install_requirements() installer.install_packages() diff --git a/modules/devices.py b/modules/devices.py index 8647105ac..cb29e6e3b 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -53,6 +53,7 @@ def get_optimal_device(): def get_device_for(task): if task in shared.cmd_opts.use_cpu: + shared.log.debug(f'Forcing CPU for task: {task}') return cpu return get_optimal_device() diff --git a/modules/hashes.py b/modules/hashes.py index 76953c6bd..d4c35483a 100644 --- a/modules/hashes.py +++ b/modules/hashes.py @@ -15,10 +15,7 @@ def dump_cache(): def cache(subsection): global cache_data # pylint: disable=global-statement if cache_data is None: - if not os.path.isfile(cache_filename): - cache_data = {} - else: - cache_data = shared.readfile(cache_filename) + cache_data = {} if not os.path.isfile(cache_filename) else shared.readfile(cache_filename) s = cache_data.get(subsection, {}) cache_data[subsection] = s return s @@ -59,7 +56,7 @@ def sha256(filename, title, use_addnet_hash=False): sha256_value = calculate_sha256(filename) hashes[title] = { "mtime": os.path.getmtime(filename), - "sha256": sha256_value, + "sha256": sha256_value } dump_cache() return sha256_value diff --git a/modules/images.py b/modules/images.py index eac7067fb..fe032ab27 100644 --- a/modules/images.py +++ b/modules/images.py @@ -448,7 +448,7 @@ def atomically_save_image(): image_format = 'JPEG' if shared.opts.image_watermark_enabled: image = set_watermark(image, shared.opts.image_watermark) - shared.log.debug(f'Saving image: {image_format} {fn} {image.size}') + shared.log.debug(f'Saving image: type={image_format} size={image.size} {fn}') # actual save exifinfo = (exifinfo or "") if shared.opts.image_metadata else "" if image_format == 'PNG': diff --git a/modules/modelloader.py b/modules/modelloader.py index 431ff3fb1..a2db0fabb 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -157,7 +157,7 @@ def load_diffusers_models(model_path: str, command_path: str = None): output.append(str(r.repo_id)) except Exception as e: shared.log.error(f"Error listing diffusers: {place} {e}") - shared.log.debug(f'Scanning diffusers cache: {model_path} {command_path} {len(output)}') + shared.log.debug(f'Scanning diffusers cache: {model_path} {command_path} items={len(output)}') return output diff --git a/modules/sd_models.py b/modules/sd_models.py index 6a07d6874..f60cbf897 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -4,6 +4,7 @@ import re import io import sys import json +import time import threading from os import mkdir from urllib import request @@ -38,7 +39,7 @@ skip_next_load = False sd_metadata_file = os.path.join(paths.data_path, "metadata.json") sd_metadata = None sd_metadata_pending = 0 - +sd_metadata_timer = 0 class CheckpointInfo: def __init__(self, filename): @@ -59,8 +60,8 @@ class CheckpointInfo: if name.startswith("\\") or name.startswith("/"): name = name[1:] self.name = name - self.hash = model_hash(self.filename) self.sha256 = hashes.sha256_from_cache(self.filename, f"checkpoint/{name}") + self.hash = self.sha256[0:8] if self.sha256 is not None else None self.path = abspath self.type = abspath.split('.')[-1].lower() self.name_for_extra = os.path.splitext(os.path.basename(filename))[0] @@ -155,7 +156,7 @@ def list_models(): shared.opts.data['sd_model_checkpoint'] = checkpoint_info.title elif shared.cmd_opts.ckpt != shared.default_sd_model_file and shared.cmd_opts.ckpt is not None: shared.log.warning(f"Checkpoint not found: {shared.cmd_opts.ckpt}") - shared.log.info(f'Available models: {shared.opts.ckpt_dir} {len(checkpoints_list)}') + shared.log.info(f'Available models: {shared.opts.ckpt_dir} items={len(checkpoints_list)}') if len(checkpoints_list) == 0: if not shared.cmd_opts.no_download: @@ -212,10 +213,14 @@ def model_hash(filename): try: with open(filename, "rb") as file: import hashlib + t0 = time.time() m = hashlib.sha256() file.seek(0x100000) m.update(file.read(0x10000)) - return m.hexdigest()[0:8] + shorthash = m.hexdigest()[0:8] + t1 = time.time() + shared.log.debug(f'Calculating short hash: {filename} hash={shorthash} time={(t1-t0):.2f}') + return shorthash except FileNotFoundError: return 'NOFILE' except Exception: @@ -284,7 +289,7 @@ def write_metadata(): shared.log.debug(f"Model metadata: {sd_metadata_file} no changes") return shared.writefile(sd_metadata, sd_metadata_file) - shared.log.info(f"Model metadata saved: {sd_metadata_file} {sd_metadata_pending}") + shared.log.info(f"Model metadata saved: {sd_metadata_file} items={sd_metadata_pending} time={sd_metadata_timer:.2f}s") sd_metadata_pending = 0 @@ -300,6 +305,7 @@ def read_metadata_from_safetensors(filename): return res res = {} try: + t0 = time.time() with open(filename, mode="rb") as file: metadata_len = file.read(8) metadata_len = int.from_bytes(metadata_len, "little") @@ -318,6 +324,9 @@ def read_metadata_from_safetensors(filename): sd_metadata[filename] = res global sd_metadata_pending # pylint: disable=global-statement sd_metadata_pending += 1 + t1 = time.time() + global sd_metadata_timer # pylint: disable=global-statement + sd_metadata_timer += (t1 - t0) except Exception as e: shared.log.error(f"Error reading metadata from: {filename} {e}") return res @@ -368,7 +377,6 @@ def get_checkpoint_state_dict(checkpoint_info: CheckpointInfo, timer): def load_model_weights(model: torch.nn.Module, checkpoint_info: CheckpointInfo, state_dict, timer): shared.log.debug(f'Model weights loading: {memory_stats()}') - sd_model_hash = checkpoint_info.calculate_shorthash() timer.record("hash") if model_data.sd_dict == 'None': shared.opts.data["sd_model_checkpoint"] = checkpoint_info.title @@ -407,7 +415,7 @@ def load_model_weights(model: torch.nn.Module, checkpoint_info: CheckpointInfo, # clean up cache if limit is reached while len(checkpoints_loaded) > shared.opts.sd_checkpoint_cache: checkpoints_loaded.popitem(last=False) - model.sd_model_hash = sd_model_hash + model.sd_model_hash = checkpoint_info.calculate_shorthash() model.sd_model_checkpoint = checkpoint_info.filename model.sd_checkpoint_info = checkpoint_info shared.opts.data["sd_checkpoint_hash"] = checkpoint_info.sha256 @@ -795,7 +803,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No shared.log.info(f"Compiling pipeline={sd_model.__class__.__name__} shape={8 * sd_model.unet.config.sample_size} mode={shared.opts.cuda_compile_backend}") import torch._dynamo # pylint: disable=unused-import,redefined-outer-name if shared.opts.cuda_compile_backend == "openvino_fx": - from modules.intel.openvino import openvino_fx + from modules.intel.openvino import openvino_fx # pylint: disable=unused-import log_level = logging.WARNING if shared.opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access if hasattr(torch, '_logging'): torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access @@ -812,9 +820,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if sd_model is None: shared.log.error('Diffuser model not loaded') return + sd_model.sd_model_hash = checkpoint_info.calculate_shorthash() # pylint: disable=attribute-defined-outside-init sd_model.sd_checkpoint_info = checkpoint_info # pylint: disable=attribute-defined-outside-init sd_model.sd_model_checkpoint = checkpoint_info.filename # pylint: disable=attribute-defined-outside-init - sd_model.sd_model_hash = checkpoint_info.hash # pylint: disable=attribute-defined-outside-init + shared.opts.data["sd_checkpoint_hash"] = checkpoint_info.sha256 if hasattr(sd_model, "set_progress_bar_config"): sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining}', ncols=80, colour='#327fba') if op == 'refiner' and shared.opts.diffusers_move_refiner and not sd_model.has_accelerate: diff --git a/modules/sd_vae.py b/modules/sd_vae.py index 8e845431d..e3521066c 100644 --- a/modules/sd_vae.py +++ b/modules/sd_vae.py @@ -98,7 +98,7 @@ def refresh_vae_list(): vae_dict[name] = os.path.dirname(filepath) else: vae_dict[name] = filepath - shared.log.info(f"Available VAEs: {vae_path} {len(vae_dict)}") + shared.log.info(f"Available VAEs: {vae_path} items={len(vae_dict)}") return vae_dict diff --git a/modules/shared.py b/modules/shared.py index 51c46e230..0906f19d7 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -66,6 +66,7 @@ restricted_opts = { "outdir_save", "outdir_init_images" } +compatibility_opts = ['clip_skip', 'uni_pc_lower_order_final', 'uni_pc_order'] def is_url(string): @@ -720,11 +721,12 @@ class Options: if self.data_labels[k].default != v: diff[k] = v else: - unused_settings.append(k) + if k not in compatibility_opts: + unused_settings.append(k) diff[k] = v writefile(diff, filename) if len(unused_settings) > 0: - log.debug(f"Unknown settings: {unused_settings}") + log.debug(f"Unused settings: {unused_settings}") except Exception as e: log.error(f'Saving settings failed: {filename} {e}') @@ -748,10 +750,10 @@ class Options: info = self.data_labels.get(k, None) if info is not None and not self.same_type(info.default, v): log.error(f"Error: bad setting value: {k}: {v} ({type(v).__name__}; expected {type(info.default).__name__})") - if info is None: + if info is None and k not in compatibility_opts: unknown_settings.append(k) if len(unknown_settings) > 0: - log.debug(f"Unknown settings: {unknown_settings}") + log.warning(f"Unknown settings: {unknown_settings}") def onchange(self, key, func, call=True): item = self.data_labels.get(key) diff --git a/modules/styles.py b/modules/styles.py index 635d08cdc..f611a2c55 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -60,7 +60,7 @@ class StyleDatabase: self.styles[row["name"]] = PromptStyle(row["name"], prompt, negative_prompt) except Exception: log.error(f'Styles error: {self.path} {row}') - log.debug(f'Loaded styles: {self.path} {len(self.styles.keys())}') + log.debug(f'Loaded styles: {self.path} items={len(self.styles.keys())}') def get_style_prompts(self, styles): return [self.styles.get(x, self.no_style).prompt for x in styles] diff --git a/modules/ui.py b/modules/ui.py index 66aae0096..dfd3b9292 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -843,7 +843,7 @@ def create_ui(startup_timer = None): (latent_index, "Latent sampler"), (denoising_strength, "Denoising strength"), (refiner_start, "Refiner start"), - (full_quality, "Full quality"), # TODO + (full_quality, "Full quality"), (restore_faces, "Face restoration"), (batch_size, "Batch size"), (batch_count, "Batch count"), @@ -968,13 +968,13 @@ def create_ui(startup_timer = None): directml_override_opts() if cmd_opts.use_openvino: if not modules.shared.opts.cuda_compile: - modules.shared.log.warn("OpenVINO: Enabling Torch Compile") + modules.shared.log.warning("OpenVINO: Enabling Torch Compile") modules.shared.opts.cuda_compile = True if modules.shared.opts.cuda_compile_backend != "openvino_fx": - modules.shared.log.warn("OpenVINO: Setting Torch Compiler backend to OpenVINO FX") + modules.shared.log.warning("OpenVINO: Setting Torch Compiler backend to OpenVINO FX") modules.shared.opts.cuda_compile_backend = "openvino_fx" if modules.shared.opts.sd_backend != "diffusers": - modules.shared.log.warn("OpenVINO: Setting backend to Diffusers") + modules.shared.log.warning("OpenVINO: Setting backend to Diffusers") modules.shared.opts.sd_backend = "diffusers" try: opts.save(modules.shared.config_filename) diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index e9e8df7f9..e3120b0eb 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -343,7 +343,7 @@ def create_ui(container, button, tabname, skip_indexing = False): ui.description_target_filename = gr.Textbox('Description save filename', elem_id=tabname+"_description_filename", visible=False) for page in ui.stored_extra_pages: - shared.log.debug(f"Create UI Extra Network Page: {page.title}") + shared.log.debug(f"Extra network page: {page.title} tab={tabname}") page_html = page.create_html(ui.tabname, skip_indexing) with gr.Tab(page.title, id=page.title.lower().replace(" ", "_"), elem_classes="extra-networks-tab"): page_elem = gr.HTML(page_html, elem_id=tabname+page.name+"_extra_page", elem_classes="extra-networks-page")