diff --git a/CHANGELOG.md b/CHANGELOG.md index 385e6710b..54286e2a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,6 @@ OPTIONAL: - pipeline `diffusers_sag_scale` [pr](https://github.com/huggingface/diffusers/issues/6443) - wuerstchen v3 [pr](https://github.com/huggingface/diffusers/pull/6487) - adetailer [fix](https://github.com/Bing-su/adetailer/issues/466) -- rebasin [issue](https://github.com/vladmandic/automatic/issues/2608) - control api - photomaker api - interrogate api @@ -19,7 +18,7 @@ OPTIONAL: - masking api - preprocess api -## Update for 2023-01-18 +## Update for 2023-01-19 Another big release, highlights being: - A lot more functionality in the **Control** module: @@ -128,6 +127,8 @@ And it also includes fixes for all reported issues so far - **extra networks** - 4x faster civitai metadata and previews lookup - better display and selection of tags & trigger words + if hashes are calculated, trigger words will only be displayed for actual model version + - better matching of previews - better search, including searching for multiple keywords or using full regex see wiki page for more details on syntax thanks @NetroScript @@ -192,6 +193,7 @@ And it also includes fixes for all reported issues so far - **fixes** - ipadapter: allow changing of model/image on-the-fly - ipadapter: fix fallback of cross-attention on unload + - rebasin iterations, thanks @AI-Casanova - prompt scheduler, thanks @AI-Casanova - python: fix python 3.9 compatibility - sdxl: fix positive prompt embeds diff --git a/extensions-builtin/Lora/network.py b/extensions-builtin/Lora/network.py index d0f3ebb7a..d5cb6444f 100644 --- a/extensions-builtin/Lora/network.py +++ b/extensions-builtin/Lora/network.py @@ -32,9 +32,8 @@ class NetworkOnDisk: m[k] = v self.metadata = m self.alias = self.metadata.get('ss_output_name', self.name) - self.hash = None - self.shorthash = None - self.set_hash(self.metadata.get('sshs_model_hash') or hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or '') + # self.set_hash(self.metadata.get('sshs_model_hash') or hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or '') + self.set_hash(hashes.sha256_from_cache(self.filename, "lora/" + self.name) or self.metadata.get('sshs_model_hash')) self.sd_version = self.detect_version() def detect_version(self): @@ -47,8 +46,8 @@ class NetworkOnDisk: return SdVersion.Unknown def set_hash(self, v): - self.hash = v - self.shorthash = self.hash[0:12] + self.hash = v or '' + self.shorthash = self.hash[0:8] def read_hash(self): if not self.hash: diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index effa20911..0d7fb2a1c 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -464,8 +464,9 @@ def list_available_networks(): except OSError as e: # should catch FileNotFoundError and PermissionError etc. shared.log.error(f"Failed to load network {name} from {filename} {e}") + candidates = list(files_cache.list_files(*directories, ext_filter=[".pt", ".ckpt", ".safetensors"])) with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: - for fn in files_cache.list_files(*directories, ext_filter=[".pt", ".ckpt", ".safetensors"]): + for fn in candidates: executor.submit(add_network, fn) shared.log.info(f'LoRA networks: available={len(available_networks)} folders={len(forbidden_network_aliases)}') diff --git a/extensions-builtin/Lora/ui_extra_networks_lora.py b/extensions-builtin/Lora/ui_extra_networks_lora.py index 265fdbca3..2ce4a0cad 100644 --- a/extensions-builtin/Lora/ui_extra_networks_lora.py +++ b/extensions-builtin/Lora/ui_extra_networks_lora.py @@ -17,7 +17,7 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): def create_item(self, name): l = networks.available_networks.get(name) try: - path, _ext = os.path.splitext(l.filename) + # path, _ext = os.path.splitext(l.filename) name = os.path.splitext(os.path.relpath(l.filename, shared.cmd_opts.lora_dir))[0] if shared.backend == shared.Backend.ORIGINAL: if l.sd_version == network.SdVersion.SDXL: @@ -37,9 +37,7 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): "name": name, "filename": l.filename, "hash": l.shorthash, - "preview": self.find_preview(l.filename), "prompt": json.dumps(f" "), - "local_preview": f"{path}.{shared.opts.samples_format}", "metadata": json.dumps(l.metadata, indent=4) if l.metadata else None, "mtime": os.path.getmtime(l.filename), "size": os.path.getsize(l.filename), @@ -58,7 +56,21 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): tag = ' '.join(words[1:]).lower() tags[tag] = words[0] - for v in info.get('modelVersions', []): # trigger words from info json + + def find_version(): + found_versions = [] + current_hash = l.hash[:8].upper() + all_versions = info.get('modelVersions', []) + for v in info.get('modelVersions', []): + for f in v.get('files', []): + if any(h.startswith(current_hash) for h in f.get('hashes', {}).values()): + found_versions.append(v) + if len(found_versions) == 0: + found_versions = all_versions + return found_versions + + find_version() + for v in find_version(): # trigger words from info json possible_tags = v.get('trainedWords', []) if isinstance(possible_tags, list): for tag_str in possible_tags: @@ -96,12 +108,15 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): return None def list_items(self): + items = [] with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: future_items = {executor.submit(self.create_item, net): net for net in networks.available_networks} for future in concurrent.futures.as_completed(future_items): item = future.result() if item is not None: - yield item + items.append(item) + self.update_all_previews(items) + return items def allowed_directories_for_previews(self): return [shared.cmd_opts.lora_dir, shared.cmd_opts.lyco_dir] diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index ba05e1ea2..7b731d157 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit ba05e1ea20df0e29a3dce9fcbe3ecc30777a5271 +Subproject commit 7b731d15776217eababd636461a6b3a25470670b diff --git a/modules/files_cache.py b/modules/files_cache.py index d6b3f3940..778bf9cbf 100644 --- a/modules/files_cache.py +++ b/modules/files_cache.py @@ -1,41 +1,29 @@ import itertools -import os.path as path +import os from collections import UserDict from dataclasses import dataclass, field -from os import scandir from typing import Callable, Dict, Iterator, List, Optional, Union +from installer import print_dict, log -from installer import print_dict -class Directory: +class Directory: # forward declaration ... -WasDirty = bool -DidDelete = bool -IsDirectory = bool -DirectoryExists = bool -IsDirectory = bool -IsDirty = bool -CachedDirectoryIsStale = bool -MTime = float -IsHidden = bool -FilePath = str -FilePathList = List[FilePath] -FilePathIterator = Iterator[FilePath] -DirectoryPath = str -DirectoryPathList = List[DirectoryPath] -DirectoryPathIterator = Iterator[DirectoryPath] +FilePathList = List[str] +FilePathIterator = Iterator[str] +DirectoryPathList = List[str] +DirectoryPathIterator = Iterator[str] DirectoryList = List[Directory] DirectoryIterator = Iterator[Directory] -DirectoryCollection = Dict[DirectoryPath, Directory] +DirectoryCollection = Dict[str, Directory] ExtensionFilter = Callable ExtensionList = list[str] RecursiveType = Union[bool,Callable] -def real_path(directory_path:DirectoryPath) -> Union[DirectoryPath, None]: +def real_path(directory_path:str) -> Union[str, None]: try: - return path.abspath(path.expanduser(directory_path)) + return os.path.abspath(os.path.expanduser(directory_path)) except Exception: pass return None @@ -43,7 +31,7 @@ def real_path(directory_path:DirectoryPath) -> Union[DirectoryPath, None]: @dataclass(frozen=True) class Directory(Directory): # pylint: disable=E0102 - path: DirectoryPath = field(default_factory=str) + path: str = field(default_factory=str) mtime: float = field(default_factory=float, init=False) files: FilePathList = field(default_factory=list) directories: DirectoryPathList = field(default_factory=list) @@ -86,22 +74,24 @@ class Directory(Directory): # pylint: disable=E0102 return str(print_dict(self, path=self.path, mtime=self.mtime, files=len(self.files), directories=len(self.directories))) # pylint: disable=unexpected-keyword-arg @property - def exists(self) -> DirectoryExists: - return self.path and path.exists(self.path) + def exists(self) -> bool: + return self.path and os.path.exists(self.path) @property - def is_directory(self) -> IsDirectory: - return self.exists and path.isdir(self.path) + def is_directory(self) -> bool: + return self.exists and os.path.isdir(self.path) @property - def live_mtime(self) -> MTime: - return path.getmtime(self.path) if self.is_directory else 0 + def live_mtime(self) -> float: + return os.path.getmtime(self.path) if self.is_directory else 0 @property - def is_stale(self) -> CachedDirectoryIsStale: + def is_stale(self) -> bool: return not self.is_directory or self.mtime != self.live_mtime + + class DirectoryCache(UserDict, DirectoryCollection): def __delattr__(self, directory_path: str) -> None: directory: Directory = get_directory(directory_path, fetch=False) @@ -139,13 +129,12 @@ def clean_directory(directory: Directory, /, recursive: RecursiveType=False) -> return is_clean -def get_directory(directory_or_path: DirectoryPath, /, fetch:bool=True) -> Union[Directory, None]: +def get_directory(directory_or_path: str, /, fetch:bool=True) -> Union[Directory, None]: if isinstance(directory_or_path, Directory): if directory_or_path.is_directory: return directory_or_path else: directory_or_path = directory_or_path.path - global cache_folders # pylint: disable=W0602 directory_or_path = real_path(directory_or_path) if not cache_folders.get(directory_or_path, None): if fetch: @@ -157,91 +146,72 @@ def get_directory(directory_or_path: DirectoryPath, /, fetch:bool=True) -> Union return cache_folders[directory_or_path] if directory_or_path in cache_folders else None -def fetch_directory(directory_path: DirectoryPath) -> Union[Directory, None]: +def fetch_directory(directory_path: str) -> Union[Directory, None]: directory: Directory - for directory in _walk(directory_path, lambda e, path: delete_cached_directory(path), recurse=False): + for directory in _walk(directory_path, recurse=False): return directory # The return is intentional, we get a generator, we only need the one return None -def _walk(top, onerror:Callable=None, /, recurse:RecursiveType=True) -> Directory: - # A near-exact copy of `path.walk()`, trimmed slightly. Probably not nessesary for most people's collections, but makes a difference on really large datasets. +def _walk(top, recurse:RecursiveType=True) -> Directory: + # reimplemented `path.walk()` nondirs = [] walk_dirs = [] try: - scandir_it = scandir(top) - except OSError as error: - if callable(onerror): - onerror(error, top) + scandir_it = os.scandir(top) + entry = next(scandir_it) + except OSError: return with scandir_it: - while True: - try: - try: - entry = next(scandir_it) - except StopIteration: - break - except OSError as error: - if callable(onerror): - onerror(error, top) - return - try: - is_dir = entry.is_dir() - except OSError: - is_dir = False - if not is_dir: + while entry: + if not entry.is_dir(): nondirs.append(entry.path) else: - try: - if entry.is_symlink() and not path.exists(entry.path): - raise NotADirectoryError('Broken Symlink') + if entry.is_symlink() and not os.path.exists(entry.path): + log.error(f'Files broken symlink: {entry.path}') + else: walk_dirs.append(entry.path) - except OSError as error: - if callable(onerror): - onerror(error, entry.path) + try: + entry = next(scandir_it) + except Exception: + entry = None yield Directory(top, nondirs, walk_dirs) if recurse: - # Recurse into sub-directories for new_path in walk_dirs: - if path.basename(new_path).startswith('models--'): - continue if callable(recurse) and not recurse(new_path): continue - yield from _walk(new_path, onerror, recurse=recurse) + yield from _walk(new_path, recurse=recurse) -def _cached_walk(top, onerror:Callable=None, /, recurse:RecursiveType=True) -> Directory: +def _cached_walk(top, recurse:RecursiveType=True) -> Directory: top = get_directory(top) if not top: return yield top if recurse: for child_directory in top.directories: - if path.basename(child_directory).startswith('models--'): + if os.path.basename(child_directory).startswith('models--'): continue if callable(recurse) and not recurse(child_directory): continue - yield from _cached_walk(child_directory, onerror, recurse=recurse) + yield from _cached_walk(child_directory, recurse=recurse) -def walk(top, onerror:Callable=None, /, recurse:RecursiveType=True, cached=True) -> Directory: - if cached: - yield from _cached_walk(top, onerror, recurse=recurse) - else: - yield from _walk(top, onerror, recurse=recurse) +def walk(top, recurse:RecursiveType=True, cached=True) -> Directory: + yield from _cached_walk(top, recurse=recurse) if cached else _walk(top, recurse=recurse) -def delete_cached_directory(directory_path:DirectoryPath) -> DidDelete: +def delete_cached_directory(directory_path:str) -> bool: global cache_folders # pylint: disable=W0602 if directory_path in cache_folders: del cache_folders[directory_path] -def is_directory(dir_path:DirectoryPath) -> IsDirectory: - return dir_path and path.exists(dir_path) and path.isdir(dir_path) +def is_directory(dir_path:str) -> bool: + return dir_path and os.path.exists(dir_path) and os.path.isdir(dir_path) -def directory_mtime(directory_path:DirectoryPath, /, recursive:RecursiveType=True) -> MTime: +def directory_mtime(directory_path:str, /, recursive:RecursiveType=True) -> float: return float(max(0, *[directory.mtime for directory in get_directories(directory_path, recursive=recursive)])) @@ -255,7 +225,7 @@ def unique_directories(directories:DirectoryPathList, /, recursive:RecursiveType yield directory if not recursive: continue - _directory = path.join(directory, '') + _directory = os.path.join(directory, '') child_directory = None while directories and directories[-1].startswith(_directory): if not callable(recursive) or not child_directory: @@ -267,10 +237,10 @@ def unique_directories(directories:DirectoryPathList, /, recursive:RecursiveType if not callable(recursive): _remove_directory = next_directory else: - for sub_directory in child_directory.split(path.sep): - next_directory = path.join(next_directory, sub_directory) + for sub_directory in child_directory.split(os.path.sep): + next_directory = os.path.join(next_directory, sub_directory) if recursive(next_directory): - _remove_directory = path.join(next_directory, '') + _remove_directory = os.path.join(next_directory, '') break while _remove_directory and directories: _d = directories.pop() @@ -319,8 +289,8 @@ def extension_filter(ext_filter: Optional[ExtensionList]=None, ext_blacklist: Op return filter_functon -def not_hidden(filepath: FilePath) -> IsHidden: - return not path.basename(filepath).startswith('.') +def not_hidden(filepath: str) -> bool: + return not os.path.basename(filepath).startswith('.') def filter_files(file_paths: FilePathList, ext_filter: Optional[ExtensionList]=None, ext_blacklist: Optional[ExtensionList]=None) -> FilePathIterator: @@ -330,8 +300,7 @@ def filter_files(file_paths: FilePathList, ext_filter: Optional[ExtensionList]=N def list_files(*directory_paths:DirectoryPathList, ext_filter: Optional[ExtensionList]=None, ext_blacklist: Optional[ExtensionList]=None, recursive:RecursiveType=True) -> FilePathIterator: return filter_files(itertools.chain.from_iterable( directory_files(directory, recursive=recursive) - for directory - in get_directories(*directory_paths, recursive=recursive) + for directory in get_directories(*directory_paths, recursive=recursive) ), ext_filter, ext_blacklist) diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index ba6c790f8..4a634c96e 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -286,11 +286,7 @@ def list_hypernetworks(path): hypernetworks = { os.path.splitext(os.path.basename(hypernetwork_path))[0]: hypernetwork_path for hypernetwork_path - in files_cache.list_files( - path, - ext_filter=['.pt'], - recursive=files_cache.not_hidden - ) + in files_cache.list_files(path, ext_filter=['.pt'], recursive=files_cache.not_hidden) } return hypernetworks diff --git a/modules/modelloader.py b/modules/modelloader.py index c9d5f0fc6..f426f67d7 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -6,10 +6,9 @@ from typing import Dict from urllib.parse import urlparse from PIL import Image import rich.progress as p -from modules import shared, errors +from modules import shared, errors, files_cache from modules.upscaler import Upscaler, UpscalerLanczos, UpscalerNearest, UpscalerNone from modules.paths import script_path, models_path -from modules.files_cache import list_files, unique_directories diffuser_repos = [] @@ -389,10 +388,10 @@ def load_models(model_path: str, model_url: str = None, command_path: str = None @param ext_filter: An optional list of filename extensions to filter by @return: A list of paths containing the desired model(s) """ - places = [model_path, command_path] + places = list(set([model_path, command_path])) output = [] try: - output:list = [*list_files(*places, ext_filter=ext_filter, ext_blacklist=ext_blacklist)] + output:list = [*files_cache.list_files(*places, ext_filter=ext_filter, ext_blacklist=ext_blacklist)] if model_url is not None and len(output) == 0: if download_name is not None: dl = load_file_from_url(model_url, model_dir=places[0], progress=True, file_name=download_name) @@ -400,7 +399,7 @@ def load_models(model_path: str, model_url: str = None, command_path: str = None else: output.append(model_url) except Exception as e: - errors.display(e,f"Error listing models: {unique_directories(places)}") + errors.display(e,f"Error listing models: {files_cache.unique_directories(places)}") return output diff --git a/modules/styles.py b/modules/styles.py index 58cf3e2a5..dc6eac4c3 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -6,6 +6,7 @@ import csv import json import time from installer import log +from modules import files_cache class Style(): @@ -125,9 +126,9 @@ class StyleDatabase: def list_folder(folder): import concurrent future_items = {} + candidates = list(files_cache.list_files(folder, ext_filter=['.json'], recursive=files_cache.not_hidden)) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - for filename in os.listdir(folder): - fn = os.path.abspath(os.path.join(folder, filename)) + for fn in candidates: if os.path.isfile(fn) and fn.lower().endswith(".json"): future_items[executor.submit(self.load_style, fn, None)] = fn # self.load_style(fn) @@ -220,18 +221,3 @@ class StyleDatabase: except Exception: log.error(f'Styles error: file="{legacy_file}" row={row}') log.info(f'Load legacy styles: file="{legacy_file}" loaded={num} created={len(list(self.styles))}') - - """ - def save_csv(self, path: str) -> None: - import tempfile - basedir = os.path.dirname(path) - if basedir is not None and len(basedir) > 0: - os.makedirs(basedir, exist_ok=True) - fd, temp_path = tempfile.mkstemp(".csv") - with os.fdopen(fd, "w", encoding="utf-8-sig", newline='') as file: - writer = csv.DictWriter(file, fieldnames=Style._fields) - writer.writeheader() - writer.writerows(style._asdict() for k, style in self.styles.items()) - log.debug(f'Saved legacy styles: {path} {len(self.styles.keys())}') - shutil.move(temp_path, path) - """ diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index ea714d656..b52a4c887 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -119,6 +119,7 @@ class ExtraNetworksPage: self.list_time = 0 self.info_time = 0 self.desc_time = 0 + self.preview_time = 0 self.dirs = {} self.view = shared.opts.extra_networks_view self.card = card_full if shared.opts.extra_networks_view == 'gallery' else card_list @@ -210,7 +211,7 @@ class ExtraNetworksPage: if skip: return f"
Extra network page not ready
Click refresh to try again
" subdirs = {} - allowed_folders = [os.path.abspath(x) for x in self.allowed_directories_for_previews()] + allowed_folders = [os.path.abspath(x) for x in self.allowed_directories_for_previews() if os.path.exists(x)] for parentdir, dirs in {d: files_cache.walk(d, cached=True, recurse=files_cache.not_hidden) for d in allowed_folders}.items(): for tgt in dirs: tgt = tgt.path @@ -252,7 +253,7 @@ class ExtraNetworksPage: self.html = f"
{subdirs_html}
{self.html}
" else: return '' - shared.log.debug(f"Extra networks: page='{self.name}' items={len(self.items)} subfolders={len(subdirs)} tab={tabname} folders={self.allowed_directories_for_previews()} list={self.list_time:.2f} desc={self.desc_time:.2f} info={self.info_time:.2f} workers={shared.max_workers}") + shared.log.debug(f"Extra networks: page='{self.name}' items={len(self.items)} subfolders={len(subdirs)} tab={tabname} folders={self.allowed_directories_for_previews()} list={self.list_time:.2f} thumb={self.preview_time:.2f} desc={self.desc_time:.2f} info={self.info_time:.2f} workers={shared.max_workers}") if len(self.missing_thumbs) > 0: threading.Thread(target=self.create_thumb).start() return self.html @@ -292,19 +293,19 @@ class ExtraNetworksPage: return "" def find_preview_file(self, path): - exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2"] if path is None: return 'html/card-no-preview.png' if os.path.join('models', 'Reference') in path: return path + exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2"] if shared.opts.diffusers_dir in path: path = os.path.relpath(path, shared.opts.diffusers_dir) reference_path = os.path.abspath(os.path.join('models', 'Reference')) fn = os.path.join(reference_path, path.replace('models--', '').replace('\\', '/').split('/')[0]) files = list(files_cache.list_files(reference_path, ext_filter=exts, recursive=False)) else: - files = list(files_cache.list_files(os.path.dirname(path), ext_filter=exts, recursive=False)) fn = os.path.splitext(path)[0] + files = list(files_cache.list_files(os.path.dirname(path), ext_filter=exts, recursive=False)) for file in [f'{fn}{mid}{ext}' for ext in exts for mid in ['.thumb.', '.', '.preview.']]: if file in files: if '.thumb.' not in file: @@ -312,10 +313,41 @@ class ExtraNetworksPage: return file return 'html/card-no-preview.png' - def find_preview(self, path): - preview_file = self.find_preview_file(path) + def find_preview(self, filename): + t0 = time.time() + preview_file = self.find_preview_file(filename) + self.preview_time += time.time() - t0 return self.link_preview(preview_file) + def update_all_previews(self, items): + t0 = time.time() + reference_path = os.path.abspath(os.path.join('models', 'Reference')) + possible_paths = list(set([os.path.dirname(item['filename']) for item in items] + [reference_path])) + exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2"] + all_previews = list(files_cache.list_files(*possible_paths, ext_filter=exts, recursive=False)) + for item in items: + if item.get('preview', None) is not None: + continue + base = os.path.splitext(item['filename'])[0] + if item.get('local_preview', None) is None: + item['local_preview'] = f'{base}.{shared.opts.samples_format}' + if shared.opts.diffusers_dir in base: + match = re.search(r"models--([^/]+)/", base) + base = os.path.join(reference_path, match[1]) + model_path = os.path.join(shared.opts.diffusers_dir, match[0]) + item['local_preview'] = f'{os.path.join(model_path, match[1])}.{shared.opts.samples_format}' + all_previews += list(files_cache.list_files(model_path, ext_filter=exts, recursive=False)) + for file in [f'{base}{mid}{ext}' for ext in exts for mid in ['.thumb.', '.', '.preview.']]: + if file in all_previews: + if '.thumb.' not in file: + self.missing_thumbs.append(file) + item['preview'] = self.link_preview(file) + break + if item.get('preview', None) is None: + item['preview'] = self.link_preview('html/card-no-preview.png') + self.preview_time += time.time() - t0 + + def find_description(self, path, info=None): t0 = time.time() class HTMLFilter(HTMLParser): diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index 612fdb3ee..be8729c28 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -52,8 +52,6 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): "title": checkpoint.title, "filename": checkpoint.filename, "hash": checkpoint.shorthash, - "preview": self.find_preview(checkpoint.filename), - "local_preview": f"{os.path.splitext(checkpoint.filename)[0]}.{shared.opts.samples_format}", "metadata": checkpoint.metadata, "onclick": '"' + html.escape(f"""return selectCheckpoint({json.dumps(name)})""") + '"', "mtime": os.path.getmtime(checkpoint.filename) if exists else 0, @@ -66,14 +64,18 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): return record def list_items(self): + # items = [self.create_item(cp) for cp in list(sd_models.checkpoints_list)] + list(self.list_reference()) + items = [] with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: future_items = {executor.submit(self.create_item, cp): cp for cp in list(sd_models.checkpoints_list.copy())} for future in concurrent.futures.as_completed(future_items): item = future.result() if item is not None: - yield item + items.append(item) for record in self.list_reference(): - yield record + items.append(record) + self.update_all_previews(items) + return items def allowed_directories_for_previews(self): if shared.backend == shared.Backend.DIFFUSERS: diff --git a/modules/ui_extra_networks_styles.py b/modules/ui_extra_networks_styles.py index d7532adc5..859fba2cd 100644 --- a/modules/ui_extra_networks_styles.py +++ b/modules/ui_extra_networks_styles.py @@ -1,7 +1,6 @@ import os import html import json -import concurrent from modules import shared, extra_networks, ui_extra_networks, styles @@ -78,7 +77,7 @@ class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage): "name": name, "title": k, "filename": style.filename, - "preview": style.preview if getattr(style, 'preview', None) is not None and style.preview.startswith('data:') else self.find_preview(fn), + "preview": style.preview if getattr(style, 'preview', None) is not None and style.preview.startswith('data:') else None, "description": style.description if getattr(style, 'description', None) is not None and len(style.description) > 0 else txt, "prompt": getattr(style, 'prompt', ''), "negative": getattr(style, 'negative_prompt', ''), @@ -93,12 +92,9 @@ class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage): return item def list_items(self): - with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: - future_items = {executor.submit(self.create_item, style): style for style in list(shared.prompt_styles.styles)} - for future in concurrent.futures.as_completed(future_items): - item = future.result() - if item is not None: - yield item + items = [self.create_item(k) for k in list(shared.prompt_styles.styles)] + self.update_all_previews(items) + return items def allowed_directories_for_previews(self): return [v for v in [shared.opts.styles_dir] if v is not None] + ['html'] diff --git a/modules/ui_extra_networks_textual_inversion.py b/modules/ui_extra_networks_textual_inversion.py index 4eadfd34f..1859274da 100644 --- a/modules/ui_extra_networks_textual_inversion.py +++ b/modules/ui_extra_networks_textual_inversion.py @@ -1,6 +1,5 @@ import json import os -import concurrent from modules import shared, sd_hijack, sd_models, ui_extra_networks, files_cache from modules.textual_inversion.textual_inversion import Embedding @@ -22,7 +21,6 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage): def create_item(self, embedding: Embedding): record = None try: - path, _ext = os.path.splitext(embedding.filename) tags = {} if embedding.tag is not None: tags[embedding.tag]=1 @@ -31,9 +29,7 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage): "type": 'Embedding', "name": name, "filename": embedding.filename, - "preview": self.find_preview(embedding.filename), "prompt": json.dumps(f" {os.path.splitext(embedding.name)[0]}"), - "local_preview": f"{path}.{shared.opts.samples_format}", "tags": tags, "mtime": os.path.getmtime(embedding.filename), "size": os.path.getsize(embedding.filename), @@ -46,14 +42,11 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage): def list_items(self): if sd_models.model_data.sd_model is None: + candidates = list(files_cache.list_files(shared.opts.embeddings_dir, ext_filter=['.pt', '.safetensors'], recursive=files_cache.not_hidden)) self.embeddings = [ Embedding(vec=0, name=os.path.basename(embedding_path), filename=embedding_path) for embedding_path - in files_cache.list_files( - shared.opts.embeddings_dir, - ext_filter=['.pt', '.safetensors'], - recursive=files_cache.not_hidden - ) + in candidates ] elif shared.backend == shared.Backend.ORIGINAL: self.embeddings = list(sd_hijack.model_hijack.embedding_db.word_embeddings.values()) @@ -63,12 +56,9 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage): self.embeddings = [] self.embeddings = sorted(self.embeddings, key=lambda emb: emb.filename) - with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: - future_items = {executor.submit(self.create_item, net): net for net in self.embeddings} - for future in concurrent.futures.as_completed(future_items): - item = future.result() - if item is not None: - yield item + items = [self.create_item(embedding) for embedding in self.embeddings] + self.update_all_previews(items) + return items def allowed_directories_for_previews(self): return list(sd_hijack.model_hijack.embedding_db.embedding_dirs)