diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d00698dc..129db54a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,8 @@ And it also includes fixes for all reported issues so far - lazy load optional imports - batch embedding load, thanks @midcoastal 10x+ faster embeddings load for large number of embeddings, now works for 1000+ embeddings + - file and folder list caching, thanks @midcoastal + if you have a lot of files and and/or are using slower or non-local storage, this speeds up file access a lot - **extra networks** - 4x faster civitai metadata and previews lookup - better display and selection of tags & trigger words diff --git a/modules/extensions.py b/modules/extensions.py index 1d6f287e0..5a8a53d29 100644 --- a/modules/extensions.py +++ b/modules/extensions.py @@ -1,13 +1,11 @@ import os from datetime import datetime import git -from modules import shared, errors, files_cache +from modules import shared, errors from modules.paths import extensions_dir, extensions_builtin_dir extensions = [] - - if not os.path.exists(extensions_dir): os.makedirs(extensions_dir) diff --git a/modules/files_cache.py b/modules/files_cache.py index 00721a6b5..ef422bf9a 100644 --- a/modules/files_cache.py +++ b/modules/files_cache.py @@ -7,6 +7,9 @@ from typing import Callable, Dict, Iterator, List, Optional, Union from installer import print_dict +class Directory: + ... + WasDirty = bool DidDelete = bool IsDirectory = bool @@ -16,25 +19,17 @@ IsDirty = bool CachedDirectoryIsStale = bool MTime = float IsHidden = bool - FilePath = str FilePathList = List[FilePath] FilePathIterator = Iterator[FilePath] - DirectoryPath = str DirectoryPathList = List[DirectoryPath] DirectoryPathIterator = Iterator[DirectoryPath] - -class Directory: - ... - DirectoryList = List[Directory] DirectoryIterator = Iterator[Directory] DirectoryCollection = Dict[DirectoryPath, Directory] - ExtensionFilter = Callable ExtensionList = list[str] - RecursiveType = Union[bool,Callable] @@ -48,18 +43,14 @@ def real_path(directory_path:DirectoryPath) -> DirectoryPath | None: @dataclass(slots=True,frozen=True) class Directory(Directory): # pylint: disable=E0102 - - path: DirectoryPath = field(default_factory=str) mtime: float = field(default_factory=float, init=False) files: FilePathList = field(default_factory=list) directories: DirectoryPathList = field(default_factory=list) - def __post_init__(self): object.__setattr__(self, 'mtime', self.live_mtime) - @classmethod def from_dict(cls, dict_object: dict) -> Directory: directory = cls.__new__(cls) @@ -69,7 +60,6 @@ class Directory(Directory): # pylint: disable=E0102 object.__setattr__(directory, 'directories', dict_object.get('directories')) return directory - def clear(self) -> None: self._update(Directory.from_dict({ 'path': None, @@ -78,13 +68,11 @@ class Directory(Directory): # pylint: disable=E0102 'directories': [] })) - def update(self, source_directory: Directory) -> Directory: if source_directory is not self: self._update(source_directory) return self - def _update(self, source:Directory) -> None: assert not source.path or source.path == self.path, f'When updating a directory, the paths must match. Attemped to update Directory `{self.path}` with `{source.path}`' for dead_path in self.directories: @@ -94,26 +82,21 @@ class Directory(Directory): # pylint: disable=E0102 self.files[:] = source.files object.__setattr__(self, 'mtime', source.mtime) - def __str__(self) -> str: - return str(print_dict(self, path=self.path, mtime=self.mtime, files=len(self.files), directories=len(self.directories))) - + return str(print_dict(self, path=self.path, mtime=self.mtime, files=len(self.files), directories=len(self.directories))) # pylint: disable=unexpected-keyword-arg @property def exists(self) -> DirectoryExists: return self.path and path.exists(self.path) - @property def is_directory(self) -> IsDirectory: return self.exists and path.isdir(self.path) - @property def live_mtime(self) -> MTime: return path.getmtime(self.path) if self.is_directory else 0 - @property def is_stale(self) -> CachedDirectoryIsStale: return not self.is_directory or self.mtime != self.live_mtime @@ -240,6 +223,7 @@ def _cached_walk(top, onerror:Callable=None, /, recurse:RecursiveType=True) -> D continue yield from _cached_walk(child_directory, onerror, recurse=recurse) + def walk(top, onerror:Callable=None, /, recurse:RecursiveType=True, cached=True) -> Directory: if cached: yield from _cached_walk(top, onerror, recurse=recurse) @@ -295,31 +279,14 @@ def unique_directories(directories:DirectoryPathList, /, recursive:RecursiveType def unique_paths(directory_paths:DirectoryPathList) -> DirectoryPathIterator: - realpaths = ( - real_path(directory_path) - for directory_path - in filter(bool, directory_paths) - ) - return { - real_directory_path: True - for real_directory_path - in filter(bool, realpaths) - }.keys() + realpaths = (real_path(directory_path) for directory_path in filter(bool, directory_paths)) + return {real_directory_path: True for real_directory_path in filter(bool, realpaths)}.keys() def get_directories(*directory_paths: DirectoryPathList, fetch:bool=True, recursive:RecursiveType=True) -> DirectoryCollection: - directory_paths = unique_directories( - directory_paths, recursive=recursive - ) - directories = ( - get_directory(directory_path, fetch=fetch) - for directory_path - in directory_paths - ) - return filter( - bool, - directories - ) + directory_paths = unique_directories(directory_paths, recursive=recursive) + directories = (get_directory(directory_path, fetch=fetch) for directory_path in directory_paths) + return filter(bool, directories) def directory_files(*directories_or_paths: DirectoryPathList|DirectoryList, recursive: RecursiveType=True) -> FilePathIterator: @@ -333,28 +300,12 @@ def directory_files(*directories_or_paths: DirectoryPathList|DirectoryList, recu for directory in filter( bool, - map( - get_directory, - filter( - ( - ( bool if recursive else False ) - if not callable(recursive) - else recursive - ), - directory_object.directories - ) - ) + map(get_directory, filter(((bool if recursive else False) if not callable(recursive) else recursive), directory_object.directories)) ) ) ) for directory_object - in filter( - bool, - map( - get_directory, - directories_or_paths - ) - ) + in filter(bool, map(get_directory, directories_or_paths)) ) @@ -380,9 +331,7 @@ def list_files(*directory_paths:DirectoryPathList, ext_filter: Optional[Extensio return filter_files(itertools.chain.from_iterable( directory_files(directory, recursive=recursive) for directory - in get_directories( - *directory_paths, recursive=recursive - ) + in get_directories(*directory_paths, recursive=recursive) ), ext_filter, ext_blacklist) diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index 7352f3bc4..ba6c790f8 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -19,6 +19,7 @@ from modules.textual_inversion.learn_schedule import LearnRateScheduler optimizer_dict = {optim_name : cls_obj for optim_name, cls_obj in inspect.getmembers(torch.optim, inspect.isclass) if optim_name != "Optimizer"} + class HypernetworkModule(torch.nn.Module): activation_dict = { "linear": torch.nn.Identity, diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index 81fbb26d5..c00f187fe 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -15,7 +15,6 @@ import modules.textual_inversion.loaders from modules.textual_inversion.learn_schedule import LearnRateScheduler from modules.textual_inversion.image_embedding import embedding_to_b64, embedding_from_b64, insert_image_data_embed, extract_image_data_embed, caption_image_overlay from modules.textual_inversion.ti_logging import save_settings_to_file -from typing import List, Optional, Union from modules.files_cache import directory_files, directory_mtime, extension_filter diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index bc6e16217..ea714d656 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -1,10 +1,10 @@ +import os import io import re import time import json import html import base64 -import os.path import urllib.parse import threading from datetime import datetime @@ -155,12 +155,11 @@ class ExtraNetworksPage: debug(f'EN create-thumb: {self.name}') created = 0 for f in self.missing_thumbs: - if not os.path.exists(f): + if os.path.join('models', 'Reference') in f or not os.path.exists(f): continue - fn, _ext = os.path.splitext(f) - fn = fn.replace('.preview', '') + fn = os.path.splitext(f)[0].replace('.preview', '') fn = f'{fn}.thumb.jpg' - if os.path.exists(fn): + if os.path.exists(fn): # thumbnail already exists continue img = None try: @@ -296,17 +295,19 @@ class ExtraNetworksPage: exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2"] if path is None: return 'html/card-no-preview.png' + if os.path.join('models', 'Reference') in path: + return path if shared.opts.diffusers_dir in path: path = os.path.relpath(path, shared.opts.diffusers_dir) - ref = os.path.join('models', 'Reference') - fn = os.path.join(ref, path.replace('models--', '').replace('\\', '/').split('/')[0]) - files = list(files_cache.list_files(ref, ext_filter=exts, recursive=False)) + reference_path = os.path.abspath(os.path.join('models', 'Reference')) + fn = os.path.join(reference_path, path.replace('models--', '').replace('\\', '/').split('/')[0]) + files = list(files_cache.list_files(reference_path, ext_filter=exts, recursive=False)) else: files = list(files_cache.list_files(os.path.dirname(path), ext_filter=exts, recursive=False)) fn = os.path.splitext(path)[0] for file in [f'{fn}{mid}{ext}' for ext in exts for mid in ['.thumb.', '.', '.preview.']]: if file in files: - if 'Reference' not in file and '.thumb.' not in file: + if '.thumb.' not in file: self.missing_thumbs.append(file) return file return 'html/card-no-preview.png'