Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 17:24:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix find preview

Browse Source
This commit is contained in:
Vladimir Mandic
2024-01-17 11:44:52 -05:00
parent f3ed66c3ef
commit f10a478c82
6 changed files with 27 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+2
View File
@@ -104,6 +104,8 @@ And it also includes fixes for all reported issues so far
- lazy load optional imports
- batch embedding load, thanks @midcoastal
10x+ faster embeddings load for large number of embeddings, now works for 1000+ embeddings
- file and folder list caching, thanks @midcoastal
if you have a lot of files and and/or are using slower or non-local storage, this speeds up file access a lot
- **extra networks**
- 4x faster civitai metadata and previews lookup
- better display and selection of tags & trigger words
modules/extensions.py
+1 -3
View File
@@ -1,13 +1,11 @@
import os
from datetime import datetime
import git
from modules import shared, errors, files_cache
from modules import shared, errors
from modules.paths import extensions_dir, extensions_builtin_dir
extensions = []
if not os.path.exists(extensions_dir):
os.makedirs(extensions_dir)
modules/files_cache.py
+13 -64
View File
@@ -7,6 +7,9 @@ from typing import Callable, Dict, Iterator, List, Optional, Union
from installer import print_dict
class Directory:
...
WasDirty = bool
DidDelete = bool
IsDirectory = bool
@@ -16,25 +19,17 @@ IsDirty = bool
CachedDirectoryIsStale = bool
MTime = float
IsHidden = bool
FilePath = str
FilePathList = List[FilePath]
FilePathIterator = Iterator[FilePath]
DirectoryPath = str
DirectoryPathList = List[DirectoryPath]
DirectoryPathIterator = Iterator[DirectoryPath]
class Directory:
...
DirectoryList = List[Directory]
DirectoryIterator = Iterator[Directory]
DirectoryCollection = Dict[DirectoryPath, Directory]
ExtensionFilter = Callable
ExtensionList = list[str]
RecursiveType = Union[bool,Callable]
@@ -48,18 +43,14 @@ def real_path(directory_path:DirectoryPath) -> DirectoryPath | None:
@dataclass(slots=True,frozen=True)
class Directory(Directory): # pylint: disable=E0102
path: DirectoryPath = field(default_factory=str)
mtime: float = field(default_factory=float, init=False)
files: FilePathList = field(default_factory=list)
directories: DirectoryPathList = field(default_factory=list)
def __post_init__(self):
object.__setattr__(self, 'mtime', self.live_mtime)
@classmethod
def from_dict(cls, dict_object: dict) -> Directory:
directory = cls.__new__(cls)
@@ -69,7 +60,6 @@ class Directory(Directory): # pylint: disable=E0102
object.__setattr__(directory, 'directories', dict_object.get('directories'))
return directory
def clear(self) -> None:
self._update(Directory.from_dict({
'path': None,
@@ -78,13 +68,11 @@ class Directory(Directory): # pylint: disable=E0102
'directories': []
}))
def update(self, source_directory: Directory) -> Directory:
if source_directory is not self:
self._update(source_directory)
return self
def _update(self, source:Directory) -> None:
assert not source.path or source.path == self.path, f'When updating a directory, the paths must match. Attemped to update Directory `{self.path}` with `{source.path}`'
for dead_path in self.directories:
@@ -94,26 +82,21 @@ class Directory(Directory): # pylint: disable=E0102
self.files[:] = source.files
object.__setattr__(self, 'mtime', source.mtime)
def __str__(self) -> str:
return str(print_dict(self, path=self.path, mtime=self.mtime, files=len(self.files), directories=len(self.directories)))
return str(print_dict(self, path=self.path, mtime=self.mtime, files=len(self.files), directories=len(self.directories))) # pylint: disable=unexpected-keyword-arg
@property
def exists(self) -> DirectoryExists:
return self.path and path.exists(self.path)
@property
def is_directory(self) -> IsDirectory:
return self.exists and path.isdir(self.path)
@property
def live_mtime(self) -> MTime:
return path.getmtime(self.path) if self.is_directory else 0
@property
def is_stale(self) -> CachedDirectoryIsStale:
return not self.is_directory or self.mtime != self.live_mtime
@@ -240,6 +223,7 @@ def _cached_walk(top, onerror:Callable=None, /, recurse:RecursiveType=True) -> D
continue
yield from _cached_walk(child_directory, onerror, recurse=recurse)
def walk(top, onerror:Callable=None, /, recurse:RecursiveType=True, cached=True) -> Directory:
if cached:
yield from _cached_walk(top, onerror, recurse=recurse)
@@ -295,31 +279,14 @@ def unique_directories(directories:DirectoryPathList, /, recursive:RecursiveType
def unique_paths(directory_paths:DirectoryPathList) -> DirectoryPathIterator:
realpaths = (
real_path(directory_path)
for directory_path
in filter(bool, directory_paths)
)
return {
real_directory_path: True
for real_directory_path
in filter(bool, realpaths)
}.keys()
realpaths = (real_path(directory_path) for directory_path in filter(bool, directory_paths))
return {real_directory_path: True for real_directory_path in filter(bool, realpaths)}.keys()
def get_directories(*directory_paths: DirectoryPathList, fetch:bool=True, recursive:RecursiveType=True) -> DirectoryCollection:
directory_paths = unique_directories(
directory_paths, recursive=recursive
)
directories = (
get_directory(directory_path, fetch=fetch)
for directory_path
in directory_paths
)
return filter(
bool,
directories
)
directory_paths = unique_directories(directory_paths, recursive=recursive)
directories = (get_directory(directory_path, fetch=fetch) for directory_path in directory_paths)
return filter(bool, directories)
def directory_files(*directories_or_paths: DirectoryPathList|DirectoryList, recursive: RecursiveType=True) -> FilePathIterator:
@@ -333,28 +300,12 @@ def directory_files(*directories_or_paths: DirectoryPathList|DirectoryList, recu
for directory
in filter(
bool,
map(
get_directory,
filter(
(
( bool if recursive else False )
if not callable(recursive)
else recursive
),
directory_object.directories
)
)
map(get_directory, filter(((bool if recursive else False) if not callable(recursive) else recursive), directory_object.directories))
)
)
)
for directory_object
in filter(
bool,
map(
get_directory,
directories_or_paths
)
)
in filter(bool, map(get_directory, directories_or_paths))
)
@@ -380,9 +331,7 @@ def list_files(*directory_paths:DirectoryPathList, ext_filter: Optional[Extensio
return filter_files(itertools.chain.from_iterable(
directory_files(directory, recursive=recursive)
for directory
in get_directories(
*directory_paths, recursive=recursive
)
in get_directories(*directory_paths, recursive=recursive)
), ext_filter, ext_blacklist)
modules/hypernetworks/hypernetwork.py
+1
View File
@@ -19,6 +19,7 @@ from modules.textual_inversion.learn_schedule import LearnRateScheduler
optimizer_dict = {optim_name : cls_obj for optim_name, cls_obj in inspect.getmembers(torch.optim, inspect.isclass) if optim_name != "Optimizer"}
class HypernetworkModule(torch.nn.Module):
activation_dict = {
"linear": torch.nn.Identity,
modules/textual_inversion/textual_inversion.py
-1
View File
@@ -15,7 +15,6 @@ import modules.textual_inversion.loaders
from modules.textual_inversion.learn_schedule import LearnRateScheduler
from modules.textual_inversion.image_embedding import embedding_to_b64, embedding_from_b64, insert_image_data_embed, extract_image_data_embed, caption_image_overlay
from modules.textual_inversion.ti_logging import save_settings_to_file
from typing import List, Optional, Union
from modules.files_cache import directory_files, directory_mtime, extension_filter
modules/ui_extra_networks.py
+10 -9
View File
@@ -1,10 +1,10 @@
import os
import io
import re
import time
import json
import html
import base64
import os.path
import urllib.parse
import threading
from datetime import datetime
@@ -155,12 +155,11 @@ class ExtraNetworksPage:
debug(f'EN create-thumb: {self.name}')
created = 0
for f in self.missing_thumbs:
if not os.path.exists(f):
if os.path.join('models', 'Reference') in f or not os.path.exists(f):
continue
fn, _ext = os.path.splitext(f)
fn = fn.replace('.preview', '')
fn = os.path.splitext(f)[0].replace('.preview', '')
fn = f'{fn}.thumb.jpg'
if os.path.exists(fn):
if os.path.exists(fn): # thumbnail already exists
continue
img = None
try:
@@ -296,17 +295,19 @@ class ExtraNetworksPage:
exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2"]
if path is None:
return 'html/card-no-preview.png'
if os.path.join('models', 'Reference') in path:
return path
if shared.opts.diffusers_dir in path:
path = os.path.relpath(path, shared.opts.diffusers_dir)
ref = os.path.join('models', 'Reference')
fn = os.path.join(ref, path.replace('models--', '').replace('\\', '/').split('/')[0])
files = list(files_cache.list_files(ref, ext_filter=exts, recursive=False))
reference_path = os.path.abspath(os.path.join('models', 'Reference'))
fn = os.path.join(reference_path, path.replace('models--', '').replace('\\', '/').split('/')[0])
files = list(files_cache.list_files(reference_path, ext_filter=exts, recursive=False))
else:
files = list(files_cache.list_files(os.path.dirname(path), ext_filter=exts, recursive=False))
fn = os.path.splitext(path)[0]
for file in [f'{fn}{mid}{ext}' for ext in exts for mid in ['.thumb.', '.', '.preview.']]:
if file in files:
if 'Reference' not in file and '.thumb.' not in file:
if '.thumb.' not in file:
self.missing_thumbs.append(file)
return file
return 'html/card-no-preview.png'