better en preview matching and optimize caching

CHANGELOG.md

@@ -11,7 +11,6 @@ OPTIONAL:
 - pipeline `diffusers_sag_scale` [pr](https://github.com/huggingface/diffusers/issues/6443)
 - wuerstchen v3 [pr](https://github.com/huggingface/diffusers/pull/6487)
 - adetailer [fix](https://github.com/Bing-su/adetailer/issues/466)
-- rebasin [issue](https://github.com/vladmandic/automatic/issues/2608)
 - control api
 - photomaker api
 - interrogate api
@@ -19,7 +18,7 @@ OPTIONAL:
 - masking api
 - preprocess api
 
-## Update for 2023-01-18
+## Update for 2023-01-19
 
 Another big release, highlights being:  
 - A lot more functionality in the **Control** module:
@@ -128,6 +127,8 @@ And it also includes fixes for all reported issues so far
   - **extra networks**  
     - 4x faster civitai metadata and previews lookup  
     - better display and selection of tags & trigger words  
+      if hashes are calculated, trigger words will only be displayed for actual model version  
+    - better matching of previews  
     - better search, including searching for multiple keywords or using full regex  
       see wiki page for more details on syntax  
       thanks @NetroScript  
@@ -192,6 +193,7 @@ And it also includes fixes for all reported issues so far
 - **fixes**  
   - ipadapter: allow changing of model/image on-the-fly  
   - ipadapter: fix fallback of cross-attention on unload  
+  - rebasin iterations, thanks @AI-Casanova
   - prompt scheduler, thanks @AI-Casanova
   - python: fix python 3.9 compatibility  
   - sdxl: fix positive prompt embeds

extensions-builtin/Lora/network.py

@@ -32,9 +32,8 @@ class NetworkOnDisk:
                 m[k] = v
             self.metadata = m
         self.alias = self.metadata.get('ss_output_name', self.name)
-        self.hash = None
-        self.shorthash = None
-        self.set_hash(self.metadata.get('sshs_model_hash') or hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or '')
+        # self.set_hash(self.metadata.get('sshs_model_hash') or hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or '')
+        self.set_hash(hashes.sha256_from_cache(self.filename, "lora/" + self.name) or self.metadata.get('sshs_model_hash'))
         self.sd_version = self.detect_version()
 
     def detect_version(self):
@@ -47,8 +46,8 @@ class NetworkOnDisk:
         return SdVersion.Unknown
 
     def set_hash(self, v):
-        self.hash = v
-        self.shorthash = self.hash[0:12]
+        self.hash = v or ''
+        self.shorthash = self.hash[0:8]
 
     def read_hash(self):
         if not self.hash:

extensions-builtin/Lora/networks.py

@@ -464,8 +464,9 @@ def list_available_networks():
         except OSError as e:  # should catch FileNotFoundError and PermissionError etc.
             shared.log.error(f"Failed to load network {name} from {filename} {e}")
 
+    candidates = list(files_cache.list_files(*directories, ext_filter=[".pt", ".ckpt", ".safetensors"]))
     with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor:
-        for fn in files_cache.list_files(*directories, ext_filter=[".pt", ".ckpt", ".safetensors"]):
+        for fn in candidates:
             executor.submit(add_network, fn)
     shared.log.info(f'LoRA networks: available={len(available_networks)} folders={len(forbidden_network_aliases)}')
 

extensions-builtin/Lora/ui_extra_networks_lora.py

@@ -17,7 +17,7 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
     def create_item(self, name):
         l = networks.available_networks.get(name)
         try:
-            path, _ext = os.path.splitext(l.filename)
+            # path, _ext = os.path.splitext(l.filename)
             name = os.path.splitext(os.path.relpath(l.filename, shared.cmd_opts.lora_dir))[0]
             if shared.backend == shared.Backend.ORIGINAL:
                 if l.sd_version == network.SdVersion.SDXL:
@@ -37,9 +37,7 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
                 "name": name,
                 "filename": l.filename,
                 "hash": l.shorthash,
-                "preview": self.find_preview(l.filename),
                 "prompt": json.dumps(f" <lora:{l.get_alias()}:{shared.opts.extra_networks_default_multiplier}>"),
-                "local_preview": f"{path}.{shared.opts.samples_format}",
                 "metadata": json.dumps(l.metadata, indent=4) if l.metadata else None,
                 "mtime": os.path.getmtime(l.filename),
                 "size": os.path.getsize(l.filename),
@@ -58,7 +56,21 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
                 tag = ' '.join(words[1:]).lower()
                 tags[tag] = words[0]
 
-            for v in info.get('modelVersions', []):  # trigger words from info json
+
+            def find_version():
+                found_versions = []
+                current_hash = l.hash[:8].upper()
+                all_versions = info.get('modelVersions', [])
+                for v in info.get('modelVersions', []):
+                    for f in v.get('files', []):
+                        if any(h.startswith(current_hash) for h in f.get('hashes', {}).values()):
+                            found_versions.append(v)
+                if len(found_versions) == 0:
+                    found_versions = all_versions
+                return found_versions
+
+            find_version()
+            for v in find_version():  # trigger words from info json
                 possible_tags = v.get('trainedWords', [])
                 if isinstance(possible_tags, list):
                     for tag_str in possible_tags:
@@ -96,12 +108,15 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
             return None
 
     def list_items(self):
+        items = []
         with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor:
             future_items = {executor.submit(self.create_item, net): net for net in networks.available_networks}
             for future in concurrent.futures.as_completed(future_items):
                 item = future.result()
                 if item is not None:
-                    yield item
+                    items.append(item)
+        self.update_all_previews(items)
+        return items
 
     def allowed_directories_for_previews(self):
         return [shared.cmd_opts.lora_dir, shared.cmd_opts.lyco_dir]

modules/files_cache.py

@@ -1,41 +1,29 @@
 import itertools
-import os.path as path
+import os
 from collections import UserDict
 from dataclasses import dataclass, field
-from os import scandir
 from typing import Callable, Dict, Iterator, List, Optional, Union
+from installer import print_dict, log
 
-from installer import print_dict
 
-class Directory:
+class Directory: # forward declaration
     ...
 
-WasDirty = bool
-DidDelete = bool
-IsDirectory = bool
-DirectoryExists = bool
-IsDirectory = bool
-IsDirty = bool
-CachedDirectoryIsStale = bool
-MTime = float
-IsHidden = bool
-FilePath = str
-FilePathList = List[FilePath]
-FilePathIterator = Iterator[FilePath]
-DirectoryPath = str
-DirectoryPathList = List[DirectoryPath]
-DirectoryPathIterator = Iterator[DirectoryPath]
+FilePathList = List[str]
+FilePathIterator = Iterator[str]
+DirectoryPathList = List[str]
+DirectoryPathIterator = Iterator[str]
 DirectoryList = List[Directory]
 DirectoryIterator = Iterator[Directory]
-DirectoryCollection = Dict[DirectoryPath, Directory]
+DirectoryCollection = Dict[str, Directory]
 ExtensionFilter = Callable
 ExtensionList = list[str]
 RecursiveType = Union[bool,Callable]
 
 
-def real_path(directory_path:DirectoryPath) -> Union[DirectoryPath, None]:
+def real_path(directory_path:str) -> Union[str, None]:
     try:
-        return path.abspath(path.expanduser(directory_path))
+        return os.path.abspath(os.path.expanduser(directory_path))
     except Exception:
         pass
     return None
@@ -43,7 +31,7 @@ def real_path(directory_path:DirectoryPath) -> Union[DirectoryPath, None]:
 
 @dataclass(frozen=True)
 class Directory(Directory): # pylint: disable=E0102
-    path: DirectoryPath = field(default_factory=str)
+    path: str = field(default_factory=str)
     mtime: float = field(default_factory=float, init=False)
     files: FilePathList = field(default_factory=list)
     directories: DirectoryPathList = field(default_factory=list)
@@ -86,22 +74,24 @@ class Directory(Directory): # pylint: disable=E0102
         return str(print_dict(self, path=self.path, mtime=self.mtime, files=len(self.files), directories=len(self.directories))) # pylint: disable=unexpected-keyword-arg
 
     @property
-    def exists(self) -> DirectoryExists:
-        return self.path and path.exists(self.path)
+    def exists(self) -> bool:
+        return self.path and os.path.exists(self.path)
 
     @property
-    def is_directory(self) -> IsDirectory:
-        return self.exists and path.isdir(self.path)
+    def is_directory(self) -> bool:
+        return self.exists and os.path.isdir(self.path)
 
     @property
-    def live_mtime(self) -> MTime:
-        return path.getmtime(self.path) if self.is_directory else 0
+    def live_mtime(self) -> float:
+        return os.path.getmtime(self.path) if self.is_directory else 0
 
     @property
-    def is_stale(self) -> CachedDirectoryIsStale:
+    def is_stale(self) -> bool:
         return not self.is_directory or self.mtime != self.live_mtime
 
 
+
+
 class DirectoryCache(UserDict, DirectoryCollection):
     def __delattr__(self, directory_path: str) -> None:
         directory: Directory = get_directory(directory_path, fetch=False)
@@ -139,13 +129,12 @@ def clean_directory(directory: Directory, /, recursive: RecursiveType=False) ->
     return is_clean
 
 
-def get_directory(directory_or_path: DirectoryPath, /, fetch:bool=True) -> Union[Directory, None]:
+def get_directory(directory_or_path: str, /, fetch:bool=True) -> Union[Directory, None]:
     if isinstance(directory_or_path, Directory):
         if directory_or_path.is_directory:
             return directory_or_path
         else:
             directory_or_path = directory_or_path.path
-    global cache_folders # pylint: disable=W0602
     directory_or_path = real_path(directory_or_path)
     if not cache_folders.get(directory_or_path, None):
         if fetch:
@@ -157,91 +146,72 @@ def get_directory(directory_or_path: DirectoryPath, /, fetch:bool=True) -> Union
     return cache_folders[directory_or_path] if directory_or_path in cache_folders else None
 
 
-def fetch_directory(directory_path: DirectoryPath) -> Union[Directory, None]:
+def fetch_directory(directory_path: str) -> Union[Directory, None]:
     directory: Directory
-    for directory in _walk(directory_path, lambda e, path: delete_cached_directory(path), recurse=False):
+    for directory in _walk(directory_path, recurse=False):
         return directory # The return is intentional, we get a generator, we only need the one
     return None
 
 
-def _walk(top, onerror:Callable=None, /, recurse:RecursiveType=True) -> Directory:
-    # A near-exact copy of `path.walk()`, trimmed slightly. Probably not nessesary for most people's collections, but makes a difference on really large datasets.
+def _walk(top, recurse:RecursiveType=True) -> Directory:
+    # reimplemented `path.walk()`
     nondirs = []
     walk_dirs = []
     try:
-        scandir_it = scandir(top)
-    except OSError as error:
-        if callable(onerror):
-            onerror(error, top)
+        scandir_it = os.scandir(top)
+        entry = next(scandir_it)
+    except OSError:
         return
     with scandir_it:
-        while True:
-            try:
-                try:
-                    entry = next(scandir_it)
-                except StopIteration:
-                    break
-            except OSError as error:
-                if callable(onerror):
-                    onerror(error, top)
-                return
-            try:
-                is_dir = entry.is_dir()
-            except OSError:
-                is_dir = False
-            if not is_dir:
+        while entry:
+            if not entry.is_dir():
                 nondirs.append(entry.path)
             else:
-                try:
-                    if entry.is_symlink() and not path.exists(entry.path):
-                        raise NotADirectoryError('Broken Symlink')
+                if entry.is_symlink() and not os.path.exists(entry.path):
+                    log.error(f'Files broken symlink: {entry.path}')
+                else:
                     walk_dirs.append(entry.path)
-                except OSError as error:
-                    if callable(onerror):
-                        onerror(error, entry.path)
+            try:
+                entry = next(scandir_it)
+            except Exception:
+                entry = None
     yield Directory(top, nondirs, walk_dirs)
     if recurse:
-        # Recurse into sub-directories
         for new_path in walk_dirs:
-            if path.basename(new_path).startswith('models--'):
-                continue
             if callable(recurse) and not recurse(new_path):
                 continue
-            yield from _walk(new_path, onerror, recurse=recurse)
+            yield from _walk(new_path, recurse=recurse)
 
 
-def _cached_walk(top, onerror:Callable=None, /, recurse:RecursiveType=True) -> Directory:
+def _cached_walk(top, recurse:RecursiveType=True) -> Directory:
     top = get_directory(top)
     if not top:
         return
     yield top
     if recurse:
         for child_directory in top.directories:
-            if path.basename(child_directory).startswith('models--'):
+            if os.path.basename(child_directory).startswith('models--'):
                 continue
             if callable(recurse) and not recurse(child_directory):
                 continue
-            yield from _cached_walk(child_directory, onerror, recurse=recurse)
+            yield from _cached_walk(child_directory, recurse=recurse)
 
 
-def walk(top, onerror:Callable=None, /, recurse:RecursiveType=True, cached=True) -> Directory:
-    if cached:
-        yield from _cached_walk(top, onerror, recurse=recurse)
-    else:
-        yield from _walk(top, onerror, recurse=recurse)
+def walk(top, recurse:RecursiveType=True, cached=True) -> Directory:
+    yield from _cached_walk(top, recurse=recurse) if cached else _walk(top, recurse=recurse)
 
 
-def delete_cached_directory(directory_path:DirectoryPath) -> DidDelete:
+def delete_cached_directory(directory_path:str) -> bool:
     global cache_folders # pylint: disable=W0602
     if directory_path in cache_folders:
         del cache_folders[directory_path]
 
 
-def is_directory(dir_path:DirectoryPath) -> IsDirectory:
-    return dir_path and path.exists(dir_path) and path.isdir(dir_path)
+def is_directory(dir_path:str) -> bool:
+    return dir_path and os.path.exists(dir_path) and os.path.isdir(dir_path)
 
 
-def directory_mtime(directory_path:DirectoryPath, /, recursive:RecursiveType=True) -> MTime:
+def directory_mtime(directory_path:str, /, recursive:RecursiveType=True) -> float:
     return float(max(0, *[directory.mtime for directory in get_directories(directory_path, recursive=recursive)]))
 
 
@@ -255,7 +225,7 @@ def unique_directories(directories:DirectoryPathList, /, recursive:RecursiveType
         yield directory
         if not recursive:
             continue
-        _directory = path.join(directory, '')
+        _directory = os.path.join(directory, '')
         child_directory = None
         while directories and directories[-1].startswith(_directory):
             if not callable(recursive) or not child_directory:
@@ -267,10 +237,10 @@ def unique_directories(directories:DirectoryPathList, /, recursive:RecursiveType
                 if not callable(recursive):
                     _remove_directory = next_directory
                 else:
-                    for sub_directory in child_directory.split(path.sep):
-                        next_directory = path.join(next_directory, sub_directory)
+                    for sub_directory in child_directory.split(os.path.sep):
+                        next_directory = os.path.join(next_directory, sub_directory)
                         if recursive(next_directory):
-                            _remove_directory = path.join(next_directory, '')
+                            _remove_directory = os.path.join(next_directory, '')
                             break
                 while _remove_directory and directories:
                     _d = directories.pop()
@@ -319,8 +289,8 @@ def extension_filter(ext_filter: Optional[ExtensionList]=None, ext_blacklist: Op
     return filter_functon
 
 
-def not_hidden(filepath: FilePath) -> IsHidden:
-    return not path.basename(filepath).startswith('.')
+def not_hidden(filepath: str) -> bool:
+    return not os.path.basename(filepath).startswith('.')
 
 
 def filter_files(file_paths: FilePathList, ext_filter: Optional[ExtensionList]=None, ext_blacklist: Optional[ExtensionList]=None) -> FilePathIterator:
@@ -330,8 +300,7 @@ def filter_files(file_paths: FilePathList, ext_filter: Optional[ExtensionList]=N
 def list_files(*directory_paths:DirectoryPathList, ext_filter: Optional[ExtensionList]=None, ext_blacklist: Optional[ExtensionList]=None, recursive:RecursiveType=True) -> FilePathIterator:
     return filter_files(itertools.chain.from_iterable(
         directory_files(directory, recursive=recursive)
-        for directory
-        in get_directories(*directory_paths, recursive=recursive)
+        for directory in get_directories(*directory_paths, recursive=recursive)
     ), ext_filter, ext_blacklist)
 
 

modules/hypernetworks/hypernetwork.py

@@ -286,11 +286,7 @@ def list_hypernetworks(path):
     hypernetworks = {
         os.path.splitext(os.path.basename(hypernetwork_path))[0]: hypernetwork_path
         for hypernetwork_path
-        in files_cache.list_files(
-            path,
-            ext_filter=['.pt'],
-            recursive=files_cache.not_hidden
-        )
+        in files_cache.list_files(path, ext_filter=['.pt'], recursive=files_cache.not_hidden)
     }
     return hypernetworks
 

modules/modelloader.py

@@ -6,10 +6,9 @@ from typing import Dict
 from urllib.parse import urlparse
 from PIL import Image
 import rich.progress as p
-from modules import shared, errors
+from modules import shared, errors, files_cache
 from modules.upscaler import Upscaler, UpscalerLanczos, UpscalerNearest, UpscalerNone
 from modules.paths import script_path, models_path
-from modules.files_cache import list_files, unique_directories
 
 
 diffuser_repos = []
@@ -389,10 +388,10 @@ def load_models(model_path: str, model_url: str = None, command_path: str = None
     @param ext_filter: An optional list of filename extensions to filter by
     @return: A list of paths containing the desired model(s)
     """
-    places = [model_path, command_path]
+    places = list(set([model_path, command_path]))
     output = []
     try:
-        output:list = [*list_files(*places, ext_filter=ext_filter, ext_blacklist=ext_blacklist)]
+        output:list = [*files_cache.list_files(*places, ext_filter=ext_filter, ext_blacklist=ext_blacklist)]
         if model_url is not None and len(output) == 0:
             if download_name is not None:
                 dl = load_file_from_url(model_url, model_dir=places[0], progress=True, file_name=download_name)
@@ -400,7 +399,7 @@ def load_models(model_path: str, model_url: str = None, command_path: str = None
             else:
                 output.append(model_url)
     except Exception as e:
-        errors.display(e,f"Error listing models: {unique_directories(places)}")
+        errors.display(e,f"Error listing models: {files_cache.unique_directories(places)}")
     return output
 
 

modules/styles.py

@@ -6,6 +6,7 @@ import csv
 import json
 import time
 from installer import log
+from modules import files_cache
 
 
 class Style():
@@ -125,9 +126,9 @@ class StyleDatabase:
         def list_folder(folder):
             import concurrent
             future_items = {}
+            candidates = list(files_cache.list_files(folder, ext_filter=['.json'], recursive=files_cache.not_hidden))
             with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-                for filename in os.listdir(folder):
-                    fn = os.path.abspath(os.path.join(folder, filename))
+                for fn in candidates:
                     if os.path.isfile(fn) and fn.lower().endswith(".json"):
                         future_items[executor.submit(self.load_style, fn, None)] = fn
                         # self.load_style(fn)
@@ -220,18 +221,3 @@ class StyleDatabase:
                 except Exception:
                     log.error(f'Styles error: file="{legacy_file}" row={row}')
             log.info(f'Load legacy styles: file="{legacy_file}" loaded={num} created={len(list(self.styles))}')
-
-    """
-    def save_csv(self, path: str) -> None:
-        import tempfile
-        basedir = os.path.dirname(path)
-        if basedir is not None and len(basedir) > 0:
-            os.makedirs(basedir, exist_ok=True)
-        fd, temp_path = tempfile.mkstemp(".csv")
-        with os.fdopen(fd, "w", encoding="utf-8-sig", newline='') as file:
-            writer = csv.DictWriter(file, fieldnames=Style._fields)
-            writer.writeheader()
-            writer.writerows(style._asdict() for k, style in self.styles.items())
-            log.debug(f'Saved legacy styles: {path} {len(self.styles.keys())}')
-        shutil.move(temp_path, path)
-    """

modules/ui_extra_networks.py

@@ -119,6 +119,7 @@ class ExtraNetworksPage:
         self.list_time = 0
         self.info_time = 0
         self.desc_time = 0
+        self.preview_time = 0
         self.dirs = {}
         self.view = shared.opts.extra_networks_view
         self.card = card_full if shared.opts.extra_networks_view == 'gallery' else card_list
@@ -210,7 +211,7 @@ class ExtraNetworksPage:
         if skip:
             return f"<div id='{tabname}_{self_name_id}_subdirs' class='extra-network-subdirs'></div><div id='{tabname}_{self_name_id}_cards' class='extra-network-cards'>Extra network page not ready<br>Click refresh to try again</div>"
         subdirs = {}
-        allowed_folders = [os.path.abspath(x) for x in self.allowed_directories_for_previews()]
+        allowed_folders = [os.path.abspath(x) for x in self.allowed_directories_for_previews() if os.path.exists(x)]
         for parentdir, dirs in {d: files_cache.walk(d, cached=True, recurse=files_cache.not_hidden) for d in allowed_folders}.items():
             for tgt in dirs:
                 tgt = tgt.path
@@ -252,7 +253,7 @@ class ExtraNetworksPage:
             self.html = f"<div id='{tabname}_{self_name_id}_subdirs' class='extra-network-subdirs'>{subdirs_html}</div><div id='{tabname}_{self_name_id}_cards' class='extra-network-cards'>{self.html}</div>"
         else:
             return ''
-        shared.log.debug(f"Extra networks: page='{self.name}' items={len(self.items)} subfolders={len(subdirs)} tab={tabname} folders={self.allowed_directories_for_previews()} list={self.list_time:.2f} desc={self.desc_time:.2f} info={self.info_time:.2f} workers={shared.max_workers}")
+        shared.log.debug(f"Extra networks: page='{self.name}' items={len(self.items)} subfolders={len(subdirs)} tab={tabname} folders={self.allowed_directories_for_previews()} list={self.list_time:.2f} thumb={self.preview_time:.2f} desc={self.desc_time:.2f} info={self.info_time:.2f} workers={shared.max_workers}")
         if len(self.missing_thumbs) > 0:
             threading.Thread(target=self.create_thumb).start()
         return self.html
@@ -292,19 +293,19 @@ class ExtraNetworksPage:
             return ""
 
     def find_preview_file(self, path):
-        exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2"]
         if path is None:
             return 'html/card-no-preview.png'
         if os.path.join('models', 'Reference') in path:
             return path
+        exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2"]
         if shared.opts.diffusers_dir in path:
             path = os.path.relpath(path, shared.opts.diffusers_dir)
             reference_path = os.path.abspath(os.path.join('models', 'Reference'))
             fn = os.path.join(reference_path, path.replace('models--', '').replace('\\', '/').split('/')[0])
             files = list(files_cache.list_files(reference_path, ext_filter=exts, recursive=False))
         else:
-            files = list(files_cache.list_files(os.path.dirname(path), ext_filter=exts, recursive=False))
             fn = os.path.splitext(path)[0]
+            files = list(files_cache.list_files(os.path.dirname(path), ext_filter=exts, recursive=False))
         for file in [f'{fn}{mid}{ext}' for ext in exts for mid in ['.thumb.', '.', '.preview.']]:
             if file in files:
                 if '.thumb.' not in file:
@@ -312,10 +313,41 @@ class ExtraNetworksPage:
                 return file
         return 'html/card-no-preview.png'
 
-    def find_preview(self, path):
-        preview_file = self.find_preview_file(path)
+    def find_preview(self, filename):
+        t0 = time.time()
+        preview_file = self.find_preview_file(filename)
+        self.preview_time += time.time() - t0
         return self.link_preview(preview_file)
 
+    def update_all_previews(self, items):
+        t0 = time.time()
+        reference_path = os.path.abspath(os.path.join('models', 'Reference'))
+        possible_paths = list(set([os.path.dirname(item['filename']) for item in items] + [reference_path]))
+        exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2"]
+        all_previews = list(files_cache.list_files(*possible_paths, ext_filter=exts, recursive=False))
+        for item in items:
+            if item.get('preview', None) is not None:
+                continue
+            base = os.path.splitext(item['filename'])[0]
+            if item.get('local_preview', None) is None:
+                item['local_preview'] = f'{base}.{shared.opts.samples_format}'
+            if shared.opts.diffusers_dir in base:
+                match = re.search(r"models--([^/]+)/", base)
+                base = os.path.join(reference_path, match[1])
+                model_path = os.path.join(shared.opts.diffusers_dir, match[0])
+                item['local_preview'] = f'{os.path.join(model_path, match[1])}.{shared.opts.samples_format}'
+                all_previews += list(files_cache.list_files(model_path, ext_filter=exts, recursive=False))
+            for file in [f'{base}{mid}{ext}' for ext in exts for mid in ['.thumb.', '.', '.preview.']]:
+                if file in all_previews:
+                    if '.thumb.' not in file:
+                        self.missing_thumbs.append(file)
+                    item['preview'] = self.link_preview(file)
+                    break
+            if item.get('preview', None) is None:
+                item['preview'] = self.link_preview('html/card-no-preview.png')
+        self.preview_time += time.time() - t0
+
+
     def find_description(self, path, info=None):
         t0 = time.time()
         class HTMLFilter(HTMLParser):

modules/ui_extra_networks_checkpoints.py

@@ -52,8 +52,6 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage):
                 "title": checkpoint.title,
                 "filename": checkpoint.filename,
                 "hash": checkpoint.shorthash,
-                "preview": self.find_preview(checkpoint.filename),
-                "local_preview": f"{os.path.splitext(checkpoint.filename)[0]}.{shared.opts.samples_format}",
                 "metadata": checkpoint.metadata,
                 "onclick": '"' + html.escape(f"""return selectCheckpoint({json.dumps(name)})""") + '"',
                 "mtime": os.path.getmtime(checkpoint.filename) if exists else 0,
@@ -66,14 +64,18 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage):
         return record
 
     def list_items(self):
+        # items = [self.create_item(cp) for cp in list(sd_models.checkpoints_list)] + list(self.list_reference())
+        items = []
         with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor:
             future_items = {executor.submit(self.create_item, cp): cp for cp in list(sd_models.checkpoints_list.copy())}
             for future in concurrent.futures.as_completed(future_items):
                 item = future.result()
                 if item is not None:
-                    yield item
+                    items.append(item)
         for record in self.list_reference():
-            yield record
+            items.append(record)
+        self.update_all_previews(items)
+        return items
 
     def allowed_directories_for_previews(self):
         if shared.backend == shared.Backend.DIFFUSERS:

modules/ui_extra_networks_styles.py

@@ -1,7 +1,6 @@
 import os
 import html
 import json
-import concurrent
 from modules import shared, extra_networks, ui_extra_networks, styles
 
 
@@ -78,7 +77,7 @@ class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage):
                 "name": name,
                 "title": k,
                 "filename": style.filename,
-                "preview": style.preview if getattr(style, 'preview', None) is not None and style.preview.startswith('data:') else self.find_preview(fn),
+                "preview": style.preview if getattr(style, 'preview', None) is not None and style.preview.startswith('data:') else None,
                 "description": style.description if getattr(style, 'description', None) is not None and len(style.description) > 0 else txt,
                 "prompt": getattr(style, 'prompt', ''),
                 "negative": getattr(style, 'negative_prompt', ''),
@@ -93,12 +92,9 @@ class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage):
         return item
 
     def list_items(self):
-        with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor:
-            future_items = {executor.submit(self.create_item, style): style for style in list(shared.prompt_styles.styles)}
-            for future in concurrent.futures.as_completed(future_items):
-                item = future.result()
-                if item is not None:
-                    yield item
+        items = [self.create_item(k) for k in list(shared.prompt_styles.styles)]
+        self.update_all_previews(items)
+        return items
 
     def allowed_directories_for_previews(self):
         return [v for v in [shared.opts.styles_dir] if v is not None] + ['html']

modules/ui_extra_networks_textual_inversion.py

@@ -1,6 +1,5 @@
 import json
 import os
-import concurrent
 from modules import shared, sd_hijack, sd_models, ui_extra_networks, files_cache
 from modules.textual_inversion.textual_inversion import Embedding
 
@@ -22,7 +21,6 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage):
     def create_item(self, embedding: Embedding):
         record = None
         try:
-            path, _ext = os.path.splitext(embedding.filename)
             tags = {}
             if embedding.tag is not None:
                 tags[embedding.tag]=1
@@ -31,9 +29,7 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage):
                 "type": 'Embedding',
                 "name": name,
                 "filename": embedding.filename,
-                "preview": self.find_preview(embedding.filename),
                 "prompt": json.dumps(f" {os.path.splitext(embedding.name)[0]}"),
-                "local_preview": f"{path}.{shared.opts.samples_format}",
                 "tags": tags,
                 "mtime": os.path.getmtime(embedding.filename),
                 "size": os.path.getsize(embedding.filename),
@@ -46,14 +42,11 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage):
 
     def list_items(self):
         if sd_models.model_data.sd_model is None:
+            candidates = list(files_cache.list_files(shared.opts.embeddings_dir, ext_filter=['.pt', '.safetensors'], recursive=files_cache.not_hidden))
             self.embeddings = [
                 Embedding(vec=0, name=os.path.basename(embedding_path), filename=embedding_path)
                 for embedding_path
-                in files_cache.list_files(
-                    shared.opts.embeddings_dir,
-                    ext_filter=['.pt', '.safetensors'],
-                    recursive=files_cache.not_hidden
-                )
+                in candidates
             ]
         elif shared.backend == shared.Backend.ORIGINAL:
             self.embeddings = list(sd_hijack.model_hijack.embedding_db.word_embeddings.values())
@@ -63,12 +56,9 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage):
             self.embeddings = []
         self.embeddings = sorted(self.embeddings, key=lambda emb: emb.filename)
 
-        with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor:
-            future_items = {executor.submit(self.create_item, net): net for net in self.embeddings}
-            for future in concurrent.futures.as_completed(future_items):
-                item = future.result()
-                if item is not None:
-                    yield item
+        items = [self.create_item(embedding) for embedding in self.embeddings]
+        self.update_all_previews(items)
+        return items
 
     def allowed_directories_for_previews(self):
         return list(sd_hijack.model_hijack.embedding_db.embedding_dirs)