From de80b74c64c16b18506ab0bd6b6b53f41dbd8262 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 6 Jan 2024 15:15:23 -0500
Subject: [PATCH] uniform listdir and use threadpool to index loras

---
 CHANGELOG.md                                  |  5 ++--
 extensions-builtin/Lora/network.py            | 21 +--------------
 extensions-builtin/Lora/networks.py           | 27 +++++++++++--------
 modules/control/units/controlnet.py           |  4 +--
 modules/control/units/lite.py                 |  4 +--
 modules/control/units/xs.py                   |  4 +--
 modules/hypernetworks/hypernetwork.py         |  2 +-
 modules/images.py                             |  2 +-
 modules/img2img.py                            |  4 +--
 modules/modelloader.py                        |  6 ++---
 modules/postprocessing.py                     |  2 +-
 modules/script_loading.py                     |  8 ++++--
 modules/sd_hijack_hypertile.py                |  4 +--
 modules/shared.py                             | 15 ++++++++---
 modules/textual_inversion/preprocess.py       | 24 +----------------
 modules/ui_extra_networks.py                  | 25 ++++-------------
 .../ui_extra_networks_textual_inversion.py    |  2 +-
 modules/ui_img2img.py                         |  2 +-
 modules/ui_interrogate.py                     |  2 +-
 modules/upscaler.py                           | 15 +----------
 20 files changed, 64 insertions(+), 114 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b178ab845..9b0202491 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -44,8 +44,9 @@ And it also includes fixes for all reported issues so far
   - **SDXL**: Base SXDL, Base ViT-H SXDL, Plus ViT-H SXDL, Plus Face ViT-H SXDL  
 - **Improvements**  
   - **server startup**: performance  
-    - faster extension load
-    - faster json parsing
+    - faster extension load  
+    - faster json parsing  
+    - faster lora indexing  
   - **offline deployment**: allow deployment without git clone  
     for example, you can now deploy a zip of the sdnext folder  
   - **latent upscale**: updated latent upscalers (some are new)  
diff --git a/extensions-builtin/Lora/network.py b/extensions-builtin/Lora/network.py
index e5828daf3..ea22e9c3e 100644
--- a/extensions-builtin/Lora/network.py
+++ b/extensions-builtin/Lora/network.py
@@ -24,20 +24,8 @@ class NetworkOnDisk:
         self.metadata = {}
         self.is_safetensors = os.path.splitext(filename)[1].lower() == ".safetensors"
 
-        def read_metadata(): #  # pylint: disable=W0612
-            metadata = sd_models.read_metadata_from_safetensors(filename)
-            metadata.pop('ssmd_cover_images', None)  # those are cover images, and they are too big to display in UI as text
-            return metadata
-
         if self.is_safetensors:
             self.metadata = sd_models.read_metadata_from_safetensors(filename)
-            """
-            try:
-                self.metadata = cache.cached_data_for_file('safetensors-metadata', "lora/" + self.name, filename, read_metadata)
-            except Exception as e:
-                errors.display(e, f"reading lora {filename}")
-            """
-
         if self.metadata:
             m = {}
             for k, v in sorted(self.metadata.items(), key=lambda x: metadata_tags_order.get(x[0], 999)):
@@ -46,11 +34,7 @@ class NetworkOnDisk:
         self.alias = self.metadata.get('ss_output_name', self.name)
         self.hash = None
         self.shorthash = None
-        self.set_hash(
-            self.metadata.get('sshs_model_hash') or
-            hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or
-            ''
-        )
+        self.set_hash(self.metadata.get('sshs_model_hash') or hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or '')
         self.sd_version = self.detect_version()
 
     def detect_version(self):
@@ -65,9 +49,6 @@ class NetworkOnDisk:
     def set_hash(self, v):
         self.hash = v
         self.shorthash = self.hash[0:12]
-        if self.shorthash:
-            import networks
-            networks.available_network_hash_lookup[self.shorthash] = self
 
     def read_hash(self):
         if not self.hash:
diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py
index 7846d63ea..4a4198e1a 100644
--- a/extensions-builtin/Lora/networks.py
+++ b/extensions-builtin/Lora/networks.py
@@ -2,7 +2,7 @@ from typing import Union, List
 import os
 import re
 import time
-from threading import Thread
+import concurrent
 import lora_patches
 import network
 import network_lora
@@ -440,20 +440,26 @@ def list_available_networks():
         shared.log.warning('LoRA directory not found: path="{shared.cmd_opts.lora_dir}"')
     if os.path.exists(shared.cmd_opts.lyco_dir):
         candidates += list(shared.walk_files(shared.cmd_opts.lyco_dir, allowed_extensions=[".pt", ".ckpt", ".safetensors"]))
-    for filename in candidates:
+
+    def add_network(filename):
         if os.path.isdir(filename):
-            continue
+            return
         name = os.path.splitext(os.path.basename(filename))[0]
         try:
             entry = network.NetworkOnDisk(name, filename)
+            available_networks[entry.name] = entry
+            if entry.alias in available_network_aliases:
+                forbidden_network_aliases[entry.alias.lower()] = 1
+            available_network_aliases[entry.name] = entry
+            available_network_aliases[entry.alias] = entry
+            if entry.shorthash:
+                available_network_hash_lookup[entry.shorthash] = entry
         except OSError as e:  # should catch FileNotFoundError and PermissionError etc.
             shared.log.error(f"Failed to load network {name} from {filename} {e}")
-            continue
-        available_networks[name] = entry
-        if entry.alias in available_network_aliases:
-            forbidden_network_aliases[entry.alias.lower()] = 1
-        available_network_aliases[name] = entry
-        available_network_aliases[entry.alias] = entry
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor:
+        for fn in candidates:
+            executor.submit(add_network, fn)
 
 
 def infotext_pasted(infotext, params): # pylint: disable=W0613
@@ -478,5 +484,4 @@ def infotext_pasted(infotext, params): # pylint: disable=W0613
         params["Prompt"] += "\n" + "".join(added)
 
 
-thread_lora = Thread(target=list_available_networks)
-thread_lora.start()
+list_available_networks()
diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py
index 0dcecdbc8..444f151c4 100644
--- a/modules/control/units/controlnet.py
+++ b/modules/control/units/controlnet.py
@@ -3,7 +3,7 @@ import time
 from typing import Union
 from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, ControlNetModel, StableDiffusionControlNetPipeline, StableDiffusionXLControlNetPipeline
 from modules.control.units import detect
-from modules.shared import log, opts
+from modules.shared import log, opts, listdir
 from modules import errors
 
 
@@ -44,7 +44,7 @@ cache_dir = 'models/control/controlnet'
 
 def find_models():
     path = os.path.join(opts.control_dir, 'controlnet')
-    files = os.listdir(path)
+    files = listdir(path)
     files = [f for f in files if f.endswith('.safetensors')]
     downloaded_models = {}
     for f in files:
diff --git a/modules/control/units/lite.py b/modules/control/units/lite.py
index 9796f77f1..cc385610c 100644
--- a/modules/control/units/lite.py
+++ b/modules/control/units/lite.py
@@ -4,7 +4,7 @@ from typing import Union
 import numpy as np
 from PIL import Image
 from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline
-from modules.shared import log, opts
+from modules.shared import log, opts, listdir
 from modules import errors
 from modules.control.units.lite_model import ControlNetLLLite
 
@@ -31,7 +31,7 @@ cache_dir = 'models/control/lite'
 
 def find_models():
     path = os.path.join(opts.control_dir, 'lite')
-    files = os.listdir(path)
+    files = listdir(path)
     files = [f for f in files if f.endswith('.safetensors')]
     downloaded_models = {}
     for f in files:
diff --git a/modules/control/units/xs.py b/modules/control/units/xs.py
index 086ce3524..673f4b90d 100644
--- a/modules/control/units/xs.py
+++ b/modules/control/units/xs.py
@@ -2,7 +2,7 @@ import os
 import time
 from typing import Union
 from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline
-from modules.shared import log, opts
+from modules.shared import log, opts, listdir
 from modules import errors
 from modules.control.units.xs_model import ControlNetXSModel
 from modules.control.units.xs_pipe import StableDiffusionControlNetXSPipeline, StableDiffusionXLControlNetXSPipeline
@@ -27,7 +27,7 @@ cache_dir = 'models/control/xs'
 
 def find_models():
     path = os.path.join(opts.control_dir, 'xs')
-    files = os.listdir(path)
+    files = listdir(path)
     files = [f for f in files if f.endswith('.safetensors')]
     downloaded_models = {}
     for f in files:
diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py
index f5d192b05..5e75d2440 100644
--- a/modules/hypernetworks/hypernetwork.py
+++ b/modules/hypernetworks/hypernetwork.py
@@ -284,7 +284,7 @@ class Hypernetwork:
 def list_hypernetworks(path):
     res = {}
     def list_folder(folder):
-        for filename in os.listdir(folder):
+        for filename in shared.listdir(folder):
             fn = os.path.join(folder, filename)
             if os.path.isfile(fn) and fn.lower().endswith(".pt"):
                 name = os.path.splitext(os.path.basename(fn))[0]
diff --git a/modules/images.py b/modules/images.py
index e117b6860..adf8d78b3 100644
--- a/modules/images.py
+++ b/modules/images.py
@@ -504,7 +504,7 @@ def get_next_sequence_number(path, basename):
     prefix_length = len(basename)
     if not os.path.isdir(path):
         return 0
-    for p in os.listdir(path):
+    for p in shared.listdir(path):
         if p.startswith(basename):
             parts = os.path.splitext(p[prefix_length:])[0].split('-')  # splits the filename (removing the basename first if one is defined, so the sequence number is always the first element)
             try:
diff --git a/modules/img2img.py b/modules/img2img.py
index c54eb6097..a26995760 100644
--- a/modules/img2img.py
+++ b/modules/img2img.py
@@ -22,10 +22,10 @@ def process_batch(p, input_files, input_dir, output_dir, inpaint_mask_dir, args)
         if not os.path.isdir(input_dir):
             shared.log.error(f"Process batch: directory not found: {input_dir}")
             return
-        image_files = shared.listfiles(input_dir)
+        image_files = shared.listdir(input_dir)
     is_inpaint_batch = False
     if inpaint_mask_dir:
-        inpaint_masks = shared.listfiles(inpaint_mask_dir)
+        inpaint_masks = shared.listdir(inpaint_mask_dir)
         is_inpaint_batch = len(inpaint_masks) > 0
     if is_inpaint_batch:
         shared.log.info(f"Process batch: inpaint batch masks={len(inpaint_masks)}")
diff --git a/modules/modelloader.py b/modules/modelloader.py
index 9cd05dc82..c52b810a6 100644
--- a/modules/modelloader.py
+++ b/modules/modelloader.py
@@ -271,7 +271,7 @@ def load_diffusers_models(model_path: str, command_path: str = None, clear=True)
                 if not os.path.isfile(os.path.join(cache_path, "hidden")):
                     output.append(str(r.repo_id))
             """
-            for folder in os.listdir(place):
+            for folder in shared.listdir(place):
                 try:
                     if "--" not in folder:
                         continue
@@ -281,7 +281,7 @@ def load_diffusers_models(model_path: str, command_path: str = None, clear=True)
                     name = name.replace("--", "/")
                     folder = os.path.join(place, folder)
                     friendly = os.path.join(place, name)
-                    snapshots = os.listdir(os.path.join(folder, "snapshots"))
+                    snapshots = shared.listdir(os.path.join(folder, "snapshots"))
                     if len(snapshots) == 0:
                         shared.log.warning(f"Diffusers folder has no snapshots: location={place} folder={folder} name={name}")
                         continue
@@ -578,7 +578,7 @@ def move_files(src_path: str, dest_path: str, ext_filter: str = None):
         if not os.path.exists(dest_path):
             os.makedirs(dest_path)
         if os.path.exists(src_path):
-            for file in os.listdir(src_path):
+            for file in shared.listdir(src_path):
                 fullpath = os.path.join(src_path, file)
                 if os.path.isfile(fullpath):
                     if ext_filter is not None:
diff --git a/modules/postprocessing.py b/modules/postprocessing.py
index 150c641dc..b5df4868e 100644
--- a/modules/postprocessing.py
+++ b/modules/postprocessing.py
@@ -38,7 +38,7 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp
     elif extras_mode == 2:
         assert not shared.cmd_opts.hide_ui_dir_config, '--hide-ui-dir-config option must be disabled'
         assert input_dir, 'input directory not selected'
-        image_list = shared.listfiles(input_dir)
+        image_list = shared.listdir(input_dir)
         for filename in image_list:
             try:
                 image = Image.open(filename)
diff --git a/modules/script_loading.py b/modules/script_loading.py
index b20d4072e..37971eaf1 100644
--- a/modules/script_loading.py
+++ b/modules/script_loading.py
@@ -7,6 +7,7 @@ from installer import setup_logging, args
 
 
 preloaded = []
+debug = os.environ.get('SD_SCRIPT_DEBUG', None)
 
 
 def load_module(path):
@@ -20,9 +21,12 @@ def load_module(path):
         if '/sd-extension-' in path: # safe extensions without stdout intercept
             module_spec.loader.exec_module(module)
         else:
-            # stdout = io.StringIO()
-            with contextlib.redirect_stdout(io.StringIO()) as stdout:
+            if debug:
                 module_spec.loader.exec_module(module)
+                stdout = io.StringIO()
+            else:
+                with contextlib.redirect_stdout(io.StringIO()) as stdout:
+                    module_spec.loader.exec_module(module)
             setup_logging() # reset since scripts can hijaack logging
             for line in stdout.getvalue().splitlines():
                 if len(line) > 0:
diff --git a/modules/sd_hijack_hypertile.py b/modules/sd_hijack_hypertile.py
index c5e9a619d..7087b1436 100644
--- a/modules/sd_hijack_hypertile.py
+++ b/modules/sd_hijack_hypertile.py
@@ -190,7 +190,7 @@ def context_hypertile_vae(p):
         # shared.log.warning('Hypertile VAE is enabled but no VAE model was found')
         return nullcontext()
     else:
-        tile_size = shared.opts.hypertile_vae_tile if shared.opts.hypertile_vae_tile > 0 else max(256, 64 * min(p.width // 128, p.height // 128))
+        tile_size = shared.opts.hypertile_vae_tile if shared.opts.hypertile_vae_tile > 0 else max(128, 64 * min(p.width // 128, p.height // 128))
         shared.log.info(f'Applying hypertile: vae={tile_size}')
         p.extra_generation_params['Hypertile VAE'] = tile_size
         return split_attention(vae, tile_size=tile_size, min_tile_size=128, swap_size=1)
@@ -216,7 +216,7 @@ def context_hypertile_unet(p):
         # shared.log.warning('Hypertile UNet is enabled but no Unet model was found')
         return nullcontext()
     else:
-        tile_size = shared.opts.hypertile_unet_tile if shared.opts.hypertile_unet_tile > 0 else max(256, 64 * min(p.width // 128, p.height // 128))
+        tile_size = shared.opts.hypertile_unet_tile if shared.opts.hypertile_unet_tile > 0 else max(128, 64 * min(p.width // 128, p.height // 128))
         shared.log.info(f'Applying hypertile: unet={tile_size}')
         p.extra_generation_params['Hypertile UNet'] = tile_size
         return split_attention(unet, tile_size=tile_size, min_tile_size=128, swap_size=1)
diff --git a/modules/shared.py b/modules/shared.py
index 93b317c30..28b521a35 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -70,6 +70,8 @@ restricted_opts = {
 resize_modes = ["None", "Fixed", "Crop", "Fill", "Latent"]
 compatibility_opts = ['clip_skip', 'uni_pc_lower_order_final', 'uni_pc_order']
 console = Console(log_time=True, log_time_format='%H:%M:%S-%f')
+dir_timestamps = {}
+dir_cache = {}
 
 
 class Backend(Enum):
@@ -888,9 +890,16 @@ def restore_defaults(restart=True):
     restart_server(restart)
 
 
-def listfiles(dirname):
-    filenames = [os.path.join(dirname, x) for x in sorted(os.listdir(dirname), key=str.lower) if not x.startswith(".")]
-    return [file for file in filenames if os.path.isfile(file)]
+def listdir(path):
+    if not os.path.exists(path):
+        return []
+    mtime = os.path.getmtime(path)
+    if path in dir_timestamps and mtime == dir_timestamps[path]:
+        return dir_cache[path]
+    else:
+        dir_cache[path] = [os.path.join(path, f) for f in os.listdir(path)]
+        dir_timestamps[path] = mtime
+        return dir_cache[path]
 
 
 def walk_files(path, allowed_extensions=None):
diff --git a/modules/textual_inversion/preprocess.py b/modules/textual_inversion/preprocess.py
index 10aee5a00..220e23077 100644
--- a/modules/textual_inversion/preprocess.py
+++ b/modules/textual_inversion/preprocess.py
@@ -25,10 +25,6 @@ def preprocess(id_task, process_src, process_dst, process_width, process_height,
             deepbooru.model.stop()
 
 
-def listfiles(dirname):
-    return os.listdir(dirname)
-
-
 class PreprocessParams:
     src = None
     dstdir = None
@@ -137,17 +133,12 @@ def preprocess_work(process_src, process_dst, process_width, process_height, pre
     dst = os.path.abspath(process_dst)
     split_threshold = max(0.0, min(1.0, split_threshold))
     overlap_ratio = max(0.0, min(0.9, overlap_ratio))
-
     assert src != dst, 'same directory specified as source and destination'
-
     os.makedirs(dst, exist_ok=True)
-
-    files = listfiles(src)
-
+    files = shared.listdir(src)
     shared.state.job = "preprocess"
     shared.state.textinfo = "Preprocessing..."
     shared.state.job_count = len(files)
-
     params = PreprocessParams()
     params.dstdir = dst
     params.flip = process_flip
@@ -155,7 +146,6 @@ def preprocess_work(process_src, process_dst, process_width, process_height, pre
     params.process_caption = process_caption
     params.process_caption_deepbooru = process_caption_deepbooru
     params.preprocess_txt_action = preprocess_txt_action
-
     pbar = tqdm(files)
     for index, imagefile in enumerate(pbar):
         params.subindex = 0
@@ -171,9 +161,7 @@ def preprocess_work(process_src, process_dst, process_width, process_height, pre
         description = f"Preprocessing image {index + 1}/{len(files)}"
         pbar.set_description(description)
         shared.state.textinfo = description
-
         params.src = filename
-
         existing_caption = None
         existing_caption_filename = f"{os.path.splitext(filename)[0]}.txt"
         if os.path.exists(existing_caption_filename):
@@ -181,32 +169,25 @@ def preprocess_work(process_src, process_dst, process_width, process_height, pre
                 existing_caption = file.read()
         else:
             existing_caption_filename = None
-
         if shared.state.interrupted:
             break
-
         if img.height > img.width:
             ratio = (img.width * height) / (img.height * width)
             inverse_xy = False
         else:
             ratio = (img.height * width) / (img.width * height)
             inverse_xy = True
-
         process_default_resize = True
-
         if process_split and ratio < 1.0 and ratio <= split_threshold:
             for splitted in split_pic(img, inverse_xy, width, height, overlap_ratio):
                 save_pic(splitted, index, params, existing_caption=existing_caption, existing_caption_filename=existing_caption_filename)
             process_default_resize = False
-
         if process_focal_crop and img.height != img.width:
-
             dnn_model_path = None
             try:
                 dnn_model_path = autocrop.download_and_cache_models(os.path.join(paths.models_path, "opencv"))
             except Exception as e:
                 print("Unable to load face detection model for auto crop selection. Falling back to lower quality haar method.", e)
-
             autocrop_settings = autocrop.Settings(
                 crop_width = width,
                 crop_height = height,
@@ -227,13 +208,10 @@ def preprocess_work(process_src, process_dst, process_width, process_height, pre
             else:
                 print(f"skipped {img.width}x{img.height} image {filename} (can't find suitable size within error threshold)")
             process_default_resize = False
-
         if process_keep_original_size:
             save_pic(img, index, params, existing_caption=existing_caption)
             process_default_resize = False
-
         if process_default_resize:
             img = images.resize_image(1, img, width, height)
             save_pic(img, index, params, existing_caption=existing_caption)
-
         shared.state.nextjob()
diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py
index 6c3aa04a9..335223824 100644
--- a/modules/ui_extra_networks.py
+++ b/modules/ui_extra_networks.py
@@ -21,8 +21,6 @@ import modules.ui_symbols as symbols
 
 
 allowed_dirs = []
-dir_timestamps = {}
-dir_cache = {} # key=path, value=(mtime, listdir(path))
 refresh_time = 0
 extra_pages = shared.extra_networks
 debug = shared.log.trace if os.environ.get('SD_EN_DEBUG', None) is not None else lambda *args, **kwargs: None
@@ -51,19 +49,6 @@ card_list = '''
 '''
 
 
-def listdir(path):
-    if not os.path.exists(path):
-        return []
-    mtime = os.path.getmtime(path)
-    if path in dir_timestamps and mtime == dir_timestamps[path]:
-        return dir_cache[path]
-    else:
-        # debug(f'EN list-dir list: {path}')
-        dir_cache[path] = [os.path.join(path, f) for f in os.listdir(path)]
-        dir_timestamps[path] = mtime
-        return dir_cache[path]
-
-
 def init_api(app):
 
     def fetch_file(filename: str = ""):
@@ -167,7 +152,7 @@ class ExtraNetworksPage:
         return filename.replace('\\', '/')
 
     def is_empty(self, folder):
-        for f in listdir(folder):
+        for f in shared.listdir(folder):
             _fn, ext = os.path.splitext(f)
             if ext.lower() in ['.ckpt', '.safetensors', '.pt', '.json'] or os.path.isdir(os.path.join(folder, f)):
                 return False
@@ -319,9 +304,9 @@ class ExtraNetworksPage:
             path = os.path.relpath(path, shared.opts.diffusers_dir)
             ref = os.path.join('models', 'Reference')
             fn = os.path.join(ref, path.replace('models--', '').replace('\\', '/').split('/')[0])
-            files = listdir(ref)
+            files = shared.listdir(ref)
         else:
-            files = listdir(os.path.dirname(path))
+            files = shared.listdir(os.path.dirname(path))
             fn = os.path.splitext(path)[0]
         exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2"]
         for file in [f'{fn}{mid}{ext}' for ext in exts for mid in ['.thumb.', '.', '.preview.']]:
@@ -346,7 +331,7 @@ class ExtraNetworksPage:
                     self.text += '\n'
 
         fn = os.path.splitext(path)[0] + '.txt'
-        if fn in listdir(os.path.dirname(path)):
+        if fn in shared.listdir(os.path.dirname(path)):
             try:
                 with open(fn, "r", encoding="utf-8", errors="replace") as f:
                     txt = f.read()
@@ -366,7 +351,7 @@ class ExtraNetworksPage:
     def find_info(self, path):
         fn = os.path.splitext(path)[0] + '.json'
         data = {}
-        if fn in listdir(os.path.dirname(path)):
+        if fn in shared.listdir(os.path.dirname(path)):
             t0 = time.time()
             data = shared.readfile(fn, silent=True)
             if type(data) is list:
diff --git a/modules/ui_extra_networks_textual_inversion.py b/modules/ui_extra_networks_textual_inversion.py
index d94ce39dc..3d9c17da1 100644
--- a/modules/ui_extra_networks_textual_inversion.py
+++ b/modules/ui_extra_networks_textual_inversion.py
@@ -48,7 +48,7 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage):
     def list_items(self):
 
         def list_folder(folder):
-            for filename in os.listdir(folder):
+            for filename in shared.listdir(folder):
                 fn = os.path.join(folder, filename)
                 if os.path.isfile(fn) and (fn.lower().endswith(".pt") or fn.lower().endswith(".safetensors")):
                     embedding = Embedding(vec=0, name=os.path.basename(fn), filename=fn)
diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py
index 42916abde..7cf2e45b2 100644
--- a/modules/ui_img2img.py
+++ b/modules/ui_img2img.py
@@ -20,7 +20,7 @@ def process_interrogate(interrogation_function, mode, ii_input_files, ii_input_d
             if not os.path.isdir(ii_input_dir):
                 shared.log.error(f"Interrogate: Input directory not found: {ii_input_dir}")
                 return [gr.update(), None]
-            images = shared.listfiles(ii_input_dir)
+            images = shared.listdir(ii_input_dir)
         if ii_output_dir != "":
             os.makedirs(ii_output_dir, exist_ok=True)
         else:
diff --git a/modules/ui_interrogate.py b/modules/ui_interrogate.py
index 98fbe9a2f..196cfee68 100644
--- a/modules/ui_interrogate.py
+++ b/modules/ui_interrogate.py
@@ -119,7 +119,7 @@ def batch_process(batch_files, batch_folder, batch_str, mode, clip_model, write)
     if batch_folder is not None:
         files += [f.name for f in batch_folder]
     if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str):
-        files += [os.path.join(batch_str, f) for f in os.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))]
+        files += [os.path.join(batch_str, f) for f in shared.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))]
     if len(files) == 0:
         shared.log.error('Interrogate batch no images')
         return ''
diff --git a/modules/upscaler.py b/modules/upscaler.py
index 575dfdc12..37b5fde93 100644
--- a/modules/upscaler.py
+++ b/modules/upscaler.py
@@ -52,7 +52,7 @@ class Upscaler:
             pass
 
     def find_folder(self, folder, scalers, loaded):
-        for fn in os.listdir(folder): # from folder
+        for fn in modules.shared.listdir(folder): # from folder
             file_name = os.path.join(folder, fn)
             if os.path.isdir(file_name):
                 self.find_folder(file_name, scalers, loaded)
@@ -83,19 +83,6 @@ class Upscaler:
         if not os.path.exists(self.user_path):
             return scalers
         self.find_folder(self.user_path, scalers, loaded)
-        """
-        for fn in os.listdir(self.user_path): # from folder
-            if not fn.endswith('.pth') and not fn.endswith('.pt'):
-                continue
-            file_name = os.path.join(self.user_path, fn)
-            if file_name not in loaded:
-                model_name = os.path.splitext(fn)[0]
-                scaler = UpscalerData(name=f'{self.name} {model_name}', path=file_name, upscaler=self)
-                scaler.custom = True
-                scalers.append(scaler)
-                loaded.append(file_name)
-                # modules.shared.log.debug(f'Upscaler type={self.name} folder="{self.user_path}" model="{model_name}" path="{file_name}"')
-        """
         return scalers
 
     @abstractmethod