From e4aaed9eee39195045dbc421316fdc9fca10c4df Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Sat, 11 Apr 2026 12:55:29 -0700 Subject: [PATCH 1/6] Improve init & data handling. Improve typing. Minor usage change. Instead of setting the hash info directly, call `add_hash` to ensure proper data structure. --- modules/civitai/api_civitai.py | 5 ++- modules/civitai/download_civitai.py | 6 +--- modules/hashes.py | 53 ++++++++++++++++------------- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/modules/civitai/api_civitai.py b/modules/civitai/api_civitai.py index 7612797d5..4a8302757 100644 --- a/modules/civitai/api_civitai.py +++ b/modules/civitai/api_civitai.py @@ -477,9 +477,8 @@ def post_check_local(request: dict): return {"found": {}} # Build reverse lookup: lowercase sha256 -> {filename, type} found = {} - hash_cache = hash_module.cache("hashes") - for title, entry in hash_cache.items(): - sha = entry.get("sha256") + for title, entry in hash_module.cache().items(): + sha = entry["sha256"] if not sha: continue parts = title.split("/", 1) diff --git a/modules/civitai/download_civitai.py b/modules/civitai/download_civitai.py index 402f2ac28..f04012c90 100644 --- a/modules/civitai/download_civitai.py +++ b/modules/civitai/download_civitai.py @@ -285,11 +285,7 @@ class DownloadManager: prefix = model_type_map.get(item.model_type, item.model_type.lower()) name = os.path.splitext(item.filename)[0] title = f"{prefix}/{name}" - hash_cache = hashes.cache("hashes") - hash_cache[title] = { - "mtime": os.path.getmtime(final_file), - "sha256": item.expected_hash.lower(), - } + hashes.cache().add_hash(title, os.path.getmtime(final_file), item.expected_hash.lower()) hashes.dump_cache() except Exception: pass diff --git a/modules/hashes.py b/modules/hashes.py index 64741253d..26e177719 100644 --- a/modules/hashes.py +++ b/modules/hashes.py @@ -1,5 +1,7 @@ import hashlib import os.path +from collections import defaultdict +from typing import TypedDict from rich import progress, errors from modules.logger import console from modules.logger import log @@ -7,28 +9,37 @@ from modules.json_helpers import readfile, writefile from modules.paths import data_path -cache_filename = os.path.join(data_path, 'data', 'cache.json') -cache_data = None +class HashEntry(TypedDict): + mtime: float + sha256: str + + +class HashStore(dict[str, HashEntry]): + def __init__(self, *args): + super().__init__(*args) + + def add_hash(self, key: str, mtime: float = 0, sha256: str | None = None): + self.__setitem__(key, {"mtime": mtime, "sha256": sha256 or ""}) + + +cache_filename = os.path.join(data_path, "data", "cache.json") +# defaultdict allows for easily using new stores without needing to define them ahead of time +_data: defaultdict[str, HashStore] = defaultdict(HashStore) progress_ok = True def init_cache(): - global cache_data # pylint: disable=global-statement - if cache_data is None: - cache_data = {} if not os.path.isfile(cache_filename) else readfile(cache_filename, lock=True, as_type="dict") + if os.path.isfile(cache_filename): + for store, data in readfile(cache_filename, lock=True, as_type="dict").items(): + _data[store] = HashStore(data) def dump_cache(): - writefile(cache_data, cache_filename) + writefile(_data, cache_filename) -def cache(subsection): - global cache_data # pylint: disable=global-statement - if cache_data is None: - cache_data = {} if not os.path.isfile(cache_filename) else readfile(cache_filename, lock=True, as_type="dict") - s = cache_data.get(subsection, {}) - cache_data[subsection] = s - return s +def cache(store: str = "hashes") -> HashStore: + return _data[store] def calculate_sha256(filename, quiet=False): @@ -55,19 +66,18 @@ def calculate_sha256(filename, quiet=False): return hash_sha256.hexdigest() -def sha256_from_cache(filename, title, use_addnet_hash=False): +def sha256_from_cache(filename: str, title: str, use_addnet_hash=False): hashes = cache("hashes-addnet") if use_addnet_hash else cache("hashes") if title not in hashes: return None - cached_sha256 = hashes[title].get("sha256", None) - cached_mtime = hashes[title].get("mtime", 0) + cached = hashes[title] ondisk_mtime = os.path.getmtime(filename) if os.path.isfile(filename) else 0 - if ondisk_mtime > cached_mtime or cached_sha256 is None: + if ondisk_mtime > cached["mtime"] or not cached["sha256"]: return None - return cached_sha256 + return cached["sha256"] -def sha256(filename, title, use_addnet_hash=False): +def sha256(filename: str, title: str, use_addnet_hash=False): from modules import shared global progress_ok # pylint: disable=global-statement hashes = cache("hashes-addnet") if use_addnet_hash else cache("hashes") @@ -92,10 +102,7 @@ def sha256(filename, title, use_addnet_hash=False): sha256_value = addnet_hash_safetensors(f) else: sha256_value = calculate_sha256(filename) - hashes[title] = { - "mtime": os.path.getmtime(filename), - "sha256": sha256_value - } + hashes.add_hash(title, os.path.getmtime(filename), sha256_value) shared.state.end(jobid) dump_cache() return sha256_value From 1647be96de59141a38dd3bd2abf97708d5ebeef9 Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Sat, 11 Apr 2026 13:14:46 -0700 Subject: [PATCH 2/6] Minor function renaming Cache no longer requires initialization. --- modules/civitai/download_civitai.py | 2 +- modules/hashes.py | 6 +++--- webui.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/civitai/download_civitai.py b/modules/civitai/download_civitai.py index f04012c90..154b93448 100644 --- a/modules/civitai/download_civitai.py +++ b/modules/civitai/download_civitai.py @@ -286,7 +286,7 @@ class DownloadManager: name = os.path.splitext(item.filename)[0] title = f"{prefix}/{name}" hashes.cache().add_hash(title, os.path.getmtime(final_file), item.expected_hash.lower()) - hashes.dump_cache() + hashes.save_cache() except Exception: pass diff --git a/modules/hashes.py b/modules/hashes.py index 26e177719..02feeffa5 100644 --- a/modules/hashes.py +++ b/modules/hashes.py @@ -28,13 +28,13 @@ _data: defaultdict[str, HashStore] = defaultdict(HashStore) progress_ok = True -def init_cache(): +def load_cache(): if os.path.isfile(cache_filename): for store, data in readfile(cache_filename, lock=True, as_type="dict").items(): _data[store] = HashStore(data) -def dump_cache(): +def save_cache(): writefile(_data, cache_filename) @@ -104,7 +104,7 @@ def sha256(filename: str, title: str, use_addnet_hash=False): sha256_value = calculate_sha256(filename) hashes.add_hash(title, os.path.getmtime(filename), sha256_value) shared.state.end(jobid) - dump_cache() + save_cache() return sha256_value diff --git a/webui.py b/webui.py index 11c48e8a5..b8d06697f 100644 --- a/webui.py +++ b/webui.py @@ -77,7 +77,7 @@ def initialize(): from concurrent.futures import ThreadPoolExecutor, as_completed modules.sd_checkpoint.init_metadata() - modules.hashes.init_cache() + modules.hashes.load_cache() modules.sd_samplers.list_samplers() timer.startup.record("samplers") From e94d715e7d98b8a9f3e1fba6c9f8ec1a8abfb78e Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Mon, 13 Apr 2026 12:25:15 -0700 Subject: [PATCH 3/6] Fix data serialization --- modules/hashes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/hashes.py b/modules/hashes.py index 02feeffa5..57afb703e 100644 --- a/modules/hashes.py +++ b/modules/hashes.py @@ -35,7 +35,7 @@ def load_cache(): def save_cache(): - writefile(_data, cache_filename) + writefile(dict(_data), cache_filename) def cache(store: str = "hashes") -> HashStore: From 5c30c993b5f14b421b26f89c40e933d960ebf230 Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Tue, 14 Apr 2026 00:11:05 -0700 Subject: [PATCH 4/6] Don't include empty hash stores --- modules/hashes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/hashes.py b/modules/hashes.py index 57afb703e..ac3e01c45 100644 --- a/modules/hashes.py +++ b/modules/hashes.py @@ -35,7 +35,9 @@ def load_cache(): def save_cache(): - writefile(dict(_data), cache_filename) + # Don't include empty hash stores + filtered = filter(lambda item: len(item[1]) > 0, _data.items()) + writefile(dict(filtered), cache_filename) def cache(store: str = "hashes") -> HashStore: From 6aff7959856fb5762a9dee154b2d3cd336cf4394 Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Tue, 14 Apr 2026 02:59:19 -0700 Subject: [PATCH 5/6] Improve extensibility --- modules/hashes.py | 23 ++++++++++++++--------- modules/lora/network.py | 4 ++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/modules/hashes.py b/modules/hashes.py index ac3e01c45..3541a7a9b 100644 --- a/modules/hashes.py +++ b/modules/hashes.py @@ -1,7 +1,7 @@ import hashlib import os.path from collections import defaultdict -from typing import TypedDict +from typing import Literal, TypeAlias, TypedDict from rich import progress, errors from modules.logger import console from modules.logger import log @@ -22,10 +22,13 @@ class HashStore(dict[str, HashEntry]): self.__setitem__(key, {"mtime": mtime, "sha256": sha256 or ""}) +default_hash_store = "hashes" +KnownHashStores: TypeAlias = Literal["hashes", "hashes-addnet"] # For autocomplete in IDE + cache_filename = os.path.join(data_path, "data", "cache.json") +progress_ok = True # defaultdict allows for easily using new stores without needing to define them ahead of time _data: defaultdict[str, HashStore] = defaultdict(HashStore) -progress_ok = True def load_cache(): @@ -40,7 +43,9 @@ def save_cache(): writefile(dict(filtered), cache_filename) -def cache(store: str = "hashes") -> HashStore: +def cache(store: KnownHashStores | str | None = None) -> HashStore: + if store is None: + return _data[default_hash_store] return _data[store] @@ -68,8 +73,8 @@ def calculate_sha256(filename, quiet=False): return hash_sha256.hexdigest() -def sha256_from_cache(filename: str, title: str, use_addnet_hash=False): - hashes = cache("hashes-addnet") if use_addnet_hash else cache("hashes") +def sha256_from_cache(filename: str, title: str, *, store: KnownHashStores | str | None = None): + hashes = cache(store) if title not in hashes: return None cached = hashes[title] @@ -79,11 +84,11 @@ def sha256_from_cache(filename: str, title: str, use_addnet_hash=False): return cached["sha256"] -def sha256(filename: str, title: str, use_addnet_hash=False): +def sha256(filename: str, title: str, *, store: KnownHashStores | str | None = None): from modules import shared global progress_ok # pylint: disable=global-statement - hashes = cache("hashes-addnet") if use_addnet_hash else cache("hashes") - sha256_value = sha256_from_cache(filename, title, use_addnet_hash) + hashes = cache(store) + sha256_value = sha256_from_cache(filename, title, store=store) if sha256_value is not None: return sha256_value if shared.cmd_opts.no_hashing: @@ -91,7 +96,7 @@ def sha256(filename: str, title: str, use_addnet_hash=False): if not os.path.isfile(filename): return None jobid = shared.state.begin("Hash") - if use_addnet_hash: + if store == "hashes-addnet": if progress_ok: try: with progress.open(filename, 'rb', description=f'[cyan]Calculating hash: [yellow]{filename}', auto_refresh=True, console=console) as f: diff --git a/modules/lora/network.py b/modules/lora/network.py index 5d3fd9fc7..a6753a500 100644 --- a/modules/lora/network.py +++ b/modules/lora/network.py @@ -40,7 +40,7 @@ class NetworkOnDisk: m[k] = v self.metadata = m self.alias = self.metadata.get('ss_output_name', self.name) - sha256 = hashes.sha256_from_cache(self.filename, "lora/" + self.name) or hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=True) or self.metadata.get('sshs_model_hash') + sha256 = hashes.sha256_from_cache(self.filename, "lora/" + self.name) or hashes.sha256_from_cache(self.filename, "lora/" + self.name, store='hashes-addnet') or self.metadata.get('sshs_model_hash') self.set_hash(sha256) self.sd_version = self.detect_version() @@ -107,7 +107,7 @@ class NetworkOnDisk: def read_hash(self): if not self.hash: - self.set_hash(hashes.sha256(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or '') + self.set_hash(hashes.sha256(self.filename, "lora/" + self.name, store='hashes-addnet' if self.is_safetensors else None) or '') def get_info(self): data = {} From c8e3473e93657dedac4c919a6d7080b4094f4013 Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Tue, 14 Apr 2026 05:22:07 -0700 Subject: [PATCH 6/6] Minor cleanup --- modules/hashes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/hashes.py b/modules/hashes.py index 3541a7a9b..487271e52 100644 --- a/modules/hashes.py +++ b/modules/hashes.py @@ -87,7 +87,6 @@ def sha256_from_cache(filename: str, title: str, *, store: KnownHashStores | str def sha256(filename: str, title: str, *, store: KnownHashStores | str | None = None): from modules import shared global progress_ok # pylint: disable=global-statement - hashes = cache(store) sha256_value = sha256_from_cache(filename, title, store=store) if sha256_value is not None: return sha256_value @@ -109,7 +108,7 @@ def sha256(filename: str, title: str, *, store: KnownHashStores | str | None = N sha256_value = addnet_hash_safetensors(f) else: sha256_value = calculate_sha256(filename) - hashes.add_hash(title, os.path.getmtime(filename), sha256_value) + cache(store).add_hash(title, os.path.getmtime(filename), sha256_value) shared.state.end(jobid) save_cache() return sha256_value