Merge pull request #5088 from CalamitousFelicitousness/refactor/json-helpers-path-lock

Refactor/json helpers path lock

modules/hashes.py

@@ -1,5 +1,6 @@
 import hashlib
 import os.path
+import threading
 from collections import defaultdict
 from typing import Literal, TypeAlias, TypedDict
 from rich import progress, errors
@@ -29,6 +30,7 @@ cache_filename = os.path.join(data_path, "data", "cache.json")
 progress_ok = True
 # defaultdict allows for easily using new stores without needing to define them ahead of time
 _data: defaultdict[str, HashStore] = defaultdict(HashStore)
+cache_lock = threading.Lock()
 
 
 def load_cache():
@@ -38,9 +40,9 @@ def load_cache():
 
 
 def save_cache():
-    # Don't include empty hash stores
-    filtered = filter(lambda item: len(item[1]) > 0, _data.items())
-    writefile(dict(filtered), cache_filename)
+    with cache_lock:  # snapshot and write together, so a later save never lands under an earlier snapshot
+        snapshot = {store: dict(data) for store, data in list(_data.items()) if len(data) > 0}  # dict() of a store runs under the GIL, so add_hash from another thread cannot interrupt it
+        writefile(snapshot, cache_filename)
 
 
 def cache(store: KnownHashStores | str | None = None) -> HashStore:

modules/json_helpers.py

@@ -1,14 +1,40 @@
 import os
 import sys
 import contextlib
+import threading
 import time
 import json
 from typing import overload, Literal
-import fasteners
 import orjson
 from modules.logger import log
 
-locking_available = True  # used by file read/write locking
+path_locks: dict[str, threading.RLock] = {}
+path_locks_guard = threading.Lock()
+
+
+def path_lock(filename: str | os.PathLike[str]) -> threading.RLock:
+    """One lock per file path; threads of this process serialize on it, other processes are covered by atomic replace."""
+    key = os.path.normcase(os.path.realpath(filename))
+    with path_locks_guard:
+        lock = path_locks.get(key)
+        if lock is None:
+            lock = path_locks[key] = threading.RLock()
+            with contextlib.suppress(OSError):
+                os.remove(f"{key}.lock")  # left behind by the file lock this replaces
+        return lock
+
+
+def read_bytes(filename: str | os.PathLike[str], attempts: int = 5, delay: float = 0.01) -> bytes:
+    """Read a whole file, waiting out an open that Windows refuses while a replace of the same name is in flight."""
+    for attempt in range(attempts):
+        try:
+            with open(filename, "rb") as file:
+                return file.read()
+        except PermissionError:
+            if attempt == attempts - 1:
+                raise
+            time.sleep(delay)
+    return b""
 
 
 def replace_file(source: str, target: str, attempts: int = 10, delay: float = 0.05):
@@ -30,51 +56,28 @@ def readfile(filename: str | os.PathLike[str], silent: bool = False, lock: bool
 @overload
 def readfile(filename: str | os.PathLike[str], silent: bool = False, lock: bool = False) -> dict | list: ...
 def readfile(filename: str | os.PathLike[str], silent: bool = False, lock: bool = False, *, as_type="") -> dict | list:
-    global locking_available  # pylint: disable=global-statement
+    """Read a JSON file; lock=True serializes with writers of the same path in this process."""
     data = {} if as_type == "dict" else []
-    lock_file = None
-    locked = False
 
-    if lock and locking_available:
+    with path_lock(filename) if lock else contextlib.nullcontext():
         try:
-            lock_file = fasteners.InterProcessReaderWriterLock(f"{filename}.lock")
-            lock_file.logger.disabled = True  # type: ignore - False positive. Bad typing in Fasteners.
-            locked = lock_file.acquire_read_lock(blocking=True, timeout=3)
-            if not locked:
-                log.warning(f'File read lock: file="{filename}" timeout')
-        except Exception as err:
-            lock_file = None
-            locking_available = False
-            log.error(f'File read lock: file="{filename}" {err}')
-            locked = False
-
-    try:
-        # if not os.path.exists(filename):
-        #    return {}
-        t0 = time.time()
-        with open(filename, "rb") as file:
-            b = file.read()
+            t0 = time.time()
+            b = read_bytes(filename)
             if len(b) == 0:
+                if not silent:
+                    log.warning(f'Read: file="{filename}" empty')
                 return {} if as_type == "dict" else []
             data = orjson.loads(b)  # pylint: disable=no-member
-        # if type(data) is str:
-        #    data = json.loads(data)
-        t1 = time.time()
-        if not silent:
-            fn = f"{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}"  # pylint: disable=protected-access
-            log.debug(f'Read: file="{filename}" json={len(data)} bytes={os.path.getsize(filename)} time={t1 - t0:.3f} fn={fn}')
-    except FileNotFoundError as err:
-        if not silent:
-            log.debug(f'Read failed: file="{filename}" {err}')
-    except Exception as err:
-        if not silent:
-            log.error(f'Read failed: file="{filename}" {err}')
-
-    try:
-        if locking_available and lock_file is not None:
-            lock_file.release_read_lock()  # the lock file stays: removing it after release races other holders
-    except Exception as err:
-        log.error(f'File read lock release: file="{filename}" {err}')
+            t1 = time.time()
+            if not silent:
+                fn = f"{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}"  # pylint: disable=protected-access
+                log.debug(f'Read: file="{filename}" json={len(data)} bytes={os.path.getsize(filename)} time={t1 - t0:.3f} fn={fn}')
+        except FileNotFoundError as err:
+            if not silent:
+                log.debug(f'Read failed: file="{filename}" {err}')
+        except Exception as err:
+            if not silent:
+                log.error(f'Read failed: file="{filename}" {err}')
 
     if isinstance(data, list) and as_type == "dict":
         if not data:
@@ -92,72 +95,55 @@ def readfile(filename: str | os.PathLike[str], silent: bool = False, lock: bool
     return data
 
 
-def writefile(obj: dict | list, filename: str | os.PathLike[str], mode="w", silent=False, atomic=False):
+def writefile(obj: dict | list, filename: str | os.PathLike[str], mode="w", silent=False, atomic=True):
+    """Write obj as JSON through a temp file and replace; writes to the same path from this process run one at a time, in call order."""
     import copy
     import tempfile
 
-    global locking_available # pylint: disable=global-statement
-    lock_file = None
-    locked = False
-
     def default(obj):
         log.error(f'Save: file="{filename}" not a valid object: {obj}')
         return str(obj)
 
-    try:
-        t0 = time.time()
-        snapshot = obj.copy() if isinstance(obj, (dict, list)) else obj # dict.copy and list.copy run under the GIL, so a concurrent insert cannot interrupt them
-        data = copy.deepcopy(snapshot)
-        for k, v in list(data.items()) if isinstance(data, dict) else []: # validate each key-by-key to avoid global exceptions
-            try:
-                _tmp = json.dumps(v, indent=2, default=default, allow_nan=False, ensure_ascii=False)
-            except Exception as err:
-                if not silent:
-                    log.error(f'Save: file="{filename}" key="{k}" value="{v}" {err}')
-                del data[k]
-        output = json.dumps(data, indent=2, default=default)
-    except Exception as err:
-        log.error(f'Save failed: file="{filename}" {err}')
-        return
+    if mode != "w":
+        atomic = False  # append cannot go through a temp file
 
-    try:
-        if locking_available:
-            lock_file = fasteners.InterProcessReaderWriterLock(f"{filename}.lock") if locking_available else None
-            lock_file.logger.disabled = True  # type: ignore - False positive. Bad typing in Fasteners.
-            locked = lock_file.acquire_write_lock(blocking=True, timeout=3) if lock_file is not None else False
-            if not locked:
-                log.warning(f'File write lock: file="{filename}" timeout')
-    except Exception as err:
-        locking_available = False
-        lock_file = None
-        log.error(f'File write lock: file="{filename}" {err}')
-        locked = False
+    with path_lock(filename):
+        try:
+            t0 = time.time()
+            snapshot = obj.copy() if isinstance(obj, (dict, list)) else obj # dict.copy and list.copy run under the GIL, so a concurrent insert cannot interrupt them
+            data = copy.deepcopy(snapshot)
+            for k, v in list(data.items()) if isinstance(data, dict) else []: # validate each key-by-key to avoid global exceptions
+                try:
+                    _tmp = json.dumps(v, indent=2, default=default, allow_nan=False, ensure_ascii=False)
+                except Exception as err:
+                    if not silent:
+                        log.error(f'Save: file="{filename}" key="{k}" value="{v}" {err}')
+                    del data[k]
+            output = json.dumps(data, indent=2, default=default)
+        except Exception as err:
+            log.error(f'Save failed: file="{filename}" {err}')
+            return
 
-    try:
-        if atomic:
-            fd, temp_name = tempfile.mkstemp(dir=os.path.dirname(os.path.abspath(filename)), prefix=f"{os.path.basename(filename)}.", suffix=".tmp")
-            try:
-                with os.fdopen(fd, mode, encoding="utf8") as f:
-                    f.write(output)
-                    f.flush()
-                    os.fsync(f.fileno())
-                replace_file(temp_name, filename)
-            except BaseException:
-                with contextlib.suppress(OSError):
-                    os.remove(temp_name)
-                raise
-        else:
-            with open(filename, mode=mode, encoding="utf8") as file:
-                file.write(output)
-        t1 = time.time()
-        if not silent:
-            datalength = len(data)
-            log.debug(f'Save: file="{filename}" json={datalength} bytes={len(output)} time={t1 - t0:.3f}')
-    except Exception as err:
-        log.error(f'Save failed: file="{filename}" {err}')
-
-    try:
-        if locking_available and lock_file is not None:
-            lock_file.release_write_lock()  # the lock file stays: removing it after release races other holders
-    except Exception as err:
-        log.error(f'File write lock release: file="{filename}" {err}')
+        try:
+            if atomic:
+                target = os.path.realpath(filename)  # replace the file a symlink points at, not the symlink
+                fd, temp_name = tempfile.mkstemp(dir=os.path.dirname(target), prefix=f"{os.path.basename(target)}.", suffix=".tmp")
+                try:
+                    with os.fdopen(fd, mode, encoding="utf8") as f:
+                        f.write(output)
+                        f.flush()
+                        os.fsync(f.fileno())
+                    replace_file(temp_name, target)
+                except BaseException:
+                    with contextlib.suppress(OSError):
+                        os.remove(temp_name)
+                    raise
+            else:
+                with open(filename, mode=mode, encoding="utf8") as file:
+                    file.write(output)
+            t1 = time.time()
+            if not silent:
+                datalength = len(data)
+                log.debug(f'Save: file="{filename}" json={datalength} bytes={len(output)} time={t1 - t0:.3f}')
+        except Exception as err:
+            log.error(f'Save failed: file="{filename}" {err}')