diff --git a/javascript/gallery.js b/javascript/gallery.js index d3d8825dc..028f622c1 100644 --- a/javascript/gallery.js +++ b/javascript/gallery.js @@ -22,7 +22,19 @@ const el = { const SUPPORTED_EXTENSIONS = ['jpg', 'jpeg', 'png', 'webp', 'tiff', 'jp2', 'jxl', 'gif', 'mp4', 'mkv', 'avi', 'mjpeg', 'mpg', 'avr']; async function awaitForIDB(num = 0) { - while (outstanding > num || idbIsCleaning) await new Promise((resolve) => setTimeout(resolve, 50)); // eslint-disable-line no-promise-executor-return + while (outstanding > num || idbIsCleaning) await new Promise((resolve) => { setTimeout(resolve, 50); }); +} + +async function awaitForGallery(folderName, num = 0) { + let timeout = 0; + const timeoutThreshold = 60; // 30 seconds (60*0.5) + while (galleryHashes.size < num && activeGalleryFolder === folderName && !idbIsCleaning && timeout++ < timeoutThreshold) await new Promise((resolve) => { setTimeout(resolve, 500); }); // longer interval because it's a low priority check + if (timeout >= timeoutThreshold) { + throw new Error('Timed out waiting for gallery to populate'); + } + if (idbIsCleaning) { + throw new Error('Another thread has already started cleaning the database'); + } } // HTML Elements @@ -554,9 +566,21 @@ async function gallerySort(btn) { updateStatusWithSort(`${arr.length.toLocaleString()} images | ${Math.floor(t1 - t0).toLocaleString()}ms`); } +/** + * Function for updating the cleaning overlay message + * @callback updateMsgCallback + * @param {number} progressPercent - Value for completion progress percentage + * @returns {void} + */ +/** + * Function for removing the cleaning overlay + * @callback clearMsgCallback + * @returns {void} + */ + /** * Generate and display the overlay to announce cleanup is in progress. - * @returns {() => void} Function for clearing the overlay + * @returns {[updateMsgCallback, clearMsgCallback]} */ function showCleaningMsg() { const parent = el.folders.parentElement; @@ -567,36 +591,54 @@ function showCleaningMsg() { parent.style.position = 'relative'; cleaningOverlay.style.cssText = 'position: absolute; height: 100%; width: 100%; background-color: hsl(210 50 20 / 0.8); display: flex; align-items: center; justify-content: center;'; msg.style.cssText = 'display: block; background-color: hsl(0 0 10); color: white; padding: 12px; border-radius: 8px; margin-right: 16px;'; - msg.innerText = 'Running thumbnail cleanup'; + msg.innerText = 'Thumbnail cleanup (0%)'; anim.classList.add('idbBusyAnim'); cleaningOverlay.append(msg, anim); parent.append(cleaningOverlay); - return () => { - parent.style.position = ''; - cleaningOverlay.remove(); - }; + return [ + (pct) => { + msg.innerText = `Thumbnail cleanup (${pct}%)`; + }, + () => { + parent.style.position = ''; + cleaningOverlay.remove(); + }, + ]; } -async function thumbCacheCleanup() { +/** + * IndexedDB thumbnail cache cleanup function + * @param {string} folder - Folder to clean + * @param {number} imgCount - Expected number of images in gallery + */ +async function thumbCacheCleanup(folder, imgCount) { if (idbIsCleaning) return; await awaitForIDB(); - idbIsCleaning = true; - - const t0 = performance.now(); - const cachedHashesCount = await idbCount() - .catch(() => 0); - if (cachedHashesCount < galleryHashes.size + 500) { - // Don't run when there aren't many excess entries - idbIsCleaning = false; + try { + await awaitForGallery(folder, imgCount); + } catch (err) { + log('Thumbnail DB cleanup:', err.message); return; } + if (activeGalleryFolder !== folder || idbIsCleaning) return; // First check for other thread activity - const removeOverlayFunc = showCleaningMsg(); - idbClean(galleryHashes, activeGalleryFolder) - .then((delcount, folder) => { + const t0 = performance.now(); + const staticGalleryHashes = new Set(galleryHashes); + const cachedHashesCount = await idbCount(folder) + .catch(() => 0); + if (cachedHashesCount < staticGalleryHashes.size + 500) { + // Don't run when there aren't many excess entries + return; + } + if (activeGalleryFolder !== folder || idbIsCleaning) return; // Second check for other thread activity + + idbIsCleaning = true; + const [updateCleaningMsg, removeOverlayFunc] = showCleaningMsg(); + idbClean(staticGalleryHashes, folder, updateCleaningMsg) + .then((delcount) => { const t1 = performance.now(); - log(`Thumbnail DB cleanup: folder=${folder} kept=${galleryHashes.size} deleted=${delcount} time=${Math.floor(t1 - t0)}ms`); + log(`Thumbnail DB cleanup: folder=${folder} kept=${staticGalleryHashes.size} deleted=${delcount} time=${Math.floor(t1 - t0)}ms`); }) .catch((err) => { error('Thumbnail DB cleanup: Cleanup failed.', err.message); @@ -633,27 +675,25 @@ async function fetchFilesHT(evt) { el.files.appendChild(fragment); const t1 = performance.now(); - activeGalleryFolder = evt.target.name; log(`gallery: folder=${evt.target.name} num=${numFiles} time=${Math.floor(t1 - t0)}ms`); updateStatusWithSort(`Folder: ${evt.target.name} | ${numFiles.toLocaleString()} images | ${Math.floor(t1 - t0).toLocaleString()}ms`); - addSeparators(); - thumbCacheCleanup(); + thumbCacheCleanup(evt.target.name, numFiles); } async function fetchFilesWS(evt) { // fetch file-by-file list over websockets - if (idbIsCleaning) return; + if (idbIsCleaning || !url) return; galleryHashes.clear(); // Only called here because fetchFilesHT isn't called directly el.files.innerHTML = ''; - if (!url) return; if (ws && ws.readyState === WebSocket.OPEN) ws.close(); // abort previous request let wsConnected = false; try { ws = new WebSocket(`${url}/sdapi/v1/browser/files`); - wsConnected = await wsConnect(ws); + wsConnected = await wsConnect(ws); // Warning. This changes "evt". } catch (err) { log('gallery: ws connect error', err); return; } + activeGalleryFolder = evt.target.name; log(`gallery: connected=${wsConnected} state=${ws?.readyState} url=${ws?.url}`); if (!wsConnected) { await fetchFilesHT(evt); // fallback to http @@ -688,11 +728,10 @@ async function fetchFilesWS(evt) { // fetch file-by-file list over websockets ws.onclose = (event) => { el.files.appendChild(fragment); // gallerySort(); - activeGalleryFolder = evt.target.name; log(`gallery: folder=${evt.target.name} num=${numFiles} time=${Math.floor(t1 - t0)}ms`); updateStatusWithSort(`Folder: ${evt.target.name} | ${numFiles.toLocaleString()} images | ${Math.floor(t1 - t0).toLocaleString()}ms`); addSeparators(); - thumbCacheCleanup(); + thumbCacheCleanup(evt.target.name, numFiles); }; ws.onerror = (event) => { log('gallery ws error', event); diff --git a/javascript/indexdb.js b/javascript/indexdb.js index ee84cde12..490eb46f6 100644 --- a/javascript/indexdb.js +++ b/javascript/indexdb.js @@ -1,9 +1,12 @@ -let db; +/** + * @type {?IDBDatabase} + */ +let db = null; async function initIndexDB() { async function createDB() { return new Promise((resolve, reject) => { - const request = indexedDB.open('SDNext'); + const request = indexedDB.open('SDNext', 2); request.onerror = (evt) => reject(evt); request.onsuccess = (evt) => { db = evt.target.result; @@ -16,9 +19,15 @@ async function initIndexDB() { }; request.onupgradeneeded = (evt) => { db = evt.target.result; - const store = db.createObjectStore('thumbs', { keyPath: 'hash' }); - store.createIndex('hash', 'hash', { unique: true }); - const index = store.index('hash'); + const oldver = evt.oldVersion; + if (oldver < 1) { + const store = db.createObjectStore('thumbs', { keyPath: 'hash' }); + store.createIndex('hash', 'hash', { unique: true }); + } + if (oldver < 2) { + const existingStore = request.transaction.objectStore('thumbs'); + existingStore.createIndex('folder', 'folder', { unique: false }); + } resolve(); }; }); @@ -75,54 +84,81 @@ async function put(record) { }); } -async function idbGetAllKeys() { +async function idbGetAllKeys(index = null, query = null) { if (!db) return null; return new Promise((resolve, reject) => { - const request = db + let request; + const store = db .transaction('thumbs', 'readonly') - .objectStore('thumbs') - .getAllKeys(); + .objectStore('thumbs'); + if (index) { + request = store.index(index).getAllKeys(query); + } else { + request = store.getAllKeys(query); + } request.onsuccess = () => resolve(request.result); request.onerror = (evt) => reject(evt); }); } -async function idbCount() { +async function idbCount(folder = null) { if (!db) return null; return new Promise((resolve, reject) => { - const request = db + let request; + const store = db .transaction('thumbs', 'readonly') - .objectStore('thumbs') - .count(); + .objectStore('thumbs'); + if (folder) { + request = store.index('folder').count(folder); + } else { + request = store.count(); + } request.onsuccess = () => resolve(request.result); request.onerror = (evt) => reject(evt); }); } -async function idbClean(keepSet, folder = null) { +/** + * @param {Set} keepSet - Set containing the hashes of the current files in the folder. + * @param {string} folder - Folder name/path + * @param {updateMsgCallback} msgCallback - Callback for updating progress display + */ +async function idbClean(keepSet, folder, msgCallback) { if (!db) return null; if (!(keepSet instanceof Set)) { throw new TypeError('IndexedDB cleaning function must be given a Set() of hashes to keep'); } - if (folder === null) { + if (!folder) { throw new Error('IndexedDB cleaning function must be told the current active folder'); } + const folderCached = new Set(await idbGetAllKeys('folder', folder)); + const removals = folderCached.difference(keepSet); + const totalRemovals = removals.size; + let counter = 0; return new Promise((resolve, reject) => { - let counter = 0; - const request = db + const folderIndex = db .transaction('thumbs', 'readwrite') .objectStore('thumbs') - .openCursor(); + .index('folder'); + const request = folderIndex.openCursor(folder); + request.onsuccess = (evt) => { const cursor = evt.target.result; if (cursor) { - if (folder === cursor.value.folder && !keepSet.has(cursor.key)) { - cursor.delete(); + if (removals.has(cursor.primaryKey)) { counter++; + cursor.delete(); + } + if (counter === totalRemovals) { + resolve(counter); // Got lucky with element order and can stop early + } else { + if (counter % 100 === 0 && counter !== 0) { + msgCallback(Math.floor((counter / totalRemovals) * 100)); + } + cursor.continue(); } - cursor.continue(); } else { - resolve(counter, folder); + resolve(counter); } }; request.onerror = (evt) => reject(evt);