improve connection monitor and server restart

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -146,6 +146,8 @@ For full details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/m
   - patch `unipc` for timesteps device placement, thanks @resonantsky
   - `civitai` search and base-model discovery improvements
   - auto-masking with `rembg`
+  - token counter formatting, thanks @awsr
+  - ui server restart
 
 ## Update for 2026-04-01
 

javascript/authWrap.js

@@ -26,7 +26,9 @@ async function authFetch(url, options = {}) {
     res = await fetch(url, options);
     if (!res.ok) error('fetch', { status: res?.status || 503, url, user, token });
   } catch (err) {
-    error('fetch', { status: res?.status || 503, url, user, token, error: err });
+    if (ConnectionMonitorState.online) {
+      error('fetch', { status: res?.status || 503, url, user, token, error: err });
+    }
   }
   return res;
 }

javascript/monitor.js

@@ -1,5 +1,8 @@
+let monitorActive = false;
+
 class ConnectionMonitorState {
   static ws = undefined;
+  static url = '';
   static delay = 1000;
   static element;
   static version = '';
@@ -8,6 +11,7 @@ class ConnectionMonitorState {
   static model = '';
   static startup = '';
   static online = false;
+  static ts = Date();
 
   static getModel() {
     const cp = opts?.sd_model_checkpoint || '';
@@ -19,7 +23,11 @@ class ConnectionMonitorState {
   }
 
   static setData({ online, data }) {
-    this.online = online;
+    if (online !== this.online) {
+      this.online = online;
+      this.ts = new Date()
+      debug('monitorState', { online: ConnectionMonitorState.online, ts: ConnectionMonitorState.ts })
+    }
     if (data?.updated) this.version = data.updated;
     if (data?.commit) this.commit = data.commit;
     if (data?.branch) this.branch = data.branch;
@@ -55,26 +63,23 @@ async function updateIndicator(online, data = {}, msg = undefined) {
   if (msg) log('monitorConnection:', { online, data, msg });
 }
 
-async function wsMonitorLoop(url) {
+async function wsMonitorLoop() {
+  const delayed = new Date() - this.ts;
+  if ((delayed > 60 * 60) && (ConnectionMonitorState.delay < 60) && !ConnectionMonitorState.online) ConnectionMonitorState.delay = 60000;
+  else if ((delayed > 60) && (this.delay < 10) && !ConnectionMonitorState.online) ConnectionMonitorState.delay = 10000;
+  else ConnectionMonitorState.delay = 2500;
   try {
-    ConnectionMonitorState.ws = new WebSocket(`${url}/queue/join`);
+    ConnectionMonitorState.ws = new WebSocket(`${ConnectionMonitorState.url}/queue/join`);
     ConnectionMonitorState.ws.onopen = () => {};
     ConnectionMonitorState.ws.onmessage = (evt) => updateIndicator(true);
-    ConnectionMonitorState.ws.onclose = () => {
-      setTimeout(() => wsMonitorLoop(url), ConnectionMonitorState.delay); // happens regularly if there is no traffic
-    };
-    ConnectionMonitorState.ws.onerror = (e) => {
-      updateIndicator(false, {}, e.message); // actual error
-      setTimeout(() => wsMonitorLoop(url), ConnectionMonitorState.delay);
-    };
+    ConnectionMonitorState.ws.onclose = () => setTimeout(wsMonitorLoop, ConnectionMonitorState.delay); // main re-check loop
+    ConnectionMonitorState.ws.onerror = (e) => updateIndicator(false, {}, e.message); // actual error
   } catch (e) {
     updateIndicator(false, {}, e.message);
     setTimeout(monitorConnection, ConnectionMonitorState.delay); // eslint-disable-line no-use-before-define
   }
 }
 
-let monitorActive = false;
-
 async function monitorConnection() {
   if (!monitorActive) { // start monitor loop only once on startup
     monitorActive = true;
@@ -91,8 +96,8 @@ async function monitorConnection() {
     data = await res.json();
     log('monitorConnection:', { data });
     ConnectionMonitorState.startup = new Date();
+    ConnectionMonitorState.url = res.url.split('/sdapi')[0].replace('https:', 'wss:').replace('http:', 'ws:'); // update global url as ws need fqdn
     updateIndicator(true, data);
-    const url = res.url.split('/sdapi')[0].replace('https:', 'wss:').replace('http:', 'ws:'); // update global url as ws need fqdn
     wsMonitorLoop(url);
   } catch {
     updateIndicator(false, data);

javascript/timers.js

@@ -7,7 +7,7 @@ async function timer(name, elapsed) {
 async function logTimers() {
   allTimers.sort((a, b) => b[1] - a[1]);
   const filteredTimers = allTimers.filter((t) => t[1] > 50);
-  log('timers', filteredTimers);
+  debug('startupTimers', filteredTimers);
   // xhrPost(`${window.api}/log`, { debug: JSON.stringify(filteredTimers) });
 }
 

javascript/ui.js

@@ -583,8 +583,15 @@ function monitorServerStatus() {
         <script>
           function monitorServerStatus() {
             fetch('${window.api}/progress?skip_current_image=true')
-              .then((res) => { !res?.ok ? setTimeout(monitorServerStatus, 1000) : location.reload(); })
-              .catch((e) => setTimeout(monitorServerStatus, 1000))
+              .then((res) => {
+                console.log('monitorServerStatus', res);
+                if (res.ok) location.reload();
+                else setTimeout(monitorServerStatus, 1000);
+              })
+              .catch((err) => {
+                console.error('monitorServerStatus', err);
+                setTimeout(monitorServerStatus, 1000);
+              })
           }
           window.onload = () => monitorServerStatus();
         </script>
@@ -594,12 +601,24 @@ function monitorServerStatus() {
   document.close();
 }
 
-function restartReload() {
+const delay = ms => new Promise((resolve) => setTimeout(resolve, ms));
+
+async function restartReload(initial=true) {
   document.body.style = 'background: #222222; font-size: 1rem; font-family:monospace; margin-top:20%; color:lightgray; text-align:center';
   document.body.innerHTML = '<h1>Server shutdown in progress...</h1>';
-  authFetch(`${window.api}/progress?skip_current_image=true`)
-    .then((res) => setTimeout(restartReload, 1000))
-    .catch((e) => setTimeout(monitorServerStatus, 500));
+  if (initial) await delay(10000);
+  try {
+    const res = await authFetch(`${window.api}/progress?skip_current_image=true`)
+    console.log('restartReload', res);
+    if (res?.ok) {
+      document.body.innerHTML = '<h1>Server restart in progress...</h1>';
+      setTimeout(() => location.reload(), 10000);
+    } else {
+      setTimeout(() => restartReload(false), 2500);
+    }
+  } catch {
+    setTimeout(() => restartReload(false), 2500);
+  }
   return [];
 }