improve server monitor and profiling

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log for SD.Next
 
-## Update for 2026-08-04
+## Update for 2026-08-05
 
 - **Models**
   - [SeFi-Image](https://huggingface.co/SeFi-Image/SeFi-Image-5B-RL) in *Base*, *Turbo* (distilled) and *RL* (finetuned) variants  
@@ -25,6 +25,7 @@
   - startup: optimized server startup
   - process: preserve audio when processing video
   - remove background: new [lucida](https://huggingface.co/egeorcun/lucida) model
+  - profile flag now logs all http requests and internal tasks
 - **API**
   - add `/sdapi/v1/storage` endpoint to return storage usage info  
 - **Internal**
@@ -36,6 +37,7 @@
   - torch reset compile cache on reload
   - bypass sdna for caption/prompt-enhance calls
   - skip sdnq for small weights  
+  - server monitor keep websocket open
 
 ## Update for 2026-07-23
 

TODO.md

@@ -19,6 +19,7 @@
 ### Unassigned
 
 - [Object clear](https://huggingface.co/jixin0101/ObjectClear) remover for Kanvas
+- [MiniMax H3](https://github.com/huggingface/diffusers/pull/14355)
 - Video models: add to Reference
 - Video models: support custom entries, finetunes
 - UI Lite vs Expert mode

modules/api/middleware.py

@@ -1,12 +1,14 @@
 import ssl
 import time
 import logging
+import asyncio
 from asyncio.exceptions import CancelledError
 import anyio
 import starlette
 import uvicorn
 import fastapi
 from starlette.responses import JSONResponse
+from starlette.websockets import WebSocket, WebSocketDisconnect
 from fastapi import FastAPI, Request, Response
 from fastapi.exceptions import HTTPException
 from fastapi.responses import RedirectResponse
@@ -30,16 +32,54 @@ def validate_subpath(endpoint: str, subpath: str | None):
         return RedirectResponse(url=url, status_code=308)
     return None
 
+class LoopInstrumentorMiddleware:
+    def __init__(self, app):
+        self.app = app
+        self.instrumented = False
+
+    async def __call__(self, scope, receive, send):
+        if not self.instrumented:
+            loop = asyncio.get_running_loop()
+            def verbose_task_factory(loop, coro, context=None):
+                coro_name = getattr(coro, '__qualname__', str(coro))
+                frame = getattr(coro, 'cr_frame', None)
+                origin = f"{frame.f_code.co_filename}:{frame.f_lineno}" if frame else "unknown"
+                log.trace(f"HTTP: coro={coro_name} fn={origin}")
+                if context is not None:
+                    return asyncio.Task(coro, loop=loop, name=coro_name, context=context)
+                return asyncio.Task(coro, loop=loop, name=coro_name)
+
+            loop.set_task_factory(verbose_task_factory)
+            self.instrumented = True
+
+        await self.app(scope, receive, send)
+
+
+def setup_logging(debug: bool = False):
+    level = logging.DEBUG if debug else logging.WARNING
+    logging.getLogger("httpcore").setLevel(level)
+    logging.getLogger("httpx").setLevel(level)
+    logging.getLogger("uvicorn.access").setLevel(level)
+    logging.getLogger("asyncio").setLevel(level)
+    if not debug:
+        logging.getLogger("uvicorn.error").disabled = True
+    if debug:
+        asyncio_logger = logging.getLogger("asyncio")
+        if not asyncio_logger.handlers:
+            handler = logging.StreamHandler()
+            handler.setFormatter(logging.Formatter("[asyncio] %(message)s"))
+            asyncio_logger.addHandler(handler)
+
 
 def setup_middleware(app: FastAPI, cmd_opts):
     ssl._create_default_https_context = ssl._create_unverified_context # pylint: disable=protected-access
-    uvicorn_logger=logging.getLogger("uvicorn.error")
-    uvicorn_logger.disabled = True
     from fastapi.middleware.cors import CORSMiddleware
     from fastapi.middleware.gzip import GZipMiddleware
     app.user_middleware = [x for x in app.user_middleware if x.cls.__name__ != 'CORSMiddleware']
     app.middleware_stack = None # reset current middleware to allow modifying user provided list
     app.add_middleware(GZipMiddleware, minimum_size=2048)
+    if cmd_opts.profile:
+        app.add_middleware(LoopInstrumentorMiddleware)
     if cmd_opts.cors_origins and cmd_opts.cors_regex:
         app.add_middleware(CORSMiddleware, allow_origins=cmd_opts.cors_origins.split(','), allow_origin_regex=cmd_opts.cors_regex, allow_methods=['*'], allow_credentials=True, allow_headers=['*'])
     elif cmd_opts.cors_origins:
@@ -123,3 +163,20 @@ def setup_middleware(app: FastAPI, cmd_opts):
 
     app.build_middleware_stack() # rebuild middleware stack on-the-fly
     log.debug(f'API middleware: {[m.cls.__name__ for m in app.user_middleware]}')
+
+    @app.websocket("/internal/monitor")
+    async def ws_monitor(ws: WebSocket):
+        await ws.accept()
+        try:
+            while True:
+                await asyncio.sleep(1.0)
+                await ws.send_json({"status": "ok"})
+        except WebSocketDisconnect:
+            pass # Expected when client navigates away or closes tab
+        except Exception as e:
+            log.error(f'WebSocket monitor: {e}')
+        finally:
+            try:
+                await ws.close()
+            except RuntimeError:
+                pass  # Socket was already closed by client

modules/api/validate.py

@@ -41,7 +41,7 @@ log_exclude_prefix = ['/assets']
 
 
 class Limiter():
-    def __init__(self, limit, subpath=None):
+    def __init__(self, limit, subpath=None, debug=False):
         import limits
         self.request_backend = limits.storage.MemoryStorage()
         self.request_limit = limit # default is 300 requests per minute
@@ -53,6 +53,7 @@ class Limiter():
         self.log_limiter = limits.parse(f"{self.log_limit}/minute")
         self.summary = {}
         self.subpath = subpath
+        self.debug = debug
         log.info(f'API: limit={self.request_limit} strategy={self.request_strategy.__class__.__name__} backend={self.request_backend.__class__.__name__} subpath={self.subpath}')
 
 
@@ -75,6 +76,8 @@ class Limiter():
         return status
 
     def check_log(self, client: str, api: str):
+        if self.debug:
+            return True
         if self.log_limit < 0:
             return True
         if any(api.endswith(s) for s in log_exclude_suffix):
@@ -99,7 +102,7 @@ def validate_request(client, endpoint):
     global limiter # pylint: disable=global-statement
     from modules.shared import opts, cmd_opts
     if opts.server_rate_limit != limiter.request_limit:
-        limiter = Limiter(opts.server_rate_limit, cmd_opts.subpath)
+        limiter = Limiter(opts.server_rate_limit, cmd_opts.subpath, cmd_opts.profile)
     api = re.match(r"^[^?#&=]+", endpoint).group(0)
 
     if (limiter.subpath is not None) and (len(limiter.subpath) > 0) and api.startswith(limiter.subpath): # strip subpath from api for rate limiting

modules/logger.py

@@ -281,11 +281,7 @@ def setup_logging(debug=None, trace=None, filename=None):
     logging.getLogger("lycoris").handlers = log.handlers
     logging.getLogger("ControlNet").handlers = log.handlers
 
-    logging.getLogger("asyncio").setLevel(logging.ERROR)
     logging.getLogger("diffusers").setLevel(logging.ERROR)
     logging.getLogger("transformers").setLevel(logging.ERROR)
-    logging.getLogger("httpcore").setLevel(logging.ERROR)
-    logging.getLogger("httpx").setLevel(logging.ERROR)
     logging.getLogger("torch").setLevel(logging.ERROR)
     logging.getLogger("urllib3").setLevel(logging.ERROR)
-    logging.getLogger("uvicorn.access").setLevel(logging.ERROR)

ui/dist/sdnext.mjs

@@ -12424,10 +12424,11 @@ async function initSettings() {
 
 // ui/monitor.ts
 var monitorActive = false;
+var wsTimer;
 var ConnectionMonitorState = class _ConnectionMonitorState {
   static ws;
   static url = "";
-  static delay = 1e3;
+  static delay = 2e3;
   static element;
   static version = "";
   static commit = "";
@@ -12476,25 +12477,46 @@ var ConnectionMonitorState = class _ConnectionMonitorState {
   }
 };
 async function updateIndicator(online, data = {}, msg) {
+  console.error("HERE", { online, data, msg });
   ConnectionMonitorState.setData({ online, data });
   ConnectionMonitorState.updateState();
   if (msg) log("monitorConnection:", { online, data, msg });
 }
+function scheduleNextLoop() {
+  if (wsTimer) {
+    clearTimeout(wsTimer);
+    wsTimer = void 0;
+  }
+  const offlineDurationMs = Date.now() - ConnectionMonitorState.ts.getTime();
+  if (!ConnectionMonitorState.online && offlineDurationMs > 60 * 60 * 1e3) ConnectionMonitorState.delay = 1e4;
+  else if (!ConnectionMonitorState.online && offlineDurationMs > 5 * 60 * 1e3) ConnectionMonitorState.delay = 5e3;
+  else ConnectionMonitorState.delay = 1e3;
+  wsTimer = setTimeout(wsMonitorLoop, ConnectionMonitorState.delay);
+}
 async function wsMonitorLoop() {
-  const delayed = Date.now() - ConnectionMonitorState.ts.getTime();
-  if (delayed > 60 * 60 && ConnectionMonitorState.delay < 10 && !ConnectionMonitorState.online) ConnectionMonitorState.delay = 1e4;
-  else if (delayed > 5 * 60 && ConnectionMonitorState.delay < 5 && !ConnectionMonitorState.online) ConnectionMonitorState.delay = 5e3;
-  else ConnectionMonitorState.delay = 2e3;
+  if (ConnectionMonitorState.ws) {
+    ConnectionMonitorState.ws.onopen = null;
+    ConnectionMonitorState.ws.onmessage = null;
+    ConnectionMonitorState.ws.onclose = null;
+    ConnectionMonitorState.ws.onerror = null;
+    try {
+      ConnectionMonitorState.ws.close();
+    } catch {
+    }
+    ConnectionMonitorState.ws = void 0;
+  }
   try {
-    ConnectionMonitorState.ws = new WebSocket(`${ConnectionMonitorState.url}/queue/join`);
-    ConnectionMonitorState.ws.onopen = () => {
+    ConnectionMonitorState.ws = new WebSocket(`${ConnectionMonitorState.url}/internal/monitor`);
+    ConnectionMonitorState.ws.onopen = () => updateIndicator(true);
+    ConnectionMonitorState.ws.onmessage = (msg) => updateIndicator(true, msg.data ? JSON.parse(msg.data) : {});
+    ConnectionMonitorState.ws.onclose = () => {
+      updateIndicator(false);
+      scheduleNextLoop();
     };
-    ConnectionMonitorState.ws.onmessage = () => updateIndicator(true);
-    ConnectionMonitorState.ws.onclose = () => setTimeout(wsMonitorLoop, ConnectionMonitorState.delay);
     ConnectionMonitorState.ws.onerror = (e) => updateIndicator(false, {}, String(e.message || "unknown error"));
   } catch (e) {
     updateIndicator(false, {}, String(e.message || e));
-    setTimeout(monitorConnection, ConnectionMonitorState.delay);
+    scheduleNextLoop();
   }
 }
 async function monitorConnection() {
@@ -12518,7 +12540,7 @@ async function monitorConnection() {
     wsMonitorLoop();
   } catch {
     updateIndicator(false, data);
-    setTimeout(monitorConnection, ConnectionMonitorState.delay);
+    scheduleNextLoop();
   }
 }
 

ui/monitor.ts

@@ -10,11 +10,12 @@ interface VersionInfo {
 }
 
 let monitorActive = false;
+let wsTimer: ReturnType<typeof setTimeout> | undefined;
 
 export class ConnectionMonitorState {
   static ws: WebSocket | undefined;
   static url = '';
-  static delay = 1000;
+  static delay = 2000;
   static element: HTMLElement | undefined;
   static version = '';
   static commit = '';
@@ -69,26 +70,51 @@ export class ConnectionMonitorState {
 }
 
 async function updateIndicator(online: boolean, data: VersionInfo = {}, msg?: string): Promise<void> {
+  console.error('HERE', { online, data, msg });
   ConnectionMonitorState.setData({ online, data });
   ConnectionMonitorState.updateState();
   if (msg) log('monitorConnection:', { online, data, msg });
 }
 
+function scheduleNextLoop() {
+  if (wsTimer) {
+    clearTimeout(wsTimer);
+    wsTimer = undefined;
+  }
+  const offlineDurationMs = Date.now() - ConnectionMonitorState.ts.getTime();
+  if (!ConnectionMonitorState.online && offlineDurationMs > (60 * 60 * 1000)) ConnectionMonitorState.delay = 10000;
+  else if (!ConnectionMonitorState.online && offlineDurationMs > (5 * 60 * 1000)) ConnectionMonitorState.delay = 5000;
+  else ConnectionMonitorState.delay = 1000;
+  wsTimer = setTimeout(wsMonitorLoop, ConnectionMonitorState.delay); // eslint-disable-line @typescript-eslint/no-use-before-define
+}
+
 async function wsMonitorLoop() {
-  const delayed = Date.now() - ConnectionMonitorState.ts.getTime();
-  if ((delayed > 60 * 60) && (ConnectionMonitorState.delay < 10) && !ConnectionMonitorState.online) ConnectionMonitorState.delay = 10000;
-  else if ((delayed > 5 * 60) && (ConnectionMonitorState.delay < 5) && !ConnectionMonitorState.online) ConnectionMonitorState.delay = 5000;
-  else ConnectionMonitorState.delay = 2000;
+  // Tear down any existing socket before creating a new one
+  if (ConnectionMonitorState.ws) {
+    ConnectionMonitorState.ws.onopen = null;
+    ConnectionMonitorState.ws.onmessage = null;
+    ConnectionMonitorState.ws.onclose = null;
+    ConnectionMonitorState.ws.onerror = null;
+    try {
+      ConnectionMonitorState.ws.close();
+    } catch {
+      // Ignore cleanup errors on stale sockets
+    }
+    ConnectionMonitorState.ws = undefined;
+  }
+
   try {
-    ConnectionMonitorState.ws = new WebSocket(`${ConnectionMonitorState.url}/queue/join`);
-    ConnectionMonitorState.ws.onopen = () => {};
-    ConnectionMonitorState.ws.onmessage = () => updateIndicator(true);
-    ConnectionMonitorState.ws.onclose = () => setTimeout(wsMonitorLoop, ConnectionMonitorState.delay); // main re-check loop
-    ConnectionMonitorState.ws.onerror = (e: Event) => updateIndicator(false, {}, String((e as ErrorEvent).message || 'unknown error')); // actual error
+    ConnectionMonitorState.ws = new WebSocket(`${ConnectionMonitorState.url}/internal/monitor`);
+    ConnectionMonitorState.ws.onopen = () => updateIndicator(true);
+    ConnectionMonitorState.ws.onmessage = (msg: MessageEvent) => updateIndicator(true, msg.data ? JSON.parse(msg.data) : {});
+    ConnectionMonitorState.ws.onclose = () => {
+      updateIndicator(false);
+      scheduleNextLoop();
+    };
+    ConnectionMonitorState.ws.onerror = (e: Event) => updateIndicator(false, {}, String((e as ErrorEvent).message || 'unknown error'));
   } catch (e) {
     updateIndicator(false, {}, String((e as Error).message || e));
-    // eslint-disable-next-line @typescript-eslint/no-use-before-define
-    setTimeout(monitorConnection, ConnectionMonitorState.delay);
+    scheduleNextLoop();
   }
 }
 
@@ -114,6 +140,6 @@ export async function monitorConnection() {
     wsMonitorLoop();
   } catch {
     updateIndicator(false, data);
-    setTimeout(monitorConnection, ConnectionMonitorState.delay);
+    scheduleNextLoop();
   }
 }

webui.py

@@ -207,31 +207,57 @@ def create_api(app):
     return api
 
 
-def async_policy():
-    _BasePolicy = asyncio.WindowsSelectorEventLoopPolicy if sys.platform == "win32" and hasattr(asyncio, "WindowsSelectorEventLoopPolicy") else asyncio.DefaultEventLoopPolicy
+def verbose_task_factory(loop, coro, context=None):
+    """Custom task factory that intercepts and logs every created task."""
+    # Retrieve the origin frame (where asyncio.create_task was called)
+    frame = coro.cr_frame if hasattr(coro, 'cr_frame') and coro.cr_frame else None
+    origin = f"{frame.f_code.co_filename}:{frame.f_lineno}" if frame else "unknown origin"
 
-    class AnyThreadEventLoopPolicy(_BasePolicy):
-        def handle_exception(self, context):
-            msg = context.get("exception", context["message"])
-            log.error(f"AsyncIO loop: {msg}")
+    # Get the coroutine function name
+    coro_name = getattr(coro, '__qualname__', str(coro))
+
+    print(f"[TASK CREATED] {coro_name} | Origin: {origin}")
+
+    # Fallback to the default Task creation (handles Python 3.11+ context kwargs)
+    if context is not None:
+        return asyncio.Task(coro, loop=loop, name=coro_name, context=context)
+    return asyncio.Task(coro, loop=loop, name=coro_name)
+
+
+def async_policy():
+    if sys.platform == "win32" and hasattr(asyncio, "WindowsSelectorEventLoopPolicy"):
+        AsyncPolicy = asyncio.WindowsSelectorEventLoopPolicy
+    else:
+        AsyncPolicy = asyncio.DefaultEventLoopPolicy
+
+    class AnyThreadEventLoopPolicy(AsyncPolicy):
+        @staticmethod
+        def handle_exception(loop, context):
+            msg = context.get("exception", context.get("message"))
+            log.error(f"AsyncIO: loop={loop}: {msg}")
+
+        def new_event_loop(self) -> asyncio.AbstractEventLoop:
+            """Ensure custom exception handler is attached whenever a loop is created."""
+            loop = super().new_event_loop()
+            if shared.cmd_opts.profile:
+                loop.slow_callback_duration = 0.001
+            loop.set_debug(shared.cmd_opts.profile)
+            log.debug(f'AsyncIO: loop={loop}')
+            loop.set_task_factory(verbose_task_factory)
+            loop.set_exception_handler(self.handle_exception)
+            return loop
 
         def get_event_loop(self) -> asyncio.AbstractEventLoop:
+            """Get current thread's event loop, creating one if none exists (thread-safe)."""
             try:
-                self.loop = super().get_event_loop()
+                return super().get_event_loop()
             except (RuntimeError, AssertionError):
-                self.loop = self.new_event_loop()
-                self.set_event_loop(self.loop)
-            return self.loop
-
-        def __init__(self):
-            super().__init__()
-            self.loop = self.get_event_loop()
-            self.loop.set_exception_handler(self.handle_exception)
-            # log.debug(f"Event loop: {self.loop}")
+                loop = self.new_event_loop()
+                self.set_event_loop(loop)
+                return loop
 
     asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy())
 
-
 def get_external_ip():
     import socket
     try:
@@ -381,6 +407,7 @@ def start_ui():
     # log.debug(f'Gradio functions: registered={len(shared.demo.fns)}')
     shared.demo.server.wants_restart = False
     modules.api.middleware.setup_middleware(app, shared.cmd_opts)
+    modules.api.middleware.setup_logging(debug=shared.cmd_opts.profile)
 
     timer.startup.record("launch")