From 4533acdd213fdef1a747a128ea92f0d960006529 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 21 Apr 2025 20:20:04 -0400 Subject: [PATCH] update memmon Signed-off-by: Vladimir Mandic --- modules/call_queue.py | 24 ++---------------------- modules/memmon.py | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/modules/call_queue.py b/modules/call_queue.py index 6963ca8a5..9bc0df11c 100644 --- a/modules/call_queue.py +++ b/modules/call_queue.py @@ -74,28 +74,8 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False, name=None): elapsed_s = elapsed % 60 elapsed_text = f"{elapsed_m}m {elapsed_s:.2f}s" if elapsed_m > 0 else f"{elapsed_s:.2f}s" summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ') - gpu = '' - cpu = '' - if not shared.mem_mon.disabled: - mem_mon_read = shared.mem_mon.read() - ooms = mem_mon_read.pop("oom") - retries = mem_mon_read.pop("retries") - vram = {k: v//1048576 for k, v in mem_mon_read.items()} - if 'active_peak' in vram: - peak = max(vram['active_peak'], vram['reserved_peak'], vram['used']) - used = round(100.0 * peak / vram['total']) if vram['total'] > 0 else 0 - else: - peak = 0 - used = 0 - if peak > 0: - gpu += f"| GPU {peak} MB" - gpu += f" {used}%" if used > 0 else '' - gpu += f" | retries {retries} oom {ooms}" if retries > 0 or ooms > 0 else '' - ram = shared.ram_stats() - if ram['used'] > 0: - cpu += f"| RAM {ram['used']} GB" - cpu += f" {round(100.0 * ram['used'] / ram['total'])}%" if ram['total'] > 0 else '' + memory = shared.mem_mon.summary() if isinstance(res, list) and isinstance(res[-1], str): - res[-1] += f"

Time: {elapsed_text} | {summary} {gpu} {cpu}

" + res[-1] += f"

Time: {elapsed_text} | {summary} {memory}

" return tuple(res) return f diff --git a/modules/memmon.py b/modules/memmon.py index 01a41d76d..944d85d83 100644 --- a/modules/memmon.py +++ b/modules/memmon.py @@ -53,3 +53,29 @@ class MemUsageMonitor(): except Exception: self.disabled = True return self.data + + def summary(self): + from modules.shared import ram_stats + gpu = '' + cpu = '' + gpu = '' + if not self.disabled: + mem_mon_read = self.read() + ooms = mem_mon_read.pop("oom") + retries = mem_mon_read.pop("retries") + vram = {k: v//1048576 for k, v in mem_mon_read.items()} + if 'active_peak' in vram: + peak = max(vram['active_peak'], vram['reserved_peak'], vram['used']) + used = round(100.0 * peak / vram['total']) if vram['total'] > 0 else 0 + else: + peak = 0 + used = 0 + if peak > 0: + gpu += f"| GPU {peak} MB" + gpu += f" {used}%" if used > 0 else '' + gpu += f" | retries {retries} oom {ooms}" if retries > 0 or ooms > 0 else '' + ram = ram_stats() + if ram['used'] > 0: + cpu += f"| RAM {ram['used']} GB" + cpu += f" {round(100.0 * ram['used'] / ram['total'])}%" if ram['total'] > 0 else '' + return f'{gpu} {cpu}'