diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b7bac782..e9c2c69ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,11 @@ # Change Log for SD.Next -## Update for 2025-03-11 +## Update for 2025-03-12 - fix installer not starting when older version of rich is installed - fix circular imports when debug flags are enabled - fix cuda errors with directml +- fix memory stats not displaying the ram usage - **ipex** - add xpu to profiler - fix untyped_storage, torch.eye and torch.cuda.device ops diff --git a/modules/call_queue.py b/modules/call_queue.py index af6f2e4d0..7e7ab16f5 100644 --- a/modules/call_queue.py +++ b/modules/call_queue.py @@ -77,15 +77,18 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False, name=None): gpu = '' cpu = '' if not shared.mem_mon.disabled: - vram = {k: -(v//-(1024*1024)) for k, v in shared.mem_mon.read().items()} + mem_mon_read = shared.mem_mon.read() + ooms = mem_mon_read.pop("oom") + retries = mem_mon_read.pop("retries") + vram = {k: v//1048576 for k, v in mem_mon_read.items()} peak = max(vram['active_peak'], vram['reserved_peak'], vram['used']) used = round(100.0 * peak / vram['total']) if vram['total'] > 0 else 0 if used > 0: gpu += f"| GPU {peak} MB {used}%" - gpu += f" | retries {vram['retries']} oom {vram['oom']}" if vram.get('retries', 0) > 0 or vram.get('oom', 0) > 0 else '' - ram = shared.ram_stats() - if ram['used'] > 0: - cpu += f"| RAM {ram['used']} GB {round(100.0 * ram['used'] / ram['total'])}%" + gpu += f" | retries {retries} oom {ooms}" if retries > 0 or ooms > 0 else '' + ram = shared.ram_stats() + if ram['used'] > 0: + cpu += f"| RAM {ram['used']} GB {round(100.0 * ram['used'] / ram['total'])}%" if isinstance(res, list): res[-1] += f"

Time: {elapsed_text} | {summary} {gpu} {cpu}

" return tuple(res) diff --git a/modules/memmon.py b/modules/memmon.py index d9fa3963d..01a41d76d 100644 --- a/modules/memmon.py +++ b/modules/memmon.py @@ -44,10 +44,10 @@ class MemUsageMonitor(): self.data["free"], self.data["total"] = torch.cuda.mem_get_info(self.device.index if self.device.index is not None else torch.cuda.current_device()) self.data["used"] = self.data["total"] - self.data["free"] torch_stats = torch.cuda.memory_stats(self.device) - self.data["active"] = torch_stats.get("active.all.current", torch_stats["active_bytes.all.current"]) - self.data["active_peak"] = torch_stats["active_bytes.all.peak"] - self.data["reserved"] = torch_stats["reserved_bytes.all.current"] - self.data["reserved_peak"] = torch_stats["reserved_bytes.all.peak"] + self.data["active"] = torch_stats.get("active.all.current", torch_stats.get("active_bytes.all.current", -1)) + self.data["active_peak"] = torch_stats.get("active_bytes.all.peak", -1) + self.data["reserved"] = torch_stats.get("reserved_bytes.all.current", -1) + self.data["reserved_peak"] = torch_stats.get("reserved_bytes.all.peak", -1) self.data['retries'] = torch_stats.get("num_alloc_retries", -1) self.data['oom'] = torch_stats.get("num_ooms", -1) except Exception: diff --git a/modules/memstats.py b/modules/memstats.py index 1ff7f721f..f01cfc2e1 100644 --- a/modules/memstats.py +++ b/modules/memstats.py @@ -77,7 +77,7 @@ def ram_stats(): process = psutil.Process(os.getpid()) res = process.memory_info() ram_total = 100 * res.rss / process.memory_percent() - ram_total = min(ram_total, docker_limit(), runpod_limit()) + ram_total = min(ram_total, get_docker_limit(), get_runpod_limit()) ram = { 'used': gb(res.rss), 'total': gb(ram_total) } return ram except Exception: