diff --git a/CHANGELOG.md b/CHANGELOG.md index 23ea9fd7c..f255a024c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ set tile size to 0 to use autodetected value - cli: sdapi.py allow manual api invoke example: `python cli/sdapi.py /sdapi/v1/sd-models` + - memory: add ram usage monitoring in addition to gpu memory usage monitoring - **Fixes** - python: fix python 3.9 compatibility - control: fix input image size diff --git a/modules/devices.py b/modules/devices.py index 0211242b5..362e87b52 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -1,5 +1,6 @@ import gc import sys +import time import contextlib import torch from modules.errors import log @@ -123,31 +124,38 @@ def get_device_for(task): def torch_gc(force=False): + t0 = time.time() mem = memstats.memory_stats() gpu = mem.get('gpu', {}) + ram = mem.get('ram', {}) oom = gpu.get('oom', 0) if backend == "directml": - used = round(100 * torch.cuda.memory_allocated() / (1 << 30) / gpu.get('total', 1)) if gpu.get('total', 1) > 1 else 0 + used_gpu = round(100 * torch.cuda.memory_allocated() / (1 << 30) / gpu.get('total', 1)) if gpu.get('total', 1) > 1 else 0 else: - used = round(100 * gpu.get('used', 0) / gpu.get('total', 1)) if gpu.get('total', 1) > 1 else 0 + used_gpu = round(100 * gpu.get('used', 0) / gpu.get('total', 1)) if gpu.get('total', 1) > 1 else 0 + used_ram = round(100 * ram.get('used', 0) / ram.get('total', 1)) if ram.get('total', 1) > 1 else 0 global previous_oom # pylint: disable=global-statement if oom > previous_oom: previous_oom = oom log.warning(f'GPU out-of-memory error: {mem}') - if used >= shared.opts.torch_gc_threshold: - log.info(f'GPU high memory utilization: {used}% {mem}') + force = True + if used_gpu >= shared.opts.torch_gc_threshold or used_ram >= shared.opts.torch_gc_threshold: + log.info(f'High memory utilization: GPU={used_gpu}% RAM={used_ram}% {mem}') force = True if not force: return - collected = gc.collect() + + # actual gc + collected = gc.collect() # python gc if cuda_ok: try: with torch.cuda.device(get_cuda_device_string()): - torch.cuda.empty_cache() + torch.cuda.empty_cache() # cuda gc torch.cuda.ipc_collect() except Exception: pass - log.debug(f'gc: collected={collected} device={torch.device(get_optimal_device_name())} {memstats.memory_stats()}') + t1 = time.time() + log.debug(f'GC: collected={collected} device={torch.device(get_optimal_device_name())} {memstats.memory_stats()} time={round(t1 - t0, 2)}') def set_cuda_sync_mode(mode): diff --git a/modules/memstats.py b/modules/memstats.py index 1d2c21162..a7c28861a 100644 --- a/modules/memstats.py +++ b/modules/memstats.py @@ -6,6 +6,7 @@ from modules import shared def memory_stats(): def gb(val: float): return round(val / 1024 / 1024 / 1024, 2) + mem = {} try: process = psutil.Process(os.getpid()) diff --git a/modules/shared.py b/modules/shared.py index 28abf6b8b..fefd0554c 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -317,7 +317,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "other_sep": OptionInfo("

Execution precision

", "", gr.HTML), "opt_channelslast": OptionInfo(False, "Use channels last as torch memory format "), "cudnn_benchmark": OptionInfo(False, "Enable full-depth cuDNN benchmark feature"), - "torch_gc_threshold": OptionInfo(80 if devices.backend == "ipex" else 90, "VRAM usage threshold before running Torch GC to clear up VRAM", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), + "torch_gc_threshold": OptionInfo(80, "Memory usage threshold before running Torch GC", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), "cuda_compile_sep": OptionInfo("

Model Compile

", "", gr.HTML), "cuda_compile": OptionInfo(False if not cmd_opts.use_openvino else True, "Compile UNet"),