diff --git a/CHANGELOG.md b/CHANGELOG.md index cc4f24636..a3ce1dfba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2025-08-09 +## Update for 2025-08-10 -### Highlights for 2025-08-09 +### Highlights for 2025-08-10 Several new models: [Qwen-Image](https://qwenlm.github.io/blog/qwen-image/) and [FLUX.1-Krea-Dev](https://www.krea.ai/blog/flux-krea-open-source-release) Several updated models: [Chroma](https://huggingface.co/lodestones/Chroma), [SkyReels-V2](https://huggingface.co/Skywork/SkyReels-V2-DF-14B-720P-Diffusers) and [Wan-VACE](https://huggingface.co/Wan-AI/Wan2.1-VACE-14B-diffusers) @@ -12,7 +12,7 @@ And (*as always*) many bugfixes and improvements to existing features! [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -### Details for 2025-08-09 +### Details for 2025-08-10 - **Models** - [Qwen-Image](https://qwenlm.github.io/blog/qwen-image/) @@ -113,7 +113,8 @@ And (*as always*) many bugfixes and improvements to existing features! - fix *Flux.1-Kontext-Dev* with variable resolution - use `utf_16_be` as primary metadata decoding - fix `sd35` width/height alignment - - fix `nudenet` api + - fix `nudenet` api + - fix global state tracking - reapply offloading on ipadapter load - api set default script-name - avoid forced gc and rely on thresholds diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index bc510ba7e..9a9c0f8ed 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit bc510ba7e5887db5342f6ad463943849755f4b8f +Subproject commit 9a9c0f8ed77e6f42f9e2528e547cbc242d801bb3 diff --git a/modules/api/nvml.py b/modules/api/nvml.py index 4ecc73b0e..177c4fc54 100644 --- a/modules/api/nvml.py +++ b/modules/api/nvml.py @@ -25,9 +25,11 @@ def get_reason(val): reason = ', '.join([throttle[i] for i in throttle if i & val]) return reason if len(reason) > 0 else 'ok' + def get_nvml(): global nvml_initialized # pylint: disable=global-statement try: + from modules.memstats import ram_stats if not nvml_initialized: install('pynvml', quiet=True) import pynvml # pylint: disable=redefined-outer-name @@ -44,33 +46,28 @@ def get_nvml(): except Exception: name = '' load = pynvml.nvmlDeviceGetUtilizationRates(dev) - """ - 'load': { - 'gpu': round(load.gpu), - 'memory': round(load.memory), - 'temp': pynvml.nvmlDeviceGetTemperature(dev, 0), - 'fan': pynvml.nvmlDeviceGetFanSpeed(dev), - }, - 'chart_val1': load.memory, - 'chart_val2': load.gpu, - } - """ mem = pynvml.nvmlDeviceGetMemoryInfo(dev) + ram = ram_stats() data = { - "CUDA": f'version {pynvml.nvmlSystemGetCudaDriverVersion()} compute {pynvml.nvmlDeviceGetCudaComputeCapability(dev)}', + "CUDA": f'Version {pynvml.nvmlSystemGetCudaDriverVersion()} Compute {pynvml.nvmlDeviceGetCudaComputeCapability(dev)}', "Driver": pynvml.nvmlSystemGetDriverVersion(), "Hardware": f'VBIOS {pynvml.nvmlDeviceGetVbiosVersion(dev)} ROM {pynvml.nvmlDeviceGetInforomImageVersion(dev)}', - "PCI link": f'gen.{pynvml.nvmlDeviceGetCurrPcieLinkGeneration(dev)} x{pynvml.nvmlDeviceGetCurrPcieLinkWidth(dev)}', + "PCI link": f'Gen.{pynvml.nvmlDeviceGetCurrPcieLinkGeneration(dev)} x{pynvml.nvmlDeviceGetCurrPcieLinkWidth(dev)}', "Power": f'{round(pynvml.nvmlDeviceGetPowerUsage(dev)/1000, 2)} W / {round(pynvml.nvmlDeviceGetEnforcedPowerLimit(dev)/1000, 2)} W', "GPU clock": f'{pynvml.nvmlDeviceGetClockInfo(dev, 0)} Mhz / {pynvml.nvmlDeviceGetMaxClockInfo(dev, 0)} Mhz', "SM clock": f'{pynvml.nvmlDeviceGetClockInfo(dev, 1)} Mhz / {pynvml.nvmlDeviceGetMaxClockInfo(dev, 1)} Mhz', - "Memory clock": f'{pynvml.nvmlDeviceGetClockInfo(dev, 2)} Mhz / {pynvml.nvmlDeviceGetMaxClockInfo(dev, 2)} Mhz', - "Memory usage": f'used {round(mem.used / 1024 / 1024)} MB | free {round(mem.free / 1024 / 1024)} MB | total {round(mem.total / 1024 / 1024)} MB', - "System load": f'GPU {load.gpu}% | Memory {load.memory}% | temp {pynvml.nvmlDeviceGetTemperature(dev, 0)}C | fan {pynvml.nvmlDeviceGetFanSpeed(dev)}%', + "VRAM clock": f'{pynvml.nvmlDeviceGetClockInfo(dev, 2)} Mhz / {pynvml.nvmlDeviceGetMaxClockInfo(dev, 2)} Mhz', + "VRAM usage": f'{round(100 * mem.used / mem.total)}% | {round(mem.used / 1024 / 1024)} MB used | {round(mem.free / 1024 / 1024)} MB free | {round(mem.total / 1024 / 1024)} MB total', + "RAM usage": f'{round(100 * ram["used"] / ram["total"])}% | {round(1024 * ram["used"])} MB used | {round(1024 * ram["free"])} MB free | {round(1024 * ram["total"])} MB total', + "System load": f'GPU {load.gpu}% | VRAM {load.memory}% | Temp {pynvml.nvmlDeviceGetTemperature(dev, 0)}C | Fan {pynvml.nvmlDeviceGetFanSpeed(dev)}%', 'State': get_reason(pynvml.nvmlDeviceGetCurrentClocksThrottleReasons(dev)), } chart = [load.memory, load.gpu] - devices.append({ 'name': name, 'data': data, 'chart': chart }) + devices.append({ + 'name': name, + 'data': data, + 'chart': chart, + }) # log.debug(f'nmvl: {devices}') return devices except Exception as e: diff --git a/modules/api/rocm_smi.py b/modules/api/rocm_smi.py index 870bfaaed..bbf7f1ba7 100644 --- a/modules/api/rocm_smi.py +++ b/modules/api/rocm_smi.py @@ -3,12 +3,14 @@ import json import subprocess as sp from enum import IntFlag + try: from installer import log except Exception: import logging log = logging.getLogger(__name__) + try: from modules.rocm import version as rocm_version except Exception: @@ -87,14 +89,19 @@ def get_rocm_smi(): "PCI link": f'Gen.{int(math.log2(float(rocm_smi_data[key].get("pcie_link_speed (0.1 GT/s)", 10)) / 10))} x{rocm_smi_data[key].get("pcie_link_width (Lanes)", "unknown")}', "Power": f'{round(float(rocm_smi_data[key].get("Average Graphics Package Power (W)", 0)), 2)} W / {round(float(rocm_smi_data[key].get("Max Graphics Package Power (W)", 0)), 2)} W', "GPU clock": f'{rocm_smi_data[key].get("average_gfxclk_frequency (MHz)", 0)} Mhz / {rocm_smi_data[key].get("Valid sclk range", "0").split(" - ")[-1].removesuffix("Mhz")} Mhz', - "Memory clock": f'{rocm_smi_data[key].get("current_uclk (MHz)", 0)} Mhz / {rocm_smi_data[key].get("Valid mclk range", "0").split(" - ")[-1].removesuffix("Mhz")} Mhz', - "Memory usage": f'used {load["memory"]}% | activity {rocm_smi_data[key].get("GPU Memory Read/Write Activity (%)", "unknown")}%', - "GPU usage": f'GPU {load["gpu"]}% | fan {load["fan"]}%', - "GPU temp": f'edge {load["temp"]}C | junction {load["temp_junction"]}C | memory {load["temp_memory"]}C', + "VRAM clock": f'{rocm_smi_data[key].get("current_uclk (MHz)", 0)} Mhz / {rocm_smi_data[key].get("Valid mclk range", "0").split(" - ")[-1].removesuffix("Mhz")} Mhz', + "VRAM usage": f'{load["memory"]}% Used | {rocm_smi_data[key].get("GPU Memory Read/Write Activity (%)", "unknown")}% Activity', + "GPU usage": f'GPU {load["gpu"]}% | Fan {load["fan"]}%', + "GPU temp": f'Edge {load["temp"]}C | Junction {load["temp_junction"]}C | Memory {load["temp_memory"]}C', 'Throttle reason': str(ThrottleStatus(int(rocm_smi_data[key].get("throttle_status", 0)))), } + name = rocm_smi_data[key].get('Device Name', 'unknown') chart = [load["memory"], load["gpu"]] - devices.append({ 'name': rocm_smi_data[key].get('Device Name', 'unknown'), 'data': data, 'chart': chart }) + devices.append({ + 'name': name, + 'data': data, + 'chart': chart, + }) return devices except Exception as e: log.error(f'ROCm SMI: {e}') diff --git a/modules/memstats.py b/modules/memstats.py index 47469fd77..a2b8c1133 100644 --- a/modules/memstats.py +++ b/modules/memstats.py @@ -7,7 +7,10 @@ from modules import shared, errors fail_once = False +ram = {} +gpu = {} mem = {} +process = None docker_limit = None runpod_limit = None @@ -40,40 +43,58 @@ def get_runpod_limit(): return runpod_limit -def memory_stats(): - global fail_once # pylint: disable=global-statement - mem.clear() +def ram_stats(): + global process, fail_once # pylint: disable=global-statement try: - process = psutil.Process(os.getpid()) + if process is None: + process = psutil.Process(os.getpid()) res = process.memory_info() - ram_total = 100 * res.rss / process.memory_percent() - ram_total = min(ram_total, get_docker_limit(), get_runpod_limit()) - ram = { 'used': gb(res.rss), 'total': gb(ram_total) } - mem.update({ 'ram': ram }) + if 'total' not in ram: + process = psutil.Process(os.getpid()) + ram_total = 100 * res.rss / process.memory_percent() + ram_total = min(ram_total, get_docker_limit(), get_runpod_limit()) + ram['total'] = gb(ram_total) + ram['used'] = gb(res.rss) + ram['free'] = ram['total'] - ram['used'] except Exception as e: + ram['error'] = str(e) if not fail_once: - shared.log.error(f'Memory stats: {e}') - errors.display(e, 'Memory stats') + shared.log.error(f'RAM stats: {e}') + errors.display(e, 'RAM stats') fail_once = True - mem.update({ 'ram': { 'error': str(e) } }) + return ram + + +def gpu_stats(): + global fail_once # pylint: disable=global-statement try: free, total = torch.cuda.mem_get_info() - gpu = { 'used': gb(total - free), 'total': gb(total) } + gpu['used'] = gb(total - free) + gpu['total'] = gb(total) stats = dict(torch.cuda.memory_stats()) if stats.get('num_ooms', 0) > 0: shared.state.oom = True - mem.update({ - 'gpu': gpu, - 'active': gb(stats.get('active_bytes.all.current', 0)), - 'peak': gb(stats.get('active_bytes.all.peak', 0)), - 'retries': stats.get('num_alloc_retries', 0), - 'oom': stats.get('num_ooms', 0), - 'job': shared.state.job, - }) - mem['swap'] = round(mem['active'] - mem['gpu']['used'], 2) if mem['active'] > mem['gpu']['used'] else 0 - return mem - except Exception: - pass + gpu['active'] = gb(stats.get('active_bytes.all.current', 0)) + gpu['peak'] = gb(stats.get('active_bytes.all.peak', 0)) + gpu['retries'] = stats.get('num_alloc_retries', 0) + gpu['oom'] = stats.get('num_ooms', 0) + except Exception as e: + gpu['error'] = str(e) + if not fail_once: + shared.log.error(f'GPU stats: {e}') + errors.display(e, 'GPU stats') + fail_once = True + return gpu + + +def memory_stats(): + mem['ram'] = ram_stats() + mem['gpu'] = gpu_stats() + mem['job'] = shared.state.job + try: + mem['gpu']['swap'] = round(mem['gpu']['active'] - mem['gpu']['used']) if mem['gpu']['active'] > mem['gpu']['used'] else 0 + except: + mem['gpu']['swap'] = 0 return mem @@ -84,18 +105,6 @@ def reset_stats(): pass -def ram_stats(): - try: - process = psutil.Process(os.getpid()) - res = process.memory_info() - ram_total = 100 * res.rss / process.memory_percent() - ram_total = min(ram_total, get_docker_limit(), get_runpod_limit()) - ram = { 'used': gb(res.rss), 'total': gb(ram_total) } - return ram - except Exception: - return { 'used': 0, 'total': 0 } - - class Object: pattern = r"'(.*?)'" diff --git a/modules/sd_models.py b/modules/sd_models.py index cbafbea80..d0c3423b1 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1057,7 +1057,7 @@ def reload_model_weights(sd_model=None, info=None, op='model', force=False, revi unload_model_weights(op=op) return None orig_state = copy.deepcopy(shared.state) - shared.state = shared_state.State() + # shared.state = shared_state.State() shared.state.begin('Load') if sd_model is None: sd_model = model_data.sd_model if op == 'model' or op == 'dict' else model_data.sd_refiner diff --git a/modules/shared_state.py b/modules/shared_state.py index c7c1cb472..616e92cce 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -50,6 +50,9 @@ class State: server_start = time.time() oom = False + def __init__(self): + log.debug(f'State initialized: id={id(self)}') + def __str__(self) -> str: status = ' ' status += 'skipped ' if self.skipped else '' @@ -208,7 +211,8 @@ class State: log.trace(f'State end: {self}') self.time_end = time.time() self.history('end') - self.job = "" + self.id = '' + self.job = '' self.job_count = 0 self.job_no = 0 self.frame_count = 0