mirror of
https://github.com/vladmandic/automatic
synced 2026-09-01 18:46:53 +02:00
fix(sdnq): reset dynamo caches at model unload
Dynamo tracks a lifetime recompile counter per compiled function that freed models leave climbing while their graphs and guards die, and the compiled dequant runs fullgraph, so crossing the accumulated limit is a hard FailOnRecompileLimitHit instead of an eager fallback; enough model or quant switches in one process got there. unload_model_weights now calls reset_compile_caches when the compiled dequant is active, dropping the dead graphs and the counters in the same sweep as the unload gc. Raised limits only move the wall; the reset removes it. - scoped to the model unload branch: the reset is global and must only run when the graphs' owner is being discarded - regression test trips the wall under a lowered limit and recovers through the same helper the unload path calls
This commit is contained in:
@@ -1573,6 +1573,8 @@ def unload_model_weights(op='model'):
|
||||
disable_offload(model_data.sd_model)
|
||||
move_model(model_data.sd_model, 'meta')
|
||||
model_data.sd_model = None
|
||||
from modules.sdnq.common import reset_compile_caches
|
||||
reset_compile_caches() # dead compiled-dequant graphs and their lifetime recompile counters otherwise accumulate across switches
|
||||
devices.torch_gc(force=True, reason='unload')
|
||||
log.debug(f'Unload {op}: {memory_stats()} fn={fn}')
|
||||
elif (op == 'refiner') and model_data.sd_refiner:
|
||||
|
||||
@@ -368,6 +368,22 @@ else:
|
||||
return fn
|
||||
|
||||
|
||||
def reset_compile_caches():
|
||||
"""Drop compiled graphs and dynamo's lifetime recompile counters at model unload.
|
||||
|
||||
Freed layers leave their guards invalid, so the dead graphs cannot be
|
||||
reused, but the per-frame lifetime compile counter keeps climbing and a
|
||||
fullgraph compile hard-fails (FailOnRecompileLimitHit) once it crosses the
|
||||
accumulated limit; enough model or quant switches in one process get there.
|
||||
The reset is global, so call it only when the compiled graphs' owner is
|
||||
being discarded.
|
||||
"""
|
||||
if check_torch_compile():
|
||||
from modules.logger import log
|
||||
torch._dynamo.reset()
|
||||
log.debug('SDNQ compile: dynamo reset')
|
||||
|
||||
|
||||
common_skip_keys = (
|
||||
".time_embed",
|
||||
".context_embedder",
|
||||
|
||||
Reference in New Issue
Block a user