From 82e3c1d20f7e5557a3cbe03bf0c67f0cf7895f78 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sat, 25 Jul 2026 22:14:47 +0100 Subject: [PATCH] feat(lora): log select stack schedules Select modes left no trace distinguishable from plain summation: apply_select counted its layers on the exact path, and the mode field in the load summary reflects the requested setting rather than what executed. A flip count can only come from a populated schedule. - report layers, initial style picks, flips, steps and gamma from finalize - deduplicate on content, since the schedule rebuilds on every pass - give select its own apply counter instead of inflating apply=exact --- modules/lora/lora_sdnq.py | 6 +++++- modules/lora/lora_stack.py | 11 ++++++++++- modules/lora/networks.py | 1 + 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/modules/lora/lora_sdnq.py b/modules/lora/lora_sdnq.py index 45a514693..375e8d05a 100644 --- a/modules/lora/lora_sdnq.py +++ b/modules/lora/lora_sdnq.py @@ -56,6 +56,7 @@ fallback_layers: list[str] = [] hosted_layers: list[tuple[str, float, bool]] = [] hosted_ranks: list[int] = [] factor_layers: list[str] = [] +select_layers: list[str] = [] routed_layers: list[str] = [] REQUANT_RATIO = 0.30 # delta rms over mean grid step above which requantize can retain the delta @@ -507,7 +508,7 @@ def apply_select(self, network_layer_name, per_net, wanted_names): del d0, d1 segments, transposed = append_factors(self, [pairs[0][0], pairs[1][0]], [pairs[0][1], pairs[1][1]]) lora_stack.register(network_layer_name, self, 'factor', scores, segments=(segments[0], segments[1], transposed), abs_sums=abs_sums) - factor_layers.append(network_layer_name) + select_layers.append(network_layer_name) # counted apart from the plain concat: both ride the svd channel but only one is a summed set return True @@ -524,6 +525,9 @@ def report_fallbacks(): if len(factor_layers) > 0: log.info(f'Network load: type=LoRA quant=sdnq apply=exact layers={len(factor_layers)}') factor_layers.clear() + if len(select_layers) > 0: + log.info(f'Network load: type=LoRA quant=sdnq apply=select layers={len(select_layers)} mode={lora_stack.mode()}') + select_layers.clear() if len(hosted_layers) > 0: energies = sorted(e for _name, e, _c in hosted_layers) median = energies[len(energies) // 2] diff --git a/modules/lora/lora_stack.py b/modules/lora/lora_stack.py index 55453c858..38f586560 100644 --- a/modules/lora/lora_stack.py +++ b/modules/lora/lora_stack.py @@ -30,7 +30,7 @@ KLORA_BETA = 0.5 # the paper's fixed ramp offset; only the slope is user-tunable ROW_CHUNK = 512 # fp32 interiors run in first-dim slices; also fixes the DARE draw sequence SAMPLE_CAP = 1 << 22 # strided subsample bound for magnitude quantiles (full-size quantile exceeds torch limits) -state: dict = {'entries': {}, 'flips': {}, 'gamma': 1.0, 'gamma_num': 0.0, 'gamma_den': 0.0, 'gamma_e': 1.0, 'gamma_e_num': 0.0, 'gamma_e_den': 0.0, 'total_steps': 0, 'finalized': False} +state: dict = {'entries': {}, 'flips': {}, 'gamma': 1.0, 'gamma_num': 0.0, 'gamma_den': 0.0, 'gamma_e': 1.0, 'gamma_e_num': 0.0, 'gamma_e_den': 0.0, 'total_steps': 0, 'finalized': False, 'reported': None} warned: set = set() @@ -208,6 +208,7 @@ def clear(): state['gamma_e_den'] = 0.0 state['total_steps'] = 0 state['finalized'] = False + state['reported'] = None def register(layer_name, module, kind, scores, segments=None, nets=None, abs_sums=None): @@ -271,13 +272,21 @@ def finalize(total_steps): state['flips'] = {} if any(e['kind'] == 'weight' for e in state['entries'].values()): materialize_model() + style_first = 0 for layer_name, entry in list(state['entries'].items()): # snapshot: apply_selection drops entries whose module died flip_at = layer_flip_step(entry['scores'], state['total_steps']) initial = 1 if flip_at == 0 else 0 + style_first += initial apply_selection(layer_name, entry, initial) if 0 < flip_at < state['total_steps']: state['flips'].setdefault(flip_at, []).append(layer_name) state['finalized'] = True + if len(state['entries']) > 0: # only a built schedule can carry a flip count, so this is the line that shows selection is live rather than requested + gamma = state['gamma_e'] if mode() == 'estlora' else state['gamma'] + report = (mode(), len(state['entries']), style_first, sum(len(v) for v in state['flips'].values()), state['total_steps'], round(gamma, 3)) + if report != state['reported']: # rebuilt every pass, so a batch would otherwise repeat one line per image + state['reported'] = report + log.info(f'Network load: type=LoRA stack={report[0]} layers={report[1]} style={report[2]} flips={report[3]} steps={report[4]} gamma={report[5]:.3f}') def reset(total_steps): diff --git a/modules/lora/networks.py b/modules/lora/networks.py index a1af90518..0987c679d 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -96,6 +96,7 @@ def network_activate(include=None, exclude=None): lora_sdnq.fallback_layers.clear() # a raise mid-pass leaves stale entries behind lora_sdnq.hosted_layers.clear() lora_sdnq.factor_layers.clear() + lora_sdnq.select_layers.clear() backup_size = 0 for component in modules.keys(): component_wanted = wanted_names if component in components else ()