feat(lora): log select stack schedules

Select modes left no trace distinguishable from plain summation: apply_select
counted its layers on the exact path, and the mode field in the load summary
reflects the requested setting rather than what executed. A flip count can only
come from a populated schedule.

- report layers, initial style picks, flips, steps and gamma from finalize
- deduplicate on content, since the schedule rebuilds on every pass
- give select its own apply counter instead of inflating apply=exact
This commit is contained in:
CalamitousFelicitousness
2026-07-25 22:14:47 +01:00
parent 3218740b20
commit 82e3c1d20f
3 changed files with 16 additions and 2 deletions
+5 -1
View File
@@ -56,6 +56,7 @@ fallback_layers: list[str] = []
hosted_layers: list[tuple[str, float, bool]] = []
hosted_ranks: list[int] = []
factor_layers: list[str] = []
select_layers: list[str] = []
routed_layers: list[str] = []
REQUANT_RATIO = 0.30 # delta rms over mean grid step above which requantize can retain the delta
@@ -507,7 +508,7 @@ def apply_select(self, network_layer_name, per_net, wanted_names):
del d0, d1
segments, transposed = append_factors(self, [pairs[0][0], pairs[1][0]], [pairs[0][1], pairs[1][1]])
lora_stack.register(network_layer_name, self, 'factor', scores, segments=(segments[0], segments[1], transposed), abs_sums=abs_sums)
factor_layers.append(network_layer_name)
select_layers.append(network_layer_name) # counted apart from the plain concat: both ride the svd channel but only one is a summed set
return True
@@ -524,6 +525,9 @@ def report_fallbacks():
if len(factor_layers) > 0:
log.info(f'Network load: type=LoRA quant=sdnq apply=exact layers={len(factor_layers)}')
factor_layers.clear()
if len(select_layers) > 0:
log.info(f'Network load: type=LoRA quant=sdnq apply=select layers={len(select_layers)} mode={lora_stack.mode()}')
select_layers.clear()
if len(hosted_layers) > 0:
energies = sorted(e for _name, e, _c in hosted_layers)
median = energies[len(energies) // 2]
+10 -1
View File
@@ -30,7 +30,7 @@ KLORA_BETA = 0.5 # the paper's fixed ramp offset; only the slope is user-tunable
ROW_CHUNK = 512 # fp32 interiors run in first-dim slices; also fixes the DARE draw sequence
SAMPLE_CAP = 1 << 22 # strided subsample bound for magnitude quantiles (full-size quantile exceeds torch limits)
state: dict = {'entries': {}, 'flips': {}, 'gamma': 1.0, 'gamma_num': 0.0, 'gamma_den': 0.0, 'gamma_e': 1.0, 'gamma_e_num': 0.0, 'gamma_e_den': 0.0, 'total_steps': 0, 'finalized': False}
state: dict = {'entries': {}, 'flips': {}, 'gamma': 1.0, 'gamma_num': 0.0, 'gamma_den': 0.0, 'gamma_e': 1.0, 'gamma_e_num': 0.0, 'gamma_e_den': 0.0, 'total_steps': 0, 'finalized': False, 'reported': None}
warned: set = set()
@@ -208,6 +208,7 @@ def clear():
state['gamma_e_den'] = 0.0
state['total_steps'] = 0
state['finalized'] = False
state['reported'] = None
def register(layer_name, module, kind, scores, segments=None, nets=None, abs_sums=None):
@@ -271,13 +272,21 @@ def finalize(total_steps):
state['flips'] = {}
if any(e['kind'] == 'weight' for e in state['entries'].values()):
materialize_model()
style_first = 0
for layer_name, entry in list(state['entries'].items()): # snapshot: apply_selection drops entries whose module died
flip_at = layer_flip_step(entry['scores'], state['total_steps'])
initial = 1 if flip_at == 0 else 0
style_first += initial
apply_selection(layer_name, entry, initial)
if 0 < flip_at < state['total_steps']:
state['flips'].setdefault(flip_at, []).append(layer_name)
state['finalized'] = True
if len(state['entries']) > 0: # only a built schedule can carry a flip count, so this is the line that shows selection is live rather than requested
gamma = state['gamma_e'] if mode() == 'estlora' else state['gamma']
report = (mode(), len(state['entries']), style_first, sum(len(v) for v in state['flips'].values()), state['total_steps'], round(gamma, 3))
if report != state['reported']: # rebuilt every pass, so a batch would otherwise repeat one line per image
state['reported'] = report
log.info(f'Network load: type=LoRA stack={report[0]} layers={report[1]} style={report[2]} flips={report[3]} steps={report[4]} gamma={report[5]:.3f}')
def reset(total_steps):
+1
View File
@@ -96,6 +96,7 @@ def network_activate(include=None, exclude=None):
lora_sdnq.fallback_layers.clear() # a raise mid-pass leaves stale entries behind
lora_sdnq.hosted_layers.clear()
lora_sdnq.factor_layers.clear()
lora_sdnq.select_layers.clear()
backup_size = 0
for component in modules.keys():
component_wanted = wanted_names if component in components else ()