mirror of
https://github.com/vladmandic/automatic
synced 2026-09-19 17:24:32 +02:00
feat(lora): project adaln deltas onto the pruned minimax basis
The pruned transformer stores W @ P for each AdaLN projection against the rank-8 basis in time_embedder.basis, so deltas trained at the released width failed the shape check. try_load_lora gains an adapt_weights hook and the MiniMax loader refits lora_down as down @ P, which is exact.
This commit is contained in:
@@ -303,19 +303,19 @@ def finalize_network(net, name, family, lora_scale, t0, unmapped=0, mismatch=0,
|
||||
return net
|
||||
|
||||
|
||||
def shapes_match(sd_module, down_w: torch.Tensor, up_w: torch.Tensor) -> bool:
|
||||
"""LoRA-style rank-and-dim sanity check against the live module weight.
|
||||
|
||||
Honors SDNQ-quantized modules by reading the original shape from the
|
||||
dequantizer rather than the packed weight tensor.
|
||||
"""
|
||||
def module_shape(sd_module):
|
||||
"""The live weight shape of a module, read from the dequantizer for SDNQ-quantized layers; None without a weight."""
|
||||
if not hasattr(sd_module, "weight"):
|
||||
return False
|
||||
return None
|
||||
if hasattr(sd_module, "sdnq_dequantizer"):
|
||||
mod_shape = sd_module.sdnq_dequantizer.original_shape
|
||||
else:
|
||||
mod_shape = sd_module.weight.shape
|
||||
if len(mod_shape) < 2 or len(down_w.shape) < 2 or len(up_w.shape) < 2:
|
||||
return tuple(sd_module.sdnq_dequantizer.original_shape)
|
||||
return tuple(sd_module.weight.shape)
|
||||
|
||||
|
||||
def shapes_match(sd_module, down_w: torch.Tensor, up_w: torch.Tensor) -> bool:
|
||||
"""LoRA-style rank-and-dim sanity check against the live module weight."""
|
||||
mod_shape = module_shape(sd_module)
|
||||
if mod_shape is None or len(mod_shape) < 2 or len(down_w.shape) < 2 or len(up_w.shape) < 2:
|
||||
return False
|
||||
return down_w.shape[1] == mod_shape[1] and up_w.shape[0] == mod_shape[0]
|
||||
|
||||
@@ -576,6 +576,7 @@ def try_load_lora(name, network_on_disk, lora_scale, *,
|
||||
network_prefix=NETWORK_PREFIX_DEFAULT,
|
||||
group_by_suffixes_fn=group_by_suffixes,
|
||||
network_alpha=None,
|
||||
adapt_weights=None,
|
||||
arch_name="generic"):
|
||||
"""Generic LoRA loader (handles DoRA via the universal ``finalize_updown`` hook).
|
||||
|
||||
@@ -584,6 +585,10 @@ def try_load_lora(name, network_on_disk, lora_scale, *,
|
||||
|
||||
``network_alpha`` is a file-level alpha for files without alpha tensors;
|
||||
a file carrying any alpha of its own keeps those and ignores it.
|
||||
|
||||
``adapt_weights(sd_module, network_key, w)`` lets an arch refit a delta onto
|
||||
a module whose live layout differs from the trained one (a pruned AdaLN
|
||||
basis, for instance) before the shape check; returning None keeps ``w``.
|
||||
"""
|
||||
t0 = time.time()
|
||||
state_dict = read_state_dict(network_on_disk.filename, what="network")
|
||||
@@ -647,6 +652,9 @@ def try_load_lora(name, network_on_disk, lora_scale, *,
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if adapt_weights is not None:
|
||||
target_w = adapt_weights(sd_module, network_key, target_w) or target_w
|
||||
|
||||
if not shapes_match(sd_module, target_w["lora_down.weight"], target_w["lora_up.weight"]):
|
||||
if l.debug:
|
||||
_module = f'{getattr(sd_module, "weight", None).shape if hasattr(sd_module, "weight") else "?"}'
|
||||
|
||||
Reference in New Issue
Block a user