mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-08-25 22:21:03 +02:00
model : GraniteSWAForCausalLM / GraniteMoeSWAForCausalLM (#25505)
* feat(convert): Add conversion for GraniteSWAForCausalLM Branch: GraniteSWAForCausalLM AI-usage: full (Bob, OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat(llama): Add granite_swa support Branch: GraniteSWAForCausalLM AI-usage: full (Bob, OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat(conversion): Add conversion infra for rope_pattern array NOTE: There is other work also targeting this, so this may be removed depending on merge order. Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix(conversion): Fix SWA pattern logic and support for non-rope layers Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat(conversion): Add support for GraniteMoeSWA Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Add llama_hparams::has_rope and arch constants NOTE: This shadows the work done for Granite Speech https://github.com/ggml-org/llama.cpp/pull/25107 Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Add support for per-layer rope determination Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * style: Fix failing flake8 for extra newlines Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * test: Write out SLIDING_WINDOW_PATTERN in llama-model-saver Branch: GraniteSWAForCausalLM AI-usage: full (OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix(convert): Fix missing registration for GraniteMoeSWAForCausalLM Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Load MoE params as optional Branch: GraniteSWAForCausalLM AI-usage: draft (OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Handle MoE params in conversion branch: GraniteSWAForCausalLM AI-usage: full (OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * style: Remove unnecessary newline AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Remove unnecessary tensor additions to GRANITE architecture Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Correctly handle naming for ffn gate inp Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Always default hparams.rope_pattern to 1s This isn't strictly necessary, but it will allow other models to rely on hparams.has_rope(il) without needting to prepopulate. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Move to has_rope for all granite model architectures Now that we have a proper hparam for this, it's better to use it and not require a hacky fallback in the hparam method itself. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: No hacky rope_finetuned fallback in has_rope Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Fully remove rope hparam filling in granitemoe There are no granitemoe models that use NoPE (it's not actually used in the layer building below), so this was just dead code. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Save out rope_pattern in model-saver Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Set hparams.rope_finetuned for round trip Since the value is _read_ from rope_finetuned, we need to persist it when the model is saved with the saver. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Code review cleanup Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> * refactor: Keep gate/up fused for MoE path Branch: GraniteSWAForCausalLM AI-usage: full (Claude + Sonnet 5) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Skip GRANITE_SWA in model saver https://github.com/ggml-org/llama.cpp/pull/25505#discussion_r3773175651 Keeping is_swa_impl in the saver can break other models. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * add sliding window pattern for model in test * style: Fix indentation Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Fix \r\n Thanks Claude! Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Keep shared expert fused Branch: GraniteSWAForCausalLM AI-usage: full (Claude + Sonnet 5) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * style: More indentation fixes Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> --------- Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
This commit is contained in:
Gabe Goodhart
GitHub
@@ -109,6 +109,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"GraniteSwitchForCausalLM": "granite",
|
||||
"GraniteSpeechForConditionalGeneration": "granite",
|
||||
"GraniteSpeechPlusForConditionalGeneration": "granite",
|
||||
"GraniteSWAForCausalLM": "granite",
|
||||
"GraniteMoeSWAForCausalLM": "granite",
|
||||
"Grok1ForCausalLM": "grok",
|
||||
"GrokForCausalLM": "grok",
|
||||
"GroveMoeForCausalLM": "grovemoe",
|
||||
|
||||
@@ -74,6 +74,108 @@ class GraniteModel(LlamaModel):
|
||||
return super().filter_tensors(item)
|
||||
|
||||
|
||||
@ModelBase.register("GraniteSWAForCausalLM")
|
||||
class GraniteSWAModel(GraniteModel):
|
||||
"""Conversion for IBM's GraniteSWAForCausalLM (interleaved sliding window attention)"""
|
||||
model_arch = gguf.MODEL_ARCH.GRANITE_SWA
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
if name.endswith("sinks"):
|
||||
name += ".weight"
|
||||
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
"""GraniteSWA uses Granite parameters plus sliding window configuration."""
|
||||
super().set_gguf_parameters()
|
||||
|
||||
# Add sliding_window from config
|
||||
sliding_window = self.hparams.get("sliding_window", 128)
|
||||
self.gguf_writer.add_sliding_window(sliding_window)
|
||||
logger.info("gguf: (granite_swa) sliding_window = %s", sliding_window)
|
||||
|
||||
# Derive sliding_window_pattern from layer_types
|
||||
if layer_types := self.hparams.get("layer_types"):
|
||||
is_swa = [t == "sliding_attention" for t in layer_types]
|
||||
self.gguf_writer.add_sliding_window_pattern(is_swa)
|
||||
logger.info("gguf: (granite_swa) sliding_window_pattern = %d SWA layers / %d total",
|
||||
sum(is_swa), len(is_swa))
|
||||
else:
|
||||
# Fall back to period-based pattern: i % 4 != 0
|
||||
# This matches the transformers default pattern
|
||||
n_layers = self.block_count
|
||||
is_swa = [i % 4 != 0 for i in range(n_layers)]
|
||||
self.gguf_writer.add_sliding_window_pattern(is_swa)
|
||||
logger.info("gguf: (granite_swa) sliding_window_pattern (inferred) = %d SWA layers / %d total",
|
||||
sum(is_swa), n_layers)
|
||||
|
||||
# Add rope_pattern from no_rope_layers
|
||||
if no_rope_layers := self.hparams.get("no_rope_layers"):
|
||||
# Convert 1/0 to bool (1 = use RoPE, 0 = NoPE)
|
||||
rope_pattern = [bool(x) for x in no_rope_layers]
|
||||
self.gguf_writer.add_rope_pattern(rope_pattern)
|
||||
logger.info("gguf: (granite_swa) rope_pattern = %d RoPE layers / %d total",
|
||||
sum(rope_pattern), len(rope_pattern))
|
||||
|
||||
|
||||
@ModelBase.register("GraniteMoeSWAForCausalLM")
|
||||
class GraniteMoeSWAModel(GraniteSWAModel):
|
||||
"""Conversion for IBM's GraniteMoeSWAForCausalLM (unified dense + MoE with iSWA)"""
|
||||
model_arch = gguf.MODEL_ARCH.GRANITE_SWA
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
if shared_intermediate_size := self.hparams.get("shared_intermediate_size"):
|
||||
self.gguf_writer.add_expert_shared_feed_forward_length(shared_intermediate_size)
|
||||
logger.info("gguf: (granitemoewa) shared_intermediate_size = %s", shared_intermediate_size)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
"""Split merged MoE tensors (gate+up) following standard MoE pattern."""
|
||||
|
||||
# Handle expert FFN tensors (merged gate+up) - swash format: experts.gate_up_proj
|
||||
# Kept fused since inference (build_moe_ffn) supports a single gate_up_exps
|
||||
# tensor for the routed experts.
|
||||
if name.endswith("block_sparse_moe.experts.gate_up_proj"):
|
||||
ffn_dim = self.hparams["intermediate_size"]
|
||||
assert data_torch.shape[-2] == 2 * ffn_dim, f"Merged FFN tensor size must be 2 * intermediate_size, got {data_torch.shape[-2]}"
|
||||
yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_UP_EXP, bid), bid)
|
||||
return
|
||||
|
||||
# Handle expert FFN down projection - swash format: experts.down_proj
|
||||
if name.endswith("block_sparse_moe.experts.down_proj"):
|
||||
yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_EXP, bid), bid)
|
||||
return
|
||||
|
||||
# Handle expert FFN tensors (merged gate+up) - standard granite format: input_linear.weight
|
||||
# Kept fused since inference (build_moe_ffn) supports a single gate_up_exps
|
||||
# tensor for the routed experts.
|
||||
if name.endswith("block_sparse_moe.input_linear.weight"):
|
||||
ffn_dim = self.hparams["intermediate_size"]
|
||||
assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * intermediate_size"
|
||||
yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_UP_EXP, bid), bid)
|
||||
return
|
||||
|
||||
# Handle shared expert FFN tensors (if present) - kept fused since
|
||||
# inference (build_ffn) supports a single ffn_up_shexp tensor with
|
||||
# LLM_FFN_SWIGLU for the shared expert.
|
||||
if name.endswith("shared_mlp.input_linear.weight"):
|
||||
ffn_dim = self.hparams.get("shared_intermediate_size", self.hparams["intermediate_size"])
|
||||
assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * shared_intermediate_size"
|
||||
yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), bid)
|
||||
return
|
||||
|
||||
# Handle shared expert output (if present)
|
||||
if name.endswith("shared_mlp.output_linear.weight"):
|
||||
yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, bid), bid)
|
||||
return
|
||||
|
||||
# Pass through to parent for all other tensors (including sinks)
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM")
|
||||
@ModelBase.example("ibm-granite/granite-3.1-3b-a800m-instruct")
|
||||
class GraniteMoeModel(GraniteModel):
|
||||
|
||||
@@ -208,6 +208,7 @@ class Keys:
|
||||
SHARED_KV_LAYERS = "{arch}.attention.shared_kv_layers"
|
||||
SLIDING_WINDOW_PATTERN = "{arch}.attention.sliding_window_pattern"
|
||||
TEMPERATURE_SCALE = "{arch}.attention.temperature_scale"
|
||||
ROPE_PATTERN = "{arch}.attention.rope_pattern"
|
||||
|
||||
class Indexer:
|
||||
HEAD_COUNT = "{arch}.attention.indexer.head_count"
|
||||
@@ -549,6 +550,7 @@ class MODEL_ARCH(IntEnum):
|
||||
GRANITE_MOE = auto()
|
||||
GRANITE_HYBRID = auto()
|
||||
GRANITE_SWITCH = auto()
|
||||
GRANITE_SWA = auto()
|
||||
CHAMELEON = auto()
|
||||
WAVTOKENIZER_DEC = auto()
|
||||
PLM = auto()
|
||||
@@ -1265,6 +1267,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
|
||||
MODEL_ARCH.GRANITE_MOE: "granitemoe",
|
||||
MODEL_ARCH.GRANITE_HYBRID: "granitehybrid",
|
||||
MODEL_ARCH.GRANITE_SWITCH: "graniteswitch",
|
||||
MODEL_ARCH.GRANITE_SWA: "granite_swa",
|
||||
MODEL_ARCH.CHAMELEON: "chameleon",
|
||||
MODEL_ARCH.WAVTOKENIZER_DEC: "wavtokenizer-dec",
|
||||
MODEL_ARCH.PLM: "plm",
|
||||
@@ -4152,6 +4155,31 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.FFN_DOWN,
|
||||
MODEL_TENSOR.FFN_UP,
|
||||
],
|
||||
MODEL_ARCH.GRANITE_SWA: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
MODEL_TENSOR.OUTPUT_NORM,
|
||||
MODEL_TENSOR.OUTPUT,
|
||||
MODEL_TENSOR.ATTN_NORM,
|
||||
MODEL_TENSOR.ATTN_Q,
|
||||
MODEL_TENSOR.ATTN_K,
|
||||
MODEL_TENSOR.ATTN_V,
|
||||
MODEL_TENSOR.ATTN_OUT,
|
||||
MODEL_TENSOR.ATTN_SINKS,
|
||||
MODEL_TENSOR.ROPE_FREQS,
|
||||
MODEL_TENSOR.FFN_NORM,
|
||||
MODEL_TENSOR.FFN_GATE,
|
||||
MODEL_TENSOR.FFN_DOWN,
|
||||
MODEL_TENSOR.FFN_UP,
|
||||
# MoE (GraniteMoeSWA)
|
||||
MODEL_TENSOR.FFN_GATE_INP,
|
||||
MODEL_TENSOR.FFN_GATE_EXP,
|
||||
MODEL_TENSOR.FFN_GATE_UP_EXP,
|
||||
MODEL_TENSOR.FFN_DOWN_EXP,
|
||||
MODEL_TENSOR.FFN_UP_EXP,
|
||||
# Shared expert - gate+up kept fused in FFN_UP_SHEXP (LLM_FFN_SWIGLU)
|
||||
MODEL_TENSOR.FFN_UP_SHEXP,
|
||||
MODEL_TENSOR.FFN_DOWN_SHEXP,
|
||||
],
|
||||
MODEL_ARCH.CHAMELEON: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
MODEL_TENSOR.OUTPUT_NORM,
|
||||
|
||||
@@ -824,6 +824,9 @@ class GGUFWriter:
|
||||
else:
|
||||
self.add_array(key, value)
|
||||
|
||||
def add_rope_pattern(self, value: Sequence[bool]) -> None:
|
||||
self.add_array(Keys.Attention.ROPE_PATTERN.format(arch=self.arch), value)
|
||||
|
||||
def add_dense_features_dims(self, dense:str, in_f:int, out_f:int) -> None:
|
||||
self.add_uint32(Keys.LLM.DENSE_FEAT_IN_SIZE.format(arch=self.arch, dense=dense), in_f)
|
||||
self.add_uint32(Keys.LLM.DENSE_FEAT_OUT_SIZE.format(arch=self.arch, dense=dense), out_f)
|
||||
|
||||
@@ -458,6 +458,7 @@ class TensorNameMap:
|
||||
"transformer.decoder_layer.{bid}.router", # Grok
|
||||
"transformer.blocks.{bid}.ffn.router.layer", # dbrx
|
||||
"model.layers.{bid}.block_sparse_moe.router.layer", # granitemoe
|
||||
"model.layers.{bid}.block_sparse_moe.router", # granite_swa
|
||||
"model.layers.{bid}.feed_forward.router", # llama4 jamba
|
||||
"encoder.layers.{bid}.mlp.router.layer", # nomic-bert-moe
|
||||
"model.layers.{bid}.mlp.router", # openai-moe
|
||||
|
||||
@@ -102,6 +102,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
|
||||
{ LLM_ARCH_GRANITE_MOE, "granitemoe" },
|
||||
{ LLM_ARCH_GRANITE_HYBRID, "granitehybrid" },
|
||||
{ LLM_ARCH_GRANITE_SWITCH, "graniteswitch" },
|
||||
{ LLM_ARCH_GRANITE_SWA, "granite_swa" },
|
||||
{ LLM_ARCH_CHAMELEON, "chameleon" },
|
||||
{ LLM_ARCH_WAVTOKENIZER_DEC, "wavtokenizer-dec" },
|
||||
{ LLM_ARCH_PLM, "plm" },
|
||||
@@ -261,6 +262,8 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
|
||||
{ LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" },
|
||||
{ LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" },
|
||||
{ LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, "%s.attention.sliding_window_pattern" },
|
||||
{ LLM_KV_ATTENTION_ROPE_PATTERN, "%s.attention.rope_pattern" },
|
||||
|
||||
{ LLM_KV_ATTENTION_SCALE, "%s.attention.scale" },
|
||||
{ LLM_KV_ATTENTION_OUTPUT_SCALE, "%s.attention.output_scale" },
|
||||
{ LLM_KV_ATTENTION_VALUE_SCALE, "%s.attention.value_scale" },
|
||||
|
||||
@@ -107,6 +107,7 @@ enum llm_arch {
|
||||
LLM_ARCH_GRANITE_MOE,
|
||||
LLM_ARCH_GRANITE_HYBRID,
|
||||
LLM_ARCH_GRANITE_SWITCH,
|
||||
LLM_ARCH_GRANITE_SWA,
|
||||
LLM_ARCH_CHAMELEON,
|
||||
LLM_ARCH_WAVTOKENIZER_DEC,
|
||||
LLM_ARCH_PLM,
|
||||
@@ -267,6 +268,8 @@ enum llm_kv {
|
||||
LLM_KV_ATTENTION_SLIDING_WINDOW,
|
||||
LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN,
|
||||
LLM_KV_ATTENTION_SCALE,
|
||||
LLM_KV_ATTENTION_ROPE_PATTERN,
|
||||
|
||||
LLM_KV_ATTENTION_OUTPUT_SCALE,
|
||||
LLM_KV_ATTENTION_VALUE_SCALE,
|
||||
LLM_KV_ATTENTION_TEMPERATURE_LENGTH,
|
||||
|
||||
@@ -291,7 +291,11 @@ bool llama_hparams::has_rope(uint32_t il) const {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
if (il < n_layer_all) {
|
||||
return rope_pattern[il] != 0;
|
||||
}
|
||||
|
||||
GGML_ABORT("%s: il (%u) out of bounds (n_layer_all: %u)\n", __func__, il, n_layer_all);
|
||||
}
|
||||
|
||||
uint32_t llama_hparams::n_layer() const {
|
||||
|
||||
@@ -144,6 +144,10 @@ struct llama_hparams {
|
||||
|
||||
std::array<int, 4> rope_sections;
|
||||
|
||||
// Per-layer RoPE enable flags (1 = use RoPE, 0 = NoPE)
|
||||
// by default, all layers use RoPE (controlled by rope_finetuned)
|
||||
std::array<uint32_t, LLAMA_MAX_LAYERS> rope_pattern;
|
||||
|
||||
// Sliding Window Attention (SWA)
|
||||
llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;
|
||||
// the size of the sliding window (0 - no SWA)
|
||||
|
||||
@@ -30,6 +30,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) {
|
||||
case LLM_ARCH_MUSE_GLIMMER:
|
||||
case LLM_ARCH_MELLUM:
|
||||
case LLM_ARCH_LAGUNA:
|
||||
case LLM_ARCH_GRANITE_SWA:
|
||||
return false;
|
||||
default:
|
||||
return true;
|
||||
@@ -272,6 +273,7 @@ void llama_model_saver::add_kv_from_model() {
|
||||
add_kv(LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, hparams.n_lora_value_res_mix);
|
||||
add_kv(LLM_KV_ATTENTION_GATE_LORA_RANK, hparams.n_lora_gate);
|
||||
add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);
|
||||
add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, true);
|
||||
add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
||||
// add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???);
|
||||
add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale);
|
||||
|
||||
@@ -246,6 +246,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
|
||||
return new llama_model_minicpm(params);
|
||||
case LLM_ARCH_GRANITE_HYBRID:
|
||||
return new llama_model_granite_hybrid(params);
|
||||
case LLM_ARCH_GRANITE_SWA:
|
||||
return new llama_model_granite_swa(params);
|
||||
case LLM_ARCH_CHAMELEON:
|
||||
return new llama_model_chameleon(params);
|
||||
case LLM_ARCH_WAVTOKENIZER_DEC:
|
||||
@@ -1157,6 +1159,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
|
||||
std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0);
|
||||
|
||||
std::fill(hparams.rope_sections.begin(), hparams.rope_sections.end(), 0);
|
||||
std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), 1);
|
||||
std::fill(hparams.is_swa_impl.begin(), hparams.is_swa_impl.end(), 0);
|
||||
std::fill(hparams.is_recr_impl.begin(), hparams.is_recr_impl.end(), llm_arch_is_recurrent(ml.get_arch()) ? 1 : 0);
|
||||
std::fill(hparams.is_indexer_full_impl.begin(), hparams.is_indexer_full_impl.end(), 0);
|
||||
@@ -2639,6 +2642,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
|
||||
case LLM_ARCH_GRANITE_MOE:
|
||||
case LLM_ARCH_GRANITE_HYBRID:
|
||||
case LLM_ARCH_GRANITE_SWITCH:
|
||||
case LLM_ARCH_GRANITE_SWA:
|
||||
case LLM_ARCH_CHAMELEON:
|
||||
case LLM_ARCH_BAILINGMOE:
|
||||
case LLM_ARCH_BAILINGMOE3:
|
||||
|
||||
@@ -16,7 +16,8 @@ void llama_model_granite_hybrid::load_arch_hparams(llama_model_loader & ml) {
|
||||
// Granite uses rope_finetuned as a switch for rope, so default to true
|
||||
bool rope_finetuned = true;
|
||||
ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);
|
||||
hparams.rope_finetuned = rope_finetuned;
|
||||
hparams.rope_finetuned = rope_finetuned; // needed for round trip save
|
||||
std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned);
|
||||
|
||||
// A layer is recurrent IFF the n_head_kv value is set to 0
|
||||
for (uint32_t i = 0; i < hparams.n_layer(); ++i) {
|
||||
@@ -147,7 +148,7 @@ llama_model_granite_hybrid::graph::graph(const llama_model & model, const llm_gr
|
||||
|
||||
// Positional embeddings populated if rope enabled
|
||||
ggml_tensor * inp_pos = nullptr;
|
||||
if (hparams.rope_finetuned) {
|
||||
if (hparams.has_rope(0)) {
|
||||
inp_pos = build_inp_pos();
|
||||
}
|
||||
|
||||
@@ -206,8 +207,7 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_attention_layer(ggml_tens
|
||||
const int il) {
|
||||
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);
|
||||
|
||||
const bool use_rope = hparams.rope_finetuned;
|
||||
if (use_rope) {
|
||||
if (hparams.has_rope(il)) {
|
||||
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
|
||||
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow);
|
||||
|
||||
@@ -7,11 +7,6 @@ void llama_model_granite_moe::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false);
|
||||
ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false);
|
||||
|
||||
// Granite uses rope_finetuned as a switch for rope, so default to true
|
||||
bool rope_finetuned = true;
|
||||
ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);
|
||||
hparams.rope_finetuned = rope_finetuned;
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 32: type = LLM_TYPE_3B; break;
|
||||
case 40: type = LLM_TYPE_3B; break;
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
#include "models.h"
|
||||
|
||||
#include <sstream>
|
||||
|
||||
void llama_model_granite_swa::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
||||
ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);
|
||||
ml.get_key(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale, false);
|
||||
ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false);
|
||||
ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false);
|
||||
|
||||
// MoE expert configuration
|
||||
ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);
|
||||
ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);
|
||||
|
||||
// iSWA configuration
|
||||
ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
|
||||
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
||||
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
||||
|
||||
// Granite4 Vision uses array deepstack_mapping
|
||||
ml.get_arr(LLM_KV_DEEPSTACK_MAPPING, hparams.deepstack_mapping_arr, false);
|
||||
|
||||
// Count the unique deepstack input indices
|
||||
std::unordered_set<uint32_t> unique_deepstack_idxs;
|
||||
for (const auto val : hparams.deepstack_mapping_arr) {
|
||||
if (val >= 0) {
|
||||
unique_deepstack_idxs.insert(val);
|
||||
}
|
||||
}
|
||||
hparams.n_deepstack_layers = unique_deepstack_idxs.size();
|
||||
|
||||
// Ensure all values are valid (avoid overflow attacks)
|
||||
for (const auto val : unique_deepstack_idxs) {
|
||||
if (val > hparams.n_deepstack_layers) {
|
||||
std::stringstream ss;
|
||||
ss << "Invalid deepstack index: " << val << " > " << hparams.n_deepstack_layers;
|
||||
throw std::runtime_error(ss.str());
|
||||
}
|
||||
}
|
||||
|
||||
// Per-layer RoPE pattern (optional)
|
||||
ml.get_arr(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, false);
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 32: type = LLM_TYPE_3B; break;
|
||||
case 40: type = LLM_TYPE_3B; break;
|
||||
// Add additional layer/vocab/etc checks here for other model sizes
|
||||
default: type = LLM_TYPE_UNKNOWN;
|
||||
}
|
||||
|
||||
// For Granite MoE Shared
|
||||
ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, /* required */ false);
|
||||
}
|
||||
|
||||
void llama_model_granite_swa::load_arch_tensors(llama_model_loader &) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
|
||||
|
||||
// output
|
||||
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
|
||||
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
|
||||
|
||||
// if output is NULL, init from the input tok embed
|
||||
if (output == NULL) {
|
||||
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
|
||||
}
|
||||
|
||||
for (int i = 0; i < n_layer; ++i) {
|
||||
auto & layer = layers[i];
|
||||
|
||||
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
|
||||
|
||||
create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
|
||||
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
|
||||
|
||||
// optional bias tensors
|
||||
layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
|
||||
|
||||
// Per-layer attention sinks for iSWA
|
||||
layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0);
|
||||
|
||||
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
|
||||
|
||||
if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) {
|
||||
layer.rope_long = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
|
||||
layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
|
||||
}
|
||||
else {
|
||||
layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
|
||||
}
|
||||
|
||||
if (n_expert == 0) {
|
||||
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
|
||||
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);
|
||||
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
|
||||
|
||||
// optional MLP bias
|
||||
layer.ffn_gate_b = create_tensor(tn(LLM_TENSOR_FFN_GATE, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED);
|
||||
} else {
|
||||
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
|
||||
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0);
|
||||
create_tensor_gate_up_exps(layer, i, n_embd, n_ff, n_expert, 0);
|
||||
|
||||
// For Granite MoE Shared - gate+up kept fused in ffn_up_shexp (see LLM_FFN_SWIGLU below)
|
||||
if (hparams.n_ff_shexp > 0) {
|
||||
layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, 2*hparams.n_ff_shexp}, 0);
|
||||
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {hparams.n_ff_shexp, n_embd}, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<llm_graph_context> llama_model_granite_swa::build_arch_graph(const llm_graph_params & params) const {
|
||||
return std::make_unique<graph>(*this, params);
|
||||
}
|
||||
|
||||
llama_model_granite_swa::graph::graph(
|
||||
const llama_model & model,
|
||||
const llm_graph_params & params)
|
||||
: llm_graph_context(params) {
|
||||
|
||||
const int64_t n_embd_head = hparams.n_embd_head_v();
|
||||
|
||||
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
|
||||
GGML_ASSERT(n_embd_head == n_rot);
|
||||
|
||||
ggml_tensor * cur;
|
||||
ggml_tensor * inpL;
|
||||
|
||||
inpL = build_inp_embd(model.tok_embd);
|
||||
|
||||
// inp_pos - built only if rope enabled
|
||||
ggml_tensor * inp_pos = build_inp_pos();
|
||||
auto * inp_attn = build_attn_inp_kv_iswa();
|
||||
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
|
||||
// Granite Vision 4.1 deepstack: inject the projector stream that
|
||||
// targets decoder layer `il` before the decoder runs.
|
||||
// NOTE: skip the first deepstack layer since that's inpL
|
||||
const auto & deepstack_emb_idx = hparams.deepstack_mapping_arr[il];
|
||||
if (il > 0 && deepstack_emb_idx >= 0) {
|
||||
ggml_tensor * ds = ggml_view_2d(ctx0,
|
||||
res->t_inp_embd, n_embd, n_tokens,
|
||||
res->t_inp_embd->nb[1],
|
||||
deepstack_emb_idx * n_embd * sizeof(float));
|
||||
inpL = ggml_add(ctx0, inpL, ds);
|
||||
cb(inpL, "deepstack_in", il);
|
||||
}
|
||||
|
||||
ggml_tensor * inpSA = inpL;
|
||||
|
||||
// norm
|
||||
cur = build_norm(inpL,
|
||||
model.layers[il].attn_norm, NULL,
|
||||
LLM_NORM_RMS, il);
|
||||
cb(cur, "attn_norm", il);
|
||||
|
||||
// self-attention
|
||||
cur = build_attention_layer(
|
||||
cur, inp_pos, inp_attn,
|
||||
model, n_embd_head, il);
|
||||
|
||||
if (il == n_layer - 1 && inp_out_ids) {
|
||||
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
||||
inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
|
||||
}
|
||||
// ffn
|
||||
cur = build_layer_ffn(cur, inpSA, model, il);
|
||||
|
||||
// input for next layer
|
||||
inpL = cur;
|
||||
}
|
||||
cur = inpL;
|
||||
|
||||
cur = build_norm(cur,
|
||||
model.output_norm, NULL,
|
||||
LLM_NORM_RMS, -1);
|
||||
|
||||
cb(cur, "result_norm", -1);
|
||||
res->t_embd = cur;
|
||||
|
||||
// lm_head
|
||||
cur = build_lora_mm(model.output, cur, model.output_s);
|
||||
|
||||
// For Granite architectures - scale logits
|
||||
cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_logit_scale);
|
||||
cb(cur, "result_output", -1);
|
||||
res->t_logits = cur;
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
}
|
||||
|
||||
ggml_tensor * llama_model_granite_swa::graph::build_attention_layer(
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * inp_pos,
|
||||
llm_graph_input_attn_kv_iswa * inp_attn,
|
||||
const llama_model & model,
|
||||
const int64_t n_embd_head,
|
||||
const int il) {
|
||||
|
||||
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
|
||||
n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);
|
||||
|
||||
const bool use_rope = hparams.has_rope(il);
|
||||
if (use_rope) {
|
||||
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
|
||||
Qcur = ggml_rope_ext(
|
||||
ctx0, Qcur, inp_pos, rope_factors,
|
||||
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow
|
||||
);
|
||||
|
||||
Kcur = ggml_rope_ext(
|
||||
ctx0, Kcur, inp_pos, rope_factors,
|
||||
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow
|
||||
);
|
||||
}
|
||||
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
|
||||
const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
|
||||
|
||||
// Pass layer.attn_sinks to build_attn for sink-based attention modulation
|
||||
cur = build_attn(inp_attn,
|
||||
model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
|
||||
Qcur, Kcur, Vcur, nullptr, model.layers[il].attn_sinks, nullptr, kq_scale, il);
|
||||
cb(cur, "attn_out", il);
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn(
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * inpSA,
|
||||
const llama_model & model,
|
||||
const int il) {
|
||||
|
||||
// For Granite architectures - scale residual
|
||||
if (hparams.f_residual_scale) {
|
||||
cur = ggml_scale(ctx0, cur, hparams.f_residual_scale);
|
||||
}
|
||||
ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
|
||||
cb(ffn_inp, "ffn_inp", il);
|
||||
|
||||
// feed-forward network (non-MoE)
|
||||
if (model.layers[il].ffn_gate_inp == nullptr) {
|
||||
|
||||
cur = build_norm(ffn_inp,
|
||||
model.layers[il].ffn_norm, NULL,
|
||||
LLM_NORM_RMS, il);
|
||||
cb(cur, "ffn_norm", il);
|
||||
|
||||
cur = build_ffn(cur,
|
||||
model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
|
||||
model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,
|
||||
model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
|
||||
NULL,
|
||||
LLM_FFN_SILU, LLM_FFN_PAR, il);
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
} else {
|
||||
// MoE branch
|
||||
cur = build_norm(ffn_inp,
|
||||
model.layers[il].ffn_norm, NULL,
|
||||
LLM_NORM_RMS, il);
|
||||
cb(cur, "ffn_norm", il);
|
||||
|
||||
ggml_tensor * moe_out = build_moe_ffn(cur,
|
||||
model.layers[il].ffn_gate_inp,
|
||||
model.layers[il].ffn_up_exps,
|
||||
model.layers[il].ffn_gate_exps,
|
||||
model.layers[il].ffn_down_exps,
|
||||
nullptr,
|
||||
n_expert, n_expert_used,
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
|
||||
il,
|
||||
nullptr, model.layers[il].ffn_gate_up_exps);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
// For Granite MoE Shared - gate+up kept fused in ffn_up_shexp
|
||||
if (hparams.n_ff_shexp > 0) {
|
||||
ggml_tensor * ffn_shexp = build_ffn(cur,
|
||||
model.layers[il].ffn_up_shexp, NULL, NULL,
|
||||
NULL, NULL, NULL,
|
||||
model.layers[il].ffn_down_shexp, NULL, NULL,
|
||||
NULL,
|
||||
LLM_FFN_SWIGLU, LLM_FFN_SEQ, il);
|
||||
cb(ffn_shexp, "ffn_shexp", il);
|
||||
|
||||
cur = ggml_add(ctx0, moe_out, ffn_shexp);
|
||||
cb(cur, "ffn_out", il);
|
||||
} else {
|
||||
cur = moe_out;
|
||||
}
|
||||
}
|
||||
|
||||
// For Granite architectures - scale residual
|
||||
if (hparams.f_residual_scale) {
|
||||
cur = ggml_scale(ctx0, cur, hparams.f_residual_scale);
|
||||
}
|
||||
cur = ggml_add(ctx0, cur, ffn_inp);
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
cur = build_cvec(cur, il);
|
||||
cb(cur, "l_out", il);
|
||||
|
||||
return cur;
|
||||
}
|
||||
@@ -11,7 +11,8 @@ void llama_model_granite_switch::load_arch_hparams(llama_model_loader & ml) {
|
||||
|
||||
bool rope_finetuned = true;
|
||||
ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);
|
||||
hparams.rope_finetuned = rope_finetuned;
|
||||
hparams.rope_finetuned = rope_finetuned; // needed for round trip save
|
||||
std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned);
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 40: type = hparams.n_embd == 4096 ? LLM_TYPE_8B : LLM_TYPE_3B; break;
|
||||
@@ -254,7 +255,7 @@ llama_model_granite_switch::graph::graph(
|
||||
cb(inpL, "inp_embd", -1);
|
||||
|
||||
ggml_tensor * inp_pos = nullptr;
|
||||
if (hparams.rope_finetuned) {
|
||||
if (hparams.has_rope(0)) {
|
||||
inp_pos = build_inp_pos();
|
||||
}
|
||||
auto * inp_attn = build_attn_inp_kv();
|
||||
@@ -361,7 +362,7 @@ ggml_tensor * llama_model_granite_switch::graph::build_attention_layer(
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
|
||||
|
||||
if (hparams.rope_finetuned) {
|
||||
if (hparams.has_rope(il)) {
|
||||
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
|
||||
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors,
|
||||
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
||||
|
||||
@@ -33,7 +33,8 @@ void llama_model_granite::load_arch_hparams(llama_model_loader & ml) {
|
||||
// Granite uses rope_finetuned as a switch for rope, so default to true
|
||||
bool rope_finetuned = true;
|
||||
ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);
|
||||
hparams.rope_finetuned = rope_finetuned;
|
||||
hparams.rope_finetuned = rope_finetuned; // needed for round trip save
|
||||
std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned);
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 32: type = LLM_TYPE_3B; break;
|
||||
@@ -127,7 +128,7 @@ llama_model_granite::graph::graph(
|
||||
|
||||
// inp_pos - built only if rope enabled
|
||||
ggml_tensor * inp_pos = nullptr;
|
||||
if (hparams.rope_finetuned) {
|
||||
if (hparams.has_rope(0)) {
|
||||
inp_pos = build_inp_pos();
|
||||
}
|
||||
auto * inp_attn = build_attn_inp_kv();
|
||||
@@ -203,8 +204,7 @@ ggml_tensor * llama_model_granite::graph::build_attention_layer(
|
||||
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
|
||||
n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);
|
||||
|
||||
const bool use_rope = hparams.rope_finetuned;
|
||||
if (use_rope) {
|
||||
if (hparams.has_rope(il)) {
|
||||
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
|
||||
Qcur = ggml_rope_ext(
|
||||
ctx0, Qcur, inp_pos, rope_factors,
|
||||
|
||||
@@ -1719,6 +1719,34 @@ struct llama_model_granite_hybrid : public llama_model_base {
|
||||
};
|
||||
|
||||
|
||||
struct llama_model_granite_swa : public llama_model_base {
|
||||
llama_model_granite_swa(const struct llama_model_params & params) : llama_model_base(params) {}
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
void load_arch_tensors(llama_model_loader & ml) override;
|
||||
|
||||
struct graph : public llm_graph_context {
|
||||
graph(const llama_model & model, const llm_graph_params & params);
|
||||
|
||||
private:
|
||||
ggml_tensor * build_attention_layer(
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * inp_pos,
|
||||
llm_graph_input_attn_kv_iswa * inp_attn,
|
||||
const llama_model & model,
|
||||
const int64_t n_embd_head,
|
||||
const int il);
|
||||
|
||||
ggml_tensor * build_layer_ffn(
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * inpSA,
|
||||
const llama_model & model,
|
||||
const int il);
|
||||
};
|
||||
|
||||
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
|
||||
};
|
||||
|
||||
|
||||
struct llama_model_chameleon : public llama_model_base {
|
||||
llama_model_chameleon(const struct llama_model_params & params) : llama_model_base(params) {}
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
|
||||
@@ -197,7 +197,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|
||||
ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);
|
||||
// SWA pattern: every 5th layer is full attention (matches E2B layer_types)
|
||||
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));
|
||||
} else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_MUSE_GLIMMER) {
|
||||
} else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA) {
|
||||
std::vector<uint32_t> pattern;
|
||||
pattern.reserve(n_layer);
|
||||
for (uint32_t il = 0; il < n_layer; il++) {
|
||||
|
||||
Reference in New Issue
Block a user