mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-09-19 17:24:57 +02:00
quantize: let --tensor-type name per_layer_token_embd
per_layer_token_embd shares the TOKEN_EMBD category with token_embd.weight, so --token-embedding-type is returned for it before any --tensor-type pattern is consulted, and there is no way to give it a tier of its own. That grouping is fine as a default and stays the default. It is a poor fit for the size, though: on qwen4exp the table is 97.7 GiB of a 337.6 GiB BF16 file and about 46% of a 4-bit one, roughly eighty times token_embd.weight, and it is read by ggml_get_rows rather than a matmul so no imatrix ever covers it. Allow an explicit --tensor-type pattern to name it, and only it. Nothing changes unless such a pattern is passed, and token_embd.weight keeps the old precedence in either case. Measured on Qwen3.8-Flash-Next, Q4_K_M with an imatrix: the table lands at q8_0 (51.9 GiB, 113.5 GiB total) by following --token-embedding-type, and pinning it q4_1 gives 30.5 GiB for 92.1 GiB total, 19% off the file.
This commit is contained in:
committed by
Daniel Han
parent
570c26d3d6
commit
4597a86f12
+17
-1
@@ -690,7 +690,23 @@ static ggml_type llama_tensor_get_type(quantize_state_impl & qs, const llama_mod
|
||||
return tensor->type;
|
||||
}
|
||||
if (params->token_embedding_type < GGML_TYPE_COUNT && tm.category == tensor_category::TOKEN_EMBD) {
|
||||
return params->token_embedding_type;
|
||||
// per_layer_token_embd shares this category with token_embd.weight and follows
|
||||
// --token-embedding-type by default. But it is a separate table and far from a
|
||||
// rounding error: qwen4exp's is ~46% of a 4-bit file. Let an explicit --tensor-type
|
||||
// name it; nothing changes unless such a pattern is passed.
|
||||
bool named = false;
|
||||
if (std::strcmp(tensor->name, "per_layer_token_embd.weight") == 0) {
|
||||
const std::string tensor_name(tensor->name);
|
||||
for (const auto & [pattern, qtype] : qs.tensor_type_patterns) {
|
||||
if (std::regex_search(tensor_name, pattern)) {
|
||||
named = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!named) {
|
||||
return params->token_embedding_type;
|
||||
}
|
||||
}
|
||||
if (params->output_tensor_type < GGML_TYPE_COUNT && tm.category == tensor_category::OUTPUT) {
|
||||
return params->output_tensor_type;
|
||||
|
||||
Reference in New Issue
Block a user