From 4597a86f124e2ef04676fa9a6cb989235d2ac63c Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 25 Aug 2026 23:28:58 +0000 Subject: [PATCH] quantize: let --tensor-type name per_layer_token_embd per_layer_token_embd shares the TOKEN_EMBD category with token_embd.weight, so --token-embedding-type is returned for it before any --tensor-type pattern is consulted, and there is no way to give it a tier of its own. That grouping is fine as a default and stays the default. It is a poor fit for the size, though: on qwen4exp the table is 97.7 GiB of a 337.6 GiB BF16 file and about 46% of a 4-bit one, roughly eighty times token_embd.weight, and it is read by ggml_get_rows rather than a matmul so no imatrix ever covers it. Allow an explicit --tensor-type pattern to name it, and only it. Nothing changes unless such a pattern is passed, and token_embd.weight keeps the old precedence in either case. Measured on Qwen3.8-Flash-Next, Q4_K_M with an imatrix: the table lands at q8_0 (51.9 GiB, 113.5 GiB total) by following --token-embedding-type, and pinning it q4_1 gives 30.5 GiB for 92.1 GiB total, 19% off the file. --- src/llama-quant.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 6e7f1ba3ba..eb94c8a212 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -690,7 +690,23 @@ static ggml_type llama_tensor_get_type(quantize_state_impl & qs, const llama_mod return tensor->type; } if (params->token_embedding_type < GGML_TYPE_COUNT && tm.category == tensor_category::TOKEN_EMBD) { - return params->token_embedding_type; + // per_layer_token_embd shares this category with token_embd.weight and follows + // --token-embedding-type by default. But it is a separate table and far from a + // rounding error: qwen4exp's is ~46% of a 4-bit file. Let an explicit --tensor-type + // name it; nothing changes unless such a pattern is passed. + bool named = false; + if (std::strcmp(tensor->name, "per_layer_token_embd.weight") == 0) { + const std::string tensor_name(tensor->name); + for (const auto & [pattern, qtype] : qs.tensor_type_patterns) { + if (std::regex_search(tensor_name, pattern)) { + named = true; + break; + } + } + } + if (!named) { + return params->token_embedding_type; + } } if (params->output_tensor_type < GGML_TYPE_COUNT && tm.category == tensor_category::OUTPUT) { return params->output_tensor_type;