Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 09:15:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

correct token count

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-09-01 12:37:26 +02:00
parent 07335164a1
commit 3a798bf2f3
5 changed files with 53 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip-impl.h
+16
View File
@@ -779,6 +779,22 @@ static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_
}
}
// deepseek4v: layout of the LLM token block built from the aligner grid
struct dsv4_block_layout {
int rows; // grid rows, padded to an even count
int row_len; // grid width + 1 newline
int pad_last; // trailing pads
int n_out; // total block size, including lead pads and the start/end sentinels
};
static inline dsv4_block_layout dsv4_get_block_layout(int n_llm_w, int n_llm_h, int lead_pad) {
dsv4_block_layout bl;
bl.rows = n_llm_h + (n_llm_h % 2);
bl.row_len = n_llm_w + 1;
bl.pad_last = (bl.rows / 2 * bl.row_len) % 2 * 2;
bl.n_out = lead_pad + 1 + bl.rows * bl.row_len + bl.pad_last + 1;
return bl;
}
//
// logging
//
tools/mtmd/clip.cpp
+29 -23
View File
@@ -4193,15 +4193,10 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
// aligner grid + newlines + pads + start/end sentinels
// keep in sync with clip_graph_deepseek4v::build
const int out_patch_size = params.patch_size * params.n_merge;
const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
const int rows = n_llm_h + (n_llm_h % 2);
const int row_len = n_llm_w + 1;
const int pad_last = (rows / 2 * row_len) % 2 * 2;
n_patches = img->lead_pad + 1 + rows * row_len + pad_last + 1;
const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out;
} break;
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_DOTS_OCR:
@@ -5077,41 +5072,39 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
case PROJECTOR_TYPE_DEEPSEEK4V:
{
// set the 2D positions (mrope layout, only the first 2 channels are used)
int n_patches_per_col = image_size_width / patch_size;
int n_patches_per_row = image_size_width / patch_size;
std::vector<int32_t> positions(n_pos * 4, 0);
for (int i = 0; i < n_pos; i++) {
positions[i] = i / n_patches_per_col; // row
positions[n_pos + i] = i % n_patches_per_col; // col
positions[i] = i / n_patches_per_row; // row
positions[n_pos + i] = i % n_patches_per_row; // col
}
set_input_i32("positions", positions);
// token block layout index (see clip_graph_deepseek4v::build)
// rows [0, n_grid) are the aligner output, the sentinels follow
const int n_merge = hparams.n_merge;
const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge;
const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge;
const int n_grid = n_llm_w * n_llm_h;
const int n_merge = hparams.n_merge;
const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge;
const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge;
const int n_grid = n_llm_w * n_llm_h;
const int idx_start = n_grid;
const int idx_end = n_grid + 1;
const int idx_newline = n_grid + 2;
const int idx_pad = n_grid + 3;
const int rows = n_llm_h + (n_llm_h % 2);
const int row_len = n_llm_w + 1;
const int pad_last = (rows / 2 * row_len) % 2 * 2;
const int lead_pad = imgs.entries[0].lead_pad;
const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad);
std::vector<int32_t> idx;
idx.reserve(lead_pad + 1 + rows * row_len + pad_last + 1);
idx.reserve(bl.n_out);
for (int i = 0; i < lead_pad; i++) {
idx.push_back(idx_pad);
}
idx.push_back(idx_start);
// pairs of adjacent rows are interleaved column-wise ("N-layout")
// ref: build_image_block in inference/image_processor.py
for (int t = 0; t < rows * row_len; t++) {
const int g = t / (2 * row_len);
const int rem = t % (2 * row_len);
for (int t = 0; t < bl.rows * bl.row_len; t++) {
const int g = t / (2 * bl.row_len);
const int rem = t % (2 * bl.row_len);
const int c = rem / 2; // column
const int r = 2 * g + rem % 2; // row
if (r >= n_llm_h) {
@@ -5122,7 +5115,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
idx.push_back(r * n_llm_w + c);
}
}
for (int i = 0; i < pad_last; i++) {
for (int i = 0; i < bl.pad_last; i++) {
idx.push_back(idx_pad);
}
idx.push_back(idx_end);
@@ -5869,6 +5862,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n");
LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens);
// TEMP debugging (parity validation), will be removed before merge
// when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data]
const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS");
if (dump_path && strcmp(dump_path, "1") != 0) {
FILE * f = fopen(dump_path, "wb");
if (f) {
const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd };
fwrite(hdr, sizeof(hdr), 1, f);
fwrite(emb_data.data(), sizeof(float), emb_data.size(), f);
fclose(f);
}
}
// Print first few values of first token
LOG_INF("Token 0 (first 16 values): ");
for (int i = 0; i < std::min((int64_t)16, n_embd); i++) {
tools/mtmd/models/deepseek4v.cpp
+4 -7
View File
@@ -9,7 +9,7 @@
//
// [PAD]*lead_pad [START] <interleaved rows> [PAD]*pad_last [END]
//
// each aligner row ends with a NEWLINE, the row count is padded to an even number with PAD rows
// each aligner row ends with a NEWLINE, an odd row count is padded with a full row of PADs
// pairs of adjacent rows are interleaved column-wise ("N-layout")
// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp)
//
@@ -83,12 +83,9 @@ ggml_cgraph * clip_graph_deepseek4v::build() {
cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, tok, n_embd_out, 1), 1);
}
const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge;
const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge;
const int rows = n_llm_h + (n_llm_h % 2);
const int row_len = n_llm_w + 1;
const int pad_last = (rows / 2 * row_len) % 2 * 2;
const int n_out = img.lead_pad + 1 + rows * row_len + pad_last + 1;
const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge;
const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge;
const int n_out = dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out;
GGML_ASSERT(n_grid == n_llm_w * n_llm_h);
ggml_tensor * layout_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out);
tools/mtmd/mtmd-image.cpp
+3 -7
View File
@@ -1104,11 +1104,7 @@ mtmd_image_preprocessor_deepseek4v::grid_info mtmd_image_preprocessor_deepseek4v
grid_info g;
g.n_llm_h = ((best_height / patch_size) + r - 1) / r;
g.n_llm_w = ((best_width / patch_size) + r - 1) / r;
g.n_tokens = g.n_llm_h * (g.n_llm_w + 1) + 2;
if (g.n_llm_h % 2 == 1) {
g.n_tokens += g.n_llm_w + 1; // rows are padded to an even count
}
g.n_tokens += (g.n_llm_h + 1) / 2 * (g.n_llm_w + 1) % 2 * 2; // trailing pads
g.n_tokens = dsv4_get_block_layout(g.n_llm_w, g.n_llm_h, 0).n_out;
return g;
}
@@ -1128,8 +1124,8 @@ void mtmd_image_preprocessor_deepseek4v::solve_resize_ratio(int height, int widt
best_height = max_h * p * r;
} else if (max_h_f < 2.0) {
const int max_h = 2;
const int max_w = ((max_n_token - 2) / max_h) - 1;
GGML_ASSERT(max_w > 1);
// guard tiny budgets; cannot be hit with the current lower bound on max_n_token
const int max_w = std::max(((max_n_token - 2) / max_h) - 1, 2);
best_width = max_w * p * r;
best_height = max_h * p * r;
} else {
tools/mtmd/mtmd.cpp
+1 -1
View File
@@ -1459,7 +1459,7 @@ struct mtmd_tokenizer {
}
if (ctx->proj_type_v() == PROJECTOR_TYPE_DEEPSEEK4V) {
// the text model preceives input in blocks of N tokens (N = COMPRESS_PAD_TO = 4)
// the text model perceives input in blocks of N tokens (N = COMPRESS_PAD_TO = 4, same as the CSA compress ratio)
// image need to be aligned to block size, while adding IMAGE_PAD embeddings to the beginning
// TODO @ngxson : maybe refactor this in the future
constexpr int32_t align = 4;