From 3a798bf2f3e0a5ee90c0a7bcef60fb3ef1b4b8b3 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 1 Sep 2026 12:37:26 +0200 Subject: [PATCH] correct token count --- tools/mtmd/clip-impl.h | 16 ++++++++++ tools/mtmd/clip.cpp | 52 ++++++++++++++++++-------------- tools/mtmd/models/deepseek4v.cpp | 11 +++---- tools/mtmd/mtmd-image.cpp | 10 ++---- tools/mtmd/mtmd.cpp | 2 +- 5 files changed, 53 insertions(+), 38 deletions(-) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index af2ac969ff..72148a4d9a 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -779,6 +779,22 @@ static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_ } } +// deepseek4v: layout of the LLM token block built from the aligner grid +struct dsv4_block_layout { + int rows; // grid rows, padded to an even count + int row_len; // grid width + 1 newline + int pad_last; // trailing pads + int n_out; // total block size, including lead pads and the start/end sentinels +}; +static inline dsv4_block_layout dsv4_get_block_layout(int n_llm_w, int n_llm_h, int lead_pad) { + dsv4_block_layout bl; + bl.rows = n_llm_h + (n_llm_h % 2); + bl.row_len = n_llm_w + 1; + bl.pad_last = (bl.rows / 2 * bl.row_len) % 2 * 2; + bl.n_out = lead_pad + 1 + bl.rows * bl.row_len + bl.pad_last + 1; + return bl; +} + // // logging // diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 19c6745c13..a054dc515e 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -4193,15 +4193,10 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { } break; case PROJECTOR_TYPE_DEEPSEEK4V: { - // aligner grid + newlines + pads + start/end sentinels - // keep in sync with clip_graph_deepseek4v::build const int out_patch_size = params.patch_size * params.n_merge; - const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size; - const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size; - const int rows = n_llm_h + (n_llm_h % 2); - const int row_len = n_llm_w + 1; - const int pad_last = (rows / 2 * row_len) % 2 * 2; - n_patches = img->lead_pad + 1 + rows * row_len + pad_last + 1; + const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size; + const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size; + n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out; } break; case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_DOTS_OCR: @@ -5077,41 +5072,39 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { case PROJECTOR_TYPE_DEEPSEEK4V: { // set the 2D positions (mrope layout, only the first 2 channels are used) - int n_patches_per_col = image_size_width / patch_size; + int n_patches_per_row = image_size_width / patch_size; std::vector positions(n_pos * 4, 0); for (int i = 0; i < n_pos; i++) { - positions[i] = i / n_patches_per_col; // row - positions[n_pos + i] = i % n_patches_per_col; // col + positions[i] = i / n_patches_per_row; // row + positions[n_pos + i] = i % n_patches_per_row; // col } set_input_i32("positions", positions); // token block layout index (see clip_graph_deepseek4v::build) // rows [0, n_grid) are the aligner output, the sentinels follow - const int n_merge = hparams.n_merge; - const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge; - const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge; - const int n_grid = n_llm_w * n_llm_h; + const int n_merge = hparams.n_merge; + const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge; + const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge; + const int n_grid = n_llm_w * n_llm_h; const int idx_start = n_grid; const int idx_end = n_grid + 1; const int idx_newline = n_grid + 2; const int idx_pad = n_grid + 3; - const int rows = n_llm_h + (n_llm_h % 2); - const int row_len = n_llm_w + 1; - const int pad_last = (rows / 2 * row_len) % 2 * 2; const int lead_pad = imgs.entries[0].lead_pad; + const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad); std::vector idx; - idx.reserve(lead_pad + 1 + rows * row_len + pad_last + 1); + idx.reserve(bl.n_out); for (int i = 0; i < lead_pad; i++) { idx.push_back(idx_pad); } idx.push_back(idx_start); // pairs of adjacent rows are interleaved column-wise ("N-layout") // ref: build_image_block in inference/image_processor.py - for (int t = 0; t < rows * row_len; t++) { - const int g = t / (2 * row_len); - const int rem = t % (2 * row_len); + for (int t = 0; t < bl.rows * bl.row_len; t++) { + const int g = t / (2 * bl.row_len); + const int rem = t % (2 * bl.row_len); const int c = rem / 2; // column const int r = 2 * g + rem % 2; // row if (r >= n_llm_h) { @@ -5122,7 +5115,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { idx.push_back(r * n_llm_w + c); } } - for (int i = 0; i < pad_last; i++) { + for (int i = 0; i < bl.pad_last; i++) { idx.push_back(idx_pad); } idx.push_back(idx_end); @@ -5869,6 +5862,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n"); LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens); + // TEMP debugging (parity validation), will be removed before merge + // when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data] + const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS"); + if (dump_path && strcmp(dump_path, "1") != 0) { + FILE * f = fopen(dump_path, "wb"); + if (f) { + const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd }; + fwrite(hdr, sizeof(hdr), 1, f); + fwrite(emb_data.data(), sizeof(float), emb_data.size(), f); + fclose(f); + } + } + // Print first few values of first token LOG_INF("Token 0 (first 16 values): "); for (int i = 0; i < std::min((int64_t)16, n_embd); i++) { diff --git a/tools/mtmd/models/deepseek4v.cpp b/tools/mtmd/models/deepseek4v.cpp index d25df9156b..ffe8f59d99 100644 --- a/tools/mtmd/models/deepseek4v.cpp +++ b/tools/mtmd/models/deepseek4v.cpp @@ -9,7 +9,7 @@ // // [PAD]*lead_pad [START] [PAD]*pad_last [END] // -// each aligner row ends with a NEWLINE, the row count is padded to an even number with PAD rows +// each aligner row ends with a NEWLINE, an odd row count is padded with a full row of PADs // pairs of adjacent rows are interleaved column-wise ("N-layout") // the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp) // @@ -83,12 +83,9 @@ ggml_cgraph * clip_graph_deepseek4v::build() { cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, tok, n_embd_out, 1), 1); } - const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge; - const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge; - const int rows = n_llm_h + (n_llm_h % 2); - const int row_len = n_llm_w + 1; - const int pad_last = (rows / 2 * row_len) % 2 * 2; - const int n_out = img.lead_pad + 1 + rows * row_len + pad_last + 1; + const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge; + const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge; + const int n_out = dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out; GGML_ASSERT(n_grid == n_llm_w * n_llm_h); ggml_tensor * layout_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out); diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 405d5fb88a..65c24f4d47 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -1104,11 +1104,7 @@ mtmd_image_preprocessor_deepseek4v::grid_info mtmd_image_preprocessor_deepseek4v grid_info g; g.n_llm_h = ((best_height / patch_size) + r - 1) / r; g.n_llm_w = ((best_width / patch_size) + r - 1) / r; - g.n_tokens = g.n_llm_h * (g.n_llm_w + 1) + 2; - if (g.n_llm_h % 2 == 1) { - g.n_tokens += g.n_llm_w + 1; // rows are padded to an even count - } - g.n_tokens += (g.n_llm_h + 1) / 2 * (g.n_llm_w + 1) % 2 * 2; // trailing pads + g.n_tokens = dsv4_get_block_layout(g.n_llm_w, g.n_llm_h, 0).n_out; return g; } @@ -1128,8 +1124,8 @@ void mtmd_image_preprocessor_deepseek4v::solve_resize_ratio(int height, int widt best_height = max_h * p * r; } else if (max_h_f < 2.0) { const int max_h = 2; - const int max_w = ((max_n_token - 2) / max_h) - 1; - GGML_ASSERT(max_w > 1); + // guard tiny budgets; cannot be hit with the current lower bound on max_n_token + const int max_w = std::max(((max_n_token - 2) / max_h) - 1, 2); best_width = max_w * p * r; best_height = max_h * p * r; } else { diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 446a126978..d2b88b1e46 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1459,7 +1459,7 @@ struct mtmd_tokenizer { } if (ctx->proj_type_v() == PROJECTOR_TYPE_DEEPSEEK4V) { - // the text model preceives input in blocks of N tokens (N = COMPRESS_PAD_TO = 4) + // the text model perceives input in blocks of N tokens (N = COMPRESS_PAD_TO = 4, same as the CSA compress ratio) // image need to be aligned to block size, while adding IMAGE_PAD embeddings to the beginning // TODO @ngxson : maybe refactor this in the future constexpr int32_t align = 4;