From 1a257585da746e6f02b407b39826c95dd8216bd7 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 2 Sep 2026 19:28:24 +0200 Subject: [PATCH] move add_special to call level --- tests/test-mtmd-c-api.c | 33 +++++++++++++++++++++++++++++++++ tools/mtmd/README-dev.md | 1 + tools/mtmd/mtmd-cli.cpp | 7 ++++--- tools/mtmd/mtmd-internal.h | 3 +-- tools/mtmd/mtmd.cpp | 28 +++++++++++++++++----------- tools/mtmd/mtmd.h | 6 ++++-- 6 files changed, 60 insertions(+), 18 deletions(-) diff --git a/tests/test-mtmd-c-api.c b/tests/test-mtmd-c-api.c index 970d8a6000..664c56cf80 100644 --- a/tests/test-mtmd-c-api.c +++ b/tests/test-mtmd-c-api.c @@ -130,6 +130,39 @@ int main(void) { } printf("Chunk save/load round-trip OK\n"); + // test input validation of mtmd_tokenize_from_parts() + // invalid parts are rejected before the ctx is used, so NULL ctx is OK here + { + mtmd_input_chunks * out = mtmd_input_chunks_init(); + mtmd_bitmap * bmp = mtmd_bitmap_init(4, 4, NULL); // placeholder bitmap + struct mtmd_input_text txt = { "hello", 5, false, false }; + struct mtmd_input_text txt_null = { NULL, 0, false, false }; + + struct mtmd_input_part part_both = { &txt, bmp }; + struct mtmd_input_part part_neither = { NULL, NULL }; + struct mtmd_input_part part_null_text = { &txt_null, NULL }; + const mtmd_input_part * parts[1]; + int32_t rc; + + parts[0] = &part_both; + rc = mtmd_tokenize_from_parts(NULL, out, parts, 1, false); + printf("tokenize part with both text and bitmap rc = %d (expect 1)\n", rc); + assert(rc == 1); + + parts[0] = &part_neither; + rc = mtmd_tokenize_from_parts(NULL, out, parts, 1, false); + printf("tokenize part with neither text nor bitmap rc = %d (expect 1)\n", rc); + assert(rc == 1); + + parts[0] = &part_null_text; + rc = mtmd_tokenize_from_parts(NULL, out, parts, 1, false); + printf("tokenize part with null text pointer rc = %d (expect 1)\n", rc); + assert(rc == 1); + + mtmd_bitmap_free(bmp); + mtmd_input_chunks_free(out); + } + // Free the chunks mtmd_input_chunks_free(chunks); diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md index e14906823a..b85627d2a4 100644 --- a/tools/mtmd/README-dev.md +++ b/tools/mtmd/README-dev.md @@ -20,6 +20,7 @@ In short: A typical pipeline of the core libmtmd is as follows: - A bitmap (RGB image or PCM audio) is created - Bitmap and the text prompt is provided to `mtmd_tokenize()` that breaks the input into chunks + - Alternatively, `mtmd_tokenize_from_parts()` takes a list of pre-split text/media parts instead of a marker-based prompt - The tokenizer function first expands a "lazy" bitmap if it finds one. Typically, this is used by video, so that one media token corresponds to one input bitmap - For models that support "fused" temporal frames like Qwen-VL, the tokenizer tries to merge pair of consecutive frames into one batch. Only bitmaps marked by `mtmd_bitmap_set_mergeable()` are merged - The preprocessor will then be called, which produces a list of chunks diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index c20f936f75..ba18b3e32b 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -287,11 +287,11 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) { return 1; } - // interleave text and media parts; only the first text part may add BOS + // interleave text and media parts std::vector texts(segments.size()); std::vector parts; for (size_t i = 0; i < segments.size(); i++) { - texts[i] = {segments[i].data(), segments[i].size(), i == 0 && add_bos, /* parse_special */ true}; + texts[i] = {segments[i].data(), segments[i].size(), /* add_special */ false, /* parse_special */ true}; parts.push_back({&texts[i], nullptr}); if (i < bitmaps_c_ptr.size()) { parts.push_back({nullptr, bitmaps_c_ptr[i]}); @@ -306,7 +306,8 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) { int32_t res = mtmd_tokenize_from_parts(ctx.ctx_vision.get(), chunks.ptr.get(), // output parts_ptr.data(), - parts_ptr.size()); + parts_ptr.size(), + add_bos); if (res != 0) { LOG_ERR("Unable to tokenize prompt, res = %d\n", res); return 1; diff --git a/tools/mtmd/mtmd-internal.h b/tools/mtmd/mtmd-internal.h index d7dea5e8bf..e7c62773e3 100644 --- a/tools/mtmd/mtmd-internal.h +++ b/tools/mtmd/mtmd-internal.h @@ -13,8 +13,7 @@ struct mtmd_internal_part { std::string text; const mtmd_bitmap * bitmap; - // tokenizer flags, only used for text parts - bool add_special = false; + // only used for text parts bool parse_special = false; }; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index f7ac1c4627..368960aa93 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1153,7 +1153,7 @@ struct mtmd_tokenizer { } parts.push_back({"", bitmaps[i_bm++]}); } else { - parts.push_back({std::move(part), nullptr, false, parse_special}); + parts.push_back({std::move(part), nullptr, parse_special}); } } @@ -1172,15 +1172,16 @@ struct mtmd_tokenizer { mtmd_tokenizer(mtmd_context * ctx, const mtmd_input_part ** input_parts, - size_t n_parts) : ctx(ctx) { - add_special = false; // add_special is controlled per text part - parse_special = true; // only used for text returned by lazy bitmaps + size_t n_parts, + bool add_special) : ctx(ctx) { + this->add_special = add_special; + parse_special = true; // only used for text returned by lazy bitmaps vocab = ctx->vocab; for (size_t i = 0; i < n_parts; i++) { const mtmd_input_part * p = input_parts[i]; if (p->text != nullptr) { - parts.push_back({std::string(p->text->text, p->text->text_len), nullptr, p->text->add_special, p->text->parse_special}); + parts.push_back({std::string(p->text->text, p->text->text_len), nullptr, p->text->parse_special}); } else { parts.push_back({"", p->bitmap}); } @@ -1213,7 +1214,7 @@ struct mtmd_tokenizer { LOG_DBG("%s: lazy callback returned bitmap with dimensions %d x %d\n", __func__, out_bm->nx, out_bm->ny); } else if (out_str) { auto & ptr = text_from_lazy.emplace_back(out_str); // remember to free it later - expanded.push_back({ptr, nullptr, false, parse_special}); + expanded.push_back({ptr, nullptr, parse_special}); LOG_DBG("%s: lazy callback returned text: %s\n", __func__, out_str); } } else if (res == -1) { @@ -1257,7 +1258,7 @@ struct mtmd_tokenizer { return res; } } else { - add_text(p.text, p.parse_special, p.add_special); + add_text(p.text, p.parse_special); } } @@ -1297,12 +1298,12 @@ struct mtmd_tokenizer { return 0; } - void add_text(const std::string & txt, bool parse_special, bool add_special = false) { + void add_text(const std::string & txt, bool parse_special) { if (vocab == nullptr) { throw std::runtime_error("llama_vocab is not provided"); } LOG_DBG("%s: %s\n", __func__, txt.c_str()); - auto tokens = mtmd_tokenize_text_internal(vocab, txt, add_special, parse_special); + auto tokens = mtmd_tokenize_text_internal(vocab, txt, /* add_special */ false, parse_special); add_text(tokens); } @@ -1730,15 +1731,20 @@ int32_t mtmd_tokenize(mtmd_context * ctx, int32_t mtmd_tokenize_from_parts(mtmd_context * ctx, mtmd_input_chunks * output, const mtmd_input_part ** parts, - size_t n_parts) { + size_t n_parts, + bool add_special) { for (size_t i = 0; i < n_parts; i++) { if ((parts[i]->text == nullptr) == (parts[i]->bitmap == nullptr)) { LOG_ERR("%s: part %zu must have either text or bitmap set, not both\n", __func__, i); return 1; } + if (parts[i]->text != nullptr && parts[i]->text->text == nullptr) { + LOG_ERR("%s: part %zu has null text pointer\n", __func__, i); + return 1; + } } try { - mtmd_tokenizer tokenizer(ctx, parts, n_parts); + mtmd_tokenizer tokenizer(ctx, parts, n_parts, add_special); return tokenizer.tokenize(output); } catch (const std::exception & e) { LOG_ERR("%s: error: %s\n", __func__, e.what()); diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 83b9ed4d64..0c2f9886e1 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -308,12 +308,14 @@ MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx, // same as mtmd_tokenize(), but takes an array of mtmd_input_part // use cases: // - when you don't want to use media markers (they will be tokenized as normal text) -// - when you want to control add_special for each text part +// - when you want to control parse_special for each text part +// note: per-part add_special will be ignored // return 1 if a part has both text and bitmap set (or neither) MTMD_API int32_t mtmd_tokenize_from_parts(mtmd_context * ctx, mtmd_input_chunks * output, const mtmd_input_part ** parts, - size_t n_parts); + size_t n_parts, + bool add_special); DEPRECATED(MTMD_API int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens), "use mtmd_encode_chunk() instead");