move add_special to call level

tests/test-mtmd-c-api.c

@@ -130,6 +130,39 @@ int main(void) {
     }
     printf("Chunk save/load round-trip OK\n");
 
+    // test input validation of mtmd_tokenize_from_parts()
+    // invalid parts are rejected before the ctx is used, so NULL ctx is OK here
+    {
+        mtmd_input_chunks * out = mtmd_input_chunks_init();
+        mtmd_bitmap * bmp = mtmd_bitmap_init(4, 4, NULL); // placeholder bitmap
+        struct mtmd_input_text txt = { "hello", 5, false, false };
+        struct mtmd_input_text txt_null = { NULL, 0, false, false };
+
+        struct mtmd_input_part part_both      = { &txt, bmp };
+        struct mtmd_input_part part_neither   = { NULL, NULL };
+        struct mtmd_input_part part_null_text = { &txt_null, NULL };
+        const mtmd_input_part * parts[1];
+        int32_t rc;
+
+        parts[0] = &part_both;
+        rc = mtmd_tokenize_from_parts(NULL, out, parts, 1, false);
+        printf("tokenize part with both text and bitmap rc = %d (expect 1)\n", rc);
+        assert(rc == 1);
+
+        parts[0] = &part_neither;
+        rc = mtmd_tokenize_from_parts(NULL, out, parts, 1, false);
+        printf("tokenize part with neither text nor bitmap rc = %d (expect 1)\n", rc);
+        assert(rc == 1);
+
+        parts[0] = &part_null_text;
+        rc = mtmd_tokenize_from_parts(NULL, out, parts, 1, false);
+        printf("tokenize part with null text pointer rc = %d (expect 1)\n", rc);
+        assert(rc == 1);
+
+        mtmd_bitmap_free(bmp);
+        mtmd_input_chunks_free(out);
+    }
+
     // Free the chunks
     mtmd_input_chunks_free(chunks);
 

tools/mtmd/README-dev.md

@@ -20,6 +20,7 @@ In short:
 A typical pipeline of the core libmtmd is as follows:
 - A bitmap (RGB image or PCM audio) is created
 - Bitmap and the text prompt is provided to `mtmd_tokenize()` that breaks the input into chunks
+    - Alternatively, `mtmd_tokenize_from_parts()` takes a list of pre-split text/media parts instead of a marker-based prompt
     - The tokenizer function first expands a "lazy" bitmap if it finds one. Typically, this is used by video, so that one media token corresponds to one input bitmap
     - For models that support "fused" temporal frames like Qwen-VL, the tokenizer tries to merge pair of consecutive frames into one batch. Only bitmaps marked by `mtmd_bitmap_set_mergeable()` are merged
     - The preprocessor will then be called, which produces a list of chunks

tools/mtmd/mtmd-cli.cpp

@@ -287,11 +287,11 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) {
         return 1;
     }
 
-    // interleave text and media parts; only the first text part may add BOS
+    // interleave text and media parts
     std::vector<mtmd_input_text> texts(segments.size());
     std::vector<mtmd_input_part> parts;
     for (size_t i = 0; i < segments.size(); i++) {
-        texts[i] = {segments[i].data(), segments[i].size(), i == 0 && add_bos, /* parse_special */ true};
+        texts[i] = {segments[i].data(), segments[i].size(), /* add_special */ false, /* parse_special */ true};
         parts.push_back({&texts[i], nullptr});
         if (i < bitmaps_c_ptr.size()) {
             parts.push_back({nullptr, bitmaps_c_ptr[i]});
@@ -306,7 +306,8 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) {
     int32_t res = mtmd_tokenize_from_parts(ctx.ctx_vision.get(),
                         chunks.ptr.get(), // output
                         parts_ptr.data(),
-                        parts_ptr.size());
+                        parts_ptr.size(),
+                        add_bos);
     if (res != 0) {
         LOG_ERR("Unable to tokenize prompt, res = %d\n", res);
         return 1;

tools/mtmd/mtmd-internal.h

@@ -13,8 +13,7 @@
 struct mtmd_internal_part {
     std::string text;
     const mtmd_bitmap * bitmap;
-    // tokenizer flags, only used for text parts
-    bool add_special   = false;
+    // only used for text parts
     bool parse_special = false;
 };
 

tools/mtmd/mtmd.cpp

@@ -1153,7 +1153,7 @@ struct mtmd_tokenizer {
                 }
                 parts.push_back({"", bitmaps[i_bm++]});
             } else {
-                parts.push_back({std::move(part), nullptr, false, parse_special});
+                parts.push_back({std::move(part), nullptr, parse_special});
             }
         }
 
@@ -1172,15 +1172,16 @@ struct mtmd_tokenizer {
 
     mtmd_tokenizer(mtmd_context * ctx,
             const mtmd_input_part ** input_parts,
-            size_t n_parts) : ctx(ctx) {
-        add_special   = false; // add_special is controlled per text part
-        parse_special = true;  // only used for text returned by lazy bitmaps
+            size_t n_parts,
+            bool add_special) : ctx(ctx) {
+        this->add_special = add_special;
+        parse_special = true; // only used for text returned by lazy bitmaps
         vocab         = ctx->vocab;
 
         for (size_t i = 0; i < n_parts; i++) {
             const mtmd_input_part * p = input_parts[i];
             if (p->text != nullptr) {
-                parts.push_back({std::string(p->text->text, p->text->text_len), nullptr, p->text->add_special, p->text->parse_special});
+                parts.push_back({std::string(p->text->text, p->text->text_len), nullptr, p->text->parse_special});
             } else {
                 parts.push_back({"", p->bitmap});
             }
@@ -1213,7 +1214,7 @@ struct mtmd_tokenizer {
                             LOG_DBG("%s: lazy callback returned bitmap with dimensions %d x %d\n", __func__, out_bm->nx, out_bm->ny);
                         } else if (out_str) {
                             auto & ptr = text_from_lazy.emplace_back(out_str); // remember to free it later
-                            expanded.push_back({ptr, nullptr, false, parse_special});
+                            expanded.push_back({ptr, nullptr, parse_special});
                             LOG_DBG("%s: lazy callback returned text: %s\n", __func__, out_str);
                         }
                     } else if (res == -1) {
@@ -1257,7 +1258,7 @@ struct mtmd_tokenizer {
                     return res;
                 }
             } else {
-                add_text(p.text, p.parse_special, p.add_special);
+                add_text(p.text, p.parse_special);
             }
         }
 
@@ -1297,12 +1298,12 @@ struct mtmd_tokenizer {
         return 0;
     }
 
-    void add_text(const std::string & txt, bool parse_special, bool add_special = false) {
+    void add_text(const std::string & txt, bool parse_special) {
         if (vocab == nullptr) {
             throw std::runtime_error("llama_vocab is not provided");
         }
         LOG_DBG("%s: %s\n", __func__, txt.c_str());
-        auto tokens = mtmd_tokenize_text_internal(vocab, txt, add_special, parse_special);
+        auto tokens = mtmd_tokenize_text_internal(vocab, txt, /* add_special */ false, parse_special);
         add_text(tokens);
     }
 
@@ -1730,15 +1731,20 @@ int32_t mtmd_tokenize(mtmd_context * ctx,
 int32_t mtmd_tokenize_from_parts(mtmd_context * ctx,
             mtmd_input_chunks * output,
             const mtmd_input_part ** parts,
-            size_t n_parts) {
+            size_t n_parts,
+            bool add_special) {
     for (size_t i = 0; i < n_parts; i++) {
         if ((parts[i]->text == nullptr) == (parts[i]->bitmap == nullptr)) {
             LOG_ERR("%s: part %zu must have either text or bitmap set, not both\n", __func__, i);
             return 1;
         }
+        if (parts[i]->text != nullptr && parts[i]->text->text == nullptr) {
+            LOG_ERR("%s: part %zu has null text pointer\n", __func__, i);
+            return 1;
+        }
     }
     try {
-        mtmd_tokenizer tokenizer(ctx, parts, n_parts);
+        mtmd_tokenizer tokenizer(ctx, parts, n_parts, add_special);
         return tokenizer.tokenize(output);
     } catch (const std::exception & e) {
         LOG_ERR("%s: error: %s\n", __func__, e.what());

tools/mtmd/mtmd.h

@@ -308,12 +308,14 @@ MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx,
 // same as mtmd_tokenize(), but takes an array of mtmd_input_part
 // use cases:
 // - when you don't want to use media markers (they will be tokenized as normal text)
-// - when you want to control add_special for each text part
+// - when you want to control parse_special for each text part
+// note: per-part add_special will be ignored
 // return 1 if a part has both text and bitmap set (or neither)
 MTMD_API int32_t mtmd_tokenize_from_parts(mtmd_context * ctx,
                                           mtmd_input_chunks * output,
                                           const mtmd_input_part ** parts,
-                                          size_t n_parts);
+                                          size_t n_parts,
+                                          bool add_special);
 
 DEPRECATED(MTMD_API int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens),
            "use mtmd_encode_chunk() instead");