diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index df18b0a42e..2ec6b23915 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -80,7 +80,7 @@ MAKE_TEST(test_temporal_merge_grouping) { // spec chars: // v = video frame, w = video frame of another size, a = audio, i = plain image, t = text auto make_parts = [&pool](const std::string & spec) { - std::vector parts; + std::vector parts; for (char c : spec) { if (c == 't') { parts.push_back({ "hello", nullptr }); diff --git a/tools/mtmd/mtmd-internal.h b/tools/mtmd/mtmd-internal.h index 067fa88b99..d7dea5e8bf 100644 --- a/tools/mtmd/mtmd-internal.h +++ b/tools/mtmd/mtmd-internal.h @@ -10,10 +10,13 @@ #define MTMD_INTERNAL_HEADER // bitmap is null for text parts -struct mtmd_input_part { +struct mtmd_internal_part { std::string text; const mtmd_bitmap * bitmap; + // tokenizer flags, only used for text parts + bool add_special = false; + bool parse_special = false; }; // [QWEN_VIDEO] merged parts are erased from `parts`, so one group always maps to one part -std::vector> mtmd_group_mergeable_bitmaps(std::vector & parts, int n_merge); +std::vector> mtmd_group_mergeable_bitmaps(std::vector & parts, int n_merge); diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 5b306180d6..f7ac1c4627 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1090,7 +1090,7 @@ void mtmd_free(mtmd_context * ctx) { delete ctx; } -std::vector> mtmd_group_mergeable_bitmaps(std::vector & parts, int n_merge) { +std::vector> mtmd_group_mergeable_bitmaps(std::vector & parts, int n_merge) { std::vector> output; for (size_t i = 0; i < parts.size(); i++) { if (parts[i].bitmap == nullptr) { @@ -1117,7 +1117,7 @@ struct mtmd_tokenizer { bool parse_special; const llama_vocab * vocab; - using part = mtmd_input_part; + using part = mtmd_internal_part; std::vector parts; // these will be freed when mtmd_tokenizer finishes std::vector bm_from_lazy; // TODO @ngxson : refactor, free bm_from_lazy progressively @@ -1153,7 +1153,7 @@ struct mtmd_tokenizer { } parts.push_back({"", bitmaps[i_bm++]}); } else { - parts.push_back({std::move(part), nullptr}); + parts.push_back({std::move(part), nullptr, false, parse_special}); } } @@ -1170,6 +1170,25 @@ struct mtmd_tokenizer { expand_lazy_bitmaps(); } + mtmd_tokenizer(mtmd_context * ctx, + const mtmd_input_part ** input_parts, + size_t n_parts) : ctx(ctx) { + add_special = false; // add_special is controlled per text part + parse_special = true; // only used for text returned by lazy bitmaps + vocab = ctx->vocab; + + for (size_t i = 0; i < n_parts; i++) { + const mtmd_input_part * p = input_parts[i]; + if (p->text != nullptr) { + parts.push_back({std::string(p->text->text, p->text->text_len), nullptr, p->text->add_special, p->text->parse_special}); + } else { + parts.push_back({"", p->bitmap}); + } + } + + expand_lazy_bitmaps(); + } + void expand_lazy_bitmaps() { std::vector expanded; expanded.reserve(parts.size()); @@ -1194,7 +1213,7 @@ struct mtmd_tokenizer { LOG_DBG("%s: lazy callback returned bitmap with dimensions %d x %d\n", __func__, out_bm->nx, out_bm->ny); } else if (out_str) { auto & ptr = text_from_lazy.emplace_back(out_str); // remember to free it later - expanded.push_back({ptr, nullptr}); + expanded.push_back({ptr, nullptr, false, parse_special}); LOG_DBG("%s: lazy callback returned text: %s\n", __func__, out_str); } } else if (res == -1) { @@ -1238,7 +1257,7 @@ struct mtmd_tokenizer { return res; } } else { - add_text(p.text, parse_special); + add_text(p.text, p.parse_special, p.add_special); } } @@ -1278,12 +1297,12 @@ struct mtmd_tokenizer { return 0; } - void add_text(const std::string & txt, bool parse_special) { + void add_text(const std::string & txt, bool parse_special, bool add_special = false) { if (vocab == nullptr) { throw std::runtime_error("llama_vocab is not provided"); } LOG_DBG("%s: %s\n", __func__, txt.c_str()); - auto tokens = mtmd_tokenize_text_internal(vocab, txt, /* add_special */ false, parse_special); + auto tokens = mtmd_tokenize_text_internal(vocab, txt, add_special, parse_special); add_text(tokens); } @@ -1708,6 +1727,25 @@ int32_t mtmd_tokenize(mtmd_context * ctx, } } +int32_t mtmd_tokenize_from_parts(mtmd_context * ctx, + mtmd_input_chunks * output, + const mtmd_input_part ** parts, + size_t n_parts) { + for (size_t i = 0; i < n_parts; i++) { + if ((parts[i]->text == nullptr) == (parts[i]->bitmap == nullptr)) { + LOG_ERR("%s: part %zu must have either text or bitmap set, not both\n", __func__, i); + return 1; + } + } + try { + mtmd_tokenizer tokenizer(ctx, parts, n_parts); + return tokenizer.tokenize(output); + } catch (const std::exception & e) { + LOG_ERR("%s: error: %s\n", __func__, e.what()); + return 2; + } +} + static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * image_tokens, std::vector & out_embd) { clip_ctx * ctx_clip = ctx->ctx_v; if (!ctx_clip) { diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index ef88efd316..e1c2829727 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -73,6 +73,12 @@ struct mtmd_input_text { bool parse_special; }; +struct mtmd_input_part { + // only text or bitmap can be set, not both + struct mtmd_input_text * text; + struct mtmd_bitmap * bitmap; +}; + // // C API // @@ -83,6 +89,7 @@ typedef struct mtmd_image_tokens mtmd_image_tokens; typedef struct mtmd_input_chunk mtmd_input_chunk; typedef struct mtmd_input_chunks mtmd_input_chunks; typedef struct mtmd_input_text mtmd_input_text; +typedef struct mtmd_input_part mtmd_input_part; typedef struct mtmd_batch mtmd_batch; typedef bool (*mtmd_progress_callback)(float progress, void * user_data); @@ -276,10 +283,10 @@ struct mtmd_decoder_pos { // return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position) MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * image_tokens, llama_pos pos_0, size_t i); -// tokenize an input text prompt and a list of bitmaps (images/audio) -// the prompt must have the input image marker (default: "<__media__>") in it +// tokenize an input text prompt and a list of bitmaps (image/audio) +// the prompt must have the input media marker (default: "<__media__>") in it // the default marker is defined by mtmd_default_marker() -// the marker will be replaced with the image/audio chunk +// the marker will be replaced with the media chunk // for example: // "here is an image: <__media__>\ndescribe it in detail." // this will gives 3 chunks: @@ -291,13 +298,23 @@ MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_im // return values: // 0 on success // 1 on number of bitmaps not matching the number of markers -// 2 on image preprocessing error +// 2 on media preprocessing error MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx, mtmd_input_chunks * output, const mtmd_input_text * text, const mtmd_bitmap ** bitmaps, size_t n_bitmaps); +// same as mtmd_tokenize(), but takes an array of mtmd_input_part +// use cases: +// - when you don't want to use media markers (they will be tokenized as normal text) +// - when you want to control add_special for each text part +// return 1 if a part has both text and bitmap set (or neither) +MTMD_API int32_t mtmd_tokenize_from_parts(mtmd_context * ctx, + mtmd_input_chunks * output, + const mtmd_input_part ** parts, + size_t n_parts); + DEPRECATED(MTMD_API int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens), "use mtmd_encode_chunk() instead");