Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 09:15:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

add mtmd_tokenize_from_parts

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-09-02 16:19:28 +02:00
parent 0f3a71be15
commit b6cd8a0cae
4 changed files with 72 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-mtmd-impl.cpp
+1 -1
View File
@@ -80,7 +80,7 @@ MAKE_TEST(test_temporal_merge_grouping) {
// spec chars:
// v = video frame, w = video frame of another size, a = audio, i = plain image, t = text
auto make_parts = [&pool](const std::string & spec) {
std::vector<mtmd_input_part> parts;
std::vector<mtmd_internal_part> parts;
for (char c : spec) {
if (c == 't') {
parts.push_back({ "hello", nullptr });
tools/mtmd/mtmd-internal.h
+5 -2
View File
@@ -10,10 +10,13 @@
#define MTMD_INTERNAL_HEADER
// bitmap is null for text parts
struct mtmd_input_part {
struct mtmd_internal_part {
std::string text;
const mtmd_bitmap * bitmap;
// tokenizer flags, only used for text parts
bool add_special = false;
bool parse_special = false;
};
// [QWEN_VIDEO] merged parts are erased from `parts`, so one group always maps to one part
std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::vector<mtmd_input_part> & parts, int n_merge);
std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::vector<mtmd_internal_part> & parts, int n_merge);
tools/mtmd/mtmd.cpp
+45 -7
View File
@@ -1090,7 +1090,7 @@ void mtmd_free(mtmd_context * ctx) {
delete ctx;
}
std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::vector<mtmd_input_part> & parts, int n_merge) {
std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::vector<mtmd_internal_part> & parts, int n_merge) {
std::vector<std::vector<const mtmd_bitmap *>> output;
for (size_t i = 0; i < parts.size(); i++) {
if (parts[i].bitmap == nullptr) {
@@ -1117,7 +1117,7 @@ struct mtmd_tokenizer {
bool parse_special;
const llama_vocab * vocab;
using part = mtmd_input_part;
using part = mtmd_internal_part;
std::vector<part> parts;
// these will be freed when mtmd_tokenizer finishes
std::vector<mtmd::bitmap> bm_from_lazy; // TODO @ngxson : refactor, free bm_from_lazy progressively
@@ -1153,7 +1153,7 @@ struct mtmd_tokenizer {
}
parts.push_back({"", bitmaps[i_bm++]});
} else {
parts.push_back({std::move(part), nullptr});
parts.push_back({std::move(part), nullptr, false, parse_special});
}
}
@@ -1170,6 +1170,25 @@ struct mtmd_tokenizer {
expand_lazy_bitmaps();
}
mtmd_tokenizer(mtmd_context * ctx,
const mtmd_input_part ** input_parts,
size_t n_parts) : ctx(ctx) {
add_special = false; // add_special is controlled per text part
parse_special = true; // only used for text returned by lazy bitmaps
vocab = ctx->vocab;
for (size_t i = 0; i < n_parts; i++) {
const mtmd_input_part * p = input_parts[i];
if (p->text != nullptr) {
parts.push_back({std::string(p->text->text, p->text->text_len), nullptr, p->text->add_special, p->text->parse_special});
} else {
parts.push_back({"", p->bitmap});
}
}
expand_lazy_bitmaps();
}
void expand_lazy_bitmaps() {
std::vector<part> expanded;
expanded.reserve(parts.size());
@@ -1194,7 +1213,7 @@ struct mtmd_tokenizer {
LOG_DBG("%s: lazy callback returned bitmap with dimensions %d x %d\n", __func__, out_bm->nx, out_bm->ny);
} else if (out_str) {
auto & ptr = text_from_lazy.emplace_back(out_str); // remember to free it later
expanded.push_back({ptr, nullptr});
expanded.push_back({ptr, nullptr, false, parse_special});
LOG_DBG("%s: lazy callback returned text: %s\n", __func__, out_str);
}
} else if (res == -1) {
@@ -1238,7 +1257,7 @@ struct mtmd_tokenizer {
return res;
}
} else {
add_text(p.text, parse_special);
add_text(p.text, p.parse_special, p.add_special);
}
}
@@ -1278,12 +1297,12 @@ struct mtmd_tokenizer {
return 0;
}
void add_text(const std::string & txt, bool parse_special) {
void add_text(const std::string & txt, bool parse_special, bool add_special = false) {
if (vocab == nullptr) {
throw std::runtime_error("llama_vocab is not provided");
}
LOG_DBG("%s: %s\n", __func__, txt.c_str());
auto tokens = mtmd_tokenize_text_internal(vocab, txt, /* add_special */ false, parse_special);
auto tokens = mtmd_tokenize_text_internal(vocab, txt, add_special, parse_special);
add_text(tokens);
}
@@ -1708,6 +1727,25 @@ int32_t mtmd_tokenize(mtmd_context * ctx,
}
}
int32_t mtmd_tokenize_from_parts(mtmd_context * ctx,
mtmd_input_chunks * output,
const mtmd_input_part ** parts,
size_t n_parts) {
for (size_t i = 0; i < n_parts; i++) {
if ((parts[i]->text == nullptr) == (parts[i]->bitmap == nullptr)) {
LOG_ERR("%s: part %zu must have either text or bitmap set, not both\n", __func__, i);
return 1;
}
}
try {
mtmd_tokenizer tokenizer(ctx, parts, n_parts);
return tokenizer.tokenize(output);
} catch (const std::exception & e) {
LOG_ERR("%s: error: %s\n", __func__, e.what());
return 2;
}
}
static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * image_tokens, std::vector<float> & out_embd) {
clip_ctx * ctx_clip = ctx->ctx_v;
if (!ctx_clip) {
tools/mtmd/mtmd.h
+21 -4
View File
@@ -73,6 +73,12 @@ struct mtmd_input_text {
bool parse_special;
};
struct mtmd_input_part {
// only text or bitmap can be set, not both
struct mtmd_input_text * text;
struct mtmd_bitmap * bitmap;
};
//
// C API
//
@@ -83,6 +89,7 @@ typedef struct mtmd_image_tokens mtmd_image_tokens;
typedef struct mtmd_input_chunk mtmd_input_chunk;
typedef struct mtmd_input_chunks mtmd_input_chunks;
typedef struct mtmd_input_text mtmd_input_text;
typedef struct mtmd_input_part mtmd_input_part;
typedef struct mtmd_batch mtmd_batch;
typedef bool (*mtmd_progress_callback)(float progress, void * user_data);
@@ -276,10 +283,10 @@ struct mtmd_decoder_pos {
// return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position)
MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * image_tokens, llama_pos pos_0, size_t i);
// tokenize an input text prompt and a list of bitmaps (images/audio)
// the prompt must have the input image marker (default: "<__media__>") in it
// tokenize an input text prompt and a list of bitmaps (image/audio)
// the prompt must have the input media marker (default: "<__media__>") in it
// the default marker is defined by mtmd_default_marker()
// the marker will be replaced with the image/audio chunk
// the marker will be replaced with the media chunk
// for example:
// "here is an image: <__media__>\ndescribe it in detail."
// this will gives 3 chunks:
@@ -291,13 +298,23 @@ MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_im
// return values:
// 0 on success
// 1 on number of bitmaps not matching the number of markers
// 2 on image preprocessing error
// 2 on media preprocessing error
MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx,
mtmd_input_chunks * output,
const mtmd_input_text * text,
const mtmd_bitmap ** bitmaps,
size_t n_bitmaps);
// same as mtmd_tokenize(), but takes an array of mtmd_input_part
// use cases:
// - when you don't want to use media markers (they will be tokenized as normal text)
// - when you want to control add_special for each text part
// return 1 if a part has both text and bitmap set (or neither)
MTMD_API int32_t mtmd_tokenize_from_parts(mtmd_context * ctx,
mtmd_input_chunks * output,
const mtmd_input_part ** parts,
size_t n_parts);
DEPRECATED(MTMD_API int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens),
"use mtmd_encode_chunk() instead");