Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-04 20:11:17 +02:00
Code Issues Packages Projects Releases Wiki Activity

args: add --video-* CLI arguments (#24318)

Browse Source 
* args: add --video-* CLI arguments

* gen docs

* nits

* add mtmd_helper_init_opt
This commit is contained in:
Xuan-Son Nguyen
2026-08-27 12:11:12 +02:00
committed by GitHub
parent 915dc6d38c
commit f29551215b
12 changed files with 138 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-common.cpp
+17 -11
View File
@@ -910,12 +910,17 @@ size_t validate_utf8(const std::string& text) {
return len;
}
server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder) {
server_tokens process_mtmd_prompt(
mtmd_context * mctx,
const std::string & prompt,
const std::vector<raw_buffer> & files,
const mtmd_helper_init_opt & init_opt,
bool is_placeholder) {
// these will be freed upon going out of scope
mtmd::bitmaps bitmaps;
std::vector<mtmd_helper::video_ptr> videos;
for (auto & file : files) {
auto out = mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder);
auto out = mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder, init_opt);
if (!out.bitmap) {
throw std::runtime_error("Failed to load image or audio file");
}
@@ -956,7 +961,7 @@ server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & promp
* - "prompt": [12, 34, "string", 56, 78]
* - "prompt": { "prompt_string": "string", "multimodal_data": [ "base64" ] }
*/
static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special) {
static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special, const mtmd_helper_init_opt & init_opt) {
constexpr char JSON_STRING_PROMPT_KEY[] = "prompt_string";
constexpr char JSON_MTMD_DATA_KEY[] = "multimodal_data";
const bool has_mtmd = mctx != nullptr;
@@ -979,7 +984,7 @@ static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_co
for (const auto & entry : json_prompt.at(JSON_MTMD_DATA_KEY)) {
files.push_back(base64_decode(entry));
}
return process_mtmd_prompt(mctx, json_prompt.at(JSON_STRING_PROMPT_KEY), files);
return process_mtmd_prompt(mctx, json_prompt.at(JSON_STRING_PROMPT_KEY), files, init_opt);
} else {
// Not multimodal, but contains a subobject.
llama_tokens tmp = tokenize_mixed(vocab, json_prompt.at(JSON_STRING_PROMPT_KEY), add_special, parse_special);
@@ -990,15 +995,15 @@ static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_co
}
}
std::vector<server_tokens> tokenize_input_prompts(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special) {
std::vector<server_tokens> tokenize_input_prompts(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special, const mtmd_helper_init_opt & init_opt) {
std::vector<server_tokens> result;
if (json_prompt.is_array() && !json_is_array_and_contains_numbers(json_prompt)) {
result.reserve(json_prompt.size());
for (const auto & p : json_prompt) {
result.push_back(tokenize_input_subprompt(vocab, mctx, p,add_special, parse_special));
result.push_back(tokenize_input_subprompt(vocab, mctx, p, add_special, parse_special, init_opt));
}
} else {
result.push_back(tokenize_input_subprompt(vocab, mctx, json_prompt, add_special, parse_special));
result.push_back(tokenize_input_subprompt(vocab, mctx, json_prompt, add_special, parse_special, init_opt));
}
if (result.empty()) {
throw std::runtime_error("\"prompt\" must not be empty");
@@ -1787,7 +1792,8 @@ server_tokens format_prompt_rerank(
const struct llama_vocab * vocab,
mtmd_context * mctx,
const std::string & query,
const std::string & doc) {
const std::string & doc,
const mtmd_helper_init_opt & init_opt) {
server_tokens result = {};
const char * rerank_prompt = llama_model_chat_template(model, "rerank");
@@ -1796,12 +1802,12 @@ server_tokens format_prompt_rerank(
std::string prompt = rerank_prompt;
string_replace_all(prompt, "{query}" , query);
string_replace_all(prompt, "{document}", doc );
server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true);
server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true, init_opt);
result.push_back(tokens);
} else {
// Get EOS token - use SEP token as fallback if EOS is not available
server_tokens query_tokens = tokenize_input_subprompt(vocab, mctx, query, false, false);
server_tokens doc_tokens = tokenize_input_subprompt(vocab, mctx, doc, false, false);
server_tokens query_tokens = tokenize_input_subprompt(vocab, mctx, query, false, false, init_opt);
server_tokens doc_tokens = tokenize_input_subprompt(vocab, mctx, doc, false, false, init_opt);
llama_token eos_token = llama_vocab_eos(vocab);
if (eos_token == LLAMA_TOKEN_NULL) {
eos_token = llama_vocab_sep(vocab);