Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 09:15:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.devops/cpu.Dockerfile
#	.devops/cuda.Dockerfile
#	.devops/intel.Dockerfile
#	.devops/musa.Dockerfile
#	.devops/openvino.Dockerfile
#	.devops/rocm.Dockerfile
#	.devops/vulkan.Dockerfile
#	.devops/zendnn.Dockerfile
#	.github/workflows/build-webgpu.yml
#	.github/workflows/release.yml
#	ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl
#	ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl
#	ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl
#	ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl
#	ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl
#	tests/CMakeLists.txt
#	tests/test-backend-ops.cpp
#	tests/test-mtmd-c-api.c
#	tools/cli/cli.cpp
#	tools/mtmd/CMakeLists.txt
#	tools/server/README.md
This commit is contained in:
Concedo
2026-06-10 17:21:05 +08:00
parent 258ca8fd90 e25a32e98c
commit afc8de2c6b
55 changed files with 1514 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-common.cpp
+22 -18
View File
@@ -701,29 +701,19 @@ size_t validate_utf8(const std::string& text) {
return len;
}
// Computes FNV-1a hash of the data
static std::string fnv_hash(const uint8_t * data, size_t len) {
const uint64_t fnv_prime = 0x100000001b3ULL;
uint64_t hash = 0xcbf29ce484222325ULL;
for (size_t i = 0; i < len; ++i) {
hash ^= data[i];
hash *= fnv_prime;
}
return std::to_string(hash);
}
server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder) {
// these will be freed upon going out of scope
mtmd::bitmaps bitmaps;
std::vector<mtmd_helper::video_ptr> videos;
for (auto & file : files) {
mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder));
if (!bmp.ptr) {
auto out = mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder);
if (!out.bitmap) {
throw std::runtime_error("Failed to load image or audio file");
}
// calculate bitmap hash (for KV caching)
std::string hash = fnv_hash(bmp.data(), bmp.n_bytes());
bmp.set_id(hash.c_str());
bitmaps.entries.push_back(std::move(bmp));
bitmaps.entries.emplace_back(out.bitmap);
if (out.video_ctx) {
videos.emplace_back(out.video_ctx);
}
}
// process prompt
std::vector<server_tokens> inputs;
@@ -1023,6 +1013,20 @@ json oaicompat_chat_params_parse(
p["text"] = get_media_marker();
p.erase("input_audio");
} else if (type == "input_video") {
if (!opt.allow_video) {
throw std::runtime_error("video input is not supported - hint: if this is unexpected, you may need to provide the mmproj");
}
json input_video = json_value(p, "input_video", json::object());
std::string data = json_value(input_video, "data", std::string());
auto decoded_data = base64_decode(data); // expected to be base64 encoded
out_files.push_back(decoded_data);
p["type"] = "media_marker";
p["text"] = get_media_marker();
p.erase("input_video");
} else if (type != "text") {
throw std::invalid_argument("unsupported content[].type");
}
tools/server/server-common.h
+1
View File
@@ -294,6 +294,7 @@ struct server_chat_params {
common_chat_templates_ptr tmpls;
bool allow_image;
bool allow_audio;
bool allow_video;
bool enable_thinking = true;
int reasoning_budget = -1;
std::string reasoning_budget_message;
tools/server/server-context.cpp
+45 -28
View File
@@ -27,6 +27,7 @@
#include <memory>
#include <filesystem>
#include <utility>
#include <fstream>
// fix problem with std::min and std::max
#if defined(_WIN32)
@@ -127,7 +128,11 @@ struct server_slot {
server_prompt prompt;
void prompt_save(server_prompt_cache & prompt_cache) const {
bool prompt_save(server_prompt_cache & prompt_cache) const {
if (prompt.tokens.size() == 0) {
return false;
}
GGML_ASSERT(prompt.data.size() == 0);
const size_t cur_size_tgt = llama_state_seq_get_size_ext(ctx_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE);
@@ -140,13 +145,15 @@ struct server_slot {
auto * cur = prompt_cache.alloc(prompt, cur_size_tgt, cur_size_dft);
if (cur == nullptr) {
return;
return false;
}
llama_state_seq_get_data_ext(ctx_tgt, cur->data.main.data(), cur_size_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE);
if (ctx_dft) {
llama_state_seq_get_data_ext(ctx_dft, cur->data.drft.data(), cur_size_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE);
}
return true;
}
bool prompt_load(server_prompt_cache & prompt_cache, const server_tokens & tokens) {
@@ -739,17 +746,6 @@ private:
llama_batch_free(batch);
}
void slot_save_and_clear(server_slot & slot) {
if (slot.prompt.n_tokens() == 0) {
return;
}
SLT_INF(slot, "%s", "saving idle slot to prompt cache\n");
SLT_DBG(slot, "%s", "__TEST_TAG_CACHE_IDLE_SLOT__\n");
slot.prompt_save(*prompt_cache);
slot.prompt_clear(false);
prompt_cache->update();
}
void handle_sleeping_state(bool new_state) {
GGML_ASSERT(sleeping != new_state);
if (new_state) {
@@ -1186,14 +1182,17 @@ private:
metrics.init();
if (params_base.cache_idle_slots) {
if (!params_base.kv_unified) {
SRV_WRN("%s", "--cache-idle-slots requires --kv-unified, disabling\n");
params_base.cache_idle_slots = false;
} else if (params_base.cache_ram_mib == 0) {
if (params_base.cache_ram_mib == 0) {
SRV_WRN("%s", "--cache-idle-slots requires --cache-ram, disabling\n");
params_base.cache_idle_slots = false;
} else {
SRV_INF("%s", "idle slots will be saved to prompt cache and cleared upon starting a new task\n");
if (params_base.kv_unified) {
SRV_INF("%s", "idle slots will be saved to prompt cache and cleared upon starting a new task\n");
} else {
// without a unified KV cache, clearing a slot frees no reusable room, so we only
// publish a RAM-cache copy of idle slots (their KV stays in VRAM) [TAG_IDLE_SLOT_CLEAR]
SRV_INF("%s", "idle slots will be saved to prompt cache upon starting a new task\n");
}
SRV_DBG("%s", "__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__\n");
}
}
@@ -1247,6 +1246,7 @@ private:
/* tmpls */ std::move(chat_templates),
/* allow_image */ mctx ? mtmd_support_vision(mctx) : false,
/* allow_audio */ mctx ? mtmd_support_audio (mctx) : false,
/* allow_video */ mctx ? mtmd_helper_support_video(mctx) : false,
/* enable_thinking */ enable_thinking,
/* reasoning_budget */ params_base.sampling.reasoning_budget_tokens,
/* reasoning_budget_msg */ params_base.sampling.reasoning_budget_message,
@@ -1356,8 +1356,6 @@ private:
}
if (ret) {
const auto & tokens = ret->prompt.tokens;
update_cache = update_cache && prompt_cache;
// cache prompts only for completion tasks
@@ -1368,10 +1366,7 @@ private:
const int64_t t_start = ggml_time_us();
// don't save the slot's state if its context is empty
if (tokens.size() > 0) {
ret->prompt_save(*prompt_cache);
}
ret->prompt_save(*prompt_cache);
if (!ret->prompt_load(*prompt_cache, task.tokens)) {
ret->prompt_clear(false);
@@ -2119,9 +2114,19 @@ private:
}
if (params_base.cache_idle_slots) {
for (auto & s : slots) {
if (!s.is_processing()) {
slot_save_and_clear(s);
for (auto & slot : slots) {
if (!slot.is_processing()) {
SLT_INF(slot, "%s", "saving idle slot to prompt cache\n");
if (slot.prompt_save(*prompt_cache)) {
SLT_DBG(slot, "%s", "__TEST_TAG_CACHE_IDLE_SLOT__\n");
prompt_cache->update();
}
if (params_base.kv_unified) {
// [TAG_IDLE_SLOT_CLEAR]
slot.prompt_clear(false);
}
}
}
}
@@ -3586,6 +3591,7 @@ server_context_meta server_context::get_meta() const {
/* has_mtmd */ impl->mctx != nullptr,
/* has_inp_image */ impl->chat_params.allow_image,
/* has_inp_audio */ impl->chat_params.allow_audio,
/* has_inp_video */ impl->chat_params.allow_video,
/* json_ui_settings */ impl->json_ui_settings,
/* json_webui_settings */ impl->json_webui_settings, // Deprecated
/* slot_n_ctx */ impl->get_slot_n_ctx(),
@@ -3714,6 +3720,16 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
// TODO: this log can become very long, put it behind a flag or think about a more compact format
//SRV_DBG("Prompt: %s\n", prompt.is_string() ? prompt.get<std::string>().c_str() : prompt.dump(2).c_str());
if (!params.path_prompts_log_dir.empty()) {
const auto file_path = std::filesystem::path(params.path_prompts_log_dir) / string_format("%012" PRId64 ".txt", ggml_time_ms());
std::ofstream f(file_path);
if (f) {
f << (prompt.is_string() ? prompt.get<std::string>().c_str() : prompt.dump(2).c_str());
} else {
SRV_ERR("failed to create %s\n", file_path.string().c_str());
}
}
// process prompt
std::vector<server_tokens> inputs;
@@ -4183,6 +4199,7 @@ void server_routes::init_routes() {
{ "model_path", meta->model_path },
{ "modalities", json {
{"vision", meta->has_inp_image},
{"video", meta->has_inp_video},
{"audio", meta->has_inp_audio},
} },
{ "media_marker", get_media_marker() },
@@ -4976,7 +4993,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const l
n_tokens = tokenize_mixed(vocab, prompt, true, true).size();
}
json response = {{"input_tokens", static_cast<int>(n_tokens)}};
json response = {{"input_tokens", static_cast<int64_t>(n_tokens)}};
if (is_oai) {
response["object"] = "response.input_tokens";
}
tools/server/server-context.h
+1
View File
@@ -21,6 +21,7 @@ struct server_context_meta {
bool has_mtmd;
bool has_inp_image;
bool has_inp_audio;
bool has_inp_video;
json json_ui_settings; // Primary: new name
json json_webui_settings; // Deprecated: use json_ui_settings instead (kept for backward compat)
int slot_n_ctx;
tools/server/server-task.cpp
+3
View File
@@ -1393,6 +1393,9 @@ json server_task_result_cmpl_final::to_json_anthropic_stream() {
//
void server_task_result_cmpl_partial::update(task_result_state & state) {
is_updated = true;
if (is_begin) {
return; // begin marker only flushes headers, skip parsing
}
state.update_chat_msg(content, true, oaicompat_msg_diffs);
// Copy current state for use in to_json_*() (reflects state BEFORE this chunk)