Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 01:05:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.devops/intel.Dockerfile
#	.github/workflows/build-cross.yml
#	.github/workflows/build-sycl.yml
#	.github/workflows/build.yml
#	.github/workflows/editorconfig.yml
#	.github/workflows/release.yml
#	cmake/riscv64-spacemit-linux-gnu-gcc.cmake
#	docs/backend/OPENVINO.md
#	docs/backend/SYCL.md
#	docs/build-riscv64-spacemit.md
#	docs/ops.md
#	docs/ops/WebGPU.csv
#	embd_res/ggml-vocab-qwen35.gguf
#	embd_res/ggml-vocab-qwen35.gguf.inp
#	embd_res/ggml-vocab-qwen35.gguf.out
#	examples/model-conversion/Makefile
#	ggml/CMakeLists.txt
#	ggml/src/ggml-cpu/CMakeLists.txt
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp/hmx-flash-attn-ops.c
#	ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c
#	ggml/src/ggml-hexagon/htp/hmx-utils.h
#	ggml/src/ggml-hexagon/htp/htp-ops.h
#	ggml/src/ggml-hexagon/htp/hvx-utils.h
#	ggml/src/ggml-hexagon/htp/main.c
#	ggml/src/ggml-hexagon/htp/unary-ops.c
#	ggml/src/ggml-opencl/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-opencl/kernels/cvt.cl
#	ggml/src/ggml-sycl/CMakeLists.txt
#	ggml/src/ggml-sycl/common.cpp
#	ggml/src/ggml-sycl/common.hpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl
#	ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl
#	ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl
#	ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl
#	ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl
#	ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl
#	ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl
#	ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl
#	ggml/src/ggml-zendnn/CMakeLists.txt
#	ggml/src/ggml-zendnn/ggml-zendnn.cpp
#	scripts/snapdragon/adb/run-completion.sh
#	tests/CMakeLists.txt
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/mtmd/clip-impl.h
#	tools/mtmd/clip.cpp
#	tools/mtmd/clip.h
#	tools/server/README.md
This commit is contained in:
Concedo
2026-05-14 19:04:04 +08:00
parent 6e42e0ebb6 67b2b7f2f2
commit cc82c3164e
62 changed files with 18693 additions and 7034 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip-impl.h
+1 -1
View File
@@ -474,10 +474,10 @@ static void clip_log_internal(enum ggml_log_level level, const char * format, ..
}
#ifndef LOG_INF
#define LOG_DBG(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
#define LOG_INF(...) clip_log_internal(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
#define LOG_WRN(...) clip_log_internal(GGML_LOG_LEVEL_WARN, __VA_ARGS__)
#define LOG_ERR(...) clip_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
#define LOG_DBG(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
#define LOG_CNT(...) clip_log_internal(GGML_LOG_LEVEL_CONT, __VA_ARGS__)
#endif
tools/mtmd/clip.cpp
+10 -2
View File
@@ -1054,7 +1054,7 @@ struct clip_model_loader {
bool has_audio = false;
// TODO @ngxson : we should not pass clip_ctx here, it should be clip_model
clip_model_loader(const char * fname) : fname(fname) {
clip_model_loader(const char * fname, bool skip_tensors = false) : fname(fname) {
struct ggml_context * meta = nullptr;
struct gguf_init_params params = {
@@ -1100,7 +1100,7 @@ struct clip_model_loader {
}
// tensors
{
if (!skip_tensors) {
for (int i = 0; i < n_tensors; ++i) {
const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
const size_t offset = gguf_get_tensor_offset(ctx_gguf.get(), i);
@@ -3020,6 +3020,14 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
return {ctx_vision, ctx_audio};
}
struct clip_cap clip_get_cap(const char * fname) {
clip_cap res;
clip_model_loader loader(fname, /* skip_tensors= */ true);
res.has_vision = loader.has_vision;
res.has_audio = loader.has_audio;
return res;
}
struct clip_image_size * clip_image_size_init() {
struct clip_image_size * load_image_size = new struct clip_image_size();
load_image_size->width = 448;
tools/mtmd/clip.h
+8
View File
@@ -126,4 +126,12 @@ bool clip_has_vision_encoder(const struct clip_ctx * ctx);
bool clip_has_audio_encoder(const struct clip_ctx * ctx);
bool clip_has_whisper_encoder(const struct clip_ctx * ctx);
bool clip_model_quantize(const char * fname_inp, const char * fname_out, const int itype) ;
struct clip_cap {
bool has_vision;
bool has_audio;
};
struct clip_cap clip_get_cap(const char * fname);
tools/mtmd/mtmd.cpp
+13
View File
@@ -1423,6 +1423,19 @@ void mtmd_log_set(ggml_log_callback log_callback, void * user_data) {
g_logger_state.log_callback_user_data = user_data;
}
struct mtmd_caps mtmd_get_cap_from_file(const char * fname) {
try {
auto tmp = clip_get_cap(fname);
mtmd_caps cap;
cap.inp_audio = tmp.has_audio;
cap.inp_vision = tmp.has_vision;
return cap;
} catch (const std::exception & e) {
LOG_ERR("%s: failed to get capabilities from file '%s': %s\n", __func__, fname, e.what());
return mtmd_caps{ false, false };
}
}
//
// Debugging API (NOT intended for public use)
//
tools/mtmd/mtmd.h
+8
View File
@@ -244,6 +244,14 @@ MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx);
// If this is not called, or NULL is supplied, everything is output on stderr.
MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data);
// EXPERIMENTAL API to get mmproj's capabilities without initializing the full context
// This is only intended to be used by llama-server, breaking changes is expected
struct mtmd_caps {
bool inp_vision;
bool inp_audio;
};
MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname);
/////////////////////////////////////////
// test function, to be used in test-mtmd-c-api.c
tools/server/public/bundle.css
+1 -1
View File
File diff suppressed because one or more lines are too long
tools/server/public/bundle.js
+3213 -3207
View File
File diff suppressed because it is too large Load Diff
tools/server/server-common.cpp
+50 -6
View File
@@ -1040,6 +1040,10 @@ json oaicompat_chat_params_parse(
inputs.use_jinja = opt.use_jinja;
inputs.parallel_tool_calls = json_value(body, "parallel_tool_calls", caps["supports_parallel_tool_calls"]);
inputs.add_generation_prompt = json_value(body, "add_generation_prompt", true);
const bool continue_final_message = json_value(body, "continue_final_message", false);
if (continue_final_message && inputs.add_generation_prompt) {
throw std::invalid_argument("Cannot set both add_generation_prompt and continue_final_message to true.");
}
inputs.reasoning_format = opt.reasoning_format;
if (body.contains("reasoning_format")) {
inputs.reasoning_format = common_reasoning_format_from_name(body.at("reasoning_format").get<std::string>());
@@ -1071,7 +1075,10 @@ json oaicompat_chat_params_parse(
// if the assistant message appears at the end of list, we do not add end-of-turn token
// for ex. this can be useful to modify the reasoning process in reasoning models
bool prefill_assistant_message = !inputs.messages.empty() && inputs.messages.back().role == "assistant" && opt.prefill_assistant;
// continue_final_message is the explicit opt in alias from the vLLM/transformers API,
// equivalent to the prefill_assistant heuristic
bool prefill_assistant_message = !inputs.messages.empty() && inputs.messages.back().role == "assistant"
&& (continue_final_message || opt.prefill_assistant);
common_chat_msg last_message;
if (prefill_assistant_message) {
last_message = inputs.messages.back();
@@ -1082,11 +1089,12 @@ json oaicompat_chat_params_parse(
throw std::invalid_argument("Cannot have 2 or more assistant messages at the end of the list.");
}
/* TODO: test this properly */
inputs.reasoning_format = COMMON_REASONING_FORMAT_NONE;
if ( inputs.enable_thinking ) {
throw std::invalid_argument("Assistant response prefill is incompatible with enable_thinking.");
// reject reasoning prefill on channel based templates that do not expose explicit thinking tags
if (!last_message.reasoning_content.empty() && inputs.enable_thinking) {
auto probe_params = common_chat_templates_apply(opt.tmpls.get(), inputs);
if (probe_params.supports_thinking && probe_params.thinking_end_tag.empty()) {
throw std::invalid_argument("Assistant prefill with reasoning_content is not supported yet for this template.");
}
}
inputs.add_generation_prompt = true;
@@ -1098,6 +1106,42 @@ json oaicompat_chat_params_parse(
/* Append assistant prefilled message */
if (prefill_assistant_message) {
const bool thinking_active = chat_params.supports_thinking && !chat_params.thinking_end_tag.empty();
const bool has_reasoning = !last_message.reasoning_content.empty();
const bool has_content = !last_message.content.empty() || !last_message.content_parts.empty();
const bool mid_reasoning = has_reasoning && !has_content;
// some templates inject thinking_start in generation_prompt, others let the model emit it
const bool gp_has_think = thinking_active
&& chat_params.generation_prompt.find(chat_params.thinking_start_tag) != std::string::npos;
// open the thinking block when reasoning is present and the template did not inject it
if (has_reasoning) {
if (thinking_active && !gp_has_think) {
chat_params.prompt += chat_params.thinking_start_tag;
}
chat_params.prompt += last_message.reasoning_content;
}
if (thinking_active) {
if (mid_reasoning) {
// model continues inside the thinking block, keep generation_prompt open on think
if (!gp_has_think) {
chat_params.generation_prompt += chat_params.thinking_start_tag;
}
} else {
// close thinking block when reasoning is followed by content, or when the template forced it open
if (has_reasoning || gp_has_think) {
chat_params.prompt += chat_params.thinking_end_tag;
}
// strip thinking_start from generation_prompt so the parser routes model output as content
auto pos = chat_params.generation_prompt.rfind(chat_params.thinking_start_tag);
if (pos != std::string::npos) {
chat_params.generation_prompt = chat_params.generation_prompt.substr(0, pos);
}
}
}
if (!last_message.content_parts.empty()) {
for (auto & p : last_message.content_parts) {
chat_params.prompt += p.text;
tools/server/server-common.h
+6 -4
View File
@@ -15,17 +15,19 @@
using json = nlohmann::ordered_json;
#define SLT_DBG(slot, fmt, ...) LOG_DBG("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)
#define SLT_TRC(slot, fmt, ...) LOG_TRC("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)
#define SLT_INF(slot, fmt, ...) LOG_INF("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)
#define SLT_CNT(slot, fmt, ...) LOG_CNT("" fmt, __VA_ARGS__)
#define SLT_WRN(slot, fmt, ...) LOG_WRN("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)
#define SLT_ERR(slot, fmt, ...) LOG_ERR("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)
#define SLT_DBG(slot, fmt, ...) LOG_DBG("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)
#define SLT_CNT(slot, fmt, ...) LOG_CNT("" fmt, __VA_ARGS__)
#define SRV_DBG(fmt, ...) LOG_DBG("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)
#define SRV_TRC(fmt, ...) LOG_TRC("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)
#define SRV_INF(fmt, ...) LOG_INF("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)
#define SRV_CNT(fmt, ...) LOG_CNT("" fmt, __VA_ARGS__)
#define SRV_WRN(fmt, ...) LOG_WRN("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)
#define SRV_ERR(fmt, ...) LOG_ERR("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)
#define SRV_DBG(fmt, ...) LOG_DBG("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)
#define SRV_CNT(fmt, ...) LOG_CNT("" fmt, __VA_ARGS__)
using raw_buffer = std::vector<uint8_t>;
tools/server/server-context.cpp
+62 -25
View File
@@ -166,6 +166,7 @@ struct server_slot {
// stats
size_t n_sent_text = 0; // number of sent text character
int64_t t_print_last = 0;
int64_t t_start_process_prompt;
int64_t t_start_generation;
@@ -233,7 +234,7 @@ struct server_slot {
}
}
SLT_INF(*this, "init sampler, took %0.2f ms, tokens: text = %d, total = %d\n",
SLT_TRC(*this, "init sampler, took %0.2f ms, tokens: text = %d, total = %d\n",
(ggml_time_us() - t_start) / 1000.0, n_text, (int) prompt.tokens.size());
}
@@ -417,6 +418,36 @@ struct server_slot {
return stop_pos;
}
void print_timings_tg() {
if (n_decoded < 100) {
return;
}
const int64_t t_now = ggml_time_us();
if (t_now - t_print_last < 3*1000*1000) {
return;
}
t_print_last = t_now;
const double n_gen_second = 1e3 / t_token_generation * n_decoded;
SLT_INF(*this, "n_decoded = %6d, tg = %6.2f t/s\n", n_decoded, n_gen_second);
}
void print_timings_pp() const {
const double n_prompt_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed;
const double f_progress = (float) prompt.n_tokens() / task->n_tokens();
if (t_prompt_processing < 3000.0) {
return;
}
SLT_INF(*this, "prompt processing, n_tokens = %6d, progress = %.2f, t = %6.2f s / %.2f tokens per second\n",
n_prompt_tokens_processed, f_progress, t_prompt_processing / 1e3, n_prompt_second);
}
void print_timings() const {
const double t_prompt = t_prompt_processing / n_prompt_tokens_processed;
const double n_prompt_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed;
@@ -588,6 +619,10 @@ public:
// note: chat_params must not be refreshed upon existing sleeping state
server_chat_params chat_params;
server_context_impl() {
mtmd_helper_log_set(common_log_default_callback, nullptr);
}
~server_context_impl() {
if (!sleeping) {
// destroy() is already called when entering sleeping state
@@ -749,10 +784,6 @@ private:
std::string & mmproj_path = params_base.mmproj.path;
if (!mmproj_path.empty()) {
if (!is_resume) {
mtmd_helper_log_set(common_log_default_callback, nullptr);
}
mtmd_context_params mparams = mtmd_context_params_default();
mparams.use_gpu = params_base.mmproj_use_gpu;
@@ -896,17 +927,17 @@ private:
if (params_base.cache_ram_mib != 0) {
if (params_base.cache_ram_mib < 0) {
SRV_WRN("prompt cache is enabled, size limit: %s\n", "no limit");
SRV_INF("prompt cache is enabled, size limit: %s\n", "no limit");
} else {
SRV_WRN("prompt cache is enabled, size limit: %d MiB\n", params_base.cache_ram_mib);
SRV_INF("prompt cache is enabled, size limit: %d MiB\n", params_base.cache_ram_mib);
}
SRV_WRN("%s", "use `--cache-ram 0` to disable the prompt cache\n");
SRV_INF("%s", "use `--cache-ram 0` to disable the prompt cache\n");
prompt_cache = std::make_unique<server_prompt_cache>(params_base.cache_ram_mib, n_ctx);
} else {
SRV_WRN("%s", "prompt cache is disabled - use `--cache-ram N` to enable it\n");
SRV_INF("%s", "prompt cache is disabled - use `--cache-ram N` to enable it\n");
}
SRV_WRN("%s", "for more info see https://github.com/ggml-org/llama.cpp/pull/16391\n");
SRV_INF("%s", "for more info see https://github.com/ggml-org/llama.cpp/pull/16391\n");
if (!params_base.model_alias.empty()) {
// backward compat: use first alias as model name
@@ -954,13 +985,13 @@ private:
if (params_base.cache_idle_slots) {
if (!params_base.kv_unified) {
SRV_WRN("%s: --cache-idle-slots requires --kv-unified, disabling\n", __func__);
SRV_WRN("%s", "--cache-idle-slots requires --kv-unified, disabling\n");
params_base.cache_idle_slots = false;
} else if (params_base.cache_ram_mib == 0) {
SRV_WRN("%s: --cache-idle-slots requires --cache-ram, disabling\n", __func__);
SRV_WRN("%s", "--cache-idle-slots requires --cache-ram, disabling\n");
params_base.cache_idle_slots = false;
} else {
SRV_INF("%s: idle slots will be saved to prompt cache and cleared upon starting a new task\n", __func__);
SRV_INF("%s", "idle slots will be saved to prompt cache and cleared upon starting a new task\n");
SRV_DBG("%s", "__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__\n");
}
}
@@ -1112,7 +1143,7 @@ private:
update_cache = update_cache && task.type == SERVER_TASK_TYPE_COMPLETION;
if (update_cache) {
SRV_WRN("%s", "updating prompt cache\n");
SRV_INF("%s", "updating prompt cache\n");
const int64_t t_start = ggml_time_us();
@@ -1127,7 +1158,7 @@ private:
prompt_cache->update();
SRV_WRN("prompt cache update took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);
SRV_INF("prompt cache update took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);
}
}
@@ -1186,10 +1217,10 @@ private:
if (!are_lora_equal(task_loras, slot.lora)) {
// if lora has changed, check to see if the cache should be cleared
if (lora_should_clear_cache(slot.lora, task_loras)) {
SLT_INF(slot, "clearing cache for lora change. %zu loras -> %zu loras\n", slot.lora.size(), task.params.lora.size());
SLT_TRC(slot, "clearing cache for lora change. %zu loras -> %zu loras\n", slot.lora.size(), task.params.lora.size());
slot.prompt.tokens.clear();
} else {
SLT_INF(slot, "keeping cache for alora. %zu target loras\n", task_loras.size());
SLT_TRC(slot, "keeping cache for alora. %zu target loras\n", task_loras.size());
}
slot.lora = task_loras;
}
@@ -1281,7 +1312,8 @@ private:
llama_set_sampler(ctx_tgt, slot.id, nullptr);
}
SLT_INF(slot, "sampler chain: %s\n", common_sampler_print(slot.smpl.get()).c_str());
SLT_TRC(slot, "sampler chain: %s\n", common_sampler_print(slot.smpl.get()).c_str());
SLT_TRC(slot, "sampler params: \n%s\n", task.params.sampling.print().c_str());
} else {
slot.smpl.reset();
}
@@ -1800,7 +1832,7 @@ private:
cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
cur.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
SLT_WRN(slot,
SLT_INF(slot,
"created context checkpoint %d of %d (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n",
(int) slot.prompt.checkpoints.size(), params_base.n_ctx_checkpoints, cur.pos_min,
cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024);
@@ -2339,7 +2371,7 @@ private:
slot.state = SLOT_STATE_PROCESSING_PROMPT;
SLT_INF(slot, "new prompt, n_ctx_slot = %d, n_keep = %d, task.n_tokens = %d\n",
SLT_TRC(slot, "new prompt, n_ctx_slot = %d, n_keep = %d, task.n_tokens = %d\n",
slot.n_ctx, slot.task->params.n_keep, slot.task->n_tokens());
// print prompt tokens (for debugging)
@@ -2454,7 +2486,7 @@ private:
}
if (n_match >= (size_t) n_cache_reuse) {
SLT_INF(slot, "reusing chunk with size %zu, shifting KV cache [%zu, %zu) -> [%zu, %zu)\n", n_match, head_c, head_c + n_match, head_p, head_p + n_match);
SLT_TRC(slot, "reusing chunk with size %zu, shifting KV cache [%zu, %zu) -> [%zu, %zu)\n", n_match, head_c, head_c + n_match, head_p, head_p + n_match);
//for (size_t i = head_p; i < head_p + n_match; i++) {
// SLT_DBG(slot, "cache token %3zu: %6d '%s'\n", i, prompt_tokens[i], common_token_to_piece(ctx_tgt, prompt_tokens[i]).c_str());
//}
@@ -2620,10 +2652,14 @@ private:
}
}
const int64_t t_current = ggml_time_us();
slot.t_prompt_processing = (t_current - slot.t_start_process_prompt) / 1e3;
slot.print_timings_pp();
// truncate any tokens that are beyond n_past for this slot
const llama_pos p0 = slot.prompt.tokens.pos_next();
SLT_INF(slot, "n_tokens = %d, memory_seq_rm [%d, end)\n", slot.prompt.n_tokens(), p0);
SLT_TRC(slot, "cached n_tokens = %d, memory_seq_rm [%d, end)\n", slot.prompt.n_tokens(), p0);
if (!llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot.id, p0, -1)) {
SLT_WRN(slot, "failed to truncate tokens with position >= %d - clearing the memory\n", p0);
@@ -2764,7 +2800,6 @@ private:
slot.i_batch = batch.n_tokens - 1;
slot.init_sampler();
SLT_INF(slot, "prompt processing done, n_tokens = %d, batch.n_tokens = %d\n", slot.prompt.n_tokens(), batch.n_tokens);
} else {
if (slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch) {
// near the end of the prompt
@@ -2786,8 +2821,6 @@ private:
}
}
}
SLT_INF(slot, "prompt processing progress, n_tokens = %d, batch.n_tokens = %d, progress = %f\n", slot.prompt.n_tokens(), batch.n_tokens, (float) slot.prompt.n_tokens() / slot.task->n_tokens());
}
const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id);
@@ -3084,6 +3117,8 @@ private:
continue;
}
slot.print_timings_tg();
}
// speculative decoding - main model sample and accept
@@ -3196,6 +3231,8 @@ private:
}
}
slot.print_timings_tg();
SLT_DBG(slot, "accepted %d/%d draft tokens, new n_tokens = %d\n", (int) ids.size() - 1, (int) n_draft, slot.prompt.n_tokens());
}
}
tools/server/server-http.cpp
+19 -19
View File
@@ -47,7 +47,7 @@ static void log_server_request(const httplib::Request & req, const httplib::Resp
// reminder: this function is not covered by httplib's exception handler; if someone does more complicated stuff, think about wrapping it in try-catch
SRV_INF("done request: %s %s %s %d\n", req.method.c_str(), req.path.c_str(), req.remote_addr.c_str(), res.status);
SRV_TRC("done request: %s %s %s %d\n", req.method.c_str(), req.path.c_str(), req.remote_addr.c_str(), res.status);
SRV_DBG("request: %s\n", req.body.c_str());
SRV_DBG("response: %s\n", res.body.c_str());
@@ -89,10 +89,10 @@ bool server_http_context::init(const common_params & params) {
hostname = params.hostname;
if (gcp.enabled) {
LOG_INF("%s: Google Cloud Platform compat: health route = %s, predict route = %s, port = %d\n", __func__, gcp.path_health.c_str(), gcp.path_predict.c_str(), gcp.port);
SRV_INF("Google Cloud Platform compat: health route = %s, predict route = %s, port = %d\n", gcp.path_health.c_str(), gcp.path_predict.c_str(), gcp.port);
if (port != gcp.port) {
LOG_WRN("%s: Google Cloud Platform compat: overriding server port %d with AIP_HTTP_PORT %d\n", __func__, port, gcp.port);
SRV_WRN("Google Cloud Platform compat: overriding server port %d with AIP_HTTP_PORT %d\n", port, gcp.port);
}
port = gcp.port;
@@ -102,17 +102,17 @@ bool server_http_context::init(const common_params & params) {
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
if (params.ssl_file_key != "" && params.ssl_file_cert != "") {
LOG_INF("Running with SSL: key = %s, cert = %s\n", params.ssl_file_key.c_str(), params.ssl_file_cert.c_str());
SRV_INF("running with SSL: key = %s, cert = %s\n", params.ssl_file_key.c_str(), params.ssl_file_cert.c_str());
srv.reset(
new httplib::SSLServer(params.ssl_file_cert.c_str(), params.ssl_file_key.c_str())
);
} else {
LOG_INF("Running without SSL\n");
SRV_INF("%s", "running without SSL\n");
srv.reset(new httplib::Server());
}
#else
if (params.ssl_file_key != "" && params.ssl_file_cert != "") {
LOG_ERR("Server is built without SSL support\n");
SRV_ERR("%s", "the server is built without SSL support\n");
return false;
}
srv.reset(new httplib::Server());
@@ -134,7 +134,7 @@ bool server_http_context::init(const common_params & params) {
res.status = 500;
res.set_content(message, "text/plain");
LOG_ERR("got exception: %s\n", message.c_str());
SRV_ERR("got exception: %s\n", message.c_str());
});
srv->set_error_handler([](const httplib::Request &, httplib::Response & res) {
@@ -162,7 +162,7 @@ bool server_http_context::init(const common_params & params) {
#ifdef SO_REUSEPORT
httplib::set_socket_opt(sock, SOL_SOCKET, SO_REUSEPORT, 1);
#else
LOG_WRN("%s: SO_REUSEPORT is not supported\n", __func__);
SRV_WRN("%s", "SO_REUSEPORT is not supported\n");
#endif
}
});
@@ -170,9 +170,9 @@ bool server_http_context::init(const common_params & params) {
if (params.api_keys.size() == 1) {
auto key = params.api_keys[0];
std::string substr = key.substr(std::max((int)(key.length() - 4), 0));
LOG_INF("%s: api_keys: ****%s\n", __func__, substr.c_str());
SRV_INF("api_keys: ****%s\n", substr.c_str());
} else if (params.api_keys.size() > 1) {
LOG_INF("%s: api_keys: %zu keys loaded\n", __func__, params.api_keys.size());
SRV_INF("api_keys: %zu keys loaded\n", params.api_keys.size());
}
//
@@ -232,7 +232,7 @@ bool server_http_context::init(const common_params & params) {
"application/json; charset=utf-8"
);
LOG_WRN("Unauthorized: Invalid API Key\n");
SRV_WRN("%s", "unauthorized: Invalid API Key\n");
return false;
};
@@ -292,7 +292,7 @@ bool server_http_context::init(const common_params & params) {
// +4 threads for monitoring, health and some threads reserved for MCP and other tasks in the future
n_threads_http = std::max(params.n_parallel + 4, (int32_t) std::thread::hardware_concurrency() - 1);
}
LOG_INF("%s: using %d threads for HTTP server\n", __func__, n_threads_http);
SRV_INF("using %d threads for HTTP server\n", n_threads_http);
srv->new_task_queue = [n_threads_http] {
// spawn n_threads_http fixed thread (always alive), while allow up to 1024 max possible additional threads
// when n_threads_http is used, server will create new "dynamic" threads that will be destroyed after processing each request
@@ -306,14 +306,14 @@ bool server_http_context::init(const common_params & params) {
//
if (!params.webui) {
LOG_INF("Web UI is disabled\n");
SRV_INF("%s", "the WebUI is disabled\n");
} else {
// register static assets routes
if (!params.public_path.empty()) {
// Set the base directory for serving static files
bool is_found = srv->set_mount_point(params.api_prefix + "/", params.public_path);
if (!is_found) {
LOG_ERR("%s: static assets path not found: %s\n", __func__, params.public_path.c_str());
SRV_ERR("static assets path not found: %s\n", params.public_path.c_str());
return 1;
}
} else {
@@ -348,13 +348,13 @@ bool server_http_context::start() {
bool is_sock = false;
if (string_ends_with(std::string(hostname), ".sock")) {
is_sock = true;
LOG_INF("%s: setting address family to AF_UNIX\n", __func__);
SRV_INF("%s", "setting address family to AF_UNIX\n");
srv->set_address_family(AF_UNIX);
// bind_to_port requires a second arg, any value other than 0 should
// simply get ignored
was_bound = srv->bind_to_port(hostname, 8080);
} else {
LOG_INF("%s: binding port with default address family\n", __func__);
SRV_INF("%s", "binding port with default address family\n");
// bind HTTP listen port
if (port == 0) {
int bound_port = srv->bind_to_any_port(hostname);
@@ -368,7 +368,7 @@ bool server_http_context::start() {
}
if (!was_bound) {
LOG_ERR("%s: couldn't bind HTTP server socket, hostname: %s, port: %d\n", __func__, hostname.c_str(), port);
SRV_ERR("couldn't bind HTTP server socket, hostname: %s, port: %d\n", hostname.c_str(), port);
return false;
}
@@ -580,7 +580,7 @@ void server_http_context::register_gcp_compat() {
}
if (handlers.count(gcp.path_predict)) {
LOG_ERR("%s: AIP_PREDICT_ROUTE=%s conflicts with an existing llama-server route\n", __func__, gcp.path_predict.c_str());
SRV_ERR("AIP_PREDICT_ROUTE=%s conflicts with an existing llama-server route\n", gcp.path_predict.c_str());
exit(1);
}
@@ -651,7 +651,7 @@ void server_http_context::register_gcp_compat() {
payload.erase("@requestFormat");
if (payload.contains("stream")) {
LOG_WRN("%s: ignoring client-provided stream field in instance, streaming is not supported in predict route\n", __func__);
SRV_WRN("%s", "ignoring client-provided stream field in instance, streaming is not supported in predict route\n");
payload["stream"] = false;
}
tools/server/server-models.cpp
+51 -7
View File
@@ -161,6 +161,30 @@ void server_model_meta::update_args(common_preset_context & ctx_preset, std::str
args = preset.to_args(bin_path);
}
void server_model_meta::update_caps() {
try {
common_params params;
preset.apply_to_params(params, {
"LLAMA_ARG_MODEL",
"LLAMA_ARG_MODEL_URL",
"LLAMA_ARG_MMPROJ",
"LLAMA_ARG_MMPROJ_URL",
"LLAMA_ARG_HF_REPO",
"LLAMA_ARG_HF_REPO_FILE",
});
params.offline = true; // avoid any unwanted network call during capability detection
common_params_handle_models(params, LLAMA_EXAMPLE_SERVER);
if (params.mmproj.path.empty()) {
multimodal = { false, false };
} else {
multimodal = mtmd_get_cap_from_file(params.mmproj.path.c_str());
}
} catch (const std::exception & e) {
LOG_WRN("failed to initialize common_params for multimodal capability detection: %s\n", e.what());
multimodal = { false, false };
}
}
//
// server_models
//
@@ -236,6 +260,7 @@ void server_models::add_model(server_model_meta && meta) {
}
meta.update_args(ctx_preset, bin_path); // render args
meta.update_caps();
std::string name = meta.name;
mapping[name] = instance_t{
/* subproc */ std::make_shared<subprocess_s>(),
@@ -346,8 +371,10 @@ void server_models::load_models() {
/* status */ SERVER_MODEL_STATUS_UNLOADED,
/* last_used */ 0,
/* args */ std::vector<std::string>(),
/* loaded_info */ {},
/* exit_code */ 0,
/* stop_timeout */ DEFAULT_STOP_TIMEOUT,
/* multimodal */ mtmd_caps{false, false},
};
add_model(std::move(meta));
}
@@ -481,6 +508,7 @@ void server_models::load_models() {
inst.meta.exit_code = 0; // clear failed state so the model can be reloaded
inst.meta.update_args(ctx_preset, bin_path);
inst.meta.update_caps();
}
// add models that are new in this reload
@@ -496,8 +524,10 @@ void server_models::load_models() {
/* status */ SERVER_MODEL_STATUS_UNLOADED,
/* last_used */ 0,
/* args */ std::vector<std::string>(),
/* loaded_info */ {},
/* exit_code */ 0,
/* stop_timeout */ DEFAULT_STOP_TIMEOUT,
/* multimodal */ mtmd_caps{false, false},
};
add_model(std::move(meta));
newly_added.push_back(name);
@@ -1206,14 +1236,28 @@ void server_models_routes::init_routes() {
status["failed"] = true;
}
// pi coding agent multimodal compatibility
json input_modalities = json::array({"text"});
if (meta.multimodal.inp_vision) {
input_modalities.push_back("image");
}
if (meta.multimodal.inp_audio) {
input_modalities.push_back("audio");
}
json architecture {
{"input_modalities", input_modalities},
{"output_modalities", json::array({"text"})},
};
json model_info = json {
{"id", meta.name},
{"aliases", meta.aliases},
{"tags", meta.tags},
{"object", "model"}, // for OAI-compat
{"owned_by", "llamacpp"}, // for OAI-compat
{"created", t}, // for OAI-compat
{"status", status},
{"id", meta.name},
{"aliases", meta.aliases},
{"tags", meta.tags},
{"object", "model"}, // for OAI-compat
{"owned_by", "llamacpp"}, // for OAI-compat
{"created", t}, // for OAI-compat
{"status", status},
{"architecture", architecture},
// TODO: add other fields, may require reading GGUF metadata
};
tools/server/server-models.h
+2
View File
@@ -66,6 +66,7 @@ struct server_model_meta {
json loaded_info; // info to be reflected via /v1/models endpoint
int exit_code = 0; // exit code of the model instance process (only valid if status == FAILED)
int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdown
mtmd_caps multimodal; // multimodal capabilities
bool is_ready() const {
return status == SERVER_MODEL_STATUS_LOADED;
@@ -80,6 +81,7 @@ struct server_model_meta {
}
void update_args(common_preset_context & ctx_presets, std::string bin_path);
void update_caps();
};
struct subprocess_s;
tools/server/server-task.cpp
+6 -6
View File
@@ -1988,7 +1988,7 @@ server_prompt * server_prompt_cache::alloc(const server_prompt & prompt, size_t
const int cur_lcp_len = it->tokens.get_common_prefix(prompt.tokens);
if (cur_lcp_len == (int) prompt.tokens.size()) {
SRV_WRN("%s", " - prompt is already in the cache, skipping\n");
SRV_INF("%s", " - prompt is already in the cache, skipping\n");
return nullptr;
}
}
@@ -2043,7 +2043,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
float sim_best = float(lcp_best) / tokens_new.size();
SRV_WRN(" - looking for better prompt, base f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);
SRV_INF(" - looking for better prompt, base f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);
auto it_best = states.end();
@@ -2068,7 +2068,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
}
if (it_best != states.end()) {
SRV_WRN(" - found better prompt with f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);
SRV_INF(" - found better prompt with f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);
{
auto & data = it_best->data.main;
@@ -2076,7 +2076,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
const size_t size = data.size();
const size_t n = llama_state_seq_set_data_ext(ctx_tgt, data.data(), size, id_slot, 0);
if (n != size) {
SRV_WRN("failed to restore state with size %zu\n", size);
SRV_ERR("failed to restore state with size %zu\n", size);
return false;
}
@@ -2145,11 +2145,11 @@ void server_prompt_cache::update() {
}
}
SRV_WRN(" - cache state: %zu prompts, %.3f MiB (limits: %.3f MiB, %zu tokens, %zu est)\n",
SRV_INF(" - cache state: %zu prompts, %.3f MiB (limits: %.3f MiB, %zu tokens, %zu est)\n",
states.size(), size() / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0), limit_tokens, limit_tokens_cur);
for (const auto & state : states) {
SRV_WRN(" - prompt %p: %7d tokens, checkpoints: %2zu, %9.3f MiB\n",
SRV_INF(" - prompt %p: %7d tokens, checkpoints: %2zu, %9.3f MiB\n",
(const void *)&state, state.n_tokens(), state.checkpoints.size(), state.size() / (1024.0 * 1024.0));
}
}
tools/server/server.cpp
+21 -23
View File
@@ -83,17 +83,22 @@ int main(int argc, char ** argv) {
return 1;
}
llama_backend_init();
llama_numa_init(params.numa);
common_params_print_info(params);
// validate batch size for embeddings
// embeddings require all tokens to be processed in a single ubatch
// see https://github.com/ggml-org/llama.cpp/issues/12836
if (params.embedding && params.n_batch > params.n_ubatch) {
LOG_WRN("%s: embeddings enabled with n_batch (%d) > n_ubatch (%d)\n", __func__, params.n_batch, params.n_ubatch);
LOG_WRN("%s: setting n_batch = n_ubatch = %d to avoid assertion failure\n", __func__, params.n_ubatch);
SRV_WRN("embeddings enabled with n_batch (%d) > n_ubatch (%d)\n", params.n_batch, params.n_ubatch);
SRV_WRN("setting n_batch = n_ubatch = %d to avoid assertion failure\n", params.n_ubatch);
params.n_batch = params.n_ubatch;
}
if (params.n_parallel < 0) {
LOG_INF("%s: n_parallel is set to auto, using n_parallel = 4 and kv_unified = true\n", __func__);
SRV_INF("%s", "n_parallel is set to auto, using n_parallel = 4 and kv_unified = true\n");
params.n_parallel = 4;
params.kv_unified = true;
@@ -107,15 +112,9 @@ int main(int argc, char ** argv) {
// struct that contains llama context and inference
server_context ctx_server;
llama_backend_init();
llama_numa_init(params.numa);
LOG_INF("build_info: %s\n", llama_build_info());
LOG_INF("%s\n", common_params_get_system_info(params).c_str());
server_http_context ctx_http;
if (!ctx_http.init(params)) {
LOG_ERR("%s: failed to initialize HTTP server\n", __func__);
SRV_ERR("%s", "failed to initialize HTTP server\n");
return 1;
}
@@ -134,7 +133,7 @@ int main(int argc, char ** argv) {
try {
models_routes.emplace(params, argc, argv);
} catch (const std::exception & e) {
LOG_ERR("%s: failed to initialize router models: %s\n", __func__, e.what());
SRV_ERR("failed to initialize router models: %s\n", e.what());
return 1;
}
@@ -222,7 +221,7 @@ int main(int argc, char ** argv) {
try {
tools.setup(params.server_tools);
} catch (const std::exception & e) {
LOG_ERR("%s: tools setup failed: %s\n", __func__, e.what());
SRV_ERR("tools setup failed: %s\n", e.what());
return 1;
}
SRV_WRN("%s", "-----------------\n");
@@ -240,7 +239,7 @@ int main(int argc, char ** argv) {
std::function<void()> clean_up;
if (is_router_server) {
LOG_INF("%s: starting router server, no model will be loaded in this process\n", __func__);
SRV_INF("%s", "starting router server, no model will be loaded in this process\n");
clean_up = [&models_routes]() {
SRV_INF("%s: cleaning up before exit...\n", __func__);
@@ -252,7 +251,7 @@ int main(int argc, char ** argv) {
if (!ctx_http.start()) {
clean_up();
LOG_ERR("%s: exiting due to HTTP server error\n", __func__);
SRV_ERR("%s", "exiting due to HTTP server error\n");
return 1;
}
ctx_http.is_ready.store(true);
@@ -273,12 +272,12 @@ int main(int argc, char ** argv) {
// start the HTTP server before loading the model to be able to serve /health requests
if (!ctx_http.start()) {
clean_up();
LOG_ERR("%s: exiting due to HTTP server error\n", __func__);
SRV_ERR("%s", "exiting due to HTTP server error\n");
return 1;
}
// load the model
LOG_INF("%s: loading model\n", __func__);
SRV_INF("%s", "loading model\n");
if (server_models::is_child_server()) {
ctx_server.on_sleeping_changed([&](bool sleeping) {
@@ -291,14 +290,14 @@ int main(int argc, char ** argv) {
if (ctx_http.thread.joinable()) {
ctx_http.thread.join();
}
LOG_ERR("%s: exiting due to model loading error\n", __func__);
SRV_ERR("%s", "exiting due to model loading error\n");
return 1;
}
routes.update_meta(ctx_server);
ctx_http.is_ready.store(true);
LOG_INF("%s: model loaded\n", __func__);
SRV_INF("%s", "model loaded\n");
shutdown_handler = [&](int) {
// this will unblock start_loop()
@@ -322,9 +321,9 @@ int main(int argc, char ** argv) {
#endif
if (is_router_server) {
LOG_INF("%s: router server is listening on %s\n", __func__, ctx_http.listening_address.c_str());
LOG_INF("%s: NOTE: router mode is experimental\n", __func__);
LOG_INF("%s: it is not recommended to use this mode in untrusted environments\n", __func__);
SRV_INF("router server is listening on %s\n", ctx_http.listening_address.c_str());
SRV_WRN("%s", "NOTE: router mode is experimental\n");
SRV_WRN("%s", " it is not recommended to use this mode in untrusted environments\n");
if (ctx_http.thread.joinable()) {
ctx_http.thread.join(); // keep the main thread alive
}
@@ -332,8 +331,7 @@ int main(int argc, char ** argv) {
// when the HTTP server stops, clean up and exit
clean_up();
} else {
LOG_INF("%s: server is listening on %s\n", __func__, ctx_http.listening_address.c_str());
LOG_INF("%s: starting the main loop...\n", __func__);
SRV_INF("server is listening on %s\n", ctx_http.listening_address.c_str());
// optionally, notify router server that this instance is ready
std::thread monitor_thread;
tools/server/tests/unit/test_chat_completion.py
+39
View File
@@ -178,6 +178,45 @@ def test_chat_template_assistant_prefill(prefill, re_prefill):
assert res.body["__verbose"]["prompt"] == f"<s> <|start_header_id|>system<|end_header_id|>\n\nBook<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nWhat is the best book<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{re_prefill}"
def test_chat_template_continue_final_message_vllm_compat():
"""continue_final_message is the vLLM/transformers explicit alias for the prefill_assistant heuristic.
Both must produce the same prompt."""
global server
server.chat_template = "llama3"
server.debug = True
server.start()
res = server.make_request("POST", "/chat/completions", data={
"max_tokens": 8,
"add_generation_prompt": False,
"continue_final_message": True,
"messages": [
{"role": "system", "content": "Book"},
{"role": "user", "content": "What is the best book"},
{"role": "assistant", "content": "Whill"},
]
})
assert res.status_code == 200
assert "__verbose" in res.body
assert res.body["__verbose"]["prompt"] == "<s> <|start_header_id|>system<|end_header_id|>\n\nBook<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nWhat is the best book<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nWhill"
def test_chat_template_continue_final_message_mutual_exclusion():
"""add_generation_prompt and continue_final_message both set to true must be rejected"""
global server
server.chat_template = "llama3"
server.start()
res = server.make_request("POST", "/chat/completions", data={
"max_tokens": 8,
"add_generation_prompt": True,
"continue_final_message": True,
"messages": [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hello"},
]
})
assert res.status_code == 400
def test_apply_chat_template():
global server
server.chat_template = "command-r"
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte
+1 -1
View File
@@ -179,7 +179,7 @@
isEditing = false;
// If canceling a new system message with placeholder content, remove it without deleting children
if (message.role === MessageRole.SYSTEM) {
if (message.role === MessageRole.SYSTEM && message.content === SYSTEM_MESSAGE_PLACEHOLDER) {
const conversationDeleted = await chatStore.removeSystemPromptPlaceholder(message.id);
if (conversationDeleted) {
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte
+1 -2
View File
@@ -74,7 +74,6 @@
const editCtx = getMessageEditContext();
const isAgentic = $derived(hasAgenticContent(message, toolMessages));
const hasReasoning = $derived(!!message.reasoningContent);
const processingState = useProcessingState();
let currentConfig = $derived(config());
@@ -329,7 +328,7 @@
{onCopy}
{onEdit}
{onRegenerate}
onContinue={currentConfig.enableContinueGeneration && !hasReasoning ? onContinue : undefined}
onContinue={currentConfig.enableContinueGeneration ? onContinue : undefined}
{onForkConversation}
{onDelete}
{onConfirmDelete}
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte
+85 -39
View File
@@ -1,4 +1,6 @@
<script lang="ts">
import { onMount } from 'svelte';
import { beforeNavigate, afterNavigate } from '$app/navigation';
import { ChatMessage, ChatMessageUserPending } from '$lib/components/app';
import { setChatActionsContext } from '$lib/contexts';
import { MessageRole } from '$lib/enums';
@@ -27,11 +29,14 @@
interface Props {
messages?: DatabaseMessage[];
onUserAction?: () => void;
onMessagesReady?: (messageCount: number) => void;
}
let { messages = [], onUserAction }: Props = $props();
let { messages = [], onUserAction, onMessagesReady }: Props = $props();
let allConversationMessages = $state<DatabaseMessage[]>([]);
let isVisible = $state(false);
let previousConversationId = $state<string | null>(null);
const currentConfig = config();
@@ -117,15 +122,51 @@
}
}
// Single effect that tracks both conversation and message changes
// Track conversation changes to trigger transition even on same route
$effect(() => {
const conversation = activeConversation();
const currentId = conversation?.id ?? null;
if (conversation) {
refreshAllMessages();
if (currentId !== previousConversationId && previousConversationId !== null) {
// Conversation changed - trigger fade out/in
isVisible = false;
requestAnimationFrame(() => {
refreshAllMessages();
previousConversationId = currentId;
requestAnimationFrame(() => {
isVisible = true;
});
});
} else {
previousConversationId = currentId;
if (conversation) {
refreshAllMessages();
}
}
});
$effect(() => {
void allConversationMessages;
onMessagesReady?.(displayMessages.length);
});
onMount(() => {
requestAnimationFrame(() => {
isVisible = true;
});
});
beforeNavigate(() => {
isVisible = false;
});
afterNavigate(() => {
requestAnimationFrame(() => {
isVisible = true;
});
});
let displayMessages = $derived.by(() => {
if (!messages.length) {
return [];
@@ -207,42 +248,47 @@
});
</script>
{#each displayMessages as { message, toolMessages, isLastAssistantMessage, siblingInfo } (message.id)}
<ChatMessage
class="mx-auto mt-12 w-full max-w-[48rem]"
{message}
{toolMessages}
{isLastAssistantMessage}
{siblingInfo}
/>
{/each}
{#if activeConversation() && agenticPendingSteeringMessageContent(activeConversation()!.id)}
{@const convId = activeConversation()!.id}
{@const pendingContent = agenticPendingSteeringMessageContent(convId)}
{#if pendingContent}
<ChatMessageUserPending
<div
class="transition-opacity delay-300 duration-500 ease-out
{isVisible ? 'opacity-100' : 'opacity-0'}"
>
{#each displayMessages as { message, toolMessages, isLastAssistantMessage, siblingInfo } (message.id)}
<ChatMessage
class="mx-auto mt-12 w-full max-w-[48rem]"
content={pendingContent}
extras={agenticPendingSteeringMessageExtras(convId)}
onSendImmediately={() => chatStore.abortCurrentFlow(convId)}
onEdit={(newContent, extras) => agenticInjectSteeringMessage(convId, newContent, extras)}
onDelete={() => agenticClearSteeringMessage(convId)}
{message}
{toolMessages}
{isLastAssistantMessage}
{siblingInfo}
/>
{/if}
{:else if activeConversation() && chatPendingMessageContent(activeConversation()!.id)}
{@const convId = activeConversation()!.id}
{@const pendingContent = chatPendingMessageContent(convId)}
{/each}
{#if pendingContent}
<ChatMessageUserPending
class="mx-auto mt-12 w-full max-w-[48rem]"
content={pendingContent}
extras={chatPendingMessageExtras(convId)}
onSendImmediately={() => chatStore.abortCurrentFlow(convId)}
onEdit={(newContent, extras) => chatInjectPendingMessage(convId, newContent, extras)}
onDelete={() => chatClearPendingMessage(convId)}
/>
{#if activeConversation() && agenticPendingSteeringMessageContent(activeConversation()!.id)}
{@const convId = activeConversation()!.id}
{@const pendingContent = agenticPendingSteeringMessageContent(convId)}
{#if pendingContent}
<ChatMessageUserPending
class="mx-auto mt-12 w-full max-w-[48rem]"
content={pendingContent}
extras={agenticPendingSteeringMessageExtras(convId)}
onSendImmediately={() => chatStore.abortCurrentFlow(convId)}
onEdit={(newContent, extras) => agenticInjectSteeringMessage(convId, newContent, extras)}
onDelete={() => agenticClearSteeringMessage(convId)}
/>
{/if}
{:else if activeConversation() && chatPendingMessageContent(activeConversation()!.id)}
{@const convId = activeConversation()!.id}
{@const pendingContent = chatPendingMessageContent(convId)}
{#if pendingContent}
<ChatMessageUserPending
class="mx-auto mt-12 w-full max-w-[48rem]"
content={pendingContent}
extras={chatPendingMessageExtras(convId)}
onSendImmediately={() => chatStore.abortCurrentFlow(convId)}
onEdit={(newContent, extras) => chatInjectPendingMessage(convId, newContent, extras)}
onDelete={() => chatClearPendingMessage(convId)}
/>
{/if}
{/if}
{/if}
</div>
tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte
+15 -4
View File
@@ -42,6 +42,8 @@
let { showCenteredEmpty = false } = $props();
const autoScroll = createAutoScrollController();
let disableAutoScroll = $derived(Boolean(config().disableAutoScroll));
let chatScrollContainer: HTMLDivElement | undefined = $state();
let dragCounter = $state(0);
@@ -49,8 +51,6 @@
let showFileErrorDialog = $state(false);
let uploadedFiles = $state<ChatUploadedFile[]>([]);
const autoScroll = createAutoScrollController({ isColumnReverse: true });
let fileErrorData = $state<{
generallyUnsupported: File[];
modalityUnsupported: File[];
@@ -322,6 +322,14 @@
}
});
function handleMessagesReady() {
if (!disableAutoScroll && !autoScroll.userScrolledUp) {
requestAnimationFrame(() => {
autoScroll.scrollToBottom('instant');
});
}
}
onMount(() => {
autoScroll.startObserving();
@@ -357,7 +365,7 @@
<div
bind:this={chatScrollContainer}
aria-label="Chat interface with file drop zone"
class="flex h-full flex-col-reverse overflow-y-auto px-4 md:px-6"
class="flex h-full flex-col overflow-y-auto px-4 md:px-6"
ondragenter={handleDragEnter}
ondragleave={handleDragLeave}
ondragover={handleDragOver}
@@ -371,8 +379,11 @@
messages={activeMessages()}
onUserAction={() => {
autoScroll.enable();
autoScroll.scrollToBottom();
if (!autoScroll.userScrolledUp) {
autoScroll.scrollToBottom();
}
}}
onMessagesReady={handleMessagesReady}
/>
{/if}
tools/server/webui/src/lib/components/app/models/ModelId.svelte
+14 -7
View File
@@ -31,9 +31,12 @@
let parsed = $derived(ModelsService.parseModelId(modelId));
let resolvedShowRaw = $derived(showRaw ?? (config().showRawModelNames as boolean) ?? false);
let displayName = $derived(parsed.modelName ?? modelId);
let allAliases = $derived(aliases ?? []);
let allTags = $derived([...(parsed.tags ?? []), ...(tags ?? [])]);
let uniqueAliases = $derived([...new Set(aliases ?? [])]);
let uniqueTags = $derived([...new Set([...(parsed.tags ?? []), ...(tags ?? [])])]);
let primaryAlias = $derived(uniqueAliases.length === 1 ? uniqueAliases[0] : null);
let displayName = $derived(primaryAlias ?? parsed.modelName ?? modelId);
</script>
{#if resolvedShowRaw}
@@ -56,14 +59,18 @@
</span>
{/if}
{#if allAliases.length > 0}
{#each allAliases as alias (alias)}
{#if primaryAlias}
{#if primaryAlias !== parsed.modelName}
<span class={badgeClass}>{parsed.modelName ?? modelId}</span>
{/if}
{:else if uniqueAliases.length > 1}
{#each uniqueAliases as alias (alias)}
<span class={badgeClass}>{alias}</span>
{/each}
{/if}
{#if allTags.length > 0}
{#each allTags as tag (tag)}
{#if uniqueTags.length > 0}
{#each uniqueTags as tag (tag)}
<span class={tagBadgeClass}>{tag}</span>
{/each}
{/if}
tools/server/webui/src/lib/constants/settings-registry.ts
+1 -1
View File
@@ -122,7 +122,7 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
{
key: SETTINGS_KEYS.ENABLE_CONTINUE_GENERATION,
label: 'Enable "Continue" button',
help: 'Enable "Continue" button for assistant messages. Currently works only with non-reasoning models.',
help: 'Enable "Continue" button for assistant messages, including reasoning models.',
defaultValue: false,
type: SettingsFieldType.CHECKBOX,
section: SETTINGS_SECTION_SLUGS.GENERAL,
tools/server/webui/src/lib/hooks/use-auto-scroll.svelte.ts
+5 -51
View File
@@ -2,7 +2,6 @@ import { AUTO_SCROLL_AT_BOTTOM_THRESHOLD, AUTO_SCROLL_INTERVAL } from '$lib/cons
export interface AutoScrollOptions {
disabled?: boolean;
isColumnReverse?: boolean;
}
/**
@@ -12,24 +11,19 @@ export interface AutoScrollOptions {
* - Auto-scrolls to bottom during streaming/loading
* - Stops auto-scroll when user manually scrolls up
* - Resumes auto-scroll when user scrolls back to bottom
* - Supports both normal and column-reverse scroll containers
*/
export class AutoScrollController {
private _autoScrollEnabled = $state(true);
private _userScrolledUp = $state(false);
private _lastScrollTop = $state(0);
private _scrollInterval: ReturnType<typeof setInterval> | undefined;
private _scrollTimeout: ReturnType<typeof setTimeout> | undefined;
private _container: HTMLElement | undefined;
private _disabled: boolean;
private _isColumnReverse: boolean;
private _mutationObserver: MutationObserver | null = null;
private _rafPending = false;
private _observerEnabled = false;
constructor(options: AutoScrollOptions = {}) {
this._disabled = options.disabled ?? false;
this._isColumnReverse = options.isColumnReverse ?? false;
}
get autoScrollEnabled(): boolean {
@@ -56,6 +50,7 @@ export class AutoScrollController {
* Updates the disabled state.
*/
setDisabled(disabled: boolean): void {
if (this._disabled === disabled) return;
this._disabled = disabled;
if (disabled) {
this._autoScrollEnabled = false;
@@ -73,20 +68,8 @@ export class AutoScrollController {
if (this._disabled || !this._container) return;
const { scrollTop, scrollHeight, clientHeight } = this._container;
let distanceFromBottom: number;
let isScrollingUp: boolean;
if (this._isColumnReverse) {
// column-reverse: scrollTop=0 at bottom, negative when scrolled up
distanceFromBottom = Math.abs(scrollTop);
isScrollingUp = scrollTop < this._lastScrollTop;
} else {
// normal: scrollTop=0 at top, increases when scrolled down
distanceFromBottom = scrollHeight - clientHeight - scrollTop;
isScrollingUp = scrollTop < this._lastScrollTop;
}
const distanceFromBottom = scrollHeight - clientHeight - scrollTop;
const isScrollingUp = scrollTop < this._lastScrollTop;
const isAtBottom = distanceFromBottom < AUTO_SCROLL_AT_BOTTOM_THRESHOLD;
if (isScrollingUp && !isAtBottom) {
@@ -97,17 +80,6 @@ export class AutoScrollController {
this._autoScrollEnabled = true;
}
if (this._scrollTimeout) {
clearTimeout(this._scrollTimeout);
}
this._scrollTimeout = setTimeout(() => {
if (isAtBottom) {
this._userScrolledUp = false;
this._autoScrollEnabled = true;
}
}, AUTO_SCROLL_INTERVAL);
this._lastScrollTop = scrollTop;
}
@@ -116,13 +88,7 @@ export class AutoScrollController {
*/
scrollToBottom(behavior: ScrollBehavior = 'smooth'): void {
if (this._disabled || !this._container) return;
if (this._isColumnReverse) {
// column-reverse: scrollTop=0 is the bottom
this._container.scrollTo({ top: 0, behavior });
} else {
this._container.scrollTo({ top: this._container.scrollHeight, behavior });
}
this._container.scrollTo({ top: this._container.scrollHeight, behavior });
}
/**
@@ -180,11 +146,6 @@ export class AutoScrollController {
destroy(): void {
this.stopInterval();
this._doStopObserving();
if (this._scrollTimeout) {
clearTimeout(this._scrollTimeout);
this._scrollTimeout = undefined;
}
}
/**
@@ -210,20 +171,13 @@ export class AutoScrollController {
private _doStartObserving(): void {
if (!this._container || this._mutationObserver) return;
const isReverse = this._isColumnReverse;
this._mutationObserver = new MutationObserver(() => {
if (!this._autoScrollEnabled || this._rafPending) return;
this._rafPending = true;
requestAnimationFrame(() => {
this._rafPending = false;
if (this._autoScrollEnabled && this._container) {
if (isReverse) {
// column-reverse: scrollTop=0 is the bottom
this._container.scrollTop = 0;
} else {
this._container.scrollTop = this._container.scrollHeight;
}
this._container.scrollTop = this._container.scrollHeight;
}
});
});
tools/server/webui/src/lib/services/chat.service.ts
+7 -1
View File
@@ -130,7 +130,8 @@ export class ChatService {
timings_per_token,
// Config options
disableReasoningParsing,
excludeReasoningFromContext
excludeReasoningFromContext,
continueFinalMessage
} = options;
const normalizedMessages: ApiChatMessageData[] = messages
@@ -209,6 +210,11 @@ export class ChatService {
? ReasoningFormat.NONE
: ReasoningFormat.AUTO;
if (continueFinalMessage) {
requestBody.continue_final_message = true;
requestBody.add_generation_prompt = false;
}
if (temperature !== undefined) requestBody.temperature = temperature;
if (max_tokens !== undefined) {
// Set max_tokens to -1 (infinite) when explicitly configured as 0 or null
tools/server/webui/src/lib/stores/chat.svelte.ts
+48 -27
View File
@@ -674,7 +674,8 @@ class ChatStore {
},
onReasoningChunk: (chunk: string) => {
streamedReasoningContent += chunk;
// Update UI to show reasoning is being received
// mark streaming state so a stop mid-thinking can persist the partial reasoning
this.setChatStreaming(convId, streamedContent, currentMessageId);
const idx = conversationsStore.findMessageIndex(currentMessageId);
conversationsStore.updateMessageAtIndex(idx, {
reasoningContent: streamedReasoningContent
@@ -989,38 +990,51 @@ class ChatStore {
const conversationId = convId || conversationsStore.activeConversation?.id;
if (!conversationId) return;
const streamingState = this.getChatStreaming(conversationId);
if (!streamingState || !streamingState.response.trim()) return;
if (!streamingState) return;
const messages =
conversationId === conversationsStore.activeConversation?.id
? conversationsStore.activeMessages
: await conversationsStore.getConversationMessages(conversationId);
if (!messages.length) return;
const lastMessage = messages[messages.length - 1];
if (lastMessage?.role === MessageRole.ASSISTANT) {
try {
const updateData: { content: string; timings?: ChatMessageTimings } = {
content: streamingState.response
};
const lastKnownState = this.getProcessingState(conversationId);
if (lastKnownState) {
updateData.timings = {
prompt_n: lastKnownState.promptTokens || 0,
prompt_ms: lastKnownState.promptMs,
predicted_n: lastKnownState.tokensDecoded || 0,
cache_n: lastKnownState.cacheTokens || 0,
predicted_ms:
lastKnownState.tokensPerSecond && lastKnownState.tokensDecoded
? (lastKnownState.tokensDecoded / lastKnownState.tokensPerSecond) * 1000
: undefined
};
}
await DatabaseService.updateMessage(lastMessage.id, updateData);
lastMessage.content = streamingState.response;
if (updateData.timings) lastMessage.timings = updateData.timings;
} catch (error) {
lastMessage.content = streamingState.response;
console.error('Failed to save partial response:', error);
if (lastMessage?.role !== MessageRole.ASSISTANT) return;
const partialContent = streamingState.response;
const partialReasoning = lastMessage.reasoningContent || '';
// nothing to persist when both content and reasoning are empty (e.g. stop before any token)
if (!partialContent.trim() && !partialReasoning.trim()) return;
try {
const updateData: {
content: string;
reasoningContent?: string;
timings?: ChatMessageTimings;
} = {
content: partialContent
};
if (partialReasoning) {
updateData.reasoningContent = partialReasoning;
}
const lastKnownState = this.getProcessingState(conversationId);
if (lastKnownState) {
updateData.timings = {
prompt_n: lastKnownState.promptTokens || 0,
prompt_ms: lastKnownState.promptMs,
predicted_n: lastKnownState.tokensDecoded || 0,
cache_n: lastKnownState.cacheTokens || 0,
predicted_ms:
lastKnownState.tokensPerSecond && lastKnownState.tokensDecoded
? (lastKnownState.tokensDecoded / lastKnownState.tokensPerSecond) * 1000
: undefined
};
}
await DatabaseService.updateMessage(lastMessage.id, updateData);
lastMessage.content = partialContent;
if (updateData.timings) lastMessage.timings = updateData.timings;
} catch (error) {
lastMessage.content = partialContent;
console.error('Failed to save partial response:', error);
}
}
@@ -1265,7 +1279,11 @@ class ChatStore {
const conversationContext = conversationsStore.activeMessages.slice(0, idx);
const contextWithContinue = [
...conversationContext,
{ role: MessageRole.ASSISTANT as const, content: originalContent }
{
role: MessageRole.ASSISTANT as const,
content: originalContent,
reasoning_content: originalReasoning || undefined
}
];
let appendedContent = '';
@@ -1283,6 +1301,7 @@ class ChatStore {
contextWithContinue,
{
...this.getApiOptions(),
continueFinalMessage: true,
onChunk: (chunk: string) => {
appendedContent += chunk;
hasReceivedContent = true;
@@ -1291,6 +1310,8 @@ class ChatStore {
onReasoningChunk: (chunk: string) => {
appendedReasoning += chunk;
hasReceivedContent = true;
// mark streaming state so a stop mid-thinking can persist the partial reasoning
this.setChatStreaming(msg.convId, originalContent + appendedContent, msg.id);
conversationsStore.updateMessageAtIndex(idx, {
reasoningContent: originalReasoning + appendedReasoning
});
tools/server/webui/src/lib/types/api.d.ts
Vendored
+3
View File
@@ -239,6 +239,9 @@ export interface ApiChatCompletionRequest {
// Custom parameters (JSON string)
custom?: Record<string, unknown>;
timings_per_token?: boolean;
// Continuation control (vLLM compat)
add_generation_prompt?: boolean;
continue_final_message?: boolean;
}
export interface ApiChatCompletionToolCallFunctionDelta {
tools/server/webui/src/lib/types/settings.d.ts
Vendored
+2
View File
@@ -92,6 +92,8 @@ export interface SettingsChatServiceOptions {
// Custom parameters
custom?: string;
timings_per_token?: boolean;
// Continuation control (vLLM compat), opt in to the explicit continue final message flag
continueFinalMessage?: boolean;
// Callbacks
onChunk?: (chunk: string) => void;
onReasoningChunk?: (chunk: string) => void;
tools/tts/tts.cpp
+1 -1
View File
@@ -604,8 +604,8 @@ int main(int argc, char ** argv) {
}
LOG_INF("sampler seed: %u\n", common_sampler_get_seed(smpl[0]));
LOG_INF("sampler params: \n%s\n", params.sampling.print().c_str());
LOG_INF("sampler chain: %s\n", common_sampler_print(smpl[0]).c_str());
LOG_INF("sampler params: \n%s\n", params.sampling.print().c_str());
LOG_INF("%s: loading done\n", __func__);