From a6b3260a4268d3c9931e0c47859fe4ff8f108bd9 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Thu, 18 Jun 2026 21:55:04 +0200 Subject: [PATCH 01/21] mtmd: add batching for mtmd-cli, add video tests (#24778) --- tools/mtmd/mtmd-cli.cpp | 117 ++++++++++++++++++++++++++++++++++------ tools/mtmd/tests.sh | 52 ++++++++++++++---- 2 files changed, 145 insertions(+), 24 deletions(-) diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index a3cad7cd0..0ad000ef0 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -32,9 +32,9 @@ static volatile bool g_is_generating = false; static volatile bool g_is_interrupted = false; /** - * Please note that this is NOT a production-ready stuff. + * Please note that this is NOT a production-ready binary. * It is a playground for trying multimodal support in llama.cpp. - * For contributors: please keep this code simple and easy to understand. + * For contributors: please keep this code simple and easy to understand. Do not add unnecessary complexity. The goal is to have a simple CLI for testing multimodal support. */ static void show_additional_info(int /*argc*/, char ** argv) { @@ -65,6 +65,14 @@ static void sigint_handler(int signo) { } #endif +// this is only used by tests.sh to capture the response ; it's not meant to be used in production +static void inject_test_response_marker() { + const char * env = std::getenv("MTMD_TEST_RESPONSE_MARKER"); + if (env) { + LOG("%s\n", env); + } +} + struct mtmd_cli_context { mtmd::context_ptr ctx_vision; common_init_result_ptr llama_init; @@ -79,6 +87,8 @@ struct mtmd_cli_context { mtmd::bitmaps bitmaps; std::vector videos; + mtmd::batch_ptr mbatch; + // chat template common_chat_templates_ptr tmpls; std::vector chat_history; @@ -233,6 +243,8 @@ static std::string chat_add_and_format(mtmd_cli_context & ctx, common_chat_msg & } static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) { + inject_test_response_marker(); + bool add_bos = ctx.chat_history.empty(); auto formatted_chat = chat_add_and_format(ctx, msg); LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.c_str()); @@ -259,20 +271,95 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) { ctx.bitmaps.entries.clear(); ctx.videos.clear(); - llama_pos new_n_past; - if (mtmd_helper_eval_chunks(ctx.ctx_vision.get(), - ctx.lctx, // lctx - chunks.ptr.get(), // chunks - ctx.n_past, // n_past - 0, // seq_id - ctx.n_batch, // n_batch - true, // logits_last - &new_n_past)) { - LOG_ERR("Unable to eval prompt\n"); - return 1; - } + // batch encode all media chunks, then decode each + size_t n_chunks = mtmd_input_chunks_size(chunks.ptr.get()); + for (size_t i = 0; i < n_chunks; i++) { + auto chunk = mtmd_input_chunks_get(chunks.ptr.get(), i); + auto chunk_type = mtmd_input_chunk_get_type(chunk); - ctx.n_past = new_n_past; + if (chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT) { + // decode text chunk + llama_pos new_n_past = ctx.n_past; + res = mtmd_helper_eval_chunk_single(ctx.ctx_vision.get(), + ctx.lctx, + chunk, + ctx.n_past, + 0, // seq_id + ctx.n_batch, + i == n_chunks - 1, // logits_last + &new_n_past); + if (res != 0) { + LOG_ERR("Unable to eval text chunk %zu\n", i); + return 1; + } + ctx.n_past = new_n_past; + } else { + // media chunk: try to get embd from existing batch, or create a new batch + float * embd = nullptr; + if (ctx.mbatch) { + embd = mtmd_batch_get_output_embd(ctx.mbatch.get(), chunk); + + if (embd) { + LOG_DBG("found embd for media chunk %zu in existing batch\n", i); + } else { + LOG_DBG("media chunk %zu not found in existing batch, creating new batch\n", i); + } + } + + if (!embd) { + // create and encode a new batch with as many media chunks as possible + ctx.mbatch.reset(mtmd_batch_init(ctx.ctx_vision.get())); + res = mtmd_batch_add_chunk(ctx.mbatch.get(), chunk); + GGML_ASSERT(res == 0); // first chunk must always succeed + + int n_added = 1; + // add as many subsequent media chunks as possible + for (size_t j = i + 1; j < n_chunks; j++) { + auto next_chunk = mtmd_input_chunks_get(chunks.ptr.get(), j); + auto next_type = mtmd_input_chunk_get_type(next_chunk); + if (next_type == MTMD_INPUT_CHUNK_TYPE_TEXT) { + break; // text chunk splits the batch + } + res = mtmd_batch_add_chunk(ctx.mbatch.get(), next_chunk); + if (res != 0) { + break; // batch full or incompatible + } + n_added++; + } + + int64_t time_start = ggml_time_ms(); + LOG_INF("encoding mtmd batch, n_chunks = %d (done = %zu, total = %zu)\n", n_added, i, n_chunks); + res = mtmd_batch_encode(ctx.mbatch.get()); + if (res != 0) { + LOG_ERR("Failed to encode mtmd batch, res = %d\n", res); + return 1; + } + LOG_INF("mtmd batch encoding done in %d ms\n", (int)(ggml_time_ms() - time_start)); + + embd = mtmd_batch_get_output_embd(ctx.mbatch.get(), chunk); + } + + GGML_ASSERT(embd != nullptr); + + llama_pos new_n_past = ctx.n_past; + res = mtmd_helper_decode_image_chunk(ctx.ctx_vision.get(), + ctx.lctx, + chunk, + embd, + ctx.n_past, + 0, // seq_id + ctx.n_batch, + &new_n_past, + nullptr, // callback + nullptr // user_data + ); + if (res != 0) { + LOG_ERR("Unable to decode media chunk %zu\n", i); + return 1; + } + ctx.n_past = new_n_past; + } + } LOG("\n"); diff --git a/tools/mtmd/tests.sh b/tools/mtmd/tests.sh index 5da48d61b..6fe26478a 100755 --- a/tools/mtmd/tests.sh +++ b/tools/mtmd/tests.sh @@ -13,6 +13,8 @@ mkdir -p $SCRIPT_DIR/output PROJ_ROOT="$SCRIPT_DIR/../.." cd $PROJ_ROOT +export MTMD_TEST_RESPONSE_MARKER="" + # Check if the first argument is "big", then run test with big models # This is useful if we're running the script on a larger machine, so we can test the big models RUN_BIG_TESTS=false @@ -28,6 +30,15 @@ if [ "${1:-}" = "huge" ]; then echo "Include BIG and HUGE models..." fi +USE_VIDEO=false +if [ "${1:-}" = "video" ]; then + USE_VIDEO=true + echo "Using video as input..." + # behavior of USE_VIDEO: + # do NOT check if the output contains "new york", only verify if the exit code is 0 + # when printing the result, print the OK/FAIL line then print the generated text +fi + # Check if the second argument is "flash", then enable flash attention # This is useful to test if flash attention off works correctly FLASH_ATTN="on" @@ -50,13 +61,20 @@ add_test_vision() { if [ $# -gt 0 ]; then extra_args=$(printf " %q" "$@") fi + if [ "$USE_VIDEO" = true ]; then + arr_file+=("test-3.mp4") + else + arr_file+=("test-1.jpeg") + fi arr_prefix+=("[vision]") arr_hf+=("$hf") arr_extra_args+=("$extra_args") - arr_file+=("test-1.jpeg") } add_test_audio() { + if [ "$USE_VIDEO" = true ]; then + return 0 + fi local hf=$1 shift local extra_args="" @@ -166,19 +184,35 @@ for i in "${!arr_hf[@]}"; do cmd+=" -p \"what is the publisher name of the newspaper?\"" fi - output=$(eval "$cmd" 2>&1 | tee /dev/tty) + exit_code=0 + output=$(eval "$cmd" 2>&1 | tee /dev/tty) || exit_code=$? echo "$output" > $SCRIPT_DIR/output/$bin-$(echo "$hf" | tr '/' '-').log - # either contains "new york" or both "men" and "walk" - if echo "$output" | grep -iq "new york" \ - || (echo "$output" | grep -iq "men" && echo "$output" | grep -iq "walk") - then - result="$prefix \033[32mOK\033[0m: $hf" + if [ "$USE_VIDEO" = true ]; then + # for video, only check exit code; do not grep for "new york" + if [ $exit_code -eq 0 ]; then + result="$prefix \033[32mOK\033[0m: $hf" + else + result="$prefix \033[31mFAIL\033[0m: $hf" + fi + # append generated text (after the response marker) + generated_text=$(echo "$output" | sed "1,/${MTMD_TEST_RESPONSE_MARKER}/d" | tail -10) + if [ -n "$generated_text" ]; then + result+="\n$generated_text" + fi + echo -e "$result" else - result="$prefix \033[31mFAIL\033[0m: $hf" + # either contains "new york" or both "men" and "walk" + if echo "$output" | grep -iq "new york" \ + || (echo "$output" | grep -iq "men" && echo "$output" | grep -iq "walk") + then + result="$prefix \033[32mOK\033[0m: $hf" + else + result="$prefix \033[31mFAIL\033[0m: $hf" + fi + echo -e "$result" fi - echo -e "$result" arr_res+=("$result") echo "" From 40f3aafc45990c1646397c67d2bd3e8eff3f543e Mon Sep 17 00:00:00 2001 From: Reguna Date: Fri, 19 Jun 2026 04:01:24 +0800 Subject: [PATCH 02/21] server: add "X-Accel-Buffering": "no" header to streaming endpoints (#24774) * server: add "X-Accel-Buffering": "no" header to streaming endpoints This header tells Nginx (as a reverse proxy) to NOT buffer responses. (only affects streaming endpoints) Without it, Nginx will break streaming with certain applications (notably the Pi coding harness). --- tools/server/server-http.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 5defee1f5..4f2abab00 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -492,6 +492,8 @@ using server_http_req_ptr = std::unique_ptr; static void process_handler_response(server_http_req_ptr && request, server_http_res_ptr & response, httplib::Response & res) { if (response->is_stream()) { res.status = response->status; + // Tell Nginx to not buffer any streamed response + response->headers["X-Accel-Buffering"] = "no"; set_headers(res, response->headers); const std::string content_type = response->content_type; // convert to shared_ptr as both chunked_content_provider() and on_complete() need to use it From 3a3edc9ac65cca79584ca497be41d70c75a58ba8 Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 18 Jun 2026 22:23:01 +0200 Subject: [PATCH 03/21] Ggml/cuda col2im 1d (#24417) * cuda: add GGML_OP_COL2IM_1D, follow-up to the CPU op * cuda: col2im_1d use fast_div_modulo for the index decomposition * cuda: col2im_1d tighten supports_op, type match and contiguous dst --- ggml/src/ggml-cuda/col2im-1d.cu | 81 ++++++++++++++++++++++++++++++++ ggml/src/ggml-cuda/col2im-1d.cuh | 3 ++ ggml/src/ggml-cuda/ggml-cuda.cu | 12 +++++ 3 files changed, 96 insertions(+) create mode 100644 ggml/src/ggml-cuda/col2im-1d.cu create mode 100644 ggml/src/ggml-cuda/col2im-1d.cuh diff --git a/ggml/src/ggml-cuda/col2im-1d.cu b/ggml/src/ggml-cuda/col2im-1d.cu new file mode 100644 index 000000000..fecd4c6a9 --- /dev/null +++ b/ggml/src/ggml-cuda/col2im-1d.cu @@ -0,0 +1,81 @@ +#include "col2im-1d.cuh" +#include "convert.cuh" + +// col2im_1d: scatter-add GEMM columns to 1D signal (gather approach) +// columns: [K*OC, T_in] -> output: [T_out, OC] +// Supports F32, F16, BF16 data with F32 accumulator. + +template +static __global__ void col2im_1d_kernel( + const T * __restrict__ col, + T * __restrict__ dst, + const int T_in, const uint3 T_out_fd, + const int OC, const int K, const int K_OC, + const int s0, const int p0, const int total) { + + const int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx >= total) return; + + // dst layout: [T_out, OC], ne[0]=T_out fastest + const uint2 qr = fast_div_modulo((uint32_t)idx, T_out_fd); // qr.x = idx / T_out, qr.y = idx % T_out + const int oc = (int)qr.x; + const int t_out = (int)qr.y; + const int t_abs = t_out + p0; // absolute position in uncropped signal + + // Gather: find all (t_in, k) where t_in*s + k == t_abs, 0 <= k < K + int t_in_min = (t_abs - K + s0) / s0; // ceil((t_abs - K + 1) / s) + if (t_in_min < 0) t_in_min = 0; + int t_in_max = t_abs / s0; + if (t_in_max >= T_in) t_in_max = T_in - 1; + + float sum = 0.0f; + for (int t_in = t_in_min; t_in <= t_in_max; t_in++) { + const int k = t_abs - t_in * s0; + // col layout: [K*OC, T_in], column index = oc * K + k + sum += ggml_cuda_cast(col[(oc * K + k) + t_in * K_OC]); + } + + dst[idx] = ggml_cuda_cast(sum); +} + +void ggml_cuda_op_col2im_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(ggml_is_contiguous(src0)); + + const int32_t s0 = ((const int32_t *)(dst->op_params))[0]; + const int32_t OC = ((const int32_t *)(dst->op_params))[1]; + const int32_t p0 = ((const int32_t *)(dst->op_params))[2]; + + const int K_OC = (int) src0->ne[0]; + const int T_in = (int) src0->ne[1]; + const int K = K_OC / OC; + const int T_out = (int) dst->ne[0]; + + const uint3 T_out_fd = init_fastdiv_values((uint32_t)T_out); + + const int total = T_out * OC; + const int block_size = 256; + const int num_blocks = (total + block_size - 1) / block_size; + + switch (src0->type) { + case GGML_TYPE_F32: { + col2im_1d_kernel<<>>( + (const float *)src0->data, (float *)dst->data, + T_in, T_out_fd, OC, K, K_OC, s0, p0, total); + } break; + case GGML_TYPE_F16: { + col2im_1d_kernel<<>>( + (const half *)src0->data, (half *)dst->data, + T_in, T_out_fd, OC, K, K_OC, s0, p0, total); + } break; + case GGML_TYPE_BF16: { + col2im_1d_kernel<<>>( + (const nv_bfloat16 *)src0->data, (nv_bfloat16 *)dst->data, + T_in, T_out_fd, OC, K, K_OC, s0, p0, total); + } break; + default: + GGML_ABORT("col2im_1d: unsupported type"); + } +} diff --git a/ggml/src/ggml-cuda/col2im-1d.cuh b/ggml/src/ggml-cuda/col2im-1d.cuh new file mode 100644 index 000000000..efc3313c4 --- /dev/null +++ b/ggml/src/ggml-cuda/col2im-1d.cuh @@ -0,0 +1,3 @@ +#include "common.cuh" + +void ggml_cuda_op_col2im_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 34cdbc81c..3d4b5f605 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -11,6 +11,7 @@ #include "ggml-cuda/argsort.cuh" #include "ggml-cuda/binbcast.cuh" #include "ggml-cuda/clamp.cuh" +#include "ggml-cuda/col2im-1d.cuh" #include "ggml-cuda/concat.cuh" #include "ggml-cuda/conv-transpose-1d.cuh" #include "ggml-cuda/conv2d.cuh" @@ -3051,6 +3052,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_CONV_TRANSPOSE_1D: ggml_cuda_op_conv_transpose_1d(ctx,dst); break; + case GGML_OP_COL2IM_1D: + ggml_cuda_op_col2im_1d(ctx, dst); + break; case GGML_OP_POOL_2D: ggml_cuda_op_pool2d(ctx, dst); break; @@ -5316,6 +5320,14 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g } return false; } break; + case GGML_OP_COL2IM_1D: + { + ggml_type src0_type = op->src[0]->type; + return (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_F16 || src0_type == GGML_TYPE_BF16) && + op->type == src0_type && + ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op); + } break; case GGML_OP_SILU_BACK: return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; break; From db52540f730de39efcf7172d4ab1f79bb50556e2 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Fri, 19 Jun 2026 01:16:16 +0200 Subject: [PATCH 04/21] mtmd: add batching support for internvl (#24775) --- tools/mtmd/clip.cpp | 2 +- tools/mtmd/models/internvl.cpp | 18 +++++++++++------- tools/mtmd/models/models.h | 1 + 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index dc6223295..17079815d 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -534,7 +534,7 @@ ggml_tensor * clip_graph::build_vit( ggml_tensor * clip_graph::build_inp() { ggml_tensor * inp_raw = build_inp_raw(); ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); - inp = ggml_reshape_2d(ctx0, inp, n_patches, n_embd); + inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, n_batch); inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); if (model.patch_bias) { inp = ggml_add(ctx0, inp, model.patch_bias); diff --git a/tools/mtmd/models/internvl.cpp b/tools/mtmd/models/internvl.cpp index 9aded3b97..65d7d5a6b 100644 --- a/tools/mtmd/models/internvl.cpp +++ b/tools/mtmd/models/internvl.cpp @@ -8,7 +8,9 @@ ggml_cgraph * clip_graph_internvl::build() { ggml_tensor * inp = build_inp(); // add CLS token - inp = ggml_concat(ctx0, inp, model.class_embedding, 1); + ggml_tensor * cls_repeated = ggml_repeat_4d(ctx0, model.class_embedding, + model.class_embedding->ne[0], 1, n_batch, 1); + inp = ggml_concat(ctx0, inp, cls_repeated, 1); // The larger models use a different ViT, which uses RMS norm instead of layer norm // ref: https://github.com/ggml-org/llama.cpp/pull/13443#issuecomment-2869786188 @@ -24,14 +26,15 @@ ggml_cgraph * clip_graph_internvl::build() { nullptr); // remove CLS token - cur = ggml_view_2d(ctx0, cur, - n_embd, n_patches, - ggml_row_size(cur->type, n_embd), 0); + cur = ggml_view_3d(ctx0, cur, + n_embd, n_patches, n_batch, + cur->nb[1], cur->nb[2], 0); + cur = ggml_cont(ctx0, cur); // pixel shuffle { const int scale_factor = model.hparams.n_merge; - const int bsz = 1; // batch size, always 1 for now since we don't support batching + const int bsz = n_batch; const int height = n_patches_y; const int width = n_patches_x; GGML_ASSERT(scale_factor > 0); @@ -44,9 +47,10 @@ ggml_cgraph * clip_graph_internvl::build() { bsz); cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); // flatten to 2D - cur = ggml_cont_2d(ctx0, cur, + cur = ggml_cont_3d(ctx0, cur, n_embd * scale_factor * scale_factor, - cur->ne[1] * cur->ne[2]); + cur->ne[1] * cur->ne[2], + cur->ne[3]); } // projector (always using GELU activation) diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 3a15f7682..12d5e6949 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -80,6 +80,7 @@ struct clip_graph_minicpmv4_6 : clip_graph { struct clip_graph_internvl : clip_graph { clip_graph_internvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; + bool support_batch() const override { return true; } }; struct clip_graph_nemotron_v2_vl : clip_graph { From 8141e730f1598780c19b153e0e212ed70a672c53 Mon Sep 17 00:00:00 2001 From: shalinib-ibm Date: Fri, 19 Jun 2026 11:25:38 +0530 Subject: [PATCH 05/21] ggml-cpu: support K tails in power10 Q8/Q4 MMA matmul (#24753) * ggml-cpu: support K tails in Power10 MMA Q8/Q4 matmul This patch removes the requirement that K be divisible by kc in the tinyBlas_Q0_PPC tiled matmul path. Process the final K panel using its actual depth and pass the reduced panel size through packing and kernel execution. This allows more workloads to use the MMA kernel and reduces fallback to mnpack. * Apply suggestion from @taronaeo Co-authored-by: Aaron Teo --------- Co-authored-by: Aaron Teo --- ggml/src/ggml-cpu/llamafile/sgemm.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index e13828e3b..0b8323e60 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -2345,7 +2345,7 @@ class tinyBLAS_Q0_PPC { else if (n_aligned % 16 == 0) nc = 16; else nc = 8; } - bool can_use_tiled = n_aligned > 0 && (m % mc == 0) && (k % kc == 0); + bool can_use_tiled = n_aligned > 0 && (m % mc == 0); if (can_use_tiled) { matmul_tiled(m, n_aligned, mc, nc, kc); if (n > n_aligned) { @@ -3063,13 +3063,14 @@ class tinyBLAS_Q0_PPC { int64_t ii = (job / xtiles) * mc; int64_t jj = (job % xtiles) * nc; for (int64_t kk = 0; kk < k; kk += kc) { + int64_t k_cur = MIN(kc, k - kk); if constexpr(is_Ablock_q4) { - packNormal_q4_fp16(A + ii * lda + kk, lda, mc, kc, (uint8_t *)A_pack); + packNormal_q4_fp16(A + ii * lda + kk, lda, mc, k_cur, (uint8_t *)A_pack); } else { - packNormal_q8_fp16(A + ii * lda + kk, lda, mc, kc, (uint8_t *)A_pack); + packNormal_q8_fp16(A + ii * lda + kk, lda, mc, k_cur, (uint8_t *)A_pack); } - packNormal_q8_fp16(B + jj * ldb + kk, ldb, nc, kc, (uint8_t *)B_pack); - KERNEL_Q0(ii, jj, mc, nc, kc, kk, A_pack, B_pack); + packNormal_q8_fp16(B + jj * ldb + kk, ldb, nc, k_cur, (uint8_t *)B_pack); + KERNEL_Q0(ii, jj, mc, nc, k_cur, kk, A_pack, B_pack); } } } From 80452d65b9b1d44b496ed729f1fb0b6c4c39d7bf Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 19 Jun 2026 09:22:34 +0300 Subject: [PATCH 06/21] server : consolidate slot selection into get_available_slot (#24755) Absorb get_slot_by_id logic into get_available_slot so slot selection is handled by a single function call. When a specific slot id is requested, the LCP similarity check still runs to enable proper prompt cache updates. Assisted-by: pi:llama.cpp/Qwen3.6-27B --- tools/server/server-context.cpp | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index aebca306a..ded622cfd 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1395,11 +1395,23 @@ private: bool update_cache = false; + // if a specific slot is requested, use it (still goes through cache update logic below) + if (task.id_slot != -1) { + ret = get_slot_by_id(task.id_slot); + if (ret) { + SLT_INF(*ret, "selected slot by id (%d)\n", task.id_slot); + } + } + // find the slot that has at least n% prompt similarity - if (ret == nullptr && slot_prompt_similarity != 0.0f) { + if (slot_prompt_similarity != 0.0f) { float sim_best = 0; for (server_slot & slot : slots) { + if (task.id_slot != -1 && slot.id != task.id_slot) { + continue; + } + // skip the slot if it is not available if (slot.is_processing()) { continue; @@ -1426,8 +1438,10 @@ private: if (ret != nullptr) { const float f_keep = (sim_best*task.tokens.size()) / ret->prompt.tokens.size(); - SLT_INF(*ret, "selected slot by LCP similarity, sim_best = %.3f (> %.3f thold), f_keep = %.3f\n", - sim_best, slot_prompt_similarity, f_keep); + if (task.id_slot == -1) { + SLT_INF(*ret, "selected slot by LCP similarity, sim_best = %.3f (> %.3f thold), f_keep = %.3f\n", + sim_best, slot_prompt_similarity, f_keep); + } // if we are about to lose a large portion of the existing context - save it in the prompt cache if (f_keep < 0.5f) { @@ -2180,10 +2194,9 @@ private: } } - const int id_slot = task.id_slot; const int id_task = task.id; - server_slot * slot = id_slot != -1 ? get_slot_by_id(id_slot) : get_available_slot(task); + server_slot * slot = get_available_slot(task); // // slot scheduling logic From 5bd21b8555edf203aee78aacedf2d2744b8702a3 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 19 Jun 2026 09:34:00 +0300 Subject: [PATCH 07/21] pi : remove docs from system prompt (#24791) --- .pi/gg/SYSTEM.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.pi/gg/SYSTEM.md b/.pi/gg/SYSTEM.md index 197173fae..17ce71cc1 100644 --- a/.pi/gg/SYSTEM.md +++ b/.pi/gg/SYSTEM.md @@ -25,13 +25,3 @@ Commits: - Do not explicitly set the git author in commits - rely on the default git config - Always use `--no-gpg-sign` when committing - Never `git push` without explicit confirmation from the user - -Resources (read on demand): -- [CONTRIBUTING.md](CONTRIBUTING.md) -- [Build documentation](docs/build.md) -- [Server usage documentation](tools/server/README.md) -- [Server development documentation](tools/server/README-dev.md) -- [PEG parser](docs/development/parsing.md) -- [Auto parser](docs/autoparser.md) -- [Jinja engine](common/jinja/README.md) -- [PR template](.github/pull_request_template.md) From 1868af13ac1c2c6dd5b0dd1a4ac43f6ccc80fd80 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 19 Jun 2026 10:14:26 +0300 Subject: [PATCH 08/21] ggml : bump version to 0.15.2 (ggml/1548) --- ggml/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 0507e0c5a..04069784f 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -5,7 +5,7 @@ project("ggml" C CXX ASM) ### GGML Version set(GGML_VERSION_MAJOR 0) set(GGML_VERSION_MINOR 15) -set(GGML_VERSION_PATCH 1) +set(GGML_VERSION_PATCH 2) set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") From 5fd2dc2c41c342a75c26f9756ca6b1814ed05fb4 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 19 Jun 2026 10:18:14 +0300 Subject: [PATCH 09/21] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 87d353ef4..499be5a58 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -3af5f5760e19a96427f5f7a93b79cbdf3d4b265b +707321c4cf6d21cb4bc831aa8b687dbf01a521ce From 159d093a43e87b977e9749b18a980c15b1a66a90 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Fri, 19 Jun 2026 10:53:44 +0200 Subject: [PATCH 10/21] server: fix non-bound n_discard value (ctx shifting) (#24786) * server: fix non-bound n_discard value * Update tools/server/server-context.cpp Co-authored-by: Georgi Gerganov --------- Co-authored-by: Georgi Gerganov --- tools/server/server-context.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index ded622cfd..a23b0405c 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2565,7 +2565,10 @@ private: n_keep = std::min(slot.n_ctx - 4, n_keep); const int n_left = slot.prompt.n_tokens() - n_keep; - const int n_discard = slot.task->params.n_discard ? slot.task->params.n_discard : (n_left / 2); + int n_discard = slot.task->params.n_discard ? slot.task->params.n_discard : (n_left / 2); + + // ref: https://github.com/ggml-org/llama.cpp/pull/24786 + n_discard = std::clamp(n_discard, 0, std::max(0, n_left - 1)); SLT_WRN(slot, "slot context shift, n_keep = %d, n_left = %d, n_discard = %d\n", n_keep, n_left, n_discard); From b14e3fb90ca8c760f4254ddc9aa7845ebbdb2edf Mon Sep 17 00:00:00 2001 From: Ruixiang Wang Date: Fri, 19 Jun 2026 12:08:50 +0200 Subject: [PATCH 11/21] spec: support eagle3 for qwen3.5 & 3.6 (#24593) * spec: support qwen3.5 & 3.6 eagle3 draft * eagle3: Add deferred boundary checkpoints restore support for hybrid models * apply suggestions Co-authored-by: Georgi Gerganov * spec: adapt to API change * spec: fix naming * cont : add TODO --------- Co-authored-by: Georgi Gerganov --- common/common.cpp | 4 +- common/common.h | 6 ++- common/speculative.cpp | 72 +++++++++++++++++++++++++++++++++ common/speculative.h | 4 ++ src/models/qwen35.cpp | 2 + src/models/qwen35moe.cpp | 2 + tools/server/server-context.cpp | 4 ++ 7 files changed, 92 insertions(+), 2 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index b01772e1c..f3f114f68 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -2034,7 +2034,7 @@ bool common_prompt_batch_decode( } size_t common_prompt_checkpoint::size() const { - return data_tgt.size() + data_dft.size(); + return data_tgt.size() + data_dft.size() + data_spec.size(); } bool common_prompt_checkpoint::empty() const { @@ -2049,6 +2049,7 @@ void common_prompt_checkpoint::clear() { data_tgt.clear(); data_dft.clear(); + data_spec.clear(); } void common_prompt_checkpoint::update_pos( @@ -2138,4 +2139,5 @@ void common_prompt_checkpoint::clear_tgt() { void common_prompt_checkpoint::clear_dft() { data_dft.clear(); + data_spec.clear(); } diff --git a/common/common.h b/common/common.h index 040b9cf23..535a4ed33 100644 --- a/common/common.h +++ b/common/common.h @@ -363,7 +363,7 @@ struct common_params_speculative { uint32_t need_n_rs_seq() const { bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { - return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP; + return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3; }); return needs_rs_seq ? draft.n_max : 0u; @@ -1065,6 +1065,10 @@ struct common_prompt_checkpoint { std::vector data_tgt; std::vector data_dft; + // (optional) speculative-decoding implementation state stashed with the checkpoint + // (e.g. eagle3's deferred-boundary g_embd row) + std::vector data_spec; + size_t size() const; bool empty() const; diff --git a/common/speculative.cpp b/common/speculative.cpp index 6f387f2cf..9c20585dc 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -161,6 +161,10 @@ struct common_speculative_impl { virtual void accept(llama_seq_id seq_id, uint16_t n_accepted, bool is_other) = 0; + // (optional) serialize/restore per-seq internal state (e.g. eagle3's deferred boundary). + virtual bool get_state(llama_seq_id /*seq_id*/, std::vector & /*data*/) const { return false; } + virtual void set_state(llama_seq_id /*seq_id*/, const std::vector & /*data*/) {} + // true if this implementation requires the target context to extract post-norm embeddings virtual bool need_embd() const = 0; @@ -841,6 +845,49 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { (size_t) n_embd_dec * sizeof(float)); } + // we only need to stash the deferred boundary's g_embd row for recurrent/hybrid targets: + // their single-position checkpoints drop it on restore + bool need_boundary_stash() const { + const llama_model * model_tgt = llama_get_model(params.ctx_tgt); + return llama_model_is_recurrent(model_tgt) || llama_model_is_hybrid(model_tgt); + } + + bool get_state(llama_seq_id seq_id, std::vector & data) const override { + if (!need_boundary_stash()) { + return false; + } + if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq || pending_pos_last[seq_id] < 0) { + return false; + } + + const llama_pos pos = pending_pos_last[seq_id]; + const std::vector & g = pending_g_last[seq_id]; + + data.resize(sizeof(llama_pos) + g.size() * sizeof(float)); + std::memcpy(data.data(), &pos, sizeof(llama_pos)); + std::memcpy(data.data() + sizeof(llama_pos), g.data(), g.size() * sizeof(float)); + return true; + } + + void set_state(llama_seq_id seq_id, const std::vector & data) override { + if (!need_boundary_stash()) { + return; + } + if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { + return; + } + if (data.size() != sizeof(llama_pos) + (size_t) n_embd_dec * sizeof(float)) { + return; + } + + llama_pos pos = -1; + std::memcpy(&pos, data.data(), sizeof(llama_pos)); + + pending_pos_last[seq_id] = pos; + pending_g_last[seq_id].resize(n_embd_dec); + std::memcpy(pending_g_last[seq_id].data(), data.data() + sizeof(llama_pos), (size_t) n_embd_dec * sizeof(float)); + } + bool need_embd() const override { return false; } @@ -2118,6 +2165,31 @@ void common_speculative_accept(common_speculative * spec, llama_seq_id seq_id, u } } +// TODO: support the case of more than one speculative implementations having a state +bool common_speculative_get_state(common_speculative * spec, llama_seq_id seq_id, std::vector & data) { + if (spec == nullptr) { + return false; + } + + for (auto & impl : spec->impls) { + if (impl->get_state(seq_id, data)) { + return true; + } + } + + return false; +} + +void common_speculative_set_state(common_speculative * spec, llama_seq_id seq_id, const std::vector & data) { + if (spec == nullptr) { + return; + } + + for (auto & impl : spec->impls) { + impl->set_state(seq_id, data); + } +} + void common_speculative_print_stats(const common_speculative * spec) { if (spec == nullptr) { return; diff --git a/common/speculative.h b/common/speculative.h index bf76ad709..c58fac3cc 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -68,6 +68,10 @@ void common_speculative_draft(common_speculative * spec); // informs the speculative context that n_accepted tokens were accepted by the target model void common_speculative_accept(common_speculative * spec, llama_seq_id, uint16_t n_accepted); +// (optional) get/set internal state +bool common_speculative_get_state(common_speculative * spec, llama_seq_id seq_id, std::vector & data); +void common_speculative_set_state(common_speculative * spec, llama_seq_id seq_id, const std::vector & data); + // print statistics about the speculative decoding void common_speculative_print_stats(const common_speculative * spec); diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 6783d98ec..d8ffe43ae 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -156,6 +156,8 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para // MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass. for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = inpL; + ggml_tensor * inpSA = inpL; cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp index eb5e9a406..7b0876cbb 100644 --- a/src/models/qwen35moe.cpp +++ b/src/models/qwen35moe.cpp @@ -179,6 +179,8 @@ llama_model_qwen35moe::graph::graph(const llama_model & model, const llm_graph_p // MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass. for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = inpL; + ggml_tensor * inpSA = inpL; cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index a23b0405c..c887beb0a 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2172,6 +2172,8 @@ private: cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); cur.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + // stash the draft's speculative state with the checkpoint + common_speculative_get_state(spec.get(), slot.id, cur.data_spec); SLT_INF(slot, "created context checkpoint %d of %d (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", @@ -2998,6 +3000,8 @@ private: // restore the context checkpoint it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); it->load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + // restore the draft's speculative state + common_speculative_set_state(spec.get(), slot.id, it->data_spec); pos_next = std::min(pos_next, std::max(it->pos_min + 1, it->pos_max)); n_past = std::min(slot.prompt.tokens.size_up_to_pos(pos_next), (size_t) it->n_tokens); From e2e7a9b2d04e0404131e551243d4bd399ccfe606 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Fri, 19 Jun 2026 12:18:36 +0200 Subject: [PATCH 12/21] mtmd: several bug fixes (#24784) * mtmd: several bug fixes * fix build * fix gemma4ua * add sanity check in get_u32() * fix build (2) * area() avoid overflow --- tools/mtmd/clip.cpp | 30 +++++++- tools/mtmd/clip.h | 3 + tools/mtmd/mtmd-audio.cpp | 139 +++++++++++++++++++++----------------- tools/mtmd/mtmd-audio.h | 14 ++-- tools/mtmd/mtmd.cpp | 5 +- 5 files changed, 119 insertions(+), 72 deletions(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 17079815d..10840a851 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1675,6 +1675,9 @@ struct clip_model_loader { // note: some models having hparams.image_size == 0, which means the image size is dynamic throw std::runtime_error(string_format("%s: image_size (%d) cannot be negative\n", __func__, hparams.image_size)); } + if (hparams.image_size > 65536) { + throw std::runtime_error(string_format("%s: image_size (%d) is too large (max 65536)\n", __func__, hparams.image_size)); + } if (hparams.patch_size <= 0) { throw std::runtime_error(string_format("%s: patch_size (%d) must be greater than 0\n", __func__, hparams.patch_size)); } @@ -1723,6 +1726,19 @@ struct clip_model_loader { LOG_INF("%s: audio_n_fft: %d\n", __func__, hparams.audio_n_fft); LOG_INF("%s: audio_window_len: %d\n", __func__, hparams.audio_window_len); LOG_INF("%s: audio_hop_len: %d\n", __func__, hparams.audio_hop_len); + + // GEMMA4UA is encoder-free: it uses n_mel_bins as a raw-waveform frame size (640) and has no FFT/filterbank, so the mel-range and FFT + // checks below do not apply to it. + const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA; + + // Validate audio hparams loaded from GGUF metadata + if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) { + throw std::runtime_error(string_format("%s: n_mel_bins (%d) must be in range [1, 256]\n", __func__, hparams.n_mel_bins)); + } + if (fft_based && (hparams.audio_sample_rate <= 0 || hparams.audio_n_fft <= 0 || hparams.audio_hop_len <= 0 || hparams.audio_window_len <= 0)) { + throw std::runtime_error(string_format("%s: audio hparams invalid: sample_rate=%d n_fft=%d window_len=%d hop_len=%d\n", + __func__, hparams.audio_sample_rate, hparams.audio_n_fft, hparams.audio_window_len, hparams.audio_hop_len)); + } } LOG_INF("\n"); LOG_INF("%s: model size: %.2f MiB\n", __func__, model_size / 1024.0 / 1024.0); @@ -2831,6 +2847,12 @@ struct clip_model_loader { img.set_size({sz, sz}, false, false); LOG_INF("%s: warmup with image size = %d x %d\n", __func__, sz, sz); } else { + // GEMMA4UA uses n_mel_bins as a raw-waveform frame size (640), not a mel-bin count, + // so the [1, 256] bound only applies to FFT-based models. + const bool fft_based = ctx_clip.model.proj_type != PROJECTOR_TYPE_GEMMA4UA; + if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) { + throw std::runtime_error(string_format("%s: invalid n_mel_bins (%d), must be in [1, 256]\n", __func__, hparams.n_mel_bins)); + } img.set_size({hparams.warmup_audio_size, hparams.n_mel_bins}, false, false); LOG_INF("%s: warmup with audio size = %d\n", __func__, hparams.warmup_audio_size); } @@ -2994,7 +3016,13 @@ struct clip_model_loader { } return; } - output = gguf_get_val_u32(ctx_gguf.get(), i); + const uint32_t val = gguf_get_val_u32(ctx_gguf.get(), i); + // sanity check + if (val > (uint32_t) INT32_MAX) { + throw std::runtime_error(string_format("%s: value %u for key '%s' exceeds INT32_MAX\n", + __func__, val, key.c_str())); + } + output = (int) val; } void get_f32(const std::string & key, float & output, bool required = true) const { diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index f66f4bc3b..e0f1d298c 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -24,6 +24,9 @@ struct clip_image_size { return !(*this == other); } int area() const { + // avoid overflow when computing area + GGML_ASSERT(width >= 0 && width <= 46000); + GGML_ASSERT(height >= 0 && height <= 46000); return width * height; } }; diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index 13f211fd9..b72fd067a 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -32,8 +32,8 @@ void mtmd_audio_cache::fill_hann_window(uint32_t length, bool periodic) { } } -void mtmd_audio_cache::fill_mel_filterbank_matrix(int n_mel, - int n_fft, +void mtmd_audio_cache::fill_mel_filterbank_matrix(int64_t n_mel, + int64_t n_fft, int sample_rate, float fmin, float fmax, @@ -86,11 +86,16 @@ void mtmd_audio_cache::fill_mel_filterbank_matrix(int n_mel, hz_pts[i] = mel_to_hz(mel_pts[i]); } - const int n_fft_bins = n_fft / 2 + 1; + const int64_t n_fft_bins = n_fft / 2 + 1; + + // Validate allocation size + if ((size_t)n_mel * (size_t)n_fft_bins > SIZE_MAX) { + GGML_ASSERT(false && "mel filterbank allocation too large"); + } // filterbank - std::vector out(n_mel * n_fft_bins, 0); - for (int m = 0; m < n_mel; ++m) { + std::vector out((size_t)n_mel * (size_t)n_fft_bins, 0); + for (int64_t m = 0; m < n_mel; ++m) { const double f_left = hz_pts[m]; const double f_center = hz_pts[m + 1]; const double f_right = hz_pts[m + 2]; @@ -266,8 +271,8 @@ static void ifft(const mtmd_audio_cache & cache, float * in, int N, float * out) } struct filter_params { - int32_t n_mel; - int32_t n_fft_bins; + int64_t n_mel; + int64_t n_fft_bins; int32_t hann_window_size; int32_t hop_length; int32_t sample_rate; @@ -293,8 +298,8 @@ static void log_mel_spectrogram_worker_thread(int ith, std::vector fft_in(frame_size * 2, 0.0); std::vector fft_out(frame_size * 2 * 2 * 2); - int n_fft_bins = params.n_fft_bins; - int i = ith; + int64_t n_fft_bins = params.n_fft_bins; + int64_t i = ith; const auto & filters = cache.filters; @@ -302,17 +307,18 @@ static void log_mel_spectrogram_worker_thread(int ith, GGML_ASSERT(n_fft_bins == 1 + (frame_size / 2)); GGML_ASSERT(cache.sin_vals.size() == cache.cos_vals.size()); // calculate FFT only when fft_in are not all zero - for (; i < std::min(n_samples / frame_step + 1, out.n_len); i += n_threads) { - const int offset = i * frame_step; + for (; i < std::min((int64_t)(n_samples / frame_step + 1), out.n_len); i += n_threads) { + const int64_t offset = i * frame_step; // apply Hann window (~10% faster) - for (int j = 0; j < std::min(frame_size, n_samples - offset); j++) { + const int valid_len = std::min(frame_size, std::max(0, n_samples - (int)offset)); + for (int j = 0; j < valid_len; j++) { fft_in[j] = hann[j] * samples[offset + j]; } // fill the rest with zeros - if (n_samples - offset < frame_size) { - std::fill(fft_in.begin() + (n_samples - offset), fft_in.end(), 0.0); + if (valid_len < frame_size) { + std::fill(fft_in.begin() + valid_len, fft_in.end(), 0.0); } // FFT @@ -325,7 +331,7 @@ static void log_mel_spectrogram_worker_thread(int ith, } // mel spectrogram - for (int j = 0; j < out.n_mel; j++) { + for (int64_t j = 0; j < out.n_mel; j++) { double sum = 0.0; // unroll loop (suggested by GH user @lunixbochs) int k = 0; @@ -339,21 +345,21 @@ static void log_mel_spectrogram_worker_thread(int ith, } // handle n_fft remainder for (; k < n_fft_bins; k++) { - sum += fft_out[k] * filters.data[j * n_fft_bins + k]; + sum += fft_out[k] * filters.data[(size_t)j * n_fft_bins + k]; } sum = std::max(sum, (double)params.mel_floor); sum = params.use_natural_log ? log(sum) : log10(sum); - out.data[j * out.n_len + i] = sum; + out.data[(size_t)j * out.n_len + i] = sum; } } // Otherwise fft_out are all zero double sum = params.use_natural_log ? log(1e-10) : log10(1e-10); for (; i < out.n_len; i += n_threads) { - for (int j = 0; j < out.n_mel; j++) { - out.data[j * out.n_len + i] = sum; + for (int64_t j = 0; j < out.n_mel; j++) { + out.data[(size_t)j * out.n_len + i] = sum; } } } @@ -437,16 +443,21 @@ static bool log_mel_spectrogram( GGML_ASSERT(params.hop_length > 0); out.n_mel = params.n_mel; out.n_len = (n_samples - frame_size) / frame_step + 1; - // TODO: handle these checks better - if (out.n_mel > 0 && (unsigned long)out.n_len > SIZE_MAX / out.n_mel) { - LOG_ERR("%s: size overflow\n", __func__); + // Validate dimensions before allocation to prevent integer overflow + if (out.n_mel <= 0 || out.n_len <= 0) { + LOG_ERR("%s: invalid mel dimensions n_mel=%lld n_len=%lld\n", __func__, (long long)out.n_mel, (long long)out.n_len); + return false; + } + const size_t total_size = (size_t)out.n_mel * (size_t)out.n_len; + if (total_size > SIZE_MAX / sizeof(float)) { + LOG_ERR("%s: size overflow: n_mel=%lld n_len=%lld\n", __func__, (long long)out.n_mel, (long long)out.n_len); return false; } if (n_samples < frame_size) { LOG_ERR("%s: not enough samples after padding\n", __func__); return false; } - out.data.resize(out.n_mel * out.n_len); + out.data.resize(total_size); { std::vector workers(n_threads - 1); @@ -464,38 +475,39 @@ static bool log_mel_spectrogram( } } - const int effective_n_len = n_samples_in / frame_step; + const int64_t effective_n_len = n_samples_in / frame_step; if (params.norm_per_feature) { GGML_ASSERT(effective_n_len > 1); - for (int i = 0; i < out.n_mel; i++) { + for (int64_t i = 0; i < out.n_mel; i++) { double mean = 0; - for (int j = 0; j < effective_n_len; ++j) { - mean += out.data[i * out.n_len + j]; + for (int64_t j = 0; j < effective_n_len; ++j) { + mean += out.data[(size_t)i * out.n_len + j]; } mean /= effective_n_len; double var = 0.0; - for (int j = 0; j < effective_n_len; ++j) { - const double value = out.data[i * out.n_len + j] - mean; + for (int64_t j = 0; j < effective_n_len; ++j) { + const double value = out.data[(size_t)i * out.n_len + j] - mean; var += value * value; } var /= effective_n_len - 1; // unbiased const double mstd = std::sqrt(var + 1e-5); - for (int j = 0; j < effective_n_len; ++j) { - auto &value = out.data[i * out.n_len + j]; + for (int64_t j = 0; j < effective_n_len; ++j) { + auto &value = out.data[(size_t)i * out.n_len + j]; value = (value - mean) / mstd; } // pad the rest with zeros - for (int j = effective_n_len; j < out.n_len; ++j) { - out.data[i * out.n_len + j] = 0.0; + for (int64_t j = effective_n_len; j < out.n_len; ++j) { + out.data[(size_t)i * out.n_len + j] = 0.0; } } } else if (!params.no_padding) { // Whisper-style clamping and normalization (NOT used by Gemma4) double mmax = -1e20; - for (int i = 0; i < out.n_mel*out.n_len; i++) { + const size_t mel_size = (size_t)out.n_mel * (size_t)out.n_len; + for (size_t i = 0; i < mel_size; i++) { if (out.data[i] > mmax) { mmax = out.data[i]; } @@ -503,7 +515,7 @@ static bool log_mel_spectrogram( mmax -= 8.0; - for (int i = 0; i < out.n_mel*out.n_len; i++) { + for (size_t i = 0; i < mel_size; i++) { if (out.data[i] < mmax) { out.data[i] = mmax; } @@ -582,13 +594,13 @@ bool mtmd_audio_preprocessor_whisper::preprocess(const float * s // because the cgraph in clip.cpp only accepts 3000 frames each, we need to split the mel // we always expect the mel to have 3000 silent frames at the end if (DEBUG) { - printf("output: n_mel = %d, n_len = %d\n", out_full.n_mel, out_full.n_len); + printf("output: n_mel = %d, n_len = %d\n", (int) out_full.n_mel, (int) out_full.n_len); } const size_t frames_per_chunk = 3000; GGML_ASSERT((size_t) out_full.n_len > frames_per_chunk); for (size_t off = 0; off < (size_t) out_full.n_len; off += frames_per_chunk) { - int n_len = std::min(frames_per_chunk, (size_t) out_full.n_len - off); - if ((size_t) n_len < frames_per_chunk) { + int64_t n_len = std::min((int64_t)frames_per_chunk, out_full.n_len - (int64_t)off); + if (n_len < (int64_t)frames_per_chunk) { break; // last incomplete chunk will always be a padded chunk, safe to ignore } @@ -596,10 +608,10 @@ bool mtmd_audio_preprocessor_whisper::preprocess(const float * s out_chunk.n_len = n_len; out_chunk.n_mel = out_full.n_mel; out_chunk.n_len_org = out_full.n_mel; // unused - out_chunk.data.reserve(out_chunk.n_mel * out_chunk.n_len); + out_chunk.data.reserve((size_t)out_chunk.n_mel * (size_t)out_chunk.n_len); - for (int i = 0; i < out_full.n_mel; i++) { - auto src = out_full.data.begin() + i * out_full.n_len + off; + for (int64_t i = 0; i < out_full.n_mel; i++) { + auto src = out_full.data.begin() + (size_t)i * out_full.n_len + off; out_chunk.data.insert(out_chunk.data.end(), src, src + frames_per_chunk); } @@ -681,8 +693,8 @@ bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * sa // The effective frame count: center-padded STFT gives ~n_samples/hop_length frames. // We take min(mel_full.n_len, n_samples/hop + 1) to avoid including excess frames. - const int n_eff = std::min(mel_full.n_len, - (int)(n_samples / hparams.audio_hop_len) + 1); + const int64_t n_eff = std::min(mel_full.n_len, + (int64_t)(n_samples / hparams.audio_hop_len) + 1); // Split into inference windows matching n_window_infer=800 from model config. // Each window is padded to the next multiple of chunk_size for the cgraph. @@ -690,18 +702,18 @@ bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * sa const int chunk_size = 100; // conv sub-chunk size (n_window * 2, n_window=50) const int window_size = 800; // mel frames per forward pass (n_window_infer=800) - for (int off = 0; off < n_eff; off += window_size) { - const int win_eff = std::min(window_size, n_eff - off); - const int n_chunks = (win_eff + chunk_size - 1) / chunk_size; - const int n_padded = n_chunks * chunk_size; + for (int64_t off = 0; off < n_eff; off += window_size) { + const int64_t win_eff = std::min((int64_t)window_size, n_eff - off); + const int64_t n_chunks = (win_eff + chunk_size - 1) / chunk_size; + const int64_t n_padded = n_chunks * chunk_size; mtmd_audio_mel out; out.n_mel = mel_full.n_mel; out.n_len = n_padded; out.n_len_org = win_eff; - out.data.assign(out.n_mel * out.n_len, 0.0f); - for (int m = 0; m < out.n_mel; m++) { - const int copy_len = std::min(win_eff, mel_full.n_len - off); + out.data.assign((size_t)out.n_mel * (size_t)out.n_len, 0.0f); + for (int64_t m = 0; m < out.n_mel; m++) { + const int64_t copy_len = std::min((int64_t)win_eff, mel_full.n_len - off); if (copy_len > 0) { std::copy(mel_full.data.begin() + (size_t)m * mel_full.n_len + off, mel_full.data.begin() + (size_t)m * mel_full.n_len + off + copy_len, @@ -823,37 +835,38 @@ bool mtmd_audio_preprocessor_granite_speech::preprocess(const float * } double mmax = -1e20; - for (int i = 0; i < mel.n_mel * mel.n_len; i++) { + const size_t mel_size = (size_t)mel.n_mel * (size_t)mel.n_len; + for (size_t i = 0; i < mel_size; i++) { if (mel.data[i] > mmax) { mmax = mel.data[i]; } } mmax -= 8.0; - for (int i = 0; i < mel.n_mel * mel.n_len; i++) { + for (size_t i = 0; i < mel_size; i++) { if (mel.data[i] < mmax) { mel.data[i] = mmax; } mel.data[i] = (mel.data[i] + 4.0) / 4.0; } - int n_frames = mel.n_len; + int64_t n_frames = mel.n_len; if (n_frames % 2 == 1) { n_frames--; } - const int n_mel = mel.n_mel; - const int n_stacked = n_frames / 2; + const int64_t n_mel = mel.n_mel; + const int64_t n_stacked = n_frames / 2; mtmd_audio_mel stacked; stacked.n_mel = 2 * n_mel; stacked.n_len = n_stacked; - stacked.n_len_org = (int)n_samples; - stacked.data.resize(2 * n_mel * n_stacked); + stacked.n_len_org = (int64_t)n_samples; + stacked.data.resize((size_t)2 * (size_t)n_mel * (size_t)n_stacked); - for (int t = 0; t < n_stacked; t++) { - for (int m = 0; m < n_mel; m++) { - stacked.data[m * n_stacked + t] = mel.data[m * mel.n_len + 2 * t]; - stacked.data[(m + n_mel) * n_stacked + t] = mel.data[m * mel.n_len + 2 * t + 1]; + for (int64_t t = 0; t < n_stacked; t++) { + for (int64_t m = 0; m < n_mel; m++) { + stacked.data[(size_t)m * n_stacked + t] = mel.data[(size_t)m * mel.n_len + 2 * t]; + stacked.data[(size_t)(m + n_mel) * n_stacked + t] = mel.data[(size_t)m * mel.n_len + 2 * t + 1]; } } @@ -921,8 +934,8 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s const int hop = hparams.audio_hop_len; const int n_with_left = (int)chunk_len + pad_left; // PyTorch: unfold(size=frame_length+1, step=hop) on semicausal-padded waveform - const int pt_frames = (n_with_left - (hparams.audio_window_len + 1)) / hop + 1; - const int n_padded_needed = (pt_frames - 1) * hop + fft_size; + const int64_t pt_frames = (n_with_left - (hparams.audio_window_len + 1)) / hop + 1; + const int64_t n_padded_needed = (pt_frames - 1) * hop + fft_size; const int total_pad = std::max((int)(n_padded_needed - (int)chunk_len), pad_left); std::vector padded_samples(total_pad + chunk_len, 0.0f); std::copy(chunk_ptr, chunk_ptr + chunk_len, padded_samples.data() + pad_left); diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index 9656e3940..ad96bd847 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -10,16 +10,16 @@ #define MTMD_INTERNAL_HEADER struct mtmd_audio_mel { - int n_len; - int n_len_org; - int n_mel; + int64_t n_len; + int64_t n_len_org; + int64_t n_mel; std::vector data; }; struct mtmd_audio_mel_filters { - int32_t n_mel; - int32_t n_fft; + int64_t n_mel; + int64_t n_fft; std::vector data; }; @@ -39,8 +39,8 @@ struct mtmd_audio_cache { // Build mel filterbank matrix [n_mel × n_fft_bins] at runtime. // n_fft_bins must be (N_fft / 2 + 1). Example: if N_fft=512 -> n_fft_bins=257. - void fill_mel_filterbank_matrix(int n_mel, - int n_fft, + void fill_mel_filterbank_matrix(int64_t n_mel, + int64_t n_fft, int sample_rate, // e.g. 16000 float fmin = 0.0f, // e.g. 0.0 float fmax = -1.0f, // e.g. sr/2; pass -1 for auto diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index abba2ebf2..cbaac1d37 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1295,9 +1295,12 @@ struct mtmd_tokenizer { for (auto & mel_spec : mel_spec_chunks) { const bool is_placeholder = mel_spec.data.empty(); + // Validate dimensions fit in clip_image_size (int) + GGML_ASSERT(mel_spec.n_len <= INT32_MAX && mel_spec.n_len >= 0); + GGML_ASSERT(mel_spec.n_mel <= INT32_MAX && mel_spec.n_mel >= 0); clip_image_f32 mel_f32; mel_f32.set_size( - {mel_spec.n_len, mel_spec.n_mel}, + {(int)mel_spec.n_len, (int)mel_spec.n_mel}, is_placeholder, /* is_audio */ true); mel_f32.cpy_buf(mel_spec.data); From 38724ab5937f6440993d4e9814563385569d3b5a Mon Sep 17 00:00:00 2001 From: Aldehir Rojas Date: Fri, 19 Jun 2026 08:32:31 -0500 Subject: [PATCH 13/21] docker : build the UI (#24794) * docker : build the UI * cont : use existing APP_VERSION --- .devops/cann.Dockerfile | 16 ++++++++++++++++ .devops/cpu.Dockerfile | 16 ++++++++++++++++ .devops/cuda.Dockerfile | 16 ++++++++++++++++ .devops/intel.Dockerfile | 16 ++++++++++++++++ .devops/musa.Dockerfile | 16 ++++++++++++++++ .devops/openvino.Dockerfile | 16 ++++++++++++++++ .devops/rocm.Dockerfile | 16 ++++++++++++++++ .devops/s390x.Dockerfile | 16 ++++++++++++++++ .devops/vulkan.Dockerfile | 16 ++++++++++++++++ .devops/zendnn.Dockerfile | 16 ++++++++++++++++ .dockerignore | 3 +++ 11 files changed, 163 insertions(+) diff --git a/.devops/cann.Dockerfile b/.devops/cann.Dockerfile index 9df86d048..dc95e3f38 100644 --- a/.devops/cann.Dockerfile +++ b/.devops/cann.Dockerfile @@ -13,6 +13,20 @@ ARG APP_REVISION=N/A # BUILD STAGE # Compile all binary files and libraries # ============================================================================== +ARG NODE_VERSION=24 + +FROM docker.io/node:$NODE_VERSION AS web + +ARG APP_VERSION + +WORKDIR /app/tools/ui + +COPY tools/ui/package.json tools/ui/package-lock.json ./ +RUN npm ci + +COPY tools/ui/ ./ +RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build + FROM ${CANN_BASE_IMAGE} AS build # -- Install build dependencies -- @@ -26,6 +40,8 @@ WORKDIR /app # -- Copy project files -- COPY . . +COPY --from=web /app/tools/ui/dist tools/ui/dist + # -- Set CANN environment variables (required for compilation) -- # Using ENV instead of `source` allows environment variables to persist across the entire image layer ENV ASCEND_TOOLKIT_HOME=/usr/local/Ascend/ascend-toolkit/latest diff --git a/.devops/cpu.Dockerfile b/.devops/cpu.Dockerfile index 9dbfdd11d..caf727bcd 100644 --- a/.devops/cpu.Dockerfile +++ b/.devops/cpu.Dockerfile @@ -3,6 +3,20 @@ ARG BUILD_DATE=N/A ARG APP_VERSION=N/A ARG APP_REVISION=N/A +ARG NODE_VERSION=24 + +FROM docker.io/node:$NODE_VERSION AS web + +ARG APP_VERSION + +WORKDIR /app/tools/ui + +COPY tools/ui/package.json tools/ui/package-lock.json ./ +RUN npm ci + +COPY tools/ui/ ./ +RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build + FROM docker.io/ubuntu:$UBUNTU_VERSION AS build ARG TARGETARCH @@ -16,6 +30,8 @@ WORKDIR /app COPY . . +COPY --from=web /app/tools/ui/dist tools/ui/dist + RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON; \ else \ diff --git a/.devops/cuda.Dockerfile b/.devops/cuda.Dockerfile index 276f82c34..b16b9a8f1 100644 --- a/.devops/cuda.Dockerfile +++ b/.devops/cuda.Dockerfile @@ -11,6 +11,20 @@ ARG BUILD_DATE=N/A ARG APP_VERSION=N/A ARG APP_REVISION=N/A +ARG NODE_VERSION=24 + +FROM docker.io/node:$NODE_VERSION AS web + +ARG APP_VERSION + +WORKDIR /app/tools/ui + +COPY tools/ui/package.json tools/ui/package-lock.json ./ +RUN npm ci + +COPY tools/ui/ ./ +RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build + FROM ${BASE_CUDA_DEV_CONTAINER} AS build ARG GCC_VERSION @@ -26,6 +40,8 @@ WORKDIR /app COPY . . +COPY --from=web /app/tools/ui/dist tools/ui/dist + RUN if [ "${CUDA_DOCKER_ARCH}" != "default" ]; then \ export CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CUDA_DOCKER_ARCH}"; \ fi && \ diff --git a/.devops/intel.Dockerfile b/.devops/intel.Dockerfile index 4d0c0a8fd..3c059eb30 100644 --- a/.devops/intel.Dockerfile +++ b/.devops/intel.Dockerfile @@ -5,6 +5,20 @@ ARG APP_REVISION=N/A ## Build Image +ARG NODE_VERSION=24 + +FROM docker.io/node:$NODE_VERSION AS web + +ARG APP_VERSION + +WORKDIR /app/tools/ui + +COPY tools/ui/package.json tools/ui/package-lock.json ./ +RUN npm ci + +COPY tools/ui/ ./ +RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build + FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build ARG GGML_SYCL_F16=ON @@ -22,6 +36,8 @@ WORKDIR /app COPY . . +COPY --from=web /app/tools/ui/dist tools/ui/dist + RUN if [ "${GGML_SYCL_F16}" = "ON" ]; then \ echo "GGML_SYCL_F16 is set" \ && export OPT_SYCL_F16="-DGGML_SYCL_F16=ON" \ diff --git a/.devops/musa.Dockerfile b/.devops/musa.Dockerfile index c98c44c95..0c23cc554 100644 --- a/.devops/musa.Dockerfile +++ b/.devops/musa.Dockerfile @@ -10,6 +10,20 @@ ARG BUILD_DATE=N/A ARG APP_VERSION=N/A ARG APP_REVISION=N/A +ARG NODE_VERSION=24 + +FROM docker.io/node:$NODE_VERSION AS web + +ARG APP_VERSION + +WORKDIR /app/tools/ui + +COPY tools/ui/package.json tools/ui/package-lock.json ./ +RUN npm ci + +COPY tools/ui/ ./ +RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build + FROM ${BASE_MUSA_DEV_CONTAINER} AS build # MUSA architecture to build for (defaults to all supported archs) @@ -29,6 +43,8 @@ WORKDIR /app COPY . . +COPY --from=web /app/tools/ui/dist tools/ui/dist + RUN if [ "${MUSA_DOCKER_ARCH}" != "default" ]; then \ export CMAKE_ARGS="-DMUSA_ARCHITECTURES=${MUSA_DOCKER_ARCH}"; \ fi && \ diff --git a/.devops/openvino.Dockerfile b/.devops/openvino.Dockerfile index 9e96244ce..fec72b1c7 100644 --- a/.devops/openvino.Dockerfile +++ b/.devops/openvino.Dockerfile @@ -22,6 +22,20 @@ ARG BUILD_DATE=N/A ARG APP_VERSION=N/A ARG APP_REVISION=N/A +ARG NODE_VERSION=24 + +FROM docker.io/node:$NODE_VERSION AS web + +ARG APP_VERSION + +WORKDIR /app/tools/ui + +COPY tools/ui/package.json tools/ui/package-lock.json ./ +RUN npm ci + +COPY tools/ui/ ./ +RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build + ## Build Image FROM docker.io/ubuntu:${UBUNTU_VERSION} AS build @@ -69,6 +83,8 @@ WORKDIR /app COPY . . +COPY --from=web /app/tools/ui/dist tools/ui/dist + # Build Stage RUN bash -c "source ${OpenVINO_DIR}/setupvars.sh && \ cmake -B build/ReleaseOV -G Ninja \ diff --git a/.devops/rocm.Dockerfile b/.devops/rocm.Dockerfile index 2ab10f411..7fad0c22e 100644 --- a/.devops/rocm.Dockerfile +++ b/.devops/rocm.Dockerfile @@ -11,6 +11,20 @@ ARG BUILD_DATE=N/A ARG APP_VERSION=N/A ARG APP_REVISION=N/A +ARG NODE_VERSION=24 + +FROM docker.io/node:$NODE_VERSION AS web + +ARG APP_VERSION + +WORKDIR /app/tools/ui + +COPY tools/ui/package.json tools/ui/package-lock.json ./ +RUN npm ci + +COPY tools/ui/ ./ +RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build + ### Build image FROM ${BASE_ROCM_DEV_CONTAINER} AS build @@ -38,6 +52,8 @@ WORKDIR /app COPY . . +COPY --from=web /app/tools/ui/dist tools/ui/dist + RUN HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \ cmake -S . -B build \ -DGGML_HIP=ON \ diff --git a/.devops/s390x.Dockerfile b/.devops/s390x.Dockerfile index d88dd2d92..149d79a61 100644 --- a/.devops/s390x.Dockerfile +++ b/.devops/s390x.Dockerfile @@ -4,6 +4,20 @@ ARG BUILD_DATE=N/A ARG APP_VERSION=N/A ARG APP_REVISION=N/A +ARG NODE_VERSION=24 + +FROM docker.io/node:$NODE_VERSION AS web + +ARG APP_VERSION + +WORKDIR /app/tools/ui + +COPY tools/ui/package.json tools/ui/package-lock.json ./ +RUN npm ci + +COPY tools/ui/ ./ +RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build + ### Build Llama.cpp stage FROM docker.io/gcc:${GCC_VERSION} AS build @@ -20,6 +34,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ WORKDIR /app COPY . . +COPY --from=web /app/tools/ui/dist tools/ui/dist + RUN --mount=type=cache,target=/root/.ccache \ --mount=type=cache,target=/app/build \ cmake -S . -B build -G Ninja \ diff --git a/.devops/vulkan.Dockerfile b/.devops/vulkan.Dockerfile index 05df94ec4..26c1902b1 100644 --- a/.devops/vulkan.Dockerfile +++ b/.devops/vulkan.Dockerfile @@ -3,6 +3,20 @@ ARG BUILD_DATE=N/A ARG APP_VERSION=N/A ARG APP_REVISION=N/A +ARG NODE_VERSION=24 + +FROM docker.io/node:$NODE_VERSION AS web + +ARG APP_VERSION + +WORKDIR /app/tools/ui + +COPY tools/ui/package.json tools/ui/package-lock.json ./ +RUN npm ci + +COPY tools/ui/ ./ +RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build + FROM docker.io/ubuntu:$UBUNTU_VERSION AS build # Install build tools @@ -17,6 +31,8 @@ WORKDIR /app COPY . . +COPY --from=web /app/tools/ui/dist tools/ui/dist + RUN cmake -B build -DGGML_NATIVE=OFF -DGGML_VULKAN=ON -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON && \ cmake --build build --config Release -j$(nproc) diff --git a/.devops/zendnn.Dockerfile b/.devops/zendnn.Dockerfile index 9f811ab27..80daf5671 100644 --- a/.devops/zendnn.Dockerfile +++ b/.devops/zendnn.Dockerfile @@ -3,6 +3,20 @@ ARG BUILD_DATE=N/A ARG APP_VERSION=N/A ARG APP_REVISION=N/A +ARG NODE_VERSION=24 + +FROM docker.io/node:$NODE_VERSION AS web + +ARG APP_VERSION + +WORKDIR /app/tools/ui + +COPY tools/ui/package.json tools/ui/package-lock.json ./ +RUN npm ci + +COPY tools/ui/ ./ +RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build + FROM docker.io/ubuntu:$UBUNTU_VERSION AS build RUN apt-get update && \ @@ -14,6 +28,8 @@ WORKDIR /app COPY . . +COPY --from=web /app/tools/ui/dist tools/ui/dist + RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_ZENDNN=ON && \ cmake --build build -j $(nproc) diff --git a/.dockerignore b/.dockerignore index 064b7c7be..a223b7e89 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,9 @@ build*/ +tools/ui/node_modules/ +tools/ui/dist/ + models/* /llama-cli From 8c2d6f6475f2586e079fa6677dec91088de85604 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Fri, 19 Jun 2026 16:06:13 +0200 Subject: [PATCH 14/21] server: add --agent arg, remove redundant webui naming compat (#24801) * server: add --agent arg, remove redundant webui naming compat * corrent env * fix the test * llama-gen-docs * nits: wordings --- common/arg.cpp | 70 +++++++++------------------------ common/common.h | 6 --- tools/cli/README.md | 3 +- tools/server/README.md | 18 ++++----- tools/server/server-context.cpp | 15 +++---- tools/server/server-models.cpp | 15 ++++--- tools/server/server-models.h | 5 +-- tools/server/server.cpp | 3 +- 8 files changed, 43 insertions(+), 92 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index bd4b113d1..52425f25e 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2830,62 +2830,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.api_prefix = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_API_PREFIX")); - // Deprecated: use --ui-config instead (kept for backward compat) add_opt(common_arg( - {"--webui-config"}, "JSON", - "[DEPRECATED: use --ui-config] JSON that provides default WebUI settings (overrides WebUI defaults)", - [](common_params & params, const std::string & value) { - params.ui_config_json = value; - params.webui_config_json = value; - } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI_CONFIG")); - - add_opt(common_arg( - {"--ui-config"}, "JSON", + {"--ui-config", "--webui-config"}, "JSON", "JSON that provides default UI settings (overrides UI defaults)", [](common_params & params, const std::string & value) { params.ui_config_json = value; - params.webui_config_json = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_CONFIG")); - - // Deprecated: use --ui-config-file instead (kept for backward compat) add_opt(common_arg( - {"--webui-config-file"}, "PATH", - "[DEPRECATED: use --ui-config-file] JSON file that provides default WebUI settings (overrides WebUI defaults)", - [](common_params & params, const std::string & value) { - params.ui_config_json = read_file(value); - params.webui_config_json = params.ui_config_json; - } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI_CONFIG_FILE")); - - add_opt(common_arg( - {"--ui-config-file"}, "PATH", + {"--ui-config-file", "--webui-config-file"}, "PATH", "JSON file that provides default UI settings (overrides UI defaults)", [](common_params & params, const std::string & value) { params.ui_config_json = read_file(value); - params.webui_config_json = params.ui_config_json; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_CONFIG_FILE")); - - // Deprecated: use --ui-mcp-proxy instead (kept for backward compat) add_opt(common_arg( - {"--webui-mcp-proxy"}, - {"--no-webui-mcp-proxy"}, - "[DEPRECATED: use --ui-mcp-proxy/--no-ui-mcp-proxy] experimental: whether to enable MCP CORS proxy", - [](common_params & params, bool value) { - params.ui_mcp_proxy = value; - params.webui_mcp_proxy = value; - } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI_MCP_PROXY")); - - add_opt(common_arg( - {"--ui-mcp-proxy"}, - {"--no-ui-mcp-proxy"}, + {"--ui-mcp-proxy", "--webui-mcp-proxy"}, + {"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"}, "experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)", [](common_params & params, bool value) { params.ui_mcp_proxy = value; - params.webui_mcp_proxy = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_MCP_PROXY")); add_opt(common_arg( @@ -2897,24 +2861,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.server_tools = parse_csv_row(value); } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS")); - // Deprecated: use --ui/--no-ui instead (kept for backward compat) - add_opt(common_arg( - {"--webui"}, - {"--no-webui"}, - "[DEPRECATED: use --ui/--no-ui] whether to enable the Web UI", + add_opt(common_arg( + {"-ag", "--agent"}, + {"-no-ag", "--no-agent"}, + "whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)", [](common_params & params, bool value) { - params.ui = value; - params.webui = value; + if (value) { + params.server_tools = {"all"}; + params.ui_mcp_proxy = true; + } else { + params.server_tools.clear(); + params.ui_mcp_proxy = false; + } } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI")); - + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_AGENT")); add_opt(common_arg( - {"--ui"}, - {"--no-ui"}, + {"--ui", "--webui"}, + {"--no-ui", "--no-webui"}, string_format("whether to enable the Web UI (default: %s)", params.ui ? "enabled" : "disabled"), [](common_params & params, bool value) { params.ui = value; - params.webui = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI")); add_opt(common_arg( diff --git a/common/common.h b/common/common.h index 535a4ed33..44c605189 100644 --- a/common/common.h +++ b/common/common.h @@ -624,12 +624,6 @@ struct common_params { // UI configs bool ui = true; - - // Deprecated: use ui, ui_mcp_proxy, ui_config_json instead - bool webui = ui; - bool webui_mcp_proxy = false; - std::string webui_config_json; - bool ui_mcp_proxy = false; std::string ui_config_json; diff --git a/tools/cli/README.md b/tools/cli/README.md index b11aa45ce..f93ae914c 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -161,7 +161,7 @@ | `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md
(env: LLAMA_ARG_MMPROJ_URL) | | `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)
(env: LLAMA_ARG_MMPROJ_AUTO) | | `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)
(env: LLAMA_ARG_MMPROJ_OFFLOAD) | -| `--image, --audio FILE` | path to an image or audio file. use with multimodal models, use comma-separated values for multiple files | +| `--image, --audio, --video FILE` | path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files | | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MIN_TOKENS) | | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MAX_TOKENS) | | `--chat-template-kwargs STRING` | sets additional params for the json template parser, must be a valid json object string, e.g. '{"key1":"value1","key2":"value2"}'
(env: LLAMA_ARG_CHAT_TEMPLATE_KWARGS) | @@ -174,6 +174,7 @@ | `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) | | `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)
(env: LLAMA_ARG_SKIP_CHAT_PARSING) | | `--simple-io` | use basic IO for better compatibility in subprocesses and limited consoles | +| `--log-prompts-dir PATH` | Log prompts to directory (only used for debugging, default: disabled) | | `--spec-draft-hf, -hfd, -hfrd, --hf-repo-draft /[:quant]` | Same as --hf-repo, but for the draft model (default: unused)
(env: LLAMA_ARG_SPEC_DRAFT_HF_REPO) | | `--spec-draft-threads, -td, --threads-draft N` | number of threads to use during generation (default: same as --threads) | | `--spec-draft-threads-batch, -tbd, --threads-batch-draft N` | number of threads to use during batch and prompt processing (default: same as --threads-draft) | diff --git a/tools/server/README.md b/tools/server/README.md index 88a507e2c..1f74ba52a 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -175,13 +175,12 @@ For the full list of features, please refer to [server's changelog](https://gith | `-np, --parallel N` | number of server slots (default: -1, -1 = auto)
(env: LLAMA_ARG_N_PARALLEL) | | `-cb, --cont-batching, -nocb, --no-cont-batching` | whether to enable continuous batching (a.k.a dynamic batching) (default: enabled)
(env: LLAMA_ARG_CONT_BATCHING) | | `-mm, --mmproj FILE` | path to a multimodal projector file. see tools/mtmd/README.md
note: if -hf is used, this argument can be omitted
(env: LLAMA_ARG_MMPROJ) | -| `-tk, --talker-model FILE` | path to the qwen3-omni talker gguf, enables the /v1/audio/speech endpoint
(env: LLAMA_ARG_TALKER_MODEL) | -| `-c2w, --code2wav-model FILE` | path to the qwen3-omni code2wav gguf, the talker code detokenizer
(env: LLAMA_ARG_CODE2WAV_MODEL) | | `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md
(env: LLAMA_ARG_MMPROJ_URL) | | `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)
(env: LLAMA_ARG_MMPROJ_AUTO) | | `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)
(env: LLAMA_ARG_MMPROJ_OFFLOAD) | | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MIN_TOKENS) | | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MAX_TOKENS) | +| `--mtmd-batch-max-tokens N` | maximum number of image tokens per batch when encoding images (default: 1024)
(env: LLAMA_ARG_MTMD_BATCH_MAX_TOKENS) | | `-a, --alias STRING` | set model name aliases, comma-separated (to be used by API)
(env: LLAMA_ARG_ALIAS) | | `--tags STRING` | set model tags, comma-separated (informational, not used for routing)
(env: LLAMA_ARG_TAGS) | | `--embd-normalize N` | normalisation for embeddings (default: 2) (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm) | @@ -190,15 +189,12 @@ For the full list of features, please refer to [server's changelog](https://gith | `--reuse-port` | allow multiple sockets to bind to the same port (default: disabled)
(env: LLAMA_ARG_REUSE_PORT) | | `--path PATH` | path to serve static files from (default: )
(env: LLAMA_ARG_STATIC_PATH) | | `--api-prefix PREFIX` | prefix path the server serves from, without the trailing slash (default: )
(env: LLAMA_ARG_API_PREFIX) | -| `--webui-config JSON` | [DEPRECATED: use --ui-config] JSON that provides default WebUI settings (overrides WebUI defaults)
(env: LLAMA_ARG_WEBUI_CONFIG) | -| `--ui-config JSON` | JSON that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG) | -| `--webui-config-file PATH` | [DEPRECATED: use --ui-config-file] JSON file that provides default WebUI settings (overrides WebUI defaults)
(env: LLAMA_ARG_WEBUI_CONFIG_FILE) | -| `--ui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG_FILE) | -| `--webui-mcp-proxy, --no-webui-mcp-proxy` | [DEPRECATED: use --ui-mcp-proxy/--no-ui-mcp-proxy] experimental: whether to enable MCP CORS proxy
(env: LLAMA_ARG_WEBUI_MCP_PROXY) | -| `--ui-mcp-proxy, --no-ui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)
(env: LLAMA_ARG_UI_MCP_PROXY) | +| `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG) | +| `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG_FILE) | +| `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)
(env: LLAMA_ARG_UI_MCP_PROXY) | | `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, apply_diff, get_datetime
(env: LLAMA_ARG_TOOLS) | -| `--webui, --no-webui` | [DEPRECATED: use --ui/--no-ui] whether to enable the Web UI
(env: LLAMA_ARG_WEBUI) | -| `--ui, --no-ui` | whether to enable the Web UI (default: enabled)
(env: LLAMA_ARG_UI) | +| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)
(env: LLAMA_ARG_AGENT) | +| `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)
(env: LLAMA_ARG_UI) | | `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)
(env: LLAMA_ARG_EMBEDDINGS) | | `--rerank, --reranking` | enable reranking endpoint on server (default: disabled)
(env: LLAMA_ARG_RERANKING) | | `--api-key KEY` | API key to use for authentication, multiple keys can be provided as a comma-separated list (default: none)
(env: LLAMA_API_KEY) | @@ -207,6 +203,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--ssl-cert-file FNAME` | path to file a PEM-encoded SSL certificate
(env: LLAMA_ARG_SSL_CERT_FILE) | | `--chat-template-kwargs STRING` | sets additional params for the json template parser, must be a valid json object string, e.g. '{"key1":"value1","key2":"value2"}'
(env: LLAMA_ARG_CHAT_TEMPLATE_KWARGS) | | `-to, --timeout N` | server read/write timeout in seconds (default: 3600)
(env: LLAMA_ARG_TIMEOUT) | +| `--sse-ping-interval N` | server SSE ping interval in seconds (-1 = disabled, default: 30)
(env: LLAMA_ARG_SSE_PING_INTERVAL) | | `--threads-http N` | number of threads used to process HTTP requests (default: -1)
(env: LLAMA_ARG_THREADS_HTTP) | | `--cache-prompt, --no-cache-prompt` | whether to enable prompt caching (default: enabled)
(env: LLAMA_ARG_CACHE_PROMPT) | | `--cache-reuse N` | min chunk size to attempt reusing from the cache via KV shifting, requires prompt caching to be enabled (default: 0)
[(card)](https://ggml.ai/f0.png)
(env: LLAMA_ARG_CACHE_REUSE) | @@ -231,6 +228,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `-sps, --slot-prompt-similarity SIMILARITY` | how much the prompt of a request must match the prompt of a slot in order to use that slot (default: 0.10, 0.0 = disabled) | | `--lora-init-without-apply` | load LoRA adapters without applying them (apply later via POST /lora-adapters) (default: disabled) | | `--sleep-idle-seconds SECONDS` | number of seconds of idleness after which the server will sleep (default: -1; -1 = disabled) | +| `--log-prompts-dir PATH` | Log prompts to directory (only used for debugging, default: disabled) | | `--spec-draft-hf, -hfd, -hfrd, --hf-repo-draft /[:quant]` | Same as --hf-repo, but for the draft model (default: unused)
(env: LLAMA_ARG_SPEC_DRAFT_HF_REPO) | | `--spec-draft-threads, -td, --threads-draft N` | number of threads to use during generation (default: same as --threads) | | `--spec-draft-threads-batch, -tbd, --threads-batch-draft N` | number of threads to use during batch and prompt processing (default: same as --threads-draft) | diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index c887beb0a..00ab31340 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1302,11 +1302,8 @@ private: } } - // populate UI settings (from either new ui_config_json or deprecated webui_config_json) { - const std::string & cfg = !params_base.ui_config_json.empty() - ? params_base.ui_config_json - : params_base.webui_config_json; + const std::string & cfg = params_base.ui_config_json; if (!cfg.empty()) { try { json json_settings = json::parse(cfg); @@ -4304,18 +4301,18 @@ void server_routes::init_routes() { { "endpoint_props", params.endpoint_props }, { "endpoint_metrics", params.endpoint_metrics }, // New keys - { "ui", params.ui }, - { "ui_settings", meta->json_ui_settings }, + { "ui", params.ui }, + { "ui_settings", meta->json_ui_settings }, // Deprecated: use ui/ui_settings instead (kept for backward compat) - { "webui", params.webui }, - { "webui_settings", meta->json_webui_settings }, + { "webui", params.ui }, + { "webui_settings", meta->json_ui_settings }, { "chat_template", tmpl_default }, { "chat_template_caps", meta->chat_template_caps }, { "bos_token", meta->bos_token_str }, { "eos_token", meta->eos_token_str }, { "build_info", meta->build_info }, { "is_sleeping", queue_tasks.is_sleeping() }, - { "cors_proxy_enabled", params.ui_mcp_proxy || params.webui_mcp_proxy }, + { "cors_proxy_enabled", params.ui_mcp_proxy }, }; if (params.use_jinja) { if (!tmpl_tools.empty()) { diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 7aaad6926..23c1f1668 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1462,9 +1462,9 @@ void server_models_routes::init_routes() { auto res = std::make_unique(); res_ok(res, { // TODO: add support for this on web UI - {"role", "router"}, - {"max_instances", params.models_max}, - {"models_autoload", params.models_autoload}, + {"role", "router"}, + {"max_instances", params.models_max}, + {"models_autoload", params.models_autoload}, // this is a dummy response to make sure the UI doesn't break {"model_alias", "llama-server"}, {"model_path", "none"}, @@ -1473,11 +1473,10 @@ void server_models_routes::init_routes() { {"n_ctx", 0}, }}, // New key - {"ui_settings", ui_settings}, - // Deprecated: use ui_settings instead (kept for backward compat) - {"webui_settings", webui_settings}, - {"build_info", std::string(llama_build_info())}, - {"cors_proxy_enabled", params.ui_mcp_proxy || params.webui_mcp_proxy}, + {"ui_settings", ui_settings}, + {"webui_settings", webui_settings}, + {"build_info", std::string(llama_build_info())}, + {"cors_proxy_enabled", params.ui_mcp_proxy}, }); return res; } diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 319c4352e..aeb0e874d 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -212,10 +212,7 @@ struct server_models_routes { server_models models; server_models_routes(const common_params & params, int argc, char ** argv) : params(params), models(params, argc, argv) { - // Support both new ui_config_json and deprecated webui_config_json - const std::string & cfg = !this->params.ui_config_json.empty() - ? this->params.ui_config_json - : this->params.webui_config_json; + const std::string & cfg = this->params.ui_config_json; if (!cfg.empty()) { try { json json_settings = json::parse(cfg); diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 78ab0318c..2a67bfcfe 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -227,8 +227,7 @@ int llama_server(int argc, char ** argv) { ctx_http.register_gcp_compat(); // CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP) - // Supports both new ui_mcp_proxy and deprecated webui_mcp_proxy fields - if (params.ui_mcp_proxy || params.webui_mcp_proxy) { + if (params.ui_mcp_proxy) { SRV_WRN("%s", "-----------------\n"); SRV_WRN("%s", "CORS proxy is enabled, do not expose server to untrusted environments\n"); SRV_WRN("%s", "This feature is EXPERIMENTAL and may be removed or changed in future versions\n"); From 0d2d9ccbf6aae92de310712297fd52becc134092 Mon Sep 17 00:00:00 2001 From: "Alessandro de Oliveira Faria (A.K.A.CABELO)" Date: Fri, 19 Jun 2026 11:16:35 -0300 Subject: [PATCH 15/21] vendor : update cpp-httplib to 0.48.0 (#24787) --- scripts/sync_vendor.py | 2 +- vendor/cpp-httplib/httplib.cpp | 241 +++++++++++++-------------------- vendor/cpp-httplib/httplib.h | 81 ++++++++--- 3 files changed, 160 insertions(+), 164 deletions(-) diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.py index 402d7bbad..f913b0c7d 100755 --- a/scripts/sync_vendor.py +++ b/scripts/sync_vendor.py @@ -5,7 +5,7 @@ import os import sys import subprocess -HTTPLIB_VERSION = "refs/tags/v0.47.0" +HTTPLIB_VERSION = "refs/tags/v0.48.0" vendor = { "https://github.com/nlohmann/json/releases/latest/download/json.hpp": "vendor/nlohmann/json.hpp", diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp index 370c0e798..1ac4fa4ba 100644 --- a/vendor/cpp-httplib/httplib.cpp +++ b/vendor/cpp-httplib/httplib.cpp @@ -5809,11 +5809,9 @@ std::string decode_query_component(const std::string &component, for (size_t i = 0; i < component.size(); i++) { if (component[i] == '%' && i + 2 < component.size()) { - std::string hex = component.substr(i + 1, 2); - char *end; - unsigned long value = std::strtoul(hex.c_str(), &end, 16); - if (end == hex.c_str() + 2) { - result += static_cast(value); + auto val = 0; + if (detail::from_hex_to_i(component, i + 1, 2, val)) { + result += static_cast(val); i += 2; } else { result += component[i]; @@ -12551,6 +12549,21 @@ bool parse_ipv4(const std::string &str, unsigned char *out) { return *p == '\0'; } +// Parse an IP literal (IPv4 or IPv6) into raw network-order bytes. +// `out` must have room for at least 16 bytes. Returns the address length +// (4 for IPv4, 16 for IPv6) on success, or 0 if the string is not an IP +// literal. Used to match a host against iPAddress SANs the same way the +// OpenSSL backend does via X509_check_ip. +size_t parse_ip_address(const std::string &str, unsigned char *out) { + if (is_ipv4_address(str)) { return parse_ipv4(str, out) ? 4 : 0; } + struct in6_addr addr6 = {}; + if (inet_pton(AF_INET6, str.c_str(), &addr6) == 1) { + memcpy(out, &addr6, 16); + return 16; + } + return 0; +} + #ifdef _WIN32 // Enumerate Windows system certificates and call callback with DER data template @@ -12852,6 +12865,30 @@ int openssl_verify_callback(int preverify_ok, X509_STORE_CTX *ctx) { return callback(verify_ctx) ? 1 : 0; } +// X509_STORE_get0_objects is deprecated since OpenSSL 4.0 because it is not +// thread-safe; X509_STORE_get1_objects (OpenSSL 3.3+) returns a snapshot +// that must be released with release_store_objects +#if !defined(OPENSSL_IS_BORINGSSL) && !defined(LIBRESSL_VERSION_NUMBER) && \ + OPENSSL_VERSION_NUMBER >= 0x30300000L +#define CPPHTTPLIB_HAS_X509_STORE_GET1_OBJECTS +#endif + +STACK_OF(X509_OBJECT) * get_store_objects(X509_STORE *store) { +#ifdef CPPHTTPLIB_HAS_X509_STORE_GET1_OBJECTS + return X509_STORE_get1_objects(store); +#else + return X509_STORE_get0_objects(store); +#endif +} + +void release_store_objects(STACK_OF(X509_OBJECT) * objs) { +#ifdef CPPHTTPLIB_HAS_X509_STORE_GET1_OBJECTS + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); +#else + (void)objs; // get0 variant returns an internal pointer; nothing to free +#endif +} + } // namespace impl ctx_t create_client_context() { @@ -13373,11 +13410,19 @@ std::string get_cert_subject_cn(cert_t cert) { auto subject_name = X509_get_subject_name(x509); if (!subject_name) return ""; - char buf[256]; - auto len = - X509_NAME_get_text_by_NID(subject_name, NID_commonName, buf, sizeof(buf)); - if (len < 0) return ""; - return std::string(buf, static_cast(len)); + // X509_NAME_get_text_by_NID is deprecated since OpenSSL 4.0 + auto idx = X509_NAME_get_index_by_NID(subject_name, NID_commonName, -1); + if (idx < 0) return ""; + + auto entry = X509_NAME_get_entry(subject_name, idx); + if (!entry) return ""; + + auto data = X509_NAME_ENTRY_get_data(entry); + if (!data) return ""; + + return std::string( + reinterpret_cast(ASN1_STRING_get0_data(data)), + static_cast(ASN1_STRING_length(data))); } std::string get_cert_issuer_name(cert_t cert) { @@ -13582,8 +13627,9 @@ size_t get_ca_certs(ctx_t ctx, std::vector &certs) { auto store = SSL_CTX_get_cert_store(ssl_ctx); if (!store) { return 0; } - auto objs = X509_STORE_get0_objects(store); + auto objs = impl::get_store_objects(store); if (!objs) { return 0; } + auto se = detail::scope_exit([&] { impl::release_store_objects(objs); }); auto count = sk_X509_OBJECT_num(objs); for (decltype(count) i = 0; i < count; i++) { @@ -13609,8 +13655,9 @@ std::vector get_ca_names(ctx_t ctx) { auto store = SSL_CTX_get_cert_store(ssl_ctx); if (!store) { return names; } - auto objs = X509_STORE_get0_objects(store); + auto objs = impl::get_store_objects(store); if (!objs) { return names; } + auto se = detail::scope_exit([&] { impl::release_store_objects(objs); }); auto count = sk_X509_OBJECT_num(objs); for (decltype(count) i = 0; i < count; i++) { @@ -13716,110 +13763,6 @@ std::string verify_error_string(long error_code) { } // namespace tls -bool SSLClient::verify_host(X509 *server_cert) const { - /* Quote from RFC2818 section 3.1 "Server Identity" - - If a subjectAltName extension of type dNSName is present, that MUST - be used as the identity. Otherwise, the (most specific) Common Name - field in the Subject field of the certificate MUST be used. Although - the use of the Common Name is existing practice, it is deprecated and - Certification Authorities are encouraged to use the dNSName instead. - - Matching is performed using the matching rules specified by - [RFC2459]. If more than one identity of a given type is present in - the certificate (e.g., more than one dNSName name, a match in any one - of the set is considered acceptable.) Names may contain the wildcard - character * which is considered to match any single domain name - component or component fragment. E.g., *.a.com matches foo.a.com but - not bar.foo.a.com. f*.com matches foo.com but not bar.com. - - In some cases, the URI is specified as an IP address rather than a - hostname. In this case, the iPAddress subjectAltName must be present - in the certificate and must exactly match the IP in the URI. - - */ - return verify_host_with_subject_alt_name(server_cert) || - verify_host_with_common_name(server_cert); -} - -bool -SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const { - auto ret = false; - - auto type = GEN_DNS; - - struct in6_addr addr6 = {}; - struct in_addr addr = {}; - size_t addr_len = 0; - -#ifndef __MINGW32__ - if (inet_pton(AF_INET6, host_.c_str(), &addr6)) { - type = GEN_IPADD; - addr_len = sizeof(struct in6_addr); - } else if (inet_pton(AF_INET, host_.c_str(), &addr)) { - type = GEN_IPADD; - addr_len = sizeof(struct in_addr); - } -#endif - - auto alt_names = static_cast( - X509_get_ext_d2i(server_cert, NID_subject_alt_name, nullptr, nullptr)); - - if (alt_names) { - auto dsn_matched = false; - auto ip_matched = false; - - auto count = sk_GENERAL_NAME_num(alt_names); - - for (decltype(count) i = 0; i < count && !dsn_matched; i++) { - auto val = sk_GENERAL_NAME_value(alt_names, i); - if (!val || val->type != type) { continue; } - - auto name = - reinterpret_cast(ASN1_STRING_get0_data(val->d.ia5)); - if (name == nullptr) { continue; } - - auto name_len = static_cast(ASN1_STRING_length(val->d.ia5)); - - switch (type) { - case GEN_DNS: - dsn_matched = - detail::match_hostname(std::string(name, name_len), host_); - break; - - case GEN_IPADD: - if (!memcmp(&addr6, name, addr_len) || !memcmp(&addr, name, addr_len)) { - ip_matched = true; - } - break; - } - } - - if (dsn_matched || ip_matched) { ret = true; } - } - - GENERAL_NAMES_free(const_cast( - reinterpret_cast(alt_names))); - return ret; -} - -bool SSLClient::verify_host_with_common_name(X509 *server_cert) const { - const auto subject_name = X509_get_subject_name(server_cert); - - if (subject_name != nullptr) { - char name[BUFSIZ]; - auto name_len = X509_NAME_get_text_by_NID(subject_name, NID_commonName, - name, sizeof(name)); - - if (name_len != -1) { - return detail::match_hostname( - std::string(name, static_cast(name_len)), host_); - } - } - - return false; -} - #endif // CPPHTTPLIB_OPENSSL_SUPPORT /* @@ -14622,10 +14565,10 @@ bool verify_hostname(cert_t cert, const char *hostname) { auto mcert = static_cast(cert); std::string host_str(hostname); - // Check if hostname is an IP address - bool is_ip = impl::is_ipv4_address(host_str); - unsigned char ip_bytes[4]; - if (is_ip) { impl::parse_ipv4(host_str, ip_bytes); } + // Check if hostname is an IP address (IPv4 or IPv6) + unsigned char ip_bytes[16]; + auto ip_len = impl::parse_ip_address(host_str, ip_bytes); + auto is_ip = ip_len > 0; // Check Subject Alternative Names (SAN) // In Mbed TLS 3.x, subject_alt_names contains raw values without ASN.1 tags @@ -14637,9 +14580,9 @@ bool verify_hostname(cert_t cert, const char *hostname) { size_t len = san->buf.len; if (is_ip) { - // Check if this SAN is an IPv4 address (4 bytes) - if (len == 4 && memcmp(p, ip_bytes, 4) == 0) { return true; } - // Check if this SAN is an IPv6 address (16 bytes) - skip for now + // For an IP host, only a matching iPAddress SAN of the same family + // (4 bytes for IPv4, 16 bytes for IPv6) may authenticate it. + if (len == ip_len && memcmp(p, ip_bytes, ip_len) == 0) { return true; } } else { // Check if this SAN is a DNS name (printable ASCII string) bool is_dns = len > 0; @@ -14654,21 +14597,25 @@ bool verify_hostname(cert_t cert, const char *hostname) { san = san->next; } - // Fallback: Check Common Name (CN) in subject - char cn[256]; - int ret = mbedtls_x509_dn_gets(cn, sizeof(cn), &mcert->subject); - if (ret > 0) { - std::string cn_str(cn); + // Fallback: Check Common Name (CN) in subject. Skipped for IP-literal hosts: + // an IP identity is only valid via an iPAddress SAN, never the CN (RFC 9110; + // the OpenSSL backend's X509_check_ip behaves the same way). + if (!is_ip) { + char cn[256]; + int ret = mbedtls_x509_dn_gets(cn, sizeof(cn), &mcert->subject); + if (ret > 0) { + std::string cn_str(cn); - // Look for "CN=" in the DN string - size_t cn_pos = cn_str.find("CN="); - if (cn_pos != std::string::npos) { - size_t start = cn_pos + 3; - size_t end = cn_str.find(',', start); - std::string cn_value = - cn_str.substr(start, end == std::string::npos ? end : end - start); + // Look for "CN=" in the DN string + size_t cn_pos = cn_str.find("CN="); + if (cn_pos != std::string::npos) { + size_t start = cn_pos + 3; + size_t end = cn_str.find(',', start); + std::string cn_value = + cn_str.substr(start, end == std::string::npos ? end : end - start); - if (detail::match_hostname(cn_value, host_str)) { return true; } + if (detail::match_hostname(cn_value, host_str)) { return true; } + } } } @@ -15774,10 +15721,10 @@ bool verify_hostname(cert_t cert, const char *hostname) { auto x509 = static_cast(cert); std::string host_str(hostname); - // Check if hostname is an IP address - bool is_ip = impl::is_ipv4_address(host_str); - unsigned char ip_bytes[4]; - if (is_ip) { impl::parse_ipv4(host_str, ip_bytes); } + // Check if hostname is an IP address (IPv4 or IPv6) + unsigned char ip_bytes[16]; + auto ip_len = impl::parse_ip_address(host_str, ip_bytes); + auto is_ip = ip_len > 0; // Check Subject Alternative Names auto *san_names = static_cast( @@ -15804,10 +15751,12 @@ bool verify_hostname(cert_t cert, const char *hostname) { } } } else if (is_ip && names->type == WOLFSSL_GEN_IPADD) { - // IP address + // IP address: only an iPAddress SAN of the same family (4 bytes for + // IPv4, 16 bytes for IPv6) may authenticate the host. unsigned char *ip_data = wolfSSL_ASN1_STRING_data(names->d.iPAddress); - int ip_len = wolfSSL_ASN1_STRING_length(names->d.iPAddress); - if (ip_data && ip_len == 4 && memcmp(ip_data, ip_bytes, 4) == 0) { + auto san_ip_len = wolfSSL_ASN1_STRING_length(names->d.iPAddress); + if (ip_data && san_ip_len == static_cast(ip_len) && + memcmp(ip_data, ip_bytes, ip_len) == 0) { wolfSSL_sk_free(san_names); return true; } @@ -15816,8 +15765,10 @@ bool verify_hostname(cert_t cert, const char *hostname) { wolfSSL_sk_free(san_names); } - // Fallback: Check Common Name (CN) in subject - WOLFSSL_X509_NAME *subject = wolfSSL_X509_get_subject_name(x509); + // Fallback: Check Common Name (CN) in subject. Skipped for IP-literal hosts: + // an IP identity is only valid via an iPAddress SAN, never the CN (RFC 9110; + // the OpenSSL backend's X509_check_ip behaves the same way). + auto subject = is_ip ? nullptr : wolfSSL_X509_get_subject_name(x509); if (subject) { char cn[256] = {}; int cn_len = wolfSSL_X509_NAME_get_text_by_NID(subject, NID_commonName, cn, diff --git a/vendor/cpp-httplib/httplib.h b/vendor/cpp-httplib/httplib.h index 94d93e88a..bfdbfc1da 100644 --- a/vendor/cpp-httplib/httplib.h +++ b/vendor/cpp-httplib/httplib.h @@ -8,8 +8,8 @@ #ifndef CPPHTTPLIB_HTTPLIB_H #define CPPHTTPLIB_HTTPLIB_H -#define CPPHTTPLIB_VERSION "0.47.0" -#define CPPHTTPLIB_VERSION_NUM "0x002f00" +#define CPPHTTPLIB_VERSION "0.48.0" +#define CPPHTTPLIB_VERSION_NUM "0x003000" #ifdef _WIN32 #if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00 @@ -686,18 +686,70 @@ inline from_chars_result from_chars(const char *first, const char *last, return {p, std::errc{}}; } -// from_chars for double (simple wrapper for strtod) +// from_chars for double (hand-written, locale-independent) +// +// The only double consumed by this library is the HTTP quality value, whose +// grammar is (RFC 9110 12.4.2): +// qvalue = ( "0" [ "." 0*3DIGIT ] ) / ( "1" [ "." 0*3("0") ] ) +// i.e. a non-negative decimal with no sign, exponent, "inf"/"nan", or wide +// magnitude. So this parser recognizes exactly 1*DIGIT [ "." *DIGIT ] with +// '.' always the decimal separator (std::strtod would instead read it from the +// global C locale, mis-parsing q-values once an embedder calls +// setlocale(LC_ALL, "") into a comma-decimal locale). The caller range-checks +// the result to [0, 1], so inputs outside that range need not be distinguished +// here. Allocation-free, single pass, and free of the overflow/rounding edge +// cases that exponent and wide-range handling would introduce. inline from_chars_result from_chars(const char *first, const char *last, double &value) { - std::string s(first, last); - char *endptr = nullptr; - errno = 0; - value = std::strtod(s.c_str(), &endptr); - if (endptr == s.c_str()) { return {first, std::errc::invalid_argument}; } - if (errno == ERANGE) { - return {first + (endptr - s.c_str()), std::errc::result_out_of_range}; + value = 0.0; + const char *p = first; + + // Each 1eN is exactly representable, so a single final division by the + // matching entry yields a correctly-rounded result. + static const double powers_of_ten[] = { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18}; + const int max_frac_digits = + static_cast(sizeof(powers_of_ten) / sizeof(powers_of_ten[0])) - 1; + + // Accumulate digits into a 64-bit integer and remember how many were + // fractional. Two independent caps keep this bounded and safe: + // * accumulation saturates before mantissa could overflow uint64_t, and + // * frac_digits is capped at max_frac_digits so it is always a valid index + // into powers_of_ten (without this an input like "0.000...0" would never + // grow mantissa, so the saturation cap alone would not bound it). + // Both caps only drop digits far beyond the precision a q-value needs; any + // value they would change is well outside [0, 1] and rejected by the caller. + uint64_t mantissa = 0; + int frac_digits = 0; + bool seen_digit = false; + + const uint64_t limit = ((std::numeric_limits::max)() - 9) / 10; + auto accumulate = [&](char c) { + if (mantissa <= limit) { + mantissa = mantissa * 10 + static_cast(c - '0'); + return true; + } + return false; + }; + + for (; p != last && '0' <= *p && *p <= '9'; ++p) { + seen_digit = true; + accumulate(*p); } - return {first + (endptr - s.c_str()), std::errc{}}; + + if (p != last && *p == '.') { + ++p; + for (; p != last && '0' <= *p && *p <= '9'; ++p) { + seen_digit = true; + if (frac_digits < max_frac_digits && accumulate(*p)) { ++frac_digits; } + } + } + + if (!seen_digit) { return {first, std::errc::invalid_argument}; } + + value = static_cast(mantissa) / powers_of_ten[frac_digits]; + return {p, std::errc{}}; } inline bool parse_port(const char *s, size_t len, int &port) { @@ -2826,13 +2878,6 @@ private: #endif friend class ClientImpl; - -#ifdef CPPHTTPLIB_OPENSSL_SUPPORT -private: - bool verify_host(X509 *server_cert) const; - bool verify_host_with_subject_alt_name(X509 *server_cert) const; - bool verify_host_with_common_name(X509 *server_cert) const; -#endif }; #endif // CPPHTTPLIB_SSL_ENABLED From fabde3bf5136940eb03821aa2490e2360093965b Mon Sep 17 00:00:00 2001 From: Mikolaj Kucharski Date: Fri, 19 Jun 2026 15:33:54 +0000 Subject: [PATCH 16/21] arg: Add comment line support to --api-key-file (#23168) --- common/arg.cpp | 4 ++-- tools/server/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 52425f25e..6fd366d33 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2911,7 +2911,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_API_KEY")); add_opt(common_arg( {"--api-key-file"}, "FNAME", - "path to file containing API keys (default: none)", + "path to file containing API keys, one per line; lines starting with a hash are treated as comments (default: none)", [](common_params & params, const std::string & value) { std::ifstream key_file(value); if (!key_file) { @@ -2919,7 +2919,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } std::string key; while (std::getline(key_file, key)) { - if (!key.empty()) { + if (!key.empty() && key[0] != '#') { params.api_keys.push_back(key); } } diff --git a/tools/server/README.md b/tools/server/README.md index 1f74ba52a..eb730e713 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -198,7 +198,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)
(env: LLAMA_ARG_EMBEDDINGS) | | `--rerank, --reranking` | enable reranking endpoint on server (default: disabled)
(env: LLAMA_ARG_RERANKING) | | `--api-key KEY` | API key to use for authentication, multiple keys can be provided as a comma-separated list (default: none)
(env: LLAMA_API_KEY) | -| `--api-key-file FNAME` | path to file containing API keys (default: none)
(env: LLAMA_ARG_API_KEY_FILE) | +| `--api-key-file FNAME` | path to file containing API keys, one per line; lines starting with a hash are treated as comments (default: none)
(env: LLAMA_ARG_API_KEY_FILE) | | `--ssl-key-file FNAME` | path to file a PEM-encoded SSL private key
(env: LLAMA_ARG_SSL_KEY_FILE) | | `--ssl-cert-file FNAME` | path to file a PEM-encoded SSL certificate
(env: LLAMA_ARG_SSL_CERT_FILE) | | `--chat-template-kwargs STRING` | sets additional params for the json template parser, must be a valid json object string, e.g. '{"key1":"value1","key2":"value2"}'
(env: LLAMA_ARG_CHAT_TEMPLATE_KWARGS) | From 175147e8f612671b7906e21fb1cdea62e4da0e21 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Fri, 19 Jun 2026 22:12:46 +0200 Subject: [PATCH 17/21] server: remove all internal mentions about "webui" (#24817) --- tools/server/server-context.cpp | 9 +-------- tools/server/server-context.h | 3 +-- tools/server/server-models.cpp | 1 - tools/server/server-models.h | 2 -- tools/server/tests/unit/test_basic.py | 8 ++++---- tools/server/tests/unit/test_proxy.py | 6 +++--- tools/server/tests/utils.py | 12 ++++++------ 7 files changed, 15 insertions(+), 26 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 00ab31340..791188b1e 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -825,8 +825,7 @@ private: server_metrics metrics; - json json_ui_settings = json::object(); // Primary: new name - json json_webui_settings = json::object(); // Deprecated: use json_ui_settings instead (kept for compat) + json json_ui_settings = json::object(); // Necessary similarity of prompt for slot selection float slot_prompt_similarity = 0.0f; @@ -1308,7 +1307,6 @@ private: try { json json_settings = json::parse(cfg); json_ui_settings = json_settings; - json_webui_settings = json_settings; // deprecated: keep in sync } catch (const std::exception & e) { SRV_ERR("%s: failed to parse UI config: %s\n", __func__, e.what()); return false; @@ -3687,7 +3685,6 @@ server_context_meta server_context::get_meta() const { /* has_inp_audio */ impl->chat_params.allow_audio, /* has_inp_video */ impl->chat_params.allow_video, /* json_ui_settings */ impl->json_ui_settings, - /* json_webui_settings */ impl->json_webui_settings, // Deprecated /* slot_n_ctx */ impl->get_slot_n_ctx(), /* pooling_type */ llama_pooling_type(impl->ctx_tgt), @@ -4300,12 +4297,8 @@ void server_routes::init_routes() { { "endpoint_slots", params.endpoint_slots }, { "endpoint_props", params.endpoint_props }, { "endpoint_metrics", params.endpoint_metrics }, - // New keys { "ui", params.ui }, { "ui_settings", meta->json_ui_settings }, - // Deprecated: use ui/ui_settings instead (kept for backward compat) - { "webui", params.ui }, - { "webui_settings", meta->json_ui_settings }, { "chat_template", tmpl_default }, { "chat_template_caps", meta->chat_template_caps }, { "bos_token", meta->bos_token_str }, diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 0e84785af..07afabb92 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -22,8 +22,7 @@ struct server_context_meta { bool has_inp_image; bool has_inp_audio; bool has_inp_video; - json json_ui_settings; // Primary: new name - json json_webui_settings; // Deprecated: use json_ui_settings instead (kept for backward compat) + json json_ui_settings; int slot_n_ctx; enum llama_pooling_type pooling_type; diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 23c1f1668..1fffa6b6e 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1474,7 +1474,6 @@ void server_models_routes::init_routes() { }}, // New key {"ui_settings", ui_settings}, - {"webui_settings", webui_settings}, {"build_info", std::string(llama_build_info())}, {"cors_proxy_enabled", params.ui_mcp_proxy}, }); diff --git a/tools/server/server-models.h b/tools/server/server-models.h index aeb0e874d..98872b046 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -207,7 +207,6 @@ public: struct server_models_routes { common_params params; json ui_settings = json::object(); // Primary: new name - json webui_settings = json::object(); // Deprecated: use ui_settings (kept for compat) std::atomic stopping = false; // for graceful disconnecting SSE clients during shutdown server_models models; server_models_routes(const common_params & params, int argc, char ** argv) @@ -217,7 +216,6 @@ struct server_models_routes { try { json json_settings = json::parse(cfg); ui_settings = json_settings; - webui_settings = json_settings; // Deprecated: keep in sync } catch (const std::exception & e) { LOG_ERR("%s: failed to parse UI config: %s\n", __func__, e.what()); throw; diff --git a/tools/server/tests/unit/test_basic.py b/tools/server/tests/unit/test_basic.py index d1b89cf1a..285726abf 100644 --- a/tools/server/tests/unit/test_basic.py +++ b/tools/server/tests/unit/test_basic.py @@ -79,9 +79,9 @@ def test_load_split_model(): assert match_regex("(little|girl)+", res.body["content"]) -def test_no_webui(): +def test_no_ui(): global server - # default: webui enabled + # default: UI enabled server.start() url = f"http://{server.server_host}:{server.server_port}" res = requests.get(url) @@ -89,8 +89,8 @@ def test_no_webui(): assert "" in res.text server.stop() - # with --no-webui - server.no_webui = True + # with --no-ui, the UI should be disabled + server.no_ui = True server.start() res = requests.get(url) assert res.status_code == 404 diff --git a/tools/server/tests/unit/test_proxy.py b/tools/server/tests/unit/test_proxy.py index b7c332618..3b86d8047 100644 --- a/tools/server/tests/unit/test_proxy.py +++ b/tools/server/tests/unit/test_proxy.py @@ -12,7 +12,7 @@ def create_server(): def test_mcp_no_proxy(): global server - server.webui_mcp_proxy = False + server.ui_mcp_proxy = False server.start() res = server.make_request("GET", "/cors-proxy") @@ -21,7 +21,7 @@ def test_mcp_no_proxy(): def test_mcp_proxy(): global server - server.webui_mcp_proxy = True + server.ui_mcp_proxy = True server.start() url = f"http://{server.server_host}:{server.server_port}/cors-proxy?url=http://example.com" @@ -32,7 +32,7 @@ def test_mcp_proxy(): def test_mcp_proxy_custom_port(): global server - server.webui_mcp_proxy = True + server.ui_mcp_proxy = True server.start() # try getting the server's models API via the proxy diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index c50c9a0f5..63a959449 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -94,7 +94,7 @@ class ServerProcess: enable_ctx_shift: int | None = False spec_draft_n_min: int | None = None spec_draft_n_max: int | None = None - no_webui: bool | None = None + no_ui: bool | None = None jinja: bool | None = None reasoning_format: Literal['deepseek', 'none', 'nothink'] | None = None reasoning: Literal['on', 'off', 'auto'] | None = None @@ -107,7 +107,7 @@ class ServerProcess: cache_ram: int | None = None no_cache_idle_slots: bool = False log_path: str | None = None - webui_mcp_proxy: bool = False + ui_mcp_proxy: bool = False backend_sampling: bool = False gcp_compat: bool = False @@ -225,8 +225,8 @@ class ServerProcess: server_args.extend(["--spec-draft-n-max", self.spec_draft_n_max]) if self.spec_draft_n_min: server_args.extend(["--spec-draft-n-min", self.spec_draft_n_min]) - if self.no_webui: - server_args.append("--no-webui") + if self.no_ui: + server_args.append("--no-ui") if self.no_models_autoload: server_args.append("--no-models-autoload") if self.jinja: @@ -251,8 +251,8 @@ class ServerProcess: server_args.extend(["--cache-ram", self.cache_ram]) if self.no_cache_idle_slots: server_args.append("--no-cache-idle-slots") - if self.webui_mcp_proxy: - server_args.append("--webui-mcp-proxy") + if self.ui_mcp_proxy: + server_args.append("--ui-mcp-proxy") if self.backend_sampling: server_args.append("--backend_sampling") if self.gcp_compat: From e475fa2b5f9fb50c3d6fc3e7c6fdf1e004465b62 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Fri, 19 Jun 2026 22:28:38 +0200 Subject: [PATCH 18/21] mtmd, arg: fix utf8 handling on windows (#24779) * mtmd, arg: fix utf8 handling on windows * also fix ggml_fopen * fix build fail * also fix CLI --- common/arg.cpp | 38 ++++++++++++++++++++++++++++++++++++++ common/common.cpp | 12 ++++++++++++ common/common.h | 3 +++ ggml/src/ggml.c | 17 +++++++---------- tools/cli/cli.cpp | 2 +- tools/mtmd/clip-impl.h | 24 ++++++++++++++++++++++++ tools/mtmd/clip.cpp | 2 +- tools/mtmd/mtmd-cli.cpp | 3 +++ tools/mtmd/mtmd-helper.cpp | 18 +++++++++++++++++- 9 files changed, 106 insertions(+), 13 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 6fd366d33..8f4f7d076 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -17,6 +17,7 @@ # define NOMINMAX #endif #include +#include #endif #define JSON_ASSERT GGML_ASSERT @@ -893,7 +894,44 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map buf; + std::vector ptrs; +}; + +static utf8_argv make_utf8_argv() { + utf8_argv out; + int wargc = 0; + LPWSTR* wargv = CommandLineToArgvW(GetCommandLineW(), &wargc); + if (!wargv) return out; + + out.buf.reserve(wargc); + for (int i = 0; i < wargc; ++i) { + int n = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, nullptr, 0, nullptr, nullptr); + if (n <= 0) { out.buf.emplace_back(); continue; } + auto& s = out.buf.emplace_back(); + s.resize(static_cast(n - 1)); + (void)WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, s.data(), n, nullptr, nullptr); + } + LocalFree(wargv); + + out.ptrs.reserve(out.buf.size() + 1); + for (auto& s : out.buf) out.ptrs.push_back(s.data()); + out.ptrs.push_back(nullptr); + return out; +} +#endif + bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) { +#ifdef _WIN32 + auto utf8 = make_utf8_argv(); + if (!utf8.ptrs.empty()) { + argc = static_cast(utf8.buf.size()); + argv = utf8.ptrs.data(); + } +#endif + auto ctx_arg = common_params_parser_init(params, ex, print_usage); const common_params params_org = ctx_arg.params; // the example can modify the default params diff --git a/common/common.cpp b/common/common.cpp index f3f114f68..a14e7bbed 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1074,6 +1074,18 @@ std::vector fs_list(const std::string & path, bool include_dir return files; } +std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode) { +#ifdef _WIN32 + int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0); + if (!wlen) { return std::ifstream(); } + std::vector wfname(wlen); + (void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen); + return std::ifstream(wfname.data(), mode); +#else + return std::ifstream(fname, mode); +#endif +} + // // TTY utils // diff --git a/common/common.h b/common/common.h index 44c605189..254454dcb 100644 --- a/common/common.h +++ b/common/common.h @@ -842,6 +842,9 @@ struct common_file_info { }; std::vector fs_list(const std::string & path, bool include_directories); +// fs open, also handle UTF8 on Windows +std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode); + // // TTY utils // diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index b43016c87..0f682fd18 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -600,18 +600,15 @@ FILE * ggml_fopen(const char * fname, const char * mode) { // convert fname (UTF-8) wchar_t * wfname = ggml_mbstowcs(fname); if (wfname) { - // convert mode (ANSI) - wchar_t * wmode = GGML_MALLOC((strlen(mode) + 1) * sizeof(wchar_t)); - wchar_t * wmode_p = wmode; - do { - *wmode_p++ = (wchar_t)*mode; - } while (*mode++); - - // open file - file = _wfopen(wfname, wmode); + // convert mode (UTF-8) + wchar_t * wmode = ggml_mbstowcs(mode); + if (wmode) { + // open file + file = _wfopen(wfname, wmode); + GGML_FREE(wmode); + } GGML_FREE(wfname); - GGML_FREE(wmode); } return file; diff --git a/tools/cli/cli.cpp b/tools/cli/cli.cpp index c03894b4b..8b7b58693 100644 --- a/tools/cli/cli.cpp +++ b/tools/cli/cli.cpp @@ -202,7 +202,7 @@ struct cli_context { // TODO: support remote files in the future (http, https, etc) std::string load_input_file(const std::string & fname, bool is_media) { - std::ifstream file(fname, std::ios::binary); + std::ifstream file = fs_open_ifstream(fname, std::ios::binary); if (!file) { return ""; } diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index f232b68e5..e7b530144 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -13,6 +13,14 @@ #include #include #include +#include + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif // Internal header for clip.cpp @@ -661,6 +669,22 @@ struct clip_image_f32_batch { // common utils // +#ifdef _WIN32 +static std::ifstream open_ifstream_binary(const std::string & fname) { + int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0); + if (!wlen) { + throw std::runtime_error("failed to convert filename to UTF-16: " + fname); + } + std::vector wfname(wlen); + (void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen); + return std::ifstream(wfname.data(), std::ios::binary); +} +#else +static std::ifstream open_ifstream_binary(const std::string & fname) { + return std::ifstream(fname, std::ios::binary); +} +#endif + static std::string string_format(const char * fmt, ...) { va_list ap; va_list ap2; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 10840a851..c713703e0 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1752,7 +1752,7 @@ struct clip_model_loader { std::map tensor_offset; std::vector tensors_to_load; - auto fin = std::ifstream(fname, std::ios::binary); + auto fin = open_ifstream_binary(fname); if (!fin) { throw std::runtime_error(string_format("%s: failed to open %s\n", __func__, fname.c_str())); } diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index 0ad000ef0..8704ea79d 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -396,6 +396,9 @@ int main(int argc, char ** argv) { int n_predict = params.n_predict < 0 ? INT_MAX : params.n_predict; + console::init(params.simple_io, params.use_color); + atexit([]() { console::cleanup(); }); + // Ctrl+C handling { #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index b5c408923..3c73db443 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -582,13 +582,29 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, } mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder) { - std::vector buf; +#ifdef _WIN32 + int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0); + if (!wlen) { + LOG_ERR("Unable to convert filename to UTF-16: %s\n", fname); + return {nullptr, nullptr}; + } + std::vector wfname(wlen); + wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, wfname.data(), wlen); + if (!wlen) { + LOG_ERR("Unable to convert filename to UTF-16: %s\n", fname); + return {nullptr, nullptr}; + } + FILE * f = _wfopen(wfname.data(), L"rb"); +#else FILE * f = fopen(fname, "rb"); +#endif if (!f) { LOG_ERR("Unable to open file %s: %s\n", fname, strerror(errno)); return {nullptr, nullptr}; } + std::vector buf; + fseek(f, 0, SEEK_END); long file_size = ftell(f); fseek(f, 0, SEEK_SET); From 4b48a53b6cc60e051f35f2acbd06264a909bb255 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Gallou=C3=ABt?= Date: Fri, 19 Jun 2026 23:26:54 +0200 Subject: [PATCH 19/21] server : optimize get_token_probabilities (#24796) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use std::partial_sort to order only the requested top-n tokens instead of the full vocabulary logprobs sort: vocab=128000 n_top=0 iters=100 full sort: 8555.6 us/op partial sort: 704.3 us/op Signed-off-by: Adrien Gallouët --- tools/server/server-common.cpp | 36 +++++++++++++++++++++++---------- tools/server/server-common.h | 2 +- tools/server/server-context.cpp | 3 +-- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 75729e62d..3dc686bb4 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -12,6 +12,7 @@ #include #include #include +#include json format_error_response(const std::string & message, const enum error_type type) { std::string type_str; @@ -1238,7 +1239,7 @@ json format_response_rerank( // other utils // -std::vector get_token_probabilities(llama_context * ctx, int idx) { +std::vector get_token_probabilities(llama_context * ctx, int idx, size_t n_top) { std::vector cur; const auto * logits = llama_get_logits_ith(ctx, idx); @@ -1257,21 +1258,34 @@ std::vector get_token_probabilities(llama_context * ctx, int i } } - // sort tokens by logits - std::sort(cur.begin(), cur.end(), [](const llama_token_data & a, const llama_token_data & b) { - return a.logit > b.logit; - }); + // sort tokens by logits (partial: only the leading `n_top` need ordering) + if (n_top > cur.size()) { + n_top = cur.size(); + } + if (n_top > 0) { + std::partial_sort(cur.begin(), cur.begin() + n_top, cur.end(), + [](const llama_token_data & a, const llama_token_data & b) { + return a.logit > b.logit; + }); + } // apply softmax - float max_l = cur[0].logit; + float max_l = -std::numeric_limits::infinity(); + if (n_top > 0) { + max_l = cur[0].logit; // partial_sort guarantees the absolute maximum is at index 0 + } else { + for (const auto & t : cur) { + max_l = std::max(max_l, t.logit); + } + } float cum_sum = 0.0f; - for (size_t i = 0; i < cur.size(); ++i) { - float p = expf(cur[i].logit - max_l); - cur[i].p = p; + for (auto & t : cur) { + float p = expf(t.logit - max_l); + t.p = p; cum_sum += p; } - for (size_t i = 0; i < cur.size(); ++i) { - cur[i].p /= cum_sum; + for (auto & t : cur) { + t.p /= cum_sum; } return cur; diff --git a/tools/server/server-common.h b/tools/server/server-common.h index f286b3d15..efd31733b 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -326,7 +326,7 @@ json format_response_rerank( // other utils // -std::vector get_token_probabilities(llama_context * ctx, int idx); +std::vector get_token_probabilities(llama_context * ctx, int idx, size_t n_top); std::string safe_json_to_str(const json & data); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 791188b1e..1f0e1bfd4 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1824,8 +1824,7 @@ private: }); } } else { - // TODO: optimize this with min-p optimization - std::vector cur = get_token_probabilities(ctx_tgt, idx); + std::vector cur = get_token_probabilities(ctx_tgt, idx, n_probs_request); const size_t max_probs = cur.size(); const size_t n_probs = std::min(max_probs, n_probs_request); From 2b686a9120e2f1f9eabb65a597a4eef03eea9d87 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Sat, 20 Jun 2026 01:02:26 +0200 Subject: [PATCH 20/21] server: refactor child --> router communication (#24821) * server: refactor child --> router communication * fix wakeup case * add docs * improve update_status() * nits --- common/arg.cpp | 2 - common/common.h | 11 ++- tools/server/README-dev.md | 11 +++ tools/server/server-context.cpp | 18 ++--- tools/server/server-context.h | 30 ++++++- tools/server/server-models.cpp | 133 ++++++++++++++++++-------------- tools/server/server-models.h | 30 +++++-- tools/server/server.cpp | 29 ++++--- 8 files changed, 173 insertions(+), 91 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 8f4f7d076..a9b1a25b2 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -303,7 +303,6 @@ static handle_model_result common_params_handle_model(struct common_params_model if (!model.docker_repo.empty()) { model.path = common_docker_resolve_model(model.docker_repo); - model.name = model.docker_repo; } else if (!model.hf_repo.empty()) { // If -m was used with -hf, treat the model "path" as the hf_file to download if (model.hf_file.empty() && !model.path.empty()) { @@ -323,7 +322,6 @@ static handle_model_result common_params_handle_model(struct common_params_model throw std::runtime_error("failed to download model from Hugging Face"); } - model.name = model.hf_repo; model.path = download_result.model_path; if (!download_result.mmproj_path.empty()) { diff --git a/common/common.h b/common/common.h index 254454dcb..f2f2202ec 100644 --- a/common/common.h +++ b/common/common.h @@ -295,7 +295,16 @@ struct common_params_model { std::string hf_repo = ""; // HF repo // NOLINT std::string hf_file = ""; // HF file // NOLINT std::string docker_repo = ""; // Docker repo // NOLINT - std::string name = ""; // in format /[:] (tag is optional) // NOLINT + + std::string get_name() { + if (!hf_repo.empty()) { + return hf_repo; + } + if (!docker_repo.empty()) { + return docker_repo; + } + return path; + } }; // draft-model-based speculative decoding parameters diff --git a/tools/server/README-dev.md b/tools/server/README-dev.md index 4c4103123..2796d2835 100644 --- a/tools/server/README-dev.md +++ b/tools/server/README-dev.md @@ -180,6 +180,17 @@ That requires `JSON.stringify` when formatted to message content: } ``` +### Router mode: how child <--> router communicates + +Upon spawning a new child process using `subprocess`, both child and router listen to the stdout/stderr (combined) + +For the direction from child to router: +- Generic messages are logs, it will be forwarded to router's stdout +- Special state update messages are prefixed by `cmd_child_to_router:state:`, followed by a JSON. See `server_models::handle_child_state` for more + +For the direction from router to child: +- When server sends `cmd_router_to_child:exit`, the child should exit gracefully --> if after `DEFAULT_STOP_TIMEOUT` and the child is still running, force-kill it + ### Model management API (router mode) Model management API was added via PR [#23976](https://github.com/ggml-org/llama.cpp/pull/23976) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 1f0e1bfd4..3de1335ec 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -63,11 +63,6 @@ enum slot_state { SLOT_STATE_GENERATING, }; -enum server_state { - SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet - SERVER_STATE_READY, // Server is ready and model is loaded -}; - struct server_slot { int id; @@ -773,6 +768,8 @@ public: // note: chat_params must not be refreshed upon existing sleeping state server_chat_params chat_params; + server_state_callback_t callback_state = [](server_state, json) -> void {}; + server_context_impl() { mtmd_helper_log_set(common_log_default_callback, nullptr); } @@ -1244,8 +1241,8 @@ private: if (!params_base.model_alias.empty()) { // backward compat: use first alias as model name model_name = *params_base.model_alias.begin(); - } else if (!params_base.model.name.empty()) { - model_name = params_base.model.name; + } else if (!params_base.model.get_name().empty()) { + model_name = params_base.model.get_name(); } else { // fallback: derive model name from file name auto model_path = std::filesystem::path(params_base.model.path); @@ -3734,8 +3731,11 @@ struct server_res_generator : server_http_res { } }; -void server_context::on_sleeping_changed(std::function callback) { - impl->queue_tasks.on_sleeping_state(std::move(callback)); +void server_context::set_state_callback(server_state_callback_t callback) { + impl->callback_state = std::move(callback); + impl->queue_tasks.on_sleeping_state([this](bool sleeping) { + impl->callback_state(sleeping ? SERVER_STATE_SLEEPING : SERVER_STATE_READY, {}); + }); } // compute the number of tokens before the last user message in the prompt diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 07afabb92..c7218a12e 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -52,6 +52,31 @@ struct server_context_meta { uint64_t model_size; }; +enum server_state { + // SERVER_STATE_DOWNLOADING, + SERVER_STATE_LOADING, + SERVER_STATE_READY, + SERVER_STATE_SLEEPING, +}; + +static std::string server_state_to_str(server_state state) { + switch (state) { + case SERVER_STATE_LOADING: return "loading"; + case SERVER_STATE_READY: return "ready"; + case SERVER_STATE_SLEEPING: return "sleeping"; + default: GGML_ASSERT(false && "invalid server_state"); + } +} + +static server_state server_state_from_str(const std::string & str) { + if (str == "loading") return SERVER_STATE_LOADING; + if (str == "ready") return SERVER_STATE_READY; + if (str == "sleeping") return SERVER_STATE_SLEEPING; + GGML_ASSERT(false && "invalid server_state string"); +} + +using server_state_callback_t = std::function; + struct server_context { std::unique_ptr impl; @@ -79,9 +104,8 @@ struct server_context { // not thread-safe, should only be used from the main thread server_context_meta get_meta() const; - // register a callback to be called when sleeping state changes - // must be set before load_model() is called - void on_sleeping_changed(std::function callback); + // note: must be set before load_model() is called + void set_state_callback(server_state_callback_t callback); }; diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 1fffa6b6e..a569c8be3 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1,5 +1,6 @@ #include "server-common.h" #include "server-models.h" +#include "server-context.h" #include "build-info.h" #include "preset.h" @@ -44,9 +45,7 @@ extern char **environ; #define DEFAULT_STOP_TIMEOUT 10 // seconds #define CMD_ROUTER_TO_CHILD_EXIT "cmd_router_to_child:exit" -#define CMD_CHILD_TO_ROUTER_READY "cmd_child_to_router:ready" // also sent when waking up from sleep -#define CMD_CHILD_TO_ROUTER_SLEEP "cmd_child_to_router:sleep" -#define CMD_CHILD_TO_ROUTER_INFO "cmd_child_to_router:info:" // followed by json string +#define CMD_CHILD_TO_ROUTER_STATE "cmd_child_to_router:state:" // followed by json string // address for child process, this is needed because router may run on 0.0.0.0 // ref: https://github.com/ggml-org/llama.cpp/issues/17862 @@ -904,12 +903,8 @@ void server_models::load(const std::string & name) { while (fgets(buffer, vec_buf.size(), stdout_file) != nullptr) { LOG("[%5d] %s", port, buffer); std::string str(buffer); - if (string_starts_with(buffer, CMD_CHILD_TO_ROUTER_READY)) { - this->update_status(name, SERVER_MODEL_STATUS_LOADED, 0); - } else if (string_starts_with(buffer, CMD_CHILD_TO_ROUTER_INFO)) { - this->update_loaded_info(name, str); - } else if (string_starts_with(buffer, CMD_CHILD_TO_ROUTER_SLEEP)) { - this->update_status(name, SERVER_MODEL_STATUS_SLEEPING, 0); + if (string_starts_with(buffer, CMD_CHILD_TO_ROUTER_STATE)) { + this->handle_child_state(name, str); } } } else { @@ -976,7 +971,10 @@ void server_models::load(const std::string & name) { subprocess_destroy(&child_proc->get()); // update status and exit code - this->update_status(name, SERVER_MODEL_STATUS_UNLOADED, exit_code); + this->update_status(name, { + SERVER_MODEL_STATUS_UNLOADED, + exit_code + }); SRV_INF("instance name=%s exited with status %d\n", name.c_str(), exit_code); }); @@ -1016,7 +1014,8 @@ struct server_models_download_res : public common_download_callback { common_download_model(model, opts); is_ok = true; } catch (const std::exception & e) { - SRV_ERR("download failed for model name=%s: %s\n", model.name.c_str(), e.what()); + auto model_name = model.get_name(); + SRV_ERR("download failed for model name=%s: %s\n", model_name.c_str(), e.what()); is_ok = false; } return is_ok; @@ -1036,7 +1035,7 @@ struct server_models_download_res : public common_download_callback { }; void server_models::download(common_params_model && model, common_download_opts && opts) { - std::string name = model.name; + std::string name = model.get_name(); GGML_ASSERT(name == model.hf_repo); std::unique_lock lk(mutex); @@ -1064,9 +1063,10 @@ void server_models::download(common_params_model && model, common_download_opts inst.th = std::thread([this, dl = std::move(dl)]() { dl->opts.callback = dl.get(); bool ok = dl->run(); + auto model_name = dl->model.get_name(); SRV_INF("download finished for model name=%s with status=%s\n", - dl->model.name.c_str(), ok ? "success" : "failure"); - update_download_progress(dl->model.name, {}, true, ok); + model_name.c_str(), ok ? "success" : "failure"); + update_download_progress(model_name, {}, true, ok); // need_reload is set inside update_download_progress under the mutex; // the next load_models() call will clean up this instance }); @@ -1130,21 +1130,27 @@ void server_models::unload_all() { } } -void server_models::update_status(const std::string & name, server_model_status status, int exit_code) { +void server_models::update_status(const std::string & name, const update_status_args & args) { std::unique_lock lk(mutex); auto it = mapping.find(name); if (it != mapping.end()) { auto & meta = it->second.meta; - meta.status = status; - meta.exit_code = exit_code; + meta.status = args.status; + meta.exit_code = args.exit_code; + if (!args.loaded_info.is_null()) { + meta.loaded_info = args.loaded_info; + } } // broadcast status change to SSE { json data = { - {"status", server_model_status_to_string(status)}, + {"status", server_model_status_to_string(args.status)}, }; - if (status == SERVER_MODEL_STATUS_UNLOADED) { - data["exit_code"] = exit_code; + if (args.status == SERVER_MODEL_STATUS_UNLOADED) { + data["exit_code"] = args.exit_code; + } + if (!args.loaded_info.is_null()) { + data["info"] = args.loaded_info; } // note: notify_sse doesn't acquire the lock, so no deadlock here notify_sse("status_change", name, data); @@ -1152,29 +1158,6 @@ void server_models::update_status(const std::string & name, server_model_status cv.notify_all(); } -void server_models::update_loaded_info(const std::string & name, std::string & raw_info) { - if (!string_starts_with(raw_info, CMD_CHILD_TO_ROUTER_INFO)) { - SRV_WRN("invalid loaded info format from child for model name=%s: %s\n", name.c_str(), raw_info.c_str()); - return; - } - - json info; - try { - info = json::parse(raw_info.substr(strlen(CMD_CHILD_TO_ROUTER_INFO))); - } catch (const std::exception & e) { - SRV_WRN("failed to parse loaded info from child for model name=%s: %s\n", name.c_str(), e.what()); - return; - } - - std::unique_lock lk(mutex); - auto it = mapping.find(name); - if (it != mapping.end()) { - auto & meta = it->second.meta; - meta.loaded_info = info; - } - cv.notify_all(); -} - void server_models::update_download_progress(const std::string & name, const common_download_progress & progress, bool done, bool ok) { json curr; { @@ -1323,21 +1306,54 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co return proxy; } -bool server_models::is_child_server() { +void server_models::handle_child_state(const std::string & name, const std::string & raw_input) { + server_state state; + json payload; + + try { + json data = json::parse(raw_input.substr(strlen(CMD_CHILD_TO_ROUTER_STATE))); + state = server_state_from_str(json_value(data, "state", std::string())); + payload = json_value(data, "payload", json{}); + } catch (const std::exception & e) { + SRV_ERR("failed to parse child state update for name=%s: %s\n", name.c_str(), e.what()); + return; + } + + switch (state) { + case SERVER_STATE_LOADING: + { + // do nothing for now + // TODO: report loading progress for first load and wakeup from sleep + } break; + case SERVER_STATE_READY: + { + update_status(name, { + SERVER_MODEL_STATUS_LOADED, + 0, + // note: payload can be empty if this is a wakeup from sleep + payload.size() > 0 ? payload : nullptr + }); + } break; + case SERVER_STATE_SLEEPING: + { + update_status(name, { SERVER_MODEL_STATUS_SLEEPING }); + } break; + default: + // should never happen, but just in case + GGML_ASSERT(false && "unexpected state from child server"); + } +} + +// +// server_child +// + +bool server_child::is_child() { const char * router_port = std::getenv("LLAMA_SERVER_ROUTER_PORT"); return router_port != nullptr; } -std::thread server_models::setup_child_server(const std::function & shutdown_handler, const json & model_info) { - // send a notification to the router server that a model instance is ready - common_log_pause(common_log_main()); - fflush(stdout); - fprintf(stdout, "%s\n", CMD_CHILD_TO_ROUTER_READY); - fflush(stdout); - fprintf(stdout, "%s%s\n", CMD_CHILD_TO_ROUTER_INFO, safe_json_to_str(model_info).c_str()); - fflush(stdout); - common_log_resume(common_log_main()); - +std::thread server_child::setup(const std::function & shutdown_handler) { // setup thread for monitoring stdin return std::thread([shutdown_handler]() { // wait for EOF on stdin @@ -1363,10 +1379,14 @@ std::thread server_models::setup_child_server(const std::function & s }); } -void server_models::notify_router_sleeping_state(bool is_sleeping) { +void server_child::notify_to_router(const std::string & state, const json & payload) { + json data = { + {"state", state}, + {"payload", payload}, + }; common_log_pause(common_log_main()); fflush(stdout); - fprintf(stdout, "%s\n", is_sleeping ? CMD_CHILD_TO_ROUTER_SLEEP : CMD_CHILD_TO_ROUTER_READY); + fprintf(stdout, "%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str()); fflush(stdout); common_log_resume(common_log_main()); } @@ -1644,7 +1664,6 @@ void server_models_routes::init_routes() { common_params_model model; common_download_opts opts; - model.name = name; model.hf_repo = name; opts.bearer_token = params.hf_token; opts.download_mmproj = true; diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 98872b046..40a0e078c 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -171,8 +171,12 @@ public: void download(common_params_model && model, common_download_opts && opts); // update the status of a model instance (thread-safe) - void update_status(const std::string & name, server_model_status status, int exit_code); - void update_loaded_info(const std::string & name, std::string & raw_info); + struct update_status_args { + server_model_status status; + int exit_code = 0; // only valid if status == UNLOADED + json loaded_info = nullptr; + }; + void update_status(const std::string & name, const update_status_args & args); void update_download_progress(const std::string & name, const common_download_progress & progress, bool done, bool ok = true); // remove a cache model from disk and update the list (thread-safe) @@ -193,15 +197,27 @@ public: // proxy an HTTP request to the model instance server_http_res_ptr proxy_request(const server_http_req & req, const std::string & method, const std::string & name, bool update_last_used); + // handle message sent from server_child::notify_to_router() + // raw input must starts with CMD_CHILD_TO_ROUTER_STATE, followed by a JSON string + // this function is not thread-safe, must be called from instance's monitoring thread + // payload per state: + // state = loading -> payload = {} (TODO: add progress info) + // state = ready -> payload = model_info (json), or {} if wakeup from sleeping + // state = sleeping -> payload = {} + void handle_child_state(const std::string & name, const std::string & raw_input); +}; + +struct server_child { // return true if the current process is a child server instance - static bool is_child_server(); + bool is_child(); - // notify the router server that a model instance is ready + // register the shutdown_handler to be called by the router // return the monitoring thread (to be joined by the caller) - static std::thread setup_child_server(const std::function & shutdown_handler, const json & model_info); + std::thread setup(const std::function & shutdown_handler); - // notify the router server that the sleeping state has changed - static void notify_router_sleeping_state(bool sleeping); + // notify router server for status changes (e.g. loading, downloading, sleeping, etc.) + // message will be handled by server_models::handle_child_state() on the router side + void notify_to_router(const std::string & state_name, const json & payload); }; struct server_models_routes { diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 2a67bfcfe..bf3680b9f 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -90,8 +90,10 @@ int llama_server(int argc, char ** argv) { llama_numa_init(params.numa); // router server never loads a model and must not touch the GPU + const bool is_router_server = params.model.path.empty() + && params.model.hf_repo.empty(); + // skip device enumeration so the CUDA primary context stays uncreated - const bool is_router_server = params.model.path.empty(); common_params_print_info(params, !is_router_server); if (!is_router_server) { @@ -113,8 +115,9 @@ int llama_server(int argc, char ** argv) { } // for consistency between server router mode and single-model mode, we set the same model name as alias - if (params.model_alias.empty() && !params.model.name.empty()) { - params.model_alias.insert(params.model.name); + auto model_name = params.model.get_name(); + if (params.model_alias.empty() && !model_name.empty()) { + params.model_alias.insert(model_name); } // struct that contains llama context and inference @@ -255,6 +258,7 @@ int llama_server(int argc, char ** argv) { // Start the server // + server_child child; // only used in non-router mode std::function clean_up; if (is_router_server) { @@ -300,15 +304,16 @@ int llama_server(int argc, char ** argv) { return 1; } - // load the model - SRV_INF("%s", "loading model\n"); - - if (server_models::is_child_server()) { - ctx_server.on_sleeping_changed([&](bool sleeping) { - server_models::notify_router_sleeping_state(sleeping); + // setup communication child --> router if necessary + if (child.is_child()) { + ctx_server.set_state_callback([&](server_state state, json payload) { + child.notify_to_router(server_state_to_str(state), payload); }); } + // load the model + SRV_INF("%s", "loading model\n"); + if (!ctx_server.load_model(params)) { clean_up(); if (ctx_http.thread.joinable()) { @@ -365,9 +370,9 @@ int llama_server(int argc, char ** argv) { // optionally, notify router server that this instance is ready std::thread monitor_thread; - if (server_models::is_child_server()) { - json model_info = routes.get_model_info(); - monitor_thread = server_models::setup_child_server(shutdown_handler, model_info); + if (child.is_child()) { + monitor_thread = child.setup(shutdown_handler); + child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info()); } // this call blocks the main thread until queue_tasks.terminate() is called From f449e0553708b895adbd94a301431cef691f632d Mon Sep 17 00:00:00 2001 From: Masashi Yoshimura Date: Sat, 20 Jun 2026 08:12:32 +0900 Subject: [PATCH 21/21] ggml-webgpu: add adapter toggles for F16 on Vulkan + NVIDIA --- ggml/src/ggml-webgpu/ggml-webgpu.cpp | 42 +++++++++++----------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index 0b605fa86..f71d1aee7 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -3788,7 +3788,7 @@ static void ggml_webgpu_init_memset_pipeline(webgpu_global_context & ctx) { ctx->memset_pipeline = ggml_webgpu_create_pipeline(ctx->device, wgsl_memset, "memset", constants); } -static void create_webgpu_device(ggml_backend_webgpu_reg_context * ctx) { +static void ggml_backend_webgpu_request_adapter(wgpu::Instance & instance, wgpu::Adapter & adapter) { wgpu::RequestAdapterOptions options = {}; #ifndef __EMSCRIPTEN__ @@ -3800,17 +3800,20 @@ static void create_webgpu_device(ggml_backend_webgpu_reg_context * ctx) { options.nextInChain = &adapterTogglesDesc; #endif - ctx->webgpu_global_ctx->instance.WaitAny( - ctx->webgpu_global_ctx->instance.RequestAdapter( - &options, wgpu::CallbackMode::AllowSpontaneous, - [&ctx](wgpu::RequestAdapterStatus status, wgpu::Adapter adapter, const char * message) { - if (status != wgpu::RequestAdapterStatus::Success) { - GGML_LOG_ERROR("ggml_webgpu: Failed to get an adapter: %s\n", message); - return; - } - ctx->webgpu_global_ctx->adapter = std::move(adapter); - }), - UINT64_MAX); + instance.WaitAny(instance.RequestAdapter( + &options, wgpu::CallbackMode::AllowSpontaneous, + [&adapter](wgpu::RequestAdapterStatus status, wgpu::Adapter _adapter, const char * message) { + if (status != wgpu::RequestAdapterStatus::Success) { + GGML_LOG_ERROR("ggml_webgpu: Failed to get an adapter: %s\n", message); + return; + } + adapter = std::move(_adapter); + }), + UINT64_MAX); +} + +static void create_webgpu_device(ggml_backend_webgpu_reg_context * ctx) { + ggml_backend_webgpu_request_adapter(ctx->webgpu_global_ctx->instance, ctx->webgpu_global_ctx->adapter); GGML_ASSERT(ctx->webgpu_global_ctx->adapter != nullptr); ctx->webgpu_global_ctx->adapter.GetLimits(&ctx->webgpu_global_ctx->capabilities.limits); @@ -4543,20 +4546,7 @@ ggml_backend_reg_t ggml_backend_webgpu_reg() { // Probe for adapter support wgpu::Adapter adapter; if (ctx->webgpu_global_ctx->instance != nullptr) { - wgpu::RequestAdapterOptions options = {}; - - // probe for adapter support - ctx->webgpu_global_ctx->instance.WaitAny( - ctx->webgpu_global_ctx->instance.RequestAdapter( - &options, wgpu::CallbackMode::AllowSpontaneous, - [&adapter](wgpu::RequestAdapterStatus status, wgpu::Adapter _adapter, const char * message) { - if (status != wgpu::RequestAdapterStatus::Success) { - GGML_LOG_ERROR("ggml_webgpu: Failed to get an adapter: %s\n", message); - return; - } - adapter = std::move(_adapter); - }), - UINT64_MAX); + ggml_backend_webgpu_request_adapter(ctx->webgpu_global_ctx->instance, adapter); } // WebGPU backend requires f16 support and, on native, implicit device synchronization.