also show model aliases

Co-authored-by: Piotr Wilkin (ilintar) <piotr.wilkin@syndatis.com>

common/arg.cpp

@@ -27,7 +27,6 @@
 #include <cinttypes>
 #include <climits>
 #include <cstdarg>
-#include <filesystem>
 #include <fstream>
 #include <list>
 #include <regex>
@@ -719,9 +718,8 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
 
         // model is required (except for server)
         // TODO @ngxson : maybe show a list of available models in CLI in this case
-        if (params.model.path.empty()
-                && !params.usage
-                && !params.completion) {
+        bool can_skip_model = params.usage || params.completion || !params.server_base.empty();
+        if (!can_skip_model && params.model.path.empty()) {
             throw std::invalid_argument("error: --model is required\n");
         }
     }
@@ -1241,6 +1239,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.completion = true;
         }
     ));
+    add_opt(common_arg(
+        {"--server-base"}, "URL",
+        string_format("connect to this server instead of starting a new one, example: 'http://localhost:8080' (default: none)"),
+        [](common_params & params, const std::string & value) {
+            params.server_base = value;
+        }
+    ).set_examples({LLAMA_EXAMPLE_CLI}));
     add_opt(common_arg(
         {"--verbose-prompt"},
         string_format("print a verbose prompt before generation (default: %s)", params.verbose_prompt ? "true" : "false"),
@@ -3452,14 +3457,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_env("LLAMA_ARG_LOG_FILE"));
     add_opt(common_arg(
         {"--log-prompts-dir"}, "PATH",
-        "Log prompts to directory (auto-created if not present; only used for debugging, default: disabled)",
+        "Log prompts to directory (only used for debugging, default: disabled)",
         [](common_params & params, const std::string & value) {
             params.path_prompts_log_dir = value;
-            std::error_code ec;
-            std::filesystem::create_directories(value, ec);
-            if (ec) {
-                fprintf(stderr, "warning: failed to create prompts-log-dir '%s': %s\n", value.c_str(), ec.message().c_str());
-            }
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
     add_opt(common_arg(

common/common.h

@@ -644,6 +644,9 @@ struct common_params {
 
     std::map<std::string, std::string> default_template_kwargs;
 
+    // CLI params
+    std::string server_base; // if set, connect to this server instead of starting a new one
+
     // UI configs
     bool ui = true;
     bool ui_mcp_proxy = false;

common/http.h

@@ -2,6 +2,16 @@
 
 #include <cpp-httplib/httplib.h>
 
+#ifdef _WIN32
+#include <winsock2.h>
+#include <windows.h>
+#else
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#endif
+
 struct common_http_url {
     std::string scheme;
     std::string user;
@@ -119,3 +129,63 @@ static std::pair<httplib::Client, common_http_url> common_http_client(const std:
 static std::string common_http_show_masked_url(const common_http_url & parts) {
     return parts.scheme + "://" + (parts.user.empty() ? "" : "****:****@") + common_http_format_host(parts.host) + parts.path;
 }
+
+static int common_http_get_free_port() {
+#ifdef _WIN32
+    WSADATA wsaData;
+    if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) {
+        return -1;
+    }
+    typedef SOCKET native_socket_t;
+#define INVALID_SOCKET_VAL INVALID_SOCKET
+#define CLOSE_SOCKET(s) closesocket(s)
+#else
+    typedef int native_socket_t;
+#define INVALID_SOCKET_VAL -1
+#define CLOSE_SOCKET(s) close(s)
+#endif
+
+    native_socket_t sock = socket(AF_INET, SOCK_STREAM, 0);
+    if (sock == INVALID_SOCKET_VAL) {
+#ifdef _WIN32
+        WSACleanup();
+#endif
+        return -1;
+    }
+
+    struct sockaddr_in serv_addr;
+    std::memset(&serv_addr, 0, sizeof(serv_addr));
+    serv_addr.sin_family = AF_INET;
+    serv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
+    serv_addr.sin_port = htons(0);
+
+    if (bind(sock, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) != 0) {
+        CLOSE_SOCKET(sock);
+#ifdef _WIN32
+        WSACleanup();
+#endif
+        return -1;
+    }
+
+#ifdef _WIN32
+    int namelen = sizeof(serv_addr);
+#else
+    socklen_t namelen = sizeof(serv_addr);
+#endif
+    if (getsockname(sock, (struct sockaddr*)&serv_addr, &namelen) != 0) {
+        CLOSE_SOCKET(sock);
+#ifdef _WIN32
+        WSACleanup();
+#endif
+        return -1;
+    }
+
+    int port = ntohs(serv_addr.sin_port);
+
+    CLOSE_SOCKET(sock);
+#ifdef _WIN32
+    WSACleanup();
+#endif
+
+    return port;
+}

examples/llama-eval/llama-eval.py

@@ -362,7 +362,7 @@ class EvalState:
             case = cases.get(task_id, {})
             status = case.get("status", "pending")
             expected = case.get("expected", "")
-            answer = case.get("answer") or "" if status == "ok" else ""
+            answer = case.get("answer", "") if status == "ok" else ""
             is_correct = case.get("correct", False) if status == "ok" else False
             response = case.get("response", "") or ""
             prompt = case.get("prompt", "") or ""
@@ -647,7 +647,7 @@ class EvalState:
             question, prompt, expected = self.get_case(i)
             case = cases.get(task_id, {})
             status = case.get("status", "pending")
-            answer = case.get("answer") or "N/A" if status == "ok" else "N/A"
+            answer = case.get("answer", "N/A") if status == "ok" else "N/A"
             tokens = case.get("tokens")
             tokens_str = str(tokens) if tokens is not None else "N/A"
             tps_gen = case.get("tps_gen")

ggml/src/ggml-cpu/ops.cpp

@@ -5025,8 +5025,8 @@ void ggml_compute_forward_get_rows(
     //}
 }
 
-template<typename src_t, typename idx_t>
-static void ggml_compute_forward_set_rows_impl(
+template<typename idx_t>
+static void ggml_compute_forward_set_rows_f32(
         const ggml_compute_params * params,
               ggml_tensor * dst) {
 
@@ -5041,7 +5041,7 @@ static void ggml_compute_forward_set_rows_impl(
     assert(ne0  == nc);
     assert(ne2  == ne02);
     assert(ne3  == ne03);
-    GGML_ASSERT(src0->type == GGML_TYPE_F32 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16));
+    assert(src0->type == GGML_TYPE_F32);
     assert(ne02 % ne11 == 0);
     assert(ne03 % ne12 == 0);
 
@@ -5055,8 +5055,6 @@ static void ggml_compute_forward_set_rows_impl(
     const int64_t ir0 = dr*ith;
     const int64_t ir1 = std::min(ir0 + dr, nr);
 
-    const size_t rs = ggml_row_size(src0->type, nc);
-
     ggml_from_float_t const from_float = ggml_get_type_traits_cpu(dst->type)->from_float;
 
     for (int64_t i03 = 0; i03 < ne03; ++i03) {
@@ -5070,18 +5068,9 @@ static void ggml_compute_forward_set_rows_impl(
 
                 GGML_ASSERT(i1 >= 0 && i1 < ne1);
 
-                if constexpr (std::is_same_v<src_t, float>) {
-                    from_float(
-                            (const float *) ((char *) src0->data +  i*nb01 + i02*nb02 + i03*nb03),
-                                            ((char *)  dst->data + i1*nb1  + i02*nb2  + i03*nb3), nc);
-                } else if constexpr (std::is_same_v<src_t, ggml_fp16_t>) {
-                    memcpy(
-                                            ((char *)  dst->data + i1*nb1  + i02*nb2  + i03*nb3),
-                                            ((char *) src0->data +  i*nb01 + i02*nb02 + i03*nb03),
-                                            rs);
-                } else {
-                    GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type));
-                }
+                from_float(
+                        (const float *) ((char *) src0->data +  i*nb01 + i02*nb02 + i03*nb03),
+                                        ((char *)  dst->data + i1*nb1  + i02*nb2  + i03*nb3), nc);
             }
         }
     }
@@ -5098,27 +5087,13 @@ void ggml_compute_forward_set_rows(
         case GGML_TYPE_F32:
             {
                 if (src1->type == GGML_TYPE_I64) {
-                    ggml_compute_forward_set_rows_impl<float, int64_t>(params, dst);
+                    ggml_compute_forward_set_rows_f32<int64_t>(params, dst);
                 } else if (src1->type == GGML_TYPE_I32) {
-                    ggml_compute_forward_set_rows_impl<float, int32_t>(params, dst);
+                    ggml_compute_forward_set_rows_f32<int32_t>(params, dst);
                 } else {
                     GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type));
                 }
             } break;
-        case GGML_TYPE_F16:
-            {
-                if (dst->type == GGML_TYPE_F16) {
-                    if (src1->type == GGML_TYPE_I64) {
-                        ggml_compute_forward_set_rows_impl<ggml_fp16_t, int64_t>(params, dst);
-                    } else if (src1->type == GGML_TYPE_I32) {
-                        ggml_compute_forward_set_rows_impl<ggml_fp16_t, int32_t>(params, dst);
-                    } else {
-                        GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type));
-                    }
-                } else {
-                    GGML_ABORT("dst->type = %d (%s) not supported with src0->type = %d (%s)", dst->type, ggml_type_name(dst->type), src0->type, ggml_type_name(src0->type));
-                }
-            } break;
         default:
             {
                 GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type));

ggml/src/ggml-cpu/simd-gemm.h

@@ -78,7 +78,7 @@ static void simd_gemm(
             for (int64_t i = 0; i < GEMM_RM; i++) {
                 float a = C[i * N + jj];
                 for (int64_t kk = 0; kk < K; kk++) {
-                    a += A[i * K + kk] * B[kk * N + jj];
+                    a += A[i + kk] * B[kk * N + jj];
                 }
                 C[i * N + jj] = a;
             }

ggml/src/ggml-metal/ggml-metal-device.cpp

@@ -160,15 +160,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_get_rows(ggml_me
     return res;
 }
 
-ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows(ggml_metal_library_t lib, const ggml_tensor * op) {
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows(ggml_metal_library_t lib, ggml_type tidx, ggml_type tdst) {
     char base[256];
     char name[256];
 
-    const auto tsrc = op->src[0]->type;
-    const auto tidx = op->src[1]->type;
-    const auto tdst = op->type;
-
-    snprintf(base, 256, "kernel_set_rows_%s_%s_%s", ggml_type_name(tsrc), ggml_type_name(tidx), ggml_type_name(tdst));
+    snprintf(base, 256, "kernel_set_rows_%s_%s", ggml_type_name(tdst), ggml_type_name(tidx));
     snprintf(name, 256, "%s", base);
 
     ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);

ggml/src/ggml-metal/ggml-metal-device.h

@@ -112,7 +112,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cpy
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pool_1d           (ggml_metal_library_t lib, const struct ggml_tensor * op, enum ggml_op_pool op_pool);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pool_2d           (ggml_metal_library_t lib, const struct ggml_tensor * op, enum ggml_op_pool op_pool);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_get_rows          (ggml_metal_library_t lib, enum ggml_type tsrc);
-struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows          (ggml_metal_library_t lib, const struct ggml_tensor * op);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows          (ggml_metal_library_t lib, enum ggml_type tidx, enum ggml_type tdst);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_diag              (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat            (ggml_metal_library_t lib, enum ggml_type tsrc);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_concat            (ggml_metal_library_t lib, enum ggml_type tsrc);

ggml/src/ggml-metal/ggml-metal-device.m

@@ -1334,7 +1334,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
             return op->src[0]->type != GGML_TYPE_NVFP4;
         case GGML_OP_SET_ROWS:
             {
-                if (op->src[0]->type != GGML_TYPE_F32 && op->src[0]->type != GGML_TYPE_F16) {
+                if (op->src[0]->type != GGML_TYPE_F32) {
                     return false;
                 }
 

ggml/src/ggml-metal/ggml-metal-ops.cpp

@@ -1202,7 +1202,7 @@ int ggml_metal_op_set_rows(ggml_metal_op_t ctx, int idx) {
     GGML_TENSOR_LOCALS( int32_t, ne,  op,         ne);
     GGML_TENSOR_LOCALS(uint64_t, nb,  op,         nb);
 
-    auto pipeline = ggml_metal_library_get_pipeline_set_rows(lib, op);
+    auto pipeline = ggml_metal_library_get_pipeline_set_rows(lib, op->src[1]->type, op->type);
 
     const int32_t nk0 = ne0/ggml_blck_size(op->type);
 

ggml/src/ggml-metal/ggml-metal.metal

@@ -42,8 +42,6 @@ typedef matrix<bfloat, 4, 4> bfloat4x4;
 typedef matrix<bfloat, 2, 4> bfloat2x4;
 #endif
 
-#define QK_NL 16
-
 constexpr constant static float kvalues_iq4nl_f[16] = {
     -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f
 };
@@ -9388,40 +9386,7 @@ kernel void kernel_get_rows_f(
     }
 }
 
-typedef decltype(kernel_get_rows_f<float, float>) get_rows_f_t;
-
-template [[host_name("kernel_get_rows_f32")]]  kernel get_rows_f_t kernel_get_rows_f<float, float>;
-template [[host_name("kernel_get_rows_f16")]]  kernel get_rows_f_t kernel_get_rows_f<half,  float>;
-template [[host_name("kernel_get_rows_i32")]]  kernel get_rows_f_t kernel_get_rows_f<int32_t, int32_t>;
-#if defined(GGML_METAL_HAS_BF16)
-template [[host_name("kernel_get_rows_bf16")]] kernel get_rows_f_t kernel_get_rows_f<bfloat, float>;
-#endif
-
-typedef decltype(kernel_get_rows_q<block_q4_0, 2, dequantize_q4_0>) get_rows_q_t;
-
-template [[host_name("kernel_get_rows_q1_0")]]    kernel get_rows_q_t kernel_get_rows_q<block_q1_0,    8, dequantize_q1_0>;
-template [[host_name("kernel_get_rows_q4_0")]]    kernel get_rows_q_t kernel_get_rows_q<block_q4_0,    2, dequantize_q4_0>;
-template [[host_name("kernel_get_rows_q4_1")]]    kernel get_rows_q_t kernel_get_rows_q<block_q4_1,    2, dequantize_q4_1>;
-template [[host_name("kernel_get_rows_q5_0")]]    kernel get_rows_q_t kernel_get_rows_q<block_q5_0,    2, dequantize_q5_0>;
-template [[host_name("kernel_get_rows_q5_1")]]    kernel get_rows_q_t kernel_get_rows_q<block_q5_1,    2, dequantize_q5_1>;
-template [[host_name("kernel_get_rows_q8_0")]]    kernel get_rows_q_t kernel_get_rows_q<block_q8_0,    2, dequantize_q8_0>;
-template [[host_name("kernel_get_rows_mxfp4")]]   kernel get_rows_q_t kernel_get_rows_q<block_mxfp4,   2, dequantize_mxfp4>;
-template [[host_name("kernel_get_rows_q2_K")]]    kernel get_rows_q_t kernel_get_rows_q<block_q2_K,    QK_NL, dequantize_q2_K>;
-template [[host_name("kernel_get_rows_q3_K")]]    kernel get_rows_q_t kernel_get_rows_q<block_q3_K,    QK_NL, dequantize_q3_K>;
-template [[host_name("kernel_get_rows_q4_K")]]    kernel get_rows_q_t kernel_get_rows_q<block_q4_K,    QK_NL, dequantize_q4_K>;
-template [[host_name("kernel_get_rows_q5_K")]]    kernel get_rows_q_t kernel_get_rows_q<block_q5_K,    QK_NL, dequantize_q5_K>;
-template [[host_name("kernel_get_rows_q6_K")]]    kernel get_rows_q_t kernel_get_rows_q<block_q6_K,    QK_NL, dequantize_q6_K>;
-template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_q_t kernel_get_rows_q<block_iq2_xxs, QK_NL, dequantize_iq2_xxs>;
-template [[host_name("kernel_get_rows_iq2_xs")]]  kernel get_rows_q_t kernel_get_rows_q<block_iq2_xs,  QK_NL, dequantize_iq2_xs>;
-template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_q_t kernel_get_rows_q<block_iq3_xxs, QK_NL, dequantize_iq3_xxs>;
-template [[host_name("kernel_get_rows_iq3_s")]]   kernel get_rows_q_t kernel_get_rows_q<block_iq3_s,   QK_NL, dequantize_iq3_s>;
-template [[host_name("kernel_get_rows_iq2_s")]]   kernel get_rows_q_t kernel_get_rows_q<block_iq2_s,   QK_NL, dequantize_iq2_s>;
-template [[host_name("kernel_get_rows_iq1_s")]]   kernel get_rows_q_t kernel_get_rows_q<block_iq1_s,   QK_NL, dequantize_iq1_s>;
-template [[host_name("kernel_get_rows_iq1_m")]]   kernel get_rows_q_t kernel_get_rows_q<block_iq1_m,   QK_NL, dequantize_iq1_m>;
-template [[host_name("kernel_get_rows_iq4_nl")]]  kernel get_rows_q_t kernel_get_rows_q<block_iq4_nl,  2,     dequantize_iq4_nl>;
-template [[host_name("kernel_get_rows_iq4_xs")]]  kernel get_rows_q_t kernel_get_rows_q<block_iq4_xs,  QK_NL, dequantize_iq4_xs>;
-
-template<typename TS, typename TI, typename block_q, void (*quantize_func)(device const float *, device block_q &)>
+template<typename TI, typename block_q, void (*quantize_func)(device const float *, device block_q &)>
 kernel void kernel_set_rows_q32(
         constant ggml_metal_kargs_set_rows & args,
         device const  void * src0,
@@ -9445,14 +9410,14 @@ kernel void kernel_set_rows_q32(
     const TI      i1  = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0];
 
           device block_q * dst_row = (      device block_q *) ((      device char *) dst  +  i1*args.nb1  + i02*args.nb2  + i03*args.nb3);
-    const device TS      * src_row = (const device TS      *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03);
+    const device float   * src_row = (const device float   *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03);
 
     for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) {
         quantize_func(src_row + 32*ind, dst_row[ind]);
     }
 }
 
-template<typename TS, typename TI, typename TD>
+template<typename T, typename TI>
 kernel void kernel_set_rows_f(
         constant ggml_metal_kargs_set_rows & args,
         device const  void * src0,
@@ -9475,47 +9440,14 @@ kernel void kernel_set_rows_f(
     const int32_t i10 = i01;
     const TI      i1  = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0];
 
-          device TD * dst_row = (      device TD *) ((      device char *) dst  +  i1*args.nb1  + i02*args.nb2  + i03*args.nb3);
-    const device TS * src_row = (const device TS *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03);
+          device T     * dst_row = (      device T     *) ((      device char *) dst  +  i1*args.nb1  + i02*args.nb2  + i03*args.nb3);
+    const device float * src_row = (const device float *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03);
 
     for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) {
-        dst_row[ind] = (TD) src_row[ind];
+        dst_row[ind] = (T) src_row[ind];
     }
 }
 
-typedef decltype(kernel_set_rows_f<float, int64_t, float>) set_rows_f_t;
-
-template [[host_name("kernel_set_rows_f32_i64_f32")]]   kernel set_rows_f_t kernel_set_rows_f<float, int64_t, float>;
-template [[host_name("kernel_set_rows_f32_i32_f32")]]   kernel set_rows_f_t kernel_set_rows_f<float, int32_t, float>;
-template [[host_name("kernel_set_rows_f32_i64_f16")]]   kernel set_rows_f_t kernel_set_rows_f<float, int64_t, half>;
-template [[host_name("kernel_set_rows_f32_i32_f16")]]   kernel set_rows_f_t kernel_set_rows_f<float, int32_t, half>;
-#if defined(GGML_METAL_HAS_BF16)
-template [[host_name("kernel_set_rows_f32_i64_bf16")]]  kernel set_rows_f_t kernel_set_rows_f<float, int64_t, bfloat>;
-template [[host_name("kernel_set_rows_f32_i32_bf16")]]  kernel set_rows_f_t kernel_set_rows_f<float, int32_t, bfloat>;
-#endif
-
-template [[host_name("kernel_set_rows_f16_i64_f16")]]   kernel set_rows_f_t kernel_set_rows_f<half, int64_t, half>;
-template [[host_name("kernel_set_rows_f16_i32_f16")]]   kernel set_rows_f_t kernel_set_rows_f<half, int32_t, half>;
-#if defined(GGML_METAL_HAS_BF16)
-template [[host_name("kernel_set_rows_bf16_i64_bf16")]] kernel set_rows_f_t kernel_set_rows_f<bfloat, int64_t, bfloat>;
-template [[host_name("kernel_set_rows_bf16_i32_bf16")]] kernel set_rows_f_t kernel_set_rows_f<bfloat, int32_t, bfloat>;
-#endif
-
-typedef decltype(kernel_set_rows_q32<float, int64_t, block_q8_0, quantize_q8_0>) set_rows_q32_t;
-
-template [[host_name("kernel_set_rows_f32_i64_q8_0")]]   kernel set_rows_q32_t kernel_set_rows_q32<float, int64_t, block_q8_0,   quantize_q8_0>;
-template [[host_name("kernel_set_rows_f32_i32_q8_0")]]   kernel set_rows_q32_t kernel_set_rows_q32<float, int32_t, block_q8_0,   quantize_q8_0>;
-template [[host_name("kernel_set_rows_f32_i64_q4_0")]]   kernel set_rows_q32_t kernel_set_rows_q32<float, int64_t, block_q4_0,   quantize_q4_0>;
-template [[host_name("kernel_set_rows_f32_i32_q4_0")]]   kernel set_rows_q32_t kernel_set_rows_q32<float, int32_t, block_q4_0,   quantize_q4_0>;
-template [[host_name("kernel_set_rows_f32_i64_q4_1")]]   kernel set_rows_q32_t kernel_set_rows_q32<float, int64_t, block_q4_1,   quantize_q4_1>;
-template [[host_name("kernel_set_rows_f32_i32_q4_1")]]   kernel set_rows_q32_t kernel_set_rows_q32<float, int32_t, block_q4_1,   quantize_q4_1>;
-template [[host_name("kernel_set_rows_f32_i64_q5_0")]]   kernel set_rows_q32_t kernel_set_rows_q32<float, int64_t, block_q5_0,   quantize_q5_0>;
-template [[host_name("kernel_set_rows_f32_i32_q5_0")]]   kernel set_rows_q32_t kernel_set_rows_q32<float, int32_t, block_q5_0,   quantize_q5_0>;
-template [[host_name("kernel_set_rows_f32_i64_q5_1")]]   kernel set_rows_q32_t kernel_set_rows_q32<float, int64_t, block_q5_1,   quantize_q5_1>;
-template [[host_name("kernel_set_rows_f32_i32_q5_1")]]   kernel set_rows_q32_t kernel_set_rows_q32<float, int32_t, block_q5_1,   quantize_q5_1>;
-template [[host_name("kernel_set_rows_f32_i64_iq4_nl")]] kernel set_rows_q32_t kernel_set_rows_q32<float, int64_t, block_iq4_nl, quantize_iq4_nl>;
-template [[host_name("kernel_set_rows_f32_i32_iq4_nl")]] kernel set_rows_q32_t kernel_set_rows_q32<float, int32_t, block_iq4_nl, quantize_iq4_nl>;
-
 kernel void kernel_diag_f32(
         constant ggml_metal_kargs_diag & args,
         device   const char * src0,
@@ -10258,6 +10190,75 @@ kernel void kernel_mul_mm_id(
     }
 }
 
+#define QK_NL 16
+
+//
+// get rows
+//
+
+typedef decltype(kernel_get_rows_f<float, float>) get_rows_f_t;
+
+template [[host_name("kernel_get_rows_f32")]]  kernel get_rows_f_t kernel_get_rows_f<float, float>;
+template [[host_name("kernel_get_rows_f16")]]  kernel get_rows_f_t kernel_get_rows_f<half,  float>;
+template [[host_name("kernel_get_rows_i32")]]  kernel get_rows_f_t kernel_get_rows_f<int32_t, int32_t>;
+#if defined(GGML_METAL_HAS_BF16)
+template [[host_name("kernel_get_rows_bf16")]] kernel get_rows_f_t kernel_get_rows_f<bfloat, float>;
+#endif
+
+typedef decltype(kernel_get_rows_q<block_q4_0, 2, dequantize_q4_0>) get_rows_q_t;
+
+template [[host_name("kernel_get_rows_q1_0")]]    kernel get_rows_q_t kernel_get_rows_q<block_q1_0,    8, dequantize_q1_0>;
+template [[host_name("kernel_get_rows_q4_0")]]    kernel get_rows_q_t kernel_get_rows_q<block_q4_0,    2, dequantize_q4_0>;
+template [[host_name("kernel_get_rows_q4_1")]]    kernel get_rows_q_t kernel_get_rows_q<block_q4_1,    2, dequantize_q4_1>;
+template [[host_name("kernel_get_rows_q5_0")]]    kernel get_rows_q_t kernel_get_rows_q<block_q5_0,    2, dequantize_q5_0>;
+template [[host_name("kernel_get_rows_q5_1")]]    kernel get_rows_q_t kernel_get_rows_q<block_q5_1,    2, dequantize_q5_1>;
+template [[host_name("kernel_get_rows_q8_0")]]    kernel get_rows_q_t kernel_get_rows_q<block_q8_0,    2, dequantize_q8_0>;
+template [[host_name("kernel_get_rows_mxfp4")]]   kernel get_rows_q_t kernel_get_rows_q<block_mxfp4,   2, dequantize_mxfp4>;
+template [[host_name("kernel_get_rows_q2_K")]]    kernel get_rows_q_t kernel_get_rows_q<block_q2_K,    QK_NL, dequantize_q2_K>;
+template [[host_name("kernel_get_rows_q3_K")]]    kernel get_rows_q_t kernel_get_rows_q<block_q3_K,    QK_NL, dequantize_q3_K>;
+template [[host_name("kernel_get_rows_q4_K")]]    kernel get_rows_q_t kernel_get_rows_q<block_q4_K,    QK_NL, dequantize_q4_K>;
+template [[host_name("kernel_get_rows_q5_K")]]    kernel get_rows_q_t kernel_get_rows_q<block_q5_K,    QK_NL, dequantize_q5_K>;
+template [[host_name("kernel_get_rows_q6_K")]]    kernel get_rows_q_t kernel_get_rows_q<block_q6_K,    QK_NL, dequantize_q6_K>;
+template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_q_t kernel_get_rows_q<block_iq2_xxs, QK_NL, dequantize_iq2_xxs>;
+template [[host_name("kernel_get_rows_iq2_xs")]]  kernel get_rows_q_t kernel_get_rows_q<block_iq2_xs,  QK_NL, dequantize_iq2_xs>;
+template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_q_t kernel_get_rows_q<block_iq3_xxs, QK_NL, dequantize_iq3_xxs>;
+template [[host_name("kernel_get_rows_iq3_s")]]   kernel get_rows_q_t kernel_get_rows_q<block_iq3_s,   QK_NL, dequantize_iq3_s>;
+template [[host_name("kernel_get_rows_iq2_s")]]   kernel get_rows_q_t kernel_get_rows_q<block_iq2_s,   QK_NL, dequantize_iq2_s>;
+template [[host_name("kernel_get_rows_iq1_s")]]   kernel get_rows_q_t kernel_get_rows_q<block_iq1_s,   QK_NL, dequantize_iq1_s>;
+template [[host_name("kernel_get_rows_iq1_m")]]   kernel get_rows_q_t kernel_get_rows_q<block_iq1_m,   QK_NL, dequantize_iq1_m>;
+template [[host_name("kernel_get_rows_iq4_nl")]]  kernel get_rows_q_t kernel_get_rows_q<block_iq4_nl,  2,     dequantize_iq4_nl>;
+template [[host_name("kernel_get_rows_iq4_xs")]]  kernel get_rows_q_t kernel_get_rows_q<block_iq4_xs,  QK_NL, dequantize_iq4_xs>;
+
+//
+// set rows
+//
+
+typedef decltype(kernel_set_rows_f<float, int64_t>) set_rows_f_t;
+
+template [[host_name("kernel_set_rows_f32_i64")]]  kernel set_rows_f_t kernel_set_rows_f<float, int64_t>;
+template [[host_name("kernel_set_rows_f32_i32")]]  kernel set_rows_f_t kernel_set_rows_f<float, int32_t>;
+template [[host_name("kernel_set_rows_f16_i64")]]  kernel set_rows_f_t kernel_set_rows_f<half, int64_t>;
+template [[host_name("kernel_set_rows_f16_i32")]]  kernel set_rows_f_t kernel_set_rows_f<half, int32_t>;
+#if defined(GGML_METAL_HAS_BF16)
+template [[host_name("kernel_set_rows_bf16_i64")]] kernel set_rows_f_t kernel_set_rows_f<bfloat, int64_t>;
+template [[host_name("kernel_set_rows_bf16_i32")]] kernel set_rows_f_t kernel_set_rows_f<bfloat, int32_t>;
+#endif
+
+typedef decltype(kernel_set_rows_q32<int64_t, block_q8_0, quantize_q8_0>) set_rows_q32_t;
+
+template [[host_name("kernel_set_rows_q8_0_i64")]]   kernel set_rows_q32_t kernel_set_rows_q32<int64_t, block_q8_0,   quantize_q8_0>;
+template [[host_name("kernel_set_rows_q8_0_i32")]]   kernel set_rows_q32_t kernel_set_rows_q32<int32_t, block_q8_0,   quantize_q8_0>;
+template [[host_name("kernel_set_rows_q4_0_i64")]]   kernel set_rows_q32_t kernel_set_rows_q32<int64_t, block_q4_0,   quantize_q4_0>;
+template [[host_name("kernel_set_rows_q4_0_i32")]]   kernel set_rows_q32_t kernel_set_rows_q32<int32_t, block_q4_0,   quantize_q4_0>;
+template [[host_name("kernel_set_rows_q4_1_i64")]]   kernel set_rows_q32_t kernel_set_rows_q32<int64_t, block_q4_1,   quantize_q4_1>;
+template [[host_name("kernel_set_rows_q4_1_i32")]]   kernel set_rows_q32_t kernel_set_rows_q32<int32_t, block_q4_1,   quantize_q4_1>;
+template [[host_name("kernel_set_rows_q5_0_i64")]]   kernel set_rows_q32_t kernel_set_rows_q32<int64_t, block_q5_0,   quantize_q5_0>;
+template [[host_name("kernel_set_rows_q5_0_i32")]]   kernel set_rows_q32_t kernel_set_rows_q32<int32_t, block_q5_0,   quantize_q5_0>;
+template [[host_name("kernel_set_rows_q5_1_i64")]]   kernel set_rows_q32_t kernel_set_rows_q32<int64_t, block_q5_1,   quantize_q5_1>;
+template [[host_name("kernel_set_rows_q5_1_i32")]]   kernel set_rows_q32_t kernel_set_rows_q32<int32_t, block_q5_1,   quantize_q5_1>;
+template [[host_name("kernel_set_rows_iq4_nl_i64")]] kernel set_rows_q32_t kernel_set_rows_q32<int64_t, block_iq4_nl, quantize_iq4_nl>;
+template [[host_name("kernel_set_rows_iq4_nl_i32")]] kernel set_rows_q32_t kernel_set_rows_q32<int32_t, block_iq4_nl, quantize_iq4_nl>;
+
 //
 // matrix-matrix multiplication
 //

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -16653,7 +16653,6 @@ static cl_mem ggml_cl_mul_mat_dequant_quant_to_f16(
         ? ggml_cl_is_q4_0_soa(tensor)
         : ggml_cl_is_q8_0_soa(tensor);
 
-    cl_mem aos = nullptr;
     if (is_soa) {
         // Reconstruct full parent AoS; view's own nb[] then index it correctly.
         const ggml_tensor * parent = tensor->view_src ? tensor->view_src : tensor;
@@ -16665,7 +16664,7 @@ static cl_mem ggml_cl_mul_mat_dequant_quant_to_f16(
         const size_t parent_nbytes = (size_t) ggml_nelements(parent) / blck_size * block_bytes;
 
         cl_int err;
-        aos = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, parent_nbytes, NULL, &err);
+        cl_mem aos = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, parent_nbytes, NULL, &err);
         CL_CHECK(err);
 
         // large q4_0/q8_0 WEIGHTS are stored transposed and small weights
@@ -16752,6 +16751,9 @@ static cl_mem ggml_cl_mul_mat_dequant_quant_to_f16(
 
         if (extra_reconstruct) {
             *extra_reconstruct = aos;
+        } else {
+            // OpenCL retains the memobj while queued kernels reference it.
+            CL_CHECK(clReleaseMemObject(aos));
         }
     } else {
         auto * extra = (ggml_tensor_extra_cl *) tensor->extra;
@@ -16815,13 +16817,6 @@ static cl_mem ggml_cl_mul_mat_dequant_quant_to_f16(
     size_t lws[3] = { 1, 1, 1 };
     CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, dq_kernel, 3, NULL, gws, lws, 0, NULL, NULL));
 
-    // release the reconstructed aos if
-    //  1. it was actually reconstructed
-    //  2. the caller didn't request it to be returned
-    // src_buf may refer to aos, so we should release after this enqueue
-    if (aos && !extra_reconstruct) {
-        CL_CHECK(clReleaseMemObject(aos));
-    }
     return out;
 }
 

ggml/src/ggml.c

@@ -3926,7 +3926,7 @@ struct ggml_tensor * ggml_set_rows(
     GGML_ASSERT(b->ne[2] % c->ne[1] == 0);
     GGML_ASSERT(b->ne[3] % c->ne[2] == 0);
     GGML_ASSERT(c->ne[3] == 1);
-    GGML_ASSERT(b->type == GGML_TYPE_F32 || b->type == GGML_TYPE_F16);
+    GGML_ASSERT(b->type == GGML_TYPE_F32);
     GGML_ASSERT(c->type == GGML_TYPE_I64 || c->type == GGML_TYPE_I32);
 
     GGML_ASSERT(ggml_is_contiguous_rows(a));

src/llama-batch.cpp

@@ -505,7 +505,7 @@ llama_ubatch llama_batch_allocr::split_simple(uint32_t n_ubatch) {
     return ubatch_add(idxs, idxs.size(), false);
 }
 
-llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail) {
+llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential) {
     if (sequential && has_cpl) {
         LLAMA_LOG_ERROR("%s: sequential split is not supported when there are coupled sequences in the input batch (you may need to use the -kvu flag)\n", __func__);
 
@@ -548,7 +548,7 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential,
         }
     }
 
-    uint32_t n_seqs = cur_seq_set.size();
+    const uint32_t n_seqs = cur_seq_set.size();
 
     // we are done
     if (n_seqs == 0) {
@@ -569,7 +569,7 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential,
     std::vector<idx_vec_t> idxs_per_seq(n_seqs);
 
     while (true) {
-        // we can only add new n_seq_tokens tokens if all the sequence sets have at least 1 more unused tokens and
+        // we can only add new n_seq_tokens tokens if all the sequence sets have at least one more unused token and
         //   if we haven't reached n_ubatch
         bool can_expand = true;
 
@@ -600,72 +600,6 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential,
         }
     }
 
-    // if n_keep_tail > 0, keep only the seqs that either finish in this ubatch or have at least
-    //   n_keep_tail tokens remaining for a future ubatch, so that the trailing n_keep_tail tokens
-    //   of each seq are never split across ubatches
-    if (n_keep_tail > 0) {
-        GGML_ASSERT(n_ubatch > n_keep_tail);
-
-        auto n_remaining = [&](uint32_t s) {
-            return (uint32_t) (seq_set_map[cur_seq_set[s]].size() - cur_idx[s]);
-        };
-
-        // keep the longest prefix of seqs that satisfy the constraint, to preserve sequential seq ids
-        uint32_t n_keep = 0;
-        while (n_keep < n_seqs) {
-            const uint32_t remaining = n_remaining(n_keep);
-
-            if (remaining != 0 && remaining < n_keep_tail) {
-                break;
-            }
-
-            n_keep++;
-        }
-
-        // all seqs violate the constraint - resolve the first one directly and emit it alone
-        if (n_keep == 0) {
-            auto & idxs = idxs_per_seq[0];
-
-            const auto & seq_idxs = seq_set_map[cur_seq_set[0]];
-
-            if (idxs.size() + n_remaining(0) <= n_ubatch) {
-                // extend the seq to completion
-                while (n_remaining(0) > 0) {
-                    const int32_t idx = seq_idxs[cur_idx[0]];
-
-                    idxs.push_back(idx);
-
-                    used[idx] = true;
-                    ++n_used;
-
-                    ++cur_idx[0];
-                }
-            } else {
-                // truncate the seq so that at least n_keep_tail tokens remain
-                while (n_remaining(0) < n_keep_tail) {
-                    used[idxs.back()] = false;
-                    --n_used;
-
-                    idxs.pop_back();
-
-                    --cur_idx[0];
-                }
-            }
-
-            n_keep = 1;
-        }
-
-        // return the tokens of the deferred seqs back to the pool
-        for (uint32_t s = n_keep; s < n_seqs; ++s) {
-            for (const int32_t idx : idxs_per_seq[s]) {
-                used[idx] = false;
-                --n_used;
-            }
-        }
-
-        n_seqs = n_keep;
-    }
-
     // concat the per-sequence-set lists
     std::vector<int32_t> idxs;
 
@@ -880,7 +814,7 @@ void llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) {
         LLAMA_LOG_DEBUG("%s:   output     = %p\n", __func__, (void *) ubatch.output);
         LLAMA_LOG_DEBUG("%s:   n_outputs  = %d\n", __func__, n_outputs);
 
-        if (debug > 0) {
+        if (debug > 1) {
             int seq_id_max = 0;
             for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {
                 for (int s = 0; s < ubatch.n_seq_id[i]; ++s) {

src/llama-batch.h

@@ -104,8 +104,7 @@ public:
 
     // make ubatches of equal-length sequences sets
     // if sequential == true, the tokens in the ubatch will have increasing sequential sequence ids
-    // n_keep_tail = minimum trailing tokens of a seq that must land in the same ubatch
-    llama_ubatch split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail);
+    llama_ubatch split_equal(uint32_t n_ubatch, bool sequential);
 
     // sequence-set-wise split - each ubatch contains a single sequence-set
     llama_ubatch split_seq(uint32_t n_ubatch);

src/llama-context.cpp

@@ -17,7 +17,6 @@
 #include <cstring>
 #include <limits>
 #include <stdexcept>
-#include <string>
 
 //
 // llama_context
@@ -31,30 +30,6 @@ static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) {
     throw std::runtime_error("Unsupported ctx type");
 }
 
-struct llm_fused_op_probe {
-    llm_fused_op op;
-    const char * name;
-    uint32_t n_tokens_per_seq;
-};
-
-static const llm_fused_op_probe llm_fused_op_flash_attn_probe = {
-    /*.op               =*/ LLM_FUSED_OP_FLASH_ATTN,
-    /*.name             =*/ "Flash Attention",
-    /*.n_tokens_per_seq =*/ 1,
-};
-
-static const llm_fused_op_probe llm_fused_op_gdn_ar_probe = {
-    /*.op               =*/ LLM_FUSED_OP_GDN_AR,
-    /*.name             =*/ "fused Gated Delta Net (autoregressive)",
-    /*.n_tokens_per_seq =*/ 1,
-};
-
-static const llm_fused_op_probe llm_fused_op_gdn_ch_probe = {
-    /*.op               =*/ LLM_FUSED_OP_GDN_CH,
-    /*.name             =*/ "fused Gated Delta Net (chunked)",
-    /*.n_tokens_per_seq =*/ 16,
-};
-
 llama_context::llama_context(
         const llama_model & model,
               llama_context_params params) :
@@ -461,69 +436,6 @@ llama_context::~llama_context() {
     ggml_opt_free(opt_ctx);
 }
 
-void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint32_t n_seqs) {
-    const char * func = __func__;
-    auto resolve = [&](const llm_fused_op_probe & probe, bool & enabled) {
-        if (!enabled) {
-            return;
-        }
-
-        const uint32_t n_tokens_probe = probe.n_tokens_per_seq*n_seqs;
-
-        auto * gf = graph_reserve(n_tokens_probe, n_seqs, n_tokens_probe, mctx, true);
-        if (!gf) {
-            throw std::runtime_error(std::string("failed to reserve graph for ") + probe.name + " check");
-        }
-
-        bool device_mismatch = false;
-        for (const auto & node : get_gf_res_reserve()->get_fused_nodes()) {
-            if (node.op != probe.op) {
-                continue;
-            }
-
-            GGML_ASSERT(node.il >= 0);
-
-            ggml_backend_t backend_fused = ggml_backend_sched_get_tensor_backend(sched.get(), node.tensor);
-            ggml_backend_dev_t device_fused = backend_fused ? ggml_backend_get_device(backend_fused) : nullptr;
-
-            // TODO: make this descriptor-specific; model.dev_layer() preserves the current behavior,
-            // but is still wrong for cases like --no-kv-offload.
-            ggml_backend_dev_t device_layer = model.dev_layer(node.il);
-
-            if (device_fused != device_layer) {
-                LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but %s "
-                        "is assigned to device %s (usually due to missing support)\n",
-                        func, node.il,
-                        device_layer ? ggml_backend_dev_name(device_layer) : "none",
-                        probe.name,
-                        device_fused ? ggml_backend_dev_name(device_fused) : "none");
-                device_mismatch = true;
-                break;
-            }
-        }
-
-        if (device_mismatch) {
-            enabled = false;
-            LLAMA_LOG_WARN("%s: %s not supported, set to disabled\n", func, probe.name);
-        } else {
-            enabled = true;
-            LLAMA_LOG_INFO("%s: %s enabled\n", func, probe.name);
-        }
-    };
-
-    if (cparams.auto_fa) {
-        resolve(llm_fused_op_flash_attn_probe, cparams.flash_attn);
-        cparams.auto_fa = false;
-    }
-
-    if (cparams.auto_fgdn) {
-        LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", func);
-        resolve(llm_fused_op_gdn_ar_probe, cparams.fused_gdn_ar);
-        resolve(llm_fused_op_gdn_ch_probe, cparams.fused_gdn_ch);
-        cparams.auto_fgdn = false;
-    }
-}
-
 void llama_context::sched_reserve() {
     if (!sched_need_reserve) {
         return;
@@ -563,7 +475,128 @@ void llama_context::sched_reserve() {
 
     LLAMA_LOG_DEBUG("%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\n", __func__, n_tokens, n_seqs, n_outputs);
 
-    resolve_fused_ops(mctx.get(), n_seqs);
+    // resolve automatic Flash Attention use
+    if (cparams.auto_fa) {
+        auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true);
+        if (!gf) {
+            throw std::runtime_error("failed to reserve graph for Flash Attention check");
+        }
+
+        const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FATTN) + 1;
+        bool fa_device_mismatch = false;
+        for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
+            ggml_tensor * n = ggml_graph_node(gf, i);
+            if (n->op != GGML_OP_FLASH_ATTN_EXT) {
+                continue;
+            }
+            ggml_backend_dev_t device_fa = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));
+
+            // TODO: instead of the tensor names, use a map to keep track of which (FA) tensors belong to which layer
+            GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FATTN "-", prefix_len) == 0);
+            const int il = std::stoi(n->name + prefix_len);
+            ggml_backend_dev_t device_kv = model.dev_layer(il);
+            if (device_fa != device_kv) {
+                LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the Flash Attention tensor "
+                        "is assigned to device %s (usually due to missing support)\n",
+                        __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_fa));
+                // FIXME: fa_device_mismatch logic is wrong for --no-kv-offload, but this is broken anyways
+                fa_device_mismatch = true;
+                break;
+            }
+        }
+
+        if (fa_device_mismatch) {
+            cparams.flash_attn = false;
+            LLAMA_LOG_WARN("%s: Flash Attention was auto, set to disabled\n", __func__);
+        } else {
+            cparams.flash_attn = true;
+            LLAMA_LOG_INFO("%s: Flash Attention was auto, set to enabled\n", __func__);
+        }
+
+        cparams.auto_fa = false;
+    }
+
+    if (cparams.auto_fgdn) {
+        LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", __func__);
+
+        if (cparams.fused_gdn_ar) {
+            auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true);
+            if (!gf) {
+                throw std::runtime_error("failed to reserve graph for fused Gated Delta Net check (autoregressive)");
+            }
+
+            const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_AR) + 1;
+            bool gdn_device_mismatch = false;
+            for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
+                ggml_tensor * n = ggml_graph_node(gf, i);
+                if (n->op != GGML_OP_GATED_DELTA_NET) {
+                    continue;
+                }
+                ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));
+
+                GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_AR "-", prefix_len) == 0);
+                const int il = std::stoi(n->name + prefix_len);
+                ggml_backend_dev_t device_kv = model.dev_layer(il);
+                if (device_gdn != device_kv) {
+                    LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor "
+                            "is assigned to device %s (usually due to missing support)\n",
+                            __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn));
+                    gdn_device_mismatch = true;
+                    break;
+                }
+            }
+
+            if (gdn_device_mismatch) {
+                cparams.fused_gdn_ar = false;
+                LLAMA_LOG_WARN("%s: fused Gated Delta Net (autoregressive) not supported, set to disabled\n", __func__);
+            } else {
+                LLAMA_LOG_INFO("%s: fused Gated Delta Net (autoregressive) enabled\n", __func__);
+            }
+        }
+
+        if (cparams.fused_gdn_ch) {
+            // more than one token in the batch per sequence in order to take the chunked path
+            // note: n_outputs must match n_tokens for embedding models with mean/rank pooling,
+            // because build_pooling creates inp_mean with shape [n_tokens, n_seqs] and multiplies
+            // it with t_embd which is reduced to [n_outputs, ...] via out_ids. if n_outputs != n_tokens,
+            // the ggml_mul_mat assertion fails.
+            const uint32_t n_tokens_ch = 16*n_seqs;
+            auto * gf = graph_reserve(n_tokens_ch, n_seqs, n_tokens_ch, mctx.get(), true);
+            if (!gf) {
+                throw std::runtime_error("failed to reserve graph for fused Gated Delta Net check (chunked)");
+            }
+
+            const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_CH) + 1;
+            bool gdn_device_mismatch = false;
+            for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
+                ggml_tensor * n = ggml_graph_node(gf, i);
+                if (n->op != GGML_OP_GATED_DELTA_NET) {
+                    continue;
+                }
+                ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));
+
+                GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_CH "-", prefix_len) == 0);
+                const int il = std::stoi(n->name + prefix_len);
+                ggml_backend_dev_t device_kv = model.dev_layer(il);
+                if (device_gdn != device_kv) {
+                    LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor "
+                            "is assigned to device %s (usually due to missing support)\n",
+                            __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn));
+                    gdn_device_mismatch = true;
+                    break;
+                }
+            }
+
+            if (gdn_device_mismatch) {
+                cparams.fused_gdn_ch = false;
+                LLAMA_LOG_WARN("%s: fused Gated Delta Net (chunked) not supported, set to disabled\n", __func__);
+            } else {
+                LLAMA_LOG_INFO("%s: fused Gated Delta Net (chunked) enabled\n", __func__);
+            }
+        }
+
+        cparams.auto_fgdn = false;
+    }
 
     // reserve worst-case graph
     int n_splits_pp = -1;

src/llama-context.h

@@ -262,10 +262,6 @@ private:
 
     llm_graph_cb graph_get_cb() const;
 
-    // disable auto fused ops (Flash Attention, Gated Delta Net) whose op lands on a device
-    // that differs from the layer it belongs to (usually due to missing backend support)
-    void resolve_fused_ops(const llama_memory_context_i * mctx, uint32_t n_seqs);
-
     // TODO: read/write lora adapters and cvec
     size_t state_write_data(llama_io_write_i & io);
     size_t state_read_data (llama_io_read_i  & io);

src/llama-graph.cpp

@@ -1192,7 +1192,6 @@ void llm_graph_result::reset() {
     params = {};
 
     inputs.clear();
-    fused_nodes.clear();
 
     buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false));
 
@@ -1294,10 +1293,6 @@ llm_graph_input_i * llm_graph_result::add_input(llm_graph_input_ptr input) {
     return inputs.back().get();
 }
 
-void llm_graph_result::add_fused_node(llm_graph_fused_node result) {
-    fused_nodes.push_back(result);
-}
-
 void llm_graph_result::set_params(const llm_graph_params & params) {
     this->params = params;
 }
@@ -1357,8 +1352,6 @@ void llm_graph_context::cb(ggml_tensor * cur, const char * name, int il) const {
     }
 }
 
-
-
 ggml_tensor * llm_graph_context::build_cvec(
          ggml_tensor * cur,
                  int   il) const {
@@ -2409,7 +2402,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(
 
         cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, hparams.f_max_alibi_bias,
                                   hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f);
-        res->add_fused_node({LLM_FUSED_OP_FLASH_ATTN, cur, il});
+        cb(cur, LLAMA_TENSOR_NAME_FATTN, il);
 
         ggml_flash_attn_ext_add_sinks(cur, sinks);
         ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32);

src/llama-graph.h

@@ -38,12 +38,6 @@ enum llm_graph_type {
     LLM_GRAPH_TYPE_DECODER_MTP,
 };
 
-enum llm_fused_op {
-    LLM_FUSED_OP_FLASH_ATTN,
-    LLM_FUSED_OP_GDN_AR,
-    LLM_FUSED_OP_GDN_CH,
-};
-
 enum llm_ffn_op_type : int {
     LLM_FFN_NONE = 0,           // sentinel: unset; archs must assign before use
     LLM_FFN_SILU,
@@ -781,12 +775,6 @@ struct llm_graph_params {
     }
 };
 
-struct llm_graph_fused_node {
-    llm_fused_op op;
-    ggml_tensor * tensor;
-    int il;
-};
-
 class llm_graph_result {
 public:
     llm_graph_result(int64_t max_nodes);
@@ -820,10 +808,6 @@ public:
 
     llm_graph_input_i * add_input(llm_graph_input_ptr input);
 
-    void add_fused_node(llm_graph_fused_node result);
-
-    const std::vector<llm_graph_fused_node> & get_fused_nodes() const { return fused_nodes; }
-
     void set_params(const llm_graph_params & params);
 
     // important graph nodes
@@ -842,7 +826,6 @@ public:
     std::map<llama_seq_id, ggml_tensor *> t_sampled_probs;
 
     std::vector<llm_graph_input_ptr> inputs;
-    std::vector<llm_graph_fused_node> fused_nodes;
 
     ggml_context_ptr ctx_compute;
 

src/llama-impl.h

@@ -103,3 +103,7 @@ std::string llama_format_tensor_shape(const std::vector<int64_t> & ne);
 std::string llama_format_tensor_shape(const struct ggml_tensor * t);
 
 std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i);
+
+#define LLAMA_TENSOR_NAME_FATTN   "__fattn__"
+#define LLAMA_TENSOR_NAME_FGDN_AR "__fgdn_ar__"
+#define LLAMA_TENSOR_NAME_FGDN_CH "__fgdn_ch__"

src/llama-kv-cache-dsa.cpp

@@ -113,7 +113,7 @@ llama_memory_context_ptr llama_kv_cache_dsa::init_batch(
 
         std::vector<llama_ubatch> ubatches;
         while (true) {
-            auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);
+            auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true);
 
             if (ubatch.n_tokens == 0) {
                 break;

src/llama-kv-cache-dsv4.cpp

@@ -1110,7 +1110,7 @@ llama_memory_context_ptr llama_kv_cache_dsv4::init_batch(
             if (has_coupled) {
                 ubatch = balloc.split_seq(n_ubatch);
             } else {
-                ubatch = balloc.split_equal(n_ubatch, raw_per_seq || comp_per_seq, 0);
+                ubatch = balloc.split_equal(n_ubatch, raw_per_seq || comp_per_seq);
             }
 
             if (ubatch.n_tokens == 0) {

src/llama-kv-cache-iswa.cpp

@@ -206,7 +206,7 @@ llama_memory_context_ptr llama_kv_cache_iswa::init_batch(llama_batch_allocr & ba
 
         std::vector<llama_ubatch> ubatches;
         while (true) {
-            auto ubatch = balloc.split_equal(n_ubatch, !unified, 0);
+            auto ubatch = balloc.split_equal(n_ubatch, !unified);
 
             if (ubatch.n_tokens == 0) {
                 break;

src/llama-kv-cache.cpp

@@ -706,7 +706,7 @@ llama_memory_context_ptr llama_kv_cache::init_batch(
 
         std::vector<llama_ubatch> ubatches;
         while (true) {
-            auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);
+            auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true);
 
             if (ubatch.n_tokens == 0) {
                 break;

src/llama-memory-hybrid-iswa.cpp

@@ -77,15 +77,15 @@ llama_memory_context_ptr llama_memory_hybrid_iswa::init_batch(llama_batch_allocr
                 // if all tokens are output, split by sequence
                 ubatch = balloc.split_seq(n_ubatch);
             } else {
-                // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
-                const bool unified = (mem_attn->get_base()->get_n_stream() == 1);
-
-                // [TAG_RECURRENT_ROLLBACK_SPLITS]
-                // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch
-                //   so that the rollback snapshots remain valid
-                const uint32_t n_rs_seq = mem_recr->n_rs_seq;
-
-                ubatch = balloc.split_equal(n_ubatch, !unified, n_rs_seq > 0 ? n_rs_seq + 1 : 0);
+                if (mem_recr->n_rs_seq > 0) {
+                    // [TAG_RECURRENT_ROLLBACK_SPLITS]
+                    // TODO: recurrent state rollback does not support equal splits
+                    ubatch = balloc.split_seq(n_ubatch);
+                } else {
+                    // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
+                    const bool unified = (mem_attn->get_base()->get_n_stream() == 1);
+                    ubatch = balloc.split_equal(n_ubatch, !unified);
+                }
             }
 
             if (ubatch.n_tokens == 0) {

src/llama-memory-hybrid.cpp

@@ -78,15 +78,15 @@ llama_memory_context_ptr llama_memory_hybrid::init_batch(llama_batch_allocr & ba
                 // if all tokens are output, split by sequence
                 ubatch = balloc.split_seq(n_ubatch);
             } else {
-                // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
-                const bool unified = (mem_attn->get_n_stream() == 1);
-
-                // [TAG_RECURRENT_ROLLBACK_SPLITS]
-                // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch
-                //   so that the rollback snapshots remain valid
-                const uint32_t n_rs_seq = mem_recr->n_rs_seq;
-
-                ubatch = balloc.split_equal(n_ubatch, !unified, n_rs_seq > 0 ? n_rs_seq + 1 : 0);
+                if (mem_recr->n_rs_seq > 0) {
+                    // [TAG_RECURRENT_ROLLBACK_SPLITS]
+                    // TODO: recurrent state rollback does not support equal splits
+                    ubatch = balloc.split_seq(n_ubatch);
+                } else {
+                    // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
+                    const bool unified = (mem_attn->get_n_stream() == 1);
+                    ubatch = balloc.split_equal(n_ubatch, !unified);
+                }
             }
 
             if (ubatch.n_tokens == 0) {

src/llama-memory-recurrent.cpp

@@ -416,12 +416,15 @@ llama_memory_context_ptr llama_memory_recurrent::init_batch(llama_batch_allocr &
                 // if all tokens are output, split by sequence
                 ubatch = balloc.split_seq(n_ubatch);
             } else {
-                // TODO: non-sequential equal split can be done if using unified KV cache
-                //       for simplicity, we always use sequential equal split for now
-                // [TAG_RECURRENT_ROLLBACK_SPLITS]
-                // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch
-                //   so that the rollback snapshots remain valid
-                ubatch = balloc.split_equal(n_ubatch, true, n_rs_seq > 0 ? n_rs_seq + 1 : 0);
+                if (n_rs_seq > 0) {
+                    // [TAG_RECURRENT_ROLLBACK_SPLITS]
+                    // TODO: recurrent state rollback does not support equal splits
+                    ubatch = balloc.split_seq(n_ubatch);
+                } else {
+                    // TODO: non-sequential equal split can be done if using unified KV cache
+                    //       for simplicity, we always use sequential equal split for now
+                    ubatch = balloc.split_equal(n_ubatch, true);
+                }
             }
 
             if (ubatch.n_tokens == 0) {

src/llama-vocab.cpp

@@ -887,6 +887,9 @@ struct llm_tokenizer_ugm : llm_tokenizer {
             // blob containing XOR-compressed compact double array (XCDA) entries
             uint32_t xcda_blob_size = *(const uint32_t *) &precompiled_charsmap[0];
             charsmap_offset += sizeof(xcda_blob_size);
+            if (xcda_blob_size + charsmap_offset >= precompiled_charsmap.size()) {
+                throw std::runtime_error("Index out of array bounds in precompiled charsmap!");
+            }
 
             // Next xcda_blob_size bytes contain entries of XOR-compressed compact
             // double array (XCDA). Each entry is bit-packed into a 32-bit integer.
@@ -1202,15 +1205,7 @@ private:
                 throw std::runtime_error("Index out of array bounds in precompiled charsmap!");
             }
             const char * prefix_replacement = &(tokenizer.prefix_replacements)[longest_prefix_offset];
-            size_t max_len = tokenizer.prefix_replacements_size - longest_prefix_offset;
-            size_t repl_len = 0;
-            while (repl_len < max_len && prefix_replacement[repl_len] != '\0') {
-                repl_len++;
-            }
-            if (repl_len == max_len) {
-                throw std::runtime_error("Unterminated string in precompiled charsmap!");
-            }
-            return { prefix_replacement, repl_len, longest_prefix_length };
+            return { prefix_replacement, strlen(prefix_replacement), longest_prefix_length };
         }
 
         // check if the input prefix contains a valid sequence of UTF-8 code units
@@ -2023,18 +2018,11 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                 const size_t n_precompiled_charsmap = gguf_get_arr_n(ctx, precompiled_charsmap_keyidx);
                 const char * pc = (const char *) gguf_get_arr_data(ctx, precompiled_charsmap_keyidx);
                 precompiled_charsmap.assign(pc, pc + n_precompiled_charsmap);
-                if (precompiled_charsmap.size() < sizeof(uint32_t)) {
-                    throw std::runtime_error("precompiled_charsmap too small for xcda_blob_size header!");
-                }
-                uint32_t * xcda_blob_size = (uint32_t *) &precompiled_charsmap[0];
-#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-                *xcda_blob_size = __builtin_bswap32(*xcda_blob_size);
-#endif
-                if (*xcda_blob_size + sizeof(uint32_t) >= precompiled_charsmap.size()) {
-                    throw std::runtime_error("Index out of array bounds in precompiled charsmap!");
-                }
 #if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
                 // correct endianness of data in precompiled_charsmap binary blob
+                uint32_t * xcda_blob_size = (uint32_t *) &precompiled_charsmap[0];
+                *xcda_blob_size = __builtin_bswap32(*xcda_blob_size);
+                assert(*xcda_blob_size + sizeof(uint32_t) < n_precompiled_charsmap);
                 size_t xcda_array_size = *xcda_blob_size / sizeof(uint32_t);
                 uint32_t * xcda_array = (uint32_t *) &precompiled_charsmap[sizeof(uint32_t)];
                 for (size_t i = 0; i < xcda_array_size; ++i) {

src/models/delta-net-base.cpp

@@ -401,9 +401,9 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne
     // K=1: output carries the final state only. state s is 4D [S_v, S_v, H_v, n_seqs].
     ggml_tensor * result = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, /*K=*/1);
     if (n_tokens == 1) {
-        res->add_fused_node({LLM_FUSED_OP_GDN_AR, result, il});
+        cb(result, LLAMA_TENSOR_NAME_FGDN_AR, il);
     } else {
-        res->add_fused_node({LLM_FUSED_OP_GDN_CH, result, il});
+        cb(result, LLAMA_TENSOR_NAME_FGDN_CH, il);
     }
 
     ggml_tensor * output = ggml_view_4d(ctx0, result,
@@ -496,8 +496,8 @@ ggml_tensor * llm_build_delta_net_base::build_conv_state(
         ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_state_last, conv_state_update));
     } else {
         // [TAG_RECURRENT_ROLLBACK_SPLITS]
-        // this logic assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are inside
-        //   the same ubatch, which `split_equal()` guarantees via its n_keep_tail argument
+        // TODO: this logic incorrectly assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are
+        //       inside the same ubatch. currently with `split_equal()` this is not correct
 
         const int64_t K = (int64_t) cparams.n_rs_seq + 1;
 
@@ -566,9 +566,9 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn(
     // state s is 4D [S_v, S_v, H_v, n_seqs]; K snapshot slots are written into the output.
     ggml_tensor * gdn_out = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, K);
     if (n_seq_tokens > 1) {
-        res->add_fused_node({LLM_FUSED_OP_GDN_CH, gdn_out, il});
+        cb(gdn_out, LLAMA_TENSOR_NAME_FGDN_CH, il);
     } else {
-        res->add_fused_node({LLM_FUSED_OP_GDN_AR, gdn_out, il});
+        cb(gdn_out, LLAMA_TENSOR_NAME_FGDN_AR, il);
     }
 
     const int64_t attn_score_elems    = S_v * H_v * n_seq_tokens * n_seqs;

tests/test-backend-ops.cpp

@@ -2393,8 +2393,7 @@ static void init_set_rows_row_ids(ggml_tensor * t, int num_rows) {
 
 // GGML_OP_SET_ROWS
 struct test_set_rows : public test_case {
-    const ggml_type type_src;
-    const ggml_type type_dst;
+    const ggml_type type;
     const ggml_type type_idx;
     const std::array<int64_t, 4> ne;
     const std::array<int, 2> nr23; // broadcast only dims 2 and 3
@@ -2402,22 +2401,21 @@ struct test_set_rows : public test_case {
     const bool v; // view (non-contiguous src1)
 
     std::string vars() override {
-        return VARS_TO_STR7(type_src, type_dst, type_idx, ne, nr23, r, v);
+        return VARS_TO_STR6(type, type_idx, ne, nr23, r, v);
     }
 
-    test_set_rows(ggml_type type_src,
-            ggml_type type_dst,
+    test_set_rows(ggml_type type,
             ggml_type type_idx,
             std::array<int64_t, 4> ne,
             std::array<int, 2> nr23,
             int r, bool v = false)
-        : type_src(type_src), type_dst(type_dst), type_idx(type_idx), ne(ne), nr23(nr23), r(r), v(v) {}
+        : type(type), type_idx(type_idx), ne(ne), nr23(nr23), r(r), v(v) {}
 
     ggml_tensor * build_graph(ggml_context * ctx) override {
-        ggml_tensor * dst = ggml_new_tensor_4d(ctx, type_dst, ne[0], ne[1], ne[2]*nr23[0], ne[3]*nr23[1]);
+        ggml_tensor * dst = ggml_new_tensor_4d(ctx, type,          ne[0], ne[1], ne[2]*nr23[0], ne[3]*nr23[1]);
         ggml_set_name(dst, "dst");
 
-        ggml_tensor * src = ggml_new_tensor_4d(ctx, type_src, ne[0], r,     ne[2]*nr23[0], ne[3]*nr23[1]);
+        ggml_tensor * src = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], r,     ne[2]*nr23[0], ne[3]*nr23[1]);
         ggml_set_name(src, "src");
 
         ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, type_idx, r, ne[2], ne[3]);
@@ -2450,17 +2448,17 @@ struct test_set_rows : public test_case {
     }
 
     double max_nmse_err() override {
-        if (type_dst == GGML_TYPE_Q4_0 || type_dst == GGML_TYPE_Q4_1 || type_dst == GGML_TYPE_IQ4_NL ||
-            type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1 || type_dst == GGML_TYPE_Q8_0) {
+        if (type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || type == GGML_TYPE_IQ4_NL ||
+            type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1 || type == GGML_TYPE_Q8_0) {
             // estimate what the max nmse error would be if one quantized value is
             // off by one. The test values are distributed in [-1,1], so it'll be
             // roughly (2.0 / 2^bits)^2, divided by the mean square value of the reference,
             // which is roughly 0.25 times the number of elements.
             double err_estimate = 1.0f/8.0f;
-            if (type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1) {
+            if (type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1) {
                 err_estimate /= 2.0f;
             }
-            if (type_dst == GGML_TYPE_Q8_0) {
+            if (type == GGML_TYPE_Q8_0) {
                 err_estimate /= 8.0f;
             }
             err_estimate *= err_estimate;
@@ -2473,7 +2471,7 @@ struct test_set_rows : public test_case {
     // See dicussion here: https://github.com/ggml-org/llama.cpp/pull/23760#issuecomment-4566312209
     double max_nmse_err(ggml_backend_t backend) override {
         ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));
-        if (type_dst == GGML_TYPE_Q8_0 && strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) {
+        if (type == GGML_TYPE_Q8_0 && strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) {
             return std::max(test_case::max_nmse_err(backend), 2e-7);
         }
         return test_case::max_nmse_err(backend);
@@ -7875,28 +7873,24 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_I32, 256, 5, 4, 1, v));
     }
 
-    test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
-    test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
-    test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_Q8_0, GGML_TYPE_I32, { 256, 5, 1, 3 }, { 1, 1, }, 1, false));
+    test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
+    test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
+    test_cases.emplace_back(new test_set_rows(GGML_TYPE_Q8_0, GGML_TYPE_I32, { 256, 5, 1, 3 }, { 1, 1, }, 1, false));
     for (ggml_type type : all_types) {
         for (int b : {1, 7}) {
             for (bool v : {false, true}) {
-                test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 256, 5,  b, 3 }, { 1, 1, }, 1, v));
-                test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 256, 11, 1, b }, { 2, 3, }, 7, v));
+                test_cases.emplace_back(new test_set_rows(type, GGML_TYPE_I64, { 256, 5,  b, 3 }, { 1, 1, }, 1, v));
+                test_cases.emplace_back(new test_set_rows(type, GGML_TYPE_I64, { 256, 11, 1, b }, { 2, 3, }, 7, v));
 
-                test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 3*ggml_blck_size(type), 3, b, 1 }, { 2, 3, }, 2, v));
+                test_cases.emplace_back(new test_set_rows(type, GGML_TYPE_I64, { 3*ggml_blck_size(type), 3, b, 1 }, { 2, 3, }, 2, v));
 
                 if (ggml_blck_size(type) == 1) {
-                    test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 31, 3, b, 1 }, { 2, 3, }, 2, v));
-                    test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 33, 5, 1, b }, { 2, 3, }, 1, v));
+                    test_cases.emplace_back(new test_set_rows(type, GGML_TYPE_I64, { 31, 3, b, 1 }, { 2, 3, }, 2, v));
+                    test_cases.emplace_back(new test_set_rows(type, GGML_TYPE_I64, { 33, 5, 1, b }, { 2, 3, }, 1, v));
                 }
             }
         }
     }
-    test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
-    test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
-    test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, true));
-    test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, true));
 
     for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION }) {
         for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {

tools/cli/CMakeLists.txt

@@ -2,11 +2,13 @@
 
 set(TARGET llama-cli-impl)
 
-add_library(${TARGET} cli.cpp)
+add_library(${TARGET} cli.cpp
+                      cli-client.cpp
+                      cli-context.cpp)
 set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
 
 target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ../server)
-target_link_libraries(${TARGET} PUBLIC server-context llama-common ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PUBLIC llama-server-impl llama-common ${CMAKE_THREAD_LIBS_INIT})
 
 if(LLAMA_TOOLS_INSTALL)
     install(TARGETS ${TARGET} LIBRARY)

tools/cli/cli-client.cpp

@@ -0,0 +1,130 @@
+#include "cli-client.h"
+
+#include "http.h"
+
+#include <algorithm>
+#include <chrono>
+#include <thread>
+
+// generation can stall for a long time during prompt processing, so the
+// read timeout must be generous
+static constexpr time_t CLI_HTTP_READ_TIMEOUT_SEC = 3600;
+
+// upper bound for the accumulated response body kept for error reporting
+static constexpr size_t CLI_HTTP_MAX_ERROR_BODY = 1024 * 1024;
+
+// returns the path with the base url's path prefix prepended (if any)
+static std::string join_path(const common_http_url & parts, const std::string & path) {
+    if (parts.path.empty() || parts.path == "/") {
+        return path;
+    }
+    std::string prefix = parts.path;
+    if (prefix.back() == '/') {
+        prefix.pop_back();
+    }
+    return prefix + path;
+}
+
+std::string cli_client::get(const std::string & path) {
+    auto [cli, parts] = common_http_client(server_base);
+    cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);
+    auto path_with_model = path + (model.empty() ? "" : ("?model=" + model));
+    auto res = cli.Get(join_path(parts, path_with_model));
+    if (!res) {
+        throw std::runtime_error("failed to connect to " + server_base + ": " + httplib::to_string(res.error()));
+    }
+    if (res->status < 200 || res->status >= 300) {
+        throw std::runtime_error("GET " + path + " failed with status " + std::to_string(res->status) + ": " + res->body);
+    }
+    return res->body;
+}
+
+std::string cli_client::post(const std::string & path, const std::string & body) {
+    auto [cli, parts] = common_http_client(server_base);
+    cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);
+    auto res = cli.Post(join_path(parts, path), body, "application/json");
+    if (!res) {
+        throw std::runtime_error("failed to connect to " + server_base + ": " + httplib::to_string(res.error()));
+    }
+    if (res->status < 200 || res->status >= 300) {
+        throw std::runtime_error("POST " + path + " failed with status " + std::to_string(res->status) + ": " + res->body);
+    }
+    return res->body;
+}
+
+std::string cli_client::post_sse(const std::string & path,
+                                  const std::string & body,
+                                  const std::function<bool()> & should_stop,
+                                  const std::function<void(const std::string &)> & on_data) {
+    auto [cli, parts] = common_http_client(server_base);
+    cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);
+
+    std::string pending;  // buffer for incomplete SSE lines
+    std::string raw_body; // accumulated body, used only for error reporting
+
+    auto receiver = [&](const char * data, size_t len) -> bool {
+        if (should_stop()) {
+            return false; // aborts the request
+        }
+        if (raw_body.size() < CLI_HTTP_MAX_ERROR_BODY) {
+            raw_body.append(data, std::min(len, CLI_HTTP_MAX_ERROR_BODY - raw_body.size()));
+        }
+        pending.append(data, len);
+        size_t pos;
+        while ((pos = pending.find('\n')) != std::string::npos) {
+            std::string line = pending.substr(0, pos);
+            pending.erase(0, pos + 1);
+            if (!line.empty() && line.back() == '\r') {
+                line.pop_back();
+            }
+            if (line.rfind("data: ", 0) != 0) {
+                continue;
+            }
+            std::string payload = line.substr(6);
+            if (payload == "[DONE]") {
+                continue;
+            }
+            on_data(payload);
+        }
+        return true;
+    };
+
+    httplib::Headers headers = {{"Accept", "text/event-stream"}};
+    auto res = cli.Post(join_path(parts, path), headers, body, "application/json", receiver);
+
+    if (!res) {
+        if (res.error() == httplib::Error::Canceled && should_stop()) {
+            return ""; // cancelled by the user
+        }
+        return "failed to connect to " + server_base + ": " + httplib::to_string(res.error());
+    }
+    if (res->status < 200 || res->status >= 300) {
+        if (!raw_body.empty()) {
+            return raw_body;
+        }
+        return "request failed with status " + std::to_string(res->status);
+    }
+    return "";
+}
+
+bool cli_client::wait_health(const std::function<bool()> & is_aborted) {
+    int connect_attempts = 0;
+    while (!is_aborted()) {
+        auto [cli, parts] = common_http_client(server_base);
+        cli.set_connection_timeout(1, 0);
+        auto res = cli.Get(join_path(parts, "/health"));
+        if (res) {
+            if (res->status == 200) {
+                return true;
+            }
+            // any other status means the server is up but not ready yet
+            // (e.g. 503 while the model is still loading)
+        } else if (++connect_attempts >= 10) {
+            last_error = "failed to connect to " + server_base + ": " + httplib::to_string(res.error());
+            return false;
+        }
+        std::this_thread::sleep_for(std::chrono::milliseconds(300));
+    }
+    last_error = "aborted while waiting for the server to become ready";
+    return false;
+}

tools/cli/cli-client.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <functional>
+#include <string>
+
+// openai-like client for CLI
+struct cli_client {
+    std::string server_base; // base url, for example "http://127.0.0.1:8080"
+    std::string last_error;  // set when wait_health() fails
+
+    std::string model; // optional, set when the server has multiple models (router mode)
+
+    // simple GET request, returns the raw response body
+    // throws std::runtime_error on transport error or non-2xx status
+    std::string get(const std::string & path);
+
+    // simple POST request, returns the raw response body
+    // throws std::runtime_error on transport error or non-2xx status
+    std::string post(const std::string & path, const std::string & body);
+
+    // POST request with an SSE streaming response
+    // on_data is invoked per "data:" event with the raw event payload
+    // returns after the stream is finished (empty string on graceful exit)
+    // otherwise, the raw error response body
+    std::string post_sse(const std::string & path,
+                          const std::string & body,
+                          const std::function<bool()> & should_stop,
+                          const std::function<void(const std::string &)> & on_data);
+
+    // poll /health until the server is ready to accept requests
+    // returns false if is_aborted returned true or the server is unreachable
+    bool wait_health(const std::function<bool()> & is_aborted);
+};

tools/cli/cli-context.cpp

@@ -0,0 +1,622 @@
+#include "cli-context.h"
+#include "cli-ui.h"
+
+#include "arg.h"
+#include "base64.hpp"
+#include "log.h"
+#include "console.h"
+
+#define JSON_ASSERT GGML_ASSERT
+#include <nlohmann/json.hpp>
+
+#include <algorithm>
+#include <cctype>
+#include <filesystem>
+#include <fstream>
+#include <map>
+#include <set>
+
+using json = nlohmann::ordered_json;
+
+struct cli_context_impl {
+    json messages      = json::array();
+    json pending_media = json::array(); // staged multimodal content parts
+};
+
+cli_context::cli_context(const common_params & params) : params(params), impl(new cli_context_impl()) {}
+
+cli_context::~cli_context() {
+    shutdown();
+}
+
+std::atomic<bool> & cli_context::interrupted() {
+    static std::atomic<bool> flag = false;
+    return flag;
+}
+
+static bool should_stop() {
+    return cli_context::interrupted().load();
+}
+
+static constexpr size_t FILE_GLOB_MAX_RESULTS = 100;
+
+const char * LLAMA_ASCII_LOGO = R"(
+▄▄ ▄▄
+██ ██
+██ ██  ▀▀█▄ ███▄███▄  ▀▀█▄    ▄████ ████▄ ████▄
+██ ██ ▄█▀██ ██ ██ ██ ▄█▀██    ██    ██ ██ ██ ██
+██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
+                                    ██    ██
+                                    ▀▀    ▀▀
+)";
+
+// number of values an arg consumes on the command line
+static int arg_num_values(const common_arg & opt) {
+    if (opt.value_hint_2 != nullptr) {
+        return 2;
+    }
+    if (opt.value_hint != nullptr) {
+        return 1;
+    }
+    return 0;
+}
+
+static std::string format_error_message(const json & err) {
+    if (err.contains("error") && err.at("error").is_object()) {
+        const auto & e = err.at("error");
+        if (e.contains("message") && e.at("message").is_string()) {
+            return e.at("message").get<std::string>();
+        }
+    }
+    return err.dump();
+}
+
+// err is the raw response body of a failed request; it may or may not be JSON
+static std::string format_error_message(const std::string & err) {
+    json parsed = json::parse(err, nullptr, false);
+    if (!parsed.is_discarded()) {
+        return format_error_message(parsed);
+    }
+    return err;
+}
+
+static std::string media_type_from_ext(const std::string & fname) {
+    std::string ext = std::filesystem::path(fname).extension().string();
+    std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); });
+    if (ext == ".wav" || ext == ".mp3") {
+        return "audio";
+    }
+    if (ext == ".mp4" || ext == ".avi" || ext == ".mkv" || ext == ".mov" || ext == ".webm") {
+        return "video";
+    }
+    return "image";
+}
+
+bool cli_context::init() {
+    ui::init(params);
+
+    std::optional<ui::spinner> spinner;
+
+    bool use_external_server = !params.server_base.empty();
+    if (use_external_server) {
+        std::string base = params.server_base;
+        while (!base.empty() && base.back() == '/') {
+            base.pop_back();
+        }
+        client.server_base = base;
+
+        spinner.emplace("Connecting to server at " + base);
+    } else {
+        if (params.model.path.empty() && params.model.url.empty() &&
+                params.model.hf_repo.empty() && params.model.docker_repo.empty()) {
+            ui::show_error(
+                "no model specified",
+                "use -m <file.gguf> or -hf <user/repo> to run a local model,\n"
+                "or --server-base <url> to connect to a running llama-server"
+            );
+            return false;
+        }
+
+        spinner.emplace("\n\nLoading model...");
+
+        server.emplace();
+        if (!server->start(params)) {
+            ui::show_error("server start failed");
+            return false;
+        }
+        if (!server->wait_ready(should_stop)) {
+            if (!should_stop()) {
+                ui::show_error("the server exited before becoming ready");
+            }
+            return false;
+        }
+        client.server_base = server->address();
+    }
+
+    // for --server-base this is the main availability check; for a spawned
+    // server it is a cheap sanity check on top of the ready signal
+    auto is_aborted = [this]() {
+        return should_stop() || (server && !server->alive());
+    };
+    bool healthy = false;
+    try {
+        healthy = client.wait_health(is_aborted);
+    } catch (const std::exception & e) {
+        client.last_error = e.what();
+    }
+    if (!healthy) {
+        if (!should_stop()) {
+            ui::show_error(client.last_error);
+        }
+        return false;
+    }
+
+    if (use_external_server) {
+        spinner.reset();
+        if (!list_and_ask_models()) {
+            return false;
+        }
+        // restore the spinner for the next step
+        spinner.emplace("Waiting for server...");
+    }
+
+    fetch_server_props();
+
+    return true;
+}
+
+void cli_context::fetch_server_props() {
+    try {
+        json props = json::parse(client.get("/props"));
+        model_name = props.value("model_alias", "");
+        if (model_name.empty()) {
+            const std::string path = props.value("model_path", "");
+            if (!path.empty()) {
+                model_name = std::filesystem::path(path).filename().string();
+            }
+        }
+        model_ftype = props.value("model_ftype", "");
+        build_info = props.value("build_info", "");
+        if (props.contains("modalities") && props.at("modalities").is_object()) {
+            const auto & modalities = props.at("modalities");
+            has_vision = modalities.value("vision", false);
+            has_audio  = modalities.value("audio", false);
+            has_video  = modalities.value("video", false);
+        }
+    } catch (const std::exception & e) {
+        // /props can be disabled on remote servers; not fatal
+        LOG_DBG("failed to fetch /props: %s\n", e.what());
+    }
+}
+
+bool cli_context::list_and_ask_models() {
+    json resp = json::parse(client.get("/v1/models"));
+    if (!resp.contains("data") || !resp.at("data").is_array()) {
+        throw std::runtime_error("invalid response from /v1/models");
+    }
+    std::vector<std::string> models;
+    std::vector<std::string> models_display;
+    for (const auto & m : resp.at("data")) {
+        if (!m.contains("id") || !m.at("id").is_string()) {
+            continue;
+        }
+        std::string name = m.at("id").get<std::string>();
+        std::string display = name;
+        if (m.contains("aliases") && m.at("aliases").is_array()) {
+            std::vector<std::string> aliases;
+            for (const auto & a : m.at("aliases")) {
+                if (a.is_string()) {
+                    aliases.push_back(a.get<std::string>());
+                }
+            }
+            if (!aliases.empty()) {
+                display += " (" + string_join(aliases, ", ") + ")";
+            }
+        }
+        models.push_back(name);
+        models_display.push_back(display);
+    }
+
+    // only one model: use it without asking
+    if (models.size() == 1) {
+        model_name = models[0];
+        client.model = model_name;
+        return true;
+    }
+
+    std::string message = "\nAvailable models:";
+    for (size_t i = 0; i < models_display.size(); ++i) {
+        message += "\n  " + std::to_string(i + 1) + ". " + models_display[i];
+    }
+    message += "\n";
+    ui::show_message(message);
+    std::string selection;
+    while (selection.empty()) {
+        if (should_stop()) {
+            return false;
+        }
+        ui::user_turn user_turn;
+        selection = user_turn.read_input(false, "Select model by number: ");
+        if (selection.empty()) {
+            continue;
+        }
+        try {
+            size_t idx = std::stoul(selection);
+            if (idx > 0 && idx <= models.size()) {
+                model_name = models[idx - 1];
+                client.model = model_name;
+                ui::show_message("Selected model: " + model_name);
+                break;
+            }
+        } catch (...) {
+            // ignore
+        }
+        ui::show_error("Invalid selection. Please enter a valid number.");
+        selection.clear();
+        continue;
+    }
+    return true;
+}
+
+void cli_context::add_system_prompt() {
+    if (!params.system_prompt.empty()) {
+        impl->messages.push_back({
+            {"role",    "system"},
+            {"content", params.system_prompt}
+        });
+    }
+}
+
+void cli_context::push_user_message(const std::string & text) {
+    json content;
+    if (impl->pending_media.empty()) {
+        content = text;
+    } else {
+        // multimodal message: media parts first, then the text
+        content = impl->pending_media;
+        content.push_back({
+            {"type", "text"},
+            {"text", text}
+        });
+        impl->pending_media = json::array();
+    }
+    impl->messages.push_back({
+        {"role",    "user"},
+        {"content", content}
+    });
+}
+
+bool cli_context::stage_media_file(const std::string & fname, const std::string & type) {
+    std::ifstream file(fname, std::ios::binary);
+    if (!file) {
+        return false;
+    }
+    std::string data((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
+    std::string encoded = base64::encode(data);
+
+    if (type == "audio") {
+        std::string ext = std::filesystem::path(fname).extension().string();
+        std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); });
+        impl->pending_media.push_back({
+            {"type", "input_audio"},
+            {"input_audio", {
+                {"data",   encoded},
+                {"format", ext == ".mp3" ? "mp3" : "wav"}
+            }}
+        });
+    } else if (type == "video") {
+        impl->pending_media.push_back({
+            {"type", "input_video"},
+            {"input_video", {
+                {"data", encoded}
+            }}
+        });
+    } else {
+        // the server detects the actual image type from the data
+        impl->pending_media.push_back({
+            {"type", "image_url"},
+            {"image_url", {
+                {"url", "data:image/unknown;base64," + encoded}
+            }}
+        });
+    }
+    return true;
+}
+
+bool cli_context::generate_completion(std::string & assistant_content, cli_timings & timings) {
+    json body = {
+        {"messages",          impl->messages},
+        {"stream",            true},
+        // in order to get timings even when we cancel mid-way
+        {"timings_per_token", true},
+    };
+    if (!client.model.empty()) {
+        body["model"] = client.model;
+    }
+
+    bool stream_error = false;
+
+    ui::assistant_turn a;
+
+    std::string err = client.post_sse("/v1/chat/completions", body.dump(), should_stop, [&](const std::string & payload) {
+        json chunk = json::parse(payload, nullptr, false);
+        if (chunk.is_discarded()) {
+            return;
+        }
+        if (chunk.contains("error")) {
+            stream_error = true;
+            ui::show_error(format_error_message(chunk));
+            return;
+        }
+        if (chunk.contains("timings")) {
+            const auto & t = chunk.at("timings");
+            timings.prompt_per_second    = t.value("prompt_per_second",    0.0);
+            timings.predicted_per_second = t.value("predicted_per_second", 0.0);
+        }
+        if (!chunk.contains("choices") || !chunk.at("choices").is_array() || chunk.at("choices").empty()) {
+            return;
+        }
+        const auto & choice = chunk.at("choices").at(0);
+        if (!choice.contains("delta")) {
+            return;
+        }
+        const auto & delta = choice.at("delta");
+        if (delta.contains("reasoning_content") && delta.at("reasoning_content").is_string()) {
+            const std::string text = delta.at("reasoning_content").get<std::string>();
+            if (!text.empty()) {
+                a.push(ui::ASSISTANT_DISPLAY_MODE_REASONING, text);
+            }
+        }
+        if (delta.contains("content") && delta.at("content").is_string()) {
+            const std::string text = delta.at("content").get<std::string>();
+            if (!text.empty()) {
+                assistant_content += text;
+                a.push(ui::ASSISTANT_DISPLAY_MODE_CONTENT, text);
+            }
+        }
+    });
+
+    cli_context::interrupted().store(false);
+
+    if (!err.empty()) {
+        ui::show_error(format_error_message(err));
+        return false;
+    }
+    return !stream_error;
+}
+
+int cli_context::run() {
+    add_system_prompt();
+
+    std::string modalities = "text";
+    if (has_vision) {
+        modalities += ", vision";
+    }
+    if (has_audio) {
+        modalities += ", audio";
+    }
+    if (has_video) {
+        modalities += ", video";
+    }
+
+    std::string banner;
+    banner += "\n";
+    banner += LLAMA_ASCII_LOGO;
+    banner += "\n";
+    banner += "build      : " + build_info + "\n";
+    banner += "model      : " + model_name + "\n";
+    if (!model_ftype.empty()) {
+        banner += "ftype      : " + model_ftype + "\n";
+    }
+    banner += "modalities : " + modalities + "\n";
+    if (!params.system_prompt.empty()) {
+        banner += "using custom system prompt\n";
+    }
+    banner += "\n";
+    banner += "available commands:\n";
+    banner += "  /exit or Ctrl+C     stop or exit\n";
+    banner += "  /regen              regenerate the last response\n";
+    banner += "  /clear              clear the chat history\n";
+    banner += "  /read <file>        add a text file\n";
+    banner += "  /glob <pattern>     add text files using globbing pattern\n";
+    if (has_vision) {
+        banner += "  /image <file>       add an image file\n";
+    }
+    if (has_audio) {
+        banner += "  /audio <file>       add an audio file\n";
+    }
+    if (has_video) {
+        banner += "  /video <file>       add a video file\n";
+    }
+    banner += "\n";
+
+    ui::show_message(banner);
+
+    // interactive loop
+    std::string cur_msg;
+
+    auto add_text_file = [&](const std::string & fname) -> bool {
+        std::ifstream file(fname, std::ios::binary);
+        if (!file) {
+            ui::show_error(string_format("file does not exist or cannot be opened: '%s'", fname.c_str()));
+            return false;
+        }
+        std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
+        cur_msg += "--- File: ";
+        cur_msg += fname;
+        cur_msg += " ---\n";
+        cur_msg += content;
+        ui::show_message(string_format("Loaded text from '%s'", fname.c_str()));
+        return true;
+    };
+
+    while (true) {
+        std::string buffer;
+        {
+            ui::user_turn user_turn;
+
+            if (params.prompt.empty()) {
+                buffer = user_turn.read_input(params.multiline_input);
+            } else {
+                // process input prompt from args
+                for (auto & fname : params.image) {
+                    if (!stage_media_file(fname, media_type_from_ext(fname))) {
+                        ui::show_error(string_format("file does not exist or cannot be opened: '%s'", fname.c_str()));
+                        break;
+                    }
+                    ui::show_message(string_format("Loaded media from '%s'", fname.c_str()));
+                }
+                buffer = params.prompt;
+                user_turn.echo(buffer);
+                params.prompt.clear(); // only use it once
+            }
+        }
+
+        if (should_stop()) {
+            cli_context::interrupted().store(false);
+            break;
+        }
+
+        // remove trailing newline
+        if (!buffer.empty() && buffer.back() == '\n') {
+            buffer.pop_back();
+        }
+
+        // skip empty messages
+        if (buffer.empty()) {
+            continue;
+        }
+
+        bool add_user_msg = true;
+
+        // process commands
+        if (string_starts_with(buffer, "/exit")) {
+            break;
+        } else if (string_starts_with(buffer, "/regen")) {
+            if (impl->messages.size() >= 2) {
+                size_t last_idx = impl->messages.size() - 1;
+                impl->messages.erase(last_idx);
+                add_user_msg = false;
+            } else {
+                ui::show_error("No message to regenerate.");
+                continue;
+            }
+        } else if (string_starts_with(buffer, "/clear")) {
+            impl->messages.clear();
+            add_system_prompt();
+
+            impl->pending_media = json::array();
+            ui::show_message("Chat history cleared.");
+            continue;
+        } else if (
+                (string_starts_with(buffer, "/image ") && has_vision) ||
+                (string_starts_with(buffer, "/audio ") && has_audio) ||
+                (string_starts_with(buffer, "/video ") && has_video)) {
+            std::string type = buffer.substr(1, 5);
+            // just in case (bad copy-paste for example), we strip all trailing/leading spaces
+            std::string fname = string_strip(buffer.substr(7));
+            if (!stage_media_file(fname, type)) {
+                ui::show_error(string_format("file does not exist or cannot be opened: '%s'", fname.c_str()));
+                continue;
+            }
+            ui::show_message(string_format("Loaded media from '%s'", fname.c_str()));
+            continue;
+        } else if (string_starts_with(buffer, "/read ")) {
+            std::string fname = string_strip(buffer.substr(6));
+            add_text_file(fname);
+            continue;
+        } else if (string_starts_with(buffer, "/glob ")) {
+            std::error_code ec;
+            size_t count = 0;
+            auto curdir = std::filesystem::current_path();
+            std::string pattern = string_strip(buffer.substr(6));
+            std::filesystem::path rel_path;
+
+            auto startglob = pattern.find_first_of("![*?");
+            if (startglob != std::string::npos && startglob != 0) {
+                auto endpath = pattern.substr(0, startglob).find_last_of('/');
+                if (endpath != std::string::npos) {
+                    std::string rel_pattern = pattern.substr(0, endpath);
+#if !defined(_WIN32)
+                    if (string_starts_with(rel_pattern, '~')) {
+                        const char * home = std::getenv("HOME");
+                        if (home && home[0]) {
+                            rel_pattern = home + rel_pattern.substr(1);
+                        }
+                    }
+#endif
+                    rel_path = rel_pattern;
+                    pattern.erase(0, endpath + 1);
+                    curdir /= rel_path;
+                }
+            }
+
+            for (const auto & entry : std::filesystem::recursive_directory_iterator(curdir,
+                    std::filesystem::directory_options::skip_permission_denied, ec)) {
+                if (!entry.is_regular_file()) {
+                    continue;
+                }
+
+                std::string rel = std::filesystem::relative(entry.path(), curdir, ec).string();
+                if (ec) {
+                    ec.clear();
+                    continue;
+                }
+                std::replace(rel.begin(), rel.end(), '\\', '/');
+
+                if (!glob_match(pattern, rel)) {
+                    continue;
+                }
+
+                if (!add_text_file((rel_path / rel).string())) {
+                    continue;
+                }
+
+                if (++count >= FILE_GLOB_MAX_RESULTS) {
+                    ui::show_error(string_format("Maximum number of globbed files allowed (%zu) reached.", FILE_GLOB_MAX_RESULTS));
+                    break;
+                }
+            }
+            continue;
+        } else {
+            // not a command
+            cur_msg += buffer;
+        }
+
+        // generate response
+        if (add_user_msg) {
+            push_user_message(cur_msg);
+            cur_msg.clear();
+        }
+        cli_timings timings;
+        std::string assistant_content;
+        generate_completion(assistant_content, timings);
+        impl->messages.push_back({
+            {"role",    "assistant"},
+            {"content", assistant_content}
+        });
+
+        if (params.show_timings) {
+            ui::show_info(string_format(
+                "\n[ Prompt: %.1f t/s | Generation: %.1f t/s ]",
+                timings.prompt_per_second,
+                timings.predicted_per_second
+            ));
+        }
+
+        if (params.single_turn) {
+            break;
+        }
+    }
+
+    ui::show_message("\n\nExiting...");
+
+    return 0;
+}
+
+void cli_context::shutdown() {
+    if (server) {
+        server->stop();
+        server.reset();
+    }
+}

tools/cli/cli-context.h

@@ -0,0 +1,66 @@
+#pragma once
+
+#include "common.h"
+
+#include "cli-client.h"
+#include "cli-server.h"
+
+#include <atomic>
+#include <memory>
+#include <optional>
+#include <string>
+
+struct cli_timings {
+    double prompt_per_second    = 0.0;
+    double predicted_per_second = 0.0;
+};
+
+struct cli_context_impl;
+
+struct cli_context {
+    common_params params;
+
+    cli_client client;                // always initialized
+    std::optional<cli_server> server; // only set when no --server-base is given
+
+    // properties of the connected server
+    // will be populated by fetch_server_props()
+    std::string model_name;
+    std::string model_ftype;
+    std::string build_info;
+    bool has_vision = false;
+    bool has_audio  = false;
+    bool has_video  = false;
+
+    cli_context(const common_params & params);
+    ~cli_context();
+
+    // connect to --server-base or spawn a local llama-server child;
+    // argc/argv are needed to forward the server-relevant args to the child
+    bool init();
+
+    // run the interactive chat loop, returns the process exit code
+    int run();
+
+    // stop the local server child (if any)
+    void shutdown();
+
+    // set by the SIGINT handler; cleared once the interrupt has been handled
+    static std::atomic<bool> & interrupted();
+
+private:
+    bool generate_completion(std::string & assistant_content, cli_timings & timings);
+    void fetch_server_props();
+    void add_system_prompt();
+    void push_user_message(const std::string & text);
+
+    // check if server have multiple models (router mode)
+    // if yes, list them then ask; do nothing otherwise
+    bool list_and_ask_models();
+
+    // read a file and stage it as a multimodal content part; type is one of
+    // "image", "audio", "video"; returns false if the file cannot be read
+    bool stage_media_file(const std::string & fname, const std::string & type);
+
+    std::unique_ptr<cli_context_impl> impl;
+};

tools/cli/cli-server.h

@@ -0,0 +1,89 @@
+#pragma once
+
+#include <thread>
+
+#include "http.h"
+
+// llama_server will be available as a dynamic library symbol
+int llama_server(common_params & params, int argc, char ** argv);
+void llama_server_terminate();
+
+struct cli_server {
+    std::thread th;
+    int port = -1;
+    std::atomic<bool> is_alive = false;
+    std::atomic<bool> is_stopping = false;
+
+    ~cli_server() {
+        stop();
+    }
+
+    void stop() {
+        if (is_stopping.exchange(true)) {
+            return;
+        }
+        if (alive()) {
+            llama_server_terminate();
+        }
+        if (th.joinable()) {
+            th.join();
+        }
+    }
+
+    // spawn llama-server in a thread and interact with it via a random port
+    bool start(common_params & params) {
+        port = common_http_get_free_port();
+        if (port <= 0) {
+            fprintf(stderr, "failed to get a free port\n");
+            exit(1);
+        }
+
+        is_alive.store(true, std::memory_order_release);
+
+        common_params server_params = params; // copy
+        server_params.port = port;
+
+        th = std::thread([this, server_params]() mutable {
+            // argc / argv are only used in router mode, we can skip them for now
+            int res = llama_server(server_params, 0, nullptr);
+            if (res != 0) {
+                fprintf(stderr, "llama_server exited with code %d\n", res);
+            }
+            is_alive.store(false, std::memory_order_release);
+        });
+
+        return true;
+    }
+
+    std::string address() const {
+        return "http://127.0.0.1:" + std::to_string(port);
+    }
+
+    bool wait_ready(std::function<bool()> should_stop) {
+        if (!alive()) {
+            return false;
+        }
+        while (!should_stop()) {
+            auto [cli, parts] = common_http_client(address());
+            cli.set_connection_timeout(1, 0);
+            auto res = cli.Get("/health");
+            if (res) {
+                if (res->status == 200) {
+                    return true;
+                }
+                // any other status means the server is up but not ready yet
+                // (e.g. 503 while the model is still loading)
+            }
+            if (!alive()) {
+                // in case server die permanently
+                return false;
+            }
+            std::this_thread::sleep_for(std::chrono::milliseconds(200));
+        }
+        return true;
+    }
+
+    bool alive() const {
+        return is_alive.load(std::memory_order_acquire);
+    }
+};

tools/cli/cli-ui.h

@@ -0,0 +1,251 @@
+#pragma once
+
+#include "common.h"
+#include "console.h"
+
+#include <array>
+#include <algorithm>
+#include <cctype>
+#include <filesystem>
+#include <string_view>
+
+// TODO?: Make this reusable, enums, docs
+static const std::array<std::string_view, 8> cmds = {
+    "/audio ",
+    "/clear",
+    "/exit",
+    "/glob ",
+    "/image ",
+    "/read ",
+    "/regen",
+    "/video ",
+};
+
+static std::vector<std::pair<std::string, size_t>> auto_completion_callback(std::string_view line, size_t cursor_byte_pos) {
+    std::vector<std::pair<std::string, size_t>> matches;
+    std::string cmd;
+
+    if (line.length() > 1 && line.front() == '/' && !std::any_of(cmds.begin(), cmds.end(), [line](std::string_view prefix) {
+        return string_starts_with(line, prefix);
+    })) {
+        auto it = cmds.begin();
+
+        while ((it = std::find_if(it, cmds.end(), [line](std::string_view cmd_line) {
+            return string_starts_with(cmd_line, line);
+        })) != cmds.end()) {
+            matches.emplace_back(*it, it->length());
+            ++it;
+        }
+    } else {
+        auto it = std::find_if(cmds.begin(), cmds.end(), [line](std::string_view prefix) {
+            return prefix.back() == ' ' && string_starts_with(line, prefix);
+        });
+
+        if (it != cmds.end()) {
+            cmd = *it;
+        }
+    }
+
+    if (!cmd.empty() && cmd != "/glob " && line.length() >= cmd.length() && cursor_byte_pos >= cmd.length()) {
+        const std::string path_prefix  = std::string(line.substr(cmd.length(), cursor_byte_pos - cmd.length()));
+        const std::string path_postfix = std::string(line.substr(cursor_byte_pos));
+        auto cur_dir = std::filesystem::current_path();
+        std::string cur_dir_str = cur_dir.string();
+        std::string expanded_prefix = path_prefix;
+
+#if !defined(_WIN32)
+        if (string_starts_with(path_prefix, '~')) {
+            const char * home = std::getenv("HOME");
+            if (home && home[0]) {
+                expanded_prefix = home + path_prefix.substr(1);
+            }
+        }
+        if (string_starts_with(expanded_prefix, '/')) {
+#else
+        if (std::isalpha(static_cast<unsigned char>(expanded_prefix[0])) && expanded_prefix.find(':') == 1) {
+#endif
+            cur_dir = std::filesystem::path(expanded_prefix).parent_path();
+            cur_dir_str.clear();
+        } else if (!path_prefix.empty()) {
+            cur_dir /= std::filesystem::path(path_prefix).parent_path();
+        }
+
+        std::error_code ec;
+        for (const auto & entry : std::filesystem::directory_iterator(cur_dir, ec)) {
+            if (ec) {
+                break;
+            }
+            if (!entry.exists(ec)) {
+                ec.clear();
+                continue;
+            }
+
+            const std::string path_full = entry.path().string();
+            std::string path_entry = !cur_dir_str.empty() && string_starts_with(path_full, cur_dir_str) ? path_full.substr(cur_dir_str.length() + 1) : path_full;
+
+            if (entry.is_directory(ec)) {
+                path_entry.push_back(std::filesystem::path::preferred_separator);
+            }
+
+            if (expanded_prefix.empty() || string_starts_with(path_entry, expanded_prefix)) {
+                const std::string updated_line = cmd + path_entry;
+                matches.emplace_back(updated_line + path_postfix, updated_line.length());
+            }
+
+            if (ec) {
+                ec.clear();
+            }
+        }
+
+        if (matches.empty()) {
+            const std::string updated_line = cmd + path_prefix;
+            matches.emplace_back(updated_line + path_postfix, updated_line.length());
+        }
+
+        // Add the longest common prefix
+        if (!expanded_prefix.empty() && matches.size() > 1) {
+            const std::string_view match0(matches[0].first);
+            const std::string_view match1(matches[1].first);
+            auto it = std::mismatch(match0.begin(), match0.end(), match1.begin(), match1.end());
+            size_t len = it.first - match0.begin();
+
+            for (size_t i = 2; i < matches.size(); ++i) {
+                const std::string_view matchi(matches[i].first);
+                auto cmp = std::mismatch(match0.begin(), match0.end(), matchi.begin(), matchi.end());
+                len = std::min(len, static_cast<size_t>(cmp.first - match0.begin()));
+            }
+
+            const std::string updated_line = std::string(match0.substr(0, len));
+            matches.emplace_back(updated_line + path_postfix, updated_line.length());
+        }
+
+        std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) {
+            return a.first.compare(0, a.second, b.first, 0, b.second) < 0;
+        });
+    }
+
+    return matches;
+}
+
+// note: make this view implementation generic, so that we can move to TUI in the future if we want to
+namespace ui {
+    static void init(const common_params & params) {
+        // TODO: avoid using atexit() here by making `console` a singleton
+        console::init(params.simple_io, params.use_color);
+        atexit([]() { console::cleanup(); });
+
+        console::set_completion_callback(auto_completion_callback);
+    }
+
+    struct spinner {
+        spinner(const std::string & message) {
+            if (!message.empty()) {
+                console::log("%s ", message.c_str());
+            }
+            console::spinner::start();
+        }
+        ~spinner() {
+            console::spinner::stop();
+        }
+    };
+
+    struct user_turn {
+        user_turn() {
+            console::set_display(DISPLAY_TYPE_USER_INPUT);
+        }
+        ~user_turn() {
+            console::set_display(DISPLAY_TYPE_RESET);
+        }
+        void echo(const std::string & buffer) {
+            if (buffer.size() > 500) {
+                console::log("\n> %s ... (truncated)\n", buffer.substr(0, 500).c_str());
+            } else {
+                console::log("\n> %s\n", buffer.c_str());
+            }
+        }
+        std::string read_input(bool multiline_input, const char * prompt = nullptr) {
+            if (prompt) {
+                console::log("%s", prompt);
+            } else {
+                console::log("\n> ");
+            }
+            std::string buffer;
+            std::string line;
+            bool another_line = true;
+            do {
+                another_line = console::readline(line, multiline_input);
+                buffer += line;
+            } while (another_line);
+            return buffer;
+        }
+    };
+
+    enum assistant_display_mode {
+        ASSISTANT_DISPLAY_MODE_REASONING,
+        ASSISTANT_DISPLAY_MODE_CONTENT,
+    };
+    struct assistant_turn {
+        assistant_display_mode mode = ASSISTANT_DISPLAY_MODE_CONTENT;
+        bool trailing_newline = true;
+        bool is_inside_reasoning = false;
+        assistant_turn() {
+            console::set_display(DISPLAY_TYPE_RESET);
+        }
+        ~assistant_turn() {
+            console::set_display(DISPLAY_TYPE_RESET);
+            add_newline_if_needed();
+        }
+        void push(assistant_display_mode m, const std::string & buffer) {
+            if (m != mode) {
+                add_newline_if_needed();
+                switch (m) {
+                    case ASSISTANT_DISPLAY_MODE_CONTENT:
+                        {
+                            if (is_inside_reasoning) {
+                                console::log("[End thinking]\n\n");
+                                is_inside_reasoning = false;
+                            }
+                            console::set_display(DISPLAY_TYPE_RESET);
+                        } break;
+                    case ASSISTANT_DISPLAY_MODE_REASONING:
+                        {
+                            console::set_display(DISPLAY_TYPE_REASONING);
+                            is_inside_reasoning = true;
+                            console::log("\n[Start thinking]\n\n");
+                        } break;
+                }
+            }
+            mode = m;
+            if (buffer.empty()) {
+                return;
+            }
+            trailing_newline = buffer.back() == '\n';
+            console::log("%s", buffer.c_str());
+            console::flush();
+        }
+        void add_newline_if_needed() {
+            if (!trailing_newline) {
+                console::log("\n");
+                console::flush();
+            }
+        }
+    };
+
+    static void show_error(const std::string & title, const std::string & message = "") {
+        console::spinner::stop();
+        console::error("Error: %s\n", title.c_str());
+        if (!message.empty()) {
+            console::log("%s\n", message.c_str());
+        }
+    }
+
+    static void show_message(const std::string & message) {
+        console::log("%s\n", message.c_str());
+    }
+
+    static void show_info(const std::string & message) {
+        console::set_display(DISPLAY_TYPE_INFO);
+        console::log("%s\n", message.c_str());
+        console::set_display(DISPLAY_TYPE_RESET);
+    }
+}

tools/cli/cli.cpp

@@ -1,20 +1,9 @@
-#include "chat.h"
-#include "common.h"
 #include "arg.h"
-#include "console.h"
-#include "fit.h"
-// #include "log.h"
+#include "common.h"
+#include "log.h"
 
-#include "server-common.h"
-#include "server-context.h"
-#include "server-task.h"
+#include "cli-context.h"
 
-#include <array>
-#include <atomic>
-#include <algorithm>
-#include <filesystem>
-#include <fstream>
-#include <thread>
 #include <signal.h>
 
 #if defined(_WIN32)
@@ -25,342 +14,19 @@
 #include <windows.h>
 #endif
 
-const char * LLAMA_ASCII_LOGO = R"(
-▄▄ ▄▄
-██ ██
-██ ██  ▀▀█▄ ███▄███▄  ▀▀█▄    ▄████ ████▄ ████▄
-██ ██ ▄█▀██ ██ ██ ██ ▄█▀██    ██    ██ ██ ██ ██
-██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
-                                    ██    ██
-                                    ▀▀    ▀▀
-)";
-
-static std::atomic<bool> g_is_interrupted = false;
-static bool should_stop() {
-    return g_is_interrupted.load();
-}
-
 #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
 static void signal_handler(int) {
-    if (g_is_interrupted.load()) {
+    if (cli_context::interrupted().load()) {
         // second Ctrl+C - exit immediately
         // make sure to clear colors before exiting (not using LOG or console.cpp here to avoid deadlock)
         fprintf(stdout, "\033[0m\n");
         fflush(stdout);
         std::exit(130);
     }
-    g_is_interrupted.store(true);
+    cli_context::interrupted().store(true);
 }
 #endif
 
-struct cli_context {
-    server_context ctx_server;
-    json messages = json::array();
-    std::vector<raw_buffer> input_files;
-    task_params defaults;
-    bool verbose_prompt;
-
-    // thread for showing "loading" animation
-    std::atomic<bool> loading_show;
-
-    cli_context(const common_params & params) {
-        defaults.sampling    = params.sampling;
-        defaults.speculative = params.speculative;
-        defaults.n_keep      = params.n_keep;
-        defaults.n_predict   = params.n_predict;
-        defaults.antiprompt  = params.antiprompt;
-
-        defaults.stream = true; // make sure we always use streaming mode
-        defaults.timings_per_token = true; // in order to get timings even when we cancel mid-way
-        // defaults.return_progress = true; // TODO: show progress
-
-        verbose_prompt = params.verbose_prompt;
-    }
-
-    std::string generate_completion(result_timings & out_timings) {
-        server_response_reader rd = ctx_server.get_response_reader();
-        auto chat_params = format_chat();
-        {
-            // TODO: reduce some copies here in the future
-            server_task task = server_task(SERVER_TASK_TYPE_COMPLETION);
-            task.id         = rd.get_new_id();
-            task.index      = 0;
-            task.params     = defaults;           // copy
-            task.cli_prompt = chat_params.prompt; // copy
-            task.cli_files  = input_files;        // copy
-            task.cli        = true;
-
-            // chat template settings
-            task.params.chat_parser_params = common_chat_parser_params(chat_params);
-            task.params.chat_parser_params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
-            if (!chat_params.parser.empty()) {
-                task.params.chat_parser_params.parser.load(chat_params.parser);
-            }
-
-            // Copy the preserved tokens into the sampling params
-            const llama_vocab * vocab = llama_model_get_vocab(
-                llama_get_model(ctx_server.get_llama_context()));
-            for (const auto & token : chat_params.preserved_tokens) {
-                auto ids = common_tokenize(vocab, token, false, true);
-                if (ids.size() == 1) {
-                    task.params.sampling.preserved_tokens.insert(ids[0]);
-                }
-            }
-
-            // reasoning budget sampler
-            if (!chat_params.thinking_end_tag.empty()) {
-                task.params.sampling.reasoning_budget_tokens = defaults.sampling.reasoning_budget_tokens;
-                task.params.sampling.generation_prompt = chat_params.generation_prompt;
-
-                if (!chat_params.thinking_start_tag.empty()) {
-                    task.params.sampling.reasoning_budget_start =
-                        common_tokenize(vocab, chat_params.thinking_start_tag, false, true);
-                }
-                task.params.sampling.reasoning_budget_end =
-                    common_tokenize(vocab, chat_params.thinking_end_tag, false, true);
-                task.params.sampling.reasoning_budget_forced =
-                    common_tokenize(vocab, defaults.sampling.reasoning_budget_message + chat_params.thinking_end_tag, false, true);
-            }
-
-            rd.post_task({std::move(task)});
-        }
-
-        if (verbose_prompt) {
-            console::set_display(DISPLAY_TYPE_PROMPT);
-            console::log("%s\n\n", chat_params.prompt.c_str());
-            console::set_display(DISPLAY_TYPE_RESET);
-        }
-
-        // wait for first result
-        console::spinner::start();
-        server_task_result_ptr result = rd.next(should_stop);
-
-        while (true) {
-            auto res_partial = dynamic_cast<server_task_result_cmpl_partial *>(result.get());
-            if (res_partial && res_partial->is_begin) {
-                // this is the "send 200 status to client" signal in streaming mode
-                // skip, do not stop the spinner
-                result = rd.next(should_stop);
-            } else {
-                console::spinner::stop();
-                break;
-            }
-        }
-
-        std::string curr_content;
-        bool is_thinking = false;
-
-        while (result) {
-            if (should_stop()) {
-                break;
-            }
-            if (result->is_error()) {
-                json err_data = result->to_json();
-                if (err_data.contains("message")) {
-                    console::error("Error: %s\n", err_data["message"].get<std::string>().c_str());
-                } else {
-                    console::error("Error: %s\n", err_data.dump().c_str());
-                }
-                return curr_content;
-            }
-            auto res_partial = dynamic_cast<server_task_result_cmpl_partial *>(result.get());
-            if (res_partial) {
-                out_timings = std::move(res_partial->timings);
-                for (const auto & diff : res_partial->oaicompat_msg_diffs) {
-                    if (!diff.content_delta.empty()) {
-                        if (is_thinking) {
-                            console::log("\n[End thinking]\n\n");
-                            console::set_display(DISPLAY_TYPE_RESET);
-                            is_thinking = false;
-                        }
-                        curr_content += diff.content_delta;
-                        console::log("%s", diff.content_delta.c_str());
-                        console::flush();
-                    }
-                    if (!diff.reasoning_content_delta.empty()) {
-                        console::set_display(DISPLAY_TYPE_REASONING);
-                        if (!is_thinking) {
-                            console::log("[Start thinking]\n");
-                        }
-                        is_thinking = true;
-                        console::log("%s", diff.reasoning_content_delta.c_str());
-                        console::flush();
-                    }
-                }
-            }
-            auto res_final = dynamic_cast<server_task_result_cmpl_final *>(result.get());
-            if (res_final) {
-                out_timings = std::move(res_final->timings);
-                break;
-            }
-            result = rd.next(should_stop);
-        }
-        g_is_interrupted.store(false);
-        // server_response_reader automatically cancels pending tasks upon destruction
-        return curr_content;
-    }
-
-    // TODO: support remote files in the future (http, https, etc)
-    std::string load_input_file(const std::string & fname, bool is_media) {
-        std::ifstream file = fs_open_ifstream(fname, std::ios::binary);
-        if (!file) {
-            return "";
-        }
-        if (is_media) {
-            raw_buffer buf;
-            buf.assign((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
-            input_files.push_back(std::move(buf));
-            return get_media_marker();
-        } else {
-            std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
-            return content;
-        }
-    }
-
-    common_chat_params format_chat() {
-        auto meta = ctx_server.get_meta();
-        auto & chat_params = meta.chat_params;
-
-        auto caps = common_chat_templates_get_caps(chat_params.tmpls.get());
-
-        common_chat_templates_inputs inputs;
-        inputs.messages              = common_chat_msgs_parse_oaicompat(messages);
-        inputs.tools                 = {}; // TODO
-        inputs.tool_choice           = COMMON_CHAT_TOOL_CHOICE_NONE;
-        inputs.json_schema           = ""; // TODO
-        inputs.grammar               = ""; // TODO
-        inputs.use_jinja             = chat_params.use_jinja;
-        inputs.parallel_tool_calls   = caps["supports_parallel_tool_calls"];
-        inputs.add_generation_prompt = true;
-        inputs.reasoning_format      = COMMON_REASONING_FORMAT_DEEPSEEK;
-        inputs.force_pure_content    = chat_params.force_pure_content;
-        inputs.enable_thinking       = chat_params.enable_thinking ? common_chat_templates_support_enable_thinking(chat_params.tmpls.get()) : false;
-
-        // Apply chat template to the list of messages
-        return common_chat_templates_apply(chat_params.tmpls.get(), inputs);
-    }
-};
-
-// TODO?: Make this reusable, enums, docs
-static const std::array<std::string_view, 8> cmds = {
-    "/audio ",
-    "/clear",
-    "/exit",
-    "/glob ",
-    "/image ",
-    "/read ",
-    "/regen",
-    "/video ",
-};
-
-static std::vector<std::pair<std::string, size_t>> auto_completion_callback(std::string_view line, size_t cursor_byte_pos) {
-    std::vector<std::pair<std::string, size_t>> matches;
-    std::string cmd;
-
-    if (line.length() > 1 && line.front() == '/' && !std::any_of(cmds.begin(), cmds.end(), [line](std::string_view prefix) {
-        return string_starts_with(line, prefix);
-    })) {
-        auto it = cmds.begin();
-
-        while ((it = std::find_if(it, cmds.end(), [line](std::string_view cmd_line) {
-            return string_starts_with(cmd_line, line);
-        })) != cmds.end()) {
-            matches.emplace_back(*it, it->length());
-            ++it;
-        }
-    } else {
-        auto it = std::find_if(cmds.begin(), cmds.end(), [line](std::string_view prefix) {
-            return prefix.back() == ' ' && string_starts_with(line, prefix);
-        });
-
-        if (it != cmds.end()) {
-            cmd = *it;
-        }
-    }
-
-    if (!cmd.empty() && cmd != "/glob " && line.length() >= cmd.length() && cursor_byte_pos >= cmd.length()) {
-        const std::string path_prefix  = std::string(line.substr(cmd.length(), cursor_byte_pos - cmd.length()));
-        const std::string path_postfix = std::string(line.substr(cursor_byte_pos));
-        auto cur_dir = std::filesystem::current_path();
-        std::string cur_dir_str = cur_dir.string();
-        std::string expanded_prefix = path_prefix;
-
-#if !defined(_WIN32)
-        if (string_starts_with(path_prefix, '~')) {
-            const char * home = std::getenv("HOME");
-            if (home && home[0]) {
-                expanded_prefix = home + path_prefix.substr(1);
-            }
-        }
-        if (string_starts_with(expanded_prefix, '/')) {
-#else
-        if (std::isalpha(expanded_prefix[0]) && expanded_prefix.find(':') == 1) {
-#endif
-            cur_dir = std::filesystem::path(expanded_prefix).parent_path();
-            cur_dir_str.clear();
-        } else if (!path_prefix.empty()) {
-            cur_dir /= std::filesystem::path(path_prefix).parent_path();
-        }
-
-        std::error_code ec;
-        for (const auto & entry : std::filesystem::directory_iterator(cur_dir, ec)) {
-            if (ec) {
-                break;
-            }
-            if (!entry.exists(ec)) {
-                ec.clear();
-                continue;
-            }
-
-            const std::string path_full = entry.path().string();
-            std::string path_entry = !cur_dir_str.empty() && string_starts_with(path_full, cur_dir_str) ? path_full.substr(cur_dir_str.length() + 1) : path_full;
-
-            if (entry.is_directory(ec)) {
-                path_entry.push_back(std::filesystem::path::preferred_separator);
-            }
-
-            if (expanded_prefix.empty() || string_starts_with(path_entry, expanded_prefix)) {
-                const std::string updated_line = cmd + path_entry;
-                matches.emplace_back(updated_line + path_postfix, updated_line.length());
-            }
-
-            if (ec) {
-                ec.clear();
-            }
-        }
-
-        if (matches.empty()) {
-            const std::string updated_line = cmd + path_prefix;
-            matches.emplace_back(updated_line + path_postfix, updated_line.length());
-        }
-
-        // Add the longest common prefix
-        if (!expanded_prefix.empty() && matches.size() > 1) {
-            const std::string_view match0(matches[0].first);
-            const std::string_view match1(matches[1].first);
-            auto it = std::mismatch(match0.begin(), match0.end(), match1.begin(), match1.end());
-            size_t len = it.first - match0.begin();
-
-            for (size_t i = 2; i < matches.size(); ++i) {
-                const std::string_view matchi(matches[i].first);
-                auto cmp = std::mismatch(match0.begin(), match0.end(), matchi.begin(), matchi.end());
-                len = std::min(len, static_cast<size_t>(cmp.first - match0.begin()));
-            }
-
-            const std::string updated_line = std::string(match0.substr(0, len));
-            matches.emplace_back(updated_line + path_postfix, updated_line.length());
-        }
-
-        std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) {
-            return a.first.compare(0, a.second, b.first, 0, b.second) < 0;
-        });
-    }
-
-    return matches;
-}
-
-static constexpr size_t FILE_GLOB_MAX_RESULTS = 100;
-
 // satisfies -Wmissing-declarations
 int llama_cli(int argc, char ** argv);
 
@@ -375,25 +41,6 @@ int llama_cli(int argc, char ** argv) {
         return 1;
     }
 
-    // TODO: maybe support it later?
-    if (params.conversation_mode == COMMON_CONVERSATION_MODE_DISABLED) {
-        console::error("--no-conversation is not supported by llama-cli\n");
-        console::error("please use llama-completion instead\n");
-    }
-
-    // struct that contains llama context and inference
-    cli_context ctx_cli(params);
-
-    llama_backend_init();
-    llama_numa_init(params.numa);
-
-    // TODO: avoid using atexit() here by making `console` a singleton
-    console::init(params.simple_io, params.use_color);
-    atexit([]() { console::cleanup(); });
-
-    console::set_display(DISPLAY_TYPE_RESET);
-    console::set_completion_callback(auto_completion_callback);
-
 #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
     struct sigaction sigint_action;
     sigint_action.sa_handler = signal_handler;
@@ -408,276 +55,11 @@ int llama_cli(int argc, char ** argv) {
     SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
 #endif
 
-    console::log("\nLoading model... "); // followed by loading animation
-    console::spinner::start();
-    if (!ctx_cli.ctx_server.load_model(params)) {
-        console::spinner::stop();
-        console::error("\nFailed to load the model\n");
+    cli_context ctx_cli(params);
+
+    if (!ctx_cli.init()) {
         return 1;
     }
 
-    ctx_cli.defaults.sampling = params.sampling;
-
-    console::spinner::stop();
-    console::log("\n");
-
-    std::thread inference_thread([&ctx_cli]() {
-        ctx_cli.ctx_server.start_loop();
-    });
-
-    auto inf = ctx_cli.ctx_server.get_meta();
-    std::string modalities = "text";
-    if (inf.has_inp_image) {
-        modalities += ", vision";
-    }
-    if (inf.has_inp_audio) {
-        modalities += ", audio";
-    }
-
-    auto add_system_prompt = [&]() {
-        if (!params.system_prompt.empty()) {
-            ctx_cli.messages.push_back({
-                {"role",    "system"},
-                {"content", params.system_prompt}
-            });
-        }
-    };
-    add_system_prompt();
-
-    console::log("\n");
-    console::log("%s\n", LLAMA_ASCII_LOGO);
-    console::log("build      : %s\n", inf.build_info.c_str());
-    console::log("model      : %s\n", inf.model_name.c_str());
-    if (!inf.model_ftype.empty()) {
-        console::log("ftype      : %s\n", inf.model_ftype.c_str());
-    }
-    console::log("modalities : %s\n", modalities.c_str());
-    if (!params.system_prompt.empty()) {
-        console::log("using custom system prompt\n");
-    }
-    console::log("\n");
-    console::log("available commands:\n");
-    console::log("  /exit or Ctrl+C     stop or exit\n");
-    console::log("  /regen              regenerate the last response\n");
-    console::log("  /clear              clear the chat history\n");
-    console::log("  /read <file>        add a text file\n");
-    console::log("  /glob <pattern>     add text files using globbing pattern\n");
-    if (inf.has_inp_image) {
-        console::log("  /image <file>       add an image file\n");
-    }
-    if (inf.has_inp_audio) {
-        console::log("  /audio <file>       add an audio file\n");
-    }
-    if (inf.has_inp_video) {
-        console::log("  /video <file>       add a video file\n");
-    }
-    console::log("\n");
-
-    // interactive loop
-    std::string cur_msg;
-
-    auto add_text_file = [&](const std::string & fname) -> bool {
-        std::string marker = ctx_cli.load_input_file(fname, false);
-        if (marker.empty()) {
-            console::error("file does not exist or cannot be opened: '%s'\n", fname.c_str());
-            return false;
-        }
-        if (inf.fim_sep_token != LLAMA_TOKEN_NULL) {
-            cur_msg += common_token_to_piece(ctx_cli.ctx_server.get_llama_context(), inf.fim_sep_token, true);
-            cur_msg += fname;
-            cur_msg.push_back('\n');
-        } else {
-            cur_msg += "--- File: ";
-            cur_msg += fname;
-            cur_msg += " ---\n";
-        }
-        cur_msg += marker;
-        console::log("Loaded text from '%s'\n", fname.c_str());
-        return true;
-    };
-
-    while (true) {
-        std::string buffer;
-        console::set_display(DISPLAY_TYPE_USER_INPUT);
-        if (params.prompt.empty()) {
-            console::log("\n> ");
-            std::string line;
-            bool another_line = true;
-            do {
-                another_line = console::readline(line, params.multiline_input);
-                buffer += line;
-            } while (another_line);
-        } else {
-            // process input prompt from args
-            for (auto & fname : params.image) {
-                std::string marker = ctx_cli.load_input_file(fname, true);
-                if (marker.empty()) {
-                    console::error("file does not exist or cannot be opened: '%s'\n", fname.c_str());
-                    break;
-                }
-                console::log("Loaded media from '%s'\n", fname.c_str());
-                cur_msg += marker;
-            }
-            buffer = params.prompt;
-            if (buffer.size() > 500) {
-                console::log("\n> %s ... (truncated)\n", buffer.substr(0, 500).c_str());
-            } else {
-                console::log("\n> %s\n", buffer.c_str());
-            }
-            params.prompt.clear(); // only use it once
-        }
-        console::set_display(DISPLAY_TYPE_RESET);
-        console::log("\n");
-
-        if (should_stop()) {
-            g_is_interrupted.store(false);
-            break;
-        }
-
-        // remove trailing newline
-        if (!buffer.empty() &&buffer.back() == '\n') {
-            buffer.pop_back();
-        }
-
-        // skip empty messages
-        if (buffer.empty()) {
-            continue;
-        }
-
-        bool add_user_msg = true;
-
-        // process commands
-        if (string_starts_with(buffer, "/exit")) {
-            break;
-        } else if (string_starts_with(buffer, "/regen")) {
-            if (ctx_cli.messages.size() >= 2) {
-                size_t last_idx = ctx_cli.messages.size() - 1;
-                ctx_cli.messages.erase(last_idx);
-                add_user_msg = false;
-            } else {
-                console::error("No message to regenerate.\n");
-                continue;
-            }
-        } else if (string_starts_with(buffer, "/clear")) {
-            ctx_cli.messages.clear();
-            add_system_prompt();
-
-            ctx_cli.input_files.clear();
-            console::log("Chat history cleared.\n");
-            continue;
-        } else if (
-                (string_starts_with(buffer, "/image ") && inf.has_inp_image) ||
-                (string_starts_with(buffer, "/audio ") && inf.has_inp_audio) ||
-                (string_starts_with(buffer, "/video ") && inf.has_inp_video)) {
-            // just in case (bad copy-paste for example), we strip all trailing/leading spaces
-            std::string fname = string_strip(buffer.substr(7));
-            std::string marker = ctx_cli.load_input_file(fname, true);
-            if (marker.empty()) {
-                console::error("file does not exist or cannot be opened: '%s'\n", fname.c_str());
-                continue;
-            }
-            cur_msg += marker;
-            console::log("Loaded media from '%s'\n", fname.c_str());
-            continue;
-        } else if (string_starts_with(buffer, "/read ")) {
-            std::string fname = string_strip(buffer.substr(6));
-            add_text_file(fname);
-            continue;
-        } else if (string_starts_with(buffer, "/glob ")) {
-            std::error_code ec;
-            size_t count = 0;
-            auto curdir = std::filesystem::current_path();
-            std::string pattern = string_strip(buffer.substr(6));
-            std::filesystem::path rel_path;
-
-            auto startglob = pattern.find_first_of("![*?");
-            if (startglob != std::string::npos && startglob != 0) {
-                auto endpath = pattern.substr(0, startglob).find_last_of('/');
-                if (endpath != std::string::npos) {
-                    std::string rel_pattern = pattern.substr(0, endpath);
-#if !defined(_WIN32)
-                    if (string_starts_with(rel_pattern, '~')) {
-                        const char * home = std::getenv("HOME");
-                        if (home && home[0]) {
-                            rel_pattern = home + rel_pattern.substr(1);
-                        }
-                    }
-#endif
-                    rel_path = rel_pattern;
-                    pattern.erase(0, endpath + 1);
-                    curdir /= rel_path;
-                }
-            }
-
-            for (const auto & entry : std::filesystem::recursive_directory_iterator(curdir,
-                    std::filesystem::directory_options::skip_permission_denied, ec)) {
-                if (!entry.is_regular_file()) {
-                    continue;
-                }
-
-                std::string rel = std::filesystem::relative(entry.path(), curdir, ec).string();
-                if (ec) {
-                    ec.clear();
-                    continue;
-                }
-                std::replace(rel.begin(), rel.end(), '\\', '/');
-
-                if (!glob_match(pattern, rel)) {
-                    continue;
-                }
-
-                if (!add_text_file((rel_path / rel).string())) {
-                    continue;
-                }
-
-                if (++count >= FILE_GLOB_MAX_RESULTS) {
-                    console::error("Maximum number of globbed files allowed (%zu) reached.\n", FILE_GLOB_MAX_RESULTS);
-                    break;
-                }
-            }
-            continue;
-        } else {
-            // not a command
-            cur_msg += buffer;
-        }
-
-        // generate response
-        if (add_user_msg) {
-            ctx_cli.messages.push_back({
-                {"role",    "user"},
-                {"content", cur_msg}
-            });
-            cur_msg.clear();
-        }
-        result_timings timings;
-        std::string assistant_content = ctx_cli.generate_completion(timings);
-        ctx_cli.messages.push_back({
-            {"role",    "assistant"},
-            {"content", assistant_content}
-        });
-        console::log("\n");
-
-        if (params.show_timings) {
-            console::set_display(DISPLAY_TYPE_INFO);
-            console::log("\n");
-            console::log("[ Prompt: %.1f t/s | Generation: %.1f t/s ]\n", timings.prompt_per_second, timings.predicted_per_second);
-            console::set_display(DISPLAY_TYPE_RESET);
-        }
-
-        if (params.single_turn) {
-            break;
-        }
-    }
-
-    console::set_display(DISPLAY_TYPE_RESET);
-
-    console::log("\nExiting...\n");
-    ctx_cli.ctx_server.terminate();
-    inference_thread.join();
-
-    // bump the log level to display timings
-    common_log_set_verbosity_thold(LOG_LEVEL_INFO);
-    common_memory_breakdown_print(ctx_cli.ctx_server.get_llama_context());
-
-    return 0;
+    return ctx_cli.run();
 }

tools/server/README-dev.md

@@ -57,7 +57,7 @@ The core architecture consists of the following components:
 - `server_tokens`: Unified representation of token sequences (supports both text and multimodal tokens); used by `server_task` and `server_slot`.
 - `server_prompt_checkpoint`: For recurrent (e.g., RWKV) and SWA models, stores snapshots of KV cache state. Enables reuse when subsequent requests share the same prompt prefix, saving redundant computation.
 - `server_models`: Standalone component for managing multiple backend instances (used in router mode). It is completely independent of `server_context`.
-- `stream_session_manager`: process wide owner of resumable SSE stream sessions, keyed by conversation id. A file-static singleton inside `server-stream.cpp`, driven through `server_stream_session_manager_start/stop`. Backs the replay buffer that lets a client reattach to a generation after an HTTP disconnect. See the "Resumable streaming" section below.
+- `stream_session_manager`: Process wide owner of resumable SSE stream sessions (`g_stream_sessions`), keyed by conversation id. Backs the replay buffer that lets a client reattach to a generation after an HTTP disconnect. See the "Resumable streaming" section below.
 
 ```mermaid
 graph TD
@@ -127,12 +127,10 @@ It is opt in via the `X-Conversation-Id` header on `POST /v1/chat/completions`.
 The feature lives entirely in `server-stream.{h,cpp}` and rests on three types:
 
 - `stream_session`: a bounded ring buffer (4 MiB cap, oldest bytes drop first) plus a condvar. `append` pushes raw SSE bytes, `read_from` drains from any offset and blocks for live bytes or finalize, `finalize` wakes readers, `cancel` stops the producer. One conv maps to at most one live session.
-- `stream_session_manager`: a file-static singleton (`g_stream_sessions`) inside `server-stream.cpp`, owns all sessions keyed by conv id, enforces the one conv one session invariant via `create_or_replace`, and runs a GC thread that drops completed sessions past their TTL. Exposed to main only through `server_stream_session_manager_start/stop`.
+- `stream_session_manager` (`g_stream_sessions`): owns all sessions keyed by conv id, enforces the one conv one session invariant via `create_or_replace`, and runs a GC thread that drops completed sessions past their TTL.
 - `stream_pipe_producer` / `stream_pipe_consumer`: the write and read ends. The producer owns the session lifetime and finalizes it on destruction; the consumer is read only and never finalizes, so a reader detaching cannot kill a running generation.
 
-The implementation is hidden in `server-stream.cpp` (pimpl). The header exposes only the route handler factories, `server_stream_session_attach_pipe`, `server_stream_aware_should_stop`, `server_stream_conv_id_from_headers` and the GC lifecycle; the session, manager and consumer types stay in the `.cpp`.
-
-Producer side: `server_res_generator` attaches a producer pipe when the header is present. The HTTP content provider mirrors every chunk into the ring before writing it to the socket. While a pipe is attached, `server_stream_aware_should_stop` ignores peer disconnect, so a dropped socket does not stop generation: only an explicit `DELETE` does. When the peer leaves early, `on_complete` calls `close()`, which drains the rest of the generation into the ring on the http worker.
+Producer side: `server_res_generator` attaches a producer pipe when the header is present. The HTTP content provider mirrors every chunk into the ring before writing it to the socket. While a pipe is attached, `stream_aware_should_stop` ignores peer disconnect, so a dropped socket does not stop generation: only an explicit `DELETE` does. When the peer leaves early, `on_complete` calls `close()`, which drains the rest of the generation into the ring on the http worker.
 
 Lifetime safety: the producer pipe holds a shared `alive` flag also captured by the session cancel hook. `~server_res_generator` calls `cleanup()` to clear that hook while the reader is still alive, so a `cancel` arriving during teardown can never call `stop()` on a freed response. This ordering is the most fragile part of the feature: finalizing or destroying the producer before `cleanup()` runs reintroduces a use after free.
 
@@ -146,7 +144,7 @@ Routes:
 
 Router mode binds the same paths to proxy handlers. A `conv_id -> child` map (`conv_models`), populated when a POST is routed, resolves the owning child in one lookup with no polling. The lookup groups ids per child; GET and DELETE proxy straight to the owner. This loopback REST hop is expected to move to a websocket IPC later, swapping only the transport.
 
-Lifecycle: `server_stream_session_manager_start()` runs in main after common init, `server_stream_session_manager_stop()` runs first in `clean_up()` and finalizes every live session so no reader hangs. Reader blocking and the post drop drain both run on httplib worker threads, which block on a condvar rather than spin.
+Lifecycle: `g_stream_sessions.start_gc()` runs in main after common init, `stop_gc()` runs first in `clean_up()` and finalizes every live session so no reader hangs. Reader blocking and the post drop drain both run on httplib worker threads, which block on a condvar rather than spin.
 
 | Constant | Value | Role |
 | --- | --- | --- |

tools/server/README.md

@@ -228,7 +228,7 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `-sps, --slot-prompt-similarity SIMILARITY` | how much the prompt of a request must match the prompt of a slot in order to use that slot (default: 0.10, 0.0 = disabled) |
 | `--lora-init-without-apply` | load LoRA adapters without applying them (apply later via POST /lora-adapters) (default: disabled) |
 | `--sleep-idle-seconds SECONDS` | number of seconds of idleness after which the server will sleep (default: -1; -1 = disabled) |
-| `--log-prompts-dir PATH` | Log prompts to directory (auto-created if not present; only used for debugging, default: disabled) |
+| `--log-prompts-dir PATH` | Log prompts to directory (only used for debugging, default: disabled) |
 | `--spec-draft-hf, -hfd, -hfrd, --hf-repo-draft <user>/<model>[:quant]` | Same as --hf-repo, but for the draft model (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_HF_REPO) |
 | `--spec-draft-threads, -td, --threads-draft N` | number of threads to use during generation (default: same as --threads) |
 | `--spec-draft-threads-batch, -tbd, --threads-batch-draft N` | number of threads to use during batch and prompt processing (default: same as --threads-draft) |

tools/server/server-context.cpp

@@ -4188,7 +4188,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
                 }
             };
 
-            auto effective_should_stop = server_stream_aware_should_stop(res_this, req.should_stop);
+            auto effective_should_stop = stream_aware_should_stop(res_this, req.should_stop);
 
             try {
                 if (effective_should_stop()) {
@@ -4284,7 +4284,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
 
     // attach a producer pipe to the response when X-Conversation-Id is present.
     // the pipe mirrors SSE chunks into the ring buffer and wires up the cancel hook.
-    server_stream_session_attach_pipe(*res, req.headers);
+    stream_session_attach_pipe(*res, req.headers);
 
     return res;
 }
@@ -4521,6 +4521,7 @@ void server_routes::init_routes() {
             { "default_generation_settings", default_generation_settings_for_props },
             { "total_slots",                 params.n_parallel },
             { "model_alias",                 meta->model_name },
+            { "model_ftype",                 meta->model_ftype },
             { "model_path",                  meta->model_path },
             { "modalities",                  json {
                 {"vision", meta->has_inp_image},

tools/server/server-models.cpp

@@ -7,6 +7,7 @@
 #include "build-info.h"
 #include "preset.h"
 #include "download.h"
+#include "http.h"
 
 #include <cpp-httplib/httplib.h> // TODO: remove this once we use HTTP client from download.h
 #include <optional>
@@ -28,14 +29,7 @@
 #include <sstream>
 #include <cstring>
 
-#ifdef _WIN32
-#include <winsock2.h>
-#include <windows.h>
-#else
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <unistd.h>
+#ifndef _WIN32
 extern char **environ;
 #endif
 
@@ -716,66 +710,6 @@ std::optional<server_model_meta> server_models::get_meta(const std::string & nam
     return std::nullopt;
 }
 
-static int get_free_port() {
-#ifdef _WIN32
-    WSADATA wsaData;
-    if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) {
-        return -1;
-    }
-    typedef SOCKET native_socket_t;
-#define INVALID_SOCKET_VAL INVALID_SOCKET
-#define CLOSE_SOCKET(s) closesocket(s)
-#else
-    typedef int native_socket_t;
-#define INVALID_SOCKET_VAL -1
-#define CLOSE_SOCKET(s) close(s)
-#endif
-
-    native_socket_t sock = socket(AF_INET, SOCK_STREAM, 0);
-    if (sock == INVALID_SOCKET_VAL) {
-#ifdef _WIN32
-        WSACleanup();
-#endif
-        return -1;
-    }
-
-    struct sockaddr_in serv_addr;
-    std::memset(&serv_addr, 0, sizeof(serv_addr));
-    serv_addr.sin_family = AF_INET;
-    serv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
-    serv_addr.sin_port = htons(0);
-
-    if (bind(sock, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) != 0) {
-        CLOSE_SOCKET(sock);
-#ifdef _WIN32
-        WSACleanup();
-#endif
-        return -1;
-    }
-
-#ifdef _WIN32
-    int namelen = sizeof(serv_addr);
-#else
-    socklen_t namelen = sizeof(serv_addr);
-#endif
-    if (getsockname(sock, (struct sockaddr*)&serv_addr, &namelen) != 0) {
-        CLOSE_SOCKET(sock);
-#ifdef _WIN32
-        WSACleanup();
-#endif
-        return -1;
-    }
-
-    int port = ntohs(serv_addr.sin_port);
-
-    CLOSE_SOCKET(sock);
-#ifdef _WIN32
-    WSACleanup();
-#endif
-
-    return port;
-}
-
 // helper to convert vector<string> to char **
 // pointers are only valid as long as the original vector is valid
 static std::vector<char *> to_char_ptr_array(const std::vector<std::string> & vec) {
@@ -879,7 +813,7 @@ void server_models::load(const std::string & name, const load_options & opts) {
     // prepare new instance info
     instance_t inst;
     inst.meta             = meta;
-    inst.meta.port        = get_free_port();
+    inst.meta.port        = common_http_get_free_port();
     inst.meta.status      = SERVER_MODEL_STATUS_LOADING;
     inst.meta.loaded_info = json{};
     inst.meta.last_used   = ggml_time_ms();
@@ -1681,7 +1615,7 @@ void server_models_routes::init_routes() {
         }
         // remember which child serves this conversation so the stream routes can route straight
         // to it without polling, keyed on the exact conv id from the header
-        std::string conv_id = server_stream_conv_id_from_headers(req.headers);
+        std::string conv_id = stream_conv_id_from_headers(req.headers);
         if (!conv_id.empty()) {
             models.conv_models.remember(conv_id, name);
         }
@@ -1896,7 +1830,7 @@ void server_models_routes::init_routes() {
         if (!from.empty()) {
             child_path += "?from=" + from;
         }
-        SRV_TRC("proxying stream resume to model %s on port %d, path=%s\n",
+        SRV_INF("proxying stream resume to model %s on port %d, path=%s\n",
                 owner->name.c_str(), owner->port, child_path.c_str());
         auto proxy = std::make_unique<server_http_proxy>(
                 "GET",

tools/server/server-stream.cpp

@@ -6,12 +6,6 @@
 #include <chrono>
 #include <memory>
 #include <utility>
-#include <shared_mutex>
-
-enum class stream_read_status {
-    OK,
-    OFFSET_LOST,
-};
 
 namespace {
 constexpr int64_t STREAM_SESSION_TTL_SECONDS         = 300;
@@ -19,6 +13,7 @@ constexpr size_t  STREAM_SESSION_MAX_BYTES           = 4 * 1024 * 1024;
 constexpr int64_t STREAM_SESSION_GC_INTERVAL_SECONDS = 60;
 constexpr int64_t STREAM_READ_WAKE_INTERVAL_MS       = 200;
 
+// returns unix time in seconds
 int64_t now_seconds() {
     return std::chrono::duration_cast<std::chrono::seconds>(
         std::chrono::system_clock::now().time_since_epoch()
@@ -26,91 +21,6 @@ int64_t now_seconds() {
 }
 }
 
-// owns all live sessions keyed by conversation_id, one conv = at most one live session.
-// a periodic GC evicts expired ones
-class stream_session_manager {
-public:
-    stream_session_manager();
-    ~stream_session_manager();
-
-    stream_session_manager(const stream_session_manager &)             = delete;
-    stream_session_manager & operator=(const stream_session_manager &) = delete;
-
-    // install a new session, evicting and cancelling any previous one. conversation_id must be non empty
-    stream_session_ptr create_or_replace(const std::string & conversation_id);
-
-    stream_session_ptr get(const std::string & conversation_id);
-
-    std::vector<stream_session_ptr> list_all() const;
-
-    void evict(const std::string & conversation_id);
-
-    void evict_and_cancel(const std::string & conversation_id);
-
-    void start_gc();
-    void stop_gc();
-
-private:
-    void gc_loop();
-
-    mutable std::shared_mutex                           map_mu;
-    std::unordered_map<std::string, stream_session_ptr> sessions; // key: conversation_id
-    std::thread                                         gc_thread;
-    bool                                                running;
-    std::mutex                                          gc_wake_mu;
-    std::condition_variable                             gc_wake_cv;
-};
-
-// process wide manager, lifecycle controlled by llama-server main() via start_gc/stop_gc
-static stream_session_manager g_stream_sessions;
-
-void server_stream_session_manager_start() {
-    g_stream_sessions.start_gc();
-}
-
-void server_stream_session_manager_stop() {
-    g_stream_sessions.stop_gc();
-}
-
-struct stream_session {
-    std::string conversation_id;
-    int64_t     started_ts; // unix seconds at construction
-
-    stream_session(std::string conversation_id_, size_t max_bytes_);
-    stream_session(const stream_session &)             = delete;
-    stream_session & operator=(const stream_session &) = delete;
-
-    bool append(const char * data, size_t len);
-
-    void finalize();
-
-    // drain from offset into sink, blocking for more bytes or finalize. OFFSET_LOST if offset
-    // fell below the dropped prefix
-    stream_read_status read_from(size_t offset,
-        const std::function<bool(const char *, size_t)> & sink,
-        const std::function<bool()> & should_stop);
-
-    bool    is_done() const;
-    bool    is_cancelled() const;
-    size_t  total_size() const;     // bytes that ever entered the session
-    size_t  dropped_prefix() const; // bytes evicted from the front due to cap
-    int64_t completed_at() const;   // 0 while alive, unix seconds after finalize
-
-    void set_stop_producer(std::function<void()> fn);
-
-    void cancel();
-
-private:
-    mutable std::mutex      mu;
-    std::condition_variable cv;
-    std::vector<char>       buffer;
-    size_t                  prefix_dropped;
-    size_t                  cap_bytes;
-    bool                    done;
-    std::atomic<bool>       cancelled; // polled lock-free by the should_stop closure, no mu
-    int64_t                 completed_ts;
-    std::function<void()>   stop_producer;
-};
 stream_session::stream_session(std::string conversation_id_, size_t max_bytes_)
     : conversation_id(std::move(conversation_id_))
     , started_ts(now_seconds())
@@ -128,7 +38,7 @@ bool stream_session::append(const char * data, size_t len) {
     }
     {
         std::lock_guard<std::mutex> lock(mu);
-        if (done) {
+        if (done.load(std::memory_order_relaxed)) {
             return false;
         }
         if (len >= cap_bytes) {
@@ -152,14 +62,11 @@ bool stream_session::append(const char * data, size_t len) {
 }
 
 void stream_session::finalize() {
-    {
-        std::lock_guard<std::mutex> lock(mu);
-        if (done) {
-            return;
-        }
-        done         = true;
-        completed_ts = now_seconds();
+    bool was_done = done.exchange(true, std::memory_order_acq_rel);
+    if (was_done) {
+        return;
     }
+    completed_ts.store(now_seconds(), std::memory_order_release);
     cv.notify_all();
 }
 
@@ -189,7 +96,7 @@ stream_read_status stream_session::read_from(size_t offset,
             lock.lock();
             continue;
         }
-        if (done) {
+        if (done.load(std::memory_order_acquire)) {
             return stream_read_status::OK;
         }
         // wait for new bytes, finalize, or a periodic wake to re check should_stop
@@ -198,8 +105,7 @@ stream_read_status stream_session::read_from(size_t offset,
 }
 
 bool stream_session::is_done() const {
-    std::lock_guard<std::mutex> lock(mu);
-    return done;
+    return done.load(std::memory_order_acquire);
 }
 
 size_t stream_session::total_size() const {
@@ -213,8 +119,7 @@ size_t stream_session::dropped_prefix() const {
 }
 
 int64_t stream_session::completed_at() const {
-    std::lock_guard<std::mutex> lock(mu);
-    return completed_ts;
+    return completed_ts.load(std::memory_order_acquire);
 }
 
 void stream_session::set_stop_producer(std::function<void()> fn) {
@@ -223,7 +128,7 @@ void stream_session::set_stop_producer(std::function<void()> fn) {
 }
 
 void stream_session::cancel() {
-    // flip cancelled first so the producer-side server_stream_aware_should_stop can break out of the
+    // flip cancelled first so the producer-side stream_aware_should_stop can break out of the
     // recv() wait even if remove_waiting_task_ids does not notify the condvar (the cancel task
     // posted by rd.stop() will eventually notify, but we do not want to depend on that timing)
     cancelled.store(true, std::memory_order_release);
@@ -332,24 +237,18 @@ void stream_session_manager::evict_and_cancel(const std::string & conversation_i
 }
 
 void stream_session_manager::start_gc() {
-    {
-        std::lock_guard<std::mutex> lock(gc_wake_mu);
-        if (running) {
-            return;
-        }
-        running = true;
+    if (running.exchange(true)) {
+        return;
     }
     gc_thread = std::thread([this] { gc_loop(); });
 }
 
 void stream_session_manager::stop_gc() {
-    bool was_running;
-    {
-        std::lock_guard<std::mutex> lock(gc_wake_mu);
-        was_running = running;
-        running = false;
-    }
+    bool was_running = running.exchange(false);
     if (was_running) {
+        {
+            std::lock_guard<std::mutex> lock(gc_wake_mu);
+        }
         gc_wake_cv.notify_all();
         if (gc_thread.joinable()) {
             gc_thread.join();
@@ -371,15 +270,15 @@ void stream_session_manager::stop_gc() {
 }
 
 void stream_session_manager::gc_loop() {
-    while (true) {
+    while (running.load(std::memory_order_acquire)) {
         {
             std::unique_lock<std::mutex> lock(gc_wake_mu);
             gc_wake_cv.wait_for(lock,
                 std::chrono::seconds(STREAM_SESSION_GC_INTERVAL_SECONDS),
-                [this] { return !running; });
-            if (!running) {
-                return;
-            }
+                [this] { return !running.load(std::memory_order_acquire); });
+        }
+        if (!running.load(std::memory_order_acquire)) {
+            return;
         }
         int64_t cutoff = now_seconds() - STREAM_SESSION_TTL_SECONDS;
         std::vector<stream_session_ptr> to_drop;
@@ -402,19 +301,10 @@ void stream_session_manager::gc_loop() {
     }
 }
 
-// stream_pipe
+// process wide manager, lifecycle controlled by llama-server main() via start_gc/stop_gc
+stream_session_manager g_stream_sessions;
 
-// consumer end: read-only replay of the ring buffer, the destructor does not finalize the session
-struct stream_pipe_consumer : stream_pipe {
-    stream_read_status read(size_t & offset,
-        const std::function<bool(const char *, size_t)> & sink,
-        const std::function<bool()> & should_stop);
-
-    static std::shared_ptr<stream_pipe_consumer> create(stream_session_ptr session);
-
-private:
-    explicit stream_pipe_consumer(stream_session_ptr session);
-};
+// stream_pipe ---------------------------------------------------------------------------------
 
 stream_pipe::stream_pipe(stream_session_ptr session)
     : session_(std::move(session)) {
@@ -518,10 +408,12 @@ static server_http_res_ptr make_error_response(int status, const std::string & m
     return res;
 }
 
-server_http_context::handler_t server_stream_make_get_handler() {
+server_http_context::handler_t make_stream_get_handler() {
     return [](const server_http_req & req) -> server_http_res_ptr {
-        // GET /v1/stream/<conv_id>?from=N replays buffered SSE bytes then blocks for live
-        // bytes until the session finalizes, streamed as text/event-stream for EventSource
+        // GET /v1/stream/<conv_id>?from=N replays the SSE bytes already buffered for the
+        // session, blocks for more bytes when the session is still running, returns when
+        // the session is finalized. the body is streamed back as text/event-stream so the
+        // browser EventSource can attach to it like a fresh request
         std::string conv_id = req.get_param("conv_id");
         if (conv_id.empty()) {
             return make_error_response(400, "Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST);
@@ -567,10 +459,11 @@ server_http_context::handler_t server_stream_make_get_handler() {
     };
 }
 
-server_http_context::handler_t server_stream_make_lookup_handler() {
+server_http_context::handler_t make_streams_lookup_handler() {
     return [](const server_http_req & req) -> server_http_res_ptr {
-        // POST /v1/streams/lookup returns the matching sessions, only for ids the caller already
-        // knows. each id matches the exact key and any "<id>::<model>" per model variant
+        // POST /v1/streams/lookup with body {"conversation_ids": ["X", "Y", ...]} returns the
+        // matching sessions, only for ids the caller already knows. each id matches the exact key
+        // and any "<id>::<model>" variant, so one lookup covers every per model session for a conv
         std::vector<std::string> requested;
         try {
             json body = json::parse(req.body);
@@ -625,10 +518,11 @@ server_http_context::handler_t server_stream_make_lookup_handler() {
     };
 }
 
-server_http_context::handler_t server_stream_make_delete_handler() {
+server_http_context::handler_t make_stream_delete_handler() {
     return [](const server_http_req & req) -> server_http_res_ptr {
-        // DELETE /v1/stream/<conv_id> is the explicit user Stop, cancels the producer and evicts
-        // the buffer. idempotent, returns 204 even if the session was already gone
+        // DELETE /v1/stream/<conv_id> is the explicit user Stop, cancels the producer hook
+        // wired by handle_completions_impl and evicts the buffer. idempotent, a session that
+        // already finalized or was never created returns 204 either way
         std::string conv_id = req.get_param("conv_id");
         if (conv_id.empty()) {
             return make_error_response(400, "Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST);
@@ -642,7 +536,7 @@ server_http_context::handler_t server_stream_make_delete_handler() {
     };
 }
 
-std::string server_stream_conv_id_from_headers(const std::map<std::string, std::string> & headers) {
+std::string stream_conv_id_from_headers(const std::map<std::string, std::string> & headers) {
     // case-insensitive scan for x-conversation-id
     static constexpr char   target[]   = "x-conversation-id";
     static constexpr size_t target_len = sizeof(target) - 1;
@@ -661,8 +555,8 @@ std::string server_stream_conv_id_from_headers(const std::map<std::string, std::
     return std::string();
 }
 
-void server_stream_session_attach_pipe(server_http_res & res, const std::map<std::string, std::string> & headers) {
-    std::string conversation_id = server_stream_conv_id_from_headers(headers);
+void stream_session_attach_pipe(server_http_res & res, const std::map<std::string, std::string> & headers) {
+    std::string conversation_id = stream_conv_id_from_headers(headers);
     SRV_TRC("conv_id=%s (empty=%d)\n", conversation_id.c_str(), conversation_id.empty() ? 1 : 0);
     if (conversation_id.empty()) {
         return;
@@ -671,7 +565,7 @@ void server_stream_session_attach_pipe(server_http_res & res, const std::map<std
     res.spipe = stream_pipe_producer::create(session, res);
 }
 
-std::function<bool()> server_stream_aware_should_stop(server_http_res * res, std::function<bool()> fallback) {
+std::function<bool()> stream_aware_should_stop(server_http_res * res, std::function<bool()> fallback) {
     return [res, fallback = std::move(fallback)]() -> bool {
         if (res->spipe) {
             return res->spipe->is_cancelled();

tools/server/server-stream.h

@@ -3,23 +3,81 @@
 #include "server-http.h"
 
 #include <atomic>
+#include <condition_variable>
 #include <cstddef>
+#include <cstdint>
 #include <functional>
 #include <memory>
+#include <mutex>
+#include <shared_mutex>
 #include <string>
+#include <thread>
+#include <unordered_map>
+#include <vector>
 
-// streaming buffer for one generation, survives HTTP disconnect. the producer appends SSE bytes,
-// readers drain from any offset via read_from. keyed by conversation_id, one conv = one live session
+enum class stream_read_status {
+    OK,
+    OFFSET_LOST,
+};
 
-struct stream_session;
+// streaming buffer for one generation, survives HTTP disconnect. the producer appends raw SSE
+// bytes, readers drain from any offset via read_from and block until more bytes or finalize.
+// keyed by conversation_id: one conv = at most one live session
+struct stream_session {
+    std::string conversation_id;
+    int64_t     started_ts; // unix seconds at construction, used by /v1/streams listing
+
+    stream_session(std::string conversation_id_, size_t max_bytes_);
+    stream_session(const stream_session &)             = delete;
+    stream_session & operator=(const stream_session &) = delete;
+
+    // append raw bytes, drops from the front if the cap is reached.
+    // returns false if the session is already finalized
+    bool append(const char * data, size_t len);
+
+    // mark the session as complete, wakes all pending readers
+    void finalize();
+
+    // drain bytes from offset, calling sink for each chunk. blocks until more
+    // bytes arrive or finalize is called. returns OK on clean exit, OFFSET_LOST
+    // if offset falls below the dropped prefix
+    stream_read_status read_from(size_t offset,
+        const std::function<bool(const char *, size_t)> & sink,
+        const std::function<bool()> & should_stop);
+
+    bool    is_done() const;
+    bool    is_cancelled() const;
+    size_t  total_size() const;     // bytes that ever entered the session
+    size_t  dropped_prefix() const; // bytes evicted from the front due to cap
+    int64_t completed_at() const;   // 0 while alive, unix seconds after finalize
+
+    // attach the producer stop hook used to cancel its reader, pass an empty function to detach
+    void set_stop_producer(std::function<void()> fn);
+
+    // signal the producer to abort its inference asap via the stop hook, idempotent
+    void cancel();
+
+private:
+    mutable std::mutex      mu;
+    std::condition_variable cv;
+    std::vector<char>       buffer;
+    size_t                  prefix_dropped;
+    size_t                  cap_bytes;
+    std::atomic<bool>       done;
+    std::atomic<bool>       cancelled;
+    std::atomic<int64_t>    completed_ts;
+    std::function<void()>   stop_producer; // protected by mu
+};
 
 using stream_session_ptr = std::shared_ptr<stream_session>;
 
-// base of the producer/consumer pipe ends. virtual dtor so each runs its own teardown:
+// one end of a stream_session pipe. the base holds the session and the shared query, the
+// producer and consumer ends derive from it. virtual dtor so each end runs its own teardown:
 // the producer finalizes the session, the consumer leaves it untouched
 struct stream_pipe {
     virtual ~stream_pipe() = default;
 
+    // true if the session was cancelled (e.g. via DELETE /v1/stream/<conv_id>)
     bool is_cancelled() const;
 
 protected:
@@ -37,6 +95,7 @@ protected:
 struct stream_pipe_producer : stream_pipe {
     ~stream_pipe_producer() override;
 
+    // append raw bytes to the session's ring buffer, returns false if already finalized
     bool write(const char * data, size_t len);
 
     // mark the natural end on the wire so a later close() is a no-op
@@ -62,21 +121,83 @@ private:
     server_http_res *                   res_ = nullptr;
 };
 
-void server_stream_session_manager_start();
-void server_stream_session_manager_stop();
+// consumer end: read-only replay of the ring buffer, the destructor does not finalize the session
+struct stream_pipe_consumer : stream_pipe {
+    // drain bytes from offset, calling sink for each available chunk. blocks until more data
+    // arrives or the session finalizes. should_stop is polled, returns OFFSET_LOST if offset
+    // fell below the dropped prefix
+    stream_read_status read(size_t & offset,
+        const std::function<bool(const char *, size_t)> & sink,
+        const std::function<bool()> & should_stop);
 
-// route handler factories wired under /v1/stream/* by server.cpp
-server_http_context::handler_t server_stream_make_get_handler();
-server_http_context::handler_t server_stream_make_lookup_handler();
-server_http_context::handler_t server_stream_make_delete_handler();
+    static std::shared_ptr<stream_pipe_consumer> create(stream_session_ptr session);
 
-// extract the X-Conversation-Id header value (case-insensitive), empty when absent
-std::string server_stream_conv_id_from_headers(const std::map<std::string, std::string> & headers);
+private:
+    explicit stream_pipe_consumer(stream_session_ptr session);
+};
 
-// on an X-Conversation-Id header, create or replace the session and attach a producer pipe to res
-void server_stream_session_attach_pipe(server_http_res & res, const std::map<std::string, std::string> & headers);
+// owns all live sessions, runs a periodic GC to evict expired ones.
+// the map is keyed by conversation_id, so the invariant "one conv = at most one
+// live session" is enforced at the type level
+class stream_session_manager {
+public:
+    stream_session_manager();
+    ~stream_session_manager();
+
+    stream_session_manager(const stream_session_manager &)             = delete;
+    stream_session_manager & operator=(const stream_session_manager &) = delete;
+
+    // install a new session for this conversation, evicting and cancelling any previous one.
+    // the conversation_id must be non empty, the caller is responsible for that check.
+    // returns the new session
+    stream_session_ptr create_or_replace(const std::string & conversation_id);
+
+    // lookup, returns null if unknown or already evicted
+    stream_session_ptr get(const std::string & conversation_id);
+
+    // list every live or recently completed session, used by GET /v1/streams without filter
+    std::vector<stream_session_ptr> list_all() const;
+
+    // remove from the map and finalize, wakes any pending readers
+    void evict(const std::string & conversation_id);
+
+    // signal the producer to cancel asap then evict, used by the explicit user Stop path
+    void evict_and_cancel(const std::string & conversation_id);
+
+    void start_gc();
+    void stop_gc();
+
+private:
+    void gc_loop();
+
+    mutable std::shared_mutex                           map_mu;
+    std::unordered_map<std::string, stream_session_ptr> sessions; // key: conversation_id
+    std::thread                                         gc_thread;
+    std::atomic<bool>                                   running;
+    std::mutex                                          gc_wake_mu;
+    std::condition_variable                             gc_wake_cv;
+};
+
+// process wide manager, linked by both llama-server and llama-cli. llama-server main() drives
+// start_gc/stop_gc, llama-cli leaves it idle. the dtor calls stop_gc() unconditionally so exit
+// is safe whether or not the GC thread ran
+extern stream_session_manager g_stream_sessions;
+
+// route handler factories operating on g_stream_sessions, wired under /v1/stream/* by server.cpp.
+// keeps the resumable stream surface confined to server-stream
+server_http_context::handler_t make_stream_get_handler();
+server_http_context::handler_t make_streams_lookup_handler();
+server_http_context::handler_t make_stream_delete_handler();
+
+// extract the X-Conversation-Id header value (case-insensitive), empty when absent. exposed so
+// the router can track which child serves a forwarded POST
+std::string stream_conv_id_from_headers(const std::map<std::string, std::string> & headers);
+
+// on an X-Conversation-Id header, create or replace the session and attach a producer pipe to
+// res. no-op when absent, called from the server_res_generator constructor
+void stream_session_attach_pipe(server_http_res & res, const std::map<std::string, std::string> & headers);
 
 // should_stop closure that ignores peer disconnect when a pipe is attached, so only an explicit
 // DELETE stops the producer and generation keeps flowing into the ring buffer. without a pipe it
 // delegates to fallback, the legacy non-resumable flow
-std::function<bool()> server_stream_aware_should_stop(server_http_res * res, std::function<bool()> fallback);
+std::function<bool()> stream_aware_should_stop(server_http_res * res, std::function<bool()> fallback);

tools/server/server.cpp

@@ -36,6 +36,19 @@ static inline void signal_handler(int signal) {
     shutdown_handler(signal);
 }
 
+// satisfies -Wmissing-declarations (used by llama command)
+int llama_server(int argc, char ** argv);
+
+// to be used via CLI (argc / argv are used by router mode only)
+int llama_server(common_params & params, int argc, char ** argv);
+void llama_server_terminate();
+void llama_server_terminate() {
+    if (shutdown_handler) {
+        shutdown_handler(0);
+    }
+}
+
+
 // wrapper function that handles exceptions and logs errors
 // this is to make sure handler_t never throws exceptions; instead, it returns an error response
 static server_http_context::handler_t ex_wrapper(server_http_context::handler_t func) {
@@ -72,9 +85,6 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t
     };
 }
 
-// satisfies -Wmissing-declarations
-int llama_server(int argc, char ** argv);
-
 int llama_server(int argc, char ** argv) {
     std::setlocale(LC_NUMERIC, "C");
 
@@ -85,7 +95,7 @@ int llama_server(int argc, char ** argv) {
 
     // start the stream session manager GC right after common init, before any HTTP route can
     // touch it. lifecycle is symmetric, stop_gc() runs in clean_up() before backend free
-    server_stream_session_manager_start();
+    g_stream_sessions.start_gc();
 
     if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {
         return 1;
@@ -94,16 +104,26 @@ int llama_server(int argc, char ** argv) {
     llama_backend_init();
     llama_numa_init(params.numa);
 
+    return llama_server(params, argc, argv);
+}
+
+int llama_server(common_params & params, int argc, char ** argv) {
+    bool is_run_by_cli = (argv == nullptr);
+
     common_models_handler models_handler;
-    try {
-        models_handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
-        if (common_models_handler_is_preset_repo(models_handler)) {
-            // apply the preset and start the server in router mode
-            common_models_handler_apply(models_handler, params);
+
+    // note: router mode also accepts -hf remote-preset, so we need to check that first
+    if (!is_run_by_cli && !params.model.hf_repo.empty()) {
+        try {
+            models_handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
+            if (common_models_handler_is_preset_repo(models_handler)) {
+                // apply the preset and start the server in router mode
+                common_models_handler_apply(models_handler, params);
+            }
+        } catch (const std::exception & e) {
+            SRV_ERR("failed to fetch model metadata: %s\n", e.what());
+            return 1;
         }
-    } catch (const std::exception & e) {
-        SRV_ERR("failed to fetch model metadata: %s\n", e.what());
-        return 1;
     }
 
     // router server never loads a model and must not touch the GPU
@@ -245,8 +265,8 @@ int llama_server(int argc, char ** argv) {
     ctx_http.post("/slots/:id_slot",           ex_wrapper(routes.post_slots));
 
     // resumable streaming, the conversation_id is the session identity end to end. router and
-    // child wire different handlers under the same paths: a child binds the local session
-    // factories, the router binds proxies that resolve the owning child through the
+    // child wire different handlers under the same paths: a child binds the local g_stream_sessions
+    // backed factories, the router binds proxies that resolve the owning child through the
     // conv_id -> model map
     server_http_context::handler_t stream_get_h;
     server_http_context::handler_t streams_lookup_h;
@@ -256,9 +276,9 @@ int llama_server(int argc, char ** argv) {
         streams_lookup_h = models_routes->router_streams_lookup;
         stream_delete_h  = models_routes->router_stream_delete;
     } else {
-        stream_get_h     = server_stream_make_get_handler();
-        streams_lookup_h = server_stream_make_lookup_handler();
-        stream_delete_h  = server_stream_make_delete_handler();
+        stream_get_h     = make_stream_get_handler();
+        streams_lookup_h = make_streams_lookup_handler();
+        stream_delete_h  = make_stream_delete_handler();
     }
     ctx_http.get ("/v1/stream/:conv_id",       ex_wrapper(stream_get_h));
     // POST /v1/streams/lookup with body {"conversation_ids": [...]}. you can only ask for ids
@@ -321,8 +341,9 @@ int llama_server(int argc, char ** argv) {
 
     if (child.is_child() && child.get_mode() == SERVER_CHILD_MODE_DOWNLOAD) {
         return child.run_download(params);
-    } else if (!is_router_server) {
+    } else if (!is_router_server && !is_run_by_cli) {
         // single-model mode (NOT spawned by router)
+        // if this is invoked by CLI, model downloading should be already handled
         try {
             common_models_handler_apply(models_handler, params);
         } catch (const std::exception & e) {
@@ -343,7 +364,7 @@ int llama_server(int argc, char ** argv) {
         clean_up = [&models_routes]() {
             SRV_INF("%s: cleaning up before exit...\n", __func__);
             // stop the session GC first, it finalizes live sessions and wakes pending readers
-            server_stream_session_manager_stop();
+            g_stream_sessions.stop_gc();
             if (models_routes.has_value()) {
                 models_routes->stopping.store(true); // maybe redundant, but just to be safe
                 models_routes->models.unload_all();
@@ -371,7 +392,7 @@ int llama_server(int argc, char ** argv) {
         clean_up = [&ctx_http, &ctx_server]() {
             SRV_INF("%s: cleaning up before exit...\n", __func__);
             // stop the session GC first, it finalizes live sessions and wakes pending readers
-            server_stream_session_manager_stop();
+            g_stream_sessions.stop_gc();
             ctx_http.stop();
             ctx_server.terminate();
             llama_backend_free();
@@ -411,20 +432,22 @@ int llama_server(int argc, char ** argv) {
         };
     }
 
-    // TODO: refactor in common/console
+    // register signal handler if not running by CLI
+    if (!is_run_by_cli) {
 #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
-    struct sigaction sigint_action;
-    sigint_action.sa_handler = signal_handler;
-    sigemptyset (&sigint_action.sa_mask);
-    sigint_action.sa_flags = 0;
-    sigaction(SIGINT, &sigint_action, NULL);
-    sigaction(SIGTERM, &sigint_action, NULL);
+        struct sigaction sigint_action;
+        sigint_action.sa_handler = signal_handler;
+        sigemptyset (&sigint_action.sa_mask);
+        sigint_action.sa_flags = 0;
+        sigaction(SIGINT, &sigint_action, NULL);
+        sigaction(SIGTERM, &sigint_action, NULL);
 #elif defined (_WIN32)
-    auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
-        return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;
-    };
-    SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
+        auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
+            return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;
+        };
+        SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
 #endif
+    }
 
     SRV_INF("listening on %s\n", ctx_http.listening_address.c_str());
 

tools/ui/package-lock.json

@@ -11,7 +11,7 @@
 				"@chromatic-com/storybook": "5.0.0",
 				"@eslint/compat": "1.4.1",
 				"@eslint/js": "9.39.2",
-				"@internationalized/date": "3.12.2",
+				"@internationalized/date": "3.10.1",
 				"@lucide/svelte": "0.515.0",
 				"@modelcontextprotocol/sdk": "1.26.0",
 				"@playwright/test": "1.56.1",
@@ -2981,9 +2981,9 @@
 			}
 		},
 		"node_modules/@internationalized/date": {
-			"version": "3.12.2",
-			"resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.12.2.tgz",
-			"integrity": "sha512-FY1Y+H64NDs+HAF6omlnWxm3mEpfgaCSWtL5l551ZZfImA+kGjPFgrnJrGjH6lfmLL0g8Z/mBu1R3kufeCp6Jw==",
+			"version": "3.10.1",
+			"resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.10.1.tgz",
+			"integrity": "sha512-oJrXtQiAXLvT9clCf1K4kxp3eKsQhIaZqxEyowkBcsvZDdZkbWrVmnGknxs5flTD0VGsxrxKgBCZty1EzoiMzA==",
 			"dev": true,
 			"license": "Apache-2.0",
 			"dependencies": {

tools/ui/package.json

@@ -30,7 +30,7 @@
 		"@chromatic-com/storybook": "5.0.0",
 		"@eslint/compat": "1.4.1",
 		"@eslint/js": "9.39.2",
-		"@internationalized/date": "3.12.2",
+		"@internationalized/date": "3.10.1",
 		"@lucide/svelte": "0.515.0",
 		"@modelcontextprotocol/sdk": "1.26.0",
 		"@playwright/test": "1.56.1",

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte

@@ -11,8 +11,7 @@
 	} from '$lib/constants';
 	import {
 		ChatFormActionAddToolsSubmenu,
-		ChatFormActionAddMcpServersSubmenu,
-		ChatFormActionAddReasoningSubmenu
+		ChatFormActionAddMcpServersSubmenu
 	} from '$lib/components/app';
 	import { useAttachmentMenu } from '$lib/hooks/use-attachment-menu.svelte';
 
@@ -93,11 +92,7 @@
 			</Tooltip.Content>
 		</Tooltip.Root>
 
-		<DropdownMenu.Content align="start" class="w-52">
-			<ChatFormActionAddReasoningSubmenu />
-
-			<DropdownMenu.Separator />
-
+		<DropdownMenu.Content align="start" class="w-48">
 			<DropdownMenu.Sub>
 				<DropdownMenu.SubTrigger class="flex cursor-pointer items-center gap-2">
 					<File class="h-4 w-4" />

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte

@@ -7,20 +7,14 @@
 		ChatFormActionModels,
 		ChatFormActionRecord,
 		ChatFormActionSubmit,
-		ChatFormContextGauge
+		ChatFormReasoningToggle
 	} from '$lib/components/app';
-	import { FileTypeCategory, MessageRole } from '$lib/enums';
+	import { FileTypeCategory } from '$lib/enums';
 	import { mcpStore } from '$lib/stores/mcp.svelte';
 	import { config } from '$lib/stores/settings.svelte';
-	import { activeMessages, conversationsStore } from '$lib/stores/conversations.svelte';
-	import {
-		activeProcessingState,
-		isChatStreaming,
-		isLoading as chatIsLoading
-	} from '$lib/stores/chat.svelte';
+	import { conversationsStore } from '$lib/stores/conversations.svelte';
 	import { getFileTypeCategory } from '$lib/utils';
 	import { goto } from '$app/navigation';
-	import { page } from '$app/state';
 	import { ROUTES } from '$lib/constants/routes';
 
 	interface Props {
@@ -99,36 +93,6 @@
 	let activeMessage = $derived(
 		conversationsStore.activeMessages[conversationsStore.activeMessages.length - 1]
 	);
-
-	let hasProcessedTokens = $derived.by(() => {
-		if (!page.params.id) return false;
-
-		const messages = activeMessages() as DatabaseMessage[];
-		let totalHistoricalTokens = 0;
-		for (const m of messages) {
-			if (m.role !== MessageRole.ASSISTANT) continue;
-			const timings = m.timings;
-			if (!timings) continue;
-			const agenticLlm = timings.agentic?.llm;
-			if (agenticLlm?.prompt_n != null || agenticLlm?.predicted_n != null) {
-				totalHistoricalTokens += (agenticLlm?.prompt_n ?? 0) + (agenticLlm?.predicted_n ?? 0);
-			} else {
-				totalHistoricalTokens += (timings.prompt_n ?? 0) + (timings.predicted_n ?? 0);
-			}
-		}
-		if (totalHistoricalTokens > 0) return true;
-
-		if (!chatIsLoading() && !isChatStreaming()) return false;
-
-		const processingState = activeProcessingState();
-		if (!processingState) return false;
-		const livePromptTokens = Math.max(
-			processingState.promptTokens ?? 0,
-			processingState.promptProgress?.processed ?? 0
-		);
-		const liveOutputTokens = processingState.outputTokensUsed ?? 0;
-		return livePromptTokens > 0 || liveOutputTokens > 0;
-	});
 </script>
 
 <div
@@ -136,7 +100,7 @@
 	style="container-type: inline-size"
 >
 	{#if showAddButton}
-		<div class="mr-auto flex items-center gap-2">
+		<div class="mr-auto flex items-center gap-3">
 			<ChatFormActionsAdd
 				{disabled}
 				{hasAudioModality}
@@ -153,10 +117,8 @@
 		</div>
 	{/if}
 
-	<div class="flex items-center gap-1.5">
-		{#if hasProcessedTokens}
-			<ChatFormContextGauge />
-		{/if}
+	<div class="flex items-center gap-2">
+		<ChatFormReasoningToggle />
 
 		{#if showModelSelector}
 			<ChatFormActionModels

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormReasoningEffortSubmenu.svelte

@@ -6,7 +6,6 @@
 	import { REASONING_EFFORT_TOKENS } from '$lib/constants/reasoning-effort-tokens';
 	import { REASONING_EFFORT_LEVELS } from '$lib/constants/reasoning-effort';
 	import type { ReasoningEffortLevel } from '$lib/types';
-	import { DIALOG_SUBMENU_CONTENT } from '$lib/constants/css-classes';
 	import {
 		modelsStore,
 		checkModelSupportsThinking,
@@ -72,7 +71,9 @@
 
 {#if modelSupportsThinking}
 	<DropdownMenu.Sub bind:open={subOpen}>
-		<DropdownMenu.SubTrigger class="flex cursor-pointer items-center gap-2">
+		<DropdownMenu.SubTrigger
+			class="flex cursor-pointer items-center gap-2 rounded-md px-2.5 py-1.5 text-sm transition-colors outline-none hover:bg-accent focus:bg-accent"
+		>
 			{#if thinkingEnabled}
 				<Lightbulb class="h-4 w-4 shrink-0 text-amber-400" />
 			{:else}
@@ -88,15 +89,23 @@
 			{/if}
 		</DropdownMenu.SubTrigger>
 
-		<DropdownMenu.SubContent class={DIALOG_SUBMENU_CONTENT}>
+		<DropdownMenu.SubContent
+			class="w-60 rounded-xl bg-popover p-3 text-popover-foreground shadow-md outline-none data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[state=open]:animate-in data-[state=open]:fade-in-0 data-[state=open]:zoom-in-95"
+		>
 			{#each REASONING_EFFORT_LEVELS as level (level.value)}
 				<button
 					type="button"
-					class="flex w-full cursor-pointer items-center gap-2"
+					class="flex w-full cursor-pointer items-center gap-2 rounded-lg px-2.5 py-2 text-left text-sm transition-colors hover:bg-accent"
 					class:bg-accent={isSelected(level)}
 					onclick={() => handleSelection(level)}
 				>
-					<span class="flex-1 text-left">{level.label}</span>
+					{#if isSelected(level)}
+						<Check class="h-4 w-4 shrink-0 text-foreground" />
+					{:else}
+						<div class="h-4 w-4 shrink-0"></div>
+					{/if}
+
+					<span class="flex-1">{level.label}</span>
 
 					{#if !level.isOff}
 						<span class="text-[11px] text-muted-foreground opacity-60">
@@ -116,10 +125,6 @@
 							</Tooltip.Content>
 						</Tooltip.Root>
 					{/if}
-
-					{#if isSelected(level)}
-						<Check class="h-4 w-4 shrink-0 text-foreground" />
-					{/if}
 				</button>
 			{/each}
 		</DropdownMenu.SubContent>

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormReasoningToggle.svelte

@@ -2,7 +2,7 @@
 	import { Lightbulb, LightbulbOff, Check, Info } from '@lucide/svelte';
 	import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
 	import * as Tooltip from '$lib/components/ui/tooltip';
-	import { ReasoningEffort } from '$lib/enums';
+	import { ReasoningEffort, MessageRole } from '$lib/enums';
 	import { REASONING_EFFORT_TOKENS } from '$lib/constants/reasoning-effort-tokens';
 	import { REASONING_EFFORT_LEVELS } from '$lib/constants/reasoning-effort';
 	import type { ReasoningEffortLevel } from '$lib/types';
@@ -18,23 +18,31 @@
 	import { isRouterMode } from '$lib/stores/server.svelte';
 	import type { DatabaseMessage } from '$lib/types/database';
 
+	let thinkingEnabled = $derived(conversationsStore.getThinkingEnabled());
+	let currentEffort = $derived(conversationsStore.getReasoningEffort());
+	let isOff = $derived(!thinkingEnabled);
+	let tooltipText = $derived(thinkingEnabled ? `${currentEffort} Reasoning` : 'Disabled Reasoning');
 	let subOpen = $state(false);
 
+	// Get conversation model from message history
 	let conversationModel = $derived(
 		chatStore.getConversationModel(activeMessages() as DatabaseMessage[])
 	);
 
+	// Fallback: if model props aren't available, check if any assistant messages
+	// for this model in the active conversation have reasoning content.
 	let modelSupportsThinkingFromMessages = $derived.by(() => {
 		const modelId = isRouterMode() ? modelsStore.selectedModelName || conversationModel : null;
 		if (!modelId) return false;
-
 		const messages = conversationsStore.activeMessages;
-
 		return messages.some(
-			(m) => m.role === 'assistant' && m.model === modelId && !!m.reasoningContent
+			(m: DatabaseMessage) =>
+				m.role === MessageRole.ASSISTANT && m.model === modelId && !!m.reasoningContent
 		);
 	});
 
+	// Check if model supports thinking. Primary: chat template from /props.
+	// Fallback: message history (reasoning content in assistant messages).
 	let modelSupportsThinking = $derived.by(() => {
 		loadedModelIds();
 		propsCacheVersion();
@@ -44,15 +52,15 @@
 			return checkModelSupportsThinking(modelId ?? '') || modelSupportsThinkingFromMessages;
 		}
 
+		// In non-router mode, use the built-in supportsThinking
 		return supportsThinking() || modelSupportsThinkingFromMessages;
 	});
 
-	let thinkingEnabled = $derived(conversationsStore.getThinkingEnabled());
-	let currentEffort = $derived(conversationsStore.getReasoningEffort());
-	let isOff = $derived(!thinkingEnabled);
-
+	// Check if current item is selected
 	function isSelected(item: ReasoningEffortLevel): boolean {
-		if (item.isOff) return isOff;
+		if (item.isOff) {
+			return isOff;
+		}
 		return thinkingEnabled && currentEffort === item.value;
 	}
 
@@ -68,30 +76,39 @@
 </script>
 
 {#if modelSupportsThinking}
-	<DropdownMenu.Sub bind:open={subOpen}>
-		<DropdownMenu.SubTrigger class="flex cursor-pointer items-center gap-2">
-			{#if thinkingEnabled}
-				<Lightbulb class="h-4 w-4 shrink-0 text-amber-400" />
-			{:else}
-				<LightbulbOff class="h-4 w-4 shrink-0 text-muted-foreground" />
-			{/if}
+	<DropdownMenu.Root bind:open={subOpen}>
+		<Tooltip.Root>
+			<Tooltip.Trigger>
+				<DropdownMenu.Trigger
+					class={[
+						'flex h-6 w-6 cursor-pointer items-center justify-center rounded-full p-0 transition-colors focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2',
+						thinkingEnabled ? 'bg-amber-400/10 hover:bg-amber-400/20' : 'bg-muted'
+					]}
+					aria-label={`${tooltipText}. Click to configure.`}
+				>
+					{#if thinkingEnabled}
+						<Lightbulb class="h-3 w-3 text-amber-400" />
+					{:else}
+						<LightbulbOff class="h-3 w-3 text-muted-foreground" />
+					{/if}
+				</DropdownMenu.Trigger>
+			</Tooltip.Trigger>
 
-			<span class="text-sm inline-flex gap-2 {!thinkingEnabled ? 'text-muted-foreground' : ''}">
-				Reasoning
+			<Tooltip.Content>
+				<p class="capitalize">{tooltipText}</p>
+			</Tooltip.Content>
+		</Tooltip.Root>
 
-				<span class="capitalize text-muted-foreground">
-					{thinkingEnabled ? currentEffort : 'off'}
-				</span>
-			</span>
-		</DropdownMenu.SubTrigger>
-
-		<DropdownMenu.SubContent
-			class="w-60 bg-popover p-1.5 text-popover-foreground shadow-md outline-none"
+		<DropdownMenu.Content
+			align="start"
+			class="w-60 rounded-xl bg-popover p-3 text-popover-foreground shadow-md outline-none"
 		>
+			<div class="mb-2 px-2.5 text-sm font-medium">Reasoning effort</div>
+
 			{#each REASONING_EFFORT_LEVELS as level (level.value)}
 				<button
 					type="button"
-					class="flex w-full cursor-pointer items-center gap-3 rounded-md px-2 py-1.75 text-left text-sm transition-colors hover:bg-accent"
+					class="flex w-full cursor-pointer items-center gap-2 rounded-lg px-2.5 py-2 text-left text-sm transition-colors hover:bg-accent"
 					class:bg-accent={isSelected(level)}
 					onclick={() => handleSelection(level)}
 				>
@@ -123,6 +140,6 @@
 					{/if}
 				</button>
 			{/each}
-		</DropdownMenu.SubContent>
-	</DropdownMenu.Sub>
+		</DropdownMenu.Content>
+	</DropdownMenu.Root>
 {/if}

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte

@@ -1,108 +0,0 @@
-<script lang="ts">
-	import { untrack } from 'svelte';
-	import * as HoverCard from '$lib/components/ui/hover-card';
-	import { activeConversation, activeMessages } from '$lib/stores/conversations.svelte';
-	import { chatStore, isChatStreaming, isLoading } from '$lib/stores/chat.svelte';
-	import { formatParameters } from '$lib/utils/formatters';
-	import { useContextGauge } from '$lib/hooks/use-context-gauge.svelte';
-	import ContextGaugeDial from './ContextGaugeDial.svelte';
-	import ContextGaugeDetails from './ContextGaugeDetails.svelte';
-	import ContextGaugeLoadModel from './ContextGaugeLoadModel.svelte';
-	import { colorLevelBgClass, colorLevelTextClass } from './context-gauge';
-
-	const gauge = useContextGauge();
-
-	$effect(() => {
-		const conv = activeConversation();
-		untrack(() => chatStore.setActiveProcessingConversation(conv?.id ?? null));
-	});
-
-	$effect(() => {
-		const conv = activeConversation();
-		const messages = activeMessages() as DatabaseMessage[];
-		if (!conv) return;
-		if (isLoading() || isChatStreaming()) return;
-
-		if (messages.length === 0) {
-			untrack(() => chatStore.clearProcessingState(conv.id));
-			return;
-		}
-
-		untrack(() => chatStore.restoreProcessingStateFromMessages(messages, conv.id));
-	});
-
-	$effect(() => {
-		gauge.startMonitoring();
-	});
-
-	const showProgressBar = $derived(
-		gauge.contextTotal !== null &&
-			gauge.contextTotal > 0 &&
-			(gauge.activeModelId !== null || gauge.isActiveModelLoaded)
-	);
-</script>
-
-<HoverCard.Root>
-	<HoverCard.Trigger class="flex h-5 w-5 cursor-default items-center justify-center">
-		<ContextGaugeDial percent={gauge.contextPercent} level={gauge.colorLevel} />
-	</HoverCard.Trigger>
-
-	<HoverCard.Content
-		side="bottom"
-		class="z-50 w-64 rounded-lg border border-border/50 bg-popover p-3 text-popover-foreground shadow-lg"
-	>
-		<div class="flex flex-col gap-2">
-			<div class="flex items-center gap-2">
-				<span class="font-medium">Context</span>
-				<span class="text-muted-foreground">·</span>
-				<span class="font-mono text-muted-foreground">
-					{formatParameters(gauge.contextUsed)}
-					/ {gauge.contextTotal !== null ? formatParameters(gauge.contextTotal) : '-'}
-				</span>
-			</div>
-
-			{#if gauge.activeModelId !== null && !gauge.isActiveModelLoaded}
-				<ContextGaugeLoadModel
-					modelId={gauge.activeModelId}
-					isLoading={gauge.isActiveModelLoading}
-					onLoad={gauge.loadModel}
-				/>
-			{:else if showProgressBar}
-				<div class="h-1.5 w-full overflow-hidden rounded-full bg-muted">
-					<div
-						class="h-full rounded-full transition-all duration-300 {colorLevelBgClass(
-							gauge.colorLevel
-						)}"
-						style="width: {gauge.contextPercent}%"
-					></div>
-				</div>
-
-				<div class="flex justify-between text-xs text-muted-foreground">
-					<span>
-						<span class={colorLevelTextClass(gauge.colorLevel)}>{gauge.contextPercent}%</span> used
-					</span>
-					<span>
-						{formatParameters((gauge.contextTotal ?? 0) - gauge.contextUsed)} remaining
-					</span>
-				</div>
-			{:else}
-				<div class="text-xs text-muted-foreground">No context info available</div>
-			{/if}
-
-			{#if gauge.hasAnyUsage}
-				<ContextGaugeDetails
-					currentRead={gauge.currentRead}
-					currentFresh={gauge.currentFresh}
-					currentCache={gauge.currentCache}
-					currentOutput={gauge.currentOutput}
-					kvTotal={gauge.kvTotal}
-					cumulativeRead={gauge.cumulativeRead}
-					cumulativeOutput={gauge.cumulativeOutput}
-					cumulativeCacheTotal={gauge.cumulativeCacheTotal}
-					averageTokensPerSecond={gauge.averageTokensPerSecond}
-					transientDetails={gauge.transientDetails}
-				/>
-			{/if}
-		</div>
-	</HoverCard.Content>
-</HoverCard.Root>

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDetailRow.svelte

@@ -1,20 +0,0 @@
-<script lang="ts">
-	interface Props {
-		label: string;
-		value: string;
-		subtitle?: string;
-	}
-
-	let { label, value, subtitle }: Props = $props();
-</script>
-
-<div class="grid gap-1.5">
-	<div class="flex items-baseline justify-between">
-		<span class="text-muted-foreground">{label}</span>
-		<span class="font-mono text-muted-foreground">{value}</span>
-	</div>
-
-	{#if subtitle}
-		<div class="text-[10px] leading-tight text-muted-foreground/70">{subtitle}</div>
-	{/if}
-</div>

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDetails.svelte

@@ -1,122 +0,0 @@
-<script lang="ts">
-	import { ChevronDown } from '@lucide/svelte';
-	import * as Collapsible from '$lib/components/ui/collapsible';
-	import { STATS_UNITS } from '$lib/constants';
-	import ContextGaugeDetailRow from './ContextGaugeDetailRow.svelte';
-
-	interface Props {
-		currentRead: number;
-		currentFresh: number;
-		currentCache: number;
-		currentOutput: number;
-		kvTotal: number;
-		cumulativeRead: number;
-		cumulativeOutput: number;
-		cumulativeCacheTotal: number;
-		averageTokensPerSecond: number | null;
-		transientDetails: string[];
-	}
-
-	let {
-		currentRead,
-		currentFresh,
-		currentCache,
-		currentOutput,
-		kvTotal,
-		cumulativeRead,
-		cumulativeOutput,
-		cumulativeCacheTotal,
-		averageTokensPerSecond,
-		transientDetails
-	}: Props = $props();
-
-	let open = $state(false);
-
-	const hasCumulative = $derived(cumulativeRead > 0 || cumulativeOutput > 0);
-	const hasCurrent = $derived(currentRead > 0 || currentOutput > 0);
-</script>
-
-<Collapsible.Root bind:open class="mt-3 border-t border-border/50 pt-4">
-	<Collapsible.Trigger
-		class="flex w-full cursor-pointer items-center gap-1 text-xs text-muted-foreground hover:text-foreground"
-	>
-		<span>Token usage details</span>
-
-		<ChevronDown class={'ml-auto h-3 w-3 transition-transform' + (open ? ' rotate-180' : '')} />
-	</Collapsible.Trigger>
-
-	<Collapsible.Content class="flex flex-col gap-4 text-xs pt-4">
-		{#if hasCumulative}
-			<div>
-				<h3 class="text-[11px] font-medium uppercase tracking-wide text-muted-foreground/70 mb-2">
-					Across all turns
-				</h3>
-
-				<div class="flex flex-col gap-2">
-					{#if cumulativeRead > 0}
-						<ContextGaugeDetailRow
-							label="Prompt tokens evaluated"
-							value={`${cumulativeRead.toLocaleString()} tok`}
-							subtitle={cumulativeCacheTotal > 0
-								? `${cumulativeCacheTotal.toLocaleString()} reused from KV cache`
-								: undefined}
-						/>
-					{/if}
-					{#if cumulativeOutput > 0}
-						<ContextGaugeDetailRow
-							label="Tokens generated"
-							value={`${cumulativeOutput.toLocaleString()} tok`}
-						/>
-					{/if}
-				</div>
-			</div>
-		{/if}
-
-		{#if hasCurrent}
-			<div>
-				<h3 class="text-[11px] font-medium uppercase tracking-wide text-muted-foreground/70 mb-2">
-					This turn · KV cache
-				</h3>
-
-				<div class="flex flex-col gap-2">
-					{#if currentRead > 0}
-						<ContextGaugeDetailRow
-							label="Prompt"
-							value={`${currentRead.toLocaleString()} tok`}
-							subtitle={currentCache > 0
-								? `${currentFresh.toLocaleString()} fresh + ${currentCache.toLocaleString()} cached`
-								: undefined}
-						/>
-					{/if}
-
-					{#if currentOutput > 0}
-						<ContextGaugeDetailRow
-							label="Generated"
-							value={`${currentOutput.toLocaleString()} tok`}
-						/>
-					{/if}
-
-					<div class="pt-1 mt-0.5 border-t border-border/30">
-						<div class="flex justify-between">
-							<span class="text-muted-foreground">KV cache total</span>
-							<span class="font-mono font-medium">{kvTotal.toLocaleString()} tok</span>
-						</div>
-					</div>
-				</div>
-			</div>
-		{/if}
-
-		{#if averageTokensPerSecond !== null}
-			<div class="pt-1.5 mt-1 border-t border-border/30">
-				<ContextGaugeDetailRow
-					label="Avg speed"
-					value={`${averageTokensPerSecond.toFixed(1)}${STATS_UNITS.TOKENS_PER_SECOND}`}
-				/>
-			</div>
-		{/if}
-
-		{#each transientDetails as detail (detail)}
-			<div class="font-mono text-muted-foreground">{detail}</div>
-		{/each}
-	</Collapsible.Content>
-</Collapsible.Root>

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte

@@ -1,43 +0,0 @@
-<script lang="ts">
-	import type { ColorLevel } from './context-gauge';
-	import { colorLevelTextClass } from './context-gauge';
-
-	interface Props {
-		percent: number | null;
-		level: ColorLevel;
-		size?: 'sm' | 'md';
-	}
-
-	let { percent, level, size = 'sm' }: Props = $props();
-
-	const RADIUS = 11;
-	const CIRCUMFERENCE = 2 * Math.PI * RADIUS;
-
-	const strokeLevelClass = $derived(colorLevelTextClass(level));
-	const dimensions = $derived(size === 'md' ? 'h-6 w-6' : 'h-5 w-5');
-	const strokeWidth = $derived(size === 'md' ? 4 : 3);
-</script>
-
-<svg viewBox="0 0 32 32" fill="none" class={dimensions}>
-	<circle
-		cx="16"
-		cy="16"
-		r={RADIUS}
-		stroke="currentColor"
-		stroke-opacity="0.1"
-		stroke-width={strokeWidth}
-	/>
-
-	<circle
-		cx="16"
-		cy="16"
-		r={RADIUS}
-		class="transition-colors duration-300 {strokeLevelClass}"
-		stroke="currentColor"
-		stroke-width={strokeWidth}
-		stroke-linecap="round"
-		stroke-dasharray={CIRCUMFERENCE}
-		stroke-dashoffset={percent !== null ? CIRCUMFERENCE * (1 - percent / 100) : CIRCUMFERENCE}
-		transform="rotate(-90 16 16)"
-	/>
-</svg>

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeLoadModel.svelte

@@ -1,24 +0,0 @@
-<script lang="ts">
-	import { Loader2 } from '@lucide/svelte';
-	import { Button } from '$lib/components/ui/button';
-
-	interface Props {
-		modelId: string | null;
-		isLoading: boolean;
-		onLoad: () => void;
-	}
-
-	let { modelId, isLoading, onLoad }: Props = $props();
-</script>
-
-{#if modelId !== null && !isLoading}
-	<div class="flex flex-col gap-2 border-t border-border/50 pt-2 text-xs text-muted-foreground">
-		<span>Available context size is only visible once the model is loaded.</span>
-		<Button size="sm" variant="secondary" class="self-start" onclick={onLoad}>Load model</Button>
-	</div>
-{:else if isLoading}
-	<div class="flex items-center gap-2 border-t border-border/50 pt-2 text-xs text-muted-foreground">
-		<Loader2 class="h-3.5 w-3.5 animate-spin" />
-		<span>Loading model...</span>
-	</div>
-{/if}

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/context-gauge.ts

@@ -1,37 +0,0 @@
-export type ColorLevel = 'ok' | 'warning' | 'critical' | 'neutral';
-
-const WARNING_THRESHOLD = 80;
-const CRITICAL_THRESHOLD = 95;
-
-export function colorLevelFromPercent(percent: number | null): ColorLevel {
-	if (percent === null) return 'neutral';
-	if (percent >= CRITICAL_THRESHOLD) return 'critical';
-	if (percent >= WARNING_THRESHOLD) return 'warning';
-	return 'ok';
-}
-
-export function colorLevelTextClass(level: ColorLevel): string {
-	switch (level) {
-		case 'critical':
-			return 'text-red-400';
-		case 'warning':
-			return 'text-amber-400';
-		case 'ok':
-			return 'text-muted-foreground';
-		default:
-			return 'text-muted-foreground';
-	}
-}
-
-export function colorLevelBgClass(level: ColorLevel): string {
-	switch (level) {
-		case 'critical':
-			return 'bg-red-500';
-		case 'warning':
-			return 'bg-amber-500';
-		case 'ok':
-			return 'bg-green-500';
-		default:
-			return 'bg-muted';
-	}
-}

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte

@@ -24,8 +24,6 @@
 		message: DatabaseMessage;
 		toolMessages?: DatabaseMessage[];
 		isLastAssistantMessage?: boolean;
-		isLastUserMessage?: boolean;
-		nextAssistantMessage?: DatabaseMessage | null;
 		siblingInfo?: ChatMessageSiblingInfo | null;
 	}
 
@@ -34,8 +32,6 @@
 		message,
 		toolMessages = [],
 		isLastAssistantMessage = false,
-		isLastUserMessage = false,
-		nextAssistantMessage = null,
 		siblingInfo = null
 	}: Props = $props();
 
@@ -363,9 +359,7 @@
 		<ChatMessageUser
 			class={className}
 			{deletionInfo}
-			{isLastUserMessage}
 			{message}
-			{nextAssistantMessage}
 			onConfirmDelete={handleConfirmDelete}
 			onCopy={handleCopy}
 			onDelete={handleDelete}

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte

@@ -11,10 +11,11 @@
 	import { useProcessingState } from '$lib/hooks/use-processing-state.svelte';
 	import { isLoading, isChatStreaming } from '$lib/stores/chat.svelte';
 	import { copyToClipboard, deriveAgenticSections, modelLoadProgressText } from '$lib/utils';
-	import { AgenticSectionType, ChatMessageStatisticsMode } from '$lib/enums';
+	import { AgenticSectionType } from '$lib/enums';
 	import { REASONING_TAGS } from '$lib/constants/agentic';
+	import { tick } from 'svelte';
 	import { fade } from 'svelte/transition';
-	import { MessageRole } from '$lib/enums';
+	import { MessageRole, ChatMessageStatsView } from '$lib/enums';
 	import { config } from '$lib/stores/settings.svelte';
 	import { isRouterMode } from '$lib/stores/server.svelte';
 	import { modelsStore } from '$lib/stores/models.svelte';
@@ -121,6 +122,62 @@
 		return parts.join('\n\n\n');
 	});
 
+	let activeStatsView = $state<ChatMessageStatsView>(ChatMessageStatsView.GENERATION);
+	let statsContainerEl: HTMLDivElement | undefined = $state();
+
+	function getScrollParent(el: HTMLElement): HTMLElement | null {
+		let parent = el.parentElement;
+		while (parent) {
+			const style = getComputedStyle(parent);
+			if (/(auto|scroll)/.test(style.overflowY)) {
+				return parent;
+			}
+			parent = parent.parentElement;
+		}
+		return null;
+	}
+
+	async function handleStatsViewChange(view: ChatMessageStatsView) {
+		const el = statsContainerEl;
+		if (!el) {
+			activeStatsView = view;
+
+			return;
+		}
+
+		const scrollParent = getScrollParent(el);
+		if (!scrollParent) {
+			activeStatsView = view;
+
+			return;
+		}
+
+		const yBefore = el.getBoundingClientRect().top;
+
+		activeStatsView = view;
+
+		await tick();
+
+		const delta = el.getBoundingClientRect().top - yBefore;
+		if (delta !== 0) {
+			scrollParent.scrollTop += delta;
+		}
+
+		// Correct any drift after browser paint
+		requestAnimationFrame(() => {
+			const drift = el.getBoundingClientRect().top - yBefore;
+
+			if (Math.abs(drift) > 1) {
+				scrollParent.scrollTop += drift;
+			}
+		});
+	}
+
+	let highlightAgenticTurns = $derived(
+		isAgentic &&
+			(currentConfig.alwaysShowAgenticTurns || activeStatsView === ChatMessageStatsView.SUMMARY)
+	);
+
 	let displayedModel = $derived(message.model ?? null);
 
 	// model being switched to while it loads, so the selector bar tracks it
@@ -234,6 +291,7 @@
 				{toolMessages}
 				isStreaming={isChatStreaming()}
 				{isLastAssistantMessage}
+				highlightTurns={highlightAgenticTurns}
 			/>
 		{/if}
 	{:else}
@@ -257,7 +315,10 @@
 
 	<div class="info my-6 grid gap-4 tabular-nums">
 		{#if displayedModel}
-			<div class="inline-flex flex-wrap items-start gap-2 text-xs text-muted-foreground">
+			<div
+				bind:this={statsContainerEl}
+				class="inline-flex flex-wrap items-start gap-2 text-xs text-muted-foreground"
+			>
 				{#if isRouter}
 					<ModelsSelectorDropdown
 						currentModel={pendingModel ?? displayedModel}
@@ -286,25 +347,28 @@
 				{#if currentConfig.showMessageStats && message.timings && message.timings.predicted_n && message.timings.predicted_ms}
 					{@const agentic = message.timings.agentic}
 					<ChatMessageStatistics
-						mode={ChatMessageStatisticsMode.GENERATION}
 						promptTokens={agentic ? agentic.llm.prompt_n : message.timings.prompt_n}
 						promptMs={agentic ? agentic.llm.prompt_ms : message.timings.prompt_ms}
 						predictedTokens={agentic ? agentic.llm.predicted_n : message.timings.predicted_n}
 						predictedMs={agentic ? agentic.llm.predicted_ms : message.timings.predicted_ms}
 						agenticTimings={agentic}
+						onActiveViewChange={handleStatsViewChange}
 					/>
 				{:else if isLoading() && currentConfig.showMessageStats}
 					{@const liveStats = processingState.getLiveProcessingStats()}
 					{@const genStats = processingState.getLiveGenerationStats()}
+					{@const promptProgress = processingState.processingState?.promptProgress}
+					{@const isStillProcessingPrompt =
+						promptProgress && promptProgress.processed < promptProgress.total}
 
-					{#if genStats}
+					{#if liveStats || genStats}
 						<ChatMessageStatistics
-							mode={ChatMessageStatisticsMode.GENERATION}
 							isLive
+							isProcessingPrompt={!!isStillProcessingPrompt}
 							promptTokens={liveStats?.tokensProcessed}
 							promptMs={liveStats?.timeMs}
-							predictedTokens={genStats.tokensGenerated}
-							predictedMs={genStats.timeMs}
+							predictedTokens={genStats?.tokensGenerated}
+							predictedMs={genStats?.timeMs}
 						/>
 					{/if}
 				{/if}

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUser.svelte

@@ -2,14 +2,10 @@
 	import {
 		ChatMessageActionIcons,
 		ChatMessageEditForm,
-		ChatMessageStatistics,
 		ChatMessageUserBubble
 	} from '$lib/components/app/chat';
 	import { getMessageEditContext } from '$lib/contexts';
-	import { useProcessingState } from '$lib/hooks/use-processing-state.svelte';
-	import { isLoading } from '$lib/stores/chat.svelte';
-	import { MessageRole, ChatMessageStatisticsMode } from '$lib/enums';
-	import { config } from '$lib/stores/settings.svelte';
+	import { MessageRole } from '$lib/enums';
 
 	interface Props {
 		class?: string;
@@ -21,8 +17,6 @@
 			assistantMessages: number;
 			messageTypes: string[];
 		} | null;
-		isLastUserMessage?: boolean;
-		nextAssistantMessage?: DatabaseMessage | null;
 		showDeleteDialog: boolean;
 		onEdit: () => void;
 		onDelete: () => void;
@@ -38,8 +32,6 @@
 		message,
 		siblingInfo = null,
 		deletionInfo,
-		isLastUserMessage = false,
-		nextAssistantMessage = null,
 		showDeleteDialog,
 		onEdit,
 		onDelete,
@@ -52,37 +44,6 @@
 
 	// Get contexts
 	const editCtx = getMessageEditContext();
-	const processingState = useProcessingState();
-
-	const currentConfig = $derived(config());
-	const isActivelyProcessing = $derived(isLastUserMessage && isLoading());
-
-	// For agentic turns, prefer the cumulative agentic.llm totals over per-call timings.
-	let storedReadingStats = $derived.by(() => {
-		const timings = nextAssistantMessage?.timings;
-		if (!timings?.prompt_n || !timings?.prompt_ms) return null;
-
-		const agentic = timings.agentic;
-
-		return {
-			promptTokens: agentic ? agentic.llm.prompt_n : timings.prompt_n,
-			promptMs: agentic ? agentic.llm.prompt_ms : timings.prompt_ms
-		};
-	});
-
-	let showStoredReadingStats = $derived(
-		Boolean(currentConfig.showMessageStats) && storedReadingStats !== null
-	);
-
-	let showLiveReadingStats = $derived(
-		Boolean(currentConfig.showMessageStats) && isActivelyProcessing && storedReadingStats === null
-	);
-
-	$effect(() => {
-		if (showLiveReadingStats) {
-			processingState.startMonitoring();
-		}
-	});
 </script>
 
 <div
@@ -99,37 +60,6 @@
 			renderMarkdown={true}
 		/>
 
-		{#if showStoredReadingStats}
-			<!-- Reading stats sourced from the assistant message that followed this turn -->
-			<div class="info my-2 grid w-full justify-items-end gap-4 tabular-nums">
-				<div
-					class="inline-flex flex-wrap items-start justify-end gap-2 text-xs text-muted-foreground"
-				>
-					<ChatMessageStatistics
-						mode={ChatMessageStatisticsMode.READING}
-						promptTokens={storedReadingStats!.promptTokens}
-						promptMs={storedReadingStats!.promptMs}
-					/>
-				</div>
-			</div>
-		{:else if showLiveReadingStats}
-			{@const liveStats = processingState.getLiveProcessingStats()}
-			{#if liveStats}
-				<div class="info my-2 grid w-full justify-items-end gap-4 tabular-nums">
-					<div
-						class="inline-flex flex-wrap items-start justify-end gap-2 text-xs text-muted-foreground"
-					>
-						<ChatMessageStatistics
-							mode={ChatMessageStatisticsMode.READING}
-							isLive
-							promptTokens={liveStats.tokensProcessed}
-							promptMs={liveStats.timeMs}
-						/>
-					</div>
-				</div>
-			{/if}
-		{/if}
-
 		{#if message.timestamp}
 			<div class="max-w-[80%]">
 				<ChatMessageActionIcons

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserPending.svelte

@@ -2,6 +2,7 @@
 	import { ActionIcon, ChatMessageEditForm, ChatMessageUserBubble } from '$lib/components/app';
 	import { fadeInView } from '$lib/actions/fade-in-view.svelte';
 	import { ArrowUp, Edit, Trash2 } from '@lucide/svelte';
+	import { getProcessingInfoContext } from '$lib/contexts';
 	import { useMessageEditContext } from '$lib/hooks/use-message-edit-context.svelte';
 
 	interface Props {
@@ -22,6 +23,9 @@
 		onDelete
 	}: Props = $props();
 
+	const processingInfoCtx = getProcessingInfoContext();
+	let showProcessingInfo = $derived(processingInfoCtx.showProcessingInfo);
+
 	const editCtx = useMessageEditContext({
 		getContent: () => content,
 		getExtras: () => extras,
@@ -32,7 +36,9 @@
 <div
 	use:fadeInView
 	aria-label="Pending user message"
-	class="group flex flex-col items-end gap-3 transition-opacity hover:opacity-80 md:gap-2 {className} sticky bottom-32"
+	class="group flex flex-col items-end gap-3 transition-opacity hover:opacity-80 md:gap-2 {className} sticky {showProcessingInfo
+		? 'bottom-44'
+		: 'bottom-32'}"
 	role="group"
 >
 	{#if editCtx.isEditing}

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte

@@ -41,13 +41,15 @@
 		toolMessages?: DatabaseMessage[];
 		isStreaming?: boolean;
 		isLastAssistantMessage?: boolean;
+		highlightTurns?: boolean;
 	}
 
 	let {
 		message,
 		toolMessages = [],
 		isStreaming = false,
-		isLastAssistantMessage = false
+		isLastAssistantMessage = false,
+		highlightTurns = false
 	}: Props = $props();
 
 	let expandedStates: Record<number, boolean> = $state({});
@@ -55,7 +57,6 @@
 	const showToolCallInProgress = $derived(config().showToolCallInProgress as boolean);
 	const showThoughtInProgress = $derived(config().showThoughtInProgress as boolean);
 	const renderThinkingAsMarkdown = $derived(config().renderThinkingAsMarkdown as boolean);
-	const showMessageStats = $derived(config().showMessageStats as boolean);
 
 	const hasReasoningError = $derived(
 		isLastAssistantMessage ? !!agenticLastError(message.convId) : false
@@ -353,17 +354,16 @@
 {/snippet}
 
 <div class="agentic-content">
-	{#if turnGroups.length > 1}
+	{#if highlightTurns && turnGroups.length > 1}
 		{#each turnGroups as turn, turnIndex (turnIndex)}
 			{@const turnStats = message?.timings?.agentic?.perTurn?.[turnIndex]}
-
-			<div class="agentic-turn group/turn grid gap-3 mb-4">
+			<div class="agentic-turn my-2 hover:bg-muted/80 dark:hover:bg-muted/30">
+				<span class="agentic-turn-label">Turn {turnIndex + 1}</span>
 				{#each turn.sections as section, sIdx (turn.flatIndices[sIdx])}
 					{@render renderSection(section, turn.flatIndices[sIdx])}
 				{/each}
-
-				{#if turnStats && showMessageStats}
-					<div class="turn-stats transition-opacity duration-150">
+				{#if turnStats}
+					<div class="turn-stats">
 						<ChatMessageStatistics
 							promptTokens={turnStats.llm.prompt_n}
 							promptMs={turnStats.llm.prompt_ms}
@@ -402,21 +402,39 @@
 	.agentic-content {
 		display: flex;
 		flex-direction: column;
+		gap: 0.5rem;
 		width: 100%;
 		max-width: 48rem;
-		gap: 1rem;
-	}
-
-	.agentic-content > :global(*),
-	.agentic-turn > :global(*) {
-		min-width: 0;
 	}
 
 	.agentic-text {
 		width: 100%;
 	}
 
+	.agentic-turn {
+		position: relative;
+		border: 1.5px dashed var(--muted-foreground);
+		border-radius: 0.75rem;
+		padding: 1rem;
+		transition: background 0.1s;
+	}
+
+	.agentic-turn-label {
+		position: absolute;
+		top: -1rem;
+		left: 0.75rem;
+		padding: 0 0.375rem;
+		background: var(--background);
+		font-size: 0.7rem;
+		font-weight: 500;
+		color: var(--muted-foreground);
+		text-transform: uppercase;
+		letter-spacing: 0.05em;
+	}
+
 	.turn-stats {
+		margin-top: 0.75rem;
+		padding-top: 0.5rem;
 		border-top: 1px solid hsl(var(--muted) / 0.5);
 	}
 </style>

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageStatistics/ChatMessageStatistics.svelte

@@ -2,7 +2,7 @@
 	import { Clock, Gauge, WholeWord, BookOpenText, Sparkles, Wrench, Layers } from '@lucide/svelte';
 	import { ChatMessageStatisticsBadge } from '$lib/components/app';
 	import * as Tooltip from '$lib/components/ui/tooltip';
-	import { ChatMessageStatsView, ChatMessageStatisticsMode } from '$lib/enums';
+	import { ChatMessageStatsView } from '$lib/enums';
 	import type { ChatMessageAgenticTimings } from '$lib/types/chat';
 	import { formatPerformanceTime } from '$lib/utils';
 	import { MS_PER_SECOND, DEFAULT_PERFORMANCE_TIME } from '$lib/constants';
@@ -19,7 +19,6 @@
 		agenticTimings?: ChatMessageAgenticTimings;
 		onActiveViewChange?: (view: ChatMessageStatsView) => void;
 		hideSummary?: boolean;
-		mode?: ChatMessageStatisticsMode;
 	}
 
 	let {
@@ -32,30 +31,19 @@
 		initialView = ChatMessageStatsView.GENERATION,
 		agenticTimings,
 		onActiveViewChange,
-		hideSummary = false,
-		mode = ChatMessageStatisticsMode.SWITCHABLE
+		hideSummary = false
 	}: Props = $props();
 
-	let isSwitchable = $derived(mode === ChatMessageStatisticsMode.SWITCHABLE);
-
-	let activeView: ChatMessageStatsView = $derived(
-		mode === ChatMessageStatisticsMode.READING
-			? ChatMessageStatsView.READING
-			: mode === ChatMessageStatisticsMode.GENERATION
-				? ChatMessageStatsView.GENERATION
-				: initialView
-	);
+	let activeView: ChatMessageStatsView = $derived(initialView);
 	let hasAutoSwitchedToGeneration = $state(false);
 
 	$effect(() => {
-		if (isSwitchable) {
-			onActiveViewChange?.(activeView);
-		}
+		onActiveViewChange?.(activeView);
 	});
 
 	// In live mode: auto-switch to GENERATION tab when prompt processing completes
 	$effect(() => {
-		if (isLive && isSwitchable) {
+		if (isLive) {
 			// Auto-switch to generation tab only when prompt processing is done (once)
 			if (
 				!hasAutoSwitchedToGeneration &&
@@ -103,7 +91,8 @@
 			formattedPromptTime !== undefined
 	);
 
-	let isGenerationDisabled = $derived(isLive && isSwitchable && !hasGenerationStats);
+	// In live mode, generation tab is disabled until we have generation stats
+	let isGenerationDisabled = $derived(isLive && !hasGenerationStats);
 
 	let hasAgenticStats = $derived(agenticTimings !== undefined && agenticTimings.toolCallsCount > 0);
 
@@ -164,44 +153,44 @@
 {/snippet}
 
 <div class="inline-flex items-center text-xs text-muted-foreground">
-	{#if isSwitchable}
-		<div class="inline-flex items-center rounded-sm bg-muted-foreground/15 p-0.5">
-			{#if hasPromptStats || isLive}
-				{@render viewButton({
-					view: ChatMessageStatsView.READING,
-					icon: BookOpenText,
-					label: 'Reading',
-					tooltipText: 'Processing'
-				})}
-			{/if}
-
+	<div class="inline-flex items-center rounded-sm bg-muted-foreground/15 p-0.5">
+		{#if hasPromptStats || isLive}
 			{@render viewButton({
-				view: ChatMessageStatsView.GENERATION,
-				icon: Sparkles,
-				label: 'Generation',
-				tooltipText: isGenerationDisabled ? 'Waiting for tokens...' : 'Generation',
-				disabled: isGenerationDisabled
+				view: ChatMessageStatsView.READING,
+				icon: BookOpenText,
+				label: 'Reading',
+				tooltipText: 'Reading (prompt processing)'
+			})}
+		{/if}
+
+		{@render viewButton({
+			view: ChatMessageStatsView.GENERATION,
+			icon: Sparkles,
+			label: 'Generation',
+			tooltipText: isGenerationDisabled
+				? 'Generation (waiting for tokens...)'
+				: 'Generation (token output)',
+			disabled: isGenerationDisabled
+		})}
+
+		{#if hasAgenticStats}
+			{@render viewButton({
+				view: ChatMessageStatsView.TOOLS,
+				icon: Wrench,
+				label: 'Tools',
+				tooltipText: 'Tool calls'
 			})}
 
-			{#if hasAgenticStats}
+			{#if !hideSummary}
 				{@render viewButton({
-					view: ChatMessageStatsView.TOOLS,
-					icon: Wrench,
-					label: 'Tools',
-					tooltipText: 'Tool calls'
+					view: ChatMessageStatsView.SUMMARY,
+					icon: Layers,
+					label: 'Summary',
+					tooltipText: 'Agentic summary'
 				})}
-
-				{#if !hideSummary}
-					{@render viewButton({
-						view: ChatMessageStatsView.SUMMARY,
-						icon: Layers,
-						label: 'Summary',
-						tooltipText: 'Agentic summary'
-					})}
-				{/if}
 			{/if}
-		</div>
-	{/if}
+		{/if}
+	</div>
 
 	<div class="flex items-center gap-1 px-2">
 		{#if activeView === ChatMessageStatsView.GENERATION && hasGenerationStats}
@@ -267,7 +256,7 @@
 				value={formattedAgenticTotalTime}
 				tooltipLabel="Total time (LLM + tools)"
 			/>
-		{:else if hasPromptStats && (mode === ChatMessageStatisticsMode.READING || isSwitchable)}
+		{:else if hasPromptStats}
 			<ChatMessageStatisticsBadge
 				class="bg-transparent"
 				icon={WholeWord}

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte

@@ -186,8 +186,6 @@
 			message: DatabaseMessage;
 			toolMessages: DatabaseMessage[];
 			isLastAssistantMessage: boolean;
-			isLastUserMessage: boolean;
-			nextAssistantMessage: DatabaseMessage | null;
 			siblingInfo: ChatMessageSiblingInfo;
 		}> = [];
 
@@ -238,36 +236,18 @@
 				message: msg,
 				toolMessages,
 				isLastAssistantMessage: false,
-				isLastUserMessage: false,
-				nextAssistantMessage: null,
 				siblingInfo
 			});
 		}
 
-		let lastAssistantIdx = -1;
+		// Mark the last assistant message
 		for (let i = result.length - 1; i >= 0; i--) {
 			if (result[i].message.role === MessageRole.ASSISTANT) {
 				result[i].isLastAssistantMessage = true;
-				lastAssistantIdx = i;
 				break;
 			}
 		}
 
-		if (lastAssistantIdx > 0 && result[lastAssistantIdx - 1].message.role === MessageRole.USER) {
-			result[lastAssistantIdx - 1].isLastUserMessage = true;
-		}
-
-		for (let i = 0; i < result.length; i++) {
-			if (result[i].message.role !== MessageRole.USER) continue;
-
-			for (let j = i + 1; j < result.length; j++) {
-				if (result[j].message.role === MessageRole.ASSISTANT) {
-					result[i].nextAssistantMessage = result[j].message;
-					break;
-				}
-			}
-		}
-
 		return result;
 	});
 </script>
@@ -277,14 +257,12 @@
 		{isVisible ? 'opacity-100' : 'opacity-0'}
 		{previousRouteId === '/(chat)/chat/[id]' ? '' : 'delay-300'}"
 >
-	{#each displayMessages as { message, toolMessages, isLastAssistantMessage, isLastUserMessage, nextAssistantMessage, siblingInfo } (message.id)}
+	{#each displayMessages as { message, toolMessages, isLastAssistantMessage, siblingInfo } (message.id)}
 		<ChatMessage
 			class="mx-auto mt-12 w-full max-w-3xl"
 			{message}
 			{toolMessages}
 			{isLastAssistantMessage}
-			{isLastUserMessage}
-			{nextAssistantMessage}
 			{siblingInfo}
 		/>
 	{/each}

tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte

@@ -4,10 +4,12 @@
 		ChatScreenForm,
 		ChatMessages,
 		ChatScreenDragOverlay,
+		ChatScreenProcessingInfo,
 		ChatScreenStreamResumeStatus,
 		ServerLoadingSplash,
 		ChatScreenServerError
 	} from '$lib/components/app';
+	import { setProcessingInfoContext } from '$lib/contexts';
 	import { createAutoScrollController } from '$lib/hooks/use-auto-scroll.svelte';
 	import { useChatScreenActiveModel } from '$lib/hooks/use-chat-screen-active-model.svelte';
 	import { useChatScreenDragAndDrop } from '$lib/hooks/use-chat-screen-drag-and-drop.svelte';
@@ -21,7 +23,8 @@
 		errorDialog,
 		isLoading,
 		isChatStreaming,
-		isEditing
+		isEditing,
+		activeProcessingState
 	} from '$lib/stores/chat.svelte';
 	import {
 		conversationsStore,
@@ -39,6 +42,12 @@
 
 	let { showCenteredEmpty = false } = $props();
 
+	setProcessingInfoContext({
+		get showProcessingInfo() {
+			return showProcessingInfo;
+		}
+	});
+
 	let disableAutoScroll = $derived(Boolean(config().disableAutoScroll) || isMobile.current);
 	let isMobileUserScrolledUp = $state(false);
 	let mobileScrollDownHint = $state(false);
@@ -54,6 +63,11 @@
 	let isServerLoading = $derived(serverLoading());
 	let hasPropsError = $derived(!!serverError());
 	let isCurrentConversationLoading = $derived(isLoading() || isChatStreaming());
+	let showProcessingInfo = $derived(
+		isCurrentConversationLoading ||
+			(config().keepStatsVisible && !!page.params.id) ||
+			activeProcessingState() !== null
+	);
 	let chatFormBottomPosition = $derived.by(() => {
 		if (!isMobile.current) return '1rem';
 		if (device.isStandalone) return '1.5rem';
@@ -284,6 +298,10 @@
 						}}
 					/>
 				{/if}
+
+				{#if showProcessingInfo}
+					<ChatScreenProcessingInfo />
+				{/if}
 			</div>
 
 			<ChatScreenForm

tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenProcessingInfo.svelte

@@ -0,0 +1,127 @@
+<script lang="ts">
+	import { untrack } from 'svelte';
+	import { PROCESSING_INFO_TIMEOUT } from '$lib/constants';
+	import { useProcessingState } from '$lib/hooks/use-processing-state.svelte';
+	import { chatStore, isLoading, isChatStreaming } from '$lib/stores/chat.svelte';
+	import { activeMessages, activeConversation } from '$lib/stores/conversations.svelte';
+	import { config } from '$lib/stores/settings.svelte';
+
+	const processingState = useProcessingState();
+
+	let isCurrentConversationLoading = $derived(isLoading());
+	let isStreaming = $derived(isChatStreaming());
+	let processingDetails = $derived(processingState.getTechnicalDetails());
+
+	let processingVisible = $derived(processingDetails.length > 0);
+
+	let { onVisibilityChange }: { onVisibilityChange?: (visible: boolean) => void } = $props();
+
+	$effect(() => {
+		onVisibilityChange?.(processingVisible);
+	});
+
+	$effect(() => {
+		const conversation = activeConversation();
+
+		untrack(() => chatStore.setActiveProcessingConversation(conversation?.id ?? null));
+	});
+
+	$effect(() => {
+		const keepStatsVisible = config().keepStatsVisible;
+		const shouldMonitor = keepStatsVisible || isCurrentConversationLoading || isStreaming;
+
+		if (shouldMonitor) {
+			processingState.startMonitoring();
+		}
+
+		if (!isCurrentConversationLoading && !isStreaming && !keepStatsVisible) {
+			const timeout = setTimeout(() => {
+				if (!config().keepStatsVisible && !isChatStreaming()) {
+					processingState.stopMonitoring();
+				}
+			}, PROCESSING_INFO_TIMEOUT);
+
+			return () => clearTimeout(timeout);
+		}
+	});
+
+	$effect(() => {
+		const conversation = activeConversation();
+		const messages = activeMessages() as DatabaseMessage[];
+		const keepStatsVisible = config().keepStatsVisible;
+
+		if (keepStatsVisible && conversation) {
+			if (messages.length === 0) {
+				untrack(() => chatStore.clearProcessingState(conversation.id));
+				return;
+			}
+
+			if (!isCurrentConversationLoading && !isStreaming) {
+				untrack(() => chatStore.restoreProcessingStateFromMessages(messages, conversation.id));
+			}
+		}
+	});
+</script>
+
+<div
+	class={[
+		'chat-processing-info-container pointer-events-none relative w-full hidden md:block',
+		processingVisible && 'visible'
+	]}
+>
+	<div class="chat-processing-info-content absolute bottom-4 left-1/2 -translate-x-1/2">
+		{#each processingDetails as detail (detail)}
+			<span class="chat-processing-info-detail pointer-events-auto backdrop-blur-sm">{detail}</span>
+		{/each}
+	</div>
+</div>
+
+<style>
+	.chat-processing-info-container {
+		position: sticky;
+		top: 0;
+		z-index: 10;
+		padding: 0 1rem 0.75rem;
+		opacity: 0;
+		transform: translateY(50%);
+		transition:
+			opacity 300ms ease-out,
+			transform 300ms ease-out;
+	}
+
+	.chat-processing-info-container.visible {
+		opacity: 1;
+		transform: translateY(0);
+	}
+
+	.chat-processing-info-content {
+		display: flex;
+		flex-wrap: wrap;
+		align-items: center;
+		gap: 1rem;
+		justify-content: center;
+		max-width: 48rem;
+		margin: 0 auto;
+	}
+
+	.chat-processing-info-detail {
+		color: var(--muted-foreground);
+		font-size: 0.75rem;
+		padding: 0.25rem 0.75rem;
+		border-radius: 0.375rem;
+		font-family:
+			ui-monospace, SFMono-Regular, 'SF Mono', Consolas, 'Liberation Mono', Menlo, monospace;
+		white-space: nowrap;
+	}
+
+	@media (max-width: 768px) {
+		.chat-processing-info-content {
+			gap: 0.5rem;
+		}
+
+		.chat-processing-info-detail {
+			font-size: 0.7rem;
+			padding: 0.2rem 0.5rem;
+		}
+	}
+</style>

tools/ui/src/lib/components/app/chat/index.ts

@@ -241,18 +241,13 @@ export { default as ChatFormActionAddToolsSubmenu } from './ChatForm/ChatFormAct
 export { default as ChatFormActionAddMcpServersSubmenu } from './ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte';
 
 /**
- * Dropdown submenu for selecting reasoning effort level.
+ * **ChatFormReasoningToggle** - Thinking toggle button with effort dropdown
  *
- * Shows a "Reasoning" sub-menu item with a lightbulb icon indicating
- * thinking status, and a nested list of effort levels.
+ * A toggle button with lightbulb icon that indicates thinking status.
+ * Shows the reasoning effort dropdown when clicked.
  * Only visible when the current model supports thinking.
  */
-export { default as ChatFormActionAddReasoningSubmenu } from './ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddReasoningSubmenu.svelte';
-
-/**
- * Compact context-usage gauge with per-turn and cumulative breakdown in the tooltip.
- */
-export { default as ChatFormContextGauge } from './ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte';
+export { default as ChatFormReasoningToggle } from './ChatForm/ChatFormActions/ChatFormReasoningToggle.svelte';
 
 /**
  * Hidden file input element for programmatic file selection.
@@ -674,6 +669,14 @@ export { default as ChatScreenDragOverlay } from './ChatScreen/ChatScreenDragOve
  */
 export { default as ChatScreenForm } from './ChatScreen/ChatScreenForm.svelte';
 
+/**
+ * Processing info display during generation. Shows real-time statistics:
+ * tokens per second, prompt/completion token counts, and elapsed time.
+ * Data sourced from slotsService polling during active generation.
+ * Only visible when `isCurrentConversationLoading` is true.
+ */
+export { default as ChatScreenProcessingInfo } from './ChatScreen/ChatScreenProcessingInfo.svelte';
+
 /**
  * Server error alert displayed when the server is unreachable.
  * Shows the error message with a retry button.

tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte

@@ -76,7 +76,7 @@
 		open = value;
 		onToggle?.();
 	}}
-	class="{className} my-0!"
+	class={className}
 >
 	<Card class="gap-0 border-muted bg-muted/30 py-0">
 		<Collapsible.Trigger class="flex w-full cursor-pointer items-start justify-between gap-2 p-3">

tools/ui/src/lib/components/app/content/SyntaxHighlightedCode.svelte

@@ -72,8 +72,8 @@
 </script>
 
 <div
-	class="code-preview-wrapper min-w-0 max-w-full overflow-x-auto rounded-lg border border-border bg-muted {className}"
-	style="max-height: {maxHeight}; {maxWidth ? `max-width: ${maxWidth};` : ''}"
+	class="code-preview-wrapper rounded-lg border border-border bg-muted {className}"
+	style="max-height: {maxHeight}; max-width: {maxWidth};"
 >
 	<!-- Needs to be formatted as single line for proper rendering -->
 	<pre class="m-0"><code class="hljs text-sm leading-relaxed">{@html highlightedHtml}</code></pre>

tools/ui/src/lib/components/app/dialogs/DialogMcpServerRecommendations.svelte

@@ -4,10 +4,9 @@
 	import * as Dialog from '$lib/components/ui/dialog';
 	import { fly } from 'svelte/transition';
 	import { McpServerCardCompact, McpServerForm } from '$lib/components/app/mcp';
-	import { RECOMMENDED_MCP_SERVERS, SETTINGS_KEYS } from '$lib/constants';
+	import { RECOMMENDED_MCP_SERVERS } from '$lib/constants';
 	import { conversationsStore } from '$lib/stores/conversations.svelte';
 	import { mcpStore } from '$lib/stores/mcp.svelte';
-	import { settingsStore } from '$lib/stores/settings.svelte';
 	import { uuid } from '$lib/utils';
 	import { MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY, MCP_SERVER_ID_PREFIX } from '$lib/constants';
 	import type { MCPServerSettingsEntry } from '$lib/types';
@@ -25,22 +24,6 @@
 	);
 
 	let addedServers = $state<MCPServerSettingsEntry[]>([]);
-	let didAddAny = $state(false);
-
-	let selectedRecommendedCount = $derived.by(
-		() => RECOMMENDED_MCP_SERVERS.filter((server) => selected[server.id]).length
-	);
-
-	let footerLabel = $derived.by(() => {
-		const recommended = selectedRecommendedCount;
-		const custom = addedServers.length;
-		const total = recommended + custom;
-
-		if (total === 0) return 'Continue';
-		if (recommended === 0) return custom === 1 ? 'Add server' : `Add ${custom} servers`;
-		if (custom === 0) return recommended === 1 ? 'Add server' : `Add ${recommended} servers`;
-		return `Add ${recommended} servers and ${custom} custom`;
-	});
 
 	let showAddForm = $state(false);
 	let newServerUrl = $state('');
@@ -61,14 +44,9 @@
 			showAddForm = false;
 			newServerUrl = '';
 			newServerHeaders = '';
-
-			if (!didAddAny) {
-				settingsStore.updateConfig(SETTINGS_KEYS.MCP_SERVERS, []);
-			}
+			addedServers = [];
 
 			localStorage.setItem(MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY, 'true');
-			addedServers = [];
-			didAddAny = false;
 		}
 		open = value;
 		onOpenChange?.(value);
@@ -81,7 +59,6 @@
 	}
 
 	function enableSelected() {
-		didAddAny = true;
 		localStorage.setItem(MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY, 'true');
 
 		for (const server of RECOMMENDED_MCP_SERVERS) {
@@ -106,8 +83,6 @@
 	function saveNewServer() {
 		if (newServerUrlError) return;
 
-		didAddAny = true;
-
 		const newServerId = uuid() ?? `${MCP_SERVER_ID_PREFIX}-${Date.now()}`;
 
 		localStorage.setItem(MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY, 'true');
@@ -199,12 +174,7 @@
 		<Dialog.Footer>
 			<Button variant="secondary" size="sm" onclick={() => handleOpenChange(false)}>Not now</Button>
 
-			<Button
-				variant="default"
-				size="sm"
-				onclick={enableSelected}
-				disabled={footerLabel === 'Continue'}>{footerLabel}</Button
-			>
+			<Button variant="default" size="sm" onclick={enableSelected}>Add selected</Button>
 		</Dialog.Footer>
 	</Dialog.Content>
 </Dialog.Root>

tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatToolsTab.svelte

@@ -39,17 +39,13 @@
 					{@const faviconUrl = group.serverId ? mcpStore.getServerFavicon(group.serverId) : null}
 
 					<span class="inline-flex min-w-0 items-center gap-1.5 font-medium">
-						{#if group.source === 'mcp'}
-							<McpServerIdentity
-								iconClass="h-4 w-4"
-								iconRounded="rounded-sm"
-								showVersion={false}
-								displayName={group.label}
-								{faviconUrl}
-							/>
-						{:else}
-							<TruncatedText text={group.label} class="font-medium" />
-						{/if}
+						<McpServerIdentity
+							iconClass="h-4 w-4"
+							iconRounded="rounded-sm"
+							showVersion={false}
+							displayName={group.label}
+							{faviconUrl}
+						/>
 					</span>
 
 					<span class="ml-auto shrink-0 text-xs text-muted-foreground">

tools/ui/src/lib/components/ui/hover-card/hover-card-content.svelte

@@ -1,31 +0,0 @@
-<script lang="ts">
-	import { LinkPreview as HoverCardPrimitive } from 'bits-ui';
-	import { cn, type WithoutChildrenOrChild } from '$lib/components/ui/utils.js';
-	import HoverCardPortal from './hover-card-portal.svelte';
-	import type { ComponentProps } from 'svelte';
-
-	let {
-		ref = $bindable(null),
-		class: className,
-		align = 'center',
-		sideOffset = 4,
-		portalProps,
-		...restProps
-	}: HoverCardPrimitive.ContentProps & {
-		portalProps?: WithoutChildrenOrChild<ComponentProps<typeof HoverCardPortal>>;
-	} = $props();
-</script>
-
-<HoverCardPortal {...portalProps}>
-	<HoverCardPrimitive.Content
-		bind:ref
-		data-slot="hover-card-content"
-		{align}
-		{sideOffset}
-		class={cn(
-			'data-open:animate-in data-closed:animate-out data-closed:fade-out-0 data-open:fade-in-0 data-closed:zoom-out-95 data-open:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 ring-foreground/10 bg-popover text-popover-foreground w-64 rounded-lg p-2.5 text-sm shadow-md ring-1 duration-100 z-50 origin-(--transform-origin) outline-hidden',
-			className
-		)}
-		{...restProps}
-	/>
-</HoverCardPortal>

tools/ui/src/lib/components/ui/hover-card/hover-card-portal.svelte

@@ -1,7 +0,0 @@
-<script lang="ts">
-	import { LinkPreview as HoverCardPrimitive } from 'bits-ui';
-
-	let { ...restProps }: HoverCardPrimitive.PortalProps = $props();
-</script>
-
-<HoverCardPrimitive.Portal {...restProps} />

tools/ui/src/lib/components/ui/hover-card/hover-card-trigger.svelte

@@ -1,7 +0,0 @@
-<script lang="ts">
-	import { LinkPreview as HoverCardPrimitive } from 'bits-ui';
-
-	let { ref = $bindable(null), ...restProps }: HoverCardPrimitive.TriggerProps = $props();
-</script>
-
-<HoverCardPrimitive.Trigger bind:ref data-slot="hover-card-trigger" {...restProps} />

tools/ui/src/lib/components/ui/hover-card/hover-card.svelte

@@ -1,7 +0,0 @@
-<script lang="ts">
-	import { LinkPreview as HoverCardPrimitive } from 'bits-ui';
-
-	let { open = $bindable(false), ...restProps }: HoverCardPrimitive.RootProps = $props();
-</script>
-
-<HoverCardPrimitive.Root bind:open {...restProps} />

tools/ui/src/lib/components/ui/hover-card/index.ts

@@ -1,15 +0,0 @@
-import Root from './hover-card.svelte';
-import Content from './hover-card-content.svelte';
-import Trigger from './hover-card-trigger.svelte';
-import Portal from './hover-card-portal.svelte';
-
-export {
-	Root,
-	Content,
-	Trigger,
-	Portal,
-	Root as HoverCard,
-	Content as HoverCardContent,
-	Trigger as HoverCardTrigger,
-	Portal as HoverCardPortal
-};

tools/ui/src/lib/constants/context-keys.ts

@@ -1,3 +1,4 @@
 export const CONTEXT_KEY_MESSAGE_EDIT = 'chat-message-edit';
 export const CONTEXT_KEY_CHAT_ACTIONS = 'chat-actions';
 export const CONTEXT_KEY_CHAT_SETTINGS_CONFIG = 'chat-settings-config';
+export const CONTEXT_KEY_PROCESSING_INFO = 'processing-info';

tools/ui/src/lib/constants/css-classes.ts

@@ -17,4 +17,3 @@ export const PANEL_CLASSES = `
 `;
 
 export const CHAT_FORM_POPOVER_MAX_HEIGHT = 'max-h-80';
-export const DIALOG_SUBMENU_CONTENT = 'w-60';

tools/ui/src/lib/constants/reasoning-effort.ts

@@ -6,7 +6,6 @@ import type { ReasoningEffortLevel } from '$lib/types';
  * Keys match the ReasoningEffort enum values for type-safe lookups.
  */
 export const REASONING_EFFORT_LABELS: Record<string, string> = {
-	[ReasoningEffort.OFF]: 'Off',
 	[ReasoningEffort.LOW]: 'Low',
 	[ReasoningEffort.MEDIUM]: 'Medium',
 	[ReasoningEffort.HIGH]: 'High',
@@ -14,7 +13,7 @@ export const REASONING_EFFORT_LABELS: Record<string, string> = {
 };
 
 export const REASONING_EFFORT_LEVELS: ReasoningEffortLevel[] = [
-	{ value: ReasoningEffort.OFF, label: 'Off', isOff: true },
+	{ value: 'off', label: 'Off', isOff: true },
 	{ value: ReasoningEffort.LOW, label: 'Low' },
 	{ value: ReasoningEffort.MEDIUM, label: 'Medium' },
 	{ value: ReasoningEffort.HIGH, label: 'High' },

tools/ui/src/lib/constants/settings-keys.ts

@@ -22,6 +22,7 @@ export const SETTINGS_KEYS = {
 	// Display
 	SHOW_MESSAGE_STATS: 'showMessageStats',
 	SHOW_THOUGHT_IN_PROGRESS: 'showThoughtInProgress',
+	KEEP_STATS_VISIBLE: 'keepStatsVisible',
 	AUTO_MIC_ON_EMPTY: 'autoMicOnEmpty',
 	RENDER_USER_CONTENT_AS_MARKDOWN: 'renderUserContentAsMarkdown',
 	DISABLE_AUTO_SCROLL: 'disableAutoScroll',
@@ -60,6 +61,7 @@ export const SETTINGS_KEYS = {
 	MCP_REQUEST_TIMEOUT_SECONDS: 'mcpRequestTimeoutSeconds',
 	MCP_DEFAULT_SERVER_OVERRIDES: 'mcpDefaultServerOverrides',
 	AGENTIC_MAX_TURNS: 'agenticMaxTurns',
+	ALWAYS_SHOW_AGENTIC_TURNS: 'alwaysShowAgenticTurns',
 	AGENTIC_MAX_TOOL_PREVIEW_LINES: 'agenticMaxToolPreviewLines',
 	SHOW_TOOL_CALL_IN_PROGRESS: 'showToolCallInProgress',
 	// Performance

tools/ui/src/lib/constants/settings-registry.ts

@@ -258,6 +258,18 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
 					paramType: SyncableParameterType.BOOLEAN
 				}
 			},
+			{
+				key: SETTINGS_KEYS.KEEP_STATS_VISIBLE,
+				label: 'Keep stats visible after generation',
+				help: 'Keep processing statistics visible after generation finishes.',
+				defaultValue: false,
+				type: SettingsFieldType.CHECKBOX,
+				section: SETTINGS_SECTION_SLUGS.DISPLAY,
+				sync: {
+					serverKey: SETTINGS_KEYS.KEEP_STATS_VISIBLE,
+					paramType: SyncableParameterType.BOOLEAN
+				}
+			},
 			{
 				key: SETTINGS_KEYS.AUTO_MIC_ON_EMPTY,
 				label: 'Show microphone on empty input',
@@ -367,6 +379,18 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
 					paramType: SyncableParameterType.BOOLEAN
 				}
 			},
+			{
+				key: SETTINGS_KEYS.ALWAYS_SHOW_AGENTIC_TURNS,
+				label: 'Always show agentic turns in conversation',
+				help: 'Always expand and display agentic loop turns in conversation messages.',
+				defaultValue: false,
+				type: SettingsFieldType.CHECKBOX,
+				section: SETTINGS_SECTION_SLUGS.DISPLAY,
+				sync: {
+					serverKey: SETTINGS_KEYS.ALWAYS_SHOW_AGENTIC_TURNS,
+					paramType: SyncableParameterType.BOOLEAN
+				}
+			},
 			{
 				key: SETTINGS_KEYS.SHOW_BUILD_VERSION,
 				label: 'Show build version information',

tools/ui/src/lib/constants/storage.ts

@@ -21,6 +21,7 @@ export const DISABLED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledTool
 /** Disabled tools keyed by stable selection identity, no migration from the name based key */
 export const DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledToolKeys`;
 export const FAVORITE_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.favoriteModels`;
+export const THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.thinkingEnabledDefault`;
 export const REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.reasoningEffortDefault`;
 /** Set when user has interacted with the MCP server recommendations dialog (checked servers, added custom server, or dismissed) */
 export const MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.mcpServersSetupDone`;

tools/ui/src/lib/contexts/index.ts

@@ -17,3 +17,9 @@ export {
 	setChatSettingsConfigContext,
 	type ChatSettingsConfigContext
 } from './chat-settings-config.context';
+
+export {
+	getProcessingInfoContext,
+	setProcessingInfoContext,
+	type ProcessingInfoContext
+} from './processing-info.context';

tools/ui/src/lib/contexts/processing-info.context.ts

@@ -0,0 +1,16 @@
+import { getContext, setContext } from 'svelte';
+import { CONTEXT_KEY_PROCESSING_INFO } from '$lib/constants';
+
+export interface ProcessingInfoContext {
+	readonly showProcessingInfo: boolean;
+}
+
+const PROCESSING_INFO_KEY = Symbol.for(CONTEXT_KEY_PROCESSING_INFO);
+
+export function setProcessingInfoContext(ctx: ProcessingInfoContext): ProcessingInfoContext {
+	return setContext(PROCESSING_INFO_KEY, ctx);
+}
+
+export function getProcessingInfoContext(): ProcessingInfoContext {
+	return getContext(PROCESSING_INFO_KEY);
+}

tools/ui/src/lib/enums/chat.enums.ts

@@ -5,12 +5,6 @@ export enum ChatMessageStatsView {
 	SUMMARY = 'summary'
 }
 
-export enum ChatMessageStatisticsMode {
-	SWITCHABLE = 'switchable',
-	READING = 'reading',
-	GENERATION = 'generation'
-}
-
 /**
  * Connection state of a streamed completion, drives the resume status indicator.
  */

tools/ui/src/lib/enums/index.ts

@@ -10,7 +10,6 @@ export { AgenticSectionType, ContinueIntentKind, ToolCallType } from './agentic.
 
 export {
 	ChatMessageStatsView,
-	ChatMessageStatisticsMode,
 	StreamConnectionState,
 	ContentPartType,
 	ConversationSelectionMode,

tools/ui/src/lib/enums/reasoning-effort.enums.ts

@@ -3,7 +3,6 @@
  * These values are sent to the server and mapped to token budgets.
  */
 export enum ReasoningEffort {
-	OFF = 'off',
 	LOW = 'low',
 	MEDIUM = 'medium',
 	HIGH = 'high',

tools/ui/src/lib/hooks/use-context-gauge.svelte.ts

@@ -1,295 +0,0 @@
-/**
- * Reactive state for the context usage gauge: resolves the active model,
- * fetches its cached props, parses live server stats, and exposes per-turn
- * read / fresh / cache / output and cumulative token counts.
- */
-
-import {
-	modelsStore,
-	modelOptions,
-	selectedModelId,
-	singleModelName
-} from '$lib/stores/models.svelte';
-import { chatStore } from '$lib/stores/chat.svelte';
-import { activeMessages } from '$lib/stores/conversations.svelte';
-import { isRouterMode } from '$lib/stores/server.svelte';
-import { MessageRole } from '$lib/enums';
-import { STATS_UNITS } from '$lib/constants';
-import type { ChatMessageTimings, DatabaseMessage } from '$lib/types';
-import { useProcessingState } from './use-processing-state.svelte';
-import {
-	colorLevelFromPercent,
-	type ColorLevel
-} from '$lib/components/app/chat/ChatForm/ChatFormContextGauge/context-gauge';
-
-interface LiveStats {
-	freshTokens: number;
-	promptTokens: number;
-	cacheTokens: number;
-	outputTokens: number;
-}
-
-export interface UseContextGaugeReturn {
-	readonly activeModelId: string | null;
-	readonly isActiveModelLoaded: boolean;
-	readonly isActiveModelLoading: boolean;
-	readonly contextTotal: number | null;
-	readonly contextUsed: number;
-	readonly currentRead: number;
-	readonly currentFresh: number;
-	readonly currentCache: number;
-	readonly currentOutput: number;
-	readonly kvTotal: number;
-	readonly cumulativeRead: number;
-	readonly cumulativeOutput: number;
-	readonly cumulativeCacheTotal: number;
-	readonly averageTokensPerSecond: number | null;
-	readonly contextPercent: number | null;
-	readonly colorLevel: ColorLevel;
-	readonly transientDetails: string[];
-	readonly hasAnyUsage: boolean;
-	loadModel(): Promise<void>;
-	startMonitoring(): void;
-}
-
-function lastAssistantTimings(messages: DatabaseMessage[]): ChatMessageTimings | undefined {
-	for (let i = messages.length - 1; i >= 0; i--) {
-		const m = messages[i];
-		if (m.role === MessageRole.ASSISTANT && m.timings) return m.timings;
-	}
-	return undefined;
-}
-
-function deriveLiveStats(
-	state: ReturnType<typeof useProcessingState>['processingState']
-): LiveStats | null {
-	if (!state || (state.status !== 'preparing' && state.status !== 'generating')) {
-		return null;
-	}
-	const promptTokens = state.promptTokens ?? 0;
-	const cacheTokens = state.cacheTokens ?? 0;
-	return {
-		freshTokens: promptTokens,
-		promptTokens: promptTokens + cacheTokens,
-		cacheTokens,
-		outputTokens: state.outputTokensUsed ?? 0
-	};
-}
-
-const TRANSIENT_DETAILS_EXCLUDED_PREFIXES = ['Context:', 'Output:'];
-
-function filterTransientDetails(raw: string[]): string[] {
-	return raw.filter((detail) => {
-		if (TRANSIENT_DETAILS_EXCLUDED_PREFIXES.some((prefix) => detail.startsWith(prefix))) {
-			return false;
-		}
-		return !detail.includes(STATS_UNITS.TOKENS_PER_SECOND);
-	});
-}
-
-export function useContextGauge(): UseContextGaugeReturn {
-	const processingState = useProcessingState();
-
-	// Resolve the model the gauge reports context for: explicit selection >
-	// last assistant model > single-model mode (mirrors useChatScreenActiveModel).
-	const activeModelId = $derived.by(() => {
-		if (!isRouterMode()) {
-			return singleModelName();
-		}
-
-		const selectedId = selectedModelId();
-		if (selectedId) {
-			const model = modelOptions().find((m) => m.id === selectedId);
-			if (model) return model.model;
-		}
-
-		return chatStore.getConversationModel(activeMessages() as DatabaseMessage[]);
-	});
-
-	const isActiveModelLoaded = $derived(
-		activeModelId !== null && modelsStore.isModelLoaded(activeModelId)
-	);
-
-	const isActiveModelLoading = $derived(
-		activeModelId !== null && modelsStore.isModelOperationInProgress(activeModelId)
-	);
-
-	// Pull /props on demand so n_ctx surfaces before the first chat request.
-	$effect(() => {
-		if (activeModelId && isActiveModelLoaded) {
-			const cached = modelsStore.getModelProps(activeModelId);
-			if (!cached) {
-				void modelsStore.fetchModelProps(activeModelId);
-			}
-		}
-	});
-
-	const contextTotal = $derived.by(() => {
-		void modelsStore.propsCacheVersion;
-		return activeModelId ? modelsStore.getModelContextSize(activeModelId) : null;
-	});
-
-	const liveStats = $derived(deriveLiveStats(processingState.processingState));
-
-	const currentRead = $derived.by(() => {
-		const timings = lastAssistantTimings(activeMessages() as DatabaseMessage[]);
-		let read = 0;
-		if (timings) {
-			read = (timings.prompt_n ?? 0) + (timings.cache_n ?? 0);
-		}
-		// live.promptTokens is already the combined reading (prompt + cache),
-		// so do not also add live.cacheTokens.
-		if (liveStats && liveStats.promptTokens > 0) {
-			read = Math.max(read, liveStats.promptTokens);
-		}
-		return read;
-	});
-
-	const currentFresh = $derived.by(() => {
-		const timings = lastAssistantTimings(activeMessages() as DatabaseMessage[]);
-		const fresh = timings?.prompt_n ?? 0;
-		return Math.max(fresh, liveStats?.freshTokens ?? 0);
-	});
-
-	const currentCache = $derived.by(() => {
-		const timings = lastAssistantTimings(activeMessages() as DatabaseMessage[]);
-		const cached = timings?.cache_n ?? 0;
-		if (liveStats && liveStats.promptTokens > 0) {
-			return Math.max(cached, liveStats.cacheTokens);
-		}
-		return cached;
-	});
-
-	const currentOutput = $derived.by(() => {
-		if (liveStats && liveStats.outputTokens > 0) return liveStats.outputTokens;
-		const timings = lastAssistantTimings(activeMessages() as DatabaseMessage[]);
-		return timings?.predicted_n ?? 0;
-	});
-
-	const kvTotal = $derived(currentRead + currentOutput);
-	const contextUsed = $derived(currentRead + currentOutput);
-
-	const cumulative = $derived.by(() => {
-		const messages = activeMessages() as DatabaseMessage[];
-
-		// Agentic sessions stamp the same agentic.llm totals onto every
-		// assistant message; cache_n is never per-turn so cache_total stays 0.
-		const agenticMessages = messages.filter(
-			(m) => m.role === MessageRole.ASSISTANT && m.timings?.agentic?.llm?.predicted_n != null
-		);
-
-		if (agenticMessages.length > 0) {
-			const llm = agenticMessages[agenticMessages.length - 1].timings!.agentic!.llm;
-			const output = llm.predicted_n ?? 0;
-			const outputMs = llm.predicted_ms ?? 0;
-			const averageTokensPerSecond = outputMs > 0 && output > 0 ? (output / outputMs) * 1000 : null;
-			return {
-				read: llm.prompt_n ?? 0,
-				output,
-				cacheTotal: 0,
-				averageTokensPerSecond
-			};
-		}
-
-		let read = 0;
-		let output = 0;
-		let outputMs = 0;
-		let cacheTotal = 0;
-		for (const m of messages) {
-			if (m.role !== MessageRole.ASSISTANT || !m.timings) continue;
-			read += m.timings.prompt_n ?? 0;
-			cacheTotal += m.timings.cache_n ?? 0;
-			output += m.timings.predicted_n ?? 0;
-			outputMs += m.timings.predicted_ms ?? 0;
-		}
-		const averageTokensPerSecond = outputMs > 0 && output > 0 ? (output / outputMs) * 1000 : null;
-		return { read, output, cacheTotal, averageTokensPerSecond };
-	});
-
-	const contextPercent = $derived.by(() => {
-		if (contextTotal === null || contextTotal <= 0) return null;
-		return Math.round((contextUsed / contextTotal) * 100);
-	});
-
-	const colorLevel = $derived(colorLevelFromPercent(contextPercent));
-
-	// Drop lines the surrounding Context / Output / speed rows already render.
-	const transientDetails = $derived(filterTransientDetails(processingState.getTechnicalDetails()));
-
-	const hasAnyUsage = $derived(
-		cumulative.read > 0 ||
-			cumulative.output > 0 ||
-			currentRead > 0 ||
-			currentOutput > 0 ||
-			cumulative.averageTokensPerSecond !== null ||
-			transientDetails.length > 0
-	);
-
-	async function loadModel() {
-		if (!activeModelId || isActiveModelLoading) return;
-		try {
-			await modelsStore.loadModel(activeModelId);
-		} catch {
-			// toast already surfaced by modelsStore.loadModel
-		}
-	}
-
-	return {
-		get activeModelId() {
-			return activeModelId;
-		},
-		get isActiveModelLoaded() {
-			return isActiveModelLoaded;
-		},
-		get isActiveModelLoading() {
-			return isActiveModelLoading;
-		},
-		get contextTotal() {
-			return contextTotal;
-		},
-		get contextUsed() {
-			return contextUsed;
-		},
-		get currentRead() {
-			return currentRead;
-		},
-		get currentFresh() {
-			return currentFresh;
-		},
-		get currentCache() {
-			return currentCache;
-		},
-		get currentOutput() {
-			return currentOutput;
-		},
-		get kvTotal() {
-			return kvTotal;
-		},
-		get cumulativeRead() {
-			return cumulative.read;
-		},
-		get cumulativeOutput() {
-			return cumulative.output;
-		},
-		get cumulativeCacheTotal() {
-			return cumulative.cacheTotal;
-		},
-		get averageTokensPerSecond() {
-			return cumulative.averageTokensPerSecond;
-		},
-		get contextPercent() {
-			return contextPercent;
-		},
-		get colorLevel() {
-			return colorLevel;
-		},
-		get transientDetails() {
-			return transientDetails;
-		},
-		get hasAnyUsage() {
-			return hasAnyUsage;
-		},
-		loadModel,
-		startMonitoring: () => processingState.startMonitoring()
-	};
-}

tools/ui/src/lib/hooks/use-mcp-recommendations.svelte.ts

@@ -54,6 +54,11 @@ export function useMcpRecommendations() {
 		// effect, and we must not wipe the timeout that was just scheduled.
 		if (checked) return;
 
+		if (mcpStore.optedInRecommendationIds.size > 0) {
+			checked = true;
+			return;
+		}
+
 		const hasRecommendations = mcpStore
 			.getServers()
 			.some((server) => RECOMMENDED_MCP_SERVER_IDS.has(server.id));

tools/ui/src/lib/hooks/use-processing-state.svelte.ts

@@ -1,4 +1,5 @@
 import { activeProcessingState } from '$lib/stores/chat.svelte';
+import { config } from '$lib/stores/settings.svelte';
 import { STATS_UNITS } from '$lib/constants';
 import type { ApiProcessingState, LiveProcessingStats, LiveGenerationStats } from '$lib/types';
 
@@ -45,6 +46,7 @@ export function useProcessingState(): UseProcessingStateReturn {
 		return activeProcessingState();
 	});
 
+	// Track last known state for keepStatsVisible functionality
 	$effect(() => {
 		if (processingState && isMonitoring) {
 			lastKnownState = processingState;
@@ -86,8 +88,14 @@ export function useProcessingState(): UseProcessingStateReturn {
 
 	function stopMonitoring(): void {
 		if (!isMonitoring) return;
-
 		isMonitoring = false;
+
+		// Only clear last known state if keepStatsVisible is disabled
+		const currentConfig = config();
+		if (!currentConfig.keepStatsVisible) {
+			lastKnownState = null;
+			lastKnownProcessingStats = null;
+		}
 	}
 
 	function getProcessingMessage(): string {

tools/ui/src/lib/services/chat.service.ts

@@ -340,7 +340,6 @@ export class ChatService {
 			if (stream && conversationId) {
 				headers['X-Conversation-Id'] = streamIdentity(conversationId, options.model);
 			}
-
 			const response = await fetch(API_CHAT.COMPLETIONS, {
 				method: 'POST',
 				headers,
@@ -1012,7 +1011,7 @@ export class ChatService {
 	 *
 	 * @param response - The fetch Response object containing the JSON data
 	 * @param onComplete - Optional callback invoked when response is successfully parsed
-	 * @param onError - Optional callback invoked if an error occurs while parsing
+	 * @param onError - Optional callback invoked if an error occurs during parsing
 	 * @returns {Promise<string>} Promise that resolves to the generated content string
 	 * @throws {Error} if the response cannot be parsed or is malformed
 	 */

tools/ui/src/lib/services/migration.service.ts

@@ -564,9 +564,9 @@ const configTypesMigration: Migration = {
 		const config = JSON.parse(configRaw);
 		let changed = false;
 
-		// Pre-schema configs persisted booleans as "true"/"false" strings; the strict server
-		// schema rejects them. No config string field holds exactly "true"/"false", so the
-		// match is unambiguous.
+		// Pre-schema configs persisted booleans as the strings "true"/"false", which the
+		// strict server schema now rejects. Coerce those back to real booleans. No config
+		// string field holds exactly "true"/"false", so the match is unambiguous.
 		for (const key of Object.keys(config)) {
 			if (config[key] === 'true') {
 				config[key] = true;

tools/ui/src/lib/stores/agentic.svelte.ts

@@ -477,7 +477,7 @@ class AgenticStore {
 		conversationId: string;
 		messages: ApiChatMessageData[];
 		options: AgenticFlowOptions;
-		tools: ReturnType<typeof toolsStore.getEnabledToolsForLLM>;
+		tools: ReturnType<typeof mcpStore.getToolDefinitionsForLLM>;
 		agenticConfig: AgenticConfig;
 		callbacks: AgenticFlowCallbacks;
 		signal?: AbortSignal;

tools/ui/src/lib/stores/chat.svelte.ts

@@ -60,7 +60,6 @@ import {
 	ErrorDialogType,
 	MessageRole,
 	MessageType,
-	ReasoningEffort,
 	StreamConnectionState
 } from '$lib/enums';
 
@@ -2335,8 +2334,7 @@ class ChatStore {
 		if (currentConfig.excludeReasoningFromContext) apiOptions.excludeReasoningFromContext = true;
 
 		apiOptions.enableThinking = conversationsStore.getThinkingEnabled();
-		const effort = conversationsStore.getReasoningEffort();
-		if (effort !== ReasoningEffort.OFF) apiOptions.reasoningEffort = effort;
+		apiOptions.reasoningEffort = conversationsStore.getReasoningEffort();
 
 		if (hasValue(currentConfig.temperature))
 			apiOptions.temperature = Number(currentConfig.temperature);

tools/ui/src/lib/stores/conversations.svelte.ts

@@ -47,6 +47,7 @@ import {
 	NON_ALPHANUMERIC_REGEX,
 	MULTIPLE_UNDERSCORE_REGEX,
 	SETTINGS_KEYS,
+	THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY,
 	REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY
 } from '$lib/constants';
 
@@ -83,16 +84,11 @@ class ConversationsStore {
 	/** Pending MCP server overrides for new conversations (before first message) */
 	pendingMcpServerOverrides = $state<McpServerOverride[]>(ConversationsStore.loadMcpDefaults());
 
-	/** Global (non-conversation-specific) thinking toggle default, derived from reasoning effort */
-	pendingThinkingEnabled = $state(false);
+	/** Global (non-conversation-specific) thinking toggle default */
+	pendingThinkingEnabled = $state(ConversationsStore.loadThinkingDefaults());
 
 	/** Global (non-conversation-specific) reasoning effort default */
-	pendingReasoningEffort = $state<ReasoningEffort | ReasoningEffort.OFF>(
-		ConversationsStore.loadReasoningEffortDefault()
-	);
-
-	/** Last non-off reasoning effort, restored when re-enabling thinking globally */
-	private lastNonOffEffort: ReasoningEffort | null = null;
+	pendingReasoningEffort = $state<ReasoningEffort>(ConversationsStore.loadReasoningEffortDefault());
 
 	private static loadMcpDefaults(): McpServerOverride[] {
 		const raw = config()[SETTINGS_KEYS.MCP_DEFAULT_SERVER_OVERRIDES];
@@ -116,14 +112,35 @@ class ConversationsStore {
 		settingsStore.updateConfig(SETTINGS_KEYS.MCP_DEFAULT_SERVER_OVERRIDES, JSON.stringify(plain));
 	}
 
+	/** Load thinking-enabled default from localStorage */
+	private static loadThinkingDefaults(): boolean {
+		if (typeof globalThis.localStorage === 'undefined') return true;
+		try {
+			const raw = localStorage.getItem(THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY);
+			if (!raw) return true;
+			return raw === 'true';
+		} catch {
+			return true;
+		}
+	}
+
+	/** Persist thinking-enabled default to localStorage */
+	private saveThinkingDefaults(): void {
+		if (typeof globalThis.localStorage === 'undefined') return;
+		localStorage.setItem(
+			THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY,
+			this.pendingThinkingEnabled ? 'true' : 'false'
+		);
+	}
+
 	/** Load reasoning effort default from localStorage */
-	private static loadReasoningEffortDefault(): ReasoningEffort | ReasoningEffort.OFF {
-		if (typeof globalThis.localStorage === 'undefined') return ReasoningEffort.OFF;
+	private static loadReasoningEffortDefault(): ReasoningEffort {
+		if (typeof globalThis.localStorage === 'undefined') return ReasoningEffort.MEDIUM;
 		try {
 			const raw = localStorage.getItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY);
-			return (raw as ReasoningEffort | ReasoningEffort.OFF) || ReasoningEffort.OFF;
+			return (raw as ReasoningEffort) || ReasoningEffort.MEDIUM;
 		} catch {
-			return ReasoningEffort.OFF;
+			return ReasoningEffort.MEDIUM;
 		}
 	}
 
@@ -286,17 +303,10 @@ class ConversationsStore {
 			this.pendingMcpServerOverrides = [];
 		}
 
-		// Inherit global thinking/reasoning defaults into the new conversation
-		const thinkingEnabled = this.getThinkingEnabled();
-		conversation.thinkingEnabled = thinkingEnabled;
-		conversation.reasoningEffort =
-			this.pendingReasoningEffort === ReasoningEffort.OFF ? undefined : this.pendingReasoningEffort;
+		// Inherit global thinking default into the new conversation
+		conversation.thinkingEnabled = this.pendingThinkingEnabled;
 		await DatabaseService.updateConversation(conversation.id, {
-			thinkingEnabled,
-			reasoningEffort:
-				this.pendingReasoningEffort === ReasoningEffort.OFF
-					? undefined
-					: this.pendingReasoningEffort
+			thinkingEnabled: this.pendingThinkingEnabled
 		});
 
 		this.conversations = [conversation, ...this.conversations];
@@ -322,6 +332,7 @@ class ConversationsStore {
 			}
 
 			this.pendingMcpServerOverrides = [];
+			this.pendingThinkingEnabled = ConversationsStore.loadThinkingDefaults();
 			this.activeConversation = conversation;
 
 			if (conversation.currNode) {
@@ -352,7 +363,7 @@ class ConversationsStore {
 		this.activeMessages = [];
 		// reload defaults so new chats inherit persisted state
 		this.pendingMcpServerOverrides = ConversationsStore.loadMcpDefaults();
-		this.pendingReasoningEffort = ConversationsStore.loadReasoningEffortDefault();
+		this.pendingThinkingEnabled = ConversationsStore.loadThinkingDefaults();
 	}
 
 	/**
@@ -783,11 +794,9 @@ class ConversationsStore {
 	 */
 	getThinkingEnabled(): boolean {
 		if (this.activeConversation) {
-			if (this.activeConversation.thinkingEnabled !== undefined) {
-				return this.activeConversation.thinkingEnabled;
-			}
+			return this.activeConversation.thinkingEnabled ?? this.pendingThinkingEnabled;
 		}
-		return this.getReasoningEffort() !== ReasoningEffort.OFF;
+		return this.pendingThinkingEnabled;
 	}
 
 	/**
@@ -797,17 +806,8 @@ class ConversationsStore {
 	 */
 	async setThinkingEnabled(enabled: boolean): Promise<void> {
 		if (!this.activeConversation) {
-			if (enabled) {
-				const effort = this.lastNonOffEffort ?? ReasoningEffort.LOW;
-				this.pendingReasoningEffort = effort;
-				this.saveReasoningEffortDefaults();
-			} else {
-				if (this.pendingReasoningEffort !== ReasoningEffort.OFF) {
-					this.lastNonOffEffort = this.pendingReasoningEffort;
-				}
-				this.pendingReasoningEffort = ReasoningEffort.OFF;
-				this.saveReasoningEffortDefaults();
-			}
+			this.pendingThinkingEnabled = enabled;
+			this.saveThinkingDefaults();
 			return;
 		}
 
@@ -831,7 +831,7 @@ class ConversationsStore {
 	 * Gets the effective reasoning effort for the active conversation.
 	 * Returns the conversation override if set, otherwise the global default.
 	 */
-	getReasoningEffort(): ReasoningEffort | ReasoningEffort.OFF {
+	getReasoningEffort(): ReasoningEffort {
 		if (this.activeConversation) {
 			return this.activeConversation.reasoningEffort ?? this.pendingReasoningEffort;
 		}

tools/ui/src/lib/stores/mcp.svelte.ts

@@ -12,22 +12,21 @@
  * - Lifecycle management (initialize, shutdown)
  * - Multi-server coordination
  * - Tool name conflict detection and resolution
+ * - OpenAI-compatible tool definition generation
  * - Automatic tool-to-server routing
  * - Health checks
  *
- * MCP connection state and raw `Tool[]` per server are owned here; the
- * OpenAI-compatible wire format for those tools is built in `toolsStore`
- * (see {@link toolsStore.mcpEntries} / {@link toolsStore.getEnabledToolsForLLM}).
- *
  * @see MCPService in services/mcp.service.ts for protocol operations
  */
 
 import { browser } from '$app/environment';
+import { SvelteSet } from 'svelte/reactivity';
 import { SETTINGS_KEYS } from '$lib/constants';
 import { MCPService } from '$lib/services/mcp.service';
 import { config, settingsStore } from '$lib/stores/settings.svelte';
 import { mcpResourceStore } from '$lib/stores/mcp-resources.svelte';
 import { serverStore } from '$lib/stores/server.svelte';
+import { conversationsStore } from '$lib/stores/conversations.svelte';
 import { mode } from 'mode-watcher';
 import {
 	parseMcpServerSettings,
@@ -41,7 +40,9 @@ import {
 	HealthCheckStatus,
 	MCPRefType,
 	ColorMode,
-	UrlProtocol
+	UrlProtocol,
+	JsonSchemaType,
+	ToolCallType
 } from '$lib/enums';
 import {
 	DEFAULT_CACHE_TTL_MS,
@@ -52,10 +53,12 @@ import {
 	MCP_RECONNECT_BACKOFF_MULTIPLIER,
 	MCP_RECONNECT_INITIAL_DELAY,
 	MCP_RECONNECT_MAX_DELAY,
-	MCP_RECONNECT_ATTEMPT_TIMEOUT_MS
+	MCP_RECONNECT_ATTEMPT_TIMEOUT_MS,
+	RECOMMENDED_MCP_SERVER_IDS
 } from '$lib/constants';
 import type {
 	MCPToolCall,
+	OpenAIToolDefinition,
 	ServerStatus,
 	ToolExecutionResult,
 	MCPClientConfig,
@@ -579,10 +582,30 @@ class MCPStore {
 	}
 
 	/**
-	 * MCP servers selectable in chat-add UIs and the settings page.
+	 * Recommended MCP server IDs the user opted in to via per-chat overrides.
+	 * Single source of truth for "which recommendations has the user accepted",
+	 * shared by the recommendations hook and the visible-servers getter.
+	 */
+	get optedInRecommendationIds(): ReadonlySet<string> {
+		const ids = new SvelteSet<string>();
+		for (const override of conversationsStore.pendingMcpServerOverrides) {
+			if (RECOMMENDED_MCP_SERVER_IDS.has(override.serverId) && override.enabled) {
+				ids.add(override.serverId);
+			}
+		}
+		return ids;
+	}
+
+	/**
+	 * MCP servers selectable in chat-add UIs and the settings page:
+	 * enabled in settings and either non-recommended or explicitly opted in.
 	 */
 	get visibleMcpServers(): MCPServerSettingsEntry[] {
-		return this.getServersSorted().filter((server) => server.enabled);
+		const optedIn = this.optedInRecommendationIds;
+		return this.getServersSorted().filter(
+			(server) =>
+				server.enabled && (!RECOMMENDED_MCP_SERVER_IDS.has(server.id) || optedIn.has(server.id))
+		);
 	}
 
 	async ensureInitialized(perChatOverrides?: McpServerOverride[]): Promise<boolean> {
@@ -956,6 +979,73 @@ class MCPStore {
 		}
 	}
 
+	getToolDefinitionsForLLM(): OpenAIToolDefinition[] {
+		const tools: OpenAIToolDefinition[] = [];
+
+		for (const connection of this.connections.values()) {
+			for (const tool of connection.tools) {
+				const rawSchema = (tool.inputSchema as Record<string, unknown>) ?? {
+					type: JsonSchemaType.OBJECT,
+					properties: {},
+					required: []
+				};
+
+				tools.push({
+					type: ToolCallType.FUNCTION as const,
+					function: {
+						name: tool.name,
+						description: tool.description,
+						parameters: this.normalizeSchemaProperties(rawSchema)
+					}
+				});
+			}
+		}
+
+		return tools;
+	}
+
+	private normalizeSchemaProperties(schema: Record<string, unknown>): Record<string, unknown> {
+		if (!schema || typeof schema !== 'object') {
+			return schema;
+		}
+
+		const normalized = { ...schema };
+		if (normalized.properties && typeof normalized.properties === 'object') {
+			const props = normalized.properties as Record<string, Record<string, unknown>>;
+			const normalizedProps: Record<string, Record<string, unknown>> = {};
+			for (const [key, prop] of Object.entries(props)) {
+				if (!prop || typeof prop !== 'object') {
+					normalizedProps[key] = prop;
+					continue;
+				}
+				const normalizedProp = { ...prop };
+				if (!normalizedProp.type && normalizedProp.default !== undefined) {
+					const defaultVal = normalizedProp.default;
+					if (typeof defaultVal === 'string') normalizedProp.type = 'string';
+					else if (typeof defaultVal === 'number')
+						normalizedProp.type = Number.isInteger(defaultVal) ? 'integer' : 'number';
+					else if (typeof defaultVal === 'boolean') normalizedProp.type = 'boolean';
+					else if (Array.isArray(defaultVal)) normalizedProp.type = 'array';
+					else if (typeof defaultVal === 'object' && defaultVal !== null)
+						normalizedProp.type = 'object';
+				}
+				if (normalizedProp.properties)
+					Object.assign(
+						normalizedProp,
+						this.normalizeSchemaProperties(normalizedProp as Record<string, unknown>)
+					);
+				if (normalizedProp.items && typeof normalizedProp.items === 'object')
+					normalizedProp.items = this.normalizeSchemaProperties(
+						normalizedProp.items as Record<string, unknown>
+					);
+				normalizedProps[key] = normalizedProp;
+			}
+			normalized.properties = normalizedProps;
+		}
+
+		return normalized;
+	}
+
 	getToolNames(): string[] {
 		return Array.from(this.toolsIndex.keys());
 	}

tools/ui/src/lib/stores/tools.svelte.ts

@@ -13,6 +13,33 @@ import {
 import { SvelteMap, SvelteSet } from 'svelte/reactivity';
 
 /** Stable selection identity for a tool, shared by the disabled set and the permission store */
+function toolKey(source: ToolSource, name: string, serverId?: string): string {
+	switch (source) {
+		case ToolSource.MCP:
+			return serverId ? `mcp-${serverId}:${name}` : `mcp:${name}`;
+		case ToolSource.CUSTOM:
+			return `custom:${name}`;
+		case ToolSource.FRONTEND:
+			return `frontend:${name}`;
+		default:
+			return `builtin:${name}`;
+	}
+}
+
+function mcpDefinition(
+	name: string,
+	description: string | undefined,
+	schema?: Record<string, unknown>
+): OpenAIToolDefinition {
+	return {
+		type: ToolCallType.FUNCTION,
+		function: {
+			name,
+			description,
+			parameters: schema ?? { type: JsonSchemaType.OBJECT, properties: {}, required: [] }
+		}
+	};
+}
 
 class ToolsStore {
 	private _builtinTools = $state<OpenAIToolDefinition[]>([]);
@@ -50,96 +77,12 @@ class ToolsStore {
 		}
 	}
 
-	private toolKey(source: ToolSource, name: string, serverId?: string): string {
-		switch (source) {
-			case ToolSource.MCP:
-				return serverId ? `mcp-${serverId}:${name}` : `mcp:${name}`;
-			case ToolSource.CUSTOM:
-				return `custom:${name}`;
-			case ToolSource.FRONTEND:
-				return `frontend:${name}`;
-			default:
-				return `builtin:${name}`;
-		}
-	}
-
-	private inferTypeFromDefault(value: unknown): string | undefined {
-		if (typeof value === 'string') return 'string';
-		if (typeof value === 'boolean') return 'boolean';
-		if (typeof value === 'number') return Number.isInteger(value) ? 'integer' : 'number';
-		if (Array.isArray(value)) return 'array';
-		if (value !== null && typeof value === 'object') return 'object';
-		return undefined;
-	}
-
-	/**
-	 * Recursively normalize a JSON Schema object: infers `type` from `default`
-	 * for properties / items that omit it, and descends into nested `properties`
-	 * and `items`. Returns a new object -- does not mutate the input.
-	 */
-	private normalizeJsonSchema(schema: Record<string, unknown>): Record<string, unknown> {
-		if (!schema || typeof schema !== 'object') return schema;
-
-		const normalized: Record<string, unknown> = { ...schema };
-
-		if (normalized.properties && typeof normalized.properties === 'object') {
-			const props = normalized.properties as Record<string, Record<string, unknown>>;
-			const normalizedProps: Record<string, Record<string, unknown>> = {};
-			for (const [key, prop] of Object.entries(props)) {
-				if (!prop || typeof prop !== 'object') {
-					normalizedProps[key] = prop;
-					continue;
-				}
-
-				const normalizedProp: Record<string, unknown> = { ...prop };
-
-				if (!normalizedProp.type && normalizedProp.default !== undefined) {
-					const inferred = this.inferTypeFromDefault(normalizedProp.default);
-					if (inferred) normalizedProp.type = inferred;
-				}
-
-				if (normalizedProp.properties) {
-					Object.assign(
-						normalizedProp,
-						this.normalizeJsonSchema(normalizedProp as Record<string, unknown>)
-					);
-				}
-
-				if (normalizedProp.items && typeof normalizedProp.items === 'object') {
-					normalizedProp.items = this.normalizeJsonSchema(
-						normalizedProp.items as Record<string, unknown>
-					);
-				}
-
-				normalizedProps[key] = normalizedProp;
-			}
-			normalized.properties = normalizedProps;
-		}
-
-		return normalized;
-	}
-
-	private mcpDefinition(
-		name: string,
-		description: string | undefined,
-		schema?: Record<string, unknown>
-	): OpenAIToolDefinition {
-		return {
-			type: ToolCallType.FUNCTION,
-			function: {
-				name,
-				description,
-				parameters: schema ?? { type: JsonSchemaType.OBJECT, properties: {}, required: [] }
-			}
-		};
-	}
-
 	get builtinTools(): OpenAIToolDefinition[] {
 		return this._builtinTools;
 	}
 
 	get mcpTools(): OpenAIToolDefinition[] {
-		return this.mcpEntries().map((e) => e.definition);
+		return mcpStore.getToolDefinitionsForLLM();
 	}
 
 	get frontendTools(): OpenAIToolDefinition[] {
@@ -181,22 +124,11 @@ class ToolsStore {
 			for (const [serverId, connection] of connections) {
 				const serverName = mcpStore.getServerDisplayName(serverId);
 				for (const tool of connection.tools) {
-					const rawSchema = (tool.inputSchema as Record<string, unknown>) ?? {
-						type: JsonSchemaType.OBJECT,
-						properties: {},
-						required: []
-					};
+					const schema = (tool.inputSchema as Record<string, unknown>) ?? undefined;
 					out.push({
 						serverId,
 						serverName,
-						definition: {
-							type: ToolCallType.FUNCTION,
-							function: {
-								name: tool.name,
-								description: tool.description,
-								parameters: this.normalizeJsonSchema(rawSchema)
-							}
-						}
+						definition: mcpDefinition(tool.name, tool.description, schema)
 					});
 				}
 			}
@@ -206,7 +138,7 @@ class ToolsStore {
 					out.push({
 						serverId,
 						serverName,
-						definition: this.mcpDefinition(tool.name, tool.description)
+						definition: mcpDefinition(tool.name, tool.description)
 					});
 				}
 			}
@@ -228,18 +160,14 @@ class ToolsStore {
 
 		for (const def of this._builtinTools) {
 			const name = def.function.name;
-			push({
-				source: ToolSource.BUILTIN,
-				key: this.toolKey(ToolSource.BUILTIN, name),
-				definition: def
-			});
+			push({ source: ToolSource.BUILTIN, key: toolKey(ToolSource.BUILTIN, name), definition: def });
 		}
 
 		for (const def of this.frontendTools) {
 			const name = def.function.name;
 			push({
 				source: ToolSource.FRONTEND,
-				key: this.toolKey(ToolSource.FRONTEND, name),
+				key: toolKey(ToolSource.FRONTEND, name),
 				definition: def
 			});
 		}
@@ -250,18 +178,14 @@ class ToolsStore {
 				source: ToolSource.MCP,
 				serverId,
 				serverName,
-				key: this.toolKey(ToolSource.MCP, name, serverId),
+				key: toolKey(ToolSource.MCP, name, serverId),
 				definition
 			});
 		}
 
 		for (const def of this.customTools) {
 			const name = def.function.name;
-			push({
-				source: ToolSource.CUSTOM,
-				key: this.toolKey(ToolSource.CUSTOM, name),
-				definition: def
-			});
+			push({ source: ToolSource.CUSTOM, key: toolKey(ToolSource.CUSTOM, name), definition: def });
 		}
 
 		return entries;
@@ -309,8 +233,7 @@ class ToolsStore {
 
 	/**
 	 * Enabled tool definitions for sending to the LLM.
-	 * MCP tool schemas are normalized here so the wire payload is consistent
-	 * across all four sources (built-in, frontend/sandbox, MCP, custom JSON).
+	 * MCP tools keep their normalized schemas from mcpStore.
 	 * The API identifies tools by name, so a name is sent at most once.
 	 */
 	getEnabledToolsForLLM(): OpenAIToolDefinition[] {
@@ -333,8 +256,7 @@ class ToolsStore {
 
 		for (const def of this._builtinTools) take(def);
 		for (const def of this.frontendTools) take(def);
-		// mcpEntries() over mcpStore directly so wire shape stays normalized and aligned with the tools UI.
-		for (const entry of this.mcpEntries()) take(entry.definition);
+		for (const def of mcpStore.getToolDefinitionsForLLM()) take(def);
 		for (const def of this.customTools) take(def);
 
 		return result;
@@ -386,17 +308,15 @@ class ToolsStore {
 		const connection = mcpStore.getConnections().get(serverId);
 		if (!connection) return;
 		for (const tool of connection.tools) {
-			this._disabledTools.delete(this.toolKey(ToolSource.MCP, tool.name, serverId));
+			this._disabledTools.delete(toolKey(ToolSource.MCP, tool.name, serverId));
 		}
 		this.persistDisabledTools();
 	}
 
 	toggleGroup(group: ToolGroup): void {
 		const allEnabled = group.tools.every((t) => this.isToolEnabled(t.key));
-		const target = !allEnabled;
 		for (const tool of group.tools) {
-			if (target) this._disabledTools.delete(tool.key);
-			else this._disabledTools.add(tool.key);
+			this.setToolEnabled(tool.key, !allEnabled);
 		}
 		this.persistDisabledTools();
 	}
@@ -412,7 +332,7 @@ class ToolsStore {
 		tools: { name: string; description?: string }[];
 	}[] {
 		const result: ReturnType<ToolsStore['getMcpToolsFromHealthChecks']> = [];
-		for (const server of mcpStore.visibleMcpServers) {
+		for (const server of mcpStore.getServersSorted().filter((s) => s.enabled)) {
 			const health = mcpStore.getHealthCheckState(server.id);
 			if (health.status === HealthCheckStatus.SUCCESS && health.tools.length > 0) {
 				result.push({

tools/ui/src/lib/utils/branching.ts

@@ -111,7 +111,15 @@ function findLeafNodeInMap(
 }
 
 /**
- * Convenience wrapper around {@link findLeafNodeInMap} for callers that have a flat message array.
+ * Convenience wrapper around {@link findLeafNodeInMap} for callers that only have
+ * a flat message array.
+ *
+ * Finds the leaf node (message with no children) for a given message branch.
+ * Traverses down the tree following the last child until reaching a leaf.
+ *
+ * @param messages - All messages in the conversation
+ * @param messageId - Starting message ID to find leaf for
+ * @returns The leaf node ID, or the original messageId if no children
  */
 export function findLeafNode(messages: readonly DatabaseMessage[], messageId: string): string {
 	const nodeMap = new Map(messages.map((msg) => [msg.id, msg] as const));
@@ -217,6 +225,7 @@ export function getMessageSiblings(
 
 /**
  * Builds sibling information for every message in a conversation.
+ * A single node map is shared across all lookups for O(1) access.
  *
  * @param messages - All messages in the conversation
  * @returns Map of message ID to its sibling information