From 2cfdb5fc08a81f6c95519fc5a60a86943a05c94c Mon Sep 17 00:00:00 2001 From: Markus Tavenrath Date: Thu, 20 Aug 2026 08:52:28 +0200 Subject: [PATCH 01/44] vulkan : add source groups for shaders (#26666) --- ggml/src/ggml-vulkan/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ggml/src/ggml-vulkan/CMakeLists.txt b/ggml/src/ggml-vulkan/CMakeLists.txt index 1dc6a145d..e733ad5cc 100644 --- a/ggml/src/ggml-vulkan/CMakeLists.txt +++ b/ggml/src/ggml-vulkan/CMakeLists.txt @@ -200,8 +200,11 @@ if (Vulkan_FOUND) set (_ggml_vk_header "${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan-shaders.hpp") set (_ggml_vk_input_dir "${CMAKE_CURRENT_SOURCE_DIR}/vulkan-shaders") set (_ggml_vk_output_dir "${CMAKE_CURRENT_BINARY_DIR}/vulkan-shaders.spv") + set (_ggml_vk_generated_shader_files ${_ggml_vk_header}) file(GLOB _ggml_vk_shader_files CONFIGURE_DEPENDS "${_ggml_vk_input_dir}/*.comp") + set_source_files_properties(${_ggml_vk_shader_files} PROPERTIES HEADER_FILE_ONLY TRUE) + target_sources(ggml-vulkan PRIVATE ${_ggml_vk_shader_files}) # Because external projects do not provide source-level tracking, # the vulkan-shaders-gen sources need to be explicitly added to @@ -241,8 +244,11 @@ if (Vulkan_FOUND) COMMENT "Generate vulkan shaders for ${file}" ) target_sources(ggml-vulkan PRIVATE ${_ggml_vk_target_cpp}) + list(APPEND _ggml_vk_generated_shader_files ${_ggml_vk_target_cpp}) endforeach() + source_group("Vulkan shaders" FILES ${_ggml_vk_shader_files}) + source_group("Generated Vulkan shaders" FILES ${_ggml_vk_generated_shader_files}) else() message(WARNING "Vulkan not found") endif() From f466cfa38fac99e80a2aa4b58b3203b33872fe9c Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 10:00:16 +0300 Subject: [PATCH 02/44] spec : avoid binding reference to null pointer (#27404) --- common/speculative.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/speculative.cpp b/common/speculative.cpp index ae55e357d..89e9b2782 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2649,6 +2649,10 @@ void common_speculative_draft(common_speculative * spec) { for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) dparams.size(); ++seq_id) { auto & dp = dparams[seq_id]; + if (!dp.drafting) { + continue; + } + auto & result = *dp.result; // a new draft has been sampled From 929d47a39163d67a2808413eaf8916097c4bb53c Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 10:00:35 +0300 Subject: [PATCH 03/44] graph : create V as a view of K in the k_iswa build_attn (#27392) build_attn with the llm_graph_input_attn_k_iswa input was using the cached K tensor itself as V. Create V as a view of K (the first v_cur->ne[0] elements of each row), like the other K-only build_attn overloads. The deepseek4 MTP call site now passes the kv tensor as v_cur. Assisted-by: pi:llama.cpp/Qwen3.8-27B --- src/llama-graph.cpp | 4 +--- src/models/deepseek4.cpp | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 1896758c5..5212e19a2 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -3099,8 +3099,6 @@ ggml_tensor * llm_graph_context::build_attn( int il) const { const bool is_swa = hparams.is_swa(il); - GGML_UNUSED(v_cur); - auto * k_rot = is_swa ? inp->self_k_rot_swa : inp->self_k_rot; if (k_rot) { @@ -3133,7 +3131,7 @@ ggml_tensor * llm_graph_context::build_attn( // MLA-style attention: the cached K is used as V ggml_tensor * q = q_cur; ggml_tensor * k = mctx_cur->get_k(ctx0, il); - ggml_tensor * v = k; + ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0); ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il); cb(cur, "kqv_out", il); diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 89cd46176..366ca2e54 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -1225,7 +1225,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl( if (inp_mtp) { out = build_attn(inp_mtp, nullptr, nullptr, nullptr, - q, kv, nullptr, + q, kv, kv, nullptr, layer.attn_sinks, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); cb(out, "attn_raw", il); From d9b6be07d0864ab09417b17ba36f9788087dd22c Mon Sep 17 00:00:00 2001 From: Alexander Heisler <126129661+heislera763@users.noreply.github.com> Date: Thu, 20 Aug 2026 03:27:51 -0400 Subject: [PATCH 04/44] ggml-cuda: provide static workspace for cuBLAS handles (#26574) * provide static workspace for cuBLAS handles * account for concurrent streams when using GGML_CUDA_GRAPH_OPT * drop cublas_handle overloads and remove direct cublasSetStream calls * Update ggml/src/ggml-cuda/common.cuh --------- Co-authored-by: Oliver Simons --- ggml/src/ggml-cuda/common.cuh | 29 ++++++++++++++++++----------- ggml/src/ggml-cuda/ggml-cuda.cu | 19 +++++++++++-------- ggml/src/ggml-cuda/out-prod.cu | 2 -- ggml/src/ggml-cuda/solve_tri.cu | 8 +++----- ggml/src/ggml-cuda/ssm-scan.cu | 1 - 5 files changed, 32 insertions(+), 27 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index d27d8acb1..14dd1098c 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1418,7 +1418,9 @@ struct ggml_backend_cuda_context { cudaEvent_t copy_event = nullptr; cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } }; - cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr}; + cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {nullptr}; + void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {nullptr}; + size_t cublas_workspace_sizes[GGML_CUDA_MAX_DEVICES] = {0}; int curr_stream_no = 0; @@ -1495,17 +1497,22 @@ struct ggml_backend_cuda_context { ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; } - cublasHandle_t cublas_handle(int device) { - if (cublas_handles[device] == nullptr) { - ggml_cuda_set_device(device); - CUBLAS_CHECK(cublasCreate(&cublas_handles[device])); - CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device], CUBLAS_TF32_TENSOR_OP_MATH)); - } - return cublas_handles[device]; - } - cublasHandle_t cublas_handle() { - return cublas_handle(device); + if (cublas_handles[device][curr_stream_no] == nullptr) { + ggml_cuda_set_device(device); + CUBLAS_CHECK(cublasCreate(&cublas_handles[device][curr_stream_no])); + CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device][curr_stream_no], CUBLAS_TF32_TENSOR_OP_MATH)); + CUBLAS_CHECK(cublasSetStream(cublas_handles[device][curr_stream_no], stream())); +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUBLAS_VER_MAJOR > 11 || (CUBLAS_VER_MAJOR == 11 && CUBLAS_VER_MINOR >= 2)) + if (cublas_workspace_sizes[device] == 0) { + const int cc = ggml_cuda_info().devices[device].cc; + cublas_workspace_sizes[device] = (cc >= GGML_CUDA_CC_HOPPER) ? 32 * 1024 * 1024 : 4 * 1024 * 1024; + } + CUDA_CHECK(cudaMalloc(&cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); + CUBLAS_CHECK(cublasSetWorkspace(cublas_handles[device][curr_stream_no], cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); +#endif + } + return cublas_handles[device][curr_stream_no]; } // pool diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 3b2a0ea85..a8a1c09ca 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -711,9 +711,12 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() { if (streams[i][j] != nullptr) { CUDA_CHECK(cudaStreamDestroy(streams[i][j])); } - } - if (cublas_handles[i] != nullptr) { - CUBLAS_CHECK(cublasDestroy(cublas_handles[i])); + if (cublas_handles[i][j] != nullptr) { + CUBLAS_CHECK(cublasDestroy(cublas_handles[i][j])); + } + if (cublas_workspaces[i][j] != nullptr) { + CUDA_CHECK(cudaFree(cublas_workspaces[i][j])); + } } } } @@ -1416,7 +1419,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const const int64_t ne_dst = ggml_nelements(dst); cudaStream_t main_stream = ctx.stream(); - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(), main_stream)); + cublasHandle_t cublas_h = ctx.cublas_handle(); const size_t src0_ts = ggml_type_size(src0->type); GGML_ASSERT(nb00 == src0_ts); @@ -1539,14 +1542,14 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const // probably because the internal kernel selection logic is suboptimal. if (compute_type == GGML_TYPE_F32 && ne12 == 1 && ne13 == 1) { CUBLAS_CHECK( - cublasSgemm(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasSgemm(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, (const float *) alpha, (const float *) src0_ptr, s01, (const float *) src1_ptr, s11, (const float *) beta, (float *) dst_ptr, ne0)); } else if (ne12 == 1 && ne13 == 1) { CUBLAS_CHECK( - cublasGemmEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, src0_ptr, cu_data_type_a, s01, src1_ptr, cu_data_type_b, s11, @@ -1561,7 +1564,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const // there is no broadcast and src0, src1 are contiguous across dims 2, 3 // use cublasGemmStridedBatchedEx CUBLAS_CHECK( - cublasGemmStridedBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmStridedBatchedEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, src0_ptr, cu_data_type_a, s01, sma, // strideA src1_ptr, cu_data_type_b, s11, smb, // strideB @@ -1599,7 +1602,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const CUDA_CHECK(cudaGetLastError()); CUBLAS_CHECK( - cublasGemmBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmBatchedEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, (const void **) (ptrs_src.get() + 0*ne23), cu_data_type_a, s01, (const void **) (ptrs_src.get() + 1*ne23), cu_data_type_b, s11, diff --git a/ggml/src/ggml-cuda/out-prod.cu b/ggml/src/ggml-cuda/out-prod.cu index 46b9f3a67..c46e0455d 100644 --- a/ggml/src/ggml-cuda/out-prod.cu +++ b/ggml/src/ggml-cuda/out-prod.cu @@ -54,8 +54,6 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const float alpha = 1.0f; const float beta = 0.0f; - CUBLAS_CHECK(cublasSetStream(handle, stream)); - const int64_t lda = nb01 / sizeof(float); const int64_t ldc = nb1 / sizeof(float); diff --git a/ggml/src/ggml-cuda/solve_tri.cu b/ggml/src/ggml-cuda/solve_tri.cu index 07ca33f51..d96783420 100644 --- a/ggml/src/ggml-cuda/solve_tri.cu +++ b/ggml/src/ggml-cuda/solve_tri.cu @@ -65,15 +65,13 @@ static void solve_tri_f32_cublas(ggml_backend_cuda_context & ctx, get_batch_pointers<<<(total_batches + 255) / 256, 256, 0, stream>>>(A, X, A_ptrs_dev, X_ptrs_dev, ne02, total_batches, s02, s03, s2, s3); - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream)); - // Yes, this is necessary, without this we get RMSE errors - CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(id), CUBLAS_DEFAULT_MATH)); - CUBLAS_CHECK(cublasStrsmBatched(ctx.cublas_handle(id), CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N, + CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(), CUBLAS_DEFAULT_MATH)); + CUBLAS_CHECK(cublasStrsmBatched(ctx.cublas_handle(), CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N, CUBLAS_DIAG_NON_UNIT, k, n, &alpha, A_ptrs_dev, n, X_ptrs_dev, k, total_batches)); // revert to standard mode from common.cuh - CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(id), CUBLAS_TF32_TENSOR_OP_MATH)); + CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(), CUBLAS_TF32_TENSOR_OP_MATH)); GGML_UNUSED_VARS(s12, s13); } diff --git a/ggml/src/ggml-cuda/ssm-scan.cu b/ggml/src/ggml-cuda/ssm-scan.cu index ef342f01f..40cb38dee 100644 --- a/ggml/src/ggml-cuda/ssm-scan.cu +++ b/ggml/src/ggml-cuda/ssm-scan.cu @@ -632,7 +632,6 @@ static void ssm_scan_ssd_f32_cuda( // Step 3: chunked SSD loop // Per chunk: pre_matmul (incl. M) + 4 cuBLAS (CB, Y, S@C, state update) + scale_state cublasHandle_t handle = ctx.cublas_handle(); - CUBLAS_CHECK(cublasSetStream(handle, stream)); const float alpha_one = 1.0f; const float beta_zero = 0.0f; const float beta_one = 1.0f; From a3b1effcda84caeb180427b1346d0212841418f6 Mon Sep 17 00:00:00 2001 From: Rock Chen Date: Thu, 20 Aug 2026 15:35:28 +0800 Subject: [PATCH 05/44] convert: fix get block count error for Nemotron 3 Ultra (#27101) * convert: fix get block count error for Nemotron Signed-off-by: Rock Chen * fix this in NemotronHModel.__init__ instead. This reverts commit ca689cbc8792ba69ea1fd5d8b3ae0485c47536ec. --------- Signed-off-by: Rock Chen --- conversion/nemotron.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/conversion/nemotron.py b/conversion/nemotron.py index 3e37c7b46..e5d167185 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -207,7 +207,9 @@ class NemotronHModel(GraniteHybridModel): # calling the parent __init__. This is because the parent constructor # uses self.model_arch to build the tensor name map, and all MoE-specific # mappings would be missed if it were called with the default non-MoE arch. - hparams = ModelBase.load_hparams(args[0], self.is_mistral_format) + hparams = kwargs.pop("hparams", None) + if hparams is None: + hparams = ModelBase.load_hparams(args[0], self.is_mistral_format) has_moe_params = ( "num_experts_per_tok" in hparams or (isinstance(hparams.get("llm_config"), dict) and "num_experts_per_tok" in hparams["llm_config"]) @@ -215,8 +217,11 @@ class NemotronHModel(GraniteHybridModel): if has_moe_params: self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE self.is_moe = True + layers_block_type = hparams.get("layers_block_type") + if layers_block_type is not None: + hparams["num_hidden_layers"] = len(layers_block_type) - super().__init__(*args, **kwargs) + super().__init__(*args, hparams=hparams, **kwargs) # Save the top-level head_dim for later self.head_dim = self.hparams.get("head_dim", self.hparams.get("attention_head_dim")) From 849798132173c3c511dffe3a03c3c760d707b05f Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Thu, 20 Aug 2026 10:42:33 +0200 Subject: [PATCH 06/44] ggml: fix backend split scheduler race condition (#26040) * ggml: fix backend split scheduler race condition splits without input were running concurrently with other splits, while potentially reusing memory the other split is accessing * only sync when split has no inputs --- ggml/src/ggml-backend.cpp | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index d5ba5b5ce..e519bdf50 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -1601,11 +1601,23 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s std::vector ids; std::vector used_ids; + int prev_backend_id = -1; + for (int split_id = 0; split_id < sched->n_splits; split_id++) { struct ggml_backend_sched_split * split = &splits[split_id]; int split_backend_id = split->backend_id; ggml_backend_t split_backend = sched->backends[split_backend_id]; + // ensure the previous split's async work has completed before we start + // this split, the allocator may have reused buffer regions across splits + if (split->n_inputs == 0 && prev_backend_id >= 0 && prev_backend_id != split_backend_id) { + if (sched->events[prev_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_synchronize(sched->events[prev_backend_id][sched->cur_copy]); + } else { + ggml_backend_synchronize(sched->backends[prev_backend_id]); + } + } + // copy the input tensors to the split backend for (int input_id = 0; input_id < split->n_inputs; input_id++) { ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]); @@ -1768,12 +1780,12 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s } } - // record the event of this copy - if (split->n_inputs > 0) { - if (sched->events[split_backend_id][sched->cur_copy] != NULL) { - ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend); - } + // record the event of this split + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend); } + + prev_backend_id = split_backend_id; } return GGML_STATUS_SUCCESS; From f20395dae59ba30ab0a10e0e0b0db6eeb8e8a282 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 13:35:15 +0300 Subject: [PATCH 07/44] Revert "tensor-split meta backend fixes (#26502)" (#27433) This reverts commit d59d455fd8ea09e5a2e87ce2a9d668267ffb5ccd. --- ggml/src/ggml-backend-impl.h | 1 - ggml/src/ggml-backend-meta.cpp | 20 ++------------------ ggml/src/ggml-backend.cpp | 2 -- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h index 40cea024c..9c56ec30c 100644 --- a/ggml/src/ggml-backend-impl.h +++ b/ggml/src/ggml-backend-impl.h @@ -83,7 +83,6 @@ extern "C" { GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers); GGML_API bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer); GGML_API void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); - GGML_API void ggml_backend_meta_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); // // Backend (meta) diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 775ae9926..7654ea1f3 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -1118,6 +1118,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( } static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync) { + GGML_ASSERT(ggml_backend_buffer_is_meta(tensor->buffer)); ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) tensor->buffer->context; return ggml_backend_meta_get_split_state(buf_ctx->get_simple_tensor_container(tensor), tensor, assume_sync); } @@ -1177,15 +1178,7 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m t_ij->flags = tensor->flags; memcpy(t_ij->op_params, tensor->op_params, sizeof(tensor->op_params)); ggml_set_name(t_ij, tensor->name); - t_ij->buffer = simple_buf; - if (simple_buf) { - // the backend that owns the buffer will set .extra - ggml_backend_buffer_init_tensor(simple_buf, t_ij); - } else { - t_ij->extra = tensor->extra; - } - t_ij->view_src = tensor->view_src; t_ij->view_offs = tensor->view_offs; if (t_ij->view_src != nullptr && ggml_backend_buffer_is_meta(t_ij->view_src->buffer)) { @@ -1216,6 +1209,7 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m t_ij->data = (char *) ggml_backend_buffer_get_base(simple_buf) + size_t(tensor->data) - size_t(ggml_backend_buffer_get_base(tensor->buffer)); } + t_ij->extra = tensor->extra; for (int i = 0; i < GGML_MAX_SRC; i++) { t_ij->src[i] = tensor->src[i]; if (tensor->src[i] == tensor) { @@ -1508,16 +1502,6 @@ bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) { return buf != nullptr && buf->iface.free_buffer == ggml_backend_meta_buffer_iface.free_buffer; } -void ggml_backend_meta_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) { - GGML_ASSERT(ggml_backend_buffer_is_meta(buffer)); - ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context; - for (size_t i = 0; i < buf_ctx->bufs.size(); i++) { - if (buf_ctx->bufs[i]) { - ggml_backend_buffer_set_usage(buf_ctx->bufs[i].get(), usage); - } - } -} - static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft); diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index e519bdf50..3d6310f3f 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -182,8 +182,6 @@ void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backe // FIXME: add a generic callback to the buffer interface if (ggml_backend_buffer_is_multi_buffer(buffer)) { ggml_backend_multi_buffer_set_usage(buffer, usage); - } else if (ggml_backend_buffer_is_meta(buffer)) { - ggml_backend_meta_buffer_set_usage(buffer, usage); } } From 70aff25250075bf23b533c207b55168a4f926350 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 13:43:59 +0300 Subject: [PATCH 08/44] metal : dequantize quantized KV to F16 before flash attention (#27390) * metal: dequantize q8_0 KV to f16 before flash attention Add a preprocessing pass for GGML_OP_FLASH_ATTN_EXT on the Metal backend: when the KV cache is quantized (Q8_0 for now), dequantize K and V into a contiguous F16 scratch buffer and run the existing F16 flash attention kernels on it, instead of the in-kernel dequantization path. - new kernel kernel_flash_attn_ext_dequant_to_f16: one thread per quant block (K then V), stride-aware so permuted KV is supported; instantiated for Q8_0 (extending to Q4_0/Q4_1/Q5_0/Q5_1 is one instantiation + one gate case) - the gate is type-only: dequantize whenever the KV is quantized, regardless of head sizes, GQA ratio or n_kv; the attention kernels themselves are untouched - the F16 copies live in the op's own scratch allocation (ggml_metal_op_flash_attn_ext_extra_dequant_f16); the KV pad kernel reads the dequantized buffers when the path is active - the FA pipeline getters gain a use_f16_kv flag selecting the existing f16 kernels and contiguous strides - ref: https://github.com/ggml-org/llama.cpp/pull/25556 Verification (M2 Ultra): - test-backend-ops test -o FLASH_ATTN_EXT: 4798/4798 pass, including the new q8_0 eval cases (decode/prompt, permuted, sinks+ALiBi+softcap, kv=113 pad path, kv=16384) - llama-perplexity on Qwen2.5-0.5B with -ctk q8_0 -ctv q8_0 matches the f16 KV reference (PPL 1.0008 vs 1.0008) Assisted-by: pi:llama.cpp/Qwen3.8-27B * metal : launch the FA KV dequant kernel separately for K and V Simplify kernel_flash_attn_ext_dequant_to_f16: it now dequantizes a single tensor (its own ne/nb and dst) with no is_v branching, and the op dispatches it twice with the same pipeline - once for K and once for V. The kargs struct shrinks to a single ne/nb set plus nblocks. Assisted-by: pi:llama.cpp/Qwen3.8-27B * metal : dequantize q4_0, q4_1, q5_0 and q5_1 KV to f16 before flash attention The dequant pass now covers all quantized KV types supported by the Metal flash attention kernels. The dequant kernel, kargs, scratch allocation and dispatch are type-generic, so each type is one kernel instantiation plus one gate case. Assisted-by: pi:llama.cpp/Qwen3.8-27B * metal : skip the redundant V dequant when V is a view of K In MLA-based models, the V of the FA op is a view of K (the first ne20 elements of each K row); the dequantized V is then a view of the dequantized K, so skip the second dequant dispatch, do not reserve the V scratch region, and let the pad and attention kernels read V from the K F16 buffer with K's strides. The detection follows the CUDA backend: V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs)) Also fix the FA pipeline getters: ns10/ns20 are function constants baked into the kernels and must be the actual K/V row widths as seen by the kernel. The dispatch now passes them explicitly (nb11_attn/nb10_attn, nb21_attn/nb20_attn) instead of the getters assuming contiguous F16 KV (ns20 = dv), which was wrong when V is read from K with K's row pitch (e.g. 576 vs 512). New test cases: 576/512 q8_0 (MLA shape, V is a view of K) at kv=113 (KV pad), nb=1 (vec) and nb=64 (non-vec). Assisted-by: pi:llama.cpp/Qwen3.8-27B * test : remove backend-specific wording from test-backend-ops comments Assisted-by: pi:llama.cpp/Qwen3.8-27B * pi : avoid backend mentions in test-backend-ops comments Assisted-by: pi:llama.cpp/Qwen3.8-27B * metal : rename the FA dequant_f16 identifiers to kv_f16 Assisted-by: pi:llama.cpp/Qwen3.8-27B * cont : clean-up * cont : remove TODO --- .pi/gg/SYSTEM.md | 1 + ggml/src/ggml-metal/ggml-metal-device.cpp | 37 ++- ggml/src/ggml-metal/ggml-metal-device.h | 14 +- ggml/src/ggml-metal/ggml-metal-impl.h | 12 + ggml/src/ggml-metal/ggml-metal-ops.cpp | 269 ++++++++++++++++++---- ggml/src/ggml-metal/ggml-metal-ops.h | 1 + ggml/src/ggml-metal/ggml-metal.cpp | 1 + ggml/src/ggml-metal/ggml-metal.metal | 47 ++++ tests/test-backend-ops.cpp | 29 +++ 9 files changed, 358 insertions(+), 53 deletions(-) diff --git a/.pi/gg/SYSTEM.md b/.pi/gg/SYSTEM.md index d39afbe03..6a757c869 100644 --- a/.pi/gg/SYSTEM.md +++ b/.pi/gg/SYSTEM.md @@ -9,6 +9,7 @@ General: Coding: - When in doubt, always refer to the CONTRIBUTING.md file of the project +- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments - When referencing issues or PRs in comments, use the format: - C/C++ code: `// ref: ` - Other (CMake, etc.): `# ref: ` diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 953c75755..52043696e 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -1409,6 +1409,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_p return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16( + ggml_metal_library_t lib, + const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + char base[256]; + + snprintf(base, 256, "kernel_flash_attn_ext_kv_%s_f16", ggml_type_name(op->src[1]->type)); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, base); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, base, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk( ggml_metal_library_t lib, const struct ggml_tensor * op, @@ -1460,7 +1477,10 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext( bool has_bias, bool has_scap, bool has_kvpad, - int32_t nsg) { + int32_t nsg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); char base[256]; @@ -1469,15 +1489,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext( const int32_t dk = (int32_t) op->src[1]->ne[0]; const int32_t dv = (int32_t) op->src[2]->ne[0]; - const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0]; - const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0]; + const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type); // do bounds checks for the mask? const bool bc_mask = op->src[3] && (op->src[3]->ne[1] % 8 != 0); snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d", "flash_attn_ext", - ggml_type_name(op->src[1]->type), + type, dk, dv); @@ -1526,7 +1545,10 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v bool has_scap, bool has_kvpad, int32_t nsg, - int32_t nwg) { + int32_t nwg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); char base[256]; @@ -1535,12 +1557,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v const int32_t dk = (int32_t) op->src[1]->ne[0]; const int32_t dv = (int32_t) op->src[2]->ne[0]; - const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0]; - const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0]; + const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type); snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d", "flash_attn_ext_vec", - ggml_type_name(op->src[1]->type), + type, dk, dv); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 7e1deeaa2..b7d466058 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -176,6 +176,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_mask, int32_t ncpsg); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16( + ggml_metal_library_t lib, + const struct ggml_tensor * op); + struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk( ggml_metal_library_t lib, const struct ggml_tensor * op, @@ -190,7 +194,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_bias, bool has_scap, bool has_kvpad, - int32_t nsg); + int32_t nsg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec( ggml_metal_library_t lib, @@ -201,7 +208,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_scap, bool has_kvpad, int32_t nsg, - int32_t nwg); + int32_t nwg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_reduce( ggml_metal_library_t lib, diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 05ea7470e..f0b779979 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -345,6 +345,18 @@ typedef struct { bool inplace; } ggml_metal_kargs_rope; +typedef struct { + int32_t ne0; + int32_t ne1; + int32_t ne2; + int32_t ne3; + uint64_t nb0; + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; + int32_t nblocks; +} ggml_metal_kargs_flash_attn_ext_kv_f16; + typedef struct { int32_t ne11; int32_t ne_12_2; // assume K and V are same shape diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index d8435e957..2dde14d8d 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2801,6 +2801,44 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) { return (ne01 < 20) && (ne00 % 32 == 0); } +// ref: https://github.com/ggml-org/llama.cpp/pull/27390 +// dequantize the quantized KV cache to F16 before running the F16 flash attention kernels +static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + switch (op->src[1]->type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + return true; + default: + return false; + } +} + +// in some models (e.g. MLA-based), V is a view of K (the first ne20 elements of each K row); +// the dequantized V is then a view of the dequantized K and does not need its own dequant or scratch +// - ref: https://github.com/ggml-org/llama.cpp/pull/13435 +static bool ggml_metal_op_flash_attn_ext_v_is_view_of_k(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + const ggml_tensor * K = op->src[1]; + const ggml_tensor * V = op->src[2]; + + return V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs)); +} + +// size of the F16 dequantized K tensor; the dequantized V tensor follows it in the same scratch buffer +static size_t ggml_metal_op_flash_attn_ext_kv_f16_k_size(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); + + return GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne10*ne11*ne12*ne13, 16); +} + size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); @@ -2816,6 +2854,18 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { size_t res = 0; const bool has_mask = op->src[3] != nullptr; + const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op); + + // when the KV is dequantized to F16, the pad kernel copies the tail chunk from the F16 scratch buffer + // note: when V is a view of K, the dequantized V is read from the dequantized K with K's row stride + const bool v_is_view_of_k = use_kv_f16 && ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + uint64_t nb11_pad = nb11; + uint64_t nb21_pad = nb21; + + if (use_kv_f16) { + nb11_pad = sizeof(ggml_fp16_t)*ne10; + nb21_pad = sizeof(ggml_fp16_t)*(v_is_view_of_k ? ne10 : ne20); + } // note: the non-vec kernel requires more extra memory, so always reserve for it GGML_ASSERT(OP_FLASH_ATTN_EXT_NCPSG >= OP_FLASH_ATTN_EXT_VEC_NCPSG); @@ -2828,8 +2878,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { if (has_kvpad) { res += OP_FLASH_ATTN_EXT_VEC_NCPSG*( - nb11*ne12*ne13 + - nb21*ne22*ne23 + + nb11_pad*ne12*ne13 + + nb21_pad*ne22*ne23 + (has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0)); } } else { @@ -2838,8 +2888,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { if (has_kvpad) { res += OP_FLASH_ATTN_EXT_NCPSG*( - nb11*ne12*ne13 + - nb21*ne22*ne23 + + nb11_pad*ne12*ne13 + + nb21_pad*ne22*ne23 + (has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0)); } } @@ -2915,6 +2965,28 @@ size_t ggml_metal_op_flash_attn_ext_extra_tmp(const ggml_tensor * op) { return res; } +size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { + return 0; + } + + GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); + + const size_t k_size = ggml_metal_op_flash_attn_ext_kv_f16_k_size(op); + + // when V is a view of K, the dequantized V is a view of the dequantized K + const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + if (v_is_view_of_k) { + return k_size; + } + + const size_t v_size = GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne20*ne21*ne22*ne23, 16); + + return k_size + v_size; +} + int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -2989,6 +3061,111 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_buffer_id bid_tmp = bid_blk; bid_tmp.offs += ggml_metal_op_flash_attn_ext_extra_blk(op); + ggml_metal_buffer_id bid_kv_f16 = bid_tmp; + bid_kv_f16.offs += ggml_metal_op_flash_attn_ext_extra_tmp(op); + + const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op); + + ggml_metal_buffer_id bid_k = bid_src1; + ggml_metal_buffer_id bid_v = bid_src2; + + uint64_t nb10_attn = nb10; + uint64_t nb11_attn = nb11; + uint64_t nb12_attn = nb12; + uint64_t nb13_attn = nb13; + uint64_t nb20_attn = nb20; + uint64_t nb21_attn = nb21; + uint64_t nb22_attn = nb22; + uint64_t nb23_attn = nb23; + + if (use_kv_f16) { + assert(ggml_metal_op_flash_attn_ext_extra_kv_f16(op) != 0); + + const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + + const int64_t nblocks1_64 = (ne10/ggml_blck_size(op->src[1]->type))*(int64_t) ne11*ne12*ne13; + GGML_ASSERT(nblocks1_64 <= INT32_MAX); + const int32_t nblocks1 = nblocks1_64; + + ggml_metal_buffer_id bid_v_f16 = bid_kv_f16; + bid_v_f16.offs += ggml_metal_op_flash_attn_ext_kv_f16_k_size(op); + + auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16(lib, op); + const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline0), 256); + + // K + ggml_metal_kargs_flash_attn_ext_kv_f16 args_k = { + /*.ne0 =*/ ne10, + /*.ne1 =*/ ne11, + /*.ne2 =*/ ne12, + /*.ne3 =*/ ne13, + /*.nb0 =*/ nb10, + /*.nb1 =*/ nb11, + /*.nb2 =*/ nb12, + /*.nb3 =*/ nb13, + /*.nblocks =*/ nblocks1, + }; + + ggml_metal_encoder_set_pipeline(enc, pipeline0); + ggml_metal_encoder_set_bytes (enc, &args_k, sizeof(args_k), 0); + ggml_metal_encoder_set_buffer (enc, bid_src1, 1); + ggml_metal_encoder_set_buffer (enc, bid_kv_f16, 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, (nblocks1 + nth - 1)/nth, 1, 1, nth, 1, 1); + + // V (skip when V is a view of K: the dequantized V is a view of the dequantized K) + if (!v_is_view_of_k) { + const int64_t nblocks2_64 = (ne20/ggml_blck_size(op->src[2]->type))*(int64_t) ne21*ne22*ne23; + GGML_ASSERT(nblocks2_64 <= INT32_MAX); + const int32_t nblocks2 = nblocks2_64; + + ggml_metal_kargs_flash_attn_ext_kv_f16 args_v = { + /*.ne0 =*/ ne20, + /*.ne1 =*/ ne21, + /*.ne2 =*/ ne22, + /*.ne3 =*/ ne23, + /*.nb0 =*/ nb20, + /*.nb1 =*/ nb21, + /*.nb2 =*/ nb22, + /*.nb3 =*/ nb23, + /*.nblocks =*/ nblocks2, + }; + + ggml_metal_encoder_set_pipeline(enc, pipeline0); + ggml_metal_encoder_set_bytes (enc, &args_v, sizeof(args_v), 0); + ggml_metal_encoder_set_buffer (enc, bid_src2, 1); + ggml_metal_encoder_set_buffer (enc, bid_v_f16, 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, (nblocks2 + nth - 1)/nth, 1, 1, nth, 1, 1); + } + + // the pad and attention kernels read the dequantized KV + ggml_metal_op_concurrency_reset(ctx); + + bid_k = bid_kv_f16; + bid_v = v_is_view_of_k ? bid_k : bid_v_f16; + + // contiguous F16 layout of the dequantized K + nb10_attn = sizeof(ggml_fp16_t); + nb11_attn = nb10_attn*ne10; + nb12_attn = nb11_attn*ne11; + nb13_attn = nb12_attn*ne12; + + // if V is a view of K, the dequantized V is read from the dequantized K with K's strides + if (v_is_view_of_k) { + nb20_attn = nb10_attn; + nb21_attn = nb11_attn; + nb22_attn = nb12_attn; + nb23_attn = nb13_attn; + } else { + // contiguous F16 layout of the dequantized V + nb20_attn = sizeof(ggml_fp16_t); + nb21_attn = nb20_attn*ne20; + nb22_attn = nb21_attn*ne21; + nb23_attn = nb22_attn*ne22; + } + } + if (!ggml_metal_op_flash_attn_ext_use_vec(op)) { // half8x8 kernel const int nqptg = OP_FLASH_ATTN_EXT_NQPSG; // queries per threadgroup @@ -3009,12 +3186,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ne11, /*.ne_12_2 =*/ne12, /*.ne_12_3 =*/ne13, - /*.nb11 =*/nb11, - /*.nb12 =*/nb12, - /*.nb13 =*/nb13, - /*.nb21 =*/nb21, - /*.nb22 =*/nb22, - /*.nb23 =*/nb23, + /*.nb11 =*/nb11_attn, + /*.nb12 =*/nb12_attn, + /*.nb13 =*/nb13_attn, + /*.nb21 =*/nb21_attn, + /*.nb22 =*/nb22_attn, + /*.nb23 =*/nb23_attn, /*.ne31 =*/ne31, /*.ne32 =*/ne32, /*.ne33 =*/ne33, @@ -3027,8 +3204,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_pipeline(enc, pipeline0); ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0); - ggml_metal_encoder_set_buffer (enc, bid_src1, 1); - ggml_metal_encoder_set_buffer (enc, bid_src2, 2); + ggml_metal_encoder_set_buffer (enc, bid_k, 1); + ggml_metal_encoder_set_buffer (enc, bid_v, 2); ggml_metal_encoder_set_buffer (enc, bid_src3, 3); ggml_metal_encoder_set_buffer (enc, bid_pad, 4); @@ -3073,7 +3250,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_op_concurrency_reset(ctx); } - const int is_q = ggml_is_quantized(op->src[1]->type) ? 1 : 0; + const int is_q = !use_kv_f16 && ggml_is_quantized(op->src[1]->type) ? 1 : 0; // 2*(2*ncpsg) // ncpsg soft_max values + ncpsg mask values @@ -3104,6 +3281,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { const size_t smem = FATTN_SMEM(nsg); + const int32_t ns10 = nb11_attn/nb10_attn; + const int32_t ns20 = nb21_attn/nb20_attn; + ggml_metal_kargs_flash_attn_ext args = { /*.ne01 =*/ ne01, /*.ne02 =*/ ne02, @@ -3114,14 +3294,14 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ ne11, /*.ne_12_2 =*/ ne12, /*.ne_12_3 =*/ ne13, - /*.ns10 =*/ int32_t(nb11/nb10), - /*.nb11 =*/ nb11, - /*.nb12 =*/ nb12, - /*.nb13 =*/ nb13, - /*.ns20 =*/ int32_t(nb21/nb20), - /*.nb21 =*/ nb21, - /*.nb22 =*/ nb22, - /*.nb23 =*/ nb23, + /*.ns10 =*/ ns10, + /*.nb11 =*/ nb11_attn, + /*.nb12 =*/ nb12_attn, + /*.nb13 =*/ nb13_attn, + /*.ns20 =*/ ns20, + /*.nb21 =*/ nb21_attn, + /*.nb22 =*/ nb22_attn, + /*.nb23 =*/ nb23_attn, /*.ne31 =*/ ne31, /*.ne32 =*/ ne32, /*.ne33 =*/ ne33, @@ -3139,13 +3319,13 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.logit_softcap =*/ logit_softcap, }; - auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg); + auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, use_kv_f16, ns10, ns20); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, bid_src0, 1); - ggml_metal_encoder_set_buffer (enc, bid_src1, 2); - ggml_metal_encoder_set_buffer (enc, bid_src2, 3); + ggml_metal_encoder_set_buffer (enc, bid_k, 2); + ggml_metal_encoder_set_buffer (enc, bid_v, 3); ggml_metal_encoder_set_buffer (enc, bid_src3, 4); ggml_metal_encoder_set_buffer (enc, bid_src4, 5); ggml_metal_encoder_set_buffer (enc, bid_pad, 6); @@ -3177,12 +3357,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ne11, /*.ne_12_2 =*/ne12, /*.ne_12_3 =*/ne13, - /*.nb11 =*/nb11, - /*.nb12 =*/nb12, - /*.nb13 =*/nb13, - /*.nb21 =*/nb21, - /*.nb22 =*/nb22, - /*.nb23 =*/nb23, + /*.nb11 =*/nb11_attn, + /*.nb12 =*/nb12_attn, + /*.nb13 =*/nb13_attn, + /*.nb21 =*/nb21_attn, + /*.nb22 =*/nb22_attn, + /*.nb23 =*/nb23_attn, /*.ne31 =*/ne31, /*.ne32 =*/ne32, /*.ne33 =*/ne33, @@ -3195,8 +3375,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_pipeline(enc, pipeline0); ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0); - ggml_metal_encoder_set_buffer (enc, bid_src1, 1); - ggml_metal_encoder_set_buffer (enc, bid_src2, 2); + ggml_metal_encoder_set_buffer (enc, bid_k, 1); + ggml_metal_encoder_set_buffer (enc, bid_v, 2); ggml_metal_encoder_set_buffer (enc, bid_src3, 3); ggml_metal_encoder_set_buffer (enc, bid_pad, 4); @@ -3242,6 +3422,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { } } + const int32_t ns10 = nb11_attn/nb10_attn; + const int32_t ns20 = nb21_attn/nb20_attn; + ggml_metal_kargs_flash_attn_ext_vec args = { /*.ne01 =*/ ne01, /*.ne02 =*/ ne02, @@ -3252,14 +3435,14 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ ne11, /*.ne_12_2 =*/ ne12, /*.ne_12_3 =*/ ne13, - /*.ns10 =*/ int32_t(nb11/nb10), - /*.nb11 =*/ nb11, - /*.nb12 =*/ nb12, - /*.nb13 =*/ nb13, - /*.ns20 =*/ int32_t(nb21/nb20), - /*.nb21 =*/ nb21, - /*.nb22 =*/ nb22, - /*.nb23 =*/ nb23, + /*.ns10 =*/ ns10, + /*.nb11 =*/ nb11_attn, + /*.nb12 =*/ nb12_attn, + /*.nb13 =*/ nb13_attn, + /*.ns20 =*/ ns20, + /*.nb21 =*/ nb21_attn, + /*.nb22 =*/ nb22_attn, + /*.nb23 =*/ nb23_attn, /*.ne31 =*/ ne31, /*.ne32 =*/ ne32, /*.ne33 =*/ ne33, @@ -3277,15 +3460,15 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.logit_softcap =*/ logit_softcap, }; - auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg); + auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg, use_kv_f16, ns10, ns20); GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, bid_src0, 1); - ggml_metal_encoder_set_buffer (enc, bid_src1, 2); - ggml_metal_encoder_set_buffer (enc, bid_src2, 3); + ggml_metal_encoder_set_buffer (enc, bid_k, 2); + ggml_metal_encoder_set_buffer (enc, bid_v, 3); ggml_metal_encoder_set_buffer (enc, bid_src3, 4); ggml_metal_encoder_set_buffer (enc, bid_src4, 5); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index b03b59e0b..159a628d0 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -42,6 +42,7 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_pad(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_blk(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_tmp(const struct ggml_tensor * op); +size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const struct ggml_tensor * op); int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx); int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx); diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index ef3c92f27..0e8d409e0 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -225,6 +225,7 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_ res += ggml_metal_op_flash_attn_ext_extra_pad(tensor); res += ggml_metal_op_flash_attn_ext_extra_blk(tensor); res += ggml_metal_op_flash_attn_ext_extra_tmp(tensor); + res += ggml_metal_op_flash_attn_ext_extra_kv_f16(tensor); } break; case GGML_OP_CUMSUM: case GGML_OP_ARGSORT: diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 0537fa4cf..949931c8d 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -6318,6 +6318,53 @@ template [[host_name("kernel_fwht_f32_128")]] kernel kernel_fwht_t kernel_fwht_f template [[host_name("kernel_fwht_f32_256")]] kernel kernel_fwht_t kernel_fwht_f32<256>; template [[host_name("kernel_fwht_f32_512")]] kernel kernel_fwht_t kernel_fwht_f32<512>; +// dequantize a quantized KV cache tensor to contiguous F16 before running the F16 flash attention kernels +// - one thread per block; dispatched separately for K and V +// - ref: https://github.com/ggml-org/llama.cpp/pull/27390 +template < + typename block_t, + short QK, + void (*deq_t4x4)(device const block_t *, short, thread float4x4 &)> +kernel void kernel_flash_attn_ext_kv_f16( + constant ggml_metal_kargs_flash_attn_ext_kv_f16 & args, + device const char * x, + device half * x_dst, + uint gid [[thread_position_in_grid]]) { + if (gid >= (uint) args.nblocks) { + return; + } + + const uint nb = args.ne0/QK; + const uint i0 = gid%nb; + uint ib = gid/nb; + const uint i1 = ib%args.ne1; + ib /= args.ne1; + const uint i2 = ib%args.ne2; + const uint i3 = ib/args.ne2; + + const uint64_t offs = i0*args.nb0 + i1*args.nb1 + i2*args.nb2 + i3*args.nb3; + + device const block_t * src = (device const block_t *) (x + offs); + device half4 * dst = (device half4 *) x_dst + (QK/4)*gid; + + for (short i = 0; i < QK/16; ++i) { + float4x4 reg; + deq_t4x4(src, i, reg); + dst[4*i + 0] = (half4) reg[0]; + dst[4*i + 1] = (half4) reg[1]; + dst[4*i + 2] = (half4) reg[2]; + dst[4*i + 3] = (half4) reg[3]; + } +} + +typedef decltype(kernel_flash_attn_ext_kv_f16) kernel_flash_attn_ext_kv_f16_t; + +template [[host_name("kernel_flash_attn_ext_kv_q4_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q4_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q5_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q5_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q8_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; + constant bool FC_flash_attn_ext_pad_has_mask [[function_constant(FC_FLASH_ATTN_EXT_PAD + 0)]]; constant int32_t FC_flash_attn_ext_pad_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_PAD + 25)]]; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c9946c7ae..17098825b 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9934,6 +9934,20 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q2_0)); test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_F16)); + // q8_0 KV cases: decode and prompt batches, KV pad, permuted KV, feature flags, and long context + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3})); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 2}, 1025, 1, true, true, 8, 30, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1025, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3})); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 16384, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + + // MLA shape (V is a view of K) with quantized KV + // (the test harness builds V as a view of K for this shape; see build_graph) + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + // large-KV F16 cases (Qwen3.6-27B geometry and a llama-class control): the upstream matrix // stops at kv=1024, blind to long-context FA bugs (e.g. the oneDNN SDPA ordering race on BMG). for (int64_t kv : { 4096, 16384 }) { @@ -10325,6 +10339,21 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + // q8_0 KV cases with long context (decode and prompt) + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 128, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 2048, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + for (int kv : { 4096, 8192, 16384, }) { for (int hs : { 64, 128, }) { for (int nr : { 1, 4, }) { From dc64a1620e8ae6f01fb33f09470b7160f69fd0f2 Mon Sep 17 00:00:00 2001 From: Aldehir Rojas Date: Thu, 20 Aug 2026 06:59:03 -0500 Subject: [PATCH 09/44] common : gracefully fallback on unsupported regex patterns in JSON schema (#26939) --- common/json-schema-to-grammar.cpp | 142 +++++++++++++++++++------- tests/test-json-schema-to-grammar.cpp | 64 ++++++++++++ 2 files changed, 171 insertions(+), 35 deletions(-) diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index b18607cd6..955b4e014 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -278,7 +278,9 @@ static std::unordered_map GRAMMAR_LITERAL_ESCAPES = { {'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}, {'\\', "\\\\"} }; -static std::unordered_set NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'}; +static const int MAX_PATTERN_DEPTH = 100; + +static std::unordered_set NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?', '^', '$'}; static std::unordered_set ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'^', '$', '.', '[', ']', '(', ')', '|', '{', '}', '*', '+', '?'}; static std::string replacePattern(const std::string & input, const std::regex & regex, const std::function & replacement) { @@ -309,6 +311,32 @@ static std::string format_literal(const std::string & literal) { std::string gbnf_format_literal(const std::string & literal) { return format_literal(literal); } +static size_t gbnf_escape_length(const std::string & pattern, size_t pos) { + if (pos + 1 >= pattern.length() || pattern[pos] != '\\') { + return 0; + } + size_t n_hex = 0; + switch (pattern[pos + 1]) { + case 'x': n_hex = 2; break; + case 'u': n_hex = 4; break; + case 'U': n_hex = 8; break; + case 't': case 'r': case 'n': case '\\': case '"': case '[': case ']': + return 2; + default: + return 0; + } + if (pos + 2 + n_hex > pattern.length()) { + return 0; + } + for (size_t i = pos + 2; i < pos + 2 + n_hex; i++) { + char h = pattern[i]; + if (!((h >= '0' && h <= '9') || (h >= 'a' && h <= 'f') || (h >= 'A' && h <= 'F'))) { + return 0; + } + } + return 2 + n_hex; +} + class common_schema_converter { private: friend class common_schema_info; @@ -345,16 +373,42 @@ private: return string_join(rules, " | "); } + // thrown when the pattern is a valid regex with no grammar equivalent + struct unsupported_pattern : public std::runtime_error { + using std::runtime_error::runtime_error; + }; + + // thrown when the pattern is not a valid regex + struct invalid_pattern : public std::runtime_error { + using std::runtime_error::runtime_error; + }; + std::string _visit_pattern(const std::string & pattern, const std::string & name) { - if (!(pattern.front() == '^' && pattern.back() == '$')) { - _errors.push_back("Pattern must start with '^' and end with '$'"); + auto rules_snapshot = _rules; + try { + return _pattern_to_rule(pattern, name); + } catch (const unsupported_pattern & err) { + // revert rules + _rules = std::move(rules_snapshot); + _warnings.push_back("pattern " + pattern + " is not supported (" + err.what() + "), accepting any string"); + return _add_rule(name, _add_primitive("string", PRIMITIVE_RULES.at("string"))); + } catch (const invalid_pattern & err) { + _rules = std::move(rules_snapshot); + _errors.push_back("Invalid pattern " + pattern + ": " + err.what()); return ""; } + } + + std::string _pattern_to_rule(const std::string & pattern, const std::string & name) { + if (pattern.length() < 2 || pattern.front() != '^' || pattern.back() != '$') { + throw unsupported_pattern("not anchored with '^' and '$'"); + } std::string sub_pattern = pattern.substr(1, pattern.length() - 2); std::unordered_map sub_rule_ids; size_t i = 0; size_t length = sub_pattern.length(); + int paren_depth = 0; using literal_or_rule = std::pair; auto to_rule = [&](const literal_or_rule & ls) { @@ -363,7 +417,6 @@ private: return is_literal ? "\"" + s + "\"" : s; }; std::function transform = [&]() -> literal_or_rule { - size_t start = i; std::vector seq; auto get_dot = [&]() { @@ -420,43 +473,42 @@ private: if (i + 1 < length && sub_pattern[i + 1] == ':') { i += 2; // skip "?:" for non-capturing group, treat as regular group } else { - // lookahead/lookbehind (?=, ?!, ?<=, ? 0) { - if (sub_pattern[i] == '\\' && i + 1 < length) { - i += 2; // skip escaped character - } else { - if (sub_pattern[i] == '(') depth++; - else if (sub_pattern[i] == ')') depth--; - i++; - } - } - continue; + // lookaround, named group, inline flags, ... + throw unsupported_pattern("unsupported group syntax"); } } + paren_depth++; + if (paren_depth > MAX_PATTERN_DEPTH) { + throw unsupported_pattern("pattern nesting too deep"); + } seq.emplace_back("(" + to_rule(transform()) + ")", false); } else if (c == ')') { i++; - if (start > 0 && sub_pattern[start - 1] != '(' && (start < 2 || sub_pattern[start - 2] != '?' || sub_pattern[start - 1] != ':')) { - _errors.push_back("Unbalanced parentheses"); + if (paren_depth == 0) { + throw invalid_pattern("unbalanced parentheses"); } + paren_depth--; return join_seq(); + } else if (c == '^' || c == '$') { + throw unsupported_pattern("anchor inside the pattern"); } else if (c == '[') { std::string square_brackets = std::string(1, c); i++; while (i < length && sub_pattern[i] != ']') { if (sub_pattern[i] == '\\') { - square_brackets += sub_pattern.substr(i, 2); - i += 2; + auto escape_length = gbnf_escape_length(sub_pattern, i); + if (escape_length == 0) { + throw unsupported_pattern("unsupported escape in character class: " + sub_pattern.substr(i, 2)); + } + square_brackets += sub_pattern.substr(i, escape_length); + i += escape_length; } else { square_brackets += sub_pattern[i]; i++; } } if (i >= length) { - _errors.push_back("Unbalanced square brackets"); + throw invalid_pattern("unterminated character class"); } square_brackets += ']'; i++; @@ -465,6 +517,9 @@ private: seq.emplace_back("|", false); i++; } else if (c == '*' || c == '+' || c == '?') { + if (seq.empty()) { + throw invalid_pattern("nothing to repeat"); + } seq.back() = std::make_pair(to_rule(seq.back()) + c, false); i++; } else if (c == '{') { @@ -475,18 +530,19 @@ private: i++; } if (i >= length) { - _errors.push_back("Unbalanced curly brackets"); + throw unsupported_pattern("unterminated curly brackets"); } curly_brackets += '}'; i++; auto nums = string_split(curly_brackets.substr(1, curly_brackets.length() - 2), ","); int min_times = 0; int max_times = std::numeric_limits::max(); + if (nums.size() != 1 && nums.size() != 2) { + throw unsupported_pattern("wrong number of values in curly brackets"); + } try { if (nums.size() == 1) { min_times = max_times = std::stoi(nums[0]); - } else if (nums.size() != 2) { - _errors.push_back("Wrong number of values in curly brackets"); } else { if (!nums[0].empty()) { min_times = std::stoi(nums[0]); @@ -495,9 +551,11 @@ private: max_times = std::stoi(nums[1]); } } - } catch (const std::invalid_argument & e) { - _errors.push_back("Invalid number in curly brackets"); - return std::make_pair("", false); + } catch (const std::logic_error &) { + throw unsupported_pattern("invalid number in curly brackets"); + } + if (seq.empty()) { + throw invalid_pattern("nothing to repeat"); } auto &last = seq.back(); auto &sub = last.first; @@ -523,15 +581,22 @@ private: return NON_LITERAL_SET.find(c) != NON_LITERAL_SET.end(); }; while (i < length) { - if (sub_pattern[i] == '\\' && i < length - 1) { + if (sub_pattern[i] == '\\') { + if (i == length - 1) { + throw invalid_pattern("trailing backslash"); + } char next = sub_pattern[i + 1]; if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.find(next) != ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.end()) { i++; literal += sub_pattern[i]; i++; } else { - literal += sub_pattern.substr(i, 2); - i += 2; + auto escape_length = gbnf_escape_length(sub_pattern, i); + if (escape_length == 0) { + throw unsupported_pattern("unsupported escape: " + sub_pattern.substr(i, 2)); + } + literal += sub_pattern.substr(i, escape_length); + i += escape_length; } } else if (sub_pattern[i] == '"') { literal += "\\\""; @@ -544,14 +609,21 @@ private: break; } } - if (!literal.empty()) { - seq.emplace_back(literal, true); + if (literal.empty()) { // nothing was consumed, ex. a stray ']' or '}' + throw unsupported_pattern(std::string("unsupported character: ") + c); } + seq.emplace_back(literal, true); } } return join_seq(); }; - return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\""); + + auto rule = to_rule(transform()); + if (paren_depth != 0) { + throw invalid_pattern("unbalanced parentheses"); + } + + return _add_rule(name, "\"\\\"\" (" + rule + ") \"\\\"\""); } /* diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index f095274cd..74b57cf1b 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -1564,6 +1564,70 @@ int main() { space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", }); + + run({ + SUCCESS, + "unanchored regexp", + R"""({ + "type": "string", + "pattern": "[0-9]+" + })""", + R"""( + char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) + root ::= string + space ::= | " " | "\n"{1,2} [ \t]{0,20} + string ::= "\"" char* "\"" + )""", + }); + + // the rules of the partial conversion (here "root-0") must not leak into the grammar + run({ + SUCCESS, + "regexp with unsupported shorthand", + R"""({ + "type": "string", + "pattern": "^[0-9]{3}\\w$" + })""", + R"""( + char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) + root ::= string + space ::= | " " | "\n"{1,2} [ \t]{0,20} + string ::= "\"" char* "\"" + )""", + }); + + // a regexp that is invalid under any flavor is still an error + run({ + FAILURE, + "regexp with unbalanced parentheses", + R"""({ + "type": "string", + "pattern": "^(a$" + })""", + "" + }); + + // only the property with the bad pattern degrades + run({ + SUCCESS, + "unsupported regexp in a property", + R"""({ + "type": "object", + "properties": { + "a": { "type": "string", "pattern": "^[a-z\\-]+$" } + }, + "required": ["a"], + "additionalProperties": false + })""", + R"""( + a ::= string + a-kv ::= "\"a\"" space ":" space a + char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) + root ::= "{" space a-kv space "}" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + string ::= "\"" char* "\"" + )""", + }); } if (getenv("LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR")) { From 2b5621094ef383cdcd8428ef6d22efe5df976532 Mon Sep 17 00:00:00 2001 From: Pranesh Gonegandla Date: Thu, 20 Aug 2026 12:36:21 +0000 Subject: [PATCH 10/44] CUDA: adding switch points per HW and quant type to tune the mvq->MMQ decode crossover (#26079) * CUDA: runtime GGML_CUDA_MMVQ_MAX to tune the mvq->MMQ decode crossover Add a runtime override of the mul_mat_vec_q -> MMQ batch crossover (default MMVQ_MAX_BATCH_SIZE). Lowering it routes batches above the threshold from the CUDA-core vector kernel to the int8 MMQ tensor-core path, which is faster once quantized decode becomes compute-bound at B>1 (measured +23-41% at B=8 on RTX 5090 for Q4_K dense, no low-batch loss). The value is parsed once and clamped to [1, MMVQ_MAX_BATCH_SIZE], since mul_mat_vec_q asserts ncols_dst <= that; invalid input warns and falls back to the default. The override is applied consistently in both the mul_mat_vec_q and MUL_MAT_ID dispatch paths. Default behavior unchanged. * Added Blackwell specific switch point, to reduce dependence on runtime env var. * Add per-HW switch point values for DGX Spark and removing runtime env var * Adding switch points for Ada, tested on RTX 4090 * Modifying DGX Spark numbers based on latest run and adding some comments and small functional changes relating to MoE * Reverting an unnecessary conditional * Update ggml/src/ggml-cuda/mmvq.cu --------- Co-authored-by: praneshgo <227579474+praneshgo@users.noreply.github.com> Co-authored-by: Oliver Simons --- ggml/src/ggml-cuda/mmvq.cu | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index c99923804..970534809 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -290,6 +290,42 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { if (!ggml_is_quantized(type)) { return false; } + // k-quants cost more to decode and mvq redoes that per column, so MMQ wins sooner. + // Only list quant-types MMQ supports, others would fall back to cuBLAS. + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ADA_LOVELACE) { + switch (type) { // tuned on RTX 4090 + case GGML_TYPE_Q2_K: + return ne11 <= 4; + case GGML_TYPE_Q3_K: + return ne11 <= 6; + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return ne11 <= 7; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_BLACKWELL) { + switch (type) { // tuned on RTX 5090 + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return ne11 <= 5; + case GGML_TYPE_Q6_K: + return ne11 <= 7; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) { + switch (type) { // tuned on DGX Spark GB10 + case GGML_TYPE_Q2_K: + return ne11 <= 6; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } if (GGML_CUDA_CC_IS_CDNA(cc)) { if (GGML_CUDA_CC_IS_CDNA1(cc)) { switch (type) { From 8a832e4bf3284ec145bace0cbb8991bd97e1e144 Mon Sep 17 00:00:00 2001 From: Aritro Bandyopadhyay <71339004+AriBandyo@users.noreply.github.com> Date: Thu, 20 Aug 2026 06:37:14 -0600 Subject: [PATCH 11/44] server : fix --docker-repo being treated as router mode (#27416) --- tools/server/server.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 01cc6633a..230b578e0 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -133,7 +133,8 @@ int llama_server(common_params & params, int argc, char ** argv) { // router server never loads a model and must not touch the GPU const bool is_router_server = params.model.path.empty() - && params.model.hf_repo.empty(); + && params.model.hf_repo.empty() + && params.model.docker_repo.empty(); // skip device enumeration so the CUDA primary context stays uncreated common_params_print_info(params, !is_router_server); From 9855ad69d38c6b8ca9e1e552646ea2566fd932d9 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Thu, 20 Aug 2026 15:22:16 +0200 Subject: [PATCH 12/44] server: (router) lazy-load startup_models after main setup (#27424) * server: (router) lazy-load startup_models after main setup * only allow is_first_load to populate it * nits * nits 2 --- tools/server/README.md | 2 +- tools/server/server-models.cpp | 70 +++++++++++++++++----------------- tools/server/server-models.h | 7 ++++ tools/server/server.cpp | 12 ++++++ 4 files changed, 55 insertions(+), 36 deletions(-) diff --git a/tools/server/README.md b/tools/server/README.md index b63a0e6da..f5d747eee 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1757,7 +1757,7 @@ The precedence rule for preset options is as follows: 3. **Global options** defined in the preset file (`[*]`) We also offer additional options that are exclusive to presets (these aren't treated as command-line arguments): -- `load-on-startup` (boolean): Controls whether the model loads automatically when the server starts +- `load-on-startup` (boolean): Controls whether the model loads automatically when the server starts. Only applies at startup: if the model list is reloaded later (for example after editing the preset file), a newly added model is listed but not loaded - `stop-timeout` (int, seconds): After requested unload, wait for this many seconds before forcing termination (default: 10) - `dedup-cache-models` (boolean): When the preset uses `hf-repo` pointing to a model that is already downloaded, hide the corresponding cached model entry from `GET /models` (the preset entry remains visible). Set it in the `[*]` section to apply to all presets. diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 35b935570..d60545194 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -672,24 +672,26 @@ void server_models::load_models() { apply_hidden(); log_available_models(); - std::vector models_to_load; - for (const auto & [name, inst] : mapping) { - std::string val; - if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) { - models_to_load.push_back(name); + // skipped on reload, see startup_models + if (startup_models.has_value()) { + std::vector models_to_load; + for (const auto & [name, inst] : mapping) { + std::string val; + if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) { + models_to_load.push_back(name); + } } - } - if ((int)models_to_load.size() > base_params.models_max) { - throw std::runtime_error(string_format( - "number of models to load on startup (%zu) exceeds models_max (%d)", - models_to_load.size(), base_params.models_max)); + if ((int)models_to_load.size() > base_params.models_max) { + throw std::runtime_error(string_format( + "number of models to load on startup (%zu) exceeds models_max (%d)", + models_to_load.size(), base_params.models_max)); + } + + // to be lazy-loaded after main() setup phase is completed + startup_models = std::move(models_to_load); } lk.unlock(); - for (const auto & name : models_to_load) { - SRV_INF("(startup) loading model %s\n", name.c_str()); - load(name); - } } else { // RELOAD: diff the new preset list against the current mapping and reconcile is_reloading = true; @@ -819,8 +821,8 @@ void server_models::load_models() { inst.meta.update_caps(); } - // add models that are new in this reload - std::vector newly_added; + // add models that are new in this reload, load-on-startup is not honored here since a + // reload never spawns an instance for (const auto & [name, preset] : final_presets) { if (mapping.find(name) == mapping.end()) { server_model_meta meta{ @@ -841,42 +843,40 @@ void server_models::load_models() { // /* need_download */ false, }; add_model(std::move(meta)); - newly_added.push_back(name); } } apply_stop_timeout(); apply_hidden(); - // clear reload flag before unlocking for autoload - load() blocks on !is_reloading, - // so clearing it here (while still locked) prevents a deadlock in the autoload calls below + // clear reload flag under the lock, this releases the load() calls waiting on !is_reloading is_reloading = false; cv.notify_all(); log_available_models(); - // collect autoload candidates while still under the lock - std::vector to_autoload; - for (const auto & name : newly_added) { - auto it = mapping.find(name); - if (it != mapping.end()) { - std::string val; - if (it->second.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) { - to_autoload.push_back(name); - } - } - } - lk.unlock(); - for (const auto & name : to_autoload) { - SRV_INF("(reload) loading new model %s\n", name.c_str()); - load(name); - } notify_sse("models_reload", "*"); } } +void server_models::load_startup_models() { + std::vector to_load; + { + std::lock_guard lk(mutex); + if (!startup_models.has_value()) { + return; // already drained + } + to_load = std::move(*startup_models); + startup_models.reset(); + } + for (const auto & name : to_load) { + SRV_INF("(startup) loading model %s\n", name.c_str()); + load(name); + } +} + void server_models::update_meta(const std::string & name, const server_model_meta & meta) { std::lock_guard lk(mutex); auto it = mapping.find(name); diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 79b231cba..5cbb6a801 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -136,6 +136,10 @@ private: // if true, the next get_meta() will trigger a reload of model list bool need_reload = false; + // models marked with load-on-startup, unset once load_startup_models() drains it + // no value means the startup phase is over, so a reload must not queue anything + std::optional> startup_models{std::in_place}; + // conv_id -> model name that currently serves its stream session, lets the resumable stream // routes go straight to the owning child instead of polling every one. populated when // proxy_request forwards a POST carrying an X-Conversation-Id. best effort: a stale entry just @@ -231,6 +235,9 @@ public: // - if a model is not running, it will be added or updated according to the source void load_models(); + // lazy-load startup_models, to be called after main() setup phase + void load_startup_models(); + // check if a model instance exists (thread-safe) bool has_model(const std::string & name); diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 230b578e0..5fe2729ba 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -424,6 +424,18 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_http.stop(); }; + try { + models_routes->models.load_startup_models(); + } catch (const std::exception & e) { + SRV_ERR("failed to load models on startup: %s\n", e.what()); + ctx_http.stop(); + if (ctx_http.thread.joinable()) { + ctx_http.thread.join(); + } + clean_up(); + return 1; + } + } else { // setup clean up function, to be called before exit clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() { From bf0040e15fd5b716262658f4d652c9cee959cf91 Mon Sep 17 00:00:00 2001 From: Oliver Simons Date: Thu, 20 Aug 2026 15:42:26 +0200 Subject: [PATCH 13/44] CI: Use LLVM's OpenMP over MSVC_DEBUG_non_redist on Windows (#26678) * CI: Use LLVM's OpenMP over MSFT_DEBUG_non_redist on Windows Currently, we ship the non-redist debug version of microsoft's libomp. This PR changes this to official LLVM's release, also packaging the license as needed. * Remove LLVM SHA from job name to increase legibility * Add temp validations to CI * Revert "Add temp validations to CI" This reverts commit eef97c88b5bac280803ebb3c3b7bb09f89b0fd88. * Build OpenMP in CI * Make OpenMP fetch self-contained in cmake and cache in CI * Robustify Licens-packaging 1. Ship OpenMP license, not LLVM's. 2. Invalidate cache also on checksum of the license * Remove stale reference in docs/build.md * No longer package base license in release This was scope-creep * Add explanatory comment to OpenMP license * Remove arm64 smoke Forgot this during conflict resolution during rebase of c54c0e9cf6030a5a54ce8bdd81b3e146d9787d42 * Remove GGML_OPENMP_FETCH_CACHE_DIR as requested by @CISC * whitespace changes --- .github/workflows/build-cpu.yml | 5 +- .github/workflows/release.yml | 3 +- cmake/arm64-windows-llvm.cmake | 1 + docs/build.md | 3 +- ggml/CMakeLists.txt | 1 + ggml/src/CMakeLists.txt | 118 +++++++++++++++++++++++++++- ggml/src/ggml-cpu/CMakeLists.txt | 2 +- ggml/src/ggml-zendnn/CMakeLists.txt | 4 +- 8 files changed, 128 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml index 2016a57f8..a63ffe94b 100644 --- a/.github/workflows/build-cpu.yml +++ b/.github/workflows/build-cpu.yml @@ -119,6 +119,7 @@ jobs: ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256 windows: + name: windows / ${{ matrix.build }} runs-on: windows-2025 env: @@ -130,13 +131,13 @@ jobs: include: - build: 'x64-cpu-static' arch: 'x64' - defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF' + defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF' - build: 'x64-openblas' arch: 'x64' defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"' - build: 'arm64' arch: 'arm64' - defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON' + defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON' steps: - name: Clone diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 20c969086..61b2f5485 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -681,6 +681,7 @@ jobs: name: llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip windows-cpu: + name: windows-cpu / ${{ matrix.arch }} needs: [check-release] if: ${{ needs.check-release.outputs.should_release == 'true' }} @@ -728,6 +729,7 @@ jobs: -DGGML_BACKEND_DL=ON ^ -DGGML_CPU_ALL_VARIANTS=${{ matrix.arch == 'x64' && 'ON' || 'OFF' }} ^ -DGGML_OPENMP=ON ^ + -DGGML_OPENMP_FETCH=ON ^ ${{ env.CMAKE_ARGS }} cmake --build build --config Release @@ -739,7 +741,6 @@ jobs: - name: Pack artifacts id: pack_artifacts run: | - Copy-Item "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Redist\MSVC\14.51.36231\debug_nonredist\${{ matrix.arch }}\Microsoft.VC145.OpenMP.LLVM\libomp140.${{ matrix.arch == 'x64' && 'x86_64' || 'aarch64' }}.dll" .\build\bin\Release\ 7z a -snl llama-bin-win-cpu-${{ matrix.arch }}.zip .\build\bin\Release\* - name: Upload artifacts diff --git a/cmake/arm64-windows-llvm.cmake b/cmake/arm64-windows-llvm.cmake index 802379680..cdba4e749 100644 --- a/cmake/arm64-windows-llvm.cmake +++ b/cmake/arm64-windows-llvm.cmake @@ -8,6 +8,7 @@ set( CMAKE_CXX_COMPILER clang++ ) set( CMAKE_C_COMPILER_TARGET ${target} ) set( CMAKE_CXX_COMPILER_TARGET ${target} ) +set( CMAKE_ASM_COMPILER_TARGET ${target} ) set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" ) set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" ) diff --git a/docs/build.md b/docs/build.md index ca086a0be..45fe7f17a 100644 --- a/docs/build.md +++ b/docs/build.md @@ -72,9 +72,10 @@ cmake --build build --config Release - Please remember to always use a Developer Command Prompt / PowerShell for VS2022 for git, build, test - For Windows on ARM (arm64, WoA) build with: ```bash - cmake --preset arm64-windows-llvm-release -D GGML_OPENMP=OFF + cmake --preset arm64-windows-llvm-release -D GGML_OPENMP_FETCH=ON cmake --build build-arm64-windows-llvm-release ``` + `GGML_OPENMP_FETCH` downloads the official LLVM OpenMP runtime and requires Clang, 7-Zip and network access during configuration. CMake selects the runtime from the target architecture, so this also works when cross-compiling for WoA from x64. The extracted header, import library, DLL and OpenMP license are placed under `build/_deps`. The build copies `libomp.dll` and `LICENSE-LLVM-OpenMP` to the runtime output directory and installs them together. Omit the option to use CMake's normal OpenMP detection, or pass `-D GGML_OPENMP=OFF` to disable OpenMP. For building with ninja generator and clang compiler as default: -set path:set LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\x64;C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.41.34120\lib\x64\uwp;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\x64 ```bash diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index b7110fa12..9d807d5c8 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -243,6 +243,7 @@ set (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING "ggml: metal minimum macOS version") set (GGML_METAL_STD "" CACHE STRING "ggml: metal standard version (-std flag)") option(GGML_OPENMP "ggml: use OpenMP" ON) +option(GGML_OPENMP_FETCH "ggml: fetch LLVM OpenMP" OFF) option(GGML_RPC "ggml: use RPC" OFF) option(GGML_SYCL "ggml: use SYCL" OFF) option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF) diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index 82e9480c2..96535b49f 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -222,9 +222,123 @@ if (GGML_SCHED_NO_REALLOC) target_compile_definitions(ggml-base PUBLIC GGML_SCHED_NO_REALLOC) endif() -if (GGML_OPENMP) +if (GGML_OPENMP_FETCH) + if (NOT GGML_OPENMP) + message(FATAL_ERROR "GGML_OPENMP_FETCH requires GGML_OPENMP") + elseif (NOT WIN32 OR NOT (CMAKE_C_COMPILER_ID MATCHES "Clang")) + message(FATAL_ERROR "GGML_OPENMP_FETCH currently requires Clang on Windows") + endif() + + set(GGML_OPENMP_LLVM_VERSION "20.1.8") + string(REGEX MATCH "^[0-9]+" GGML_OPENMP_LLVM_VERSION_MAJOR "${GGML_OPENMP_LLVM_VERSION}") + string(REGEX MATCH "^[0-9]+" GGML_OPENMP_COMPILER_VERSION_MAJOR "${CMAKE_C_COMPILER_VERSION}") + if (NOT GGML_OPENMP_COMPILER_VERSION_MAJOR STREQUAL GGML_OPENMP_LLVM_VERSION_MAJOR) + message(FATAL_ERROR "LLVM OpenMP ${GGML_OPENMP_LLVM_VERSION} requires Clang ${GGML_OPENMP_LLVM_VERSION_MAJOR}.x") + endif() + + string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" GGML_OPENMP_SYSTEM_PROCESSOR) + if (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(amd64|x86_64)$") + set(GGML_OPENMP_ARCH "x64") + set(GGML_OPENMP_INSTALLER_SUFFIX "win64") + set(GGML_OPENMP_INSTALLER_SHA256 "3197846a2b19063687dd56e93e34cd941e3548d907f23a6131571321bdf9fe7b") + elseif (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") + set(GGML_OPENMP_ARCH "arm64") + set(GGML_OPENMP_INSTALLER_SUFFIX "woa64") + set(GGML_OPENMP_INSTALLER_SHA256 "7c4ac97eb2ae6b960ca5f9caf3ff6124c8d2a18cc07a7840a4d2ea15537bad8e") + else() + message(FATAL_ERROR "GGML_OPENMP_FETCH does not support ${CMAKE_SYSTEM_PROCESSOR}") + endif() + + set(GGML_OPENMP_CACHE_DIR "${CMAKE_BINARY_DIR}/_deps") + set(GGML_OPENMP_ROOT "${GGML_OPENMP_CACHE_DIR}/llvm-openmp-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_ARCH}") + set(GGML_OPENMP_LIBRARY "${GGML_OPENMP_ROOT}/lib/libomp.lib") + set(GGML_OPENMP_RUNTIME "${GGML_OPENMP_ROOT}/bin/libomp.dll") + set(GGML_OPENMP_HEADER "${GGML_OPENMP_ROOT}/include/omp.h") + set(GGML_OPENMP_LICENSE "${GGML_OPENMP_ROOT}/LICENSE.TXT") + set(GGML_OPENMP_LICENSE_SHA256 "fdad1758a9e1f9d5a81e18879b3406772115edc92c24bfa36b70c654f325e8e4") + + if (NOT EXISTS "${GGML_OPENMP_LIBRARY}" OR NOT EXISTS "${GGML_OPENMP_RUNTIME}" OR NOT EXISTS "${GGML_OPENMP_HEADER}") + find_program(GGML_OPENMP_7Z NAMES 7z 7zz 7za) + if (NOT GGML_OPENMP_7Z) + message(FATAL_ERROR "GGML_OPENMP_FETCH requires 7-Zip to extract the LLVM installer") + endif() + + set(GGML_OPENMP_INSTALLER "${GGML_OPENMP_ROOT}/LLVM-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_INSTALLER_SUFFIX}.exe") + set(GGML_OPENMP_EXTRACT_DIR "${GGML_OPENMP_ROOT}/extract") + set(GGML_OPENMP_INSTALLER_URL "https://github.com/llvm/llvm-project/releases/download/llvmorg-${GGML_OPENMP_LLVM_VERSION}/LLVM-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_INSTALLER_SUFFIX}.exe") + + file(MAKE_DIRECTORY "${GGML_OPENMP_EXTRACT_DIR}") + file(DOWNLOAD "${GGML_OPENMP_INSTALLER_URL}" "${GGML_OPENMP_INSTALLER}" + EXPECTED_HASH "SHA256=${GGML_OPENMP_INSTALLER_SHA256}" + SHOW_PROGRESS + STATUS GGML_OPENMP_DOWNLOAD_STATUS) + list(GET GGML_OPENMP_DOWNLOAD_STATUS 0 GGML_OPENMP_DOWNLOAD_RESULT) + if (NOT GGML_OPENMP_DOWNLOAD_RESULT EQUAL 0) + list(GET GGML_OPENMP_DOWNLOAD_STATUS 1 GGML_OPENMP_DOWNLOAD_ERROR) + message(FATAL_ERROR "Failed to download LLVM OpenMP: ${GGML_OPENMP_DOWNLOAD_ERROR}") + endif() + + execute_process( + COMMAND "${GGML_OPENMP_7Z}" e -y "-o${GGML_OPENMP_EXTRACT_DIR}" "${GGML_OPENMP_INSTALLER}" -r libomp.lib libomp.dll omp.h + RESULT_VARIABLE GGML_OPENMP_EXTRACT_RESULT + OUTPUT_QUIET) + if (NOT GGML_OPENMP_EXTRACT_RESULT EQUAL 0 OR + NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/libomp.lib" OR + NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/libomp.dll" OR + NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/omp.h") + message(FATAL_ERROR "Failed to extract libomp from ${GGML_OPENMP_INSTALLER}") + endif() + + file(MAKE_DIRECTORY "${GGML_OPENMP_ROOT}/lib" "${GGML_OPENMP_ROOT}/bin" "${GGML_OPENMP_ROOT}/include") + file(COPY "${GGML_OPENMP_EXTRACT_DIR}/libomp.lib" DESTINATION "${GGML_OPENMP_ROOT}/lib") + file(COPY "${GGML_OPENMP_EXTRACT_DIR}/libomp.dll" DESTINATION "${GGML_OPENMP_ROOT}/bin") + file(COPY "${GGML_OPENMP_EXTRACT_DIR}/omp.h" DESTINATION "${GGML_OPENMP_ROOT}/include") + file(REMOVE_RECURSE "${GGML_OPENMP_INSTALLER}" "${GGML_OPENMP_EXTRACT_DIR}") + endif() + + # The NSIS installer embeds LLVM's general license in its UI but does not install it as a file; use OpenMP's license to include its additional notices. + if (EXISTS "${GGML_OPENMP_LICENSE}") + file(SHA256 "${GGML_OPENMP_LICENSE}" GGML_OPENMP_LICENSE_ACTUAL_SHA256) + endif() + if (NOT GGML_OPENMP_LICENSE_ACTUAL_SHA256 STREQUAL GGML_OPENMP_LICENSE_SHA256) + file(DOWNLOAD "https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-${GGML_OPENMP_LLVM_VERSION}/openmp/LICENSE.TXT" "${GGML_OPENMP_LICENSE}" + EXPECTED_HASH "SHA256=${GGML_OPENMP_LICENSE_SHA256}") + endif() + + if (COMMAND license_add_file) + license_add_file("LLVM OpenMP" "${GGML_OPENMP_LICENSE}") + endif() + + add_library(ggml-openmp-c INTERFACE) + target_compile_options(ggml-openmp-c INTERFACE "$<$:-fopenmp=libomp>") + target_include_directories(ggml-openmp-c SYSTEM INTERFACE "${GGML_OPENMP_ROOT}/include") + target_link_libraries(ggml-openmp-c INTERFACE "${GGML_OPENMP_LIBRARY}") + + add_library(ggml-openmp-cxx INTERFACE) + target_compile_options(ggml-openmp-cxx INTERFACE "$<$:-fopenmp=libomp>") + target_include_directories(ggml-openmp-cxx SYSTEM INTERFACE "${GGML_OPENMP_ROOT}/include") + target_link_libraries(ggml-openmp-cxx INTERFACE "${GGML_OPENMP_LIBRARY}") + + set(GGML_OPENMP_RUNTIME_OUTPUT_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}") + if (CMAKE_CONFIGURATION_TYPES) + string(APPEND GGML_OPENMP_RUNTIME_OUTPUT_DIR "/$") + endif() + add_custom_target(ggml-openmp-runtime ALL + COMMAND ${CMAKE_COMMAND} -E make_directory "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${GGML_OPENMP_RUNTIME}" "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}/libomp.dll" + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${GGML_OPENMP_LICENSE}" "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}/LICENSE-LLVM-OpenMP") + add_dependencies(ggml-base ggml-openmp-runtime) + install(FILES "${GGML_OPENMP_RUNTIME}" DESTINATION ${CMAKE_INSTALL_BINDIR}) + install(FILES "${GGML_OPENMP_LICENSE}" DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME LICENSE-LLVM-OpenMP) + + set(GGML_OPENMP_TARGET_C ggml-openmp-c) + set(GGML_OPENMP_TARGET_CXX ggml-openmp-cxx) + set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "") +elseif (GGML_OPENMP) find_package(OpenMP) if (OpenMP_FOUND) + set(GGML_OPENMP_TARGET_C OpenMP::OpenMP_C) + set(GGML_OPENMP_TARGET_CXX OpenMP::OpenMP_CXX) set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "") else() set(GGML_OPENMP_ENABLED "OFF" CACHE INTERNAL "") @@ -236,7 +350,7 @@ endif() if (GGML_OPENMP_ENABLED) target_compile_definitions(ggml-base PRIVATE GGML_USE_OPENMP) - target_link_libraries(ggml-base PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX) + target_link_libraries(ggml-base PRIVATE ${GGML_OPENMP_TARGET_C} ${GGML_OPENMP_TARGET_CXX}) endif() add_library(ggml diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 836bae4d0..a6cc49586 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -74,7 +74,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name) if (GGML_OPENMP_ENABLED) target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_OPENMP) - target_link_libraries(${GGML_CPU_NAME} PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX) + target_link_libraries(${GGML_CPU_NAME} PRIVATE ${GGML_OPENMP_TARGET_C} ${GGML_OPENMP_TARGET_CXX}) endif() if (GGML_LLAMAFILE) diff --git a/ggml/src/ggml-zendnn/CMakeLists.txt b/ggml/src/ggml-zendnn/CMakeLists.txt index 87d721f6d..6e393d6b6 100644 --- a/ggml/src/ggml-zendnn/CMakeLists.txt +++ b/ggml/src/ggml-zendnn/CMakeLists.txt @@ -86,6 +86,6 @@ endif() target_link_libraries(ggml-zendnn PRIVATE m pthread) -if (GGML_OPENMP) - target_link_libraries(ggml-zendnn PRIVATE OpenMP::OpenMP_CXX) +if (GGML_OPENMP_ENABLED) + target_link_libraries(ggml-zendnn PRIVATE ${GGML_OPENMP_TARGET_CXX}) endif() From 63b64a50a37600243c7dfbc4bbb92bd360a11ff7 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 17:00:54 +0300 Subject: [PATCH 14/44] metal : dequant kv cache only for large batches (#27438) --- ggml/src/ggml-metal/ggml-metal-ops.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 2dde14d8d..8311544b3 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2806,6 +2806,13 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) { static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); + // depending on compute/bandwidth ratio, dequant to f16 kv is not always beneficial + // ref: https://github.com/ggml-org/llama.cpp/pull/27390#issuecomment-5355152767 + // TODO: tune per device + if (op->src[0]->ne[1] < 32) { + return false; + } + switch (op->src[1]->type) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -2968,9 +2975,10 @@ size_t ggml_metal_op_flash_attn_ext_extra_tmp(const ggml_tensor * op) { size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); - if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { - return 0; - } + // note: always reserve the temp buffer to avoid graph reallocations + //if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { + // return 0; + //} GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); From 78ec4c378031811671d1c76a067acbee4f4c56ce Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Thu, 20 Aug 2026 09:18:11 -0500 Subject: [PATCH 15/44] vulkan: FA MMQ should use fp32 for Q quantization calculations (#27413) Codex found that qd could be a denorm and 1/qd would overflow. --- .../ggml-vulkan/vulkan-shaders/flash_attn.comp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp index 6c264c786..0c1b6d067 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp @@ -121,13 +121,13 @@ void main() { const uint buf_ib = r * qf_stride + d / 8; const uint buf_iqs = d % 8; - FLOAT_TYPEV4 vals = is_in_bounds ? FLOAT_TYPEV4(data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale) : FLOAT_TYPEV4(0.0f); - const FLOAT_TYPEV4 abs_vals = abs(vals); + vec4 vals = is_in_bounds ? data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale : vec4(0.0f); + const vec4 abs_vals = abs(vals); - const FLOAT_TYPE thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w)); - const FLOAT_TYPE amax = subgroupClusteredMax(thread_max, 8); - const FLOAT_TYPE qd = amax / FLOAT_TYPE(127.0); - const FLOAT_TYPE qd_inv = qd != FLOAT_TYPE(0.0) ? FLOAT_TYPE(1.0) / qd : FLOAT_TYPE(0.0); + const float thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w)); + const float amax = subgroupClusteredMax(thread_max, 8); + const float qd = amax / 127.0f; + const float qd_inv = qd != 0.0f ? 1.0f / qd : 0.0f; vals = round(vals * qd_inv); Qf[buf_ib].qs[buf_iqs] = pack32(i8vec4(vals)); @@ -136,11 +136,11 @@ void main() { // the row-sum scaled by qd, used in k_dot_correction. if (FaTypeK == FA_TYPE_Q8_0) { if (buf_iqs == 0) { - Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0); + Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0f); } } else { - const FLOAT_TYPE thread_sum = vals.x + vals.y + vals.z + vals.w; - const FLOAT_TYPE sum = subgroupClusteredAdd(thread_sum, 8); + const float thread_sum = vals.x + vals.y + vals.z + vals.w; + const float sum = subgroupClusteredAdd(thread_sum, 8); if (buf_iqs == 0) { Qf[buf_ib].ds = FLOAT_TYPEV2(qd, sum * qd); From 07822bddf80d73f1168e592c52e69caaff820f9c Mon Sep 17 00:00:00 2001 From: Tarek Dakhran Date: Thu, 20 Aug 2026 16:36:57 +0200 Subject: [PATCH 16/44] model : support DSpark for LFM2 models (#27383) --- conversion/__init__.py | 1 + conversion/qwen.py | 15 ++++++++++++++- src/llama-arch.cpp | 2 ++ src/models/lfm2.cpp | 25 +++++++++++++++++-------- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/conversion/__init__.py b/conversion/__init__.py index 5ae6ad819..4b8817ead 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -57,6 +57,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "Qwen3DSparkModel": "qwen", "DSparkDraftModel": "qwen", "DSparkSpeculator": "qwen", + "Lfm2DSparkDraftModel": "qwen", "DeepseekV4ForCausalLM": "deepseek", "DeepseekV4DSparkModel": "deepseek", "DistilBertForMaskedLM": "bert", diff --git a/conversion/qwen.py b/conversion/qwen.py index 26b10452b..355365763 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -709,7 +709,7 @@ class DFlashModel(Qwen3Model): yield from super().modify_tensors(data_torch, name, bid) -@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator") +@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator", "Lfm2DSparkDraftModel") @ModelBase.example("satgeze/Qwen3.6-27B-DSpark") class DSparkModel(DFlashModel): # DSpark = DFlash + a semi-autoregressive Markov head. @@ -759,6 +759,13 @@ class DSparkModel(DFlashModel): return None return super().filter_tensors(item) + _ROPE_PERMUTE_SUFFIXES = ( + "self_attn.q_proj.weight", + "self_attn.k_proj.weight", + "self_attn.q_norm.weight", + "self_attn.k_norm.weight", + ) + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if name == "model.d2t": self._d2t = data_torch @@ -767,6 +774,12 @@ class DSparkModel(DFlashModel): if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith(("embed_tokens.weight", "lm_head.weight")): return + # interleaved-rope checkpoints (rope_is_neox_style = false) -> NeoX layout: per head, even dims first then odd + if not self.hparams.get("rope_is_neox_style", True) and name.endswith(self._ROPE_PERMUTE_SUFFIXES): + head_dim = self.hparams["head_dim"] + shape = data_torch.shape + data_torch = data_torch.reshape(-1, head_dim // 2, 2, *shape[1:]).transpose(1, 2).reshape(shape) + yield from super().modify_tensors(data_torch, name, bid) def prepare_tensors(self): diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 955c2d796..408954401 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -1032,6 +1032,8 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) { case LLM_ARCH_DEEPSEEK4: case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: + case LLM_ARCH_LFM2: + case LLM_ARCH_LFM2MOE: return true; default: return false; diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp index 70e837d6e..9a4295557 100644 --- a/src/models/lfm2.cpp +++ b/src/models/lfm2.cpp @@ -2,6 +2,8 @@ #include "../llama-memory-hybrid-iswa.h" #include "../llama-memory-hybrid.h" +#include + void llama_model_lfm2::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -202,15 +204,20 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ } GGML_ASSERT(bx->ne[0] > conv->ne[0]); - // last d_conv columns is a new conv state - auto * new_conv = ggml_view_3d(ctx0, bx, conv->ne[0], bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2], - (bx->ne[0] - conv->ne[0]) * ggml_element_size(bx)); - GGML_ASSERT(ggml_are_same_shape(conv, new_conv)); + // write conv states: slot 0 = the final state, slot s = the state s tokens back (partial rollback) + const int64_t K = hparams.causal_attn && cparams.n_rs_seq > 0 ? (int64_t) cparams.n_rs_seq + 1 : 1; + const int64_t n_written = std::min(n_seq_tokens, K); + const auto mem_size = mctx_cur->get_size(); + const size_t row_size = ggml_row_size(conv_state->type, (int64_t) d_conv * n_embd); - // write new conv conv state - ggml_build_forward_expand(gf, ggml_cpy(ctx0, new_conv, - ggml_view_1d(ctx0, conv_state, ggml_nelements(new_conv), - kv_head * d_conv * n_embd * ggml_element_size(new_conv)))); + for (int64_t slot = 0; slot < n_written; ++slot) { + auto * conv_snap = ggml_view_3d(ctx0, bx, d_conv, bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2], + (bx->ne[0] - d_conv - slot) * ggml_element_size(bx)); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_snap, + ggml_view_2d(ctx0, conv_state, (int64_t) d_conv * n_embd, n_seqs, + conv_state->nb[1], + ((size_t) slot * mem_size + kv_head) * row_size))); + } auto * conv_kernel = model.layers[il].shortconv.conv; auto * conv_out = ggml_ssm_conv(ctx0, bx, conv_kernel); @@ -242,6 +249,8 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = cur; + const bool is_moe_layer = il >= static_cast(hparams.n_layer_dense_lead); auto * prev_cur = cur; From 681c29d36a13be54d317ee147b272da9163dbef3 Mon Sep 17 00:00:00 2001 From: John-Henry Lim <42513874+Interpause@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:45:37 +0800 Subject: [PATCH 17/44] mtmd: add --mmproj-device argument (#23255) * feat: add --mmproj-device arg & backwards compatible MTMD_BACKEND_DEVICE env var * feat: load mmproj device backend immediately, add -mmdev shortflag * fix: its a pointer now get the name * clean up * gen docs * nits --------- Co-authored-by: Xuan Son Nguyen --- common/arg.cpp | 20 ++++++++++++++++++++ common/common.h | 7 ++++--- tools/cli/README.md | 1 + tools/mtmd/clip.cpp | 11 +++++------ tools/mtmd/clip.h | 1 + tools/mtmd/debug/mtmd-debug.cpp | 1 + tools/mtmd/mtmd-cli.cpp | 1 + tools/mtmd/mtmd.cpp | 2 ++ tools/mtmd/mtmd.h | 1 + tools/server/README.md | 5 +++-- tools/server/server-context.cpp | 1 + tools/tts/tts.cpp | 1 + 12 files changed, 41 insertions(+), 11 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 6f5fe377d..0a479c6aa 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2595,6 +2595,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.mmproj_use_gpu = value; } ).set_examples(mmproj_examples).set_env("LLAMA_ARG_MMPROJ_OFFLOAD")); + add_opt(common_arg( + // note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet + {"-mmdev", "--mmproj-device"}, "DEVICE", + "device to use for multimodal projector (none = don't offload, default: auto)\n" + "use --list-devices to see a list of available devices", + [](common_params & params, const std::string & value) { + if (value == "none") { + params.mmproj_use_gpu = false; + params.mmproj_device = nullptr; + return; + } + auto devices = parse_device_list(value); + // parse_device_list pushes nullptr at back so devices is length 2 for single device. + if (devices.size() > 2) { + throw std::invalid_argument("only one device may be specified for mmproj"); + } + params.mmproj_use_gpu = true; + params.mmproj_device = devices.front(); + } + ).set_examples(mmproj_examples).set_env("MTMD_BACKEND_DEVICE")); // no LLAMA_ARG_ prefix for backward compatibility reason add_opt(common_arg( {"--image", "--audio", "--video"}, "FILE", "path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files\n", diff --git a/common/common.h b/common/common.h index d8a16897b..de49dac9f 100644 --- a/common/common.h +++ b/common/common.h @@ -581,9 +581,10 @@ struct common_params { // multimodal models (see tools/mtmd) struct common_params_model mmproj; - bool mmproj_use_gpu = true; // use GPU for multimodal model - bool no_mmproj = false; // explicitly disable multimodal model - std::vector image; // path to image file(s) ; TODO: change the name to "media" + bool mmproj_use_gpu = true; // use GPU for multimodal model + ggml_backend_dev_t mmproj_device = nullptr; // GPU device to use for multimodal model + bool no_mmproj = false; // explicitly disable multimodal model + std::vector image; // path to image file(s) ; TODO: change the name to "media" int image_min_tokens = -1; int image_max_tokens = -1; int mtmd_batch_max_tokens = 1024; diff --git a/tools/cli/README.md b/tools/cli/README.md index b3543ed4d..c9cbacafc 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -162,6 +162,7 @@ | `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md
(env: LLAMA_ARG_MMPROJ_URL) | | `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)
(env: LLAMA_ARG_MMPROJ_AUTO) | | `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)
(env: LLAMA_ARG_MMPROJ_OFFLOAD) | +| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: auto)
use --list-devices to see a list of available devices
(env: MTMD_BACKEND_DEVICE) | | `--image, --audio, --video FILE` | path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files | | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MIN_TOKENS) | | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MAX_TOKENS) | diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index b9dd5e845..45e33042d 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -186,14 +186,13 @@ struct clip_ctx { throw std::runtime_error("failed to initialize CPU backend"); } if (ctx_params.use_gpu) { - auto * backend_name = std::getenv("MTMD_BACKEND_DEVICE"); - if (backend_name != nullptr) { - backend = ggml_backend_init_by_name(backend_name, nullptr); + if (ctx_params.device != nullptr) { + backend = ggml_backend_dev_init(ctx_params.device, nullptr); if (!backend) { - LOG_WRN("%s: Warning: Failed to initialize \"%s\" backend, falling back to default GPU backend\n", __func__, backend_name); + throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n", + __func__, ggml_backend_dev_name(ctx_params.device))); } - } - if (!backend) { + } else { backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr); backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr); } diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index a5b713775..e07f25815 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -48,6 +48,7 @@ enum clip_flash_attn_type { struct clip_context_params { bool use_gpu; + ggml_backend_dev_t device; enum clip_flash_attn_type flash_attn_type; int image_min_tokens; int image_max_tokens; diff --git a/tools/mtmd/debug/mtmd-debug.cpp b/tools/mtmd/debug/mtmd-debug.cpp index b88a16f0f..2719dae9b 100644 --- a/tools/mtmd/debug/mtmd-debug.cpp +++ b/tools/mtmd/debug/mtmd-debug.cpp @@ -84,6 +84,7 @@ int main(int argc, char ** argv) { const char * clip_path = params.mmproj.path.c_str(); mtmd_context_params mparams = mtmd_context_params_default(); mparams.use_gpu = params.mmproj_use_gpu; + mparams.device = params.mmproj_device; mparams.print_timings = true; mparams.n_threads = params.cpuparams.n_threads; mparams.flash_attn_type = params.flash_attn_type; diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index 07b45b644..f6c787fdb 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -154,6 +154,7 @@ struct mtmd_cli_context { const char * clip_path = params.mmproj.path.c_str(); mtmd_context_params mparams = mtmd_context_params_default(); mparams.use_gpu = params.mmproj_use_gpu; + mparams.device = params.mmproj_device; mparams.print_timings = true; mparams.n_threads = params.cpuparams.n_threads; mparams.flash_attn_type = params.flash_attn_type; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 0f5cb8c7a..95f17f7af 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -456,6 +456,7 @@ static clip_flash_attn_type mtmd_get_clip_flash_attn_type(enum llama_flash_attn_ mtmd_context_params mtmd_context_params_default() { mtmd_context_params params { /* use_gpu */ true, + /* device */ nullptr, /* print_timings */ true, /* n_threads */ 4, /* image_marker */ nullptr, @@ -564,6 +565,7 @@ struct mtmd_context { clip_context_params ctx_clip_params { /* use_gpu */ ctx_params.use_gpu, + /* device */ ctx_params.device, /* flash_attn_type */ mtmd_get_clip_flash_attn_type(ctx_params.flash_attn_type), /* image_min_tokens */ ctx_params.image_min_tokens, /* image_max_tokens */ ctx_params.image_max_tokens, diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index ef4f99c0b..ef88efd31 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -89,6 +89,7 @@ typedef bool (*mtmd_progress_callback)(float progress, void * user_data); struct mtmd_context_params { bool use_gpu; + ggml_backend_dev_t device; bool print_timings; int n_threads; const char * image_marker; // deprecated, use media_marker instead diff --git a/tools/server/README.md b/tools/server/README.md index f5d747eee..93736c3ed 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -178,6 +178,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md
(env: LLAMA_ARG_MMPROJ_URL) | | `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)
(env: LLAMA_ARG_MMPROJ_AUTO) | | `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)
(env: LLAMA_ARG_MMPROJ_OFFLOAD) | +| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: auto)
use --list-devices to see a list of available devices
(env: MTMD_BACKEND_DEVICE) | | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MIN_TOKENS) | | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MAX_TOKENS) | | `--mtmd-batch-max-tokens N` | maximum number of image tokens per batch when encoding images (default: 1024)
(env: LLAMA_ARG_MTMD_BATCH_MAX_TOKENS) | @@ -196,11 +197,11 @@ For the full list of features, please refer to [server's changelog](https://gith | `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG) | | `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG_FILE) | | `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)
(env: LLAMA_ARG_UI_MCP_PROXY) | -| `--tools TOOL1,TOOL2,...` | experimental: whether to enable server tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | +| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | | `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)
available options:
'docker:', 'podman:': spin up a new container and reuse it for all invocations, clean up on server exit
'docker-container:', 'podman-container:': use an existing container by ID, won't stop on server exit
'ssh:': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required

(env: LLAMA_ARG_TOOLS_RUNTIME) | | `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_CONFIG) | | `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_JSON) | -| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all server tools - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_AGENT) | +| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_AGENT) | | `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)
(env: LLAMA_ARG_UI) | | `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)
(env: LLAMA_ARG_EMBEDDINGS) | | `--rerank, --reranking` | enable reranking endpoint on server (default: disabled)
(env: LLAMA_ARG_RERANKING) | diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 21ff78394..1293c8640 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -998,6 +998,7 @@ private: mtmd_context_params mparams = mtmd_context_params_default(); if (has_mmproj) { mparams.use_gpu = params_base.mmproj_use_gpu; + mparams.device = params_base.mmproj_device; mparams.print_timings = false; mparams.n_threads = params_base.cpuparams.n_threads; mparams.flash_attn_type = params_base.flash_attn_type; diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp index fd7522f8d..368123baf 100644 --- a/tools/tts/tts.cpp +++ b/tools/tts/tts.cpp @@ -86,6 +86,7 @@ int main(int argc, char ** argv) { mtmd_context_params mtmd_params = mtmd_context_params_default(); mtmd_params.use_gpu = params.mmproj_use_gpu; + mtmd_params.device = params.mmproj_device; mtmd::context_ptr mctx(mtmd_init_from_file(params.mmproj.path.c_str(), model, mtmd_params)); if (!mctx) { LOG_ERR("failed to load mmproj %s\n", params.mmproj.path.c_str()); From 521a64cd01979bb5b1a466152c576a9d809b068d Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Thu, 20 Aug 2026 19:02:04 +0200 Subject: [PATCH 18/44] ui: Stores split refactor (#27240) * ui: Extract server stream lifecycle from chatStore into ChatStreamManager Discovery, attach/replay, resume retry and the remote-running snapshot formed a cohesive cluster inside chatStore. It now lives in chat-streams.svelte.ts as ChatStreamManager, owned by chatStore, which keeps the public entry points as delegates so components are unchanged. chatStore: 2877 -> 2418 lines. * ui: Extract user interaction gates from agenticStore into AgenticGates Tool permission requests, turn-limit continue prompts and queued steering messages are the state the loop waits on between turns. They had no coupling to session state, so they now live in agentic-gates.svelte.ts; agenticStore keeps delegates so components are unchanged. agenticStore: 1196 -> 1073 lines. * ui: Compose MCP resources under mcpStore.resources Resource state was a second import scope next to mcpStore. Consumers now go through mcpStore.resources, so the MCP surface is one store; mcp-resources.svelte.ts stays a separate file owned by mcpStore. * ui: Reorganize stores into domain namespaces * fix: Update stale doc comments * ui: Consolidate conv running-state into a chat activity ledger Running-state was split across chatStore.chatLoadingStates (local pipes), ChatStreamManager.remoteRunningConvs (backend sessions) and attachingConvs (attach lifecycle), unioned by hand in getAllLoadingChats and cross-cleaned by setChatLoading calling streams.clearRemoteRunning - the 'spinner ghosts until tab toggle' workaround. chatActivityStore now owns both sets with one transition per event: markLocal / localEnded (local pipe end also drops the stale remote hint, no cross-owner call) / applyRemoteSnapshot (diffed). The sidebar reads chatStore.activity.loadingConvs through the unchanged getAllLoadingChats entry point. Consequences: - isStreamingActive and its five manual writers are gone; isStreaming() now reports whether the active conversation has a live streaming pipe, which is what all four consumers (assistant row, stop action, context gauge, chat screen) actually check - isLoading/isReasoning become derived from the per-conv maps plus the active conversation, dropping the manual resync in syncLoadingStateForChat and clearUIState - attachingConvs and the last-attach coordination disappear from ChatStreamManager - getAllStreamingChats (no consumers) is removed * ui: Give store collaborators narrow host interfaces Collaborators took 'host: typeof ', i.e. the store's entire public surface, which is how chatStore's streamChatCompletion, createAssistantMessage, getApiOptions and setStreamingActive got widened to public. Replace with per-collaborator interfaces carrying only the members each one drives: - ChatStreamHost (chat/streams) - activity, processing, streaming states, abort controller, loading/streaming setters - ChatFlowsHost (chat/flows) - streaming core, message creation, per-conv state setters - McpHealthHost (mcp/health) - connection registry + reconnection - ModelPropsHost / ModelStatusHost (models) - model rows, feed updates; the managers write modalities/status back onto the host's rows, so those members stay writable - ConversationsPreferencesHost (conversations) - the active row and the conversation list The store classes now declare 'implements ' so the contract is visible at the class level, and the 'import type { }' back references in the collaborators disappear entirely - the host contract is local to each collaborator file, and collaborators can no longer reach around their slice. Members stay public (structural typing), but the collaborator side is now compiler-enforced. * test: Chat Activity store test * refactor: Cleanup * chore: Remove legacy architecture docs * ui: Memoize findMessageIndex for the streaming hot path Streaming looks up the same message index on every chunk, a linear scan of activeMessages each time. Cache the last lookup and reuse it after validating the id still sits at the same position (O(1)); any structural change to the array fails validation and falls back to a full scan. * ui: Throttle per-chunk stream state writes to localStorage saveStreamState ran JSON.stringify + a synchronous localStorage.setItem on every decoded chunk of the stream. The read loop now goes through a new saveStreamStateThrottled (one write per conversation per 500ms, latest value held pending); the public saveStreamState keeps its immediate-write contract for stream start and pre-fetch, and also resets the throttle window. A pending offset is force-flushed at resume boundaries (resumeStream reads the offset back from localStorage), on visibilitychange->hidden and on pagehide, so a reload always finds a usable offset. The resume offset only needs to be roughly current since the server retransmits from a line boundary and the client discards its partial line. Adds unit tests for the throttled/flush/clear interplay. * ui: Compute context gauge timing stats in one pass currentRead/Fresh/Cache/Output were separate deriveds, each running a full reverse scan of activeMessages for the last assistant timings, and cumulative ran its own forward scan plus an agentic filter - 4-5 O(n) passes per chunk while streaming. Replace with a single summarizeAssistantTimings() pass (last assistant timings, last agentic llm totals and the cumulative sums) feeding a shared derived snapshot. Semantics unchanged, including the live-stats overrides and the agentic llm-totals branch. * agentic : clear session state when a conversation is deleted Every conversation that ran an agentic flow left an AgenticSession in the store forever; clearSession was never called. conversationsStore now notifies deletion listeners and agenticStore drops the matching sessions, avoiding a circular import back into conversationsStore. * chat : extract ChatService.normalizeMessagesForApi The DB->API message normalization (convert + drop empty system messages) was duplicated in sendMessage, preEncode and the agentic flow. Extract it into one shared method and call it from all three. * sse : share record splitting and data extraction splitSseRecords and extractSseDataPayload centralize the record-boundary splitting and data: line extraction used by parseSseJsonStream and the models status feed. chat.service keeps its own line-based parser for resume support. * api : delegate apiFetchWithParams to apiFetch apiFetchWithParams duplicated apiFetch's headers/fetch/error handling body-for-body; it only differs in URL construction. Build the URL and delegate. * chat flows : dedupe title, timings and cleanup handling - conversationsStore.applyTitleFromContent centralizes the title-from-first- message logic duplicated in 5 places - ChatProcessingStore.applyStreamTimings centralizes the onTimings handler shared by the chat and continue flows - host.cleanupStreaming centralizes the loading/streaming/processing reset repeated across the continue flow's exit paths * conversations : centralize conversation update mirroring rename, pin, mcp override, reasoning effort and cwd all repeated the same write-DB-then-mirror-into-list-and-active dance. A single applyConversationUpdate(id, updates) on the host collapses all five and removes the forgot-to-mirror-one-field bug class. Drops the redundant array reassignment in setCwd (deep field assignment is reactive). * mcp : dedupe tool execution, server parsing and tool indexing - executeTool delegates to executeToolByName (only diff was argument parsing) - drop the private #parseServerSettings copy; use parseMcpServerSettings - cache getServers() keyed on the raw config value (hot path) - indexServerTools() unifies the three identical toolsIndex rebuild loops Assisted-by: Claude * mcp : share cursor pagination and tool indexing - MCPService.paginate() collapses the identical do-while loops in listAllResources and listAllResourceTemplates - promoteHealthCheckToConnection now uses indexServerTools like the other connect paths Assisted-by: Claude * database : share message parent-child bookkeeping - addChildToParent() dedups the append-to-children update in createMessageBranch and createSystemMessage - removeChildFromParent() dedups the remove-from-children cleanup in deleteMessage and deleteMessageCascading - bulkAdd the cloned messages when forking a conversation instead of one add per message Assisted-by: Claude * chore: Lint/format * fix: `pagehide` event from `window` * refactor: Api Fetch util * docs : rewrite architecture sections in README Update the high-level diagram, routes, hooks, stores, services and data flow tables to match the current UI structure (mcp/settings/search routes, agentic/tools/mcp stores, MCPService/ToolsService/SandboxService, /tools API). Fix stale architectural patterns for per-conversation state and modality validation. * chore : add ESLint rule for blank lines between accessors Enforce a blank line between consecutive class accessors. The core padding-line-between-statements rule does not cover class members, so a local rule is needed. * refactor : reorder store members and unify naming Order store class members as public fields, private fields, constructor, getters, public methods, then private methods. Normalize private naming to the `private` keyword (drop `#` and the `_` prefix where there is no matching public getter). Rename conversationsStore.init() to initialize() to match the other stores. * refactor : prefix lookup methods with get in agentic and chat stores Unify bare-name lookup methods with the get* prefix used across the other stores (mcp, models, tools, settings). Renames currentTurn, totalToolCalls, lastError, streamingToolCall, executingToolCallId, pendingPermissionRequest, pendingContinueRequest, pendingSteeringMessageContent, pendingSteeringMessageExtras in the agentic store and pendingMessageContent, pendingMessageExtras in the chat store. Updates the two consuming components and a doc comment. * refactor: Clean up comments in stores' and services' code * chore : add ESLint rule for class member ordering Enforce structural order (public fields -> private fields -> constructor -> getters -> setters -> public methods -> private methods) with alphabetical sorting within each group via perfectionist/sort-classes. Dependency detection keeps Svelte $derived fields in a valid dependency order instead of alphabetizing them, since Svelte rejects forward references. Assisted-by: Claude * refactor : reorder class members to match new ESLint rule Apply the sort-classes rule across stores, services, hooks and utils. Pure reordering - verified no logic changes by comparing sorted line multisets before/after. All tests and svelte-check pass. --- tools/ui/README.md | 166 +- .../high-level-architecture-simplified.md | 145 - .../architecture/high-level-architecture.md | 373 --- tools/ui/docs/flows/chat-flow.md | 228 -- tools/ui/docs/flows/conversations-flow.md | 183 -- .../flows/data-flow-simplified-model-mode.md | 45 - .../flows/data-flow-simplified-router-mode.md | 77 - tools/ui/docs/flows/database-flow.md | 174 - tools/ui/docs/flows/mcp-flow.md | 226 -- tools/ui/docs/flows/models-flow.md | 181 -- tools/ui/docs/flows/server-flow.md | 76 - tools/ui/docs/flows/settings-flow.md | 156 - tools/ui/eslint.config.js | 84 +- .../ChatAttachmentsPreview.svelte | 2 +- .../app/chat/ChatForm/ChatForm.svelte | 12 +- .../ChatFormActionAddMcpServersSubmenu.svelte | 4 +- .../ChatFormActionAddSheet.svelte | 10 +- .../ChatFormActionModels.svelte | 12 +- .../ChatFormActions/ChatFormActions.svelte | 6 +- .../ChatFormContextGauge.svelte | 6 +- .../ChatForm/ChatFormMcpResourcesList.svelte | 6 +- .../ChatFormPickerMcpPrompts.svelte | 2 +- .../ChatMessageAssistant.svelte | 2 +- .../ChatMessageAssistantModel.svelte | 2 +- .../ChatMessageAgenticContent.svelte | 8 +- .../app/chat/ChatMessages/ChatMessages.svelte | 12 +- .../dialogs/DialogMcpResourcesBrowser.svelte | 18 +- .../app/dialogs/DialogMcpServerAddNew.svelte | 2 +- .../app/dialogs/DialogModelInformation.svelte | 4 +- .../app/mcp/McpActiveServersAvatars.svelte | 4 +- .../McpResourcesBrowser.svelte | 6 +- .../app/models/ModelsSelectorDropdown.svelte | 6 +- .../app/models/ModelsSelectorOption.svelte | 12 +- .../app/models/ModelsSelectorSheet.svelte | 4 +- .../settings/SettingsChat/SettingsChat.svelte | 2 +- .../SettingsChat/SettingsChatFields.svelte | 4 +- .../app/settings/SettingsMcpServers.svelte | 8 +- .../constants/attachment-menu.constants.ts | 2 +- tools/ui/src/lib/constants/cache.constants.ts | 10 - tools/ui/src/lib/constants/url.constants.ts | 6 + .../src/lib/hooks/use-auto-scroll.svelte.ts | 178 +- .../use-chat-screen-active-model.svelte.ts | 10 +- .../src/lib/hooks/use-context-gauge.svelte.ts | 8 +- .../lib/hooks/use-models-selector.svelte.ts | 8 +- .../lib/hooks/use-processing-state.svelte.ts | 2 +- .../lib/hooks/use-reasoning-menu.svelte.ts | 11 +- .../src/lib/hooks/use-tools-panel.svelte.ts | 4 +- tools/ui/src/lib/services/chat.service.ts | 1975 ++++++------ .../services/conversation-transfer.service.ts | 346 +- tools/ui/src/lib/services/database.service.ts | 765 +++-- tools/ui/src/lib/services/index.ts | 28 +- tools/ui/src/lib/services/mcp.service.ts | 1426 ++++---- .../ui/src/lib/services/migration.service.ts | 21 +- tools/ui/src/lib/services/models.service.ts | 267 +- .../lib/services/parameter-sync.service.ts | 176 +- tools/ui/src/lib/services/props.service.ts | 16 +- .../ui/src/lib/services/read-media.service.ts | 9 +- tools/ui/src/lib/services/router.service.ts | 7 + tools/ui/src/lib/services/sandbox-harness.ts | 7 + tools/ui/src/lib/services/sandbox.service.ts | 10 +- tools/ui/src/lib/services/tools.service.ts | 25 +- .../ui/src/lib/stores/agentic/gates.svelte.ts | 208 ++ .../index.svelte.ts} | 489 +-- tools/ui/src/lib/stores/chat.svelte.ts | 2868 ----------------- .../ui/src/lib/stores/chat/activity.svelte.ts | 74 + .../stores/{ => chat}/context-stats.svelte.ts | 248 +- .../drafts.svelte.ts} | 20 +- tools/ui/src/lib/stores/chat/flows.svelte.ts | 794 +++++ tools/ui/src/lib/stores/chat/index.svelte.ts | 1441 +++++++++ .../src/lib/stores/chat/processing.svelte.ts | 188 ++ .../ui/src/lib/stores/chat/streams.svelte.ts | 494 +++ .../index.svelte.ts} | 1195 +++---- .../conversations/preferences.svelte.ts | 254 ++ tools/ui/src/lib/stores/device.svelte.ts | 4 +- tools/ui/src/lib/stores/index.ts | 28 +- tools/ui/src/lib/stores/init.ts | 6 +- tools/ui/src/lib/stores/mcp/health.svelte.ts | 298 ++ .../{mcp.svelte.ts => mcp/index.svelte.ts} | 2547 ++++++--------- .../resources.svelte.ts} | 616 ++-- tools/ui/src/lib/stores/models.svelte.ts | 1077 ------- .../ui/src/lib/stores/models/index.svelte.ts | 451 +++ .../ui/src/lib/stores/models/props.svelte.ts | 273 ++ .../ui/src/lib/stores/models/status.svelte.ts | 278 ++ tools/ui/src/lib/stores/permissions.svelte.ts | 48 +- tools/ui/src/lib/stores/server.svelte.ts | 128 +- .../index.svelte.ts} | 810 +++-- .../referrer.svelte.ts} | 7 + tools/ui/src/lib/stores/tools.svelte.ts | 863 ++--- tools/ui/src/lib/types/agentic.d.ts | 2 +- tools/ui/src/lib/utils/api-fetch.ts | 32 +- tools/ui/src/lib/utils/api-headers.ts | 2 +- tools/ui/src/lib/utils/api-key-validation.ts | 2 +- tools/ui/src/lib/utils/audio-recording.ts | 58 +- tools/ui/src/lib/utils/cache-ttl.ts | 204 +- .../utils/chat-form-input-rich-tokenizer.ts | 2 +- .../src/lib/utils/convert-files-to-extra.ts | 6 +- tools/ui/src/lib/utils/index.ts | 7 +- tools/ui/src/lib/utils/mcp.ts | 150 +- .../src/lib/utils/process-uploaded-files.ts | 6 +- tools/ui/src/lib/utils/source-history.ts | 26 +- tools/ui/src/lib/utils/sse.ts | 28 +- tools/ui/src/routes/(chat)/+page.svelte | 6 +- tools/ui/src/routes/+layout.svelte | 4 +- .../client/agentic-stream.perf.svelte.test.ts | 2 +- .../tests/client/apikey-splash.svelte.test.ts | 2 +- .../chat-form-enter-code-block.svelte.test.ts | 2 +- .../components/ChatMessagesPerfWrapper.svelte | 2 +- .../client/mcp-display-name.svelte.test.ts | 4 +- .../client/sandbox.service.svelte.test.ts | 2 +- ...ettings-registry-invariants.svelte.test.ts | 2 +- ...tings-render-keys-migration.svelte.test.ts | 2 +- .../client/ui-settings-sync.svelte.test.ts | 2 +- .../update-message-in-place.svelte.test.ts | 2 +- .../tests/stories/ChatMessage.stories.svelte | 14 +- .../stories/ModelsSelector.stories.svelte | 2 +- .../stories/SidebarNavigation.stories.svelte | 6 +- .../tests/stories/fixtures/storybook-mocks.ts | 2 +- tools/ui/tests/unit/chat-activity.test.ts | 77 + .../tests/unit/mcp-override-fallback.test.ts | 38 +- tools/ui/tests/unit/stream-resume.test.ts | 61 + 120 files changed, 11220 insertions(+), 12829 deletions(-) delete mode 100644 tools/ui/docs/architecture/high-level-architecture-simplified.md delete mode 100644 tools/ui/docs/architecture/high-level-architecture.md delete mode 100644 tools/ui/docs/flows/chat-flow.md delete mode 100644 tools/ui/docs/flows/conversations-flow.md delete mode 100644 tools/ui/docs/flows/data-flow-simplified-model-mode.md delete mode 100644 tools/ui/docs/flows/data-flow-simplified-router-mode.md delete mode 100644 tools/ui/docs/flows/database-flow.md delete mode 100644 tools/ui/docs/flows/mcp-flow.md delete mode 100644 tools/ui/docs/flows/models-flow.md delete mode 100644 tools/ui/docs/flows/server-flow.md delete mode 100644 tools/ui/docs/flows/settings-flow.md create mode 100644 tools/ui/src/lib/stores/agentic/gates.svelte.ts rename tools/ui/src/lib/stores/{agentic.svelte.ts => agentic/index.svelte.ts} (77%) delete mode 100644 tools/ui/src/lib/stores/chat.svelte.ts create mode 100644 tools/ui/src/lib/stores/chat/activity.svelte.ts rename tools/ui/src/lib/stores/{ => chat}/context-stats.svelte.ts (56%) rename tools/ui/src/lib/stores/{draft-messages.svelte.ts => chat/drafts.svelte.ts} (76%) create mode 100644 tools/ui/src/lib/stores/chat/flows.svelte.ts create mode 100644 tools/ui/src/lib/stores/chat/index.svelte.ts create mode 100644 tools/ui/src/lib/stores/chat/processing.svelte.ts create mode 100644 tools/ui/src/lib/stores/chat/streams.svelte.ts rename tools/ui/src/lib/stores/{conversations.svelte.ts => conversations/index.svelte.ts} (61%) create mode 100644 tools/ui/src/lib/stores/conversations/preferences.svelte.ts create mode 100644 tools/ui/src/lib/stores/mcp/health.svelte.ts rename tools/ui/src/lib/stores/{mcp.svelte.ts => mcp/index.svelte.ts} (67%) rename tools/ui/src/lib/stores/{mcp-resources.svelte.ts => mcp/resources.svelte.ts} (96%) delete mode 100644 tools/ui/src/lib/stores/models.svelte.ts create mode 100644 tools/ui/src/lib/stores/models/index.svelte.ts create mode 100644 tools/ui/src/lib/stores/models/props.svelte.ts create mode 100644 tools/ui/src/lib/stores/models/status.svelte.ts rename tools/ui/src/lib/stores/{settings.svelte.ts => settings/index.svelte.ts} (89%) rename tools/ui/src/lib/stores/{settings-referrer.svelte.ts => settings/referrer.svelte.ts} (50%) create mode 100644 tools/ui/tests/unit/chat-activity.test.ts diff --git a/tools/ui/README.md b/tools/ui/README.md index 53b5925e2..99abfaa41 100644 --- a/tools/ui/README.md +++ b/tools/ui/README.md @@ -239,31 +239,44 @@ Routes → Components → Hooks → Stores → Services → Storage/API ### High-Level Architecture -See: [`docs/architecture/high-level-architecture-simplified.md`](docs/architecture/high-level-architecture-simplified.md) - ```mermaid flowchart TB subgraph Routes["📍 Routes"] R1["/ (Welcome)"] R2["/chat/[id]"] + R3["/mcp-servers"] + R4["/search"] + R5["/settings"] RL["+layout.svelte"] end subgraph Components["🧩 Components"] - C_Sidebar["ChatSidebar"] C_Screen["ChatScreen"] C_Form["ChatForm"] C_Messages["ChatMessages"] - C_ModelsSelector["ModelsSelector"] + C_Sidebar["ChatSidebar"] + C_Models["ModelsSelector"] C_Settings["ChatSettings"] + C_Mcp["McpServers"] + end + + subgraph Hooks["🔌 Hooks"] + H1["use-chat-screen-active-model"] + H2["use-processing-state"] + H3["use-context-gauge"] + H4["use-models-selector"] + H5["use-tools-panel"] end subgraph Stores["🗄️ Stores"] S1["chatStore"] S2["conversationsStore"] S3["modelsStore"] - S4["serverStore"] - S5["settingsStore"] + S4["mcpStore"] + S5["agenticStore"] + S6["serverStore"] + S7["settingsStore"] + S8["toolsStore"] end subgraph Services["⚙️ Services"] @@ -271,6 +284,9 @@ flowchart TB SV2["ModelsService"] SV3["PropsService"] SV4["DatabaseService"] + SV5["MCPService"] + SV6["ToolsService"] + SV7["SandboxService"] end subgraph Storage["💾 Storage"] @@ -282,19 +298,28 @@ flowchart TB API1["/v1/chat/completions"] API2["/props"] API3["/models/*"] + API4["/tools"] end R1 & R2 --> C_Screen RL --> C_Sidebar C_Screen --> C_Form & C_Messages & C_Settings - C_Screen --> S1 & S2 - C_ModelsSelector --> S3 & S4 + C_Screen --> H1 & H2 & H3 + C_Models --> H4 + C_Mcp --> S4 + C_Screen --> S1 & S2 & S3 + C_Models --> S3 + H1 --> S3 S1 --> SV1 & SV4 + S2 --> SV4 S3 --> SV2 & SV3 + S4 --> SV5 + S5 --> SV1 & SV5 & SV6 & SV7 SV4 --> ST1 SV1 --> API1 SV2 --> API3 SV3 --> API2 + SV6 --> API4 ``` ### Layer Breakdown @@ -303,6 +328,9 @@ flowchart TB - **`/`** - Welcome screen, creates new conversation - **`/chat/[id]`** - Active chat interface +- **`/mcp-servers`** - MCP server management +- **`/search`** - Conversation search +- **`/settings`** - Settings (optional `[[section]]`) - **`+layout.svelte`** - Sidebar, navigation, global initialization #### Components (`src/lib/components/`) @@ -348,28 +376,68 @@ Components are organized in `app/` (application-specific) and `ui/` (shadcn-svel #### Hooks (`src/lib/hooks/`) -- **`useModelChangeValidation`** - Validates model switch against conversation modalities -- **`useProcessingState`** - Tracks streaming progress and token generation +Hooks are the thin view-layer between components and stores: they own UI concerns (scroll, drag-and-drop, keyboard shortcuts, pickers, selection) and translate store state into view state. + +| Hook | Responsibility | +| ------------------------------- | -------------------------------------------------------------- | +| `use-chat-screen-active-model` | Active model resolution + modality capability detection | +| `use-processing-state` | View over `chatStore.processing` for streaming progress/tokens | +| `use-context-gauge` | View over `contextStatsStore` for the context usage gauge | +| `use-models-selector` | Model selector dropdown state (loaded/available groups) | +| `use-tools-panel` | Tools panel state | +| `use-reasoning-menu` | Reasoning-effort menu state | +| `use-attachment-menu` | Attachment menu + modality flags | +| `use-draft-messages` | Per-chat draft message/files persistence | +| `use-chat-form-pickers` | Chat form pickers (commands, mentions) | +| `use-debounced-search` | Shared debounced async search for pickers | +| `use-picker-navigation` | Picker keyboard navigation | +| `use-chat-message-edit-context` | Message edit context (content + extras) | +| `use-chat-screen-drag-and-drop` | Drag-and-drop state machine | +| `use-chat-screen-file-upload` | File upload queue + capability validation | +| `use-chat-screen-scroll` | Scroll container binding + navigation guard | +| `use-auto-scroll` | Auto-scroll controller for streaming | +| `use-marquee-selection` | Shift+click / marquee range selection | +| `use-keyboard-shortcuts` | Global keyboard shortcuts | +| `use-settings-navigation` | Settings section navigation | +| `use-pwa` | PWA install/update + version mismatch detection | #### Stores (`src/lib/stores/`) -| Store | Responsibility | -| -------------------- | --------------------------------------------------------- | -| `chatStore` | Message sending, streaming, abort control, error handling | -| `conversationsStore` | CRUD for conversations, message branching, navigation | -| `modelsStore` | Model list, selection, loading/unloading (ROUTER) | -| `serverStore` | Server properties, role detection, modalities | -| `settingsStore` | User preferences, parameter sync with server defaults | +Stores own reactive application state as Svelte 5 runes. Larger stores are split into directories and compose focused sub-stores behind a narrow host interface (see Architectural Patterns). + +| Store | Responsibility | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | +| `chatStore` | Chat lifecycle, streaming, abort control, error handling; composes `processing`, `activity`, `streams`, `flows` | +| `conversationsStore` | Conversation CRUD, message branching, navigation, import/export; composes `preferences` | +| `modelsStore` | Model list, selection, loading/unloading (ROUTER); composes `props`, `status` | +| `mcpStore` | MCP host role: multi-server lifecycle, tool routing; composes `health`, `resources` | +| `agenticStore` | Multi-turn agentic loop orchestration, tool execution; composes `gates` | +| `serverStore` | Server connection state, `/props`, role detection, modalities | +| `settingsStore` | User preferences, theme, parameter sync with server defaults | +| `toolsStore` | Tool registry: server + MCP tools, enabled set for the LLM | +| `permissionsStore` | Persisted tool permission grants | +| `contextStatsStore` | Context window usage for the active conversation | +| `draftMessagesStore` | Per-chat draft message/files | +| `deviceStore` | Browser environment signals (mobile, OS, theme) | +| `versionStore` | Build version information | #### Services (`src/lib/services/`) -| Service | Responsibility | -| ---------------------- | ----------------------------------------------- | -| `ChatService` | API calls to`/v1/chat/completions`, SSE parsing | -| `ModelsService` | `/models`, `/models/load`, `/models/unload` | -| `PropsService` | `/props`, `/props?model=` | -| `DatabaseService` | IndexedDB operations via Dexie | -| `ParameterSyncService` | Syncs settings with server defaults | +Services are a stateless protocol layer: static methods, pure I/O, no reactive state. Stores consume them for all API and storage access. + +| Service | Responsibility | +| ----------------------------- | ------------------------------------------------------------------------- | +| `ChatService` | `/v1/chat/completions` streaming + SSE parsing, message format conversion | +| `ModelsService` | `/models`, `/models/load`, `/models/unload` | +| `PropsService` | `/props`, `/props?model=` | +| `DatabaseService` | IndexedDB operations via Dexie | +| `MCPService` | MCP protocol: transports, connect, list/execute tools, prompts, resources | +| `ToolsService` | Server tool list/execute/stream (`/tools`) | +| `SandboxService` | Browser JS execution in a sandboxed worker | +| `ParameterSyncService` | Syncs settings with server defaults | +| `ConversationTransferService` | Conversation import/export JSONL + ZIP format | +| `MigrationService` | Non-destructive localStorage/IndexedDB migrations | +| `RouterService` | Dynamic route URL construction | --- @@ -377,8 +445,6 @@ Components are organized in `app/` (application-specific) and `ui/` (shadcn-svel ### MODEL Mode (Single Model) -See: [`docs/flows/data-flow-simplified-model-mode.md`](docs/flows/data-flow-simplified-model-mode.md) - ```mermaid sequenceDiagram participant User @@ -388,8 +454,9 @@ sequenceDiagram participant API as llama-server Note over User,API: Initialization - UI->>Stores: initialize() - Stores->>DB: load conversations + UI->>Stores: initStores() (awaited by route loads) + Stores->>Stores: run migrations + Stores->>DB: load conversations (background) Stores->>API: GET /props API-->>Stores: server config Stores->>API: GET /v1/models @@ -408,8 +475,6 @@ sequenceDiagram ### ROUTER Mode (Multi-Model) -See: [`docs/flows/data-flow-simplified-router-mode.md`](docs/flows/data-flow-simplified-router-mode.md) - ```mermaid sequenceDiagram participant User @@ -441,17 +506,6 @@ sequenceDiagram end ``` -### Detailed Flow Diagrams - -| Flow | Description | File | -| ------------- | ------------------------------------------ | ----------------------------------------------------------- | -| Chat | Message lifecycle, streaming, regeneration | [`chat-flow.md`](docs/flows/chat-flow.md) | -| Models | Loading, unloading, modality caching | [`models-flow.md`](docs/flows/models-flow.md) | -| Server | Props fetching, role detection | [`server-flow.md`](docs/flows/server-flow.md) | -| Conversations | CRUD, branching, import/export | [`conversations-flow.md`](docs/flows/conversations-flow.md) | -| Database | IndexedDB schema, operations | [`database-flow.md`](docs/flows/database-flow.md) | -| Settings | Parameter sync, user overrides | [`settings-flow.md`](docs/flows/settings-flow.md) | - --- ## Architectural Patterns @@ -505,13 +559,14 @@ Components dispatch actions to stores, stores coordinate with services for I/O, ### 3. Per-Conversation State -Enables concurrent streaming across multiple conversations: +Enables concurrent streaming across multiple conversations. Loading is tracked +per conversation by the activity ledger (`chatStore.activity`), while streaming +state and abort controllers live in per-conversation maps: ```typescript class ChatStore { - chatLoadingStates = new Map(); - chatStreamingStates = new Map(); - abortControllers = new Map(); + chatStreamingStates = new SvelteMap(); + abortControllers = new SvelteMap(); } ``` @@ -567,20 +622,14 @@ get isRouterMode() { ### 7. Modality Validation -Prevents sending attachments to incompatible models: +Prevents sending attachments to incompatible models. The +`use-chat-screen-active-model` hook derives the active model's capabilities +from `modelsStore.props`: ```typescript -// useModelChangeValidation hook -const validate = (modelId: string) => { - const modelModalities = modelsStore.getModelModalities(modelId); - const conversationModalities = conversationsStore.usedModalities; - - // Check if model supports all used modalities - if (conversationModalities.hasImages && !modelModalities.vision) { - return { valid: false, reason: 'Model does not support images' }; - } - // ... -}; +// use-chat-screen-active-model hook +const hasVisionModality = $derived.by(() => modelsStore.props.modelSupportsVision(activeModelId)); +const hasAudioModality = $derived.by(() => modelsStore.props.modelSupportsAudio(activeModelId)); ``` ### 8. Persistent Storage Strategy @@ -673,9 +722,6 @@ tools/ui/ │ └── styles/ # Global styles ├── static/ # Static assets ├── tests/ # Test files -├── docs/ # Architecture diagrams -│ ├── architecture/ # High-level architecture -│ └── flows/ # Feature-specific flows └── .storybook/ # Storybook configuration ``` diff --git a/tools/ui/docs/architecture/high-level-architecture-simplified.md b/tools/ui/docs/architecture/high-level-architecture-simplified.md deleted file mode 100644 index 500f477c9..000000000 --- a/tools/ui/docs/architecture/high-level-architecture-simplified.md +++ /dev/null @@ -1,145 +0,0 @@ -```mermaid -flowchart TB - subgraph Routes["📍 Routes"] - R1["/ (Welcome)"] - R2["/chat/[id]"] - RL["+layout.svelte"] - end - - subgraph Components["🧩 Components"] - C_Sidebar["ChatSidebar"] - C_Screen["ChatScreen"] - C_Form["ChatForm"] - C_Messages["ChatMessages"] - C_Message["ChatMessage"] - C_ChatMessageAgenticContent["ChatMessageAgenticContent"] - C_MessageEditForm["ChatMessageEditForm"] - C_ModelsSelector["ModelsSelector"] - C_Settings["ChatSettings"] - C_McpSettings["McpServersSettings"] - C_McpResourceBrowser["McpResourceBrowser"] - C_McpServersSelector["McpServersSelector"] - end - - subgraph Hooks["🪝 Hooks"] - H1["useModelChangeValidation"] - H2["useProcessingState"] - end - - subgraph Stores["🗄️ Stores"] - S1["chatStore
Chat interactions & streaming"] - SA["agenticStore
Multi-turn agentic loop orchestration"] - S2["conversationsStore
Conversation data, messages & MCP overrides"] - S3["modelsStore
Model selection & loading"] - S4["serverStore
Server props & role detection"] - S5["settingsStore
User configuration incl. MCP"] - S6["mcpStore
MCP servers, tools, prompts"] - S7["mcpResourceStore
MCP resources & attachments"] - end - - subgraph Services["⚙️ Services"] - SV1["ChatService"] - SV2["ModelsService"] - SV3["PropsService"] - SV4["DatabaseService"] - SV5["ParameterSyncService"] - SV6["MCPService
protocol operations"] - end - - subgraph Storage["💾 Storage"] - ST1["IndexedDB
conversations, messages"] - ST2["LocalStorage
config, userOverrides, mcpServers"] - end - - subgraph APIs["🌐 llama-server API"] - API1["/v1/chat/completions"] - API2["/props"] - API3["/models/*"] - API4["/v1/models"] - end - - subgraph ExternalMCP["🔌 External MCP Servers"] - EXT1["MCP Server 1
WebSocket/HTTP/SSE"] - EXT2["MCP Server N"] - end - - %% Routes → Components - R1 & R2 --> C_Screen - RL --> C_Sidebar - - %% Layout runs MCP health checks - RL --> S6 - - %% Component hierarchy - C_Screen --> C_Form & C_Messages & C_Settings - C_Messages --> C_Message - C_Message --> C_ChatMessageAgenticContent - C_Message --> C_MessageEditForm - C_Form & C_MessageEditForm --> C_ModelsSelector - C_Form --> C_McpServersSelector - C_Settings --> C_McpSettings - C_McpSettings --> C_McpResourceBrowser - - %% Components → Hooks → Stores - C_Form & C_Messages --> H1 & H2 - H1 --> S3 & S4 - H2 --> S1 & S5 - - %% Components → Stores - C_Screen --> S1 & S2 - C_Sidebar --> S2 - C_ModelsSelector --> S3 & S4 - C_Settings --> S5 - C_McpSettings --> S6 - C_McpResourceBrowser --> S6 & S7 - C_McpServersSelector --> S6 - C_Form --> S6 - - %% chatStore → agenticStore → mcpStore (agentic loop) - S1 --> SA - SA --> SV1 - SA --> S6 - - %% Stores → Services - S1 --> SV1 & SV4 - S2 --> SV4 - S3 --> SV2 & SV3 - S4 --> SV3 - S5 --> SV5 - S6 --> SV6 - S7 --> SV6 - - %% Services → Storage - SV4 --> ST1 - SV5 --> ST2 - - %% Services → APIs - SV1 --> API1 - SV2 --> API3 & API4 - SV3 --> API2 - - %% MCP → External Servers - SV6 --> EXT1 & EXT2 - - %% Styling - classDef routeStyle fill:#e1f5fe,stroke:#01579b,stroke-width:2px - classDef componentStyle fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px - classDef hookStyle fill:#fff8e1,stroke:#ff8f00,stroke-width:2px - classDef storeStyle fill:#fff3e0,stroke:#e65100,stroke-width:2px - classDef serviceStyle fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px - classDef storageStyle fill:#fce4ec,stroke:#c2185b,stroke-width:2px - classDef apiStyle fill:#e3f2fd,stroke:#1565c0,stroke-width:2px - classDef mcpStyle fill:#e0f2f1,stroke:#00695c,stroke-width:2px - classDef agenticStyle fill:#e8eaf6,stroke:#283593,stroke-width:2px - classDef externalStyle fill:#f3e5f5,stroke:#6a1b9a,stroke-width:2px,stroke-dasharray: 5 5 - - class R1,R2,RL routeStyle - class C_Sidebar,C_Screen,C_Form,C_Messages,C_Message,C_ChatMessageAgenticContent,C_MessageEditForm,C_ModelsSelector,C_Settings componentStyle - class C_McpSettings,C_McpResourceBrowser,C_McpServersSelector componentStyle - class H1,H2 hookStyle - class S1,S2,S3,S4,S5,SA,S6,S7 storeStyle - class SV1,SV2,SV3,SV4,SV5,SV6 serviceStyle - class ST1,ST2 storageStyle - class API1,API2,API3,API4 apiStyle - class EXT1,EXT2 externalStyle -``` diff --git a/tools/ui/docs/architecture/high-level-architecture.md b/tools/ui/docs/architecture/high-level-architecture.md deleted file mode 100644 index 42ddb3f4f..000000000 --- a/tools/ui/docs/architecture/high-level-architecture.md +++ /dev/null @@ -1,373 +0,0 @@ -```mermaid -flowchart TB -subgraph Routes["📍 Routes"] -R1["/ (+page.svelte)"] -R2["/chat/[id]"] -RL["+layout.svelte"] -end - - subgraph Components["🧩 Components"] - direction TB - subgraph LayoutComponents["Layout"] - C_Sidebar["ChatSidebar"] - C_Screen["ChatScreen"] - end - subgraph ChatUIComponents["Chat UI"] - C_Form["ChatForm"] - C_Messages["ChatMessages"] - C_Message["ChatMessage"] - C_MessageUser["ChatMessageUser"] - C_MessageEditForm["ChatMessageEditForm"] - C_Attach["ChatAttachments"] - C_ModelsSelector["ModelsSelector"] - C_Settings["ChatSettings"] - end - subgraph MCPComponents["MCP UI"] - C_McpSettings["McpServersSettings"] - C_McpServerCard["McpServerCard"] - C_McpResourceBrowser["McpResourceBrowser"] - C_McpResourcePreview["McpResourcePreview"] - C_McpServersSelector["McpServersSelector"] - end - end - - subgraph Hooks["🪝 Hooks"] - H1["useModelChangeValidation"] - H2["useProcessingState"] - H3["isMobile"] - end - - subgraph Stores["🗄️ Stores"] - direction TB - subgraph S1["chatStore"] - S1State["State:
isLoading, currentResponse
errorDialogState
activeProcessingState
chatLoadingStates
chatStreamingStates
abortControllers
processingStates
activeConversationId
isStreamingActive"] - S1LoadState["Loading State:
setChatLoading()
isChatLoading()
syncLoadingStateForChat()
clearUIState()
isChatLoadingPublic()
getAllLoadingChats()
getAllStreamingChats()"] - S1ProcState["Processing State:
setActiveProcessingConversation()
getProcessingState()
clearProcessingState()
getActiveProcessingState()
updateProcessingStateFromTimings()
getCurrentProcessingStateSync()
restoreProcessingStateFromMessages()"] - S1Stream["Streaming:
streamChatCompletion()
startStreaming()
stopStreaming()
stopGeneration()
isStreaming()"] - S1Error["Error Handling:
showErrorDialog()
dismissErrorDialog()
isAbortError()"] - S1Msg["Message Operations:
addMessage()
sendMessage()
updateMessage()
deleteMessage()
getDeletionInfo()"] - S1Regen["Regeneration:
regenerateMessage()
regenerateMessageWithBranching()
continueAssistantMessage()"] - S1Edit["Editing:
editAssistantMessage()
editUserMessagePreserveResponses()
editMessageWithBranching()
clearEditMode()
isEditModeActive()
getAddFilesHandler()
setEditModeActive()"] - S1Utils["Utilities:
getApiOptions()
parseTimingData()
getOrCreateAbortController()
getConversationModel()"] - end - subgraph SA["agenticStore"] - SAState["State:
sessions (Map)
isAnyRunning"] - SASession["Session Management:
getSession()
updateSession()
clearSession()
getActiveSessions()
isRunning()
currentTurn()
totalToolCalls()
lastError()
streamingToolCall()"] - SAConfig["Configuration:
getConfig()
maxTurns, maxToolPreviewLines"] - SAFlow["Agentic Loop:
runAgenticFlow()
executeAgenticLoop()
normalizeToolCalls()
emitToolCallResult()
extractBase64Attachments()"] - end - subgraph S2["conversationsStore"] - S2State["State:
conversations
activeConversation
activeMessages
isInitialized
pendingMcpServerOverrides
titleUpdateConfirmationCallback"] - S2Lifecycle["Lifecycle:
initialize()
loadConversations()
clearActiveConversation()"] - S2ConvCRUD["Conversation CRUD:
createConversation()
loadConversation()
deleteConversation()
deleteAll()
updateConversationName()
updateConversationTitleWithConfirmation()"] - S2MsgMgmt["Message Management:
refreshActiveMessages()
addMessageToActive()
updateMessageAtIndex()
findMessageIndex()
sliceActiveMessages()
removeMessageAtIndex()
getConversationMessages()"] - S2Nav["Navigation:
navigateToSibling()
updateCurrentNode()
updateConversationTimestamp()"] - S2McpOverrides["MCP Per-Chat Overrides:
getMcpServerOverride()
getAllMcpServerOverrides()
setMcpServerOverride()
toggleMcpServerForChat()
removeMcpServerOverride()
isMcpServerEnabledForChat()
clearPendingMcpServerOverrides()"] - S2Export["Import/Export:
downloadConversation()
exportAllConversations()
importConversations()
importConversationsData()
triggerDownload()"] - S2Utils["Utilities:
setTitleUpdateConfirmationCallback()"] - end - subgraph S3["modelsStore"] - S3State["State:
models, routerModels
selectedModelId
selectedModelName
loading, updating, error
modelLoadingStates
modelPropsCache
modelPropsFetching
propsCacheVersion"] - S3Getters["Computed Getters:
selectedModel
loadedModelIds
loadingModelIds
singleModelName"] - S3Modal["Modalities:
getModelModalities()
modelSupportsVision()
modelSupportsAudio()
getModelModalitiesArray()
getModelProps()
updateModelModalities()"] - S3Status["Status Queries:
isModelLoaded()
isModelOperationInProgress()
getModelStatus()
isModelPropsFetching()"] - S3Fetch["Data Fetching:
fetch()
fetchRouterModels()
fetchModelProps()
fetchModalitiesForLoadedModels()"] - S3Select["Model Selection:
selectModelById()
selectModelByName()
clearSelection()
findModelByName()
findModelById()
hasModel()"] - S3LoadUnload["Loading/Unloading Models:
loadModel()
unloadModel()
ensureModelLoaded()
waitForModelStatus()
pollForModelStatus()"] - S3Utils["Utilities:
toDisplayName()
clear()"] - end - subgraph S4["serverStore"] - S4State["State:
props
loading, error
role
fetchPromise"] - S4Getters["Getters:
defaultParams
contextSize
isRouterMode
isModelMode"] - S4Data["Data Handling:
fetch()
getErrorMessage()
clear()"] - S4Utils["Utilities:
detectRole()"] - end - subgraph S5["settingsStore"] - S5State["State:
config
theme
isInitialized
userOverrides"] - S5Lifecycle["Lifecycle:
initialize()
loadConfig()
saveConfig()
loadTheme()
saveTheme()"] - S5Update["Config Updates:
updateConfig()
updateMultipleConfig()
updateTheme()"] - S5Reset["Reset:
resetConfig()
resetTheme()
resetAll()
resetParameterToServerDefault()"] - S5Sync["Server Sync:
syncWithServerDefaults()
forceSyncWithServerDefaults()"] - S5Utils["Utilities:
getConfig()
getAllConfig()
getParameterInfo()
getParameterDiff()
getServerDefaults()
clearAllUserOverrides()"] - end - subgraph S6["mcpStore"] - S6State["State:
isInitializing, error
toolCount, connectedServers
healthChecks (Map)
connections (Map)
toolsIndex (Map)"] - S6Lifecycle["Lifecycle:
ensureInitialized()
initialize()
shutdown()
acquireConnection()
releaseConnection()"] - S6Health["Health Checks:
runHealthCheck()
runHealthChecksForServers()
updateHealthCheck()
getHealthCheckState()
clearHealthCheck()"] - S6Servers["Server Management:
getServers()
addServer()
updateServer()
removeServer()
getServerById()
getServerDisplayName()"] - S6Tools["Tool Operations:
getToolDefinitionsForLLM()
getToolNames()
hasTool()
getToolServer()
executeTool()
executeToolByName()"] - S6Prompts["Prompt Operations:
getAllPrompts()
getPrompt()
hasPromptsCapability()
getPromptCompletions()"] - end - subgraph S7["mcpResourceStore"] - S7State["State:
serverResources (Map)
cachedResources (Map)
subscriptions (Map)
attachments[]
isLoading"] - S7Resources["Resource Discovery:
setServerResources()
getServerResources()
getAllResourceInfos()
getAllTemplateInfos()
clearServerResources()"] - S7Cache["Caching:
cacheResourceContent()
getCachedContent()
invalidateCache()
clearCache()"] - S7Subs["Subscriptions:
addSubscription()
removeSubscription()
isSubscribed()
handleResourceUpdate()"] - S7Attach["Attachments:
addAttachment()
updateAttachmentContent()
removeAttachment()
clearAttachments()
toMessageExtras()"] - end - - subgraph ReactiveExports["⚡ Reactive Exports"] - direction LR - subgraph ChatExports["chatStore"] - RE1["isLoading()"] - RE2["currentResponse()"] - RE3["errorDialog()"] - RE4["activeProcessingState()"] - RE5["isChatStreaming()"] - RE6["isChatLoading()"] - RE7["getChatStreaming()"] - RE8["getAllLoadingChats()"] - RE9["getAllStreamingChats()"] - RE9a["isEditModeActive()"] - RE9b["getAddFilesHandler()"] - RE9c["setEditModeActive()"] - RE9d["clearEditMode()"] - end - subgraph AgenticExports["agenticStore"] - REA1["agenticIsRunning()"] - REA2["agenticCurrentTurn()"] - REA3["agenticTotalToolCalls()"] - REA4["agenticLastError()"] - REA5["agenticStreamingToolCall()"] - REA6["agenticIsAnyRunning()"] - end - subgraph ConvExports["conversationsStore"] - RE10["conversations()"] - RE11["activeConversation()"] - RE12["activeMessages()"] - RE13["isConversationsInitialized()"] - end - subgraph ModelsExports["modelsStore"] - RE15["modelOptions()"] - RE16["routerModels()"] - RE17["modelsLoading()"] - RE18["modelsUpdating()"] - RE19["modelsError()"] - RE20["selectedModelId()"] - RE21["selectedModelName()"] - RE22["selectedModelOption()"] - RE23["loadedModelIds()"] - RE24["loadingModelIds()"] - RE25["propsCacheVersion()"] - RE26["singleModelName()"] - end - subgraph ServerExports["serverStore"] - RE27["serverProps()"] - RE28["serverLoading()"] - RE29["serverError()"] - RE30["serverRole()"] - RE31["defaultParams()"] - RE32["contextSize()"] - RE33["isRouterMode()"] - RE34["isModelMode()"] - end - subgraph SettingsExports["settingsStore"] - RE35["config()"] - RE36["theme()"] - RE37["isInitialized()"] - end - subgraph MCPExports["mcpStore / mcpResourceStore"] - RE38["mcpResources()"] - RE39["mcpResourceAttachments()"] - RE40["mcpHasResourceAttachments()"] - RE41["mcpTotalResourceCount()"] - RE42["mcpResourcesLoading()"] - end - end - end - - subgraph Services["⚙️ Services"] - direction TB - subgraph SV1["ChatService"] - SV1Msg["Messaging:
sendMessage()"] - SV1Stream["Streaming:
handleStreamResponse()
handleNonStreamResponse()"] - SV1Convert["Conversion:
convertDbMessageToApiChatMessageData()
mergeToolCallDeltas()"] - SV1Utils["Utilities:
stripReasoningContent()
extractModelName()
parseErrorResponse()"] - end - subgraph SV2["ModelsService"] - SV2List["Listing:
list()
listRouter()"] - SV2LoadUnload["Load/Unload:
load()
unload()"] - SV2Status["Status:
isModelLoaded()
isModelLoading()"] - end - subgraph SV3["PropsService"] - SV3Fetch["Fetching:
fetch()
fetchForModel()"] - end - subgraph SV4["DatabaseService"] - SV4Conv["Conversations:
createConversation()
getConversation()
getAllConversations()
updateConversation()
deleteConversation()"] - SV4Msg["Messages:
createMessageBranch()
createRootMessage()
createSystemMessage()
getConversationMessages()
updateMessage()
deleteMessage()
deleteMessageCascading()"] - SV4Node["Navigation:
updateCurrentNode()"] - SV4Import["Import:
importConversations()"] - end - subgraph SV5["ParameterSyncService"] - SV5Extract["Extraction:
extractServerDefaults()"] - SV5Merge["Merging:
mergeWithServerDefaults()"] - SV5Info["Info:
getParameterInfo()
canSyncParameter()
getSyncableParameterKeys()
validateServerParameter()"] - SV5Diff["Diff:
createParameterDiff()"] - end - subgraph SV6["MCPService"] - SV6Transport["Transport:
createTransport()
WebSocket / StreamableHTTP / SSE"] - SV6Conn["Connection:
connect()
disconnect()"] - SV6Tools["Tools:
listTools()
callTool()"] - SV6Prompts["Prompts:
listPrompts()
getPrompt()"] - SV6Resources["Resources:
listResources()
listResourceTemplates()
readResource()
subscribeResource()
unsubscribeResource()"] - SV6Complete["Completions:
complete()"] - end - end - - subgraph ExternalMCP["🔌 External MCP Servers"] - EXT1["MCP Server 1
(WebSocket/StreamableHTTP/SSE)"] - EXT2["MCP Server N"] - end - - subgraph Storage["💾 Storage"] - ST1["IndexedDB"] - ST2["conversations"] - ST3["messages"] - ST5["LocalStorage"] - ST6["config"] - ST7["userOverrides"] - ST8["mcpServers"] - end - - subgraph APIs["🌐 llama-server API"] - API1["/v1/chat/completions"] - API2["/props
/props?model="] - API3["/models
/models/load
/models/unload"] - API4["/v1/models"] - end - - %% Routes render Components - R1 --> C_Screen - R2 --> C_Screen - RL --> C_Sidebar - - %% Layout runs MCP health checks on startup - RL --> S6 - - %% Component hierarchy - C_Screen --> C_Form & C_Messages & C_Settings - C_Messages --> C_Message - C_Message --> C_MessageUser - C_MessageUser --> C_MessageEditForm - C_MessageEditForm --> C_ModelsSelector - C_MessageEditForm --> C_Attach - C_Form --> C_ModelsSelector - C_Form --> C_Attach - C_Form --> C_McpServersSelector - C_Message --> C_Attach - - %% MCP Components hierarchy - C_Settings --> C_McpSettings - C_McpSettings --> C_McpServerCard - C_McpServerCard --> C_McpResourceBrowser - C_McpResourceBrowser --> C_McpResourcePreview - - %% Components use Hooks - C_Form --> H1 - C_Message --> H1 & H2 - C_MessageEditForm --> H1 - C_Screen --> H2 - - %% Hooks use Stores - H1 --> S3 & S4 - H2 --> S1 & S5 - - %% Components use Stores - C_Screen --> S1 & S2 - C_Messages --> S2 - C_Message --> S1 & S2 & S3 - C_Form --> S1 & S3 & S6 - C_Sidebar --> S2 - C_ModelsSelector --> S3 & S4 - C_Settings --> S5 - C_McpSettings --> S6 - C_McpServerCard --> S6 - C_McpResourceBrowser --> S6 & S7 - C_McpServersSelector --> S6 - - %% Stores export Reactive State - S1 -. exports .-> ChatExports - SA -. exports .-> AgenticExports - S2 -. exports .-> ConvExports - S3 -. exports .-> ModelsExports - S4 -. exports .-> ServerExports - S5 -. exports .-> SettingsExports - S6 -. exports .-> MCPExports - S7 -. exports .-> MCPExports - - %% chatStore → agenticStore (agentic loop orchestration) - S1 --> SA - SA --> SV1 - SA --> S6 - - %% Stores use Services - S1 --> SV1 & SV4 - S2 --> SV4 - S3 --> SV2 & SV3 - S4 --> SV3 - S5 --> SV5 - S6 --> SV6 - S7 --> SV6 - - %% Services to Storage - SV4 --> ST1 - ST1 --> ST2 & ST3 - SV5 --> ST5 - ST5 --> ST6 & ST7 & ST8 - - %% Services to APIs - SV1 --> API1 - SV2 --> API3 & API4 - SV3 --> API2 - - %% MCP → External Servers - SV6 --> EXT1 & EXT2 - - %% Styling - classDef routeStyle fill:#e1f5fe,stroke:#01579b,stroke-width:2px - classDef componentStyle fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px - classDef componentGroupStyle fill:#e1bee7,stroke:#7b1fa2,stroke-width:1px - classDef hookStyle fill:#fff8e1,stroke:#ff8f00,stroke-width:2px - classDef storeStyle fill:#fff3e0,stroke:#e65100,stroke-width:2px - classDef stateStyle fill:#ffe0b2,stroke:#e65100,stroke-width:1px - classDef methodStyle fill:#ffecb3,stroke:#e65100,stroke-width:1px - classDef reactiveStyle fill:#fffde7,stroke:#f9a825,stroke-width:1px - classDef serviceStyle fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px - classDef serviceMStyle fill:#c8e6c9,stroke:#2e7d32,stroke-width:1px - classDef externalStyle fill:#f3e5f5,stroke:#6a1b9a,stroke-width:2px,stroke-dasharray: 5 5 - classDef storageStyle fill:#fce4ec,stroke:#c2185b,stroke-width:2px - classDef apiStyle fill:#e3f2fd,stroke:#1565c0,stroke-width:2px - - class R1,R2,RL routeStyle - class C_Sidebar,C_Screen,C_Form,C_Messages,C_Message,C_MessageUser,C_MessageEditForm componentStyle - class C_ModelsSelector,C_Settings componentStyle - class C_Attach componentStyle - class C_McpSettings,C_McpServerCard,C_McpResourceBrowser,C_McpResourcePreview,C_McpServersSelector componentStyle - class H1,H2,H3 hookStyle - class LayoutComponents,ChatUIComponents,MCPComponents componentGroupStyle - class Hooks hookStyle - classDef agenticStyle fill:#e8eaf6,stroke:#283593,stroke-width:2px - classDef agenticMethodStyle fill:#c5cae9,stroke:#283593,stroke-width:1px - - class S1,S2,S3,S4,S5,SA,S6,S7 storeStyle - class S1State,S2State,S3State,S4State,S5State,SAState,S6State,S7State stateStyle - class S1Msg,S1Regen,S1Edit,S1Stream,S1LoadState,S1ProcState,S1Error,S1Utils methodStyle - class SASession,SAConfig,SAFlow methodStyle - class S2Lifecycle,S2ConvCRUD,S2MsgMgmt,S2Nav,S2McpOverrides,S2Export,S2Utils methodStyle - class S3Getters,S3Modal,S3Status,S3Fetch,S3Select,S3LoadUnload,S3Utils methodStyle - class S4Getters,S4Data,S4Utils methodStyle - class S5Lifecycle,S5Update,S5Reset,S5Sync,S5Utils methodStyle - class S6Lifecycle,S6Health,S6Servers,S6Tools,S6Prompts methodStyle - class S7Resources,S7Cache,S7Subs,S7Attach methodStyle - class ChatExports,AgenticExports,ConvExports,ModelsExports,ServerExports,SettingsExports,MCPExports reactiveStyle - class SV1,SV2,SV3,SV4,SV5,SV6 serviceStyle - class SV6Transport,SV6Conn,SV6Tools,SV6Prompts,SV6Resources,SV6Complete serviceMStyle - class EXT1,EXT2 externalStyle - class SV1Msg,SV1Stream,SV1Convert,SV1Utils serviceMStyle - class SV2List,SV2LoadUnload,SV2Status serviceMStyle - class SV3Fetch serviceMStyle - class SV4Conv,SV4Msg,SV4Node,SV4Import serviceMStyle - class SV5Extract,SV5Merge,SV5Info,SV5Diff serviceMStyle - class ST1,ST2,ST3,ST5,ST6,ST7,ST8 storageStyle - class API1,API2,API3,API4 apiStyle -``` diff --git a/tools/ui/docs/flows/chat-flow.md b/tools/ui/docs/flows/chat-flow.md deleted file mode 100644 index 296693c6a..000000000 --- a/tools/ui/docs/flows/chat-flow.md +++ /dev/null @@ -1,228 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 ChatForm / ChatMessage - participant chatStore as 🗄️ chatStore - participant agenticStore as 🗄️ agenticStore - participant convStore as 🗄️ conversationsStore - participant settingsStore as 🗄️ settingsStore - participant mcpStore as 🗄️ mcpStore - participant ChatSvc as ⚙️ ChatService - participant DbSvc as ⚙️ DatabaseService - participant API as 🌐 /v1/chat/completions - - Note over chatStore: State:
isLoading, currentResponse
errorDialogState, activeProcessingState
chatLoadingStates (Map)
chatStreamingStates (Map)
abortControllers (Map)
processingStates (Map) - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 💬 SEND MESSAGE - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: sendMessage(content, extras) - activate chatStore - - chatStore->>chatStore: setChatLoading(convId, true) - chatStore->>chatStore: clearChatStreaming(convId) - - alt no active conversation - chatStore->>convStore: createConversation() - Note over convStore: → see conversations-flow.mmd - end - - chatStore->>mcpStore: consumeResourceAttachmentsAsExtras() - Note right of mcpStore: Converts pending MCP resource
attachments into message extras - - chatStore->>chatStore: addMessage("user", content, extras) - chatStore->>DbSvc: createMessageBranch(userMsg, parentId) - chatStore->>convStore: addMessageToActive(userMsg) - chatStore->>convStore: updateCurrentNode(userMsg.id) - - chatStore->>chatStore: createAssistantMessage(userMsg.id) - chatStore->>DbSvc: createMessageBranch(assistantMsg, userMsg.id) - chatStore->>convStore: addMessageToActive(assistantMsg) - - chatStore->>chatStore: streamChatCompletion(messages, assistantMsg) - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🌊 STREAMING (with agentic flow detection) - %% ═══════════════════════════════════════════════════════════════════════════ - - activate chatStore - chatStore->>chatStore: startStreaming() - Note right of chatStore: isStreamingActive = true - - chatStore->>chatStore: setActiveProcessingConversation(convId) - chatStore->>chatStore: getOrCreateAbortController(convId) - Note right of chatStore: abortControllers.set(convId, new AbortController()) - - chatStore->>chatStore: getApiOptions() - Note right of chatStore: Merge from settingsStore.config:
temperature, max_tokens, top_p, etc. - - alt agenticConfig.enabled && mcpStore has connected servers - chatStore->>agenticStore: runAgenticFlow(convId, messages, assistantMsg, options, signal) - Note over agenticStore: Multi-turn agentic loop:
1. Call ChatService.sendMessage()
2. If response has tool_calls → execute via mcpStore
3. Append tool results as messages
4. Loop until no more tool_calls or maxTurns
→ see agentic flow details below - agenticStore-->>chatStore: final response with timings - else standard (non-agentic) flow - chatStore->>ChatSvc: sendMessage(messages, options, signal) - end - - activate ChatSvc - - ChatSvc->>ChatSvc: convertDbMessageToApiChatMessageData(messages) - Note right of ChatSvc: DatabaseMessage[] → ApiChatMessageData[]
Process attachments (images, PDFs, audio) - - ChatSvc->>API: POST /v1/chat/completions - Note right of API: {messages, model?, stream: true, ...params} - - loop SSE chunks - API-->>ChatSvc: data: {"choices":[{"delta":{...}}]} - ChatSvc->>ChatSvc: handleStreamResponse(response) - - alt content chunk - ChatSvc-->>chatStore: onChunk(content) - chatStore->>chatStore: setChatStreaming(convId, response, msgId) - Note right of chatStore: currentResponse = $state(accumulated) - chatStore->>convStore: updateMessageAtIndex(idx, {content}) - end - - alt reasoning chunk - ChatSvc-->>chatStore: onReasoningChunk(reasoning) - chatStore->>convStore: updateMessageAtIndex(idx, {thinking}) - end - - alt tool_calls chunk - ChatSvc-->>chatStore: onToolCallChunk(toolCalls) - chatStore->>convStore: updateMessageAtIndex(idx, {toolCalls}) - end - - alt model info - ChatSvc-->>chatStore: onModel(modelName) - chatStore->>chatStore: recordModel(modelName) - chatStore->>DbSvc: updateMessage(msgId, {model}) - end - - alt timings (during stream) - ChatSvc-->>chatStore: onTimings(timings, promptProgress) - chatStore->>chatStore: updateProcessingStateFromTimings() - end - - chatStore-->>UI: reactive $state update - end - - API-->>ChatSvc: data: [DONE] - ChatSvc-->>chatStore: onComplete(content, reasoning, timings, toolCalls) - deactivate ChatSvc - - chatStore->>chatStore: stopStreaming() - chatStore->>DbSvc: updateMessage(msgId, {content, timings, model}) - chatStore->>convStore: updateCurrentNode(msgId) - chatStore->>chatStore: setChatLoading(convId, false) - chatStore->>chatStore: clearChatStreaming(convId) - chatStore->>chatStore: clearProcessingState(convId) - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ⏹️ STOP GENERATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: stopGeneration() - activate chatStore - chatStore->>chatStore: savePartialResponseIfNeeded(convId) - Note right of chatStore: Save currentResponse to DB if non-empty - chatStore->>chatStore: abortControllers.get(convId).abort() - Note right of chatStore: fetch throws AbortError → caught by isAbortError() - chatStore->>chatStore: stopStreaming() - chatStore->>chatStore: setChatLoading(convId, false) - chatStore->>chatStore: clearChatStreaming(convId) - chatStore->>chatStore: clearProcessingState(convId) - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🔁 REGENERATE - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: regenerateMessageWithBranching(msgId, model?) - activate chatStore - chatStore->>convStore: findMessageIndex(msgId) - chatStore->>chatStore: Get parent of target message - chatStore->>chatStore: createAssistantMessage(parentId) - chatStore->>DbSvc: createMessageBranch(newAssistantMsg, parentId) - chatStore->>convStore: refreshActiveMessages() - Note right of chatStore: Same streaming flow - chatStore->>chatStore: streamChatCompletion(...) - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ➡️ CONTINUE - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: continueAssistantMessage(msgId) - activate chatStore - chatStore->>chatStore: Get existing content from message - chatStore->>chatStore: streamChatCompletion(..., existingContent) - Note right of chatStore: Appends to existing message content - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ✏️ EDIT USER MESSAGE - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: editMessageWithBranching(msgId, newContent, extras) - activate chatStore - chatStore->>chatStore: Get parent of target message - chatStore->>DbSvc: createMessageBranch(editedMsg, parentId) - chatStore->>convStore: refreshActiveMessages() - Note right of chatStore: Creates new branch, original preserved - chatStore->>chatStore: createAssistantMessage(editedMsg.id) - chatStore->>chatStore: streamChatCompletion(...) - Note right of chatStore: Automatically regenerates response - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ❌ ERROR HANDLING - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over chatStore: On stream error (non-abort): - chatStore->>chatStore: showErrorDialog(type, message) - Note right of chatStore: errorDialogState = {type: 'timeout'|'server', message} - chatStore->>convStore: removeMessageAtIndex(failedMsgIdx) - chatStore->>DbSvc: deleteMessage(failedMsgId) - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🤖 AGENTIC LOOP (when agenticConfig.enabled) - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over agenticStore: agenticStore.runAgenticFlow(convId, messages, assistantMsg, options, signal) - activate agenticStore - agenticStore->>agenticStore: getSession(convId) or create new - agenticStore->>agenticStore: updateSession(turn: 0, running: true) - - loop executeAgenticLoop (until no tool_calls or maxTurns) - agenticStore->>agenticStore: turn++ - agenticStore->>ChatSvc: sendMessage(messages, options, signal) - ChatSvc->>API: POST /v1/chat/completions - API-->>ChatSvc: response with potential tool_calls - ChatSvc-->>agenticStore: onComplete(content, reasoning, timings, toolCalls) - - alt response has tool_calls - agenticStore->>agenticStore: normalizeToolCalls(toolCalls) - loop for each tool_call - agenticStore->>agenticStore: updateSession(streamingToolCall) - agenticStore->>mcpStore: executeTool(mcpCall, signal) - mcpStore-->>agenticStore: tool result - agenticStore->>agenticStore: extractBase64Attachments(result) - agenticStore->>agenticStore: emitToolCallResult(convId, ...) - agenticStore->>convStore: addMessageToActive(toolResultMsg) - agenticStore->>DbSvc: createMessageBranch(toolResultMsg) - end - agenticStore->>agenticStore: Create new assistantMsg for next turn - Note right of agenticStore: Continue loop with updated messages - else no tool_calls (final response) - agenticStore->>agenticStore: buildFinalTimings(allTurns) - Note right of agenticStore: Break loop, return final response - end - end - - agenticStore->>agenticStore: updateSession(running: false) - agenticStore-->>chatStore: final content, timings, model - deactivate agenticStore -``` diff --git a/tools/ui/docs/flows/conversations-flow.md b/tools/ui/docs/flows/conversations-flow.md deleted file mode 100644 index bd2309bc0..000000000 --- a/tools/ui/docs/flows/conversations-flow.md +++ /dev/null @@ -1,183 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 ChatSidebar / ChatScreen - participant convStore as 🗄️ conversationsStore - participant chatStore as 🗄️ chatStore - participant DbSvc as ⚙️ DatabaseService - participant IDB as 💾 IndexedDB - - Note over convStore: State:
conversations: DatabaseConversation[]
activeConversation: DatabaseConversation | null
activeMessages: DatabaseMessage[]
isInitialized: boolean
pendingMcpServerOverrides: Map<string, McpServerOverride> - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 🚀 INITIALIZATION - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over convStore: Auto-initialized in constructor (browser only) - convStore->>convStore: initialize() - activate convStore - convStore->>convStore: loadConversations() - convStore->>DbSvc: getAllConversations() - DbSvc->>IDB: SELECT * FROM conversations ORDER BY lastModified DESC - IDB-->>DbSvc: Conversation[] - DbSvc-->>convStore: conversations - convStore->>convStore: conversations = $state(data) - convStore->>convStore: isInitialized = true - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: ➕ CREATE CONVERSATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: createConversation(name?) - activate convStore - convStore->>DbSvc: createConversation(name || "New Chat") - DbSvc->>IDB: INSERT INTO conversations - IDB-->>DbSvc: conversation {id, name, lastModified, currNode: ""} - DbSvc-->>convStore: conversation - convStore->>convStore: conversations.unshift(conversation) - convStore->>convStore: activeConversation = $state(conversation) - convStore->>convStore: activeMessages = $state([]) - - alt pendingMcpServerOverrides has entries - loop each pending override - convStore->>DbSvc: Store MCP server override for new conversation - end - convStore->>convStore: clearPendingMcpServerOverrides() - end - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 📂 LOAD CONVERSATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: loadConversation(convId) - activate convStore - convStore->>DbSvc: getConversation(convId) - DbSvc->>IDB: SELECT * FROM conversations WHERE id = ? - IDB-->>DbSvc: conversation - convStore->>convStore: activeConversation = $state(conversation) - - convStore->>convStore: refreshActiveMessages() - convStore->>DbSvc: getConversationMessages(convId) - DbSvc->>IDB: SELECT * FROM messages WHERE convId = ? - IDB-->>DbSvc: allMessages[] - convStore->>convStore: filterByLeafNodeId(allMessages, currNode) - Note right of convStore: Filter to show only current branch path - convStore->>convStore: activeMessages = $state(filtered) - - Note right of convStore: Route (+page.svelte) then calls:
chatStore.syncLoadingStateForChat(convId) - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 🌳 MESSAGE BRANCHING MODEL - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over IDB: Message Tree Structure:
- Each message has parent (null for root)
- Each message has children[] array
- Conversation.currNode points to active leaf
- filterByLeafNodeId() traverses from root to currNode - - rect rgb(240, 240, 255) - Note over convStore: Example Branch Structure: - Note over convStore: root → user1 → assistant1 → user2 → assistant2a (currNode)
↘ assistant2b (alt branch) - end - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: ↔️ BRANCH NAVIGATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: navigateToSibling(msgId, direction) - activate convStore - convStore->>convStore: Find message in activeMessages - convStore->>convStore: Get parent message - convStore->>convStore: Find sibling in parent.children[] - convStore->>convStore: findLeafNode(siblingId, allMessages) - Note right of convStore: Navigate to leaf of sibling branch - convStore->>convStore: updateCurrentNode(leafId) - convStore->>DbSvc: updateCurrentNode(convId, leafId) - DbSvc->>IDB: UPDATE conversations SET currNode = ? - convStore->>convStore: refreshActiveMessages() - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 📝 UPDATE CONVERSATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: updateConversationName(convId, newName) - activate convStore - convStore->>DbSvc: updateConversation(convId, {name: newName}) - DbSvc->>IDB: UPDATE conversations SET name = ? - convStore->>convStore: Update in conversations array - deactivate convStore - - Note over convStore: Auto-title update (after first response): - convStore->>convStore: updateConversationTitleWithConfirmation() - convStore->>convStore: titleUpdateConfirmationCallback?() - Note right of convStore: Shows dialog if title would change - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 🗑️ DELETE CONVERSATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: deleteConversation(convId) - activate convStore - convStore->>DbSvc: deleteConversation(convId) - DbSvc->>IDB: DELETE FROM conversations WHERE id = ? - DbSvc->>IDB: DELETE FROM messages WHERE convId = ? - convStore->>convStore: conversations.filter(c => c.id !== convId) - alt deleted active conversation - convStore->>convStore: clearActiveConversation() - end - deactivate convStore - - UI->>convStore: deleteAll() - activate convStore - convStore->>DbSvc: Delete all conversations and messages - convStore->>convStore: conversations = [] - convStore->>convStore: clearActiveConversation() - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: � MCP SERVER PER-CHAT OVERRIDES - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over convStore: Conversations can override which MCP servers are enabled. - Note over convStore: Uses pendingMcpServerOverrides before conversation
is created, then persists to conversation metadata. - - UI->>convStore: setMcpServerOverride(convId, serverName, override) - Note right of convStore: override = {enabled: boolean} - - UI->>convStore: toggleMcpServerForChat(convId, serverName, enabled) - activate convStore - convStore->>convStore: setMcpServerOverride(convId, serverName, {enabled}) - deactivate convStore - - UI->>convStore: isMcpServerEnabledForChat(convId, serverName) - Note right of convStore: Check override → fall back to global MCP config - - UI->>convStore: getAllMcpServerOverrides(convId) - Note right of convStore: Returns all overrides for a conversation - - UI->>convStore: removeMcpServerOverride(convId, serverName) - UI->>convStore: getMcpServerOverride(convId, serverName) - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 📤 EXPORT / 📥 IMPORT - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: exportAllConversations() - activate convStore - convStore->>DbSvc: getAllConversations() - loop each conversation - convStore->>DbSvc: getConversationMessages(convId) - end - convStore->>convStore: triggerDownload(JSON blob) - deactivate convStore - - UI->>convStore: importConversations(file) - activate convStore - convStore->>convStore: Parse JSON file - convStore->>convStore: importConversationsData(parsed) - convStore->>DbSvc: importConversations(parsed) - Note right of DbSvc: Skips duplicate conversations
(checks existing by ID) - DbSvc->>IDB: INSERT conversations + messages (skip existing) - convStore->>convStore: loadConversations() - deactivate convStore -``` diff --git a/tools/ui/docs/flows/data-flow-simplified-model-mode.md b/tools/ui/docs/flows/data-flow-simplified-model-mode.md deleted file mode 100644 index 07b362147..000000000 --- a/tools/ui/docs/flows/data-flow-simplified-model-mode.md +++ /dev/null @@ -1,45 +0,0 @@ -```mermaid -%% MODEL Mode Data Flow (single model) -%% Detailed flows: ./flows/server-flow.mmd, ./flows/models-flow.mmd, ./flows/chat-flow.mmd - -sequenceDiagram - participant User as 👤 User - participant UI as 🧩 UI - participant Stores as 🗄️ Stores - participant DB as 💾 IndexedDB - participant API as 🌐 llama-server - - Note over User,API: 🚀 Initialization (see: server-flow.mmd, models-flow.mmd) - - UI->>Stores: initialize() - Stores->>DB: load conversations - Stores->>API: GET /props - API-->>Stores: server config + modalities - Stores->>API: GET /v1/models - API-->>Stores: single model (auto-selected) - - Note over User,API: 💬 Chat Flow (see: chat-flow.mmd) - - User->>UI: send message - UI->>Stores: sendMessage() - Stores->>DB: save user message - Stores->>API: POST /v1/chat/completions (stream) - loop streaming - API-->>Stores: SSE chunks - Stores-->>UI: reactive update - end - API-->>Stores: done + timings - Stores->>DB: save assistant message - - Note over User,API: 🔁 Regenerate - - User->>UI: regenerate - Stores->>DB: create message branch - Note right of Stores: same streaming flow - - Note over User,API: ⏹️ Stop - - User->>UI: stop - Stores->>Stores: abort stream - Stores->>DB: save partial response -``` diff --git a/tools/ui/docs/flows/data-flow-simplified-router-mode.md b/tools/ui/docs/flows/data-flow-simplified-router-mode.md deleted file mode 100644 index bccacf568..000000000 --- a/tools/ui/docs/flows/data-flow-simplified-router-mode.md +++ /dev/null @@ -1,77 +0,0 @@ -```mermaid -%% ROUTER Mode Data Flow (multi-model) -%% Detailed flows: ./flows/server-flow.mmd, ./flows/models-flow.mmd, ./flows/chat-flow.mmd - -sequenceDiagram - participant User as 👤 User - participant UI as 🧩 UI - participant Stores as 🗄️ Stores - participant DB as 💾 IndexedDB - participant API as 🌐 llama-server - - Note over User,API: 🚀 Initialization (see: server-flow.mmd, models-flow.mmd) - - UI->>Stores: initialize() - Stores->>DB: load conversations - Stores->>API: GET /props - API-->>Stores: {role: "router"} - Stores->>API: GET /v1/models - API-->>Stores: models[] with status (loaded/available) - loop each loaded model - Stores->>API: GET /props?model=X - API-->>Stores: modalities (vision/audio) - end - - Note over User,API: 🔄 Model Selection (see: models-flow.mmd) - - User->>UI: select model - alt model not loaded - Stores->>API: POST /models/load - loop poll status - Stores->>API: GET /v1/models - API-->>Stores: check if loaded - end - Stores->>API: GET /props?model=X - API-->>Stores: cache modalities - end - Stores->>Stores: validate modalities vs conversation - alt valid - Stores->>Stores: select model - else invalid - Stores->>API: POST /models/unload - UI->>User: show error toast - end - - Note over User,API: 💬 Chat Flow (see: chat-flow.mmd) - - User->>UI: send message - UI->>Stores: sendMessage() - Stores->>DB: save user message - Stores->>API: POST /v1/chat/completions {model: X} - Note right of API: router forwards to model - loop streaming - API-->>Stores: SSE chunks + model info - Stores-->>UI: reactive update - end - API-->>Stores: done + timings - Stores->>DB: save assistant message + model used - - Note over User,API: 🔁 Regenerate (optional: different model) - - User->>UI: regenerate - Stores->>Stores: validate modalities up to this message - Stores->>DB: create message branch - Note right of Stores: same streaming flow - - Note over User,API: ⏹️ Stop - - User->>UI: stop - Stores->>Stores: abort stream - Stores->>DB: save partial response - - Note over User,API: 🗑️ LRU Unloading - - Note right of API: Server auto-unloads LRU models
when cache full - User->>UI: select unloaded model - Note right of Stores: triggers load flow again -``` diff --git a/tools/ui/docs/flows/database-flow.md b/tools/ui/docs/flows/database-flow.md deleted file mode 100644 index 38cd6941c..000000000 --- a/tools/ui/docs/flows/database-flow.md +++ /dev/null @@ -1,174 +0,0 @@ -```mermaid -sequenceDiagram - participant Store as 🗄️ Stores - participant DbSvc as ⚙️ DatabaseService - participant Dexie as 📦 Dexie ORM - participant IDB as 💾 IndexedDB - - Note over DbSvc: Stateless service - all methods static
Database: "LlamacppWebui" - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 📊 SCHEMA - %% ═══════════════════════════════════════════════════════════════════════════ - - rect rgb(240, 248, 255) - Note over IDB: conversations table:
id (PK), lastModified, currNode, name - end - - rect rgb(255, 248, 240) - Note over IDB: messages table:
id (PK), convId (FK), type, role, timestamp,
parent, children[], content, thinking,
toolCalls, extra[], model, timings - end - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 💬 CONVERSATIONS CRUD - %% ═══════════════════════════════════════════════════════════════════════════ - - Store->>DbSvc: createConversation(name) - activate DbSvc - DbSvc->>DbSvc: Generate UUID - DbSvc->>Dexie: db.conversations.add({id, name, lastModified, currNode: ""}) - Dexie->>IDB: INSERT - IDB-->>Dexie: success - DbSvc-->>Store: DatabaseConversation - deactivate DbSvc - - Store->>DbSvc: getConversation(convId) - DbSvc->>Dexie: db.conversations.get(convId) - Dexie->>IDB: SELECT WHERE id = ? - IDB-->>DbSvc: DatabaseConversation - - Store->>DbSvc: getAllConversations() - DbSvc->>Dexie: db.conversations.orderBy('lastModified').reverse().toArray() - Dexie->>IDB: SELECT ORDER BY lastModified DESC - IDB-->>DbSvc: DatabaseConversation[] - - Store->>DbSvc: updateConversation(convId, updates) - DbSvc->>Dexie: db.conversations.update(convId, {...updates, lastModified}) - Dexie->>IDB: UPDATE - - Store->>DbSvc: deleteConversation(convId) - activate DbSvc - DbSvc->>Dexie: db.conversations.delete(convId) - Dexie->>IDB: DELETE FROM conversations - DbSvc->>Dexie: db.messages.where('convId').equals(convId).delete() - Dexie->>IDB: DELETE FROM messages WHERE convId = ? - deactivate DbSvc - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 📝 MESSAGES CRUD - %% ═══════════════════════════════════════════════════════════════════════════ - - Store->>DbSvc: createRootMessage(convId) - activate DbSvc - DbSvc->>DbSvc: Create root message {type: "root", parent: null} - DbSvc->>Dexie: db.messages.add(rootMsg) - Dexie->>IDB: INSERT - DbSvc-->>Store: rootMessageId - deactivate DbSvc - - Store->>DbSvc: createSystemMessage(convId, content, parentId) - activate DbSvc - DbSvc->>DbSvc: Create message {role: "system", parent: parentId} - DbSvc->>Dexie: db.messages.add(systemMsg) - Dexie->>IDB: INSERT - DbSvc-->>Store: DatabaseMessage - deactivate DbSvc - - Store->>DbSvc: createMessageBranch(message, parentId) - activate DbSvc - DbSvc->>DbSvc: Generate UUID for new message - DbSvc->>Dexie: db.messages.add({...message, id, parent: parentId}) - Dexie->>IDB: INSERT message - - alt parentId exists - DbSvc->>Dexie: db.messages.get(parentId) - Dexie->>IDB: SELECT parent - DbSvc->>DbSvc: parent.children.push(newId) - DbSvc->>Dexie: db.messages.update(parentId, {children}) - Dexie->>IDB: UPDATE parent.children - end - - DbSvc->>Dexie: db.conversations.update(convId, {currNode: newId}) - Dexie->>IDB: UPDATE conversation.currNode - DbSvc-->>Store: DatabaseMessage - deactivate DbSvc - - Store->>DbSvc: getConversationMessages(convId) - DbSvc->>Dexie: db.messages.where('convId').equals(convId).toArray() - Dexie->>IDB: SELECT WHERE convId = ? - IDB-->>DbSvc: DatabaseMessage[] - - Store->>DbSvc: updateMessage(msgId, updates) - DbSvc->>Dexie: db.messages.update(msgId, updates) - Dexie->>IDB: UPDATE - - Store->>DbSvc: deleteMessage(msgId) - DbSvc->>Dexie: db.messages.delete(msgId) - Dexie->>IDB: DELETE - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 🌳 BRANCHING OPERATIONS - %% ═══════════════════════════════════════════════════════════════════════════ - - Store->>DbSvc: updateCurrentNode(convId, nodeId) - DbSvc->>Dexie: db.conversations.update(convId, {currNode: nodeId, lastModified}) - Dexie->>IDB: UPDATE - - Store->>DbSvc: deleteMessageCascading(msgId) - activate DbSvc - DbSvc->>DbSvc: findDescendantMessages(msgId, allMessages) - Note right of DbSvc: Recursively find all children - loop each descendant - DbSvc->>Dexie: db.messages.delete(descendantId) - Dexie->>IDB: DELETE - end - DbSvc->>Dexie: db.messages.delete(msgId) - Dexie->>IDB: DELETE target message - - alt target message has a parent - DbSvc->>Dexie: db.messages.get(parentId) - DbSvc->>DbSvc: parent.children.filter(id !== msgId) - DbSvc->>Dexie: db.messages.update(parentId, {children}) - Note right of DbSvc: Remove deleted message from parent's children[] - end - deactivate DbSvc - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 📥 IMPORT - %% ═══════════════════════════════════════════════════════════════════════════ - - Store->>DbSvc: importConversations(data) - activate DbSvc - loop each conversation in data - DbSvc->>Dexie: db.conversations.get(conv.id) - alt conversation already exists - Note right of DbSvc: Skip duplicate (keep existing) - else conversation is new - DbSvc->>Dexie: db.conversations.add(conversation) - Dexie->>IDB: INSERT conversation - loop each message - DbSvc->>Dexie: db.messages.add(message) - Dexie->>IDB: INSERT message - end - end - end - deactivate DbSvc - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 🔗 MESSAGE TREE UTILITIES - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over DbSvc: Used by stores (imported from utils): - - rect rgb(240, 255, 240) - Note over DbSvc: filterByLeafNodeId(messages, leafId)
→ Returns path from root to leaf
→ Used to display current branch - end - - rect rgb(240, 255, 240) - Note over DbSvc: findLeafNode(startId, messages)
→ Traverse to deepest child
→ Used for branch navigation - end - - rect rgb(240, 255, 240) - Note over DbSvc: findDescendantMessages(msgId, messages)
→ Find all children recursively
→ Used for cascading deletes - end -``` diff --git a/tools/ui/docs/flows/mcp-flow.md b/tools/ui/docs/flows/mcp-flow.md deleted file mode 100644 index c8aa66659..000000000 --- a/tools/ui/docs/flows/mcp-flow.md +++ /dev/null @@ -1,226 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 McpServersSettings / ChatForm - participant chatStore as 🗄️ chatStore - participant mcpStore as 🗄️ mcpStore - participant mcpResStore as 🗄️ mcpResourceStore - participant convStore as 🗄️ conversationsStore - participant MCPSvc as ⚙️ MCPService - participant LS as 💾 LocalStorage - participant ExtMCP as 🔌 External MCP Server - - Note over mcpStore: State:
isInitializing, error
toolCount, connectedServers
healthChecks (Map)
connections (Map)
toolsIndex (Map)
serverConfigs (Map) - - Note over mcpResStore: State:
serverResources (Map)
cachedResources (Map)
subscriptions (Map)
attachments[] - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 🚀 INITIALIZATION (App Startup) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpStore: ensureInitialized() - activate mcpStore - - mcpStore->>LS: get(MCP_SERVERS_LOCALSTORAGE_KEY) - LS-->>mcpStore: MCPServerSettingsEntry[] - - mcpStore->>mcpStore: parseServerSettings(servers) - Note right of mcpStore: Filter enabled servers
Build MCPServerConfig objects
Per-chat overrides checked via convStore - - loop For each enabled server - mcpStore->>mcpStore: runHealthCheck(serverId) - mcpStore->>mcpStore: updateHealthCheck(id, CONNECTING) - - mcpStore->>MCPSvc: connect(serverName, config, clientInfo, capabilities, onPhase) - activate MCPSvc - - MCPSvc->>MCPSvc: createTransport(config) - Note right of MCPSvc: WebSocket / StreamableHTTP / SSE
with optional CORS proxy - - MCPSvc->>ExtMCP: Transport handshake - ExtMCP-->>MCPSvc: Connection established - - MCPSvc->>ExtMCP: Initialize request - Note right of ExtMCP: Exchange capabilities
Server info, protocol version - - ExtMCP-->>MCPSvc: InitializeResult (serverInfo, capabilities) - - MCPSvc->>ExtMCP: listTools() - ExtMCP-->>MCPSvc: Tool[] - - MCPSvc-->>mcpStore: MCPConnection - deactivate MCPSvc - - mcpStore->>mcpStore: connections.set(serverName, connection) - mcpStore->>mcpStore: indexTools(connection.tools, serverName) - Note right of mcpStore: toolsIndex.set(toolName, serverName)
Handle name conflicts with prefixes - - mcpStore->>mcpStore: updateHealthCheck(id, SUCCESS) - mcpStore->>mcpStore: _connectedServers.push(serverName) - - alt Server supports resources - mcpStore->>MCPSvc: listAllResources(connection) - MCPSvc->>ExtMCP: listResources() - ExtMCP-->>MCPSvc: MCPResource[] - MCPSvc-->>mcpStore: resources - - mcpStore->>MCPSvc: listAllResourceTemplates(connection) - MCPSvc->>ExtMCP: listResourceTemplates() - ExtMCP-->>MCPSvc: MCPResourceTemplate[] - MCPSvc-->>mcpStore: templates - - mcpStore->>mcpResStore: setServerResources(serverName, resources, templates) - end - end - - mcpStore->>mcpStore: _isInitializing = false - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 🔧 TOOL EXECUTION (Chat with Tools) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpStore: executeTool(mcpCall: MCPToolCall, signal?) - activate mcpStore - - mcpStore->>mcpStore: toolsIndex.get(mcpCall.function.name) - Note right of mcpStore: Resolve serverName from toolsIndex
MCPToolCall = {id, type, function: {name, arguments}} - - mcpStore->>mcpStore: acquireConnection() - Note right of mcpStore: activeFlowCount++
Prevent shutdown during execution - - mcpStore->>mcpStore: connection = connections.get(serverName) - - mcpStore->>MCPSvc: callTool(connection, {name, arguments}, signal) - activate MCPSvc - - MCPSvc->>MCPSvc: throwIfAborted(signal) - MCPSvc->>ExtMCP: callTool(name, arguments) - - alt Tool execution success - ExtMCP-->>MCPSvc: ToolCallResult (content, isError) - MCPSvc->>MCPSvc: formatToolResult(result) - Note right of MCPSvc: Handle text, image (base64),
embedded resource content - MCPSvc-->>mcpStore: ToolExecutionResult - else Tool execution error - ExtMCP-->>MCPSvc: Error - MCPSvc-->>mcpStore: throw Error - else Aborted - MCPSvc-->>mcpStore: throw AbortError - end - - deactivate MCPSvc - - mcpStore->>mcpStore: releaseConnection() - Note right of mcpStore: activeFlowCount-- - - mcpStore-->>UI: ToolExecutionResult - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: � RESOURCE ATTACHMENT CONSUMPTION - %% ═══════════════════════════════════════════════════════════════════════════ - - chatStore->>mcpStore: consumeResourceAttachmentsAsExtras() - activate mcpStore - mcpStore->>mcpResStore: getAttachments() - mcpResStore-->>mcpStore: MCPResourceAttachment[] - mcpStore->>mcpStore: Convert attachments to message extras - mcpStore->>mcpResStore: clearAttachments() - mcpStore-->>chatStore: MessageExtra[] (for user message) - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: �📝 PROMPT OPERATIONS - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpStore: getAllPrompts() - activate mcpStore - - loop For each connected server with prompts capability - mcpStore->>MCPSvc: listPrompts(connection) - MCPSvc->>ExtMCP: listPrompts() - ExtMCP-->>MCPSvc: Prompt[] - MCPSvc-->>mcpStore: prompts - end - - mcpStore-->>UI: MCPPromptInfo[] (with serverName) - deactivate mcpStore - - UI->>mcpStore: getPrompt(serverName, promptName, args?) - activate mcpStore - - mcpStore->>MCPSvc: getPrompt(connection, name, args) - MCPSvc->>ExtMCP: getPrompt({name, arguments}) - ExtMCP-->>MCPSvc: GetPromptResult (messages) - MCPSvc-->>mcpStore: GetPromptResult - - mcpStore-->>UI: GetPromptResult - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 📁 RESOURCE OPERATIONS - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpResStore: addAttachment(resourceInfo) - activate mcpResStore - mcpResStore->>mcpResStore: Create MCPResourceAttachment (loading: true) - mcpResStore-->>UI: attachment - - UI->>mcpStore: readResource(serverName, uri) - activate mcpStore - - mcpStore->>MCPSvc: readResource(connection, uri) - MCPSvc->>ExtMCP: readResource({uri}) - ExtMCP-->>MCPSvc: MCPReadResourceResult (contents) - MCPSvc-->>mcpStore: contents - - mcpStore-->>UI: MCPResourceContent[] - deactivate mcpStore - - UI->>mcpResStore: updateAttachmentContent(attachmentId, content) - mcpResStore->>mcpResStore: cacheResourceContent(resource, content) - deactivate mcpResStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 🔄 AUTO-RECONNECTION - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over mcpStore: On WebSocket close or connection error: - mcpStore->>mcpStore: autoReconnect(serverName, attempt) - activate mcpStore - - mcpStore->>mcpStore: Calculate backoff delay - Note right of mcpStore: delay = min(30s, 1s * 2^attempt) - - mcpStore->>mcpStore: Wait for delay - mcpStore->>mcpStore: reconnectServer(serverName) - - alt Reconnection success - mcpStore->>mcpStore: updateHealthCheck(id, SUCCESS) - else Max attempts reached - mcpStore->>mcpStore: updateHealthCheck(id, ERROR) - end - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 🛑 SHUTDOWN - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpStore: shutdown() - activate mcpStore - - mcpStore->>mcpStore: Wait for activeFlowCount == 0 - - loop For each connection - mcpStore->>MCPSvc: disconnect(connection) - MCPSvc->>MCPSvc: transport.onclose = undefined - MCPSvc->>ExtMCP: close() - end - - mcpStore->>mcpStore: connections.clear() - mcpStore->>mcpStore: toolsIndex.clear() - mcpStore->>mcpStore: _connectedServers = [] - - mcpStore->>mcpResStore: clear() - deactivate mcpStore -``` diff --git a/tools/ui/docs/flows/models-flow.md b/tools/ui/docs/flows/models-flow.md deleted file mode 100644 index c3031b729..000000000 --- a/tools/ui/docs/flows/models-flow.md +++ /dev/null @@ -1,181 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 ModelsSelector - participant Hooks as 🪝 useModelChangeValidation - participant modelsStore as 🗄️ modelsStore - participant serverStore as 🗄️ serverStore - participant convStore as 🗄️ conversationsStore - participant ModelsSvc as ⚙️ ModelsService - participant PropsSvc as ⚙️ PropsService - participant API as 🌐 llama-server - - Note over modelsStore: State:
models: ModelOption[]
routerModels: ApiModelDataEntry[]
selectedModelId, selectedModelName
loading, updating, error
modelLoadingStates (Map)
modelPropsCache (Map)
propsCacheVersion - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🚀 INITIALIZATION (MODEL mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>modelsStore: fetch() - activate modelsStore - modelsStore->>modelsStore: loading = true - - alt serverStore.props not loaded - modelsStore->>serverStore: fetch() - Note over serverStore: → see server-flow.mmd - end - - modelsStore->>ModelsSvc: list() - ModelsSvc->>API: GET /v1/models - API-->>ModelsSvc: ApiModelListResponse {data: [model]} - - modelsStore->>modelsStore: models = $state(mapped) - Note right of modelsStore: Map to ModelOption[]:
{id, name, model, description, capabilities} - - Note over modelsStore: MODEL mode: Get modalities from serverStore.props - modelsStore->>modelsStore: modelPropsCache.set(model.id, serverStore.props) - modelsStore->>modelsStore: models[0].modalities = props.modalities - - modelsStore->>modelsStore: Auto-select single model - Note right of modelsStore: selectedModelId = models[0].id - modelsStore->>modelsStore: loading = false - deactivate modelsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🚀 INITIALIZATION (ROUTER mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>modelsStore: fetch() - activate modelsStore - modelsStore->>ModelsSvc: list() - ModelsSvc->>API: GET /v1/models - API-->>ModelsSvc: ApiModelListResponse - modelsStore->>modelsStore: models = $state(mapped) - deactivate modelsStore - - Note over UI: After models loaded, layout triggers: - UI->>modelsStore: fetchRouterModels() - activate modelsStore - modelsStore->>ModelsSvc: listRouter() - ModelsSvc->>API: GET /v1/models - API-->>ModelsSvc: ApiRouterModelsListResponse - Note right of API: {data: [{id, status, path, in_cache}]} - modelsStore->>modelsStore: routerModels = $state(data) - - modelsStore->>modelsStore: fetchModalitiesForLoadedModels() - loop each model where status === "loaded" - modelsStore->>PropsSvc: fetchForModel(modelId) - PropsSvc->>API: GET /props?model={modelId} - API-->>PropsSvc: ApiLlamaCppServerProps - modelsStore->>modelsStore: modelPropsCache.set(modelId, props) - end - modelsStore->>modelsStore: propsCacheVersion++ - deactivate modelsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🔄 MODEL SELECTION (ROUTER mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>Hooks: useModelChangeValidation({getRequiredModalities, onSuccess?, onValidationFailure?}) - Note over Hooks: Hook configured per-component:
ChatForm: getRequiredModalities = usedModalities
ChatMessage: getRequiredModalities = getModalitiesUpToMessage(msgId) - - UI->>Hooks: handleModelChange(modelId, modelName) - activate Hooks - Hooks->>Hooks: previousSelectedModelId = modelsStore.selectedModelId - Hooks->>modelsStore: isModelLoaded(modelName)? - - alt model NOT loaded - Hooks->>modelsStore: loadModel(modelName) - Note over modelsStore: → see LOAD MODEL section below - end - - Note over Hooks: Always fetch props (from cache or API) - Hooks->>modelsStore: fetchModelProps(modelName) - modelsStore-->>Hooks: props - - Hooks->>convStore: getRequiredModalities() - convStore-->>Hooks: {vision, audio} - - Hooks->>Hooks: Validate: model.modalities ⊇ required? - - alt validation PASSED - Hooks->>modelsStore: selectModelById(modelId) - Hooks-->>UI: return true - else validation FAILED - Hooks->>UI: toast.error("Model doesn't support required modalities") - alt model was just loaded - Hooks->>modelsStore: unloadModel(modelName) - end - alt onValidationFailure provided - Hooks->>modelsStore: selectModelById(previousSelectedModelId) - end - Hooks-->>UI: return false - end - deactivate Hooks - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ⬆️ LOAD MODEL (ROUTER mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - modelsStore->>modelsStore: loadModel(modelId) - activate modelsStore - - alt already loaded - modelsStore-->>modelsStore: return (no-op) - end - - modelsStore->>modelsStore: modelLoadingStates.set(modelId, true) - modelsStore->>ModelsSvc: load(modelId) - ModelsSvc->>API: POST /models/load {model: modelId} - API-->>ModelsSvc: {status: "loading"} - - modelsStore->>modelsStore: pollForModelStatus(modelId, LOADED) - loop poll every 500ms (max 60 attempts) - modelsStore->>modelsStore: fetchRouterModels() - modelsStore->>ModelsSvc: listRouter() - ModelsSvc->>API: GET /v1/models - API-->>ModelsSvc: models[] - modelsStore->>modelsStore: getModelStatus(modelId) - alt status === LOADED - Note right of modelsStore: break loop - else status === LOADING - Note right of modelsStore: wait 500ms, continue - end - end - - modelsStore->>modelsStore: updateModelModalities(modelId) - modelsStore->>PropsSvc: fetchForModel(modelId) - PropsSvc->>API: GET /props?model={modelId} - API-->>PropsSvc: props with modalities - modelsStore->>modelsStore: modelPropsCache.set(modelId, props) - modelsStore->>modelsStore: propsCacheVersion++ - - modelsStore->>modelsStore: modelLoadingStates.set(modelId, false) - deactivate modelsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ⬇️ UNLOAD MODEL (ROUTER mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - modelsStore->>modelsStore: unloadModel(modelId) - activate modelsStore - modelsStore->>modelsStore: modelLoadingStates.set(modelId, true) - modelsStore->>ModelsSvc: unload(modelId) - ModelsSvc->>API: POST /models/unload {model: modelId} - - modelsStore->>modelsStore: pollForModelStatus(modelId, UNLOADED) - loop poll until unloaded - modelsStore->>ModelsSvc: listRouter() - ModelsSvc->>API: GET /v1/models - end - - modelsStore->>modelsStore: modelLoadingStates.set(modelId, false) - deactivate modelsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 📊 COMPUTED GETTERS - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over modelsStore: Getters:
- selectedModel: ModelOption | null
- loadedModelIds: string[] (from routerModels)
- loadingModelIds: string[] (from modelLoadingStates)
- singleModelName: string | null (MODEL mode only) - - Note over modelsStore: Modality helpers:
- getModelModalities(modelId): {vision, audio}
- modelSupportsVision(modelId): boolean
- modelSupportsAudio(modelId): boolean -``` diff --git a/tools/ui/docs/flows/server-flow.md b/tools/ui/docs/flows/server-flow.md deleted file mode 100644 index d6a1611f6..000000000 --- a/tools/ui/docs/flows/server-flow.md +++ /dev/null @@ -1,76 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 +layout.svelte - participant serverStore as 🗄️ serverStore - participant PropsSvc as ⚙️ PropsService - participant API as 🌐 llama-server - - Note over serverStore: State:
props: ApiLlamaCppServerProps | null
loading, error
role: ServerRole | null (MODEL | ROUTER)
fetchPromise (deduplication) - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🚀 INITIALIZATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>serverStore: fetch() - activate serverStore - - alt fetchPromise exists (already fetching) - serverStore-->>UI: return fetchPromise - Note right of serverStore: Deduplicate concurrent calls - end - - serverStore->>serverStore: loading = true - serverStore->>serverStore: fetchPromise = new Promise() - - serverStore->>PropsSvc: fetch() - PropsSvc->>API: GET /props - API-->>PropsSvc: ApiLlamaCppServerProps - Note right of API: {role, model_path, model_alias,
modalities, default_generation_settings, ...} - - PropsSvc-->>serverStore: props - serverStore->>serverStore: props = $state(data) - - serverStore->>serverStore: detectRole(props) - Note right of serverStore: role = props.role === "router"
? ServerRole.ROUTER
: ServerRole.MODEL - - serverStore->>serverStore: loading = false - serverStore->>serverStore: fetchPromise = null - deactivate serverStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 📊 COMPUTED GETTERS - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over serverStore: Getters from props: - - rect rgb(240, 255, 240) - Note over serverStore: defaultParams
→ props.default_generation_settings.params
(temperature, top_p, top_k, etc.) - end - - rect rgb(240, 255, 240) - Note over serverStore: contextSize
→ props.default_generation_settings.n_ctx - end - - rect rgb(255, 240, 240) - Note over serverStore: isRouterMode
→ role === ServerRole.ROUTER - end - - rect rgb(255, 240, 240) - Note over serverStore: isModelMode
→ role === ServerRole.MODEL - end - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🔗 RELATIONSHIPS - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over serverStore: Used by: - Note right of serverStore: - modelsStore: role detection, MODEL mode modalities
- settingsStore: syncWithServerDefaults (defaultParams)
- chatStore: contextSize for processing state
- UI components: isRouterMode for conditional rendering - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ❌ ERROR HANDLING - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over serverStore: getErrorMessage(): string | null
Returns formatted error for UI display - - Note over serverStore: clear(): void
Resets all state (props, error, loading, role) -``` diff --git a/tools/ui/docs/flows/settings-flow.md b/tools/ui/docs/flows/settings-flow.md deleted file mode 100644 index 260713a17..000000000 --- a/tools/ui/docs/flows/settings-flow.md +++ /dev/null @@ -1,156 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 ChatSettings - participant settingsStore as 🗄️ settingsStore - participant serverStore as 🗄️ serverStore - participant ParamSvc as ⚙️ ParameterSyncService - participant LS as 💾 LocalStorage - - Note over settingsStore: State:
config: SettingsConfigType
theme: string ("auto" | "light" | "dark")
isInitialized: boolean
userOverrides: Set<string> - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 🚀 INITIALIZATION - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over settingsStore: Auto-initialized in constructor (browser only) - settingsStore->>settingsStore: initialize() - activate settingsStore - - settingsStore->>settingsStore: loadConfig() - settingsStore->>LS: get("llama-config") - LS-->>settingsStore: StoredConfig | null - - alt config exists - settingsStore->>settingsStore: Merge with SETTING_CONFIG_DEFAULT - Note right of settingsStore: Fill missing keys with defaults - else no config - settingsStore->>settingsStore: config = SETTING_CONFIG_DEFAULT - end - - settingsStore->>LS: get("llama-userOverrides") - LS-->>settingsStore: string[] | null - settingsStore->>settingsStore: userOverrides = new Set(data) - - settingsStore->>settingsStore: loadTheme() - settingsStore->>LS: get("llama-theme") - LS-->>settingsStore: theme | "auto" - - settingsStore->>settingsStore: isInitialized = true - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 🔄 SYNC WITH SERVER DEFAULTS - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over UI: Triggered from +layout.svelte when serverStore.props loaded - UI->>settingsStore: syncWithServerDefaults() - activate settingsStore - - settingsStore->>serverStore: defaultParams - serverStore-->>settingsStore: {temperature, top_p, top_k, ...} - - loop each SYNCABLE_PARAMETER - alt key NOT in userOverrides - settingsStore->>settingsStore: config[key] = serverDefault[key] - Note right of settingsStore: Non-overridden params adopt server default - else key in userOverrides - Note right of settingsStore: Keep user value, skip server default - end - end - - alt serverStore.props has uiSettings - settingsStore->>settingsStore: Apply uiSettings from server - Note right of settingsStore: Server-provided UI settings
(e.g. showRawOutputSwitch) - end - - settingsStore->>settingsStore: saveConfig() - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: ⚙️ UPDATE CONFIG - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>settingsStore: updateConfig(key, value) - activate settingsStore - settingsStore->>settingsStore: config[key] = value - - alt value matches server default for key - settingsStore->>settingsStore: userOverrides.delete(key) - Note right of settingsStore: Matches server default, remove override - else value differs from server default - settingsStore->>settingsStore: userOverrides.add(key) - Note right of settingsStore: Mark as user-modified (won't be overwritten) - end - - settingsStore->>settingsStore: saveConfig() - settingsStore->>LS: set(CONFIG_LOCALSTORAGE_KEY, config) - settingsStore->>LS: set(USER_OVERRIDES_LOCALSTORAGE_KEY, [...userOverrides]) - deactivate settingsStore - - UI->>settingsStore: updateMultipleConfig({key1: val1, key2: val2}) - activate settingsStore - Note right of settingsStore: Batch update, single save - settingsStore->>settingsStore: For each key: config[key] = value - settingsStore->>settingsStore: For each key: userOverrides.add(key) - settingsStore->>settingsStore: saveConfig() - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 🔄 RESET - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>settingsStore: resetConfig() - activate settingsStore - settingsStore->>settingsStore: config = {...SETTING_CONFIG_DEFAULT} - settingsStore->>settingsStore: userOverrides.clear() - Note right of settingsStore: All params reset to defaults
Next syncWithServerDefaults will adopt server values - settingsStore->>settingsStore: saveConfig() - deactivate settingsStore - - UI->>settingsStore: resetParameterToServerDefault(key) - activate settingsStore - settingsStore->>settingsStore: userOverrides.delete(key) - settingsStore->>serverStore: defaultParams[key] - settingsStore->>settingsStore: config[key] = serverDefault - settingsStore->>settingsStore: saveConfig() - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 🎨 THEME - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>settingsStore: updateTheme(newTheme) - activate settingsStore - settingsStore->>settingsStore: theme = newTheme - settingsStore->>settingsStore: saveTheme() - settingsStore->>LS: set("llama-theme", theme) - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 📊 PARAMETER INFO - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>settingsStore: getParameterInfo(key) - settingsStore->>ParamSvc: getParameterInfo(key, config, serverDefaults, userOverrides) - ParamSvc-->>settingsStore: ParameterInfo - Note right of ParamSvc: {
currentValue,
serverDefault,
isUserOverride: boolean,
canSync: boolean,
isDifferentFromServer: boolean
} - - UI->>settingsStore: getParameterDiff() - settingsStore->>ParamSvc: createParameterDiff(config, serverDefaults, userOverrides) - ParamSvc-->>settingsStore: ParameterDiff[] - Note right of ParamSvc: Array of parameters where user != server - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 📋 CONFIG CATEGORIES - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over settingsStore: Syncable with server (from /props): - rect rgb(240, 255, 240) - Note over settingsStore: temperature, top_p, top_k, min_p
repeat_penalty, presence_penalty, frequency_penalty
dynatemp_range, dynatemp_exponent
typ_p, xtc_probability, xtc_threshold
dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n - end - - Note over settingsStore: UI-only (not synced): - rect rgb(255, 240, 240) - Note over settingsStore: systemMessage, custom (JSON)
showStatistics, enableContinueGeneration
autoMicOnEmpty, disableAutoScroll
apiKey, pdfAsImage, disableReasoningParsing, showRawOutputSwitch - end -``` diff --git a/tools/ui/eslint.config.js b/tools/ui/eslint.config.js index b8bdb216e..6ad065f5a 100644 --- a/tools/ui/eslint.config.js +++ b/tools/ui/eslint.config.js @@ -12,6 +12,49 @@ import { fileURLToPath } from 'node:url'; import ts from 'typescript-eslint'; const gitignorePath = fileURLToPath(new URL('./.gitignore', import.meta.url)); +// Require a blank line between consecutive class accessors (get/set). The core +// `padding-line-between-statements` rule only handles statements, not class +// members, so this is enforced with a small custom rule. +const blankLineBetweenAccessors = { + create(context) { + return { + MethodDefinition(node) { + if (node.kind !== 'get' && node.kind !== 'set') return; + + const body = node.parent; + + if (!body || body.type !== 'ClassBody') return; + + const index = body.body.indexOf(node); + + if (index <= 0) return; + + const prev = body.body[index - 1]; + + if (prev.type !== 'MethodDefinition' || (prev.kind !== 'get' && prev.kind !== 'set')) + return; + + if (node.loc.start.line - prev.loc.end.line <= 1) { + context.report({ + fix(fixer) { + // Insert after the previous accessor's closing brace so the blank + // line keeps the current accessor's indentation. + return fixer.insertTextAfter(prev, '\n'); + }, + message: 'Expected a blank line between class accessors (get/set).', + node + }); + } + } + }; + }, + meta: { + docs: { description: 'Require a blank line between consecutive class accessors (get/set).' }, + fixable: 'whitespace', + schema: [], + type: 'layout' + } +}; export default ts.config( includeIgnoreFile(gitignorePath), @@ -22,7 +65,11 @@ export default ts.config( ...svelte.configs.prettier, { languageOptions: { globals: { ...globals.browser, ...globals.node } }, - plugins: { perfectionist, 'simple-import-sort': simpleImportSort }, + plugins: { + local: { rules: { 'blank-line-between-accessors': blankLineBetweenAccessors } }, + perfectionist, + 'simple-import-sort': simpleImportSort + }, rules: { // Snippet bodies often ignore one or more of the parent's params // (e.g. `{#snippet children(_meta, ctx)}` when only ctx is read). @@ -30,8 +77,11 @@ export default ts.config( 'error', { argsIgnorePattern: '^_', varsIgnorePattern: '^_' } ], + // Enforce empty line at end of file 'eol-last': 'error', + // Enforce a blank line between consecutive get/set accessors + 'local/blank-line-between-accessors': 'error', // typescript-eslint strongly recommend that you do not use the no-undef lint rule on TypeScript projects. // see: https://typescript-eslint.io/troubleshooting/faqs/eslint/#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors 'no-undef': 'off', @@ -61,6 +111,38 @@ export default ts.config( { blankLine: 'always', next: ['return', 'throw', 'break', 'continue'], prev: '*' } ], + // Class member order: public fields -> private fields -> constructor -> getters + // -> setters -> public methods -> private methods, alphabetical within each. + // Svelte $derived fields must stay in dependency order (forward references are + // rejected), so the two stores that rely on that are exempted below. + 'perfectionist/sort-classes': [ + 'error', + { + customGroups: [ + { groupName: 'public-field', modifiers: ['public'], selector: 'property' }, + { groupName: 'private-field', modifiers: ['private'], selector: 'property' }, + { groupName: 'get-method', selector: 'get-method' }, + { groupName: 'set-method', selector: 'set-method' }, + { groupName: 'public-method', modifiers: ['public'], selector: 'method' }, + { groupName: 'private-method', modifiers: ['private'], selector: 'method' } + ], + groups: [ + 'public-field', + 'private-field', + 'constructor', + 'get-method', + 'set-method', + 'public-method', + 'private-method', + 'unknown' + ], + type: 'natural', + // Keep members in dependency order (Svelte rejects forward references in + // $derived fields), while still sorting the rest alphabetically. + useExperimentalDependencyDetection: true + } + ], + // Alphabetical order for enum members 'perfectionist/sort-enums': ['error', { type: 'natural' }], diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte index 8e8949172..304e5a600 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte @@ -139,7 +139,7 @@ let fileSize = $derived(currentItem?.size ? formatFileSize(currentItem.size) : ''); let hasVisionModality = $derived( - currentItem && activeModelId ? modelsStore.modelSupportsVision(activeModelId) : false + currentItem && activeModelId ? modelsStore.props.modelSupportsVision(activeModelId) : false ); let audioSrc = $derived( diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte index e937d2730..18061728a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -28,7 +28,6 @@ import { chatStore, conversationsStore, - mcpResourceStore, mcpStore, modelsStore, serverStore, @@ -140,7 +139,9 @@ // float above the box. let mentionAnchor: HTMLDivElement | null = $state(null); - let cwd = $derived(conversationsStore.activeConversation?.cwd ?? conversationsStore.pendingCwd); + let cwd = $derived( + conversationsStore.activeConversation?.cwd ?? conversationsStore.preferences.pendingCwd + ); const pickers = useChatFormPickers({ focusInput: refocusInput, @@ -151,7 +152,8 @@ getShowModelSelector: () => showModelSelector, getValue: () => value, hasCwdTools: () => toolsStore.hasEnabledCwdTools, - hasPrompts: () => mcpStore.hasPromptsCapability(conversationsStore.getAllMcpServerOverrides()), + hasPrompts: () => + mcpStore.hasPromptsCapability(conversationsStore.preferences.getAllMcpServerOverrides()), openModelSelector: () => chatFormActionsRef?.openModelSelector(), setCaretOffset: (offset) => inputRef?.setCaretOffset(offset), setValue: (v) => { @@ -170,7 +172,7 @@ onValueChange?.(''); } - await conversationsStore.setCwd(newDir); + await conversationsStore.preferences.setCwd(newDir); if (conversationsStore.activeConversation) { await chatStore.recordCwdChange(newDir?.trim() || null); @@ -595,7 +597,7 @@ {useRichInput} /> - {#if mcpResourceStore.hasAttachments} + {#if mcpStore.resources.hasAttachments} { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte index 3d04d14cb..92f5b9349 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte @@ -38,11 +38,11 @@ } function isServerEnabledForChat(serverId: string): boolean { - return conversationsStore.isMcpServerEnabledForChat(serverId); + return conversationsStore.preferences.isMcpServerEnabledForChat(serverId); } async function toggleServerForChat(serverId: string) { - await conversationsStore.toggleMcpServerForChat(serverId); + await conversationsStore.preferences.toggleMcpServerForChat(serverId); } function handleMcpSubMenuOpen(open: boolean) { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte index 63a8c267d..2e61bb07d 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte @@ -218,12 +218,15 @@ {@const hasError = healthState.status === HealthCheckStatus.ERROR} {@const displayName = mcpStore.getServerLabel(server)} {@const faviconUrl = mcpStore.getServerFavicon(server.id)} - {@const isEnabled = conversationsStore.isMcpServerEnabledForChat(server.id)} + {@const isEnabled = conversationsStore.preferences.isMcpServerEnabledForChat( + server.id + )} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte index 518dee5d9..f76333a1c 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte @@ -81,10 +81,10 @@ $effect(() => { if (activeModelId) { - const cached = modelsStore.getModelProps(activeModelId); + const cached = modelsStore.props.getModelProps(activeModelId); if (!cached) { - modelsStore.fetchModelProps(activeModelId).then(() => { + modelsStore.props.fetchModelProps(activeModelId).then(() => { modelPropsVersion++; }); } @@ -94,19 +94,21 @@ $effect(() => { void modelPropsVersion; - hasAudioModality = activeModelId ? modelsStore.modelSupportsAudio(activeModelId) : false; + hasAudioModality = activeModelId ? modelsStore.props.modelSupportsAudio(activeModelId) : false; }); $effect(() => { void modelPropsVersion; - hasVideoModality = activeModelId ? modelsStore.modelSupportsVideo(activeModelId) : false; + hasVideoModality = activeModelId ? modelsStore.props.modelSupportsVideo(activeModelId) : false; }); $effect(() => { void modelPropsVersion; - hasVisionModality = activeModelId ? modelsStore.modelSupportsVision(activeModelId) : false; + hasVisionModality = activeModelId + ? modelsStore.props.modelSupportsVision(activeModelId) + : false; }); $effect(() => { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte index d8fad772d..118e54a0a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte @@ -58,13 +58,13 @@ let currentConfig = $derived(settingsStore.config); let hasMcpPromptsSupport = $derived.by(() => { - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); return mcpStore.hasPromptsCapability(perChatOverrides); }); let hasMcpResourcesSupport = $derived.by(() => { - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); return mcpStore.hasResourcesCapability(perChatOverrides); }); @@ -121,7 +121,7 @@ if (!chatStore.isLoading && !chatStore.isStreaming()) return false; - const processingState = chatStore.activeProcessingState; + const processingState = chatStore.processing.activeState; if (!processingState) return false; diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte index 7b071d99e..d6bad98dc 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte @@ -16,7 +16,7 @@ $effect(() => { const conv = conversationsStore.activeConversation; - untrack(() => chatStore.setActiveProcessingConversation(conv?.id ?? null)); + untrack(() => chatStore.processing.setActiveConversation(conv?.id ?? null)); }); $effect(() => { @@ -28,12 +28,12 @@ if (chatStore.isLoading || chatStore.isStreaming()) return; if (messages.length === 0) { - untrack(() => chatStore.clearProcessingState(conv.id)); + untrack(() => chatStore.processing.setState(conv.id, null)); return; } - untrack(() => chatStore.restoreProcessingStateFromMessages(messages, conv.id)); + untrack(() => chatStore.processing.restoreFromMessages(messages, conv.id)); }); $effect(() => { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte index 3f178da18..452fd7a68 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte @@ -3,7 +3,7 @@ ChatAttachmentsListItemMcpResource, HorizontalScrollCarousel } from '$lib/components/app'; - import { mcpResourceStore, mcpStore } from '$lib/stores'; + import { mcpStore } from '$lib/stores'; interface Props { class?: string; @@ -12,8 +12,8 @@ let { class: className, onResourceClick }: Props = $props(); - const attachments = $derived(mcpResourceStore.attachments); - const hasAttachments = $derived(mcpResourceStore.hasAttachments); + const attachments = $derived(mcpStore.resources.attachments); + const hasAttachments = $derived(mcpStore.resources.hasAttachments); function handleRemove(attachmentId: string) { mcpStore.removeResourceAttachment(attachmentId); diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte index 9b5a57b9b..9a5c3e747 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte @@ -87,7 +87,7 @@ isLoading = true; try { - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); const initialized = await mcpStore.ensureInitialized(perChatOverrides); if (!initialized) { diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte index b92be9fbd..c92af719a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte @@ -59,7 +59,7 @@ message.model ?? chatStore.getResumeModel(message.convId) ?? modelsStore.selectedModelName ); let modelLoadProgress = $derived( - isRouter && loadTargetModel ? modelsStore.getLoadProgress(loadTargetModel) : null + isRouter && loadTargetModel ? modelsStore.status.getLoadProgress(loadTargetModel) : null ); let modelLoadingText = $derived(modelLoadProgressText(modelLoadProgress)); diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte index d3fb33a00..c5b80f156 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte @@ -31,7 +31,7 @@ pendingModel = modelId; try { - await modelsStore.loadModel(modelId); + await modelsStore.status.load(modelId); } finally { pendingModel = null; } diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte index 584979979..7a21c6766 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte @@ -43,14 +43,14 @@ ); const hasReasoningError = $derived( - isLastAssistantMessage ? !!agenticStore.lastError(message.convId) : false + isLastAssistantMessage ? !!agenticStore.getLastError(message.convId) : false ); let permissionDismissed = $state(false); const pendingPermission = $derived( isStreaming && isLastAssistantMessage - ? agenticStore.pendingPermissionRequest(message.convId) + ? agenticStore.getPendingPermissionRequest(message.convId) : null ); @@ -74,7 +74,7 @@ const pendingContinue = $derived( isStreaming && isLastAssistantMessage - ? agenticStore.pendingContinueRequest(message.convId) + ? agenticStore.getPendingContinueRequest(message.convId) : false ); @@ -97,7 +97,7 @@ const sections = $derived(deriveAgenticSections(message, toolMessages, [], isStreaming)); const currentlyExecutingToolCallId = $derived( - isStreaming ? agenticStore.executingToolCallId(message.convId) : null + isStreaming ? agenticStore.getExecutingToolCallId(message.convId) : null ); type TurnGroup = { diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte index 2a8f45ba5..c32c66d91 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte @@ -238,30 +238,30 @@ /> {/each} - {#if conversationsStore.activeConversation && agenticStore.pendingSteeringMessageContent(conversationsStore.activeConversation!.id)} + {#if conversationsStore.activeConversation && agenticStore.getPendingSteeringMessageContent(conversationsStore.activeConversation!.id)} {@const convId = conversationsStore.activeConversation!.id} - {@const pendingContent = agenticStore.pendingSteeringMessageContent(convId)} + {@const pendingContent = agenticStore.getPendingSteeringMessageContent(convId)} {#if pendingContent} chatStore.abortCurrentFlow(convId)} onEdit={(newContent, extras) => agenticStore.injectSteeringMessage(convId, newContent, extras)} onDelete={() => agenticStore.clearSteeringMessage(convId)} /> {/if} - {:else if conversationsStore.activeConversation && chatStore.pendingMessageContent(conversationsStore.activeConversation!.id)} + {:else if conversationsStore.activeConversation && chatStore.getPendingMessageContent(conversationsStore.activeConversation!.id)} {@const convId = conversationsStore.activeConversation!.id} - {@const pendingContent = chatStore.pendingMessageContent(convId)} + {@const pendingContent = chatStore.getPendingMessageContent(convId)} {#if pendingContent} chatStore.abortCurrentFlow(convId)} onEdit={(newContent, extras) => chatStore.injectPendingMessage(convId, newContent, extras)} onDelete={() => chatStore.clearPendingMessage(convId)} diff --git a/tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte b/tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte index 1ddad694b..c48dcb38c 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte @@ -8,7 +8,7 @@ import { Button } from '$lib/components/ui/button'; import * as Dialog from '$lib/components/ui/dialog'; import { ICON_CLASS_DEFAULT } from '$lib/constants'; - import { conversationsStore, mcpResourceStore, mcpStore } from '$lib/stores'; + import { conversationsStore, mcpStore } from '$lib/stores'; import type { MCPResourceContent, MCPResourceInfo, MCPResourceTemplateInfo } from '$lib/types'; import { getResourceDisplayName } from '$lib/utils'; import { SvelteSet } from 'svelte/reactivity'; @@ -33,7 +33,7 @@ let templatePreviewLoading = $state(false); let templatePreviewError = $state(null); - const totalCount = $derived(mcpResourceStore.totalResourceCount); + const totalCount = $derived(mcpStore.resources.totalResourceCount); $effect(() => { if (open) { @@ -48,7 +48,7 @@ }); async function loadResources() { - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); const initialized = await mcpStore.ensureInitialized(perChatOverrides); if (initialized) { @@ -126,16 +126,16 @@ isAttaching = true; try { - const knownResource = mcpResourceStore.findResourceByUri(templatePreviewUri); + const knownResource = mcpStore.resources.findResourceByUri(templatePreviewUri); if (knownResource) { - if (!mcpResourceStore.isAttached(knownResource.uri)) { + if (!mcpStore.resources.isAttached(knownResource.uri)) { await mcpStore.attachResource(knownResource.uri); } toast.success(`Resource attached: ${knownResource.title || knownResource.name}`); } else { - if (mcpResourceStore.isAttached(templatePreviewUri)) { + if (mcpStore.resources.isAttached(templatePreviewUri)) { toast.info('Resource already attached'); handleOpenChange(false); @@ -147,9 +147,9 @@ serverName: selectedTemplate.serverName, uri: templatePreviewUri }; - const attachment = mcpResourceStore.addAttachment(resourceInfo); + const attachment = mcpStore.resources.addAttachment(resourceInfo); - mcpResourceStore.updateAttachmentContent(attachment.id, templatePreviewContent); + mcpStore.resources.updateAttachmentContent(attachment.id, templatePreviewContent); toast.success(`Resource attached: ${resourceInfo.name}`); } @@ -199,7 +199,7 @@ function getAllResourcesFlatInTreeOrder(): MCPResourceInfo[] { const allResources: MCPResourceInfo[] = []; - const resourcesMap = mcpResourceStore.serverResources; + const resourcesMap = mcpStore.resources.serverResources; for (const [serverName, serverRes] of resourcesMap.entries()) { for (const resource of serverRes.resources) { diff --git a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte index 9123dcef9..a339c6a42 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte @@ -234,7 +234,7 @@ useProxy: newServerUseProxy }); - conversationsStore.setMcpServerOverride(newServerId, true); + conversationsStore.preferences.setMcpServerOverride(newServerId, true); handleOpenChange(false); } diff --git a/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte b/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte index 61155fceb..fb7498870 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte @@ -42,7 +42,7 @@ let modalities = $derived.by(() => { if (!firstModel?.id) return []; - return modelsStore.getModelModalitiesArray(firstModel.id); + return modelsStore.props.getModelModalitiesArray(firstModel.id); }); // Ensure models are fetched when dialog opens @@ -56,7 +56,7 @@ $effect(() => { if (open && isRouter && modelId) { isLoadingRouterProps = true; - modelsStore + modelsStore.props .fetchModelProps(modelId) .then((props) => { routerModelProps = props; diff --git a/tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte b/tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte index 301c39699..a8772aa1c 100644 --- a/tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte @@ -14,7 +14,9 @@ let mcpServers = $derived(mcpStore.getServers().filter((s) => s.enabled)); let enabledMcpServersForChat = $derived( - mcpServers.filter((s) => conversationsStore.isMcpServerEnabledForChat(s.id) && s.url.trim()) + mcpServers.filter( + (s) => conversationsStore.preferences.isMcpServerEnabledForChat(s.id) && s.url.trim() + ) ); let healthyEnabledMcpServers = $derived( enabledMcpServersForChat.filter((s) => { diff --git a/tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte b/tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte index 18e974653..c8cf8bbbc 100644 --- a/tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte @@ -2,7 +2,7 @@ import McpResourcesBrowserEmptyState from './McpResourcesBrowserEmptyState.svelte'; import McpResourcesBrowserHeader from './McpResourcesBrowserHeader.svelte'; import McpResourcesBrowserServerItem from './McpResourcesBrowserServerItem.svelte'; - import { mcpResourceStore, mcpStore } from '$lib/stores'; + import { mcpStore } from '$lib/stores'; import type { MCPResourceInfo, MCPResourceTemplateInfo, MCPServerResources } from '$lib/types'; import { parseResourcePath } from '$lib/utils'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; @@ -31,8 +31,8 @@ let expandedFolders = new SvelteSet(); let searchQuery = $state(''); - const resources = $derived(mcpResourceStore.serverResources); - const isLoading = $derived(mcpResourceStore.isLoading); + const resources = $derived(mcpStore.resources.serverResources); + const isLoading = $derived(mcpStore.resources.isLoading); const filteredResources = $derived.by(() => { if (!searchQuery.trim()) { diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte index c23bca1ae..1cd56c124 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte @@ -116,7 +116,7 @@ if (status === ServerModelStatus.LOADING) return; - await modelsStore.unloadModel(modelId); + await modelsStore.status.unload(modelId); } export function open() { @@ -174,9 +174,9 @@ {@const triggerLoading = !!triggerModel && (triggerStatus === ServerModelStatus.LOADING || - modelsStore.isModelOperationInProgress(triggerModel))} + modelsStore.status.isOperationInProgress(triggerModel))} {@const triggerLoadPercent = triggerLoading - ? Math.round(modelLoadFraction(modelsStore.getLoadProgress(triggerModel)) * 100) + ? Math.round(modelLoadFraction(modelsStore.status.getLoadProgress(triggerModel)) * 100) : 0} {#if ms.isRouter} diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte index 18c885a62..acdb36bca 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte @@ -47,7 +47,7 @@ return (model?.status?.value as ServerModelStatus) ?? null; }); - let isOperationInProgress = $derived(modelsStore.isModelOperationInProgress(option.model)); + let isOperationInProgress = $derived(modelsStore.status.isOperationInProgress(option.model)); let isFailed = $derived(serverStatus === ServerModelStatus.FAILED); let isSleeping = $derived(serverStatus === ServerModelStatus.SLEEPING); let isLoaded = $derived( @@ -55,7 +55,7 @@ ); let isLoading = $derived(serverStatus === ServerModelStatus.LOADING || isOperationInProgress); - let loadProgress = $derived(isLoading ? modelsStore.getLoadProgress(option.model) : null); + let loadProgress = $derived(isLoading ? modelsStore.status.getLoadProgress(option.model) : null); let loadPercent = $derived(Math.round(modelLoadFraction(loadProgress) * 100)); let loadTitle = $derived(modelLoadProgressText(loadProgress)); @@ -138,7 +138,7 @@ icon={RotateCw} tooltip="Retry loading model" class="h-3 w-3 text-red-500 hover:text-foreground" - onclick={() => modelsStore.loadModel(option.model)} + onclick={() => modelsStore.status.load(option.model)} stopPropagationOnClick /> @@ -157,7 +157,7 @@ class="h-3 w-3 text-red-500 hover:text-red-600 [@media(pointer:coarse)]:text-amber-500 [@media(pointer:coarse)]:hover:text-amber-600" onclick={(e) => { e?.stopPropagation(); - modelsStore.unloadModel(option.model); + modelsStore.status.unload(option.model); }} /> @@ -174,7 +174,7 @@ icon={PowerOff} tooltip="Unload model" class="h-3 w-3 text-red-500 hover:text-red-600 [@media(pointer:coarse)]:text-green-500 [@media(pointer:coarse)]:hover:text-green-600" - onclick={() => modelsStore.unloadModel(option.model)} + onclick={() => modelsStore.status.unload(option.model)} stopPropagationOnClick /> @@ -191,7 +191,7 @@ icon={Power} tooltip="Load model" class="h-3 w-3 [@media(pointer:coarse)]:text-muted-foreground" - onclick={() => modelsStore.loadModel(option.model)} + onclick={() => modelsStore.status.load(option.model)} stopPropagationOnClick /> diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte index 7228a2e74..0d10dd106 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte @@ -72,9 +72,9 @@ {@const triggerLoading = !!triggerModel && (triggerStatus === ServerModelStatus.LOADING || - modelsStore.isModelOperationInProgress(triggerModel))} + modelsStore.status.isOperationInProgress(triggerModel))} {@const triggerLoadPercent = triggerLoading - ? Math.round(modelLoadFraction(modelsStore.getLoadProgress(triggerModel)) * 100) + ? Math.round(modelLoadFraction(modelsStore.status.getLoadProgress(triggerModel)) * 100) : 0} {#if ms.isRouter} diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte index c8b2c814c..4233039ef 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte @@ -52,7 +52,7 @@ void modelsStore .fetch() .then(() => modelsStore.fetchRouterModels()) - .then(() => modelsStore.fetchModalitiesForLoadedModels()) + .then(() => modelsStore.props.fetchModalitiesForLoadedModels()) .then(() => modelsStore.ensureFirstModelSelected()); } }); diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte index 30f2b9b2a..d5d93e11d 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte @@ -23,13 +23,13 @@ let { fields, localConfig, onConfigChange, onThemeChange }: Props = $props(); let currentModelParams = $derived.by(() => { - void modelsStore.propsCacheVersion; + void modelsStore.props.cacheVersion; if (serverStore.isRouterMode) { const currentModelName = modelsStore.selectedModelName; if (currentModelName) { - const currentModelProps = modelsStore.getModelProps(currentModelName); + const currentModelProps = modelsStore.props.getModelProps(currentModelName); return (currentModelProps?.default_generation_settings?.params ?? {}) as Record< string, diff --git a/tools/ui/src/lib/components/app/settings/SettingsMcpServers.svelte b/tools/ui/src/lib/components/app/settings/SettingsMcpServers.svelte index 4ea428532..23736ef1c 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsMcpServers.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsMcpServers.svelte @@ -121,11 +121,13 @@ {:else} { - const wasEnabled = conversationsStore.isMcpServerEnabledForChat(server.id); + const wasEnabled = conversationsStore.preferences.isMcpServerEnabledForChat( + server.id + ); - await conversationsStore.toggleMcpServerForChat(server.id); + await conversationsStore.preferences.toggleMcpServerForChat(server.id); if (!wasEnabled) { // Promote the connection so tools/prompts/resources become diff --git a/tools/ui/src/lib/constants/attachment-menu.constants.ts b/tools/ui/src/lib/constants/attachment-menu.constants.ts index 62e03bea6..07ca17fad 100644 --- a/tools/ui/src/lib/constants/attachment-menu.constants.ts +++ b/tools/ui/src/lib/constants/attachment-menu.constants.ts @@ -74,7 +74,7 @@ export const ATTACHMENT_PROMPT_ITEMS: AttachmentMenuItem[] = [ enabledWhen: AttachmentItemEnabledWhen.ALWAYS, icon: Zap, id: AttachmentMenuItemId.MCP_PROMPT, - label: 'MCP Prompt', + label: 'MCP Prompts', visibleWhen: AttachmentItemVisibleWhen.HAS_MCP_PROMPTS_SUPPORT } ]; diff --git a/tools/ui/src/lib/constants/cache.constants.ts b/tools/ui/src/lib/constants/cache.constants.ts index b60792d99..9c6bfadf8 100644 --- a/tools/ui/src/lib/constants/cache.constants.ts +++ b/tools/ui/src/lib/constants/cache.constants.ts @@ -32,13 +32,3 @@ export const MCP_RESOURCE_CACHE = { /** TTL for MCP resource cache entries in milliseconds (5 minutes) */ TTL_MS: 5 * 60 * 1000 } as const; - -/** - * Limits for pruning inactive conversation states held in memory. - */ -export const INACTIVE_CONVERSATION = { - /** Maximum age (in ms) for inactive conversation states before cleanup (30 minutes) */ - MAX_AGE_MS: 30 * 60 * 1000, - /** Maximum number of inactive conversation states to keep in memory */ - MAX_STATES: 10 -} as const; diff --git a/tools/ui/src/lib/constants/url.constants.ts b/tools/ui/src/lib/constants/url.constants.ts index 214c8afba..8df442934 100644 --- a/tools/ui/src/lib/constants/url.constants.ts +++ b/tools/ui/src/lib/constants/url.constants.ts @@ -1,3 +1,5 @@ +import { UrlProtocol } from '$lib/enums'; + const STD = ['com', 'net', 'org', 'gov', 'edu'] as const; const STD_MIL = [...STD, 'mil'] as const; const ccTLD_PREFIXES: Record = { @@ -184,3 +186,7 @@ export const WILDCARD_PUBLIC_SUFFIXES = buildSuffixSet(WILDCARD_BASES); // Matches one or more trailing "/" characters at the end of a URL/path. export const TRAILING_SLASHES_REGEX = /\/+$/; + +// Protocols that apiFetch treats as absolute and passes through untouched. +// Add a protocol here when a caller needs to fetch an absolute URL with it. +export const API_ABSOLUTE_URL_PROTOCOLS = [UrlProtocol.HTTP, UrlProtocol.HTTPS] as const; diff --git a/tools/ui/src/lib/hooks/use-auto-scroll.svelte.ts b/tools/ui/src/lib/hooks/use-auto-scroll.svelte.ts index 6ebce15da..d55574efe 100644 --- a/tools/ui/src/lib/hooks/use-auto-scroll.svelte.ts +++ b/tools/ui/src/lib/hooks/use-auto-scroll.svelte.ts @@ -14,18 +14,14 @@ export interface AutoScrollOptions { */ export class AutoScrollController { private _autoScrollEnabled = $state(true); - private _userScrolledUp = $state(false); - private _lastScrollTop = $state(0); - private _scrollInterval: ReturnType | undefined; private _container: HTMLElement | undefined; private _disabled: boolean; + private _lastScrollTop = $state(0); private _mutationObserver: MutationObserver | null = null; - private _rafPending = false; private _observerEnabled = false; - constructor(options: AutoScrollOptions = {}) { - this._disabled = options.disabled ?? false; - } - + private _rafPending = false; + private _scrollInterval: ReturnType | undefined; + private _userScrolledUp = $state(false); get autoScrollEnabled(): boolean { return this._autoScrollEnabled; } @@ -34,6 +30,71 @@ export class AutoScrollController { return this._userScrolledUp; } + constructor(options: AutoScrollOptions = {}) { + this._disabled = options.disabled ?? false; + } + + /** + * Cleans up resources. Call this in onDestroy or when the component unmounts. + */ + destroy(): void { + this.stopInterval(); + this._doStopObserving(); + } + + /** + * Enables auto-scroll (e.g., when user sends a message). + */ + enable(): void { + if (this._disabled) return; + + this._userScrolledUp = false; + this._autoScrollEnabled = true; + } + + /** + * Handles scroll events to detect user scroll direction and toggle auto-scroll. + */ + handleScroll(): void { + if (this._disabled || !this._container) return; + + const { clientHeight, scrollHeight, scrollTop } = this._container; + const distanceFromBottom = scrollHeight - clientHeight - scrollTop; + const isScrollingUp = scrollTop < this._lastScrollTop; + const isAtBottom = distanceFromBottom < AUTO_SCROLL_AT_BOTTOM_THRESHOLD; + + if (isScrollingUp && !isAtBottom) { + this._userScrolledUp = true; + this._autoScrollEnabled = false; + } else if (isAtBottom && this._userScrolledUp) { + this._userScrolledUp = false; + this._autoScrollEnabled = true; + } + + this._lastScrollTop = scrollTop; + } + + /** + * Resets scroll state when switching conversations. + */ + resetScrollState(): void { + this._userScrolledUp = false; + this._autoScrollEnabled = !this._disabled; + + if (this._container) { + this._lastScrollTop = this._container.scrollTop; + } + } + + /** + * Scrolls the container to the bottom instantly. + */ + scrollToBottom(): void { + if (this._disabled || !this._container) return; + + this._container.scrollTop = this._container.scrollHeight; + } + /** * Binds the controller to a scrollable container element. */ @@ -63,59 +124,6 @@ export class AutoScrollController { } } - /** - * Handles scroll events to detect user scroll direction and toggle auto-scroll. - */ - handleScroll(): void { - if (this._disabled || !this._container) return; - - const { clientHeight, scrollHeight, scrollTop } = this._container; - const distanceFromBottom = scrollHeight - clientHeight - scrollTop; - const isScrollingUp = scrollTop < this._lastScrollTop; - const isAtBottom = distanceFromBottom < AUTO_SCROLL_AT_BOTTOM_THRESHOLD; - - if (isScrollingUp && !isAtBottom) { - this._userScrolledUp = true; - this._autoScrollEnabled = false; - } else if (isAtBottom && this._userScrolledUp) { - this._userScrolledUp = false; - this._autoScrollEnabled = true; - } - - this._lastScrollTop = scrollTop; - } - - /** - * Scrolls the container to the bottom instantly. - */ - scrollToBottom(): void { - if (this._disabled || !this._container) return; - - this._container.scrollTop = this._container.scrollHeight; - } - - /** - * Enables auto-scroll (e.g., when user sends a message). - */ - enable(): void { - if (this._disabled) return; - - this._userScrolledUp = false; - this._autoScrollEnabled = true; - } - - /** - * Resets scroll state when switching conversations. - */ - resetScrollState(): void { - this._userScrolledUp = false; - this._autoScrollEnabled = !this._disabled; - - if (this._container) { - this._lastScrollTop = this._container.scrollTop; - } - } - /** * Starts the auto-scroll interval for continuous scrolling during streaming. */ @@ -127,6 +135,18 @@ export class AutoScrollController { }, AUTO_SCROLL_INTERVAL); } + /** + * Starts a MutationObserver on the container that auto-scrolls to bottom + * on content changes. More responsive than interval-based polling. + */ + startObserving(): void { + this._observerEnabled = true; + + if (this._container && !this._disabled && !this._mutationObserver) { + this._doStartObserving(); + } + } + /** * Stops the auto-scroll interval. */ @@ -137,6 +157,14 @@ export class AutoScrollController { } } + /** + * Stops the MutationObserver. + */ + stopObserving(): void { + this._observerEnabled = false; + this._doStopObserving(); + } + /** * Updates the auto-scroll interval based on streaming state. * Call this in a $effect to automatically manage the interval. @@ -157,34 +185,6 @@ export class AutoScrollController { } } - /** - * Cleans up resources. Call this in onDestroy or when the component unmounts. - */ - destroy(): void { - this.stopInterval(); - this._doStopObserving(); - } - - /** - * Starts a MutationObserver on the container that auto-scrolls to bottom - * on content changes. More responsive than interval-based polling. - */ - startObserving(): void { - this._observerEnabled = true; - - if (this._container && !this._disabled && !this._mutationObserver) { - this._doStartObserving(); - } - } - - /** - * Stops the MutationObserver. - */ - stopObserving(): void { - this._observerEnabled = false; - this._doStopObserving(); - } - private _doStartObserving(): void { if (!this._container || this._mutationObserver) return; diff --git a/tools/ui/src/lib/hooks/use-chat-screen-active-model.svelte.ts b/tools/ui/src/lib/hooks/use-chat-screen-active-model.svelte.ts index ceffdd8a3..b5a5d85ce 100644 --- a/tools/ui/src/lib/hooks/use-chat-screen-active-model.svelte.ts +++ b/tools/ui/src/lib/hooks/use-chat-screen-active-model.svelte.ts @@ -22,10 +22,10 @@ export function useChatScreenActiveModel() { $effect(() => { if (activeModelId) { - const cached = modelsStore.getModelProps(activeModelId); + const cached = modelsStore.props.getModelProps(activeModelId); if (!cached) { - modelsStore.fetchModelProps(activeModelId).then(() => { + modelsStore.props.fetchModelProps(activeModelId).then(() => { modelPropsVersion++; }); } @@ -36,7 +36,7 @@ export function useChatScreenActiveModel() { if (activeModelId) { void modelPropsVersion; - return modelsStore.modelSupportsAudio(activeModelId); + return modelsStore.props.modelSupportsAudio(activeModelId); } return false; @@ -45,7 +45,7 @@ export function useChatScreenActiveModel() { if (activeModelId) { void modelPropsVersion; - return modelsStore.modelSupportsVideo(activeModelId); + return modelsStore.props.modelSupportsVideo(activeModelId); } return false; @@ -54,7 +54,7 @@ export function useChatScreenActiveModel() { if (activeModelId) { void modelPropsVersion; - return modelsStore.modelSupportsVision(activeModelId); + return modelsStore.props.modelSupportsVision(activeModelId); } return false; diff --git a/tools/ui/src/lib/hooks/use-context-gauge.svelte.ts b/tools/ui/src/lib/hooks/use-context-gauge.svelte.ts index 07d380224..c6d55e393 100644 --- a/tools/ui/src/lib/hooks/use-context-gauge.svelte.ts +++ b/tools/ui/src/lib/hooks/use-context-gauge.svelte.ts @@ -54,10 +54,10 @@ export function useContextGauge(): UseContextGaugeReturn { const modelId = contextStatsStore.activeModelId; if (modelId && contextStatsStore.isActiveModelLoaded) { - const cached = modelsStore.getModelProps(modelId); + const cached = modelsStore.props.getModelProps(modelId); if (!cached) { - void modelsStore.fetchModelProps(modelId); + void modelsStore.props.fetchModelProps(modelId); } } }); @@ -80,9 +80,9 @@ export function useContextGauge(): UseContextGaugeReturn { if (!modelId || contextStatsStore.isActiveModelLoading) return; try { - await modelsStore.loadModel(modelId); + await modelsStore.status.load(modelId); } catch { - // toast already surfaced by modelsStore.loadModel + // toast already surfaced by modelsStore.status.load } } diff --git a/tools/ui/src/lib/hooks/use-models-selector.svelte.ts b/tools/ui/src/lib/hooks/use-models-selector.svelte.ts index d56eeefcd..7d2770a26 100644 --- a/tools/ui/src/lib/hooks/use-models-selector.svelte.ts +++ b/tools/ui/src/lib/hooks/use-models-selector.svelte.ts @@ -47,7 +47,7 @@ export interface UseModelsSelectorReturn { export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSelectorReturn { const options = $derived( modelsStore.models.filter((option) => { - const modelProps = modelsStore.getModelProps(option.model); + const modelProps = modelsStore.props.getModelProps(option.model); return modelProps?.ui !== false; }) @@ -103,7 +103,7 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele if (open) { modelsStore.fetchRouterModels().then(() => { - modelsStore.fetchModalitiesForLoadedModels(); + modelsStore.props.fetchModalitiesForLoadedModels(); }); } @@ -143,8 +143,8 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele if (!onModelChange && isRouter && !modelsStore.isModelLoaded(option.model)) { isLoadingModel = true; - modelsStore - .loadModel(option.model) + modelsStore.status + .load(option.model) .catch((error) => console.error('Failed to load model:', error)) .finally(() => (isLoadingModel = false)); } diff --git a/tools/ui/src/lib/hooks/use-processing-state.svelte.ts b/tools/ui/src/lib/hooks/use-processing-state.svelte.ts index 37e0748bc..8a6f332f3 100644 --- a/tools/ui/src/lib/hooks/use-processing-state.svelte.ts +++ b/tools/ui/src/lib/hooks/use-processing-state.svelte.ts @@ -43,7 +43,7 @@ export function useProcessingState(): UseProcessingStateReturn { } // Read directly from the reactive state - return chatStore.activeProcessingState; + return chatStore.processing.activeState; }); $effect(() => { diff --git a/tools/ui/src/lib/hooks/use-reasoning-menu.svelte.ts b/tools/ui/src/lib/hooks/use-reasoning-menu.svelte.ts index 2ff67c939..2cb9c9060 100644 --- a/tools/ui/src/lib/hooks/use-reasoning-menu.svelte.ts +++ b/tools/ui/src/lib/hooks/use-reasoning-menu.svelte.ts @@ -42,19 +42,20 @@ export function useReasoningMenu(): UseReasoningMenuReturn { }); const modelSupportsThinking = $derived.by(() => { void modelsStore.loadedModelIds; - void modelsStore.propsCacheVersion; + void modelsStore.props.cacheVersion; if (serverStore.isRouterMode) { const modelId = modelsStore.selectedModelName || conversationModel; return ( - modelsStore.checkModelSupportsThinking(modelId ?? '') || modelSupportsThinkingFromMessages + modelsStore.props.checkModelSupportsThinking(modelId ?? '') || + modelSupportsThinkingFromMessages ); } - return modelsStore.supportsThinking || modelSupportsThinkingFromMessages; + return modelsStore.props.supportsThinking || modelSupportsThinkingFromMessages; }); - const currentEffort = $derived(conversationsStore.getReasoningEffort()); + const currentEffort = $derived(conversationsStore.preferences.getReasoningEffort()); const thinkingEnabled = $derived( currentEffort !== ReasoningEffort.OFF && currentEffort !== ReasoningEffort.DEFAULT ); @@ -76,7 +77,7 @@ export function useReasoningMenu(): UseReasoningMenuReturn { return modelSupportsThinking; }, select(level: ReasoningEffortLevel): void { - conversationsStore.setReasoningEffort(level.value as ReasoningEffort); + conversationsStore.preferences.setReasoningEffort(level.value as ReasoningEffort); }, get thinkingEnabled() { return thinkingEnabled; diff --git a/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts b/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts index 80b3b85a9..e9dc0dcab 100644 --- a/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts +++ b/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts @@ -35,7 +35,7 @@ export function useToolsPanel(): UseToolsPanelReturn { (g) => g.source !== ToolSource.MCP || !g.serverId || - conversationsStore.isMcpServerEnabledForChat(g.serverId) + conversationsStore.preferences.isMcpServerEnabledForChat(g.serverId) ) ); const totalToolCount = $derived(activeGroups.reduce((n, g) => n + g.tools.length, 0)); @@ -73,7 +73,7 @@ export function useToolsPanel(): UseToolsPanelReturn { return ( group.source === ToolSource.MCP && !!group.serverId && - !conversationsStore.isMcpServerEnabledForChat(group.serverId) + !conversationsStore.preferences.isMcpServerEnabledForChat(group.serverId) ); } diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index f609b4f4e..b008b16db 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -1,4 +1,11 @@ -import { settingsStore } from '../stores/settings.svelte'; +/** + * ChatService - Stateless chat completion and streaming API layer + * + * Wraps the /chat/completions and /stream endpoints: request building, SSE + * parsing, streaming callbacks, resume/probe logic and pre-encode KV-cache + * warming. No reactive state; consumed by chatStore and its managers. + */ + import { getAudioInputFormat } from '../utils/audio-format'; import { capImageDataURLSize } from '../utils/cap-img-size'; import { @@ -25,7 +32,8 @@ import { ReasoningFormat, StreamConnectionState } from '$lib/enums'; -import { modelsStore } from '$lib/stores/models.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { DatabaseMessageExtraMcpPrompt, DatabaseMessageExtraMcpResource } from '$lib/types'; import type { ApiChatCompletionToolCall, @@ -53,13 +61,310 @@ function streamStorageKey(conversationId: string): string { } export class ChatService { + // Per-chunk localStorage writes are throttled to at most one per + // conversation per interval (saveStreamStateThrottled). The resume offset + // only needs to be roughly current: on resume the server retransmits from + // a line boundary and the client discards its partial line. Guaranteed + // immediate writes happen at stream start, at resume boundaries and when + // the page goes hidden or away (pagehide/visibilitychange), so a reload + // always finds a usable offset. + private static readonly STREAM_STATE_SAVE_INTERVAL_MS = 500; + + private static streamStateSaveTrackers = new Map< + string, + { lastSavedAt: number; model: string | null; pendingBytes: number | null } + >(); + /** + * Checks whether all server slots are currently idle (not processing any requests). + * Queries the /slots endpoint (requires --slots flag on the server). + * Returns true if all slots are idle, false if any is processing. + * If the endpoint is unavailable or errors out, returns true (best-effort fallback). * - * - * Title Generation - * - * + * @param signal - Optional AbortSignal to cancel the request if needed + * @param model - Optional model name to check slots for (required in ROUTER mode) + * @returns {Promise} Promise that resolves to true if all slots are idle, false if any is processing */ + static async areAllSlotsIdle(model?: string | null, signal?: AbortSignal): Promise { + try { + const url = model ? `${API_SLOTS.LIST}?model=${encodeURIComponent(model)}` : API_SLOTS.LIST; + const res = await fetch(url, { signal }); + + if (!res.ok) return true; + + const slots: { is_processing: boolean }[] = await res.json(); + + return slots.every((s) => !s.is_processing); + } catch { + return true; + } + } + + /** + * Cancels the server-side replay buffer for a conversation, freeing its slot. + */ + static async cancelServerStream(conversationId: string, model?: string | null): Promise { + if (!conversationId) return; + + try { + const id = streamIdentity(conversationId, model); + + await fetch(ChatService.buildStreamUrl(id), { + headers: getAuthHeaders(), + method: 'DELETE' + }); + } catch (e) { + console.warn('cancelServerStream failed:', e); + } + } + + static clearStreamState(conversationId: string): void { + if (!conversationId) return; + + ChatService.streamStateSaveTrackers.delete(conversationId); + + try { + localStorage.removeItem(streamStorageKey(conversationId)); + } catch { + // nothing to do + } + } + + /** + * Converts a database message with attachments to API chat message format. + * Processes various attachment types (images, text files, PDFs) and formats them + * as content parts suitable for the chat completion API. + */ + static async convertDbMessageToApiChatMessageData( + message: DatabaseMessage & { extra?: DatabaseMessageExtra[] } + ): Promise { + // Handle tool result messages (role: 'tool') + if (message.role === MessageRole.TOOL && message.toolCallId) { + return { + content: message.content, + role: MessageRole.TOOL, + tool_call_id: message.toolCallId + }; + } + + // Parse tool calls for assistant messages + let toolCalls: ApiChatCompletionToolCall[] | undefined; + + if (message.toolCalls) { + try { + toolCalls = JSON.parse(message.toolCalls); + } catch { + // Ignore parse errors for malformed tool calls + } + } + + if (!message.extra || message.extra.length === 0) { + const result: ApiChatMessageData = { + content: message.content, + role: message.role as MessageRole + }; + + if (message.reasoningContent) { + result.reasoning_content = message.reasoningContent; + } + + if (toolCalls && toolCalls.length > 0) { + result.tool_calls = toolCalls; + } + + return result; + } + + const contentParts: ApiChatMessageContentPart[] = []; + const textFiles = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraTextFile => + extra.type === AttachmentType.TEXT + ); + + for (const textFile of textFiles) { + contentParts.push({ + text: formatAttachmentText(AttachmentLabel.FILE, textFile.name, textFile.content), + type: ContentPartType.TEXT + }); + } + + // Handle legacy 'context' type from the old UI (pasted content) + const legacyContextFiles = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraLegacyContext => + extra.type === AttachmentType.LEGACY_CONTEXT + ); + + for (const legacyContextFile of legacyContextFiles) { + contentParts.push({ + text: formatAttachmentText( + AttachmentLabel.FILE, + legacyContextFile.name, + legacyContextFile.content + ), + type: ContentPartType.TEXT + }); + } + + const imageFiles = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraImageFile => + extra.type === AttachmentType.IMAGE + ); + + for (const image of imageFiles) { + const maxImageResolution = settingsStore.getConfig(SETTINGS_KEYS.MAX_IMAGE_RESOLUTION); + // Caps the resolution and bakes the jpeg exif orientation in one pass, + // untouched images pass through as is + const base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution); + + contentParts.push({ + image_url: { url: base64Url }, + type: ContentPartType.IMAGE_URL + }); + } + + const audioFiles = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraAudioFile => + extra.type === AttachmentType.AUDIO + ); + + for (const audio of audioFiles) { + contentParts.push({ + input_audio: { + data: audio.base64Data, + format: getAudioInputFormat(audio.mimeType) + }, + type: ContentPartType.INPUT_AUDIO + }); + } + + if (message.content) { + contentParts.push({ + text: message.content, + type: ContentPartType.TEXT + }); + } + + const videoFiles = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraVideoFile => + extra.type === AttachmentType.VIDEO + ); + + for (const video of videoFiles) { + contentParts.push({ + input_video: { + data: video.base64Data, + format: video.mimeType.includes('mp4') + ? 'mp4' + : video.mimeType.includes('ogg') + ? 'ogg' + : 'auto' + }, + type: ContentPartType.INPUT_VIDEO + }); + } + + const pdfFiles = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraPdfFile => + extra.type === AttachmentType.PDF + ); + + for (const pdfFile of pdfFiles) { + if (pdfFile.processedAsImages && pdfFile.images) { + for (let i = 0; i < pdfFile.images.length; i++) { + contentParts.push({ + image_url: { url: pdfFile.images[i] }, + type: ContentPartType.IMAGE_URL + }); + } + } else { + contentParts.push({ + text: formatAttachmentText(AttachmentLabel.PDF_FILE, pdfFile.name, pdfFile.content), + type: ContentPartType.TEXT + }); + } + } + + const mcpPrompts = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraMcpPrompt => + extra.type === AttachmentType.MCP_PROMPT + ); + + for (const mcpPrompt of mcpPrompts) { + contentParts.push({ + text: formatAttachmentText( + AttachmentLabel.MCP_PROMPT, + mcpPrompt.name, + mcpPrompt.content, + mcpPrompt.serverName + ), + type: ContentPartType.TEXT + }); + } + + const mcpResources = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraMcpResource => + extra.type === AttachmentType.MCP_RESOURCE + ); + + for (const mcpResource of mcpResources) { + contentParts.push({ + text: formatAttachmentText( + AttachmentLabel.MCP_RESOURCE, + mcpResource.name, + mcpResource.content, + mcpResource.serverName + ), + type: ContentPartType.TEXT + }); + } + + const result: ApiChatMessageData = { + content: contentParts, + role: message.role as MessageRole + }; + + if (message.reasoningContent) { + result.reasoning_content = message.reasoningContent; + } + + if (toolCalls && toolCalls.length > 0) { + result.tool_calls = toolCalls; + } + + return result; + } + + /** + * Fetch the full replay of a server-side stream from byte 0. Returns the raw Response so the + * caller can pipe it through the SSE parser like a fresh stream. + */ + static async fetchStreamReplay(streamId: string): Promise { + const resp = await fetch(ChatService.buildStreamUrl(streamId, 0), { + headers: getAuthHeaders() + }); + + if (!resp.ok) { + throw new ApiError(`Stream replay failed with HTTP ${resp.status}`, resp.status); + } + + return resp; + } + + // write a throttled-but-not-yet-persisted offset immediately; used at + // resume boundaries and on pagehide/visibilitychange so the persisted + // offset is the freshest one when it matters + static flushStreamState(conversationId: string): void { + const tracker = ChatService.streamStateSaveTrackers.get(conversationId); + + if (!tracker || tracker.pendingBytes === null) return; + + const { model, pendingBytes } = tracker; + + tracker.lastSavedAt = Date.now(); + tracker.pendingBytes = null; + + ChatService.writeStreamState(conversationId, pendingBytes, model); + } /** * Sends a streaming chat completion request for generating a chat title. @@ -99,13 +404,610 @@ export class ChatService { return titleResponse; } + static getStreamState(conversationId: string): ResumableStreamState | null { + if (!conversationId) return null; + + try { + const raw = localStorage.getItem(streamStorageKey(conversationId)); + + if (!raw) return null; + + const parsed = JSON.parse(raw) as ResumableStreamState; + + if (!parsed || typeof parsed.bytesReceived !== 'number') return null; + + return parsed; + } catch { + return null; + } + } + /** - * - * - * Messaging - * - * + * Handles streaming response from the chat completion API. */ + static async handleStreamResponse( + response: Response, + onChunk?: (chunk: string) => void, + onComplete?: ( + response: string, + reasoningContent?: string, + timings?: ChatMessageTimings, + toolCalls?: string + ) => void, + onError?: (error: Error) => void, + onReasoningChunk?: (chunk: string) => void, + onToolCallChunk?: (chunk: string) => void, + onModel?: (model: string) => void, + onCompletionId?: (id: string) => void, + onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void, + conversationId?: string, + abortSignal?: AbortSignal, + onConnectionState?: (state: StreamConnectionState) => void, + streamModel?: string | null + ): Promise { + let reader = response.body?.getReader(); + + if (!reader) { + throw new Error('No response body'); + } + + // bytesParsed is the absolute server side buffer offset of the next byte to parse + // segmentStartOffset is the absolute offset where the current reader started, reset on resume + // segmentBytesRead is wire bytes read by the current reader + let bytesParsed = 0; + let segmentStartOffset = 0; + let segmentBytesRead = 0; + let lastByteAt = Date.now(); + // each resume must produce at least one byte to be retried again + // if a resume returns 200 but yields nothing, we abandon + // since the session has a bounded size, the total number of retries is bounded by construction + let madeProgress = true; + + const encoder = new TextEncoder(); + + if (conversationId) { + ChatService.saveStreamState(conversationId, 0, streamModel); + } + + onConnectionState?.(StreamConnectionState.STREAMING); + + let decoder = new TextDecoder(); + let aggregatedContent = ''; + let fullReasoningContent = ''; + let aggregatedToolCalls: ApiChatCompletionToolCall[] = []; + let lastTimings: ChatMessageTimings | undefined; + let streamFinished = false; + let modelEmitted = false; + let idEmitted = false; + let toolCallIndexOffset = 0; + let hasOpenToolCallBatch = false; + + const finalizeOpenToolCallBatch = () => { + if (!hasOpenToolCallBatch) { + return; + } + + toolCallIndexOffset = aggregatedToolCalls.length; + hasOpenToolCallBatch = false; + }; + const processToolCallDelta = (toolCalls?: ApiChatCompletionToolCallDelta[]) => { + if (!toolCalls || toolCalls.length === 0) { + return; + } + + aggregatedToolCalls = ChatService.mergeToolCallDeltas( + aggregatedToolCalls, + toolCalls, + toolCallIndexOffset + ); + + if (aggregatedToolCalls.length === 0) { + return; + } + + hasOpenToolCallBatch = true; + + const serializedToolCalls = JSON.stringify(aggregatedToolCalls); + + if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log('[ChatService] Aggregated tool calls:', serializedToolCalls); + } + + if (!serializedToolCalls) { + return; + } + + if (!abortSignal?.aborted) { + onToolCallChunk?.(serializedToolCalls); + } + }; + const onVisibilityChange = () => { + if (typeof document === 'undefined') return; + + if (document.visibilityState === 'hidden') { + // the tab is going to the background and the OS may throttle or + // drop the socket shortly; persist the freshest resume offset now + if (conversationId) ChatService.flushStreamState(conversationId); + + return; + } + + if (streamFinished) return; + + if (!conversationId) return; + + // the bytes have been quiet for too long, the OS likely killed the socket + // kicking the reader unblocks reader.read with done=true so the outer loop can resume + if (Date.now() - lastByteAt > STREAM_VISIBILITY_KICK_MS) { + reader!.cancel().catch(() => {}); + } + }; + const onPageHide = () => { + // a reload or navigation is about to happen; make sure the resume + // offset that getStreamState() will read is not a stale throttled one + if (conversationId) ChatService.flushStreamState(conversationId); + }; + + if (typeof document !== 'undefined') { + document.addEventListener('visibilitychange', onVisibilityChange); + window.addEventListener('pagehide', onPageHide); + } + + try { + let chunk = ''; + + // outer loop drives the resume cycle, swaps reader on premature end of stream + while (true) { + while (true) { + if (abortSignal?.aborted) break; + + let done: boolean; + let value: Uint8Array | undefined; + + try { + const r = await reader.read(); + + done = r.done; + value = r.value; + } catch (readErr) { + // reader.read() rejects with TypeError when the underlying connection drops + // instead of just resolving with done=true. treat it like done so the outer + // loop swaps reader via the resume path + if (isAbortError(readErr)) { + throw readErr; + } + + console.warn('reader.read() rejected, treating as premature end:', readErr); + done = true; + value = undefined; + } + + if (done) break; + + if (abortSignal?.aborted) break; + + if (value && value.byteLength > 0) { + segmentBytesRead += value.byteLength; + lastByteAt = Date.now(); + + if (!madeProgress) { + madeProgress = true; + onConnectionState?.(StreamConnectionState.STREAMING); + } + } + + chunk += decoder.decode(value, { stream: true }); + const lines = chunk.split(SSE_LINE_SEPARATOR); + + chunk = lines.pop() || ''; + + // the persisted offset must point right after the last fully parsed line, + // the trailing `chunk` is partial bytes still waiting for a newline + if (conversationId) { + const tailBytes = encoder.encode(chunk).byteLength; + + bytesParsed = segmentStartOffset + segmentBytesRead - tailBytes; + ChatService.saveStreamStateThrottled(conversationId, bytesParsed, streamModel); + } + + for (const line of lines) { + if (abortSignal?.aborted) break; + + if (line.startsWith(SSE_DATA_PREFIX)) { + const data = line.slice(SSE_DATA_PREFIX.length).trim(); + + if (data === SSE_DONE_MARKER) { + streamFinished = true; + + continue; + } + + try { + const parsed: ApiChatCompletionStreamChunk = JSON.parse(data); + const choice = parsed.choices?.[0]; + const content = choice?.delta?.content; + const reasoningContent = choice?.delta?.reasoning_content; + const toolCalls = choice?.delta?.tool_calls; + const timings = parsed.timings; + const promptProgress = parsed.prompt_progress; + const chunkModel = ChatService.extractModelName(parsed); + + if (chunkModel && !modelEmitted) { + modelEmitted = true; + onModel?.(chunkModel); + } + + if (parsed.id && !idEmitted) { + idEmitted = true; + onCompletionId?.(parsed.id); + } + + if (promptProgress) { + ChatService.notifyTimings(undefined, promptProgress, onTimings); + } + + if (timings) { + ChatService.notifyTimings(timings, promptProgress, onTimings); + lastTimings = timings; + } + + if (content) { + finalizeOpenToolCallBatch(); + aggregatedContent += content; + + if (!abortSignal?.aborted) { + onChunk?.(content); + } + } + + if (reasoningContent) { + finalizeOpenToolCallBatch(); + fullReasoningContent += reasoningContent; + + if (!abortSignal?.aborted) { + onReasoningChunk?.(reasoningContent); + } + } + + processToolCallDelta(toolCalls); + } catch (e) { + console.error('Error parsing JSON chunk:', e); + } + } + } + + if (abortSignal?.aborted) break; + + if (streamFinished) break; + } + + // inner reader done, decide whether to try a resume + if (abortSignal?.aborted) break; + + if (streamFinished) break; + + if (!conversationId) break; + + if (!madeProgress) { + onConnectionState?.(StreamConnectionState.LOST); + onError?.(new Error('Stream resume produced no new bytes, giving up')); + + break; + } + + onConnectionState?.(StreamConnectionState.RESUMING); + madeProgress = false; + + // the server resends starting at bytesParsed, discard any partial line we held, it + // will be retransmitted from a clean line boundary. reuse the frozen model, not the + // live dropdown + // resumeStream reads the offset from localStorage, so persist the + // freshest bytesParsed before asking the server to replay from it + ChatService.flushStreamState(conversationId); + const resumeResp = await ChatService.resumeStream( + conversationId, + abortSignal, + streamModel + ).catch(() => null); + + // an abort landing during the resume request is intentional, not a lost connection + if (abortSignal?.aborted) break; + + if (!resumeResp || resumeResp.status !== 200) { + onConnectionState?.(StreamConnectionState.LOST); + onError?.(new Error('Stream connection lost and could not be resumed')); + + break; + } + + const newReader = resumeResp.body?.getReader(); + + if (!newReader) break; + + try { + reader.releaseLock(); + } catch { + /* ignore */ + } + reader = newReader; + decoder = new TextDecoder(); + chunk = ''; + segmentStartOffset = bytesParsed; + segmentBytesRead = 0; + lastByteAt = Date.now(); + } + + if (abortSignal?.aborted) return; + + if (streamFinished) { + finalizeOpenToolCallBatch(); + + if (conversationId) { + ChatService.clearStreamState(conversationId); + } + + const finalToolCalls = + aggregatedToolCalls.length > 0 ? JSON.stringify(aggregatedToolCalls) : undefined; + + onComplete?.( + aggregatedContent, + fullReasoningContent || undefined, + lastTimings, + finalToolCalls + ); + } + } catch (error) { + const err = error instanceof Error ? error : new Error('Stream error'); + + onError?.(err); + + throw err; + } finally { + if (typeof document !== 'undefined') { + document.removeEventListener('visibilitychange', onVisibilityChange); + window.removeEventListener('pagehide', onPageHide); + } + + try { + reader.releaseLock(); + } catch { + /* ignore */ + } + } + } + + /** + * Look up server-side stream sessions for the given conversation ids. Ids carry the frozen + * conv::model identity when a model was bound at POST time. + */ + static async lookupStreamSessions(conversationIds: string[]): Promise { + const resp = await fetch(API_STREAM.LOOKUP, { + body: JSON.stringify({ conversation_ids: conversationIds }), + headers: getJsonHeaders(), + method: 'POST' + }); + + if (!resp.ok) { + throw new ApiError(`Stream lookup failed with HTTP ${resp.status}`, resp.status); + } + + const body = (await resp.json()) as unknown; + + if (!Array.isArray(body)) { + throw new Error('Stream lookup returned a non-array response'); + } + + return body as ApiStreamSession[]; + } + + /** + * Normalizes an array of messages (database or already-API-shaped) into + * API chat message data, converting DB messages and dropping empty system + * messages. Shared by sendMessage, preEncode and the agentic flow. + */ + static async normalizeMessagesForApi( + messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[] + ): Promise { + return ( + await Promise.all( + messages.map((msg) => { + if ('id' in msg && 'convId' in msg && 'timestamp' in msg) { + return ChatService.convertDbMessageToApiChatMessageData( + msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] } + ); + } + + return msg as ApiChatMessageData; + }) + ) + ).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => { + // Filter out empty system messages + if (msg.role === MessageRole.SYSTEM) { + const content = typeof msg.content === 'string' ? msg.content : ''; + + return content.trim().length > 0; + } + + return true; + }); + } + + /** + * Fire-and-forget request to pre-encode the conversation in the server's KV cache. + * Re-submits the full conversation with n_predict=0 so the server processes the prompt + * without generating tokens, warming the cache for the next turn. + */ + static async preEncode( + messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[], + model?: string | null, + excludeReasoning?: boolean, + signal?: AbortSignal + ): Promise { + const normalizedMessages: ApiChatMessageData[] = + await ChatService.normalizeMessagesForApi(messages); + const requestBody: Record = { + messages: normalizedMessages.map((msg: ApiChatMessageData) => { + const mapped: Record = { + content: excludeReasoning ? ChatService.stripReasoningContent(msg.content) : msg.content, + role: msg.role, + tool_call_id: msg.tool_call_id, + tool_calls: msg.tool_calls + }; + + if (!excludeReasoning && msg.reasoning_content) { + mapped.reasoning_content = msg.reasoning_content; + } + + return mapped; + }), + n_predict: 0, + stream: false + }; + + if (model) { + requestBody.model = model; + } + + try { + await fetch(API_CHAT.COMPLETIONS, { + body: JSON.stringify(requestBody), + headers: getJsonHeaders(), + method: 'POST', + signal + }); + } catch (error) { + if (!isAbortError(error)) { + console.warn('[ChatService] Pre-encode request failed:', error); + } + } + } + + // probe the resume route status without consuming the stream: the SSE route has no HEAD, + // so issue the GET and abort it right after the status line. 0 on network error + static async probeResumeStatus(streamId: string): Promise { + if (!streamId) return 0; + + const ac = new AbortController(); + + try { + const resp = await fetch(ChatService.buildStreamUrl(streamId, 0), { + headers: getAuthHeaders(), + signal: ac.signal + }); + + ac.abort(); + + return resp.status; + } catch { + return 0; + } + } + + static async resumeStream( + conversationId: string, + signal?: AbortSignal, + model?: string | null + ): Promise { + if (!conversationId) return null; + + const state = ChatService.getStreamState(conversationId); + const from = state?.bytesReceived ?? 0; + const id = streamIdentity(conversationId, model); + const url = ChatService.buildStreamUrl(id, from); + + return await fetch(url, { headers: getAuthHeaders(), method: 'GET', signal }); + } + + /** + * Rebuild the stream identity for a resume. The model persisted at POST time wins, including a + * stored null which means the POST carried no explicit model so the identity stays the bare conv + * id. Only fall back to the caller supplied current model when nothing was persisted. + */ + static resumeStreamIdentity( + conversationId: string, + state: ResumableStreamState | null, + fallbackModel: string | null + ): string { + const model = state && state.model !== undefined ? state.model : fallbackModel; + + return streamIdentity(conversationId, model); + } + + // persist the running byte count and the frozen model for a conversation, a later visit + // resumes the SSE replay at the right offset under the same conv::model + // identity. Writes immediately; the per-chunk read loop uses the throttled + // variant instead. + static saveStreamState( + conversationId: string, + bytesReceived: number, + model?: string | null + ): void { + if (!conversationId) return; + + ChatService.writeStreamState(conversationId, bytesReceived, model); + // record the write so a throttled save landing inside the interval + // holds its value pending instead of re-writing + ChatService.streamStateSaveTrackers.set(conversationId, { + lastSavedAt: Date.now(), + model: model ?? null, + pendingBytes: null + }); + } + + // throttled variant for the per-chunk read loop: writes at most once per + // conversation per STREAM_STATE_SAVE_INTERVAL_MS, holding the latest value + // pending until the interval elapses or flushStreamState() forces it out + static saveStreamStateThrottled( + conversationId: string, + bytesReceived: number, + model?: string | null + ): void { + if (!conversationId) return; + + const tracker = ChatService.streamStateSaveTrackers.get(conversationId) ?? { + lastSavedAt: 0, + model: null, + pendingBytes: null + }; + + tracker.model = model ?? null; + + if (Date.now() - tracker.lastSavedAt >= ChatService.STREAM_STATE_SAVE_INTERVAL_MS) { + tracker.lastSavedAt = Date.now(); + tracker.pendingBytes = null; + ChatService.writeStreamState(conversationId, bytesReceived, model); + } else { + tracker.pendingBytes = bytesReceived; + } + + ChatService.streamStateSaveTrackers.set(conversationId, tracker); + } + + /** + * Pick the running session to splice into when discoverActiveStream lists candidates for a + * conversation. Finalized sessions are not candidates: their final content was already written + * to the DB by the original onComplete handler, so attaching to them would replay a buffer that + * may not match what the DB holds. A continue session's buffer holds only the appended deltas, + * not the pre continue prefix, so replaying it as a fresh generation would erase the original. + * + * Among running sessions we tie break on the most recent started_at, which covers the case of + * multiple inferences left running on the same conversation. + */ + static selectActiveStream( + sessions: ApiStreamSession[] | null | undefined + ): ApiStreamSession | null { + if (!Array.isArray(sessions) || sessions.length === 0) { + return null; + } + + const running = sessions.filter((s) => !s.is_done); + + if (running.length === 0) { + return null; + } + + return running.reduce((best, cur) => (cur.started_at > best.started_at ? cur : best)); + } /** * Sends a chat completion request to the llama-server. @@ -169,31 +1071,11 @@ export class ChatService { xtc_probability, xtc_threshold } = options; - const normalizedMessages: ApiChatMessageData[] = ( - await Promise.all( - messages.map((msg) => { - if ('id' in msg && 'convId' in msg && 'timestamp' in msg) { - const dbMsg = msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }; - - return ChatService.convertDbMessageToApiChatMessageData(dbMsg); - } else { - return msg as ApiChatMessageData; - } - }) - ) - ).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => { - // Filter out empty system messages - if (msg.role === MessageRole.SYSTEM) { - const content = typeof msg.content === 'string' ? msg.content : ''; - - return content.trim().length > 0; - } - - return true; - }); + const normalizedMessages: ApiChatMessageData[] = + await ChatService.normalizeMessagesForApi(messages); // Filter out image attachments if the model doesn't support vision - if (options.model && !modelsStore.modelSupportsVision(options.model)) { + if (options.model && !modelsStore.props.modelSupportsVision(options.model)) { normalizedMessages.forEach((msg) => { if (Array.isArray(msg.content)) { msg.content = msg.content.filter((part: ApiChatMessageContentPart) => { @@ -436,31 +1318,6 @@ export class ChatService { } } - /** - * Checks whether all server slots are currently idle (not processing any requests). - * Queries the /slots endpoint (requires --slots flag on the server). - * Returns true if all slots are idle, false if any is processing. - * If the endpoint is unavailable or errors out, returns true (best-effort fallback). - * - * @param signal - Optional AbortSignal to cancel the request if needed - * @param model - Optional model name to check slots for (required in ROUTER mode) - * @returns {Promise} Promise that resolves to true if all slots are idle, false if any is processing - */ - static async areAllSlotsIdle(model?: string | null, signal?: AbortSignal): Promise { - try { - const url = model ? `${API_SLOTS.LIST}?model=${encodeURIComponent(model)}` : API_SLOTS.LIST; - const res = await fetch(url, { signal }); - - if (!res.ok) return true; - - const slots: { is_processing: boolean }[] = await res.json(); - - return slots.every((s) => !s.is_processing); - } catch { - return true; - } - } - /** * Ends the current reasoning block of a running completion, targeted by its * chat completion id (streamed back as `id`). Matching the completion rather @@ -510,167 +1367,6 @@ export class ChatService { } } - /** - * Sends a fire-and-forget request to pre-encode the conversation in the server's KV cache. - * After a response completes, this re-submits the full conversation - * using n_predict=0 and stream=false so the server processes the prompt without generating tokens. - * This warms the cache for the next turn, making it faster. - * - * When excludeReasoningFromContext is true, reasoning content is stripped from the messages - * to match what sendMessage would send on the next turn (avoiding cache misses). - * When false, reasoning_content is preserved so the cached prompt matches the next request. - * - * @param messages - The full conversation including the latest assistant response - * @param model - Optional model name (required in ROUTER mode) - * @param excludeReasoning - Whether to strip reasoning content (should match excludeReasoningFromContext setting) - * @param signal - Optional AbortSignal to cancel the pre-encode request - */ - static async cancelServerStream(conversationId: string, model?: string | null): Promise { - if (!conversationId) return; - - try { - const id = streamIdentity(conversationId, model); - - await fetch(ChatService.buildStreamUrl(id), { - headers: getAuthHeaders(), - method: 'DELETE' - }); - } catch (e) { - console.warn('cancelServerStream failed:', e); - } - } - - /** - * Look up server-side stream sessions for the given conversation ids. Ids carry the frozen - * conv::model identity when a model was bound at POST time. - */ - static async lookupStreamSessions(conversationIds: string[]): Promise { - const resp = await fetch(API_STREAM.LOOKUP, { - body: JSON.stringify({ conversation_ids: conversationIds }), - headers: getJsonHeaders(), - method: 'POST' - }); - - if (!resp.ok) { - throw new ApiError(`Stream lookup failed with HTTP ${resp.status}`, resp.status); - } - - const body = (await resp.json()) as unknown; - - if (!Array.isArray(body)) { - throw new Error('Stream lookup returned a non-array response'); - } - - return body as ApiStreamSession[]; - } - - /** - * Fetch the full replay of a server-side stream from byte 0. Returns the raw Response so the - * caller can pipe it through the SSE parser like a fresh stream. - */ - static async fetchStreamReplay(streamId: string): Promise { - const resp = await fetch(ChatService.buildStreamUrl(streamId, 0), { - headers: getAuthHeaders() - }); - - if (!resp.ok) { - throw new ApiError(`Stream replay failed with HTTP ${resp.status}`, resp.status); - } - - return resp; - } - - /** - * Pick the running session to splice into when discoverActiveStream lists candidates for a - * conversation. Finalized sessions are not candidates: their final content was already written - * to the DB by the original onComplete handler, so attaching to them would replay a buffer that - * may not match what the DB holds. A continue session's buffer holds only the appended deltas, - * not the pre continue prefix, so replaying it as a fresh generation would erase the original. - * - * Among running sessions we tie break on the most recent started_at, which covers the case of - * multiple inferences left running on the same conversation. - */ - static selectActiveStream( - sessions: ApiStreamSession[] | null | undefined - ): ApiStreamSession | null { - if (!Array.isArray(sessions) || sessions.length === 0) { - return null; - } - - const running = sessions.filter((s) => !s.is_done); - - if (running.length === 0) { - return null; - } - - return running.reduce((best, cur) => (cur.started_at > best.started_at ? cur : best)); - } - - // persist the running byte count and the frozen model for a conversation, a later visit - // resumes the SSE replay at the right offset under the same conv::model identity - static saveStreamState( - conversationId: string, - bytesReceived: number, - model?: string | null - ): void { - if (!conversationId) return; - - try { - const state: ResumableStreamState = { - bytesReceived, - model: model ?? null, - updatedAt: Date.now() - }; - - localStorage.setItem(streamStorageKey(conversationId), JSON.stringify(state)); - } catch { - // localStorage may be full or disabled, silently ignore - } - } - - static getStreamState(conversationId: string): ResumableStreamState | null { - if (!conversationId) return null; - - try { - const raw = localStorage.getItem(streamStorageKey(conversationId)); - - if (!raw) return null; - - const parsed = JSON.parse(raw) as ResumableStreamState; - - if (!parsed || typeof parsed.bytesReceived !== 'number') return null; - - return parsed; - } catch { - return null; - } - } - - static clearStreamState(conversationId: string): void { - if (!conversationId) return; - - try { - localStorage.removeItem(streamStorageKey(conversationId)); - } catch { - // nothing to do - } - } - - /** - * Rebuild the stream identity for a resume. The model persisted at POST time wins, including a - * stored null which means the POST carried no explicit model so the identity stays the bare conv - * id. Only fall back to the caller supplied current model when nothing was persisted. - */ - static resumeStreamIdentity( - conversationId: string, - state: ResumableStreamState | null, - fallbackModel: string | null - ): string { - const model = state && state.model !== undefined ? state.model : fallbackModel; - - return streamIdentity(conversationId, model); - } - // build the replay route url for a stream identity, from is the resume byte offset, omitted // for the cancel route private static buildStreamUrl(streamId: string, from?: number): string { @@ -681,462 +1377,58 @@ export class ChatService { } /** - * Reconnect to an interrupted stream for this conversation. Returns the fetch Response so the - * existing SSE parser drains it like a fresh stream. The server returns 200 on success, 404 if - * no session exists for the conv_id, and 400 if the offset is below the dropped prefix. + * Extracts model name from Chat Completions API response data. + * Handles various response formats including streaming chunks and final responses. + * + * WORKAROUND: In single model mode, llama-server returns a default/incorrect model name + * in the response. We override it with the actual model name from serverStore. + * + * @param data - Raw response data from the Chat Completions API + * @returns Model name string if found, undefined otherwise + * @private */ - // probe the resume route status without consuming the stream: the SSE route has no HEAD, - // so issue the GET and abort it right after the status line. 0 on network error - static async probeResumeStatus(streamId: string): Promise { - if (!streamId) return 0; - - const ac = new AbortController(); - - try { - const resp = await fetch(ChatService.buildStreamUrl(streamId, 0), { - headers: getAuthHeaders(), - signal: ac.signal - }); - - ac.abort(); - - return resp.status; - } catch { - return 0; - } - } - - static async resumeStream( - conversationId: string, - signal?: AbortSignal, - model?: string | null - ): Promise { - if (!conversationId) return null; - - const state = ChatService.getStreamState(conversationId); - const from = state?.bytesReceived ?? 0; - const id = streamIdentity(conversationId, model); - const url = ChatService.buildStreamUrl(id, from); - - return await fetch(url, { headers: getAuthHeaders(), method: 'GET', signal }); - } - - static async preEncode( - messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[], - model?: string | null, - excludeReasoning?: boolean, - signal?: AbortSignal - ): Promise { - const normalizedMessages: ApiChatMessageData[] = ( - await Promise.all( - messages.map((msg) => { - if ('id' in msg && 'convId' in msg && 'timestamp' in msg) { - return ChatService.convertDbMessageToApiChatMessageData( - msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] } - ); - } - - return msg as ApiChatMessageData; - }) - ) - ).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => { - if (msg.role === MessageRole.SYSTEM) { - const content = typeof msg.content === 'string' ? msg.content : ''; - - return content.trim().length > 0; - } - - return true; - }); - const requestBody: Record = { - messages: normalizedMessages.map((msg: ApiChatMessageData) => { - const mapped: Record = { - content: excludeReasoning ? ChatService.stripReasoningContent(msg.content) : msg.content, - role: msg.role, - tool_call_id: msg.tool_call_id, - tool_calls: msg.tool_calls - }; - - if (!excludeReasoning && msg.reasoning_content) { - mapped.reasoning_content = msg.reasoning_content; - } - - return mapped; - }), - n_predict: 0, - stream: false + private static extractModelName(data: unknown): string | undefined { + const asRecord = (value: unknown): Record | undefined => { + return typeof value === 'object' && value !== null + ? (value as Record) + : undefined; }; - - if (model) { - requestBody.model = model; - } - - try { - await fetch(API_CHAT.COMPLETIONS, { - body: JSON.stringify(requestBody), - headers: getJsonHeaders(), - method: 'POST', - signal - }); - } catch (error) { - if (!isAbortError(error)) { - console.warn('[ChatService] Pre-encode request failed:', error); - } - } - } - - /** - * - * - * Streaming - * - * - */ - - /** - * Handles streaming response from the chat completion API - * @param response - The Response object from the fetch request - * @param onChunk - Optional callback invoked for each content chunk received - * @param onComplete - Optional callback invoked when the stream is complete with full response - * @param onError - Optional callback invoked if an error occurs during streaming - * @param onReasoningChunk - Optional callback invoked for each reasoning content chunk - * @param conversationId - Optional conversation ID for per-conversation state tracking - * @returns {Promise} Promise that resolves when streaming is complete - * @throws {Error} if the stream cannot be read or parsed - */ - static async handleStreamResponse( - response: Response, - onChunk?: (chunk: string) => void, - onComplete?: ( - response: string, - reasoningContent?: string, - timings?: ChatMessageTimings, - toolCalls?: string - ) => void, - onError?: (error: Error) => void, - onReasoningChunk?: (chunk: string) => void, - onToolCallChunk?: (chunk: string) => void, - onModel?: (model: string) => void, - onCompletionId?: (id: string) => void, - onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void, - conversationId?: string, - abortSignal?: AbortSignal, - onConnectionState?: (state: StreamConnectionState) => void, - streamModel?: string | null - ): Promise { - let reader = response.body?.getReader(); - - if (!reader) { - throw new Error('No response body'); - } - - // bytesParsed is the absolute server side buffer offset of the next byte to parse - // segmentStartOffset is the absolute offset where the current reader started, reset on resume - // segmentBytesRead is wire bytes read by the current reader - let bytesParsed = 0; - let segmentStartOffset = 0; - let segmentBytesRead = 0; - let lastByteAt = Date.now(); - // each resume must produce at least one byte to be retried again - // if a resume returns 200 but yields nothing, we abandon - // since the session has a bounded size, the total number of retries is bounded by construction - let madeProgress = true; - - const encoder = new TextEncoder(); - - if (conversationId) { - ChatService.saveStreamState(conversationId, 0, streamModel); - } - - onConnectionState?.(StreamConnectionState.STREAMING); - - let decoder = new TextDecoder(); - let aggregatedContent = ''; - let fullReasoningContent = ''; - let aggregatedToolCalls: ApiChatCompletionToolCall[] = []; - let lastTimings: ChatMessageTimings | undefined; - let streamFinished = false; - let modelEmitted = false; - let idEmitted = false; - let toolCallIndexOffset = 0; - let hasOpenToolCallBatch = false; - - const finalizeOpenToolCallBatch = () => { - if (!hasOpenToolCallBatch) { - return; - } - - toolCallIndexOffset = aggregatedToolCalls.length; - hasOpenToolCallBatch = false; + const getTrimmedString = (value: unknown): string | undefined => { + return typeof value === 'string' && value.trim() ? value.trim() : undefined; }; - const processToolCallDelta = (toolCalls?: ApiChatCompletionToolCallDelta[]) => { - if (!toolCalls || toolCalls.length === 0) { - return; - } + const root = asRecord(data); - aggregatedToolCalls = ChatService.mergeToolCallDeltas( - aggregatedToolCalls, - toolCalls, - toolCallIndexOffset - ); + if (!root) return undefined; - if (aggregatedToolCalls.length === 0) { - return; - } + // 1) root (some implementations provide `model` at the top level) + const rootModel = getTrimmedString(root.model); - hasOpenToolCallBatch = true; - - const serializedToolCalls = JSON.stringify(aggregatedToolCalls); - - if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { - console.log('[ChatService] Aggregated tool calls:', serializedToolCalls); - } - - if (!serializedToolCalls) { - return; - } - - if (!abortSignal?.aborted) { - onToolCallChunk?.(serializedToolCalls); - } - }; - const onVisibilityChange = () => { - if (typeof document === 'undefined') return; - - if (document.visibilityState !== 'visible') return; - - if (streamFinished) return; - - if (!conversationId) return; - - // the bytes have been quiet for too long, the OS likely killed the socket - // kicking the reader unblocks reader.read with done=true so the outer loop can resume - if (Date.now() - lastByteAt > STREAM_VISIBILITY_KICK_MS) { - reader!.cancel().catch(() => {}); - } - }; - - if (typeof document !== 'undefined') { - document.addEventListener('visibilitychange', onVisibilityChange); + if (rootModel) { + return rootModel; } - try { - let chunk = ''; + // 2) streaming choice (delta) or final response (message) + const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined; - // outer loop drives the resume cycle, swaps reader on premature end of stream - while (true) { - while (true) { - if (abortSignal?.aborted) break; - - let done: boolean; - let value: Uint8Array | undefined; - - try { - const r = await reader.read(); - - done = r.done; - value = r.value; - } catch (readErr) { - // reader.read() rejects with TypeError when the underlying connection drops - // instead of just resolving with done=true. treat it like done so the outer - // loop swaps reader via the resume path - if (isAbortError(readErr)) { - throw readErr; - } - - console.warn('reader.read() rejected, treating as premature end:', readErr); - done = true; - value = undefined; - } - - if (done) break; - - if (abortSignal?.aborted) break; - - if (value && value.byteLength > 0) { - segmentBytesRead += value.byteLength; - lastByteAt = Date.now(); - - if (!madeProgress) { - madeProgress = true; - onConnectionState?.(StreamConnectionState.STREAMING); - } - } - - chunk += decoder.decode(value, { stream: true }); - const lines = chunk.split(SSE_LINE_SEPARATOR); - - chunk = lines.pop() || ''; - - // the persisted offset must point right after the last fully parsed line, - // the trailing `chunk` is partial bytes still waiting for a newline - if (conversationId) { - const tailBytes = encoder.encode(chunk).byteLength; - - bytesParsed = segmentStartOffset + segmentBytesRead - tailBytes; - ChatService.saveStreamState(conversationId, bytesParsed, streamModel); - } - - for (const line of lines) { - if (abortSignal?.aborted) break; - - if (line.startsWith(SSE_DATA_PREFIX)) { - const data = line.slice(SSE_DATA_PREFIX.length).trim(); - - if (data === SSE_DONE_MARKER) { - streamFinished = true; - - continue; - } - - try { - const parsed: ApiChatCompletionStreamChunk = JSON.parse(data); - const choice = parsed.choices?.[0]; - const content = choice?.delta?.content; - const reasoningContent = choice?.delta?.reasoning_content; - const toolCalls = choice?.delta?.tool_calls; - const timings = parsed.timings; - const promptProgress = parsed.prompt_progress; - const chunkModel = ChatService.extractModelName(parsed); - - if (chunkModel && !modelEmitted) { - modelEmitted = true; - onModel?.(chunkModel); - } - - if (parsed.id && !idEmitted) { - idEmitted = true; - onCompletionId?.(parsed.id); - } - - if (promptProgress) { - ChatService.notifyTimings(undefined, promptProgress, onTimings); - } - - if (timings) { - ChatService.notifyTimings(timings, promptProgress, onTimings); - lastTimings = timings; - } - - if (content) { - finalizeOpenToolCallBatch(); - aggregatedContent += content; - - if (!abortSignal?.aborted) { - onChunk?.(content); - } - } - - if (reasoningContent) { - finalizeOpenToolCallBatch(); - fullReasoningContent += reasoningContent; - - if (!abortSignal?.aborted) { - onReasoningChunk?.(reasoningContent); - } - } - - processToolCallDelta(toolCalls); - } catch (e) { - console.error('Error parsing JSON chunk:', e); - } - } - } - - if (abortSignal?.aborted) break; - - if (streamFinished) break; - } - - // inner reader done, decide whether to try a resume - if (abortSignal?.aborted) break; - - if (streamFinished) break; - - if (!conversationId) break; - - if (!madeProgress) { - onConnectionState?.(StreamConnectionState.LOST); - onError?.(new Error('Stream resume produced no new bytes, giving up')); - - break; - } - - onConnectionState?.(StreamConnectionState.RESUMING); - madeProgress = false; - - // the server resends starting at bytesParsed, discard any partial line we held, it - // will be retransmitted from a clean line boundary. reuse the frozen model, not the - // live dropdown - const resumeResp = await ChatService.resumeStream( - conversationId, - abortSignal, - streamModel - ).catch(() => null); - - // an abort landing during the resume request is intentional, not a lost connection - if (abortSignal?.aborted) break; - - if (!resumeResp || resumeResp.status !== 200) { - onConnectionState?.(StreamConnectionState.LOST); - onError?.(new Error('Stream connection lost and could not be resumed')); - - break; - } - - const newReader = resumeResp.body?.getReader(); - - if (!newReader) break; - - try { - reader.releaseLock(); - } catch { - /* ignore */ - } - reader = newReader; - decoder = new TextDecoder(); - chunk = ''; - segmentStartOffset = bytesParsed; - segmentBytesRead = 0; - lastByteAt = Date.now(); - } - - if (abortSignal?.aborted) return; - - if (streamFinished) { - finalizeOpenToolCallBatch(); - - if (conversationId) { - ChatService.clearStreamState(conversationId); - } - - const finalToolCalls = - aggregatedToolCalls.length > 0 ? JSON.stringify(aggregatedToolCalls) : undefined; - - onComplete?.( - aggregatedContent, - fullReasoningContent || undefined, - lastTimings, - finalToolCalls - ); - } - } catch (error) { - const err = error instanceof Error ? error : new Error('Stream error'); - - onError?.(err); - - throw err; - } finally { - if (typeof document !== 'undefined') { - document.removeEventListener('visibilitychange', onVisibilityChange); - } - - try { - reader.releaseLock(); - } catch { - /* ignore */ - } + if (!firstChoice) { + return undefined; } + + // priority: delta.model (first chunk) else message.model (final response) + const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model); + + if (deltaModel) { + return deltaModel; + } + + const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model); + + if (messageModel) { + return messageModel; + } + + // avoid guessing from non-standard locations (metadata, etc.) + return undefined; } /** @@ -1271,253 +1563,23 @@ export class ChatService { } /** + * Calls the onTimings callback with timing data from streaming response. * - * - * Conversion - * - * + * @param timings - Timing information from the Chat Completions API response + * @param promptProgress - Prompt processing progress data + * @param onTimingsCallback - Callback function to invoke with timing data + * @private */ + private static notifyTimings( + timings: ChatMessageTimings | undefined, + promptProgress: ChatMessagePromptProgress | undefined, + onTimingsCallback: + | ((timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void) + | undefined + ): void { + if (!onTimingsCallback || (!timings && !promptProgress)) return; - /** - * Converts a database message with attachments to API chat message format. - * Processes various attachment types (images, text files, PDFs) and formats them - * as content parts suitable for the chat completion API. - * - * @param message - Database message object with optional extra attachments - * @param message.content - The text content of the message - * @param message.role - The role of the message sender (user, assistant, system) - * @param message.extra - Optional array of message attachments (images, files, etc.) - * @returns {ApiChatMessageData} object formatted for the chat completion API - * @static - */ - static async convertDbMessageToApiChatMessageData( - message: DatabaseMessage & { extra?: DatabaseMessageExtra[] } - ): Promise { - // Handle tool result messages (role: 'tool') - if (message.role === MessageRole.TOOL && message.toolCallId) { - return { - content: message.content, - role: MessageRole.TOOL, - tool_call_id: message.toolCallId - }; - } - - // Parse tool calls for assistant messages - let toolCalls: ApiChatCompletionToolCall[] | undefined; - - if (message.toolCalls) { - try { - toolCalls = JSON.parse(message.toolCalls); - } catch { - // Ignore parse errors for malformed tool calls - } - } - - if (!message.extra || message.extra.length === 0) { - const result: ApiChatMessageData = { - content: message.content, - role: message.role as MessageRole - }; - - if (message.reasoningContent) { - result.reasoning_content = message.reasoningContent; - } - - if (toolCalls && toolCalls.length > 0) { - result.tool_calls = toolCalls; - } - - return result; - } - - const contentParts: ApiChatMessageContentPart[] = []; - const textFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraTextFile => - extra.type === AttachmentType.TEXT - ); - - for (const textFile of textFiles) { - contentParts.push({ - text: formatAttachmentText(AttachmentLabel.FILE, textFile.name, textFile.content), - type: ContentPartType.TEXT - }); - } - - // Handle legacy 'context' type from the old UI (pasted content) - const legacyContextFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraLegacyContext => - extra.type === AttachmentType.LEGACY_CONTEXT - ); - - for (const legacyContextFile of legacyContextFiles) { - contentParts.push({ - text: formatAttachmentText( - AttachmentLabel.FILE, - legacyContextFile.name, - legacyContextFile.content - ), - type: ContentPartType.TEXT - }); - } - - const imageFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraImageFile => - extra.type === AttachmentType.IMAGE - ); - - for (const image of imageFiles) { - const maxImageResolution = settingsStore.getConfig(SETTINGS_KEYS.MAX_IMAGE_RESOLUTION); - // Caps the resolution and bakes the jpeg exif orientation in one pass, - // untouched images pass through as is - const base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution); - - contentParts.push({ - image_url: { url: base64Url }, - type: ContentPartType.IMAGE_URL - }); - } - - const audioFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraAudioFile => - extra.type === AttachmentType.AUDIO - ); - - for (const audio of audioFiles) { - contentParts.push({ - input_audio: { - data: audio.base64Data, - format: getAudioInputFormat(audio.mimeType) - }, - type: ContentPartType.INPUT_AUDIO - }); - } - - if (message.content) { - contentParts.push({ - text: message.content, - type: ContentPartType.TEXT - }); - } - - const videoFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraVideoFile => - extra.type === AttachmentType.VIDEO - ); - - for (const video of videoFiles) { - contentParts.push({ - input_video: { - data: video.base64Data, - format: video.mimeType.includes('mp4') - ? 'mp4' - : video.mimeType.includes('ogg') - ? 'ogg' - : 'auto' - }, - type: ContentPartType.INPUT_VIDEO - }); - } - - const pdfFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraPdfFile => - extra.type === AttachmentType.PDF - ); - - for (const pdfFile of pdfFiles) { - if (pdfFile.processedAsImages && pdfFile.images) { - for (let i = 0; i < pdfFile.images.length; i++) { - contentParts.push({ - image_url: { url: pdfFile.images[i] }, - type: ContentPartType.IMAGE_URL - }); - } - } else { - contentParts.push({ - text: formatAttachmentText(AttachmentLabel.PDF_FILE, pdfFile.name, pdfFile.content), - type: ContentPartType.TEXT - }); - } - } - - const mcpPrompts = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraMcpPrompt => - extra.type === AttachmentType.MCP_PROMPT - ); - - for (const mcpPrompt of mcpPrompts) { - contentParts.push({ - text: formatAttachmentText( - AttachmentLabel.MCP_PROMPT, - mcpPrompt.name, - mcpPrompt.content, - mcpPrompt.serverName - ), - type: ContentPartType.TEXT - }); - } - - const mcpResources = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraMcpResource => - extra.type === AttachmentType.MCP_RESOURCE - ); - - for (const mcpResource of mcpResources) { - contentParts.push({ - text: formatAttachmentText( - AttachmentLabel.MCP_RESOURCE, - mcpResource.name, - mcpResource.content, - mcpResource.serverName - ), - type: ContentPartType.TEXT - }); - } - - const result: ApiChatMessageData = { - content: contentParts, - role: message.role as MessageRole - }; - - if (message.reasoningContent) { - result.reasoning_content = message.reasoningContent; - } - - if (toolCalls && toolCalls.length > 0) { - result.tool_calls = toolCalls; - } - - return result; - } - - /** - * - * - * Utilities - * - * - */ - - /** - * Strips legacy inline reasoning content tags from message content. - * Handles both plain string content and multipart content arrays. - */ - private static stripReasoningContent( - content: string | ApiChatMessageContentPart[] - ): string | ApiChatMessageContentPart[] { - const stripFromString = (text: string): string => - text.replace(LEGACY_AGENTIC_REGEX.REASONING_BLOCK, '').trim(); - - if (typeof content === 'string') { - return stripFromString(content); - } - - return content.map((part) => { - if (part.type === ContentPartType.TEXT && part.text) { - return { ...part, text: stripFromString(part.text) }; - } - - return part; - }); + onTimingsCallback(timings, promptProgress); } /** @@ -1560,77 +1622,44 @@ export class ChatService { } /** - * Extracts model name from Chat Completions API response data. - * Handles various response formats including streaming chunks and final responses. - * - * WORKAROUND: In single model mode, llama-server returns a default/incorrect model name - * in the response. We override it with the actual model name from serverStore. - * - * @param data - Raw response data from the Chat Completions API - * @returns Model name string if found, undefined otherwise - * @private + * Strips legacy inline reasoning content tags from message content. + * Handles both plain string content and multipart content arrays. */ - private static extractModelName(data: unknown): string | undefined { - const asRecord = (value: unknown): Record | undefined => { - return typeof value === 'object' && value !== null - ? (value as Record) - : undefined; - }; - const getTrimmedString = (value: unknown): string | undefined => { - return typeof value === 'string' && value.trim() ? value.trim() : undefined; - }; - const root = asRecord(data); + private static stripReasoningContent( + content: string | ApiChatMessageContentPart[] + ): string | ApiChatMessageContentPart[] { + const stripFromString = (text: string): string => + text.replace(LEGACY_AGENTIC_REGEX.REASONING_BLOCK, '').trim(); - if (!root) return undefined; - - // 1) root (some implementations provide `model` at the top level) - const rootModel = getTrimmedString(root.model); - - if (rootModel) { - return rootModel; + if (typeof content === 'string') { + return stripFromString(content); } - // 2) streaming choice (delta) or final response (message) - const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined; + return content.map((part) => { + if (part.type === ContentPartType.TEXT && part.text) { + return { ...part, text: stripFromString(part.text) }; + } - if (!firstChoice) { - return undefined; - } - - // priority: delta.model (first chunk) else message.model (final response) - const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model); - - if (deltaModel) { - return deltaModel; - } - - const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model); - - if (messageModel) { - return messageModel; - } - - // avoid guessing from non-standard locations (metadata, etc.) - return undefined; + return part; + }); } - /** - * Calls the onTimings callback with timing data from streaming response. - * - * @param timings - Timing information from the Chat Completions API response - * @param promptProgress - Prompt processing progress data - * @param onTimingsCallback - Callback function to invoke with timing data - * @private - */ - private static notifyTimings( - timings: ChatMessageTimings | undefined, - promptProgress: ChatMessagePromptProgress | undefined, - onTimingsCallback: - | ((timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void) - | undefined + // write the resume state straight to localStorage, bypassing the throttle + private static writeStreamState( + conversationId: string, + bytesReceived: number, + model?: string | null ): void { - if (!onTimingsCallback || (!timings && !promptProgress)) return; + try { + const state: ResumableStreamState = { + bytesReceived, + model: model ?? null, + updatedAt: Date.now() + }; - onTimingsCallback(timings, promptProgress); + localStorage.setItem(streamStorageKey(conversationId), JSON.stringify(state)); + } catch { + // localStorage may be full or disabled, silently ignore + } } } diff --git a/tools/ui/src/lib/services/conversation-transfer.service.ts b/tools/ui/src/lib/services/conversation-transfer.service.ts index acef58005..40a09477a 100644 --- a/tools/ui/src/lib/services/conversation-transfer.service.ts +++ b/tools/ui/src/lib/services/conversation-transfer.service.ts @@ -16,187 +16,6 @@ import { import { strFromU8, strToU8, unzipSync, zipSync } from 'fflate'; export class ConversationTransferService { - /** - * - * - * JSONL Session Format - * - * - */ - - /** - * Serializes a session (a conversation with its messages) as JSONL. - * The first line is the session header (a `SessionRecordType.SESSION` record - * carrying the conversation properties); each subsequent line is a single message. - * @param data - The exported conversation payload - * @returns The JSONL string (one record per line) - */ - static serializeSessionToJsonl(data: ExportedConversation): string { - const { conv, messages } = data; - const sessionLine = JSON.stringify({ - harness: EXPORT_CONV.HARNESS, - type: SessionRecordType.SESSION, - ...conv - }); - const messageLines = messages.map((message: DatabaseMessage) => { - // `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it. - const { toolCalls, ...rest } = message; - const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest; - - return JSON.stringify({ message: normalized, type: SessionRecordType.MESSAGE }); - }); - - return [sessionLine, ...messageLines].join(NEWLINE); - } - - /** - * Parses the JSONL session format produced by {@link serializeSessionToJsonl}. - * A `SessionRecordType.SESSION` line starts a new session; following - * `SessionRecordType.MESSAGE` lines are appended to it. Supports multiple - * sessions in a single file. - * @param text - The JSONL file contents - * @returns The parsed conversations with their messages - */ - static parseSessionsJsonl(text: string): ExportedConversation[] { - const sessions: ExportedConversation[] = []; - - let current: ExportedConversation | null = null; - - for (const line of text.split(NEWLINE)) { - const trimmed = line.trim(); - - if (!trimmed) continue; - - const record = JSON.parse(trimmed); - - if (record.type === SessionRecordType.SESSION) { - // Drop the discriminator and harness marker; the rest is the conversation. - const conv = { ...record }; - - delete conv.type; - delete conv.harness; - current = { conv: conv as DatabaseConversation, messages: [] }; - sessions.push(current); - } else if (record.type === SessionRecordType.MESSAGE) { - if (!current) { - throw new Error('Invalid JSONL: message record before any session record'); - } - - const message = record.message as DatabaseMessage; - - // `toolCalls` is parsed to an array on export; the DB stores it as a string. - if (message.toolCalls !== undefined && typeof message.toolCalls !== 'string') { - message.toolCalls = JSON.stringify(message.toolCalls); - } - - current.messages.push(message); - } - // Ignore unknown record types for forward compatibility. - } - - return sessions; - } - - /** - * Reports whether the text is the JSONL session format, whose first non-empty - * line is a `SessionRecordType.SESSION` record. A legacy JSON export starts - * with an array or an object that has no such discriminator. - * @param text - The file contents - */ - private static isSessionsJsonl(text: string): boolean { - const trimmed = text.trimStart(); - const lineEnd = trimmed.indexOf(NEWLINE); - const firstLine = lineEnd === -1 ? trimmed : trimmed.slice(0, lineEnd); - - try { - return JSON.parse(firstLine).type === SessionRecordType.SESSION; - } catch { - // Not a standalone JSON record, so not the JSONL format. - return false; - } - } - - /** - * Parses an import file into conversations, accepting the current JSONL and - * ZIP formats as well as the legacy JSON format. The format comes from the - * contents, so an import works whatever the file is named. - * @param file - The user-selected file - * @returns The parsed conversations with their messages - */ - static async parseImportFile(file: File): Promise { - const bytes = new Uint8Array(await file.arrayBuffer()); - - if (ZIP_MAGIC.every((byte, index) => bytes[index] === byte)) { - const entries = unzipSync(bytes); - const sessions: ExportedConversation[] = []; - - for (const [entryName, entryBytes] of Object.entries(entries)) { - if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue; - - sessions.push(...ConversationTransferService.parseSessionsJsonl(strFromU8(entryBytes))); - } - - return sessions; - } - - const text = strFromU8(bytes); - - if (ConversationTransferService.isSessionsJsonl(text)) { - return ConversationTransferService.parseSessionsJsonl(text); - } - - // Legacy JSON format: an array of conversations or a single conversation object. - const parsed = JSON.parse(text); - - if (Array.isArray(parsed)) { - return parsed; - } - - if (parsed && typeof parsed === 'object' && 'conv' in parsed && 'messages' in parsed) { - return [parsed]; - } - - throw new Error( - 'Invalid file format: expected array of conversations or single conversation object' - ); - } - - /** - * - * - * Downloads - * - * - */ - - /** - * Generates a sanitized filename for a conversation export - * @param conversation - The conversation metadata - * @param msgs - Optional array of messages belonging to the conversation - * @returns The generated filename string - */ - static generateConversationFilename( - conversation: { id?: string; name?: string }, - msgs?: DatabaseMessage[] - ): string { - const conversationName = (conversation.name ?? '').trim().toLowerCase(); - const sanitizedName = conversationName - .replace(EXPORT_CONV.NON_ALPHANUMERIC_REGEX, EXPORT_CONV.NONALNUM_REPLACEMENT) - .replace(EXPORT_CONV.MULTIPLE_UNDERSCORE_REGEX, '_') - .substring(0, EXPORT_CONV.NAME_SUFFIX_MAX_LENGTH); - // If we have messages, use the timestamp of the newest message - const referenceDate = msgs?.length - ? new Date(Math.max(...msgs.map((m) => m.timestamp))) - : new Date(); - const iso = referenceDate.toISOString().slice(0, EXPORT_CONV.ISO_TIMESTAMP_SLICE); - const formattedDate = iso - .replace(EXPORT_CONV.ISO_DATE_TIME_SEPARATOR, EXPORT_CONV.ISO_DATE_TIME_SEPARATOR_REPLACEMENT) - .replaceAll(EXPORT_CONV.ISO_TIME_SEPARATOR, EXPORT_CONV.ISO_TIME_SEPARATOR_REPLACEMENT); - const trimmedConvId = conversation.id?.slice(0, EXPORT_CONV.ID_TRIM_LENGTH) ?? ''; - - return `${formattedDate}_conv_${trimmedConvId}_${sanitizedName}${FileExtensionText.JSONL}`; - } - /** * Triggers a browser download of the provided exported conversation data * @param data - The exported conversation payload (a single conversation with its messages) @@ -262,6 +81,171 @@ export class ConversationTransferService { ConversationTransferService.triggerDownload(blob, archiveName); } + /** + * Generates a sanitized filename for a conversation export + * @param conversation - The conversation metadata + * @param msgs - Optional array of messages belonging to the conversation + * @returns The generated filename string + */ + static generateConversationFilename( + conversation: { id?: string; name?: string }, + msgs?: DatabaseMessage[] + ): string { + const conversationName = (conversation.name ?? '').trim().toLowerCase(); + const sanitizedName = conversationName + .replace(EXPORT_CONV.NON_ALPHANUMERIC_REGEX, EXPORT_CONV.NONALNUM_REPLACEMENT) + .replace(EXPORT_CONV.MULTIPLE_UNDERSCORE_REGEX, '_') + .substring(0, EXPORT_CONV.NAME_SUFFIX_MAX_LENGTH); + // If we have messages, use the timestamp of the newest message + const referenceDate = msgs?.length + ? new Date(Math.max(...msgs.map((m) => m.timestamp))) + : new Date(); + const iso = referenceDate.toISOString().slice(0, EXPORT_CONV.ISO_TIMESTAMP_SLICE); + const formattedDate = iso + .replace(EXPORT_CONV.ISO_DATE_TIME_SEPARATOR, EXPORT_CONV.ISO_DATE_TIME_SEPARATOR_REPLACEMENT) + .replaceAll(EXPORT_CONV.ISO_TIME_SEPARATOR, EXPORT_CONV.ISO_TIME_SEPARATOR_REPLACEMENT); + const trimmedConvId = conversation.id?.slice(0, EXPORT_CONV.ID_TRIM_LENGTH) ?? ''; + + return `${formattedDate}_conv_${trimmedConvId}_${sanitizedName}${FileExtensionText.JSONL}`; + } + + /** + * Parses an import file into conversations, accepting the current JSONL and + * ZIP formats as well as the legacy JSON format. The format comes from the + * contents, so an import works whatever the file is named. + * @param file - The user-selected file + * @returns The parsed conversations with their messages + */ + static async parseImportFile(file: File): Promise { + const bytes = new Uint8Array(await file.arrayBuffer()); + + if (ZIP_MAGIC.every((byte, index) => bytes[index] === byte)) { + const entries = unzipSync(bytes); + const sessions: ExportedConversation[] = []; + + for (const [entryName, entryBytes] of Object.entries(entries)) { + if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue; + + sessions.push(...ConversationTransferService.parseSessionsJsonl(strFromU8(entryBytes))); + } + + return sessions; + } + + const text = strFromU8(bytes); + + if (ConversationTransferService.isSessionsJsonl(text)) { + return ConversationTransferService.parseSessionsJsonl(text); + } + + // Legacy JSON format: an array of conversations or a single conversation object. + const parsed = JSON.parse(text); + + if (Array.isArray(parsed)) { + return parsed; + } + + if (parsed && typeof parsed === 'object' && 'conv' in parsed && 'messages' in parsed) { + return [parsed]; + } + + throw new Error( + 'Invalid file format: expected array of conversations or single conversation object' + ); + } + + /** + * Parses the JSONL session format produced by {@link serializeSessionToJsonl}. + * A `SessionRecordType.SESSION` line starts a new session; following + * `SessionRecordType.MESSAGE` lines are appended to it. Supports multiple + * sessions in a single file. + * @param text - The JSONL file contents + * @returns The parsed conversations with their messages + */ + static parseSessionsJsonl(text: string): ExportedConversation[] { + const sessions: ExportedConversation[] = []; + + let current: ExportedConversation | null = null; + + for (const line of text.split(NEWLINE)) { + const trimmed = line.trim(); + + if (!trimmed) continue; + + const record = JSON.parse(trimmed); + + if (record.type === SessionRecordType.SESSION) { + // Drop the discriminator and harness marker; the rest is the conversation. + const conv = { ...record }; + + delete conv.type; + delete conv.harness; + current = { conv: conv as DatabaseConversation, messages: [] }; + sessions.push(current); + } else if (record.type === SessionRecordType.MESSAGE) { + if (!current) { + throw new Error('Invalid JSONL: message record before any session record'); + } + + const message = record.message as DatabaseMessage; + + // `toolCalls` is parsed to an array on export; the DB stores it as a string. + if (message.toolCalls !== undefined && typeof message.toolCalls !== 'string') { + message.toolCalls = JSON.stringify(message.toolCalls); + } + + current.messages.push(message); + } + // Ignore unknown record types for forward compatibility. + } + + return sessions; + } + + /** + * Serializes a session (a conversation with its messages) as JSONL. + * The first line is the session header (a `SessionRecordType.SESSION` record + * carrying the conversation properties); each subsequent line is a single message. + * @param data - The exported conversation payload + * @returns The JSONL string (one record per line) + */ + static serializeSessionToJsonl(data: ExportedConversation): string { + const { conv, messages } = data; + const sessionLine = JSON.stringify({ + harness: EXPORT_CONV.HARNESS, + type: SessionRecordType.SESSION, + ...conv + }); + const messageLines = messages.map((message: DatabaseMessage) => { + // `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it. + const { toolCalls, ...rest } = message; + const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest; + + return JSON.stringify({ message: normalized, type: SessionRecordType.MESSAGE }); + }); + + return [sessionLine, ...messageLines].join(NEWLINE); + } + + /** + * Reports whether the text is the JSONL session format, whose first non-empty + * line is a `SessionRecordType.SESSION` record. A legacy JSON export starts + * with an array or an object that has no such discriminator. + * @param text - The file contents + */ + private static isSessionsJsonl(text: string): boolean { + const trimmed = text.trimStart(); + const lineEnd = trimmed.indexOf(NEWLINE); + const firstLine = lineEnd === -1 ? trimmed : trimmed.slice(0, lineEnd); + + try { + return JSON.parse(firstLine).type === SessionRecordType.SESSION; + } catch { + // Not a standalone JSON record, so not the JSONL format. + return false; + } + } + /** * Triggers a browser download of a blob under the given filename. */ diff --git a/tools/ui/src/lib/services/database.service.ts b/tools/ui/src/lib/services/database.service.ts index 89dc58b00..a466f8483 100644 --- a/tools/ui/src/lib/services/database.service.ts +++ b/tools/ui/src/lib/services/database.service.ts @@ -1,3 +1,11 @@ +/** + * DatabaseService - IndexedDB persistence for conversations and messages + * + * Thin Dexie layer over the conversations/messages tables: CRUD, tree + * navigation (descendants, reparenting) and cascading deletes. No reactive + * state; consumed by conversationsStore and the chat flows. + */ + import { IDXDB_STORES, IDXDB_TABLES, STORAGE_APP_NAME } from '$lib/constants'; import { MessageRole } from '$lib/enums'; import type { McpServerOverride } from '$lib/types/database'; @@ -20,12 +28,99 @@ const db = new LlamaUiDatabase(); export class DatabaseService { /** + * Deletes multiple conversations in a single transaction. Each deleted + * conversation has its direct children reparented to the nearest surviving + * ancestor (or promoted to top-level). Children also in `ids` are dropped + * entirely rather than reparented. * - * - * Conversations - * - * + * @param ids - Conversation IDs to delete */ + static async bulkDeleteConversations(ids: string[]): Promise { + const cleanIds = ids.filter((id): id is string => typeof id === 'string' && id.length > 0); + + if (cleanIds.length === 0) return; + + const idSet = new Set(cleanIds); + + await db.transaction( + 'rw', + [db[IDXDB_TABLES.conversations], db[IDXDB_TABLES.messages]], + async () => { + // Pre-load each to-delete conversation so the per-id reparent + // walk-up doesn't ping-pong the same ancestry chain. + const prefetched = new Map(); + + let frontier = [...cleanIds]; + + const requested = new Set(frontier); + + while (frontier.length > 0) { + const fetched = await db[IDXDB_TABLES.conversations].bulkGet(frontier); + + frontier = []; + for (let i = 0; i < fetched.length; i++) { + const conv = fetched[i]; + + if (!conv || !conv.id) continue; + + prefetched.set(conv.id, conv); + const ancestor = conv.forkedFromConversationId; + + if (ancestor && !prefetched.has(ancestor) && !requested.has(ancestor)) { + frontier.push(ancestor); + requested.add(ancestor); + } + } + } + + for (const id of cleanIds) { + await this.reparentDirectChildren(id, idSet, prefetched); + } + + await db[IDXDB_TABLES.conversations].bulkDelete(cleanIds); + await db[IDXDB_TABLES.messages].where('convId').anyOf(cleanIds).delete(); + } + ); + } + + /** + * Toggles the pinned status of each conversation in `ids` inside a single + * transaction. Treats `pinned === undefined` as `false`, matching the + * semantics of {@link toggleConversationPin} where `!undefined` evaluates + * to `true`. Returns the resulting pinned state for every id that was + * updated; missing ids are omitted from the map. + * + * @param ids - Conversation IDs to toggle + * @returns Map of id -> new pinned state + */ + static async bulkToggleConversationPins(ids: string[]): Promise> { + const cleanIds = ids.filter((id): id is string => typeof id === 'string' && id.length > 0); + const result = new Map(); + + if (cleanIds.length === 0) return result; + + await db.transaction('rw', db[IDXDB_TABLES.conversations], async () => { + const convs = await db[IDXDB_TABLES.conversations].bulkGet(cleanIds); + const updates: DatabaseConversation[] = []; + + for (let i = 0; i < cleanIds.length; i++) { + const conv = convs[i]; + + if (!conv) continue; + + const newPinned = !conv.pinned; + + updates.push({ ...conv, pinned: newPinned }); + result.set(cleanIds[i], newPinned); + } + + if (updates.length === 0) return; + + await db[IDXDB_TABLES.conversations].bulkPut(updates); + }); + + return result; + } /** * Creates a new conversation. @@ -51,14 +146,6 @@ export class DatabaseService { return conversation; } - /** - * - * - * Messages - * - * - */ - /** * Creates a new message branch by adding a message and updating parent/child relationships. * Also updates the conversation's currNode to point to the new message. @@ -96,13 +183,7 @@ export class DatabaseService { // Update parent's children array if parent exists if (parentId !== null) { - const parentMessage = await db[IDXDB_TABLES.messages].get(parentId); - - if (parentMessage) { - await db[IDXDB_TABLES.messages].update(parentId, { - children: [...parentMessage.children, newMessage.id] - }); - } + await this.addChildToParent(parentId, newMessage.id); } await this.updateConversation(message.convId, { @@ -178,9 +259,7 @@ export class DatabaseService { }; await db[IDXDB_TABLES.messages].add(systemMessage); - await db[IDXDB_TABLES.messages].update(parentId, { - children: [...parentMessage.children, systemMessage.id] - }); + await this.addChildToParent(parentId, systemMessage.id); return systemMessage; }); @@ -230,121 +309,6 @@ export class DatabaseService { ); } - /** - * Reparents direct children of `parentId` to the nearest surviving - * ancestor (or promotes them to top-level when the immediate parent was - * top-level). Walking skips any ancestor listed in `excludeIds`, since - * those will be deleted in the same batch — leaving a grandchild pointing - * at an `excludeIds` entry would orphan it. Children whose own id is in - * `excludeIds` are dropped from the updates (the bulk-delete pass will - * remove them). `prefetched` may carry a pre-fetched ancestor map to - * avoid repeat reads inside a bulk transaction. - */ - private static async reparentDirectChildren( - parentId: string, - excludeIds: ReadonlySet = new Set(), - prefetched?: ReadonlyMap - ): Promise { - const conv = prefetched?.get(parentId) ?? (await db[IDXDB_TABLES.conversations].get(parentId)); - - if (!conv) return; - - let newParent = conv.forkedFromConversationId; - - const visited = new Set([parentId]); - - while (newParent && excludeIds.has(newParent)) { - if (visited.has(newParent)) { - newParent = undefined; - - break; - } - - visited.add(newParent); - const next = - prefetched?.get(newParent) ?? (await db[IDXDB_TABLES.conversations].get(newParent)); - - if (!next) { - newParent = undefined; - - break; - } - - newParent = next.forkedFromConversationId; - } - - const directChildren = await db[IDXDB_TABLES.conversations] - .filter((c) => c.forkedFromConversationId === parentId) - .toArray(); - const updates: DatabaseConversation[] = []; - - for (const child of directChildren) { - if (excludeIds.has(child.id)) continue; - - updates.push({ ...child, forkedFromConversationId: newParent }); - } - - if (updates.length === 0) return; - - await db[IDXDB_TABLES.conversations].bulkPut(updates); - } - - /** - * Deletes multiple conversations in a single transaction. Each deleted - * conversation has its direct children reparented to the nearest surviving - * ancestor (or promoted to top-level). Children also in `ids` are dropped - * entirely rather than reparented. - * - * @param ids - Conversation IDs to delete - */ - static async bulkDeleteConversations(ids: string[]): Promise { - const cleanIds = ids.filter((id): id is string => typeof id === 'string' && id.length > 0); - - if (cleanIds.length === 0) return; - - const idSet = new Set(cleanIds); - - await db.transaction( - 'rw', - [db[IDXDB_TABLES.conversations], db[IDXDB_TABLES.messages]], - async () => { - // Pre-load each to-delete conversation so the per-id reparent - // walk-up doesn't ping-pong the same ancestry chain. - const prefetched = new Map(); - - let frontier = [...cleanIds]; - - const requested = new Set(frontier); - - while (frontier.length > 0) { - const fetched = await db[IDXDB_TABLES.conversations].bulkGet(frontier); - - frontier = []; - for (let i = 0; i < fetched.length; i++) { - const conv = fetched[i]; - - if (!conv || !conv.id) continue; - - prefetched.set(conv.id, conv); - const ancestor = conv.forkedFromConversationId; - - if (ancestor && !prefetched.has(ancestor) && !requested.has(ancestor)) { - frontier.push(ancestor); - requested.add(ancestor); - } - } - } - - for (const id of cleanIds) { - await this.reparentDirectChildren(id, idSet, prefetched); - } - - await db[IDXDB_TABLES.conversations].bulkDelete(cleanIds); - await db[IDXDB_TABLES.messages].where('convId').anyOf(cleanIds).delete(); - } - ); - } - /** * Deletes a message and removes it from its parent's children array. * @@ -356,17 +320,8 @@ export class DatabaseService { if (!message) return; - // Remove this message from its parent's children array - if (message.parent) { - const parent = await db[IDXDB_TABLES.messages].get(message.parent); + await this.removeChildFromParent(messageId); - if (parent) { - parent.children = parent.children.filter((childId: string) => childId !== messageId); - await db[IDXDB_TABLES.messages].put(parent); - } - } - - // Delete the message await db[IDXDB_TABLES.messages].delete(messageId); }); } @@ -389,20 +344,10 @@ export class DatabaseService { .where('convId') .equals(conversationId) .toArray(); - // Find all descendant messages const descendants = findDescendantMessages(allMessages, messageId); const allToDelete = [messageId, ...descendants]; - // Get the message to delete for parent cleanup - const message = await db[IDXDB_TABLES.messages].get(messageId); - if (message && message.parent) { - const parent = await db[IDXDB_TABLES.messages].get(message.parent); - - if (parent) { - parent.children = parent.children.filter((childId: string) => childId !== messageId); - await db[IDXDB_TABLES.messages].put(parent); - } - } + await this.removeChildFromParent(messageId); // Delete all messages in the branch await db[IDXDB_TABLES.messages].bulkDelete(allToDelete); @@ -411,243 +356,6 @@ export class DatabaseService { }); } - /** - * Gets all conversations, sorted by last modified time (newest first). - * - * @returns Array of conversations - */ - static async getAllConversations(): Promise { - return await db[IDXDB_TABLES.conversations].orderBy('lastModified').reverse().toArray(); - } - - /** - * Gets a conversation by ID. - * - * @param id - Conversation ID - * @returns The conversation if found, otherwise undefined - */ - static async getConversation(id: string): Promise { - return await db[IDXDB_TABLES.conversations].get(id); - } - - /** - * Gets all messages in a conversation, sorted by timestamp (oldest first). - * - * @param convId - Conversation ID - * @returns Array of messages in the conversation - */ - static async getConversationMessages(convId: string): Promise { - return await db[IDXDB_TABLES.messages].where('convId').equals(convId).sortBy('timestamp'); - } - - /** - * Loads multiple conversations with all of their messages in two bulk - * reads. Missing conversations are silently omitted from the result. - * - * @param convIds - Conversation IDs to load - * @returns Map of id -> { conv, messages }. Messages are sorted ascending by timestamp. - */ - static async getConversationsWithMessages( - convIds: string[] - ): Promise> { - const result = new Map(); - const cleanIds = convIds.filter((id): id is string => typeof id === 'string' && id.length > 0); - - if (cleanIds.length === 0) return result; - - const [convs, allMessages] = await Promise.all([ - db[IDXDB_TABLES.conversations].bulkGet(cleanIds), - db[IDXDB_TABLES.messages].where('convId').anyOf(cleanIds).toArray() - ]); - const messagesByConv = new Map(); - - for (const msg of allMessages) { - const bucket = messagesByConv.get(msg.convId); - - if (bucket) bucket.push(msg); - else messagesByConv.set(msg.convId, [msg]); - } - - for (let i = 0; i < cleanIds.length; i++) { - const conv = convs[i]; - - if (!conv) continue; - - const messages = (messagesByConv.get(conv.id) ?? []).sort( - (a, b) => a.timestamp - b.timestamp - ); - - result.set(conv.id, { conv, messages }); - } - - return result; - } - - /** - * Updates a conversation. `lastModified` is never stamped implicitly; - * pass it in `updates` to bump the conversation in recency ordering. - * - * @param id - Conversation ID - * @param updates - Partial updates to apply - * @returns Promise that resolves when the conversation is updated - */ - static async updateConversation( - id: string, - updates: Partial> - ): Promise { - await db[IDXDB_TABLES.conversations].update(id, updates); - } - - /** - * - * - * Navigation - * - * - */ - - /** - * Toggles the pinned status of a conversation. - * - * @param id - Conversation ID - * @returns The new pinned status - */ - static async toggleConversationPin(id: string): Promise { - const conversation = await db[IDXDB_TABLES.conversations].get(id); - - if (!conversation) { - throw new Error(`Conversation ${id} not found`); - } - - const newPinnedState = !conversation.pinned; - - await this.updateConversation(id, { pinned: newPinnedState }); - - return newPinnedState; - } - - /** - * Toggles the pinned status of each conversation in `ids` inside a single - * transaction. Treats `pinned === undefined` as `false`, matching the - * semantics of {@link toggleConversationPin} where `!undefined` evaluates - * to `true`. Returns the resulting pinned state for every id that was - * updated; missing ids are omitted from the map. - * - * @param ids - Conversation IDs to toggle - * @returns Map of id -> new pinned state - */ - static async bulkToggleConversationPins(ids: string[]): Promise> { - const cleanIds = ids.filter((id): id is string => typeof id === 'string' && id.length > 0); - const result = new Map(); - - if (cleanIds.length === 0) return result; - - await db.transaction('rw', db[IDXDB_TABLES.conversations], async () => { - const convs = await db[IDXDB_TABLES.conversations].bulkGet(cleanIds); - const updates: DatabaseConversation[] = []; - - for (let i = 0; i < cleanIds.length; i++) { - const conv = convs[i]; - - if (!conv) continue; - - const newPinned = !conv.pinned; - - updates.push({ ...conv, pinned: newPinned }); - result.set(cleanIds[i], newPinned); - } - - if (updates.length === 0) return; - - await db[IDXDB_TABLES.conversations].bulkPut(updates); - }); - - return result; - } - - /** - * Updates the conversation's current node (active branch). - * This determines which conversation path is currently being viewed. - * - * @param convId - Conversation ID - * @param nodeId - Message ID to set as current node - */ - static async updateCurrentNode(convId: string, nodeId: string): Promise { - await this.updateConversation(convId, { - currNode: nodeId - }); - } - - /** - * Updates a message. - * - * @param id - Message ID - * @param updates - Partial updates to apply - * @returns Promise that resolves when the message is updated - */ - static async updateMessage( - id: string, - updates: Partial> - ): Promise { - await db[IDXDB_TABLES.messages].update(id, updates); - } - - /** - * - * - * Import - * - * - */ - - /** - * Imports multiple conversations and their messages. - * Skips conversations that already exist. - * - * @param data - Array of { conv, messages } objects - * @returns The conversations written to the database and the ones skipped - */ - static async importConversations( - data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[] - ): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> { - const imported: DatabaseConversation[] = []; - const skipped: DatabaseConversation[] = []; - - return await db.transaction( - 'rw', - [db[IDXDB_TABLES.conversations], db[IDXDB_TABLES.messages]], - async () => { - for (const item of data) { - const { conv, messages } = item; - const existing = await db[IDXDB_TABLES.conversations].get(conv.id); - - if (existing) { - skipped.push(conv); - - continue; - } - - await db[IDXDB_TABLES.conversations].add(conv); - for (const msg of messages) { - await db[IDXDB_TABLES.messages].put(msg); - } - - imported.push(conv); - } - - return { imported, skipped }; - } - ); - } - - /** - * - * - * Forking - * - * - */ - /** * Forks a conversation at a specific message, creating a new conversation * containing all messages from the root up to (and including) the target message. @@ -726,13 +434,272 @@ export class DatabaseService { }; await db[IDXDB_TABLES.conversations].add(newConv); - - for (const msg of clonedMessages) { - await db[IDXDB_TABLES.messages].add(msg); - } + await db[IDXDB_TABLES.messages].bulkAdd(clonedMessages); return newConv; } ); } + + /** + * Gets all conversations, sorted by last modified time (newest first). + * + * @returns Array of conversations + */ + static async getAllConversations(): Promise { + return await db[IDXDB_TABLES.conversations].orderBy('lastModified').reverse().toArray(); + } + + /** + * Gets a conversation by ID. + * + * @param id - Conversation ID + * @returns The conversation if found, otherwise undefined + */ + static async getConversation(id: string): Promise { + return await db[IDXDB_TABLES.conversations].get(id); + } + + /** + * Gets all messages in a conversation, sorted by timestamp (oldest first). + * + * @param convId - Conversation ID + * @returns Array of messages in the conversation + */ + static async getConversationMessages(convId: string): Promise { + return await db[IDXDB_TABLES.messages].where('convId').equals(convId).sortBy('timestamp'); + } + + /** + * Loads multiple conversations with all of their messages in two bulk + * reads. Missing conversations are silently omitted from the result. + * + * @param convIds - Conversation IDs to load + * @returns Map of id -> { conv, messages }. Messages are sorted ascending by timestamp. + */ + static async getConversationsWithMessages( + convIds: string[] + ): Promise> { + const result = new Map(); + const cleanIds = convIds.filter((id): id is string => typeof id === 'string' && id.length > 0); + + if (cleanIds.length === 0) return result; + + const [convs, allMessages] = await Promise.all([ + db[IDXDB_TABLES.conversations].bulkGet(cleanIds), + db[IDXDB_TABLES.messages].where('convId').anyOf(cleanIds).toArray() + ]); + const messagesByConv = new Map(); + + for (const msg of allMessages) { + const bucket = messagesByConv.get(msg.convId); + + if (bucket) bucket.push(msg); + else messagesByConv.set(msg.convId, [msg]); + } + + for (let i = 0; i < cleanIds.length; i++) { + const conv = convs[i]; + + if (!conv) continue; + + const messages = (messagesByConv.get(conv.id) ?? []).sort( + (a, b) => a.timestamp - b.timestamp + ); + + result.set(conv.id, { conv, messages }); + } + + return result; + } + + /** + * Imports multiple conversations and their messages. + * Skips conversations that already exist. + * + * @param data - Array of { conv, messages } objects + * @returns The conversations written to the database and the ones skipped + */ + static async importConversations( + data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[] + ): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> { + const imported: DatabaseConversation[] = []; + const skipped: DatabaseConversation[] = []; + + return await db.transaction( + 'rw', + [db[IDXDB_TABLES.conversations], db[IDXDB_TABLES.messages]], + async () => { + for (const item of data) { + const { conv, messages } = item; + const existing = await db[IDXDB_TABLES.conversations].get(conv.id); + + if (existing) { + skipped.push(conv); + + continue; + } + + await db[IDXDB_TABLES.conversations].add(conv); + for (const msg of messages) { + await db[IDXDB_TABLES.messages].put(msg); + } + + imported.push(conv); + } + + return { imported, skipped }; + } + ); + } + + /** + * Toggles the pinned status of a conversation. + * + * @param id - Conversation ID + * @returns The new pinned status + */ + static async toggleConversationPin(id: string): Promise { + const conversation = await db[IDXDB_TABLES.conversations].get(id); + + if (!conversation) { + throw new Error(`Conversation ${id} not found`); + } + + const newPinnedState = !conversation.pinned; + + await this.updateConversation(id, { pinned: newPinnedState }); + + return newPinnedState; + } + + /** + * Updates a conversation. `lastModified` is never stamped implicitly; + * pass it in `updates` to bump the conversation in recency ordering. + * + * @param id - Conversation ID + * @param updates - Partial updates to apply + * @returns Promise that resolves when the conversation is updated + */ + static async updateConversation( + id: string, + updates: Partial> + ): Promise { + await db[IDXDB_TABLES.conversations].update(id, updates); + } + + /** + * Updates the conversation's current node (active branch). + * This determines which conversation path is currently being viewed. + * + * @param convId - Conversation ID + * @param nodeId - Message ID to set as current node + */ + static async updateCurrentNode(convId: string, nodeId: string): Promise { + await this.updateConversation(convId, { + currNode: nodeId + }); + } + + /** + * Updates a message. + * + * @param id - Message ID + * @param updates - Partial updates to apply + * @returns Promise that resolves when the message is updated + */ + static async updateMessage( + id: string, + updates: Partial> + ): Promise { + await db[IDXDB_TABLES.messages].update(id, updates); + } + + /** + * Appends a child id to a parent message's children array. + */ + private static async addChildToParent(parentId: string, childId: string): Promise { + const parent = await db[IDXDB_TABLES.messages].get(parentId); + + if (!parent) return; + + await db[IDXDB_TABLES.messages].update(parentId, { + children: [...parent.children, childId] + }); + } + + /** + * Removes a child id from its parent message's children array. + */ + private static async removeChildFromParent(messageId: string): Promise { + const message = await db[IDXDB_TABLES.messages].get(messageId); + + if (!message?.parent) return; + + const parent = await db[IDXDB_TABLES.messages].get(message.parent); + + if (!parent) return; + + parent.children = parent.children.filter((childId: string) => childId !== messageId); + await db[IDXDB_TABLES.messages].put(parent); + } + + /** + * Reparents direct children of `parentId` to the nearest surviving + * ancestor (or promotes them to top-level when the immediate parent was + * top-level). Walking skips any ancestor listed in `excludeIds`, since + * those will be deleted in the same batch — leaving a grandchild pointing + * at an `excludeIds` entry would orphan it. Children whose own id is in + * `excludeIds` are dropped from the updates (the bulk-delete pass will + * remove them). `prefetched` may carry a pre-fetched ancestor map to + * avoid repeat reads inside a bulk transaction. + */ + private static async reparentDirectChildren( + parentId: string, + excludeIds: ReadonlySet = new Set(), + prefetched?: ReadonlyMap + ): Promise { + const conv = prefetched?.get(parentId) ?? (await db[IDXDB_TABLES.conversations].get(parentId)); + + if (!conv) return; + + let newParent = conv.forkedFromConversationId; + + const visited = new Set([parentId]); + + while (newParent && excludeIds.has(newParent)) { + if (visited.has(newParent)) { + newParent = undefined; + + break; + } + + visited.add(newParent); + const next = + prefetched?.get(newParent) ?? (await db[IDXDB_TABLES.conversations].get(newParent)); + + if (!next) { + newParent = undefined; + + break; + } + + newParent = next.forkedFromConversationId; + } + + const directChildren = await db[IDXDB_TABLES.conversations] + .filter((c) => c.forkedFromConversationId === parentId) + .toArray(); + const updates: DatabaseConversation[] = []; + + for (const child of directChildren) { + if (excludeIds.has(child.id)) continue; + + updates.push({ ...child, forkedFromConversationId: newParent }); + } + + if (updates.length === 0) return; + + await db[IDXDB_TABLES.conversations].bulkPut(updates); + } } diff --git a/tools/ui/src/lib/services/index.ts b/tools/ui/src/lib/services/index.ts index fe739a3bc..7ae9e23d4 100644 --- a/tools/ui/src/lib/services/index.ts +++ b/tools/ui/src/lib/services/index.ts @@ -53,9 +53,9 @@ * - Reasoning content stripping from prompt history to avoid KV cache pollution * - Error translation (network, timeout, server errors → user-friendly messages) * - * @see chatStore in stores/chat.svelte.ts — primary consumer for chat state management - * @see agenticStore in stores/agentic.svelte.ts — uses ChatService for agentic loop streaming - * @see conversationsStore in stores/conversations.svelte.ts — provides message context + * @see chatStore in stores/chat/index.svelte.ts — primary consumer for chat state management + * @see agenticStore in stores/agentic/index.svelte.ts — uses ChatService for agentic loop streaming + * @see conversationsStore in stores/conversations/index.svelte.ts — provides message context */ export { ChatService } from './chat.service'; @@ -98,8 +98,8 @@ export { ChatService } from './chat.service'; * enabling conversation branching and alternative response paths. The conversation's * `currNode` tracks the currently active branch endpoint. * - * @see conversationsStore in stores/conversations.svelte.ts — reactive layer on top of DatabaseService - * @see chatStore in stores/chat.svelte.ts — uses DatabaseService directly for message CRUD during streaming + * @see conversationsStore in stores/conversations/index.svelte.ts — reactive layer on top of DatabaseService + * @see chatStore in stores/chat/index.svelte.ts — uses DatabaseService directly for message CRUD during streaming */ export { DatabaseService } from './database.service'; @@ -143,7 +143,7 @@ export { ConversationTransferService } from './conversation-transfer.service'; * - `POST /models/load` — Load a model (ROUTER mode only) * - `POST /models/unload` — Unload a model (ROUTER mode only) * - * @see modelsStore in stores/models.svelte.ts — primary consumer for reactive model state + * @see modelsStore in stores/models/index.svelte.ts — primary consumer for reactive model state */ export { ModelsService } from './models.service'; @@ -174,8 +174,8 @@ export { ModelsService } from './models.service'; * - `&autoload=false` → Prevents model auto-loading when querying props * * @see serverStore in stores/server.svelte.ts — consumes global server props - * @see modelsStore in stores/models.svelte.ts — consumes per-model props for modalities - * @see settingsStore in stores/settings.svelte.ts — syncs default generation params from props + * @see modelsStore in stores/models/index.svelte.ts — consumes per-model props for modalities + * @see settingsStore in stores/settings/index.svelte.ts — syncs default generation params from props */ export { PropsService } from './props.service'; @@ -217,7 +217,7 @@ export { PropsService } from './props.service'; * - `ParameterSyncService` class — static methods for sync logic * - `SYNCABLE_PARAMETERS` — mapping of UI setting keys to server parameter keys * - * @see settingsStore in stores/settings.svelte.ts — primary consumer for settings sync + * @see settingsStore in stores/settings/index.svelte.ts — primary consumer for settings sync * @see SettingsChatParameterSourceIndicator — displays parameter source badges in UI */ export { ParameterSyncService } from './parameter-sync.service'; @@ -241,7 +241,7 @@ export { ParameterSyncService } from './parameter-sync.service'; * - Manages connection lifecycle, health checks, reconnection * - Handles tool name conflict resolution and server coordination * - * - **mcpResourceStore**: Reactive resource state + * - **mcpResourceStore** (composed as mcpStore.resources): Reactive resource state * - Receives resource data fetched via MCPService * - Manages resource caching, subscriptions, and attachments * @@ -263,9 +263,9 @@ export { ParameterSyncService } from './parameter-sync.service'; * 2. **StreamableHTTP** — modern HTTP-based, supports CORS proxy * 3. **SSE** — legacy fallback, supports CORS proxy * - * @see mcpStore in stores/mcp.svelte.ts — reactive business logic facade on top of MCPService - * @see mcpResourceStore in stores/mcp-resources.svelte.ts — reactive resource state management - * @see agenticStore in stores/agentic.svelte.ts — uses MCPService (via mcpStore) for tool execution + * @see mcpStore in stores/mcp/index.svelte.ts — reactive business logic facade on top of MCPService + * @see mcpStore.resources in stores/mcp/resources.svelte.ts — reactive resource state management + * @see agenticStore in stores/agentic/index.svelte.ts — uses MCPService (via mcpStore) for tool execution * @see MCP Protocol Specification: https://modelcontextprotocol.io/specification/2025-06-18 */ export { MCPService } from './mcp.service'; @@ -286,7 +286,7 @@ export { MCPService } from './mcp.service'; * - **agenticStore**: Dispatches ToolSource.BROWSER calls here * * @see buildSandboxToolDefinition in utils/sandbox-tool - tool schema sent to the LLM - * @see agenticStore in stores/agentic.svelte.ts - tool dispatch + * @see agenticStore in stores/agentic/index.svelte.ts - tool dispatch */ export { SandboxService } from './sandbox.service'; diff --git a/tools/ui/src/lib/services/mcp.service.ts b/tools/ui/src/lib/services/mcp.service.ts index 65e9e59d6..7b857fd43 100644 --- a/tools/ui/src/lib/services/mcp.service.ts +++ b/tools/ui/src/lib/services/mcp.service.ts @@ -1,3 +1,11 @@ +/** + * MCPService - Stateless MCP protocol layer + * + * Implements the client side of the MCP spec over WebSocket, StreamableHTTP + * and SSE transports: connect, tool/prompt/resource operations and result + * formatting. No reactive state; consumed by mcpStore and its managers. + */ + import { Client } from '@modelcontextprotocol/sdk/client'; import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js'; import { @@ -88,493 +96,79 @@ interface DiagnosticRequestDetails { export class MCPService { /** - * Create a connection log entry for phase tracking. + * Execute a tool call on a connection. + * Supports abort signal for cancellable operations (e.g., when user stops generation). + * Formats the raw tool result into a string representation. * - * @param phase - The connection phase this log belongs to - * @param message - Human-readable log message - * @param level - Log severity level (default: INFO) - * @param details - Optional structured details for debugging - * @returns Formatted connection log entry + * @param connection - The MCP connection to execute against + * @param params - Tool name and arguments to execute + * @param signal - Optional AbortSignal for cancellation support + * @returns Formatted tool execution result with content string and error flag + * @throws {Error} If tool execution fails or is aborted */ - private static createLog( - phase: MCPConnectionPhase, - message: string, - level: MCPLogLevel = MCPLogLevel.INFO, - details?: unknown - ): MCPConnectionLog { - return { - details, - level, - message, - phase, - timestamp: new Date() - }; - } - - private static createDiagnosticRequestDetails( - input: RequestInfo | URL, - init: RequestInit | undefined, - baseInit: RequestInit, - requestHeaders: Headers, - extraRedactedHeaders?: Iterable - ): DiagnosticRequestDetails { - const body = getRequestBody(input, init); - const details: DiagnosticRequestDetails = { - body: summarizeRequestBody(body), - credentials: init?.credentials ?? baseInit.credentials, - headers: sanitizeHeaders(requestHeaders, extraRedactedHeaders, HEADERS.PARTIAL_REDACT), - method: getRequestMethod(input, init, baseInit).toUpperCase(), - mode: init?.mode ?? baseInit.mode, - url: getRequestUrl(input) - }; - const jsonRpcMethods = extractJsonRpcMethods(body); - - if (jsonRpcMethods) { - details.jsonRpcMethods = jsonRpcMethods; - } - - return details; - } - - private static addRequestHeaders( - requestHeaders: Headers, - headers: HeadersInit, - useProxy: boolean - ) { - for (const [key, value] of new Headers(headers).entries()) { - const proxiedKey = - useProxy && !key.toLowerCase().startsWith(CORS_PROXY.HEADER_PREFIX) - ? `${CORS_PROXY.HEADER_PREFIX}${key}` - : key; - - requestHeaders.set(proxiedKey, value); - } - } - - private static summarizeError(error: unknown): Record { - if (error instanceof Error) { - return { - cause: - error.cause instanceof Error - ? { message: error.cause.message, name: error.cause.name } - : error.cause, - message: error.message, - name: error.name, - stack: error.stack?.split('\n').slice(0, 6).join('\n') - }; - } - - return { value: String(error) }; - } - - private static getBrowserContext( - targetUrl: URL, - useProxy: boolean - ): Record | undefined { - if (typeof window === 'undefined') { - return undefined; - } - - return { - isSecureContext: window.isSecureContext, - location: window.location.href, - origin: window.location.origin, - protocol: window.location.protocol, - sameOrigin: window.location.origin === targetUrl.origin, - targetOrigin: targetUrl.origin, - targetProtocol: targetUrl.protocol, - useProxy - }; - } - - private static getConnectionHints( - targetUrl: URL, - config: MCPServerConfig, - error: unknown - ): string[] { - const hints: string[] = []; - const message = error instanceof Error ? error.message : String(error); - const headerNames = Object.keys(config.headers ?? {}); - - if (typeof window !== 'undefined') { - if ( - window.location.protocol === 'https:' && - targetUrl.protocol === 'http:' && - !config.useProxy - ) { - hints.push( - 'The page is running over HTTPS but the MCP server is HTTP. Browsers often block this as mixed content; enable the proxy or use HTTPS/WSS for the MCP server.' - ); - } - - if (window.location.origin !== targetUrl.origin && !config.useProxy) { - hints.push( - 'This is a cross-origin browser request. If the server is reachable from curl or Node but not from the browser, missing CORS headers are the most likely cause.' - ); - } - } - - if (headerNames.length > 0) { - hints.push( - `Custom request headers are configured (${headerNames.join(', ')}). That triggers a CORS preflight, so the server must allow OPTIONS and include the matching Access-Control-Allow-Headers response.` - ); - } - - if (config.credentials && config.credentials !== 'omit') { - hints.push( - 'Credentials are enabled for this connection. Cross-origin credentialed requests need Access-Control-Allow-Credentials: true and cannot use a wildcard Access-Control-Allow-Origin.' - ); - } - - if (message.includes('Failed to fetch')) { - hints.push( - '"Failed to fetch" is a browser-level network failure. Common causes are CORS rejection, mixed-content blocking, certificate/TLS errors, DNS failures, or nothing listening on the target port.' - ); - } - - return hints; - } - - private static createDiagnosticFetch( - serverName: string, - config: MCPServerConfig, - baseInit: RequestInit, - targetUrl: URL, - useProxy: boolean, - onLog?: (log: MCPConnectionLog) => void - ): { - fetch: typeof fetch; - disable: () => void; - } { - let enabled = true; - - const logIfEnabled = (log: MCPConnectionLog) => { - if (enabled) { - onLog?.(log); - } - }; - - return { - disable: () => { - enabled = false; - }, - fetch: async (input, init) => { - if (useProxy && typeof window !== 'undefined') { - let requestUrlStr = ''; - - if (typeof input === 'string') { - requestUrlStr = input; - } else if (input instanceof URL) { - requestUrlStr = input.href; - } - - if (requestUrlStr) { - const parsedRequestUrl = new URL(requestUrlStr, window.location.origin); - - if ( - parsedRequestUrl.origin === window.location.origin && - !parsedRequestUrl.pathname.includes(CORS_PROXY_ENDPOINT) - ) { - const originalConfigUrl = new URL(config.url); - const realTargetUrl = new URL( - parsedRequestUrl.pathname + parsedRequestUrl.search, - originalConfigUrl.origin - ); - const proxiedUrl = buildProxiedUrl(realTargetUrl.href); - - if (typeof input === 'string') { - input = proxiedUrl.href; - } else if (input instanceof URL) { - input = proxiedUrl; - } - } - } - } - - const startedAt = performance.now(); - const requestHeaders = new Headers(baseInit.headers); - - if (typeof Request !== 'undefined' && input instanceof Request) { - this.addRequestHeaders(requestHeaders, input.headers, useProxy); - } - - if (init?.headers) { - this.addRequestHeaders(requestHeaders, init.headers, useProxy); - } - - const request = this.createDiagnosticRequestDetails( - input, - init, - baseInit, - requestHeaders, - Object.keys(config.headers ?? {}) - ); - const { method, url } = request; - - logIfEnabled( - this.createLog( - MCPConnectionPhase.INITIALIZING, - `HTTP ${method} ${url}`, - MCPLogLevel.INFO, - { - request, - serverName - } - ) - ); - - if (method === 'DELETE' && url.includes(CORS_PROXY_ENDPOINT)) { - const response = new Response(null, { status: 200, statusText: 'OK' }); - - logIfEnabled( - this.createLog( - MCPConnectionPhase.INITIALIZING, - `HTTP 200 ${method} ${url} (fake response)`, - MCPLogLevel.INFO, - { - response: { - durationMs: 0, - isFake: true, - status: response.status, - statusText: response.statusText, - url - } - } - ) - ); - - // fake response, bypass real fetch() - return response; - } - - try { - const response = await fetch(input, { - ...baseInit, - ...init, - headers: requestHeaders - }); - const durationMs = Math.round(performance.now() - startedAt); - - logIfEnabled( - this.createLog( - MCPConnectionPhase.INITIALIZING, - `HTTP ${response.status} ${method} ${url} (${durationMs}ms)`, - response.ok ? MCPLogLevel.INFO : MCPLogLevel.WARN, - { - response: { - durationMs, - headers: sanitizeHeaders(response.headers, undefined, HEADERS.PARTIAL_REDACT), - status: response.status, - statusText: response.statusText, - url - } - } - ) - ); - - return response; - } catch (error) { - const durationMs = Math.round(performance.now() - startedAt); - - logIfEnabled( - this.createLog( - MCPConnectionPhase.ERROR, - `HTTP ${method} ${url} failed: ${formatDiagnosticErrorMessage(error)}`, - MCPLogLevel.ERROR, - { - browser: this.getBrowserContext(targetUrl, useProxy), - durationMs, - error: this.summarizeError(error), - hints: this.getConnectionHints(targetUrl, config, error), - request, - serverName - } - ) - ); - - throw error; - } - } - }; - } - - /** - * Detect if an error indicates an expired/invalidated MCP session. - * Per MCP spec 2025-11-25: HTTP 404 means session invalidated, client MUST - * discard its session ID and start a new session with a fresh initialize request. - * - * @param error - The caught error to inspect - * @returns true if the error is a StreamableHTTP 404 (session not found) - */ - static isSessionExpiredError(error: unknown): boolean { - return error instanceof StreamableHTTPError && error.code === 404; - } - - /** - * Create transport based on server configuration. - * Supports WebSocket, StreamableHTTP (modern), and SSE (legacy) transports. - * When `useProxy` is enabled, routes HTTP requests through llama-server's CORS proxy. - * - * **Fallback Order:** - * 1. WebSocket — if explicitly configured (no CORS proxy support) - * 2. StreamableHTTP — default for HTTP connections - * 3. SSE — automatic fallback if StreamableHTTP fails - * - * @param config - Server configuration with url, transport type, proxy, and auth settings - * @returns Object containing the created transport and the transport type used - * @throws {Error} If url is missing, WebSocket + proxy combination, or all transports fail - */ - static createTransport( - serverName: string, - config: MCPServerConfig, - onLog?: (log: MCPConnectionLog) => void - ): { - transport: Transport; - type: MCPTransportType; - stopPhaseLogging: () => void; - } { - if (!config.url) { - throw new Error('MCP server configuration is missing url'); - } - - const useProxy = config.useProxy ?? false; - const requestInit: RequestInit = {}; - - if (config.headers) { - requestInit.headers = config.useProxy ? buildProxiedHeaders(config.headers) : config.headers; - } - - if (useProxy) { - requestInit.headers = { - ...getAuthHeaders(), - ...(requestInit.headers as Record) - }; - } - - if (config.credentials) { - requestInit.credentials = config.credentials; - } - - if (config.transport === MCPTransportType.WEBSOCKET) { - if (useProxy) { - throw new Error( - 'WebSocket transport is not supported when using CORS proxy. Use HTTP transport instead.' - ); - } - - const url = new URL(config.url); - - if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { - console.log(`[MCPService] Creating WebSocket transport for ${url.href}`); - } - - return { - stopPhaseLogging: () => {}, - transport: new WebSocketClientTransport(url), - type: MCPTransportType.WEBSOCKET - }; - } - - if (config.transport === MCPTransportType.SSE) { - const url = useProxy ? buildProxiedUrl(config.url) : new URL(config.url); - const { disable: stopPhaseLogging, fetch: diagnosticFetch } = this.createDiagnosticFetch( - serverName, - config, - requestInit, - url, - useProxy, - onLog - ); - - if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { - console.log(`[MCPService] Creating SSE transport for ${url.href}`); - } - - return { - stopPhaseLogging, - transport: new SSEClientTransport(url, { - eventSourceInit: { fetch: diagnosticFetch }, - fetch: diagnosticFetch, - requestInit - }), - type: MCPTransportType.SSE - }; - } - - const url = useProxy ? buildProxiedUrl(config.url) : new URL(config.url); - const { disable: stopPhaseLogging, fetch: diagnosticFetch } = this.createDiagnosticFetch( - serverName, - config, - requestInit, - url, - useProxy, - onLog - ); - - if (useProxy && import.meta.env.DEV && import.meta.env.VITE_DEBUG) { - console.log(`[MCPService] Using CORS proxy for ${config.url} -> ${url.href}`); - } + static async callTool( + connection: MCPConnection, + params: ToolCallParams, + signal?: AbortSignal + ): Promise { + throwIfAborted(signal); try { - if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { - console.log(`[MCPService] Creating StreamableHTTP transport for ${url.href}`); - } + const result = await connection.client.callTool( + { arguments: params.arguments, name: params.name }, + undefined, + { signal, timeout: connection.requestTimeoutMs } + ); return { - stopPhaseLogging, - transport: new StreamableHTTPClientTransport(url, { - fetch: diagnosticFetch, - requestInit - }), - type: MCPTransportType.STREAMABLE_HTTP + content: this.formatToolResult(result as ToolCallResult), + isError: (result as ToolCallResult).isError ?? false }; - } catch (httpError) { - console.warn(`[MCPService] StreamableHTTP failed, trying SSE transport...`, httpError); - - try { - return { - stopPhaseLogging, - transport: new SSEClientTransport(url, { - eventSourceInit: { fetch: diagnosticFetch }, - fetch: diagnosticFetch, - requestInit - }), - type: MCPTransportType.SSE - }; - } catch (sseError) { - const httpMsg = httpError instanceof Error ? httpError.message : String(httpError); - const sseMsg = sseError instanceof Error ? sseError.message : String(sseError); - - throw new Error(`Failed to create transport. StreamableHTTP: ${httpMsg}; SSE: ${sseMsg}`); + } catch (error) { + if (isAbortError(error)) { + throw error; } + + // Let session-expired errors propagate unwrapped for reconnection handling + if (this.isSessionExpiredError(error)) { + throw error; + } + + const message = error instanceof Error ? error.message : String(error); + + throw new Error( + `Tool "${params.name}" execution failed on server "${connection.serverName}": ${message}`, + { cause: error instanceof Error ? error : undefined } + ); } } /** - * Extract server info from SDK Implementation type. - * Normalizes the SDK's server version response into our MCPServerInfo type. + * Request completion suggestions from a server. + * Used for autocompleting prompt arguments or resource URI templates. * - * @param impl - Raw Implementation object from MCP SDK - * @returns Normalized server info or undefined if input is empty + * @param connection - The MCP connection to use + * @param ref - Reference to the prompt or resource template + * @param argument - The argument being completed (name and current value) + * @returns Completion result with suggested values */ - private static extractServerInfo(impl: Implementation | undefined): MCPServerInfo | undefined { - if (!impl) { - return undefined; - } + static async complete( + connection: MCPConnection, + ref: { type: MCPRefType.PROMPT; name: string } | { type: MCPRefType.RESOURCE; uri: string }, + argument: { name: string; value: string } + ): Promise<{ values: string[]; total?: number; hasMore?: boolean } | null> { + try { + const result = await connection.client.complete({ + argument, + ref + }); - return { - description: impl.description, - icons: impl.icons?.map((icon: MCPResourceIcon) => ({ - mimeType: icon.mimeType, - sizes: icon.sizes, - src: icon.src, - theme: icon.theme - })), - name: impl.name, - title: impl.title, - version: impl.version, - websiteUrl: impl.websiteUrl - }; + return result.completion; + } catch (error) { + console.error(`[MCPService] Failed to get completions:`, error); + + return null; + } } /** @@ -847,6 +441,146 @@ export class MCPService { }; } + /** + * Create transport based on server configuration. + * Supports WebSocket, StreamableHTTP (modern), and SSE (legacy) transports. + * When `useProxy` is enabled, routes HTTP requests through llama-server's CORS proxy. + * + * **Fallback Order:** + * 1. WebSocket — if explicitly configured (no CORS proxy support) + * 2. StreamableHTTP — default for HTTP connections + * 3. SSE — automatic fallback if StreamableHTTP fails + * + * @param config - Server configuration with url, transport type, proxy, and auth settings + * @returns Object containing the created transport and the transport type used + * @throws {Error} If url is missing, WebSocket + proxy combination, or all transports fail + */ + static createTransport( + serverName: string, + config: MCPServerConfig, + onLog?: (log: MCPConnectionLog) => void + ): { + transport: Transport; + type: MCPTransportType; + stopPhaseLogging: () => void; + } { + if (!config.url) { + throw new Error('MCP server configuration is missing url'); + } + + const useProxy = config.useProxy ?? false; + const requestInit: RequestInit = {}; + + if (config.headers) { + requestInit.headers = config.useProxy ? buildProxiedHeaders(config.headers) : config.headers; + } + + if (useProxy) { + requestInit.headers = { + ...getAuthHeaders(), + ...(requestInit.headers as Record) + }; + } + + if (config.credentials) { + requestInit.credentials = config.credentials; + } + + if (config.transport === MCPTransportType.WEBSOCKET) { + if (useProxy) { + throw new Error( + 'WebSocket transport is not supported when using CORS proxy. Use HTTP transport instead.' + ); + } + + const url = new URL(config.url); + + if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log(`[MCPService] Creating WebSocket transport for ${url.href}`); + } + + return { + stopPhaseLogging: () => {}, + transport: new WebSocketClientTransport(url), + type: MCPTransportType.WEBSOCKET + }; + } + + if (config.transport === MCPTransportType.SSE) { + const url = useProxy ? buildProxiedUrl(config.url) : new URL(config.url); + const { disable: stopPhaseLogging, fetch: diagnosticFetch } = this.createDiagnosticFetch( + serverName, + config, + requestInit, + url, + useProxy, + onLog + ); + + if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log(`[MCPService] Creating SSE transport for ${url.href}`); + } + + return { + stopPhaseLogging, + transport: new SSEClientTransport(url, { + eventSourceInit: { fetch: diagnosticFetch }, + fetch: diagnosticFetch, + requestInit + }), + type: MCPTransportType.SSE + }; + } + + const url = useProxy ? buildProxiedUrl(config.url) : new URL(config.url); + const { disable: stopPhaseLogging, fetch: diagnosticFetch } = this.createDiagnosticFetch( + serverName, + config, + requestInit, + url, + useProxy, + onLog + ); + + if (useProxy && import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log(`[MCPService] Using CORS proxy for ${config.url} -> ${url.href}`); + } + + try { + if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log(`[MCPService] Creating StreamableHTTP transport for ${url.href}`); + } + + return { + stopPhaseLogging, + transport: new StreamableHTTPClientTransport(url, { + fetch: diagnosticFetch, + requestInit + }), + type: MCPTransportType.STREAMABLE_HTTP + }; + } catch (httpError) { + console.warn(`[MCPService] StreamableHTTP failed, trying SSE transport...`, httpError); + + try { + return { + stopPhaseLogging, + transport: new SSEClientTransport(url, { + eventSourceInit: { fetch: diagnosticFetch }, + fetch: diagnosticFetch, + requestInit + }), + type: MCPTransportType.SSE + }; + } catch (sseError) { + const httpMsg = httpError instanceof Error ? httpError.message : String(httpError); + const sseMsg = sseError instanceof Error ? sseError.message : String(sseError); + + throw new Error(`Failed to create transport. StreamableHTTP: ${httpMsg}; SSE: ${sseMsg}`); + } + } + } + /** * Disconnect from a server. * Clears the `onclose` handler to prevent reconnection attempts on voluntary disconnect. @@ -882,29 +616,68 @@ export class MCPService { } /** - * List tools from a connection. - * Silently returns empty array on failure (logged as warning). + * Get a specific prompt with arguments. + * Unlike list operations, this throws on failure since the caller explicitly + * requested a specific prompt and needs to handle the error. * - * @param connection - The MCP connection to query - * @returns Array of available tools, or empty array on error + * @param connection - The MCP connection to use + * @param name - The prompt name to retrieve + * @param args - Optional key-value arguments to pass to the prompt + * @returns The prompt result with messages and metadata + * @throws {Error} If the prompt retrieval fails */ - static async listTools(connection: MCPConnection): Promise { + static async getPrompt( + connection: MCPConnection, + name: string, + args?: Record + ): Promise { try { - const result = await connection.client.listTools(); - - return result.tools ?? []; + return await connection.client.getPrompt({ arguments: args, name }); } catch (error) { - // Let session-expired errors propagate for reconnection handling - if (this.isSessionExpiredError(error)) { - throw error; - } + console.error(`[MCPService][${connection.serverName}] Failed to get prompt:`, error); - console.warn(`[MCPService][${connection.serverName}] Failed to list tools:`, error); - - return []; + throw error; } } + /** + * Detect if an error indicates an expired/invalidated MCP session. + * Per MCP spec 2025-11-25: HTTP 404 means session invalidated, client MUST + * discard its session ID and start a new session with a fresh initialize request. + * + * @param error - The caught error to inspect + * @returns true if the error is a StreamableHTTP 404 (session not found) + */ + static isSessionExpiredError(error: unknown): boolean { + return error instanceof StreamableHTTPError && error.code === 404; + } + + /** + * List all resources from a connection (handles pagination automatically). + * @param connection - The MCP connection to use + * @returns Array of all available resources + */ + static async listAllResources(connection: MCPConnection): Promise { + return this.paginate( + connection, + (cursor) => this.listResources(connection, cursor), + (result) => result.resources + ); + } + + /** + * List all resource templates from a connection (handles pagination automatically). + * @param connection - The MCP connection to use + * @returns Array of all available resource templates + */ + static async listAllResourceTemplates(connection: MCPConnection): Promise { + return this.paginate( + connection, + (cursor) => this.listResourceTemplates(connection, cursor), + (result) => result.resourceTemplates + ); + } + /** * List prompts from a connection. * Silently returns empty array on failure (logged as warning). @@ -929,177 +702,6 @@ export class MCPService { } } - /** - * Get a specific prompt with arguments. - * Unlike list operations, this throws on failure since the caller explicitly - * requested a specific prompt and needs to handle the error. - * - * @param connection - The MCP connection to use - * @param name - The prompt name to retrieve - * @param args - Optional key-value arguments to pass to the prompt - * @returns The prompt result with messages and metadata - * @throws {Error} If the prompt retrieval fails - */ - static async getPrompt( - connection: MCPConnection, - name: string, - args?: Record - ): Promise { - try { - return await connection.client.getPrompt({ arguments: args, name }); - } catch (error) { - console.error(`[MCPService][${connection.serverName}] Failed to get prompt:`, error); - - throw error; - } - } - - /** - * Execute a tool call on a connection. - * Supports abort signal for cancellable operations (e.g., when user stops generation). - * Formats the raw tool result into a string representation. - * - * @param connection - The MCP connection to execute against - * @param params - Tool name and arguments to execute - * @param signal - Optional AbortSignal for cancellation support - * @returns Formatted tool execution result with content string and error flag - * @throws {Error} If tool execution fails or is aborted - */ - static async callTool( - connection: MCPConnection, - params: ToolCallParams, - signal?: AbortSignal - ): Promise { - throwIfAborted(signal); - - try { - const result = await connection.client.callTool( - { arguments: params.arguments, name: params.name }, - undefined, - { signal, timeout: connection.requestTimeoutMs } - ); - - return { - content: this.formatToolResult(result as ToolCallResult), - isError: (result as ToolCallResult).isError ?? false - }; - } catch (error) { - if (isAbortError(error)) { - throw error; - } - - // Let session-expired errors propagate unwrapped for reconnection handling - if (this.isSessionExpiredError(error)) { - throw error; - } - - const message = error instanceof Error ? error.message : String(error); - - throw new Error( - `Tool "${params.name}" execution failed on server "${connection.serverName}": ${message}`, - { cause: error instanceof Error ? error : undefined } - ); - } - } - - /** - * Format tool result content items to a single string. - * Handles text, image (base64 data URL), and embedded resource content types. - * - * @param result - Raw tool call result from MCP SDK - * @returns Concatenated string representation of all content items - */ - private static formatToolResult(result: ToolCallResult): string { - const content = result.content; - - if (!Array.isArray(content)) return ''; - - const formatted = content - .map((item) => this.formatSingleContent(item)) - .filter(Boolean) - .join(NEWLINE); - - if (formatted !== '') { - return formatted; - } - - if (result.structuredContent && typeof result.structuredContent === 'object') { - return JSON.stringify(result.structuredContent); - } - - return ''; - } - - private static formatSingleContent(content: ToolResultContentItem): string { - if (content.type === MCPContentType.TEXT && content.text) { - return content.text; - } - - if (content.type === MCPContentType.IMAGE && content.data) { - return createBase64DataUrl(content.mimeType ?? DEFAULT_IMAGE_MIME_TYPE, content.data); - } - - if (content.type === MCPContentType.RESOURCE && content.resource) { - const resource = content.resource; - - if (resource.text) return resource.text; - - if (resource.blob) return resource.blob; - - return JSON.stringify(resource); - } - - if (content.data && content.mimeType) { - return createBase64DataUrl(content.mimeType, content.data); - } - - return JSON.stringify(content); - } - - /** - * - * - * Completions Operations - * - * - */ - - /** - * Request completion suggestions from a server. - * Used for autocompleting prompt arguments or resource URI templates. - * - * @param connection - The MCP connection to use - * @param ref - Reference to the prompt or resource template - * @param argument - The argument being completed (name and current value) - * @returns Completion result with suggested values - */ - static async complete( - connection: MCPConnection, - ref: { type: MCPRefType.PROMPT; name: string } | { type: MCPRefType.RESOURCE; uri: string }, - argument: { name: string; value: string } - ): Promise<{ values: string[]; total?: number; hasMore?: boolean } | null> { - try { - const result = await connection.client.complete({ - argument, - ref - }); - - return result.completion; - } catch (error) { - console.error(`[MCPService] Failed to get completions:`, error); - - return null; - } - } - - /** - * - * - * Resources Operations - * - * - */ - /** * List resources from a connection. * @param connection - The MCP connection to use @@ -1128,26 +730,6 @@ export class MCPService { } } - /** - * List all resources from a connection (handles pagination automatically). - * @param connection - The MCP connection to use - * @returns Array of all available resources - */ - static async listAllResources(connection: MCPConnection): Promise { - const allResources: MCPResource[] = []; - - let cursor: string | undefined; - - do { - const result = await this.listResources(connection, cursor); - - allResources.push(...result.resources); - cursor = result.nextCursor; - } while (cursor); - - return allResources; - } - /** * List resource templates from a connection. * @param connection - The MCP connection to use @@ -1180,23 +762,27 @@ export class MCPService { } /** - * List all resource templates from a connection (handles pagination automatically). - * @param connection - The MCP connection to use - * @returns Array of all available resource templates + * List tools from a connection. + * Silently returns empty array on failure (logged as warning). + * + * @param connection - The MCP connection to query + * @returns Array of available tools, or empty array on error */ - static async listAllResourceTemplates(connection: MCPConnection): Promise { - const allTemplates: MCPResourceTemplate[] = []; + static async listTools(connection: MCPConnection): Promise { + try { + const result = await connection.client.listTools(); - let cursor: string | undefined; + return result.tools ?? []; + } catch (error) { + // Let session-expired errors propagate for reconnection handling + if (this.isSessionExpiredError(error)) { + throw error; + } - do { - const result = await this.listResourceTemplates(connection, cursor); + console.warn(`[MCPService][${connection.serverName}] Failed to list tools:`, error); - allTemplates.push(...result.resourceTemplates); - cursor = result.nextCursor; - } while (cursor); - - return allTemplates; + return []; + } } /** @@ -1244,28 +830,6 @@ export class MCPService { } } - /** - * Unsubscribe from updates for a resource. - * @param connection - The MCP connection to use - * @param uri - The URI of the resource to unsubscribe from - */ - static async unsubscribeResource(connection: MCPConnection, uri: string): Promise { - try { - await connection.client.unsubscribeResource({ uri }); - - if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { - console.log(`[MCPService][${connection.serverName}] Unsubscribed from resource: ${uri}`); - } - } catch (error) { - console.error( - `[MCPService][${connection.serverName}] Failed to unsubscribe from resource:`, - error - ); - - throw error; - } - } - /** * Check if a connection supports resources. * Per MCP spec: presence of the `resources` key (even as empty object `{}`) indicates support. @@ -1288,4 +852,440 @@ export class MCPService { static supportsResourceSubscriptions(connection: MCPConnection): boolean { return !!connection.serverCapabilities?.resources?.subscribe; } + + /** + * Unsubscribe from updates for a resource. + * @param connection - The MCP connection to use + * @param uri - The URI of the resource to unsubscribe from + */ + static async unsubscribeResource(connection: MCPConnection, uri: string): Promise { + try { + await connection.client.unsubscribeResource({ uri }); + + if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log(`[MCPService][${connection.serverName}] Unsubscribed from resource: ${uri}`); + } + } catch (error) { + console.error( + `[MCPService][${connection.serverName}] Failed to unsubscribe from resource:`, + error + ); + + throw error; + } + } + + private static addRequestHeaders( + requestHeaders: Headers, + headers: HeadersInit, + useProxy: boolean + ) { + for (const [key, value] of new Headers(headers).entries()) { + const proxiedKey = + useProxy && !key.toLowerCase().startsWith(CORS_PROXY.HEADER_PREFIX) + ? `${CORS_PROXY.HEADER_PREFIX}${key}` + : key; + + requestHeaders.set(proxiedKey, value); + } + } + + private static createDiagnosticFetch( + serverName: string, + config: MCPServerConfig, + baseInit: RequestInit, + targetUrl: URL, + useProxy: boolean, + onLog?: (log: MCPConnectionLog) => void + ): { + fetch: typeof fetch; + disable: () => void; + } { + let enabled = true; + + const logIfEnabled = (log: MCPConnectionLog) => { + if (enabled) { + onLog?.(log); + } + }; + + return { + disable: () => { + enabled = false; + }, + fetch: async (input, init) => { + if (useProxy && typeof window !== 'undefined') { + let requestUrlStr = ''; + + if (typeof input === 'string') { + requestUrlStr = input; + } else if (input instanceof URL) { + requestUrlStr = input.href; + } + + if (requestUrlStr) { + const parsedRequestUrl = new URL(requestUrlStr, window.location.origin); + + if ( + parsedRequestUrl.origin === window.location.origin && + !parsedRequestUrl.pathname.includes(CORS_PROXY_ENDPOINT) + ) { + const originalConfigUrl = new URL(config.url); + const realTargetUrl = new URL( + parsedRequestUrl.pathname + parsedRequestUrl.search, + originalConfigUrl.origin + ); + const proxiedUrl = buildProxiedUrl(realTargetUrl.href); + + if (typeof input === 'string') { + input = proxiedUrl.href; + } else if (input instanceof URL) { + input = proxiedUrl; + } + } + } + } + + const startedAt = performance.now(); + const requestHeaders = new Headers(baseInit.headers); + + if (typeof Request !== 'undefined' && input instanceof Request) { + this.addRequestHeaders(requestHeaders, input.headers, useProxy); + } + + if (init?.headers) { + this.addRequestHeaders(requestHeaders, init.headers, useProxy); + } + + const request = this.createDiagnosticRequestDetails( + input, + init, + baseInit, + requestHeaders, + Object.keys(config.headers ?? {}) + ); + const { method, url } = request; + + logIfEnabled( + this.createLog( + MCPConnectionPhase.INITIALIZING, + `HTTP ${method} ${url}`, + MCPLogLevel.INFO, + { + request, + serverName + } + ) + ); + + if (method === 'DELETE' && url.includes(CORS_PROXY_ENDPOINT)) { + const response = new Response(null, { status: 200, statusText: 'OK' }); + + logIfEnabled( + this.createLog( + MCPConnectionPhase.INITIALIZING, + `HTTP 200 ${method} ${url} (fake response)`, + MCPLogLevel.INFO, + { + response: { + durationMs: 0, + isFake: true, + status: response.status, + statusText: response.statusText, + url + } + } + ) + ); + + // fake response, bypass real fetch() + return response; + } + + try { + const response = await fetch(input, { + ...baseInit, + ...init, + headers: requestHeaders + }); + const durationMs = Math.round(performance.now() - startedAt); + + logIfEnabled( + this.createLog( + MCPConnectionPhase.INITIALIZING, + `HTTP ${response.status} ${method} ${url} (${durationMs}ms)`, + response.ok ? MCPLogLevel.INFO : MCPLogLevel.WARN, + { + response: { + durationMs, + headers: sanitizeHeaders(response.headers, undefined, HEADERS.PARTIAL_REDACT), + status: response.status, + statusText: response.statusText, + url + } + } + ) + ); + + return response; + } catch (error) { + const durationMs = Math.round(performance.now() - startedAt); + + logIfEnabled( + this.createLog( + MCPConnectionPhase.ERROR, + `HTTP ${method} ${url} failed: ${formatDiagnosticErrorMessage(error)}`, + MCPLogLevel.ERROR, + { + browser: this.getBrowserContext(targetUrl, useProxy), + durationMs, + error: this.summarizeError(error), + hints: this.getConnectionHints(targetUrl, config, error), + request, + serverName + } + ) + ); + + throw error; + } + } + }; + } + + private static createDiagnosticRequestDetails( + input: RequestInfo | URL, + init: RequestInit | undefined, + baseInit: RequestInit, + requestHeaders: Headers, + extraRedactedHeaders?: Iterable + ): DiagnosticRequestDetails { + const body = getRequestBody(input, init); + const details: DiagnosticRequestDetails = { + body: summarizeRequestBody(body), + credentials: init?.credentials ?? baseInit.credentials, + headers: sanitizeHeaders(requestHeaders, extraRedactedHeaders, HEADERS.PARTIAL_REDACT), + method: getRequestMethod(input, init, baseInit).toUpperCase(), + mode: init?.mode ?? baseInit.mode, + url: getRequestUrl(input) + }; + const jsonRpcMethods = extractJsonRpcMethods(body); + + if (jsonRpcMethods) { + details.jsonRpcMethods = jsonRpcMethods; + } + + return details; + } + + /** + * Create a connection log entry for phase tracking. + * + * @param phase - The connection phase this log belongs to + * @param message - Human-readable log message + * @param level - Log severity level (default: INFO) + * @param details - Optional structured details for debugging + * @returns Formatted connection log entry + */ + private static createLog( + phase: MCPConnectionPhase, + message: string, + level: MCPLogLevel = MCPLogLevel.INFO, + details?: unknown + ): MCPConnectionLog { + return { + details, + level, + message, + phase, + timestamp: new Date() + }; + } + + /** + * Extract server info from SDK Implementation type. + * Normalizes the SDK's server version response into our MCPServerInfo type. + * + * @param impl - Raw Implementation object from MCP SDK + * @returns Normalized server info or undefined if input is empty + */ + private static extractServerInfo(impl: Implementation | undefined): MCPServerInfo | undefined { + if (!impl) { + return undefined; + } + + return { + description: impl.description, + icons: impl.icons?.map((icon: MCPResourceIcon) => ({ + mimeType: icon.mimeType, + sizes: icon.sizes, + src: icon.src, + theme: icon.theme + })), + name: impl.name, + title: impl.title, + version: impl.version, + websiteUrl: impl.websiteUrl + }; + } + + private static formatSingleContent(content: ToolResultContentItem): string { + if (content.type === MCPContentType.TEXT && content.text) { + return content.text; + } + + if (content.type === MCPContentType.IMAGE && content.data) { + return createBase64DataUrl(content.mimeType ?? DEFAULT_IMAGE_MIME_TYPE, content.data); + } + + if (content.type === MCPContentType.RESOURCE && content.resource) { + const resource = content.resource; + + if (resource.text) return resource.text; + + if (resource.blob) return resource.blob; + + return JSON.stringify(resource); + } + + if (content.data && content.mimeType) { + return createBase64DataUrl(content.mimeType, content.data); + } + + return JSON.stringify(content); + } + + /** + * Format tool result content items to a single string. + * Handles text, image (base64 data URL), and embedded resource content types. + * + * @param result - Raw tool call result from MCP SDK + * @returns Concatenated string representation of all content items + */ + private static formatToolResult(result: ToolCallResult): string { + const content = result.content; + + if (!Array.isArray(content)) return ''; + + const formatted = content + .map((item) => this.formatSingleContent(item)) + .filter(Boolean) + .join(NEWLINE); + + if (formatted !== '') { + return formatted; + } + + if (result.structuredContent && typeof result.structuredContent === 'object') { + return JSON.stringify(result.structuredContent); + } + + return ''; + } + + private static getBrowserContext( + targetUrl: URL, + useProxy: boolean + ): Record | undefined { + if (typeof window === 'undefined') { + return undefined; + } + + return { + isSecureContext: window.isSecureContext, + location: window.location.href, + origin: window.location.origin, + protocol: window.location.protocol, + sameOrigin: window.location.origin === targetUrl.origin, + targetOrigin: targetUrl.origin, + targetProtocol: targetUrl.protocol, + useProxy + }; + } + + private static getConnectionHints( + targetUrl: URL, + config: MCPServerConfig, + error: unknown + ): string[] { + const hints: string[] = []; + const message = error instanceof Error ? error.message : String(error); + const headerNames = Object.keys(config.headers ?? {}); + + if (typeof window !== 'undefined') { + if ( + window.location.protocol === 'https:' && + targetUrl.protocol === 'http:' && + !config.useProxy + ) { + hints.push( + 'The page is running over HTTPS but the MCP server is HTTP. Browsers often block this as mixed content; enable the proxy or use HTTPS/WSS for the MCP server.' + ); + } + + if (window.location.origin !== targetUrl.origin && !config.useProxy) { + hints.push( + 'This is a cross-origin browser request. If the server is reachable from curl or Node but not from the browser, missing CORS headers are the most likely cause.' + ); + } + } + + if (headerNames.length > 0) { + hints.push( + `Custom request headers are configured (${headerNames.join(', ')}). That triggers a CORS preflight, so the server must allow OPTIONS and include the matching Access-Control-Allow-Headers response.` + ); + } + + if (config.credentials && config.credentials !== 'omit') { + hints.push( + 'Credentials are enabled for this connection. Cross-origin credentialed requests need Access-Control-Allow-Credentials: true and cannot use a wildcard Access-Control-Allow-Origin.' + ); + } + + if (message.includes('Failed to fetch')) { + hints.push( + '"Failed to fetch" is a browser-level network failure. Common causes are CORS rejection, mixed-content blocking, certificate/TLS errors, DNS failures, or nothing listening on the target port.' + ); + } + + return hints; + } + + /** + * Walk a cursor-paginated MCP list endpoint, collecting every page. + */ + private static async paginate( + connection: MCPConnection, + fetchPage: (cursor?: string) => Promise, + extract: (result: R) => T[] + ): Promise { + const all: T[] = []; + + let cursor: string | undefined; + + do { + const result = await fetchPage(cursor); + + all.push(...extract(result)); + cursor = result.nextCursor; + } while (cursor); + + return all; + } + + private static summarizeError(error: unknown): Record { + if (error instanceof Error) { + return { + cause: + error.cause instanceof Error + ? { message: error.cause.message, name: error.cause.name } + : error.cause, + message: error.message, + name: error.name, + stack: error.stack?.split('\n').slice(0, 6).join('\n') + }; + } + + return { value: String(error) }; + } } diff --git a/tools/ui/src/lib/services/migration.service.ts b/tools/ui/src/lib/services/migration.service.ts index 2626a42b3..5d321b3ba 100644 --- a/tools/ui/src/lib/services/migration.service.ts +++ b/tools/ui/src/lib/services/migration.service.ts @@ -1,20 +1,11 @@ /** - * Migration Service - Unified data migration hook + * MigrationService - Unified data migration hook * - * Centralizes all data migrations (localStorage, IndexedDB, legacy formats) into a single - * initialization point. Each migration copies data to new format WITHOUT deleting the old. - * - * **Architecture:** - * - Migrations are defined as objects with `id` and `run()` methods - * - Migration state is tracked in localStorage to avoid re-running - * - `runAllMigrations()` should be called once at app startup - * - All migrations are NON-DESTRUCTIVE - legacy data is preserved for downgrade compatibility - * - * **Current Migrations:** - * 1. localStorage prefix: Copy LlamaCppWebui.* → LlamaUi.* (both preserved) - * 2. IndexedDB database: Copy LlamacppWebui → LlamaUi (both preserved) - * 3. Legacy message format: Transform in-place (preserves structure, migrates markers) - * 4. Theme key: Copy standalone `theme` → config object (both preserved) + * Centralizes all data migrations (localStorage, IndexedDB, legacy formats) + * into a single initialization point. Each migration copies data to the new + * format WITHOUT deleting the old, and state is tracked in localStorage so + * `runAllMigrations()` (called once at startup) never re-runs a completed + * migration. All migrations are non-destructive for downgrade compatibility. */ import { diff --git a/tools/ui/src/lib/services/models.service.ts b/tools/ui/src/lib/services/models.service.ts index 84832e086..bb1bbd356 100644 --- a/tools/ui/src/lib/services/models.service.ts +++ b/tools/ui/src/lib/services/models.service.ts @@ -1,25 +1,55 @@ +/** + * ModelsService - Stateless model management API layer + * + * Wraps the /models endpoints (list, load, unload) and the /models/sse + * status feed in MODEL and ROUTER modes. No reactive state; consumed by + * modelsStore and its status manager. + */ + import { base } from '$app/paths'; -import { - API_MODELS, - MODEL_ID, - SSE_DATA_PREFIX, - SSE_LINE_SEPARATOR, - SSE_RECORD_SEPARATOR -} from '$lib/constants'; +import { API_MODELS, MODEL_ID } from '$lib/constants'; import { ServerModelStatus } from '$lib/enums'; import type { ParsedModelId } from '$lib/types/models'; -import { apiFetch, apiPost, normalizeModelName } from '$lib/utils'; +import { + apiFetch, + apiPost, + extractSseDataPayload, + normalizeModelName, + splitSseRecords +} from '$lib/utils'; import { getAuthHeaders } from '$lib/utils/api-headers'; export class ModelsService { + private static readonly SSE_RECONNECT_MS = 1000; + + /** + * Check if a model is loaded based on its metadata. + * + * @param model - Model data entry from the API response + * @returns True if the model status is LOADED + */ + static isModelLoaded(model: ApiModelDataEntry): boolean { + return model.status.value === ServerModelStatus.LOADED; + } + /** * * - * Listing + * Load/Unload * * */ + /** + * Check if a model is currently loading. + * + * @param model - Model data entry from the API response + * @returns True if the model status is LOADING + */ + static isModelLoading(model: ApiModelDataEntry): boolean { + return model.status.value === ServerModelStatus.LOADING; + } + /** * Fetch list of models from OpenAI-compatible endpoint. * Works in both MODEL and ROUTER modes. @@ -41,14 +71,6 @@ export class ModelsService { return apiFetch(API_MODELS.LIST); } - /** - * - * - * Load/Unload - * - * - */ - /** * Load a model (ROUTER mode only). * Sends POST request to `/models/load`. Note: the endpoint returns success @@ -68,137 +90,6 @@ export class ModelsService { return apiPost(API_MODELS.LOAD, payload); } - /** - * Unload a model (ROUTER mode only). - * Sends POST request to `/models/unload`. Note: the endpoint returns success - * before unloading completes — use polling to await actual unload status. - * - * @param modelId - Model identifier to unload - * @returns Unload response from the server - */ - static async unload(modelId: string): Promise { - return apiPost(API_MODELS.UNLOAD, { model: modelId }); - } - - /** - * - * - * Status - * - * - */ - - /** - * Check if a model is loaded based on its metadata. - * - * @param model - Model data entry from the API response - * @returns True if the model status is LOADED - */ - static isModelLoaded(model: ApiModelDataEntry): boolean { - return model.status.value === ServerModelStatus.LOADED; - } - - /** - * Check if a model is currently loading. - * - * @param model - Model data entry from the API response - * @returns True if the model status is LOADING - */ - static isModelLoading(model: ApiModelDataEntry): boolean { - return model.status.value === ServerModelStatus.LOADING; - } - - /** - * - * - * Status Feed - * - * - */ - - private static readonly SSE_RECONNECT_MS = 1000; - - /** - * Read the /models/sse feed and invoke onEvent for each parsed envelope. - * Reconnects on network drops until the signal aborts. Splits the byte - * stream into SSE records on the blank line boundary; the payload rides in - * the data lines as a JSON envelope with its own model, event and data fields. - */ - static async watchModelEvents( - signal: AbortSignal, - onEvent: (event: ApiModelsSseEvent) => void - ): Promise { - const decoder = new TextDecoder(); - - while (!signal.aborted) { - try { - const response = await fetch(`${base}${API_MODELS.SSE}`, { - headers: getAuthHeaders(), - signal - }); - - if (response.ok && response.body) { - const reader = response.body.getReader(); - - let buffer = ''; - - while (!signal.aborted) { - const { done, value } = await reader.read(); - - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - - let boundary = buffer.indexOf(SSE_RECORD_SEPARATOR); - - while (boundary !== -1) { - const event = ModelsService.parseStatusRecord(buffer.slice(0, boundary)); - - if (event) onEvent(event); - - buffer = buffer.slice(boundary + SSE_RECORD_SEPARATOR.length); - boundary = buffer.indexOf(SSE_RECORD_SEPARATOR); - } - } - } - } catch { - // network drop or abort falls through to the reconnect delay - } - - if (signal.aborted) return; - - await new Promise((resolve) => setTimeout(resolve, ModelsService.SSE_RECONNECT_MS)); - } - } - - /** - * Parse one SSE record into its JSON envelope, or null when the record - * carries no data payload or malformed JSON. - */ - private static parseStatusRecord(record: string): ApiModelsSseEvent | null { - const payload = record - .split(SSE_LINE_SEPARATOR) - .filter((line) => line.startsWith(SSE_DATA_PREFIX)) - .map((line) => line.slice(SSE_DATA_PREFIX.length).trim()) - .join(SSE_LINE_SEPARATOR); - - if (payload.length === 0) return null; - - try { - return JSON.parse(payload) as ApiModelsSseEvent; - } catch { - return null; - } - } - - /** - * - * - * Parsing - * - * - */ - /** * Parse a model ID string into its structured components. * @@ -311,4 +202,84 @@ export class ModelsService { return result; } + + /** + * Unload a model (ROUTER mode only). + * Sends POST request to `/models/unload`. Note: the endpoint returns success + * before unloading completes — use polling to await actual unload status. + * + * @param modelId - Model identifier to unload + * @returns Unload response from the server + */ + static async unload(modelId: string): Promise { + return apiPost(API_MODELS.UNLOAD, { model: modelId }); + } + + /** + * Read the /models/sse feed and invoke onEvent for each parsed envelope. + * Reconnects on network drops until the signal aborts. Splits the byte + * stream into SSE records on the blank line boundary; the payload rides in + * the data lines as a JSON envelope with its own model, event and data fields. + */ + static async watchModelEvents( + signal: AbortSignal, + onEvent: (event: ApiModelsSseEvent) => void + ): Promise { + const decoder = new TextDecoder(); + + while (!signal.aborted) { + try { + const response = await fetch(`${base}${API_MODELS.SSE}`, { + headers: getAuthHeaders(), + signal + }); + + if (response.ok && response.body) { + const reader = response.body.getReader(); + + let buffer = ''; + + while (!signal.aborted) { + const { done, value } = await reader.read(); + + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + const { records, rest } = splitSseRecords(buffer); + + buffer = rest; + + for (const record of records) { + const event = ModelsService.parseStatusRecord(record); + + if (event) onEvent(event); + } + } + } + } catch { + // network drop or abort falls through to the reconnect delay + } + + if (signal.aborted) return; + + await new Promise((resolve) => setTimeout(resolve, ModelsService.SSE_RECONNECT_MS)); + } + } + + /** + * Parse one SSE record into its JSON envelope, or null when the record + * carries no data payload or malformed JSON. + */ + private static parseStatusRecord(record: string): ApiModelsSseEvent | null { + const payload = extractSseDataPayload(record); + + if (payload.length === 0) return null; + + try { + return JSON.parse(payload) as ApiModelsSseEvent; + } catch { + return null; + } + } } diff --git a/tools/ui/src/lib/services/parameter-sync.service.ts b/tools/ui/src/lib/services/parameter-sync.service.ts index 0ed9ebd48..467e7c2db 100644 --- a/tools/ui/src/lib/services/parameter-sync.service.ts +++ b/tools/ui/src/lib/services/parameter-sync.service.ts @@ -1,3 +1,11 @@ +/** + * ParameterSyncService - Syncs sampling parameters with the server + * + * Decides for each sampling parameter whether the user's setting is an + * override of the server default, and normalizes floating-point values. + * No reactive state; consumed by settingsStore. + */ + import { SETTINGS_KEYS, SYNCABLE_PARAMETERS } from '$lib/constants'; import { ParameterSource, SyncableParameterType } from '$lib/enums'; import type { ParameterInfo, ParameterRecord, ParameterValue } from '$lib/types'; @@ -5,22 +13,47 @@ import { normalizeFloatingPoint } from '$lib/utils'; export class ParameterSyncService { /** + * Check if a parameter can be synced from server. * - * - * Extraction - * - * + * @param key - The parameter key to check + * @returns True if the parameter is in the syncable parameters list */ + static canSyncParameter(key: string): boolean { + return SYNCABLE_PARAMETERS.some((param) => param.key === key && param.canSync); + } /** - * Round floating-point numbers to avoid JavaScript precision issues. - * E.g., 0.1 + 0.2 = 0.30000000000000004 → 0.3 + * Create a diff between current settings and server defaults. + * Shows which parameters differ from server values, useful for debugging + * and for the "Reset to defaults" functionality. * - * @param value - Parameter value to normalize - * @returns Precision-normalized value + * @param currentSettings - Current parameter values in the settings store + * @param serverDefaults - Default values extracted from server props + * @returns Record of parameter diffs with current value, server value, and whether they differ */ - private static roundFloatingPoint(value: ParameterValue): ParameterValue { - return normalizeFloatingPoint(value) as ParameterValue; + static createParameterDiff( + currentSettings: ParameterRecord, + serverDefaults: ParameterRecord + ): Record { + const diff: Record< + string, + { current: ParameterValue; server: ParameterValue; differs: boolean } + > = {}; + + for (const key of this.getSyncableParameterKeys()) { + const currentValue = currentSettings[key]; + const serverValue = serverDefaults[key]; + + if (serverValue !== undefined) { + diff[key] = { + current: currentValue, + differs: currentValue !== serverValue, + server: serverValue + }; + } + } + + return diff; } /** @@ -59,49 +92,6 @@ export class ParameterSyncService { return extracted; } - /** - * - * - * Merging - * - * - */ - - /** - * Merge server defaults with current user settings. - * User overrides always take priority — only parameters not in `userOverrides` - * set will be updated from server defaults. - * - * @param currentSettings - Current parameter values in the settings store - * @param serverDefaults - Default values extracted from server props - * @param userOverrides - Set of parameter keys explicitly overridden by the user - * @returns Merged parameter record with user overrides preserved - */ - static mergeWithServerDefaults( - currentSettings: ParameterRecord, - serverDefaults: ParameterRecord, - userOverrides: Set = new Set() - ): ParameterRecord { - const merged = { ...currentSettings }; - - for (const [key, serverValue] of Object.entries(serverDefaults)) { - // Only update if user hasn't explicitly overridden this parameter - if (!userOverrides.has(key)) { - merged[key] = this.roundFloatingPoint(serverValue); - } - } - - return merged; - } - - /** - * - * - * Info - * - * - */ - /** * Get parameter information including source and values. * Used by SettingsChatParameterSourceIndicator to display the correct badge @@ -132,16 +122,6 @@ export class ParameterSyncService { }; } - /** - * Check if a parameter can be synced from server. - * - * @param key - The parameter key to check - * @returns True if the parameter is in the syncable parameters list - */ - static canSyncParameter(key: string): boolean { - return SYNCABLE_PARAMETERS.some((param) => param.key === key && param.canSync); - } - /** * Get all syncable parameter keys. * @@ -151,6 +131,33 @@ export class ParameterSyncService { return SYNCABLE_PARAMETERS.filter((param) => param.canSync).map((param) => param.key); } + /** + * Merge server defaults with current user settings. + * User overrides always take priority — only parameters not in `userOverrides` + * set will be updated from server defaults. + * + * @param currentSettings - Current parameter values in the settings store + * @param serverDefaults - Default values extracted from server props + * @param userOverrides - Set of parameter keys explicitly overridden by the user + * @returns Merged parameter record with user overrides preserved + */ + static mergeWithServerDefaults( + currentSettings: ParameterRecord, + serverDefaults: ParameterRecord, + userOverrides: Set = new Set() + ): ParameterRecord { + const merged = { ...currentSettings }; + + for (const [key, serverValue] of Object.entries(serverDefaults)) { + // Only update if user hasn't explicitly overridden this parameter + if (!userOverrides.has(key)) { + merged[key] = this.roundFloatingPoint(serverValue); + } + } + + return merged; + } + /** * Validate a server parameter value against its expected type. * @@ -176,44 +183,13 @@ export class ParameterSyncService { } /** + * Round floating-point numbers to avoid JavaScript precision issues. + * E.g., 0.1 + 0.2 = 0.30000000000000004 → 0.3 * - * - * Diff - * - * + * @param value - Parameter value to normalize + * @returns Precision-normalized value */ - - /** - * Create a diff between current settings and server defaults. - * Shows which parameters differ from server values, useful for debugging - * and for the "Reset to defaults" functionality. - * - * @param currentSettings - Current parameter values in the settings store - * @param serverDefaults - Default values extracted from server props - * @returns Record of parameter diffs with current value, server value, and whether they differ - */ - static createParameterDiff( - currentSettings: ParameterRecord, - serverDefaults: ParameterRecord - ): Record { - const diff: Record< - string, - { current: ParameterValue; server: ParameterValue; differs: boolean } - > = {}; - - for (const key of this.getSyncableParameterKeys()) { - const currentValue = currentSettings[key]; - const serverValue = serverDefaults[key]; - - if (serverValue !== undefined) { - diff[key] = { - current: currentValue, - differs: currentValue !== serverValue, - server: serverValue - }; - } - } - - return diff; + private static roundFloatingPoint(value: ParameterValue): ParameterValue { + return normalizeFloatingPoint(value) as ParameterValue; } } diff --git a/tools/ui/src/lib/services/props.service.ts b/tools/ui/src/lib/services/props.service.ts index 46f4915fa..488a67b64 100644 --- a/tools/ui/src/lib/services/props.service.ts +++ b/tools/ui/src/lib/services/props.service.ts @@ -1,14 +1,14 @@ +/** + * PropsService - Fetches server properties from /props + * + * Returns global server settings and capabilities, including per-model + * modalities in MODEL mode. No reactive state; consumed by serverStore and + * the model props manager. + */ + import { apiFetchWithParams } from '$lib/utils'; export class PropsService { - /** - * - * - * Fetching - * - * - */ - /** * Fetches global server properties from the `/props` endpoint. * In MODEL mode, returns modalities for the single loaded model. diff --git a/tools/ui/src/lib/services/read-media.service.ts b/tools/ui/src/lib/services/read-media.service.ts index 8de9bbbea..2858795e8 100644 --- a/tools/ui/src/lib/services/read-media.service.ts +++ b/tools/ui/src/lib/services/read-media.service.ts @@ -1,3 +1,10 @@ +/** + * ReadMediaService - Reads local media files for the read_media tool + * + * Encodes image and audio files as base64 data URLs with the metadata the + * model needs. No reactive state; consumed by toolsStore. + */ + import { ToolsService } from './tools.service'; import { FILE_EXTENSION_SEPARATOR, @@ -40,7 +47,7 @@ function fileExtension(path: string): string { * actually use the result - the server has no idea which model is selected. * * @see buildReadMediaToolDefinition in constants/read-media.ts - tool schema sent to the LLM - * @see agenticStore in stores/agentic.svelte.ts - tool dispatch and attachment extraction + * @see agenticStore in stores/agentic/index.svelte.ts - tool dispatch and attachment extraction */ export class ReadMediaService { static async executeTool( diff --git a/tools/ui/src/lib/services/router.service.ts b/tools/ui/src/lib/services/router.service.ts index 59de4cb6f..217de38f3 100644 --- a/tools/ui/src/lib/services/router.service.ts +++ b/tools/ui/src/lib/services/router.service.ts @@ -1,3 +1,10 @@ +/** + * RouterService - Builds app route paths + * + * Returns chat and settings route strings from a single source of truth + * (ROUTES). No state. + */ + import { ROUTES } from '$lib/constants'; export class RouterService { diff --git a/tools/ui/src/lib/services/sandbox-harness.ts b/tools/ui/src/lib/services/sandbox-harness.ts index 189ff59a5..29f9ad2a5 100644 --- a/tools/ui/src/lib/services/sandbox-harness.ts +++ b/tools/ui/src/lib/services/sandbox-harness.ts @@ -1,3 +1,10 @@ +/** + * Sandbox harness - builds the srcdoc document for the sandboxed iframe + * + * Produces the HTML/CSP/worker shim that runs untrusted model code in an + * opaque origin. Consumed by sandbox.service. + */ + import WORKER_SHIM from './sandbox-worker.js?raw'; import { NEWLINE } from '$lib/constants'; diff --git a/tools/ui/src/lib/services/sandbox.service.ts b/tools/ui/src/lib/services/sandbox.service.ts index 27da9d263..bdc63e4ed 100644 --- a/tools/ui/src/lib/services/sandbox.service.ts +++ b/tools/ui/src/lib/services/sandbox.service.ts @@ -1,3 +1,11 @@ +/** + * SandboxService - Runs untrusted code in a sandboxed worker + * + * Executes model-generated code inside a CSP-restricted, opaque-origin + * iframe worker with output and timeout limits. No reactive state; consumed + * by toolsStore for code-execution tools. + */ + import { buildSandboxHarness } from './sandbox-harness'; import { NEWLINE, @@ -8,7 +16,7 @@ import { SANDBOX_TOOL_NAME, SANDBOX_TRUNCATION_NOTICE } from '$lib/constants'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { ToolExecutionResult } from '$lib/types'; /** Cached harnesses keyed by whether nerdamer is included. */ diff --git a/tools/ui/src/lib/services/tools.service.ts b/tools/ui/src/lib/services/tools.service.ts index 2b3a2c0dc..78229756c 100644 --- a/tools/ui/src/lib/services/tools.service.ts +++ b/tools/ui/src/lib/services/tools.service.ts @@ -1,3 +1,10 @@ +/** + * ToolsService - Stateless server tools API layer + * + * Fetches the server's /tools listing and streams tool execution results. + * No reactive state; consumed by toolsStore. + */ + import { base } from '$app/paths'; import { API_TOOLS, HEADERS } from '$lib/constants'; import { ToolResponseField } from '$lib/enums'; @@ -7,15 +14,6 @@ import { getJsonHeaders } from '$lib/utils/api-headers'; import { parseSseJsonStream, type SseJsonEvent } from '$lib/utils/sse'; export class ToolsService { - /** - * Fetch the list of server tools from the server. - * - * @returns Array of tool definitions in OpenAI-compatible format - */ - static async list(): Promise { - return apiFetch(API_TOOLS.LIST); - } - /** * Execute a server tool on the server. * @@ -76,6 +74,15 @@ export class ToolsService { }); } + /** + * Fetch the list of server tools from the server. + * + * @returns Array of tool definitions in OpenAI-compatible format + */ + static async list(): Promise { + return apiFetch(API_TOOLS.LIST); + } + /** * Stream a server tool's output chunks from the server. The server * `POST /tools` endpoint with `{stream: true}` emits `data: {"chunk": "..."}` diff --git a/tools/ui/src/lib/stores/agentic/gates.svelte.ts b/tools/ui/src/lib/stores/agentic/gates.svelte.ts new file mode 100644 index 000000000..6b52fa3af --- /dev/null +++ b/tools/ui/src/lib/stores/agentic/gates.svelte.ts @@ -0,0 +1,208 @@ +/** + * AgenticGates - User interaction gates for the agentic loop + * + * Owns the state the loop waits on between turns: tool permission requests, + * turn-limit continue prompts and queued steering messages. The loop awaits + * requestPermission/requestContinue; the UI resolves them through + * resolvePermission/resolveContinue. Owned by agenticStore, no host coupling. + */ + +import { ToolPermissionDecision } from '$lib/enums'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { permissionsStore } from '$lib/stores/permissions.svelte'; +import { toolsStore } from '$lib/stores/tools.svelte'; +import type { DatabaseMessageExtra, SteeringMessage } from '$lib/types'; +import { SvelteMap } from 'svelte/reactivity'; + +export class AgenticGates { + /** Resolve functions for pending continue Promises; nothing derives from this map */ + private continueResolvers = new SvelteMap void>(); + /** Dedicated reactive state for pending continue requests (turn limit reached) */ + private pendingContinueRequests = new SvelteMap(); + + /** Dedicated reactive state for pending permission requests (ensures immediate UI updates) */ + private pendingPermissions = new SvelteMap< + string, + { toolName: string; serverLabel: string } | null + >(); + /** Resolve functions for pending permission Promises; nothing derives from this map */ + private permissionResolvers = new SvelteMap void>(); + + /** Reactive: queued steering messages to inject between turns */ + private steeringMessages = new SvelteMap(); + + /** + * Drop all pending gate state for a conversation, e.g. when a flow exits. + */ + clear(conversationId: string): void { + this.pendingPermissions.set(conversationId, null); + this.permissionResolvers.delete(conversationId); + this.pendingContinueRequests.set(conversationId, false); + this.continueResolvers.delete(conversationId); + this.steeringMessages.delete(conversationId); + } + + /** + * Clear the pending steering message without consuming it. + */ + clearSteeringMessage(conversationId: string): void { + this.steeringMessages.delete(conversationId); + } + + /** + * Consume and return the pending steering message for re-sending. + * Called by chatStore after the agentic flow exits. + */ + consumePendingSteeringMessage(conversationId: string): SteeringMessage | null { + const msg = this.steeringMessages.get(conversationId); + + if (!msg) return null; + + this.steeringMessages.delete(conversationId); + + return msg; + } + + getPendingContinueRequest(conversationId: string): boolean { + return this.pendingContinueRequests.get(conversationId) ?? false; + } + + getPendingPermissionRequest( + conversationId: string + ): { toolName: string; serverLabel: string } | null { + return this.pendingPermissions.get(conversationId) ?? null; + } + + getPendingSteeringMessageContent(conversationId: string): string | null { + return this.steeringMessages.get(conversationId)?.content ?? null; + } + + getPendingSteeringMessageExtras(conversationId: string): DatabaseMessageExtra[] | undefined { + return this.steeringMessages.get(conversationId)?.extras; + } + + hasPendingSteeringMessage(conversationId: string): boolean { + return this.steeringMessages.has(conversationId); + } + + /** + * Queue a steering message. When the current agentic turn completes, + * the flow exits and the caller re-sends the message as a normal chat message. + */ + injectSteeringMessage( + conversationId: string, + content: string, + extras?: DatabaseMessageExtra[] + ): void { + this.steeringMessages.set(conversationId, { content, extras }); + } + + async requestContinue(conversationId: string, signal?: AbortSignal): Promise { + this.pendingContinueRequests.set(conversationId, true); + + return new Promise((resolve) => { + if (signal?.aborted) { + this.pendingContinueRequests.set(conversationId, false); + resolve(false); + + return; + } + + this.continueResolvers.set(conversationId, (shouldContinue) => { + this.pendingContinueRequests.set(conversationId, false); + resolve(shouldContinue); + }); + + signal?.addEventListener( + 'abort', + () => { + const resolver = this.continueResolvers.get(conversationId); + + if (resolver) { + this.continueResolvers.delete(conversationId); + this.pendingContinueRequests.set(conversationId, false); + resolve(false); + } + }, + { once: true } + ); + }); + } + + async requestPermission( + conversationId: string, + toolName: string, + serverLabel: string, + signal?: AbortSignal + ): Promise { + const permissionKey = toolsStore.getPermissionKey(toolName); + + if (permissionKey && permissionsStore.hasTool(permissionKey)) { + return ToolPermissionDecision.ONCE; + } + + this.pendingPermissions.set(conversationId, { serverLabel, toolName }); + + return new Promise((resolve) => { + if (signal?.aborted) { + this.pendingPermissions.set(conversationId, null); + resolve(ToolPermissionDecision.DENY); + + return; + } + + this.permissionResolvers.set(conversationId, (decision) => { + this.pendingPermissions.set(conversationId, null); + + if (decision === ToolPermissionDecision.ALWAYS && permissionKey) { + permissionsStore.allowTool(permissionKey); + } else if (decision === ToolPermissionDecision.ALWAYS_SERVER) { + const serverToolKeys = toolsStore.allTools + .filter((t) => + t.serverName + ? t.serverName === serverLabel + : toolsStore.getToolServerLabel(t.definition.function.name) === serverLabel + ) + .map((t) => toolsStore.getPermissionKey(t.definition.function.name)!) + .filter((k): k is string => k !== null); + + permissionsStore.allowTools(serverToolKeys); + } + + resolve(decision); + }); + + signal?.addEventListener( + 'abort', + () => { + const resolver = this.permissionResolvers.get(conversationId); + + if (resolver) { + this.permissionResolvers.delete(conversationId); + this.pendingPermissions.set(conversationId, null); + resolve(ToolPermissionDecision.DENY); + } + }, + { once: true } + ); + }); + } + + resolveContinue(conversationId: string, shouldContinue: boolean): void { + const resolver = this.continueResolvers.get(conversationId); + + if (resolver) { + this.continueResolvers.delete(conversationId); + resolver(shouldContinue); + } + } + + resolvePermission(conversationId: string, decision: ToolPermissionDecision): void { + const resolver = this.permissionResolvers.get(conversationId); + + if (resolver) { + this.permissionResolvers.delete(conversationId); + resolver(decision); + } + } +} diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic/index.svelte.ts similarity index 77% rename from tools/ui/src/lib/stores/agentic.svelte.ts rename to tools/ui/src/lib/stores/agentic/index.svelte.ts index d2a2ea887..a91e0ba46 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic/index.svelte.ts @@ -1,23 +1,13 @@ /** - * agenticStore - Reactive State Store for Agentic Loop Orchestration + * AgenticStore - Multi-turn agentic loop orchestration * - * Manages multi-turn agentic loop with MCP tools: - * - LLM streaming with tool call detection - * - Tool execution via mcpStore - * - Session state management - * - Turn limit enforcement + * Drives the agentic loop over MCP tools: streams each LLM turn, detects + * tool calls, executes them via mcpStore, and enforces the turn limit. Each + * turn produces one assistant message (with tool_calls) and one tool result + * message per executed call, persisted as separate DB rows. * - * Each agentic turn produces separate DB messages: - * - One assistant message per LLM turn (with tool_calls if any) - * - One tool result message per tool call execution - * - * **Architecture & Relationships:** - * - **ChatService**: Stateless API layer (sendMessage, streaming) - * - **mcpStore**: MCP connection management and tool execution - * - **agenticStore** (this): Reactive state + business logic - * - * @see ChatService in services/chat.service.ts for API operations - * @see mcpStore in stores/mcp.svelte.ts for MCP operations + * Uses ChatService for streaming and mcpStore for tool execution; waits on + * the permission/continue/steering gates owned by {@link AgenticGates}. */ import { DEFAULT_AGENTIC_CONFIG, NEWLINE } from '$lib/constants'; @@ -43,11 +33,11 @@ import { ReadMediaService } from '$lib/services/read-media.service'; import { SandboxService } from '$lib/services/sandbox.service'; import { ToolsService } from '$lib/services/tools.service'; // direct imports between stores, not via the barrel, to avoid circular deps -import { conversationsStore } from '$lib/stores/conversations.svelte'; -import { mcpStore } from '$lib/stores/mcp.svelte'; -import { modelsStore } from '$lib/stores/models.svelte'; -import { permissionsStore } from '$lib/stores/permissions.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { AgenticGates } from '$lib/stores/agentic/gates.svelte'; +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import { mcpStore } from '$lib/stores/mcp/index.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { toolsStore } from '$lib/stores/tools.svelte'; import type { AgenticConfig, @@ -152,160 +142,45 @@ function toAgenticMessages(messages: ApiChatMessageData[]): AgenticMessage[] { } class AgenticStore { - private _sessions = new SvelteMap(); - /** Dedicated reactive state for pending permission requests (ensures immediate UI updates) */ - private _pendingPermissions = new SvelteMap< - string, - { toolName: string; serverLabel: string } | null - >(); - /** Non-reactive: stores resolve functions for pending permission Promises */ - private _permissionResolvers = new Map void>(); + // permission, continue and steering gates the loop waits on between turns + private gates = new AgenticGates(); + private sessions = new SvelteMap(); - /** Dedicated reactive state for pending continue requests (turn limit reached) */ - private _pendingContinueRequests = new SvelteMap(); - /** Non-reactive: stores resolve functions for pending continue Promises */ - private _continueResolvers = new Map void>(); - - /** Reactive: queued steering messages to inject between turns */ - private _steeringMessages = new SvelteMap(); - - get isReady(): boolean { - return true; - } get isAnyRunning(): boolean { - for (const session of this._sessions.values()) { + for (const session of this.sessions.values()) { if (session.isRunning) return true; } return false; } - getSession(conversationId: string): AgenticSession { - let session = this._sessions.get(conversationId); - - if (!session) { - session = createDefaultSession(); - this._sessions.set(conversationId, session); - } - - return session; - } - - private updateSession(conversationId: string, update: Partial): void { - const session = this.getSession(conversationId); - - this._sessions.set(conversationId, { ...session, ...update }); - } - - clearSession(conversationId: string): void { - this._sessions.delete(conversationId); - } - - getActiveSessions(): Array<{ conversationId: string; session: AgenticSession }> { - const active: Array<{ conversationId: string; session: AgenticSession }> = []; - - for (const [conversationId, session] of this._sessions.entries()) { - if (session.isRunning) active.push({ conversationId, session }); - } - - return active; - } - - isRunning(conversationId: string): boolean { - return this._sessions.get(conversationId)?.isRunning ?? false; - } - - // read-only: safe to call from derivations, unlike getSession - getLiveLlmTotals(conversationId: string): AgenticSession['liveLlm'] { - return this._sessions.get(conversationId)?.liveLlm ?? null; - } - - // read-only: safe to call from derivations, unlike getSession - getFlowRootMessageId(conversationId: string): string | null { - return this._sessions.get(conversationId)?.flowRootMessageId ?? null; - } - - currentTurn(conversationId: string): number { - return this._sessions.get(conversationId)?.currentTurn ?? 0; - } - - totalToolCalls(conversationId: string): number { - return this._sessions.get(conversationId)?.totalToolCalls ?? 0; - } - - lastError(conversationId: string): Error | null { - return this._sessions.get(conversationId)?.lastError ?? null; - } - - streamingToolCall(conversationId: string): { name: string; arguments: string } | null { - return this._sessions.get(conversationId)?.streamingToolCall ?? null; - } - - executingToolCallId(conversationId: string): string | null { - return this._sessions.get(conversationId)?.executingToolCallId ?? null; - } - - pendingPermissionRequest( - conversationId: string - ): { toolName: string; serverLabel: string } | null { - return this._pendingPermissions.get(conversationId) ?? null; - } - - pendingContinueRequest(conversationId: string): boolean { - return this._pendingContinueRequests.get(conversationId) ?? false; - } - - resolveContinue(conversationId: string, shouldContinue: boolean): void { - const resolver = this._continueResolvers.get(conversationId); - - if (resolver) { - this._continueResolvers.delete(conversationId); - resolver(shouldContinue); - } - } - - resolvePermission(conversationId: string, decision: ToolPermissionDecision): void { - const resolver = this._permissionResolvers.get(conversationId); - - if (resolver) { - this._permissionResolvers.delete(conversationId); - resolver(decision); - } + get isReady(): boolean { + return true; } clearError(conversationId: string): void { this.updateSession(conversationId, { lastError: null }); } - hasPendingSteeringMessage(conversationId: string): boolean { - return this._steeringMessages.has(conversationId); - } - - pendingSteeringMessageContent(conversationId: string): string | null { - return this._steeringMessages.get(conversationId)?.content ?? null; - } - - pendingSteeringMessageExtras(conversationId: string): DatabaseMessageExtra[] | undefined { - return this._steeringMessages.get(conversationId)?.extras; - } - - /** - * Queue a steering message. When the current agentic turn completes, - * the flow exits and the caller re-sends the message as a normal chat message. - */ - injectSteeringMessage( - conversationId: string, - content: string, - extras?: DatabaseMessageExtra[] - ): void { - this._steeringMessages.set(conversationId, { content, extras }); + clearSession(conversationId: string): void { + this.sessions.delete(conversationId); } /** * Clear the pending steering message without consuming it. */ clearSteeringMessage(conversationId: string): void { - this._steeringMessages.delete(conversationId); + this.gates.clearSteeringMessage(conversationId); + } + + constructor() { + // drop per-conversation session state when the conversation is deleted, + // otherwise every conversation that ever ran a flow leaks a session here + conversationsStore.onConversationsDeleted((convIds) => { + for (const convId of convIds) { + this.sessions.delete(convId); + } + }); } /** @@ -313,13 +188,17 @@ class AgenticStore { * Called by chatStore after the agentic flow exits. */ consumePendingSteeringMessage(conversationId: string): SteeringMessage | null { - const msg = this._steeringMessages.get(conversationId); + return this.gates.consumePendingSteeringMessage(conversationId); + } - if (!msg) return null; + getActiveSessions(): Array<{ conversationId: string; session: AgenticSession }> { + const active: Array<{ conversationId: string; session: AgenticSession }> = []; - this._steeringMessages.delete(conversationId); + for (const [conversationId, session] of this.sessions.entries()) { + if (session.isRunning) active.push({ conversationId, session }); + } - return msg; + return active; } getConfig(settings: SettingsConfigType, perChatOverrides?: McpServerOverride[]): AgenticConfig { @@ -336,105 +215,91 @@ class AgenticStore { }; } - private parseToolArguments(args: string | Record): Record { - if (typeof args === 'object') return args; - - const trimmed = args.trim(); - - if (trimmed === '') return {}; - - return JSON.parse(trimmed) as Record; + getCurrentTurn(conversationId: string): number { + return this.sessions.get(conversationId)?.currentTurn ?? 0; } - private async requestPermission( - conversationId: string, - toolName: string, - serverLabel: string, - signal?: AbortSignal - ): Promise { - const permissionKey = toolsStore.getPermissionKey(toolName); + getExecutingToolCallId(conversationId: string): string | null { + return this.sessions.get(conversationId)?.executingToolCallId ?? null; + } - if (permissionKey && permissionsStore.hasTool(permissionKey)) { - return ToolPermissionDecision.ONCE; + // read-only: safe to call from derivations, unlike getSession + getFlowRootMessageId(conversationId: string): string | null { + return this.sessions.get(conversationId)?.flowRootMessageId ?? null; + } + + getLastError(conversationId: string): Error | null { + return this.sessions.get(conversationId)?.lastError ?? null; + } + + // read-only: safe to call from derivations, unlike getSession + getLiveLlmTotals(conversationId: string): AgenticSession['liveLlm'] { + return this.sessions.get(conversationId)?.liveLlm ?? null; + } + + getPendingContinueRequest(conversationId: string): boolean { + return this.gates.getPendingContinueRequest(conversationId); + } + + getPendingPermissionRequest( + conversationId: string + ): { toolName: string; serverLabel: string } | null { + return this.gates.getPendingPermissionRequest(conversationId); + } + + getPendingSteeringMessageContent(conversationId: string): string | null { + return this.gates.getPendingSteeringMessageContent(conversationId); + } + + getPendingSteeringMessageExtras(conversationId: string): DatabaseMessageExtra[] | undefined { + return this.gates.getPendingSteeringMessageExtras(conversationId); + } + + getSession(conversationId: string): AgenticSession { + let session = this.sessions.get(conversationId); + + if (!session) { + session = createDefaultSession(); + this.sessions.set(conversationId, session); } - this._pendingPermissions.set(conversationId, { serverLabel, toolName }); - - return new Promise((resolve) => { - if (signal?.aborted) { - this._pendingPermissions.set(conversationId, null); - resolve(ToolPermissionDecision.DENY); - - return; - } - - this._permissionResolvers.set(conversationId, (decision) => { - this._pendingPermissions.set(conversationId, null); - - if (decision === ToolPermissionDecision.ALWAYS && permissionKey) { - permissionsStore.allowTool(permissionKey); - } else if (decision === ToolPermissionDecision.ALWAYS_SERVER) { - const serverToolKeys = toolsStore.allTools - .filter((t) => - t.serverName - ? t.serverName === serverLabel - : toolsStore.getToolServerLabel(t.definition.function.name) === serverLabel - ) - .map((t) => toolsStore.getPermissionKey(t.definition.function.name)!) - .filter((k): k is string => k !== null); - - permissionsStore.allowTools(serverToolKeys); - } - - resolve(decision); - }); - - signal?.addEventListener( - 'abort', - () => { - const resolver = this._permissionResolvers.get(conversationId); - - if (resolver) { - this._permissionResolvers.delete(conversationId); - this._pendingPermissions.set(conversationId, null); - resolve(ToolPermissionDecision.DENY); - } - }, - { once: true } - ); - }); + return session; } - private async requestContinue(conversationId: string, signal?: AbortSignal): Promise { - this._pendingContinueRequests.set(conversationId, true); + getStreamingToolCall(conversationId: string): { name: string; arguments: string } | null { + return this.sessions.get(conversationId)?.streamingToolCall ?? null; + } - return new Promise((resolve) => { - if (signal?.aborted) { - this._pendingContinueRequests.set(conversationId, false); - resolve(false); + getTotalToolCalls(conversationId: string): number { + return this.sessions.get(conversationId)?.totalToolCalls ?? 0; + } - return; - } + hasPendingSteeringMessage(conversationId: string): boolean { + return this.gates.hasPendingSteeringMessage(conversationId); + } - this._continueResolvers.set(conversationId, (shouldContinue) => { - this._pendingContinueRequests.set(conversationId, false); - resolve(shouldContinue); - }); + /** + * Queue a steering message. When the current agentic turn completes, + * the flow exits and the caller re-sends the message as a normal chat message. + */ + injectSteeringMessage( + conversationId: string, + content: string, + extras?: DatabaseMessageExtra[] + ): void { + this.gates.injectSteeringMessage(conversationId, content, extras); + } - signal?.addEventListener( - 'abort', - () => { - const resolver = this._continueResolvers.get(conversationId); + isRunning(conversationId: string): boolean { + return this.sessions.get(conversationId)?.isRunning ?? false; + } - if (resolver) { - this._continueResolvers.delete(conversationId); - this._pendingContinueRequests.set(conversationId, false); - resolve(false); - } - }, - { once: true } - ); - }); + resolveContinue(conversationId: string, shouldContinue: boolean): void { + this.gates.resolveContinue(conversationId, shouldContinue); + } + + resolvePermission(conversationId: string, decision: ToolPermissionDecision): void { + this.gates.resolvePermission(conversationId, decision); } async runAgenticFlow(params: AgenticFlowParams): Promise { @@ -449,11 +314,7 @@ class AgenticStore { } = params; // Clear any pending permissions/continue requests for this conversation when starting a new flow - this._pendingPermissions.set(conversationId, null); - this._permissionResolvers.delete(conversationId); - this._pendingContinueRequests.set(conversationId, false); - this._continueResolvers.delete(conversationId); - this._steeringMessages.delete(conversationId); + this.gates.clear(conversationId); // Ensure server tools are fetched before checking if agentic is enabled if (toolsStore.serverTools.length === 0 && !toolsStore.loading) { @@ -482,26 +343,8 @@ class AgenticStore { console.log(`[AgenticStore] Starting agentic flow with ${tools.length} tools`); - const normalizedMessages: ApiChatMessageData[] = ( - await Promise.all( - messages.map((msg) => { - if ('id' in msg && 'convId' in msg && 'timestamp' in msg) - return ChatService.convertDbMessageToApiChatMessageData( - msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] } - ); - - return msg as ApiChatMessageData; - }) - ) - ).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => { - if (msg.role === MessageRole.SYSTEM) { - const content = typeof msg.content === 'string' ? msg.content : ''; - - return content.trim().length > 0; - } - - return true; - }); + const normalizedMessages: ApiChatMessageData[] = + await ChatService.normalizeMessagesForApi(messages); this.updateSession(conversationId, { currentTurn: 0, @@ -550,6 +393,30 @@ class AgenticStore { } } + private buildAttachmentName(mimeType: string, index: number): string { + const extension = mimeType.startsWith(MimeTypePrefix.AUDIO) + ? (AUDIO_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_AUDIO_EXTENSION) + : (IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION); + + return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; + } + + private buildFinalTimings( + capturedTimings: ChatMessageTimings | undefined, + agenticTimings: ChatMessageAgenticTimings + ): ChatMessageTimings | undefined { + if (agenticTimings.toolCallsCount === 0) return capturedTimings; + + return { + agentic: agenticTimings, + cache_n: capturedTimings?.cache_n, + predicted_ms: capturedTimings?.predicted_ms, + predicted_n: capturedTimings?.predicted_n, + prompt_ms: capturedTimings?.prompt_ms, + prompt_n: capturedTimings?.prompt_n + }; + } + private async executeAgenticLoop(params: { conversationId: string; messages: ApiChatMessageData[]; @@ -596,7 +463,7 @@ class AgenticStore { while (true) { if (turn >= maxTurns) { // Turn limit reached - ask user whether to continue - const shouldContinue = await this.requestContinue(conversationId, signal); + const shouldContinue = await this.gates.requestContinue(conversationId, signal); // Yield to allow Svelte to flush the UI update await new Promise((r) => setTimeout(r, 0)); @@ -769,7 +636,7 @@ class AgenticStore { // === Steering check: if a user message was queued during this turn, exit the flow. // The caller (chatStore) will consume the pending message and re-send it normally. - if (this._steeringMessages.has(conversationId)) { + if (this.gates.hasPendingSteeringMessage(conversationId)) { console.log('[AgenticStore] Steering message detected after turn, exiting agentic flow'); await onAssistantTurnComplete?.( turnContent, @@ -847,7 +714,7 @@ class AgenticStore { } // Check for pending steering message - skip remaining tool calls - if (this._steeringMessages.has(conversationId)) { + if (this.gates.hasPendingSteeringMessage(conversationId)) { console.log( `[AgenticStore] Steering message detected, skipping ${normalizedCalls.length - i} remaining tool call(s)` ); @@ -872,7 +739,7 @@ class AgenticStore { const toolName = toolCall.function.name; const serverLabel = toolsStore.getToolServerLabel(toolName); // Ask for permission before executing the tool - const permission = await this.requestPermission( + const permission = await this.gates.requestPermission( conversationId, toolName, serverLabel, @@ -959,8 +826,8 @@ class AgenticStore { executionResult = await ReadMediaService.executeTool( args, { - audio: modelsStore.modelSupportsAudio(effectiveModel), - vision: modelsStore.modelSupportsVision(effectiveModel) + audio: modelsStore.props.modelSupportsAudio(effectiveModel), + vision: modelsStore.props.modelSupportsVision(effectiveModel) }, signal, conversationsStore.activeConversation?.cwd @@ -1058,7 +925,7 @@ class AgenticStore { for (const attachment of attachments) { if (attachment.type === AttachmentType.AUDIO) { - if (modelsStore.modelSupportsAudio(effectiveModel)) { + if (modelsStore.props.modelSupportsAudio(effectiveModel)) { contentParts.push({ input_audio: { data: (attachment as DatabaseMessageExtraAudioFile).base64Data, @@ -1070,7 +937,7 @@ class AgenticStore { }); } } else if (attachment.type === AttachmentType.IMAGE) { - if (modelsStore.modelSupportsVision(effectiveModel)) { + if (modelsStore.props.modelSupportsVision(effectiveModel)) { contentParts.push({ image_url: { url: (attachment as DatabaseMessageExtraImageFile).base64Url @@ -1101,7 +968,7 @@ class AgenticStore { } // If tools were interrupted by a steering message, exit now instead of starting another LLM turn - if (this._steeringMessages.has(conversationId)) { + if (this.gates.hasPendingSteeringMessage(conversationId)) { console.log( '[AgenticStore] Steering message detected after tool execution, exiting agentic flow' ); @@ -1114,35 +981,6 @@ class AgenticStore { } } - private buildFinalTimings( - capturedTimings: ChatMessageTimings | undefined, - agenticTimings: ChatMessageAgenticTimings - ): ChatMessageTimings | undefined { - if (agenticTimings.toolCallsCount === 0) return capturedTimings; - - return { - agentic: agenticTimings, - cache_n: capturedTimings?.cache_n, - predicted_ms: capturedTimings?.predicted_ms, - predicted_n: capturedTimings?.predicted_n, - prompt_ms: capturedTimings?.prompt_ms, - prompt_n: capturedTimings?.prompt_n - }; - } - - private normalizeToolCalls(toolCalls: ApiChatCompletionToolCall[]): AgenticToolCallList { - if (!toolCalls) return []; - - return toolCalls.map((call, index) => ({ - function: { - arguments: call?.function?.arguments ?? '', - name: call?.function?.name ?? '' - }, - id: call?.id ?? `tool_${index}`, - type: (call?.type as ToolCallType.FUNCTION) ?? ToolCallType.FUNCTION - })); - } - private extractBase64Attachments(result: string): { cleanedResult: string; attachments: DatabaseMessageExtra[]; @@ -1198,12 +1036,33 @@ class AgenticStore { return { attachments, cleanedResult: cleanedLines.join(NEWLINE) }; } - private buildAttachmentName(mimeType: string, index: number): string { - const extension = mimeType.startsWith(MimeTypePrefix.AUDIO) - ? (AUDIO_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_AUDIO_EXTENSION) - : (IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION); + private normalizeToolCalls(toolCalls: ApiChatCompletionToolCall[]): AgenticToolCallList { + if (!toolCalls) return []; - return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; + return toolCalls.map((call, index) => ({ + function: { + arguments: call?.function?.arguments ?? '', + name: call?.function?.name ?? '' + }, + id: call?.id ?? `tool_${index}`, + type: (call?.type as ToolCallType.FUNCTION) ?? ToolCallType.FUNCTION + })); + } + + private parseToolArguments(args: string | Record): Record { + if (typeof args === 'object') return args; + + const trimmed = args.trim(); + + if (trimmed === '') return {}; + + return JSON.parse(trimmed) as Record; + } + + private updateSession(conversationId: string, update: Partial): void { + const session = this.getSession(conversationId); + + this.sessions.set(conversationId, { ...session, ...update }); } } diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts deleted file mode 100644 index b7add7777..000000000 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ /dev/null @@ -1,2868 +0,0 @@ -/** - * chatStore - Reactive State Store for Chat Operations - * - * Manages chat lifecycle, streaming, message operations, and processing state. - * - * **Architecture & Relationships:** - * - **ChatService**: Stateless API layer (sendMessage, streaming) - * - **chatStore** (this): Reactive state + business logic - * - **conversationsStore**: Conversation persistence and navigation - * - * @see ChatService in services/chat.service.ts for API operations - */ - -import { - CONVERSATION_ID_SEPARATOR, - CWD_CLEARED_TEXT, - INACTIVE_CONVERSATION, - STREAM_RESUME_RETRY_MS, - SYSTEM_MESSAGE_PLACEHOLDER, - TITLE_GENERATION -} from '$lib/constants'; -import { - ContinueIntentKind, - ErrorDialogType, - MessageRole, - MessageType, - ReasoningEffort, - StreamConnectionState -} from '$lib/enums'; -import { ChatService } from '$lib/services/chat.service'; -import { DatabaseService } from '$lib/services/database.service'; -// direct imports between stores, not via the barrel, to avoid circular deps -import { agenticStore } from '$lib/stores/agentic.svelte'; -import { conversationsStore } from '$lib/stores/conversations.svelte'; -import { mcpStore } from '$lib/stores/mcp.svelte'; -import { modelsStore } from '$lib/stores/models.svelte'; -import { serverStore } from '$lib/stores/server.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; -import { toolsStore } from '$lib/stores/tools.svelte'; -import type { - ApiChatMessageData, - ApiProcessingState, - ApiStreamSession, - ChatMessagePromptProgress, - ChatMessageTimings, - ChatStreamCallbacks, - DatabaseMessage, - DatabaseMessageExtra, - ErrorDialogState -} from '$lib/types'; -import { - classifyContinueIntent, - filterByLeafNodeId, - findDescendantMessages, - findLeafNode, - findMessageById, - formatCwdMessage, - generateConversationTitle, - getConversationModel, - isAbortError, - normalizeModelName, - streamIdentity -} from '$lib/utils'; -import { SvelteMap, SvelteSet } from 'svelte/reactivity'; - -interface ConversationStateEntry { - lastAccessed: number; -} - -class ChatStore { - activeProcessingState = $state(null); - currentResponse = $state(''); - errorDialogState = $state(null); - isLoading = $state(false); - // true while the active conversation streams reasoning content but no visible content yet - isReasoning = $state(false); - // resumable stream connection state for the active conversation - // streaming -> bytes flowing normally, resuming -> waiting on /v1/stream reconnect, lost -> unrecoverable - streamConnectionState = $state(StreamConnectionState.STREAMING); - chatLoadingStates = new SvelteMap(); - chatReasoningStates = new SvelteMap(); - chatStreamingStates = new SvelteMap< - string, - { response: string; messageId: string; model?: string | null } - >(); - // convs that the backend reports as having a running session, populated by the global sync - // at app mount and on visibilitychange. it does not overlap with chatLoadingStates which - // tracks inferences driven by this browser, both are unioned to feed the sidebar spinners - private remoteRunningConvs = new SvelteSet(); - // per conv attach lifecycle, used to derive the global streaming flag without flipping it - // off when one conv finishes while another is still streaming. mirrors chatLoadingStates - // in scope but tracks the attach + tee replay path specifically - private attachingConvs = new SvelteSet(); - // pending resume retry timers while an owning model loads, one per conv - private resumeRetryTimers = new SvelteMap>(); - // convs whose resume waits on a model load: their loading state belongs to the retry loop, - // so discoverActiveStream must not treat it as a live send and bail - private resumePendingConvs = new SvelteSet(); - // in-flight discoverActiveStream guard, keyed by conv id - private discoveringConvs = new SvelteSet(); - private abortControllers = new SvelteMap(); - private preEncodeAbortController: AbortController | null = null; - private processingStates = new SvelteMap(); - private conversationStateTimestamps = new SvelteMap(); - private activeConversationId = $state(null); - private isStreamingActive = $state(false); - private isEditModeActive = $state(false); - private addFilesHandler: ((files: File[]) => void) | null = $state(null); - pendingEditMessageId = $state(null); - private _pendingDraftMessage = $state(''); - private _pendingDraftFiles = $state([]); - - /** Reactive: queued pending messages for non-agentic streaming */ - private _pendingMessages = new SvelteMap< - string, - { content: string; extras?: DatabaseMessageExtra[] } - >(); - - private setChatLoading(convId: string, loading: boolean): void { - this.touchConversationState(convId); - - if (loading) { - this.chatLoadingStates.set(convId, true); - - if (convId === conversationsStore.activeConversation?.id) this.isLoading = true; - } else { - this.chatLoadingStates.delete(convId); - - if (convId === conversationsStore.activeConversation?.id) this.isLoading = false; - - this.setChatReasoning(convId, false); - // the local pipe is the authoritative observer of session end: when it finishes (clean - // onComplete or explicit Stop), the backend session is finalized too, so we drop the - // sidebar hint for this conv right away instead of waiting for the next visibilitychange - // snapshot. without this the spinner ghosts until the user toggles the tab - this.remoteRunningConvs.delete(convId); - } - } - - private setChatReasoning(convId: string, reasoning: boolean): void { - if (reasoning) { - this.chatReasoningStates.set(convId, true); - - if (convId === conversationsStore.activeConversation?.id) this.isReasoning = true; - } else { - this.chatReasoningStates.delete(convId); - - if (convId === conversationsStore.activeConversation?.id) this.isReasoning = false; - } - } - private setChatStreaming( - convId: string, - response: string, - messageId: string, - model?: string | null - ): void { - this.touchConversationState(convId); - this.chatStreamingStates.set(convId, { - messageId, - model: model ?? this.chatStreamingStates.get(convId)?.model, - response - }); - - if (convId === conversationsStore.activeConversation?.id) this.currentResponse = response; - } - private clearChatStreaming(convId: string, messageId?: string): void { - // session aware: a stale generation must not wipe a newer one's streaming state on the - // same conversation, that would drop the frozen stop identity and stop the wrong session - if (messageId !== undefined) { - const cur = this.chatStreamingStates.get(convId); - - if (cur && cur.messageId !== messageId) return; - } - - this.chatStreamingStates.delete(convId); - - if (convId === conversationsStore.activeConversation?.id) this.currentResponse = ''; - } - private getChatStreamingState( - convId: string - ): { response: string; messageId: string } | undefined { - return this.chatStreamingStates.get(convId); - } - syncLoadingStateForChat(convId: string): void { - this.isLoading = this.chatLoadingStates.get(convId) || false; - this.isReasoning = this.chatReasoningStates.get(convId) || false; - const s = this.chatStreamingStates.get(convId); - - this.currentResponse = s?.response || ''; - this.isStreamingActive = s !== undefined; - this.setActiveProcessingConversation(convId); - - // Sync streaming content to activeMessages so UI displays current content - if (s?.response && s?.messageId) { - const idx = conversationsStore.findMessageIndex(s.messageId); - - if (idx !== -1) { - conversationsStore.updateMessageAtIndex(idx, { content: s.response }); - } - } - } - /** - * Server side stream discovery, split in three pieces: - * - * probeServerStream(convId) -> hits POST /v1/streams/lookup with the conv id, returns the session to attach - * to or null. Pure read, no side effect, no UI lock. Safe to fire in parallel with anything. - * - * attachServerStream(convId) -> flips the spinner immediately, fetches the replay stream - * from byte 0, finds the assistant slot to splice into (creates a placeholder if the conv has - * no assistant message yet, for cross device or fresh local DB cases), and pipes the SSE bytes - * into the message via handleStreamResponse. - * - * discoverActiveStream(convId) -> probe + attach in one call. Used by callers that do not need - * to overlap the probe with other async work. - * - * The mount of the chat page in +page.svelte calls probeServerStream in parallel with - * loadConversation, then attachServerStream once both have settled. This gives the earliest - * possible time to spinner and avoids racing against an empty activeMessages array. - */ - async probeServerStream(convId: string): Promise { - if (!convId) return null; - - let sessions: ApiStreamSession[]; - - try { - sessions = await ChatService.lookupStreamSessions([convId]); - } catch (e) { - console.warn(`probeServerStream failed for conv ${convId}:`, e); - - return null; - } - - return ChatService.selectActiveStream(sessions); - } - - async attachServerStream(convId: string, streamId?: string): Promise { - if (!convId) return; - - if (this.chatStreamingStates.has(convId)) return; - - // flip the spinner immediately, the user sees activity as soon as the conv becomes active. - // the global isStreamingActive flag is derived from attachingConvs.size, so adding here - // turns it on, and removing in unlock only turns it off when this is the last attach - this.setChatLoading(convId, true); - this.attachingConvs.add(convId); - this.setStreamingActive(true); - - // only set the active processing conv if we are looking at it, otherwise a background - // attach would steal the indicator from the conv the user is currently viewing - if (convId === conversationsStore.activeConversation?.id) { - this.setActiveProcessingConversation(convId); - } - - const unlock = () => { - this.attachingConvs.delete(convId); - - // flip the global flag off only when no other conv is still attaching - if (this.attachingConvs.size === 0) { - this.setStreamingActive(false); - } - - this.setChatLoading(convId, false); - this.clearChatStreaming(convId); - }; - // fetch the replay stream from byte 0, rebuild the assistant message from scratch. - // resolve the server side identity, fall back to streamIdentity when the caller does not - // pass a streamId. probeServerStream returns the full id (with ::model suffix when present) - const id = streamId || streamIdentity(convId, modelsStore.selectedModelName); - - let response: Response; - - try { - response = await ChatService.fetchStreamReplay(id); - } catch (e) { - console.error(`attachServerStream replay failed for conv ${convId}:`, e); - unlock(); - - return; - } - - // load the target conversation messages by id, not via the active store. when multiple - // attaches run in parallel the active store may reflect another conv and writing through - // its index mixes content across convs (CoT flicker, message bleed). by going through the - // DB we stay isolated, and only mirror into the active store when the attached conv is - // the one currently displayed - let messages: DatabaseMessage[]; - - try { - messages = await DatabaseService.getConversationMessages(convId); - } catch (e) { - console.error('attachServerStream load messages failed:', e); - unlock(); - - return; - } - - // locate the slot to splice into, create a placeholder assistant message if there is none. - // we use the conv-scoped findLastAssistantIdx helpers, they only depend on the array - let targetIdx = this.findLastAssistantIdx(messages); - - if (targetIdx === -1) { - const lastUserIdx = this.findLastUserIdx(messages); - - if (lastUserIdx === -1) { - console.warn( - `attachServerStream: conv ${convId} has no user or assistant message, cannot splice` - ); - unlock(); - - return; - } - - try { - const placeholder = await DatabaseService.createMessageBranch( - { - children: [], - content: '', - convId, - parent: messages[lastUserIdx].id, - role: MessageRole.ASSISTANT, - timestamp: Date.now(), - toolCalls: '', - type: MessageType.TEXT - } as Omit, - messages[lastUserIdx].id - ); - - messages = [...messages, placeholder]; - targetIdx = messages.length - 1; - - // only push into the active store when this conv is the one displayed right now - if (convId === conversationsStore.activeConversation?.id) { - conversationsStore.addMessageToActive(placeholder); - } - } catch (e) { - console.error('attachServerStream placeholder creation failed:', e); - unlock(); - - return; - } - } - - if (targetIdx === -1) { - unlock(); - - return; - } - - const targetMessage = messages[targetIdx]; - const targetMessageId = targetMessage.id; - // when the assistant slot already has content, the running session is a continue or - // another append flow and its buffer holds only the appended deltas. preserve the prefix - // and let the replay add to it. when the slot is empty the session buffer holds the whole - // message so we wipe and rebuild from byte 0 - const existingContent = targetMessage.content ?? ''; - const existingReasoning = targetMessage.reasoningContent ?? ''; - const isAppendMode = existingContent.length > 0; - // helper: write to the active store only when the attached conv is currently displayed. - // the lookup by message id is robust to reordering of activeMessages, two parallel attaches - // can no longer step on each other's indices - const writeActive = (updates: Partial) => { - if (convId !== conversationsStore.activeConversation?.id) { - return; - } - - const liveIdx = conversationsStore.findMessageIndex(targetMessageId); - - if (liveIdx === -1) return; - - conversationsStore.updateMessageAtIndex(liveIdx, updates); - }; - - if (!isAppendMode) { - writeActive({ content: '', reasoningContent: undefined }); - } - - // extract the model suffix, the resume calls in handleStreamResponse must reuse the model - // the session was tagged with, not the live dropdown - const sepIdx = id.indexOf(CONVERSATION_ID_SEPARATOR); - const attachedModel: string | null = sepIdx === -1 ? null : id.slice(sepIdx + 2); - - this.setChatStreaming(convId, existingContent, targetMessageId, attachedModel); - const abortController = this.getOrCreateAbortController(convId); - - let streamedContent = ''; - let streamedReasoningContent = ''; - - const cleanup = () => { - unlock(); - this.setProcessingState(convId, null); - }; - - try { - await ChatService.handleStreamResponse( - response, - (chunk: string) => { - streamedContent += chunk; - const displayed = isAppendMode ? existingContent + streamedContent : streamedContent; - - writeActive({ content: displayed }); - this.setChatStreaming(convId, displayed, targetMessageId); - }, - async ( - finalContent?: string, - reasoningContent?: string, - timings?: ChatMessageTimings, - toolCalls?: string - ) => { - const streamed = streamedContent || finalContent || ''; - const streamedR = streamedReasoningContent || reasoningContent || ''; - const content = isAppendMode ? existingContent + streamed : streamed; - const reasoning = isAppendMode ? existingReasoning + streamedR : streamedR; - - // the DB write is the source of truth, mirror to the active store only when - // the conv is currently displayed - await DatabaseService.updateMessage(targetMessageId, { - content, - reasoningContent: reasoning || undefined, - timings, - toolCalls: toolCalls || '' - }); - writeActive({ - content, - reasoningContent: reasoning || undefined, - timings - }); - cleanup(); - }, - (err: Error) => { - console.error('attachServerStream pipe error:', err); - cleanup(); - }, - (chunk: string) => { - streamedReasoningContent += chunk; - const displayed = isAppendMode - ? existingReasoning + streamedReasoningContent - : streamedReasoningContent; - - writeActive({ reasoningContent: displayed }); - }, - undefined, - undefined, - undefined, - undefined, - convId, - abortController.signal, - (connState: StreamConnectionState) => { - if (convId === conversationsStore.activeConversation?.id) { - this.streamConnectionState = connState; - } - }, - attachedModel - ); - } catch (e) { - console.error('attachServerStream pipe crashed:', e); - cleanup(); - } - } - - /** - * Model frozen at send time for a stream awaiting resume, from the persisted stream state. - * The load progress indicator targets it after a reload, when the message row has no model - * yet and the dropdown selection may not be restored. - */ - getResumeModel(convId: string): string | null { - return ChatService.getStreamState(convId)?.model ?? null; - } - - async discoverActiveStream(convId: string): Promise { - if (!convId) return; - - if (this.chatStreamingStates.has(convId)) return; - - if (this.chatLoadingStates.get(convId) && !this.resumePendingConvs.has(convId)) return; - - // concurrency guard: another discover may already be running for this conv (typical race - // between mount and visibilitychange on tab switch). a second concurrent fetch on the same - // /v1/stream would duplicate every byte into the DB message, this guard bounces it - if (this.discoveringConvs.has(convId)) return; - - this.discoveringConvs.add(convId); - - try { - // the model is frozen at POST time, rebuild the exact conv::model identity from the - // persisted state so the lookup key matches what the server stored. null means a single - // model conv with no ::suffix, only guess from the dropdown with no persisted state - const localState = ChatService.getStreamState(convId); - const streamId = ChatService.resumeStreamIdentity( - convId, - localState, - modelsStore.selectedModelName - ); - // primary path: ask the server which sessions exist for this identity - const serverTarget = await this.probeServerStream(streamId); - - if (serverTarget) { - // pass the full server side identity (may carry a ::model suffix) so the GET routes - // straight to the owning session, no probe or fan out - await this.attachServerStream(convId, serverTarget.conversation_id); - - return; - } - - // fallback: local state remembers an interrupted byte offset for this conv, the server may - // still have a live session matching that identity (we just lost the bytes mid stream). retry - // with the frozen identity, the server probe inside attachServerStream tells us if it exists - if (!localState) { - return; - } - - // quiet status probe first: a full attach flips the loading UI on every try, probing - // keeps the retry loop invisible while the owning model is still loading (503) - const status = await ChatService.probeResumeStatus(streamId); - - if (status === 503) { - // make the wait visible: the empty assistant row persisted at send time renders - // the processing info, whose model load percentage flows from the models feed - this.resumePendingConvs.add(convId); - this.setChatLoading(convId, true); - - if (!this.resumeRetryTimers.has(convId)) { - this.resumeRetryTimers.set( - convId, - setTimeout(() => { - this.resumeRetryTimers.delete(convId); - void this.discoverActiveStream(convId); - }, STREAM_RESUME_RETRY_MS) - ); - } - - return; - } - - if (this.resumePendingConvs.delete(convId) && status !== 200) { - // the wait is over without a session to attach, drop the visible loading state - this.setChatLoading(convId, false); - } - - if (status === 0) { - // transient network failure, the next mount or visibility change retries - return; - } - - if (status !== 200) { - // the session is gone (stopped, TTL expired), nothing to resume anymore - ChatService.clearStreamState(convId); - - return; - } - - await this.attachServerStream(convId, streamId); - - // if attachServerStream failed (session gone, TTL expired), clear the local state to avoid retrying forever - if (!this.chatStreamingStates.has(convId) && !this.chatLoadingStates.get(convId)) { - ChatService.clearStreamState(convId); - } - } finally { - this.discoveringConvs.delete(convId); - } - } - - private findLastAssistantIdx(messages: DatabaseMessage[]): number { - for (let i = messages.length - 1; i >= 0; i--) { - if (messages[i].role === MessageRole.ASSISTANT) return i; - } - - return -1; - } - - private findLastUserIdx(messages: DatabaseMessage[]): number { - for (let i = messages.length - 1; i >= 0; i--) { - if (messages[i].role === MessageRole.USER) return i; - } - - return -1; - } - - clearUIState(): void { - this.isLoading = false; - this.currentResponse = ''; - this.isStreamingActive = false; - } - - setActiveProcessingConversation(conversationId: string | null): void { - this.activeConversationId = conversationId; - this.activeProcessingState = conversationId - ? this.processingStates.get(conversationId) || null - : null; - } - - getProcessingState(conversationId: string): ApiProcessingState | null { - return this.processingStates.get(conversationId) || null; - } - - private setProcessingState(conversationId: string, state: ApiProcessingState | null): void { - if (state === null) this.processingStates.delete(conversationId); - else this.processingStates.set(conversationId, state); - - if (conversationId === this.activeConversationId) this.activeProcessingState = state; - } - - clearProcessingState(conversationId: string): void { - this.processingStates.delete(conversationId); - - if (conversationId === this.activeConversationId) this.activeProcessingState = null; - } - - getActiveProcessingState(): ApiProcessingState | null { - return this.activeProcessingState; - } - - getCurrentProcessingStateSync(): ApiProcessingState | null { - return this.activeProcessingState; - } - - private setStreamingActive(active: boolean): void { - this.isStreamingActive = active; - } - - isStreaming(): boolean { - return this.isStreamingActive; - } - - private getOrCreateAbortController(convId: string): AbortController { - let c = this.abortControllers.get(convId); - - if (!c || c.signal.aborted) { - c = new AbortController(); - this.abortControllers.set(convId, c); - } - - return c; - } - - private abortRequest(convId?: string): void { - if (convId) { - const c = this.abortControllers.get(convId); - - if (c) { - c.abort(); - this.abortControllers.delete(convId); - } - } else { - for (const c of this.abortControllers.values()) c.abort(); - this.abortControllers.clear(); - } - } - - /** - * Abort the current agentic flow signal without clearing loading state. - * Used by "Send immediately" to force the agentic loop to exit so that - * the pending steering message can be re-sent. - * - * Any tool calls captured mid-stream are dropped before the abort so the - * pending message (or a manual follow-up) does not re-send a half-received - * tool call with invalid JSON arguments to the server. Mirrors what the - * Stop button already does through stopGenerationForChat. - */ - async abortCurrentFlow(convId: string): Promise { - await this.savePartialResponseIfNeeded(convId); - const c = this.abortControllers.get(convId); - - if (c) { - c.abort(); - this.abortControllers.delete(convId); - } - } - - private showErrorDialog(state: ErrorDialogState | null): void { - this.errorDialogState = state; - } - - dismissErrorDialog(): void { - this.errorDialogState = null; - } - - clearEditMode(): void { - this.isEditModeActive = false; - this.addFilesHandler = null; - } - - isEditing(): boolean { - return this.isEditModeActive; - } - - setEditModeActive(handler: (files: File[]) => void): void { - this.isEditModeActive = true; - this.addFilesHandler = handler; - } - - getAddFilesHandler(): ((files: File[]) => void) | null { - return this.addFilesHandler; - } - - clearPendingEditMessageId(): void { - this.pendingEditMessageId = null; - } - - savePendingDraft(message: string, files: ChatUploadedFile[]): void { - this._pendingDraftMessage = message; - this._pendingDraftFiles = [...files]; - } - - consumePendingDraft(): { message: string; files: ChatUploadedFile[] } | null { - if (!this._pendingDraftMessage && this._pendingDraftFiles.length === 0) return null; - - const d = { files: [...this._pendingDraftFiles], message: this._pendingDraftMessage }; - - this._pendingDraftMessage = ''; - this._pendingDraftFiles = []; - - return d; - } - - hasPendingDraft(): boolean { - return Boolean(this._pendingDraftMessage) || this._pendingDraftFiles.length > 0; - } - - getAllLoadingChats(): string[] { - // union of local (this browser is piping) and remote (backend reports a running session - // for this conv but no local pipe yet) sources. the sidebar shows one spinner per entry - const out = new SvelteSet(this.chatLoadingStates.keys()); - - for (const id of this.remoteRunningConvs) { - out.add(id); - } - - return Array.from(out); - } - - getAllStreamingChats(): string[] { - return Array.from(this.chatStreamingStates.keys()); - } - - /** - * Resync the remote running convs set from the backend. Called by the layout at mount and on - * visibilitychange, no polling. A snapshot semantic: the set is replaced wholesale, stale entries - * for sessions that finalized while the browser was elsewhere are dropped naturally. - */ - async syncRemoteRunningStreams(): Promise { - // the conversations store loads from IndexedDB asynchronously, the +layout onMount caller - // fires before that finishes. read ids straight from the DB so the result does not depend - // on the store init race, and the sidebar spinners light up at first paint for every conv - // the user owns even if it has not been hydrated into the store yet - let ids: string[]; - - try { - const all = await DatabaseService.getAllConversations(); - - ids = all.map((c) => c.id).filter((id) => !!id); - } catch (e) { - console.warn('syncRemoteRunningStreams DB read failed:', e); - - return; - } - - // only ask about conv ids the user already owns - if (ids.length === 0) { - for (const id of Array.from(this.remoteRunningConvs)) { - this.remoteRunningConvs.delete(id); - } - - return; - } - - // rebuild the frozen conv::model identity per conv so a session started with a model still - // matches. the server response is mapped back to the bare id below for the sidebar set - const lookupIds = ids.map((id) => - ChatService.resumeStreamIdentity(id, ChatService.getStreamState(id), null) - ); - - let sessions: ApiStreamSession[]; - - try { - sessions = await ChatService.lookupStreamSessions(lookupIds); - } catch (e) { - console.warn('syncRemoteRunningStreams lookup failed:', e); - - return; - } - const running = new SvelteSet(); - - for (const s of sessions) { - if (s && !s.is_done && typeof s.conversation_id === 'string' && s.conversation_id) { - // strip the optional ::model suffix, the sidebar set is keyed by the bare conv id - const sepIdx = s.conversation_id.indexOf(CONVERSATION_ID_SEPARATOR); - const bareId = sepIdx === -1 ? s.conversation_id : s.conversation_id.slice(0, sepIdx); - - running.add(bareId); - } - } - for (const id of Array.from(this.remoteRunningConvs)) { - if (!running.has(id)) { - this.remoteRunningConvs.delete(id); - } - } - for (const id of running) { - this.remoteRunningConvs.add(id); - } - } - - getChatStreaming(convId: string): { response: string; messageId: string } | undefined { - return this.getChatStreamingState(convId); - } - - isChatLoading(convId: string): boolean { - return this.chatLoadingStates.get(convId) || false; - } - - private isChatLoadingInternal(convId: string): boolean { - return this.chatLoadingStates.has(convId) || this.chatStreamingStates.has(convId); - } - - hasPendingMessage(convId: string): boolean { - return this._pendingMessages.has(convId); - } - - pendingMessageContent(convId: string): string | null { - return this._pendingMessages.get(convId)?.content ?? null; - } - - pendingMessageExtras(convId: string): DatabaseMessageExtra[] | undefined { - return this._pendingMessages.get(convId)?.extras; - } - - injectPendingMessage(convId: string, content: string, extras?: DatabaseMessageExtra[]): void { - this._pendingMessages.set(convId, { content, extras }); - } - - clearPendingMessage(convId: string): void { - this._pendingMessages.delete(convId); - } - - consumePendingMessage( - convId: string - ): { content: string; extras?: DatabaseMessageExtra[] } | null { - const msg = this._pendingMessages.get(convId); - - if (!msg) return null; - - this._pendingMessages.delete(convId); - - return msg; - } - - private touchConversationState(convId: string): void { - this.conversationStateTimestamps.set(convId, { lastAccessed: Date.now() }); - } - - cleanupOldConversationStates(activeConversationIds?: string[]): number { - const now = Date.now(); - const activeIdsList = activeConversationIds ?? []; - const preserveIds = this.activeConversationId - ? [...activeIdsList, this.activeConversationId] - : activeIdsList; - const allConvIds = [ - ...new Set([ - ...this.chatLoadingStates.keys(), - ...this.chatStreamingStates.keys(), - ...this.abortControllers.keys(), - ...this.processingStates.keys(), - ...this.conversationStateTimestamps.keys() - ]) - ]; - const cleanupCandidates: Array<{ convId: string; lastAccessed: number }> = []; - - for (const convId of allConvIds) { - if (preserveIds.includes(convId)) continue; - - if (this.chatLoadingStates.get(convId)) continue; - - if (this.chatStreamingStates.has(convId)) continue; - - const ts = this.conversationStateTimestamps.get(convId); - - cleanupCandidates.push({ convId, lastAccessed: ts?.lastAccessed ?? 0 }); - } - cleanupCandidates.sort((a, b) => a.lastAccessed - b.lastAccessed); - let cleanedUp = 0; - - for (const { convId, lastAccessed } of cleanupCandidates) { - if ( - cleanupCandidates.length - cleanedUp > INACTIVE_CONVERSATION.MAX_STATES || - now - lastAccessed > INACTIVE_CONVERSATION.MAX_AGE_MS - ) { - this.cleanupConversationState(convId); - cleanedUp++; - } - } - - return cleanedUp; - } - private cleanupConversationState(convId: string): void { - const c = this.abortControllers.get(convId); - - if (c && !c.signal.aborted) c.abort(); - - this.chatLoadingStates.delete(convId); - this.chatStreamingStates.delete(convId); - this.abortControllers.delete(convId); - this.processingStates.delete(convId); - this.conversationStateTimestamps.delete(convId); - } - getTrackedConversationCount(): number { - return new Set([ - ...this.chatLoadingStates.keys(), - ...this.chatStreamingStates.keys(), - ...this.abortControllers.keys(), - ...this.processingStates.keys() - ]).size; - } - - private getMessageByIdWithRole( - messageId: string, - expectedRole?: MessageRole - ): { message: DatabaseMessage; index: number } | null { - const index = conversationsStore.findMessageIndex(messageId); - - if (index === -1) return null; - - const message = conversationsStore.activeMessages[index]; - - if (expectedRole && message.role !== expectedRole) return null; - - return { index, message }; - } - - async addMessage( - role: MessageRole, - content: string, - type: MessageType = MessageType.TEXT, - parent: string = '-1', - extras?: DatabaseMessageExtra[], - isSynthetic?: boolean - ): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) throw new Error('No active conversation'); - - let parentId: string | null = null; - - if (parent === '-1') { - const am = conversationsStore.activeMessages; - - if (am.length > 0) parentId = am[am.length - 1].id; - else { - const all = await conversationsStore.getConversationMessages(activeConv.id); - const r = all.find((m) => m.parent === null && m.type === 'root'); - - parentId = r ? r.id : await DatabaseService.createRootMessage(activeConv.id); - } - } else parentId = parent; - - const message = await DatabaseService.createMessageBranch( - { - children: [], - content, - convId: activeConv.id, - extra: extras, - isSynthetic, - role, - timestamp: Date.now(), - toolCalls: '', - type - }, - parentId - ); - - conversationsStore.addMessageToActive(message); - await conversationsStore.updateCurrentNode(message.id); - conversationsStore.updateConversationTimestamp(); - - return message; - } - - /** - * Record a working-directory change into chat history as a synthetic - * user message, so the model sees it on its next turn (the client - * sends the cwd itself via the x-tool-cwd header on tool calls). - * A plain user message is used because some chat templates reject - * tool messages without a preceding tool call. - */ - async recordCwdChange(cwd: string | null): Promise { - const content = cwd - ? formatCwdMessage(cwd, await toolsStore.resolveServerHome()) - : CWD_CLEARED_TEXT; - // Reuse the trailing cwd row when it is already the last message, so - // repeated picks update it in place instead of stacking another row. - const last = conversationsStore.activeMessages[conversationsStore.activeMessages.length - 1]; - - if (last && last.role === MessageRole.USER && last.isSynthetic === true) { - await DatabaseService.updateMessage(last.id, { content, isSynthetic: true }); - conversationsStore.updateMessageAtIndex(conversationsStore.activeMessages.length - 1, { - content, - isSynthetic: true - }); - - return; - } - - await this.addMessage(MessageRole.USER, content, MessageType.TEXT, '-1', undefined, true); - } - - async addSystemPrompt(): Promise { - let activeConv = conversationsStore.activeConversation; - - if (!activeConv) { - await conversationsStore.createConversation(); - activeConv = conversationsStore.activeConversation; - } - - if (!activeConv) return; - - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); - const rootId = rootMessage - ? rootMessage.id - : await DatabaseService.createRootMessage(activeConv.id); - const existingSystemMessage = allMessages.find( - (m) => m.role === MessageRole.SYSTEM && m.parent === rootId - ); - - if (existingSystemMessage) { - this.pendingEditMessageId = existingSystemMessage.id; - - if (!conversationsStore.activeMessages.some((m) => m.id === existingSystemMessage.id)) - conversationsStore.activeMessages.unshift(existingSystemMessage); - - return; - } - - const am = conversationsStore.activeMessages; - const firstActiveMessage = am.find((m) => m.parent === rootId); - const systemMessage = await DatabaseService.createSystemMessage( - activeConv.id, - SYSTEM_MESSAGE_PLACEHOLDER, - rootId - ); - - if (firstActiveMessage) { - await DatabaseService.updateMessage(firstActiveMessage.id, { - parent: systemMessage.id - }); - await DatabaseService.updateMessage(systemMessage.id, { - children: [firstActiveMessage.id] - }); - const updatedRootChildren = rootMessage - ? rootMessage.children.filter((id: string) => id !== firstActiveMessage.id) - : []; - - await DatabaseService.updateMessage(rootId, { - children: [ - ...updatedRootChildren.filter((id: string) => id !== systemMessage.id), - systemMessage.id - ] - }); - const firstMsgIndex = conversationsStore.findMessageIndex(firstActiveMessage.id); - - if (firstMsgIndex !== -1) - conversationsStore.updateMessageAtIndex(firstMsgIndex, { - parent: systemMessage.id - }); - } - - conversationsStore.activeMessages.unshift(systemMessage); - this.pendingEditMessageId = systemMessage.id; - conversationsStore.updateConversationTimestamp(); - } catch (error) { - console.error('Failed to add system prompt:', error); - } - } - - async removeSystemPromptPlaceholder(messageId: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return false; - - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const systemMessage = findMessageById(allMessages, messageId); - - if (!systemMessage || systemMessage.role !== MessageRole.SYSTEM) return false; - - const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); - - if (!rootMessage) return false; - - if (allMessages.length === 2 && systemMessage.children.length === 0) { - await conversationsStore.deleteConversation(activeConv.id); - - return true; - } - - for (const childId of systemMessage.children) { - await DatabaseService.updateMessage(childId, { parent: rootMessage.id }); - const childIndex = conversationsStore.findMessageIndex(childId); - - if (childIndex !== -1) - conversationsStore.updateMessageAtIndex(childIndex, { parent: rootMessage.id }); - } - await DatabaseService.updateMessage(rootMessage.id, { - children: [ - ...rootMessage.children.filter((id: string) => id !== messageId), - ...systemMessage.children - ] - }); - await DatabaseService.deleteMessage(messageId); - const systemIndex = conversationsStore.findMessageIndex(messageId); - - if (systemIndex !== -1) conversationsStore.activeMessages.splice(systemIndex, 1); - - conversationsStore.updateConversationTimestamp(); - - return false; - } catch (error) { - console.error('Failed to remove system prompt placeholder:', error); - - return false; - } - } - - private async createAssistantMessage(parentId?: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) throw new Error('No active conversation'); - - return await DatabaseService.createMessageBranch( - { - children: [], - content: '', - convId: activeConv.id, - model: null, - role: MessageRole.ASSISTANT, - timestamp: Date.now(), - toolCalls: '', - type: MessageType.TEXT - }, - parentId || null - ); - } - - async sendMessage(content: string, extras?: DatabaseMessageExtra[]): Promise { - if (!content.trim() && (!extras || extras.length === 0)) return; - - const activeConv = conversationsStore.activeConversation; - - // If agentic loop is running, inject as a steering message instead of starting a new flow - if (activeConv && agenticStore.isRunning(activeConv.id)) { - agenticStore.injectSteeringMessage(activeConv.id, content, extras); - - return; - } - - // If non-agentic streaming is active, queue as a pending message to send after completion - if (activeConv && this.isChatLoadingInternal(activeConv.id)) { - this.injectPendingMessage(activeConv.id, content, extras); - - return; - } - - // Cancel any in-flight pre-encode request - this.cancelPreEncode(); - - // Consume MCP resource attachments - converts them to extras and clears the live store - const resourceExtras = mcpStore.consumeResourceAttachmentsAsExtras(); - const allExtras = resourceExtras.length > 0 ? [...(extras || []), ...resourceExtras] : extras; - - let isNewConversation = false; - - if (!activeConv) { - await conversationsStore.createConversation(); - isNewConversation = true; - } - - const currentConv = conversationsStore.activeConversation; - - if (!currentConv) return; - - this.showErrorDialog(null); - this.setChatLoading(currentConv.id, true); - this.clearChatStreaming(currentConv.id); - try { - let parentIdForUserMessage: string | undefined; - - if (isNewConversation) { - const rootId = await DatabaseService.createRootMessage(currentConv.id); - const currentConfig = settingsStore.config; - const systemPrompt = currentConfig.systemMessage?.toString().trim(); - - let sysOrRootId = rootId; - - if (systemPrompt) { - const systemMessage = await DatabaseService.createSystemMessage( - currentConv.id, - systemPrompt, - rootId - ); - - conversationsStore.addMessageToActive(systemMessage); - sysOrRootId = systemMessage.id; - } - - // Reflect a working directory picked on the new-chat screen into - // chat history before the first user message, so the model sees - // it on its first turn. createConversation() has already threaded - // the pending pick onto the conversation. - if (currentConv.cwd) { - const cwdMessage = await this.addMessage( - MessageRole.USER, - formatCwdMessage(currentConv.cwd, await toolsStore.resolveServerHome()), - MessageType.TEXT, - sysOrRootId, - undefined, - true - ); - - parentIdForUserMessage = cwdMessage.id; - } else { - parentIdForUserMessage = sysOrRootId; - } - } - - const userMessage = await this.addMessage( - MessageRole.USER, - content, - MessageType.TEXT, - parentIdForUserMessage ?? '-1', - allExtras - ); - - if (isNewConversation && content) - await conversationsStore.updateConversationName( - currentConv.id, - generateConversationTitle( - content, - Boolean(settingsStore.config.titleGenerationUseFirstLine) - ) - ); - - const assistantMessage = await this.createAssistantMessage(userMessage.id); - - conversationsStore.addMessageToActive(assistantMessage); - await this.streamChatCompletion( - conversationsStore.activeMessages.slice(0, -1), - assistantMessage, - undefined, - undefined, - undefined, - settingsStore.config.titleGenerationUseLLM && isNewConversation ? content : undefined - ); - } catch (error) { - if (isAbortError(error)) { - this.setChatLoading(currentConv.id, false); - - return; - } - - console.error('Failed to send message:', error); - this.setChatLoading(currentConv.id, false); - const dialogType = - error instanceof Error && error.name === 'TimeoutError' - ? ErrorDialogType.TIMEOUT - : ErrorDialogType.SERVER; - const contextInfo = ( - error as Error & { contextInfo?: { n_prompt_tokens: number; n_ctx: number } } - ).contextInfo; - - this.showErrorDialog({ - contextInfo, - message: error instanceof Error ? error.message : 'Unknown error', - type: dialogType - }); - } - } - - private async streamChatCompletion( - allMessages: DatabaseMessage[], - assistantMessage: DatabaseMessage, - onComplete?: (content: string) => Promise, - onError?: (error: Error) => void, - modelOverride?: string | null, - firstUserMessageContent?: string - ): Promise { - // the ::model suffix in the stream identity is only for router mode, where it routes to the - // owning child. in single-model mode the identity stays the bare conv id so that attach, stop - // and reattach all agree, regardless of fresh send vs regenerate passing a resolved model - let effectiveModel: string | null | undefined = undefined; - - if (serverStore.isRouterMode) { - const conversationModel = getConversationModel(allMessages); - - effectiveModel = modelOverride || modelsStore.selectedModelName || conversationModel; - } - - if (serverStore.isRouterMode && effectiveModel) { - if (!modelsStore.getModelProps(effectiveModel)) - await modelsStore.fetchModelProps(effectiveModel); - } - - // Mutable state for the current message being streamed - let currentMessageId = assistantMessage.id; - let streamedContent = ''; - let streamedReasoningContent = ''; - let resolvedModel: string | null = null; - let modelPersisted = false; - - const convId = assistantMessage.convId; - - // Tracks the last message created in this flow. Used as the parent for the next - // turn's assistant message so createAssistantMessage does not have to read - // conversationsStore.activeMessages, which may belong to a different conversation - // after the user navigates while the loop is still running. - let lastCreatedInFlow = currentMessageId; - - // freeze the POST identity from t0 so a stop cancels with the exact session key, - // never a stale or empty model resolved later - this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); - - const recordModel = (modelName: string | null | undefined, persistImmediately = true): void => { - if (!modelName) return; - - const n = normalizeModelName(modelName); - - if (!n || n === resolvedModel) return; - - resolvedModel = n; - const idx = conversationsStore.findMessageIndex(currentMessageId); - - conversationsStore.updateMessageAtIndex(idx, { model: n }); - - if (persistImmediately && !modelPersisted) { - modelPersisted = true; - DatabaseService.updateMessage(currentMessageId, { model: n }).catch(() => { - modelPersisted = false; - resolvedModel = null; - }); - } - }; - - let completionIdRecorded = false; - - const recordCompletionId = (id: string): void => { - if (!id || completionIdRecorded) return; - - completionIdRecorded = true; - const idx = conversationsStore.findMessageIndex(currentMessageId); - - conversationsStore.updateMessageAtIndex(idx, { completionId: id }); - DatabaseService.updateMessage(currentMessageId, { completionId: id }).catch(() => { - completionIdRecorded = false; - }); - }; - const updateStreamingUI = () => { - this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); - const idx = conversationsStore.findMessageIndex(currentMessageId); - - conversationsStore.updateMessageAtIndex(idx, { content: streamedContent }); - }; - const cleanupStreamingState = () => { - this.setStreamingActive(false); - this.setChatLoading(convId, false); - this.clearChatStreaming(convId, currentMessageId); - this.setProcessingState(convId, null); - }; - - this.setStreamingActive(true); - this.setActiveProcessingConversation(convId); - const abortController = this.getOrCreateAbortController(convId); - const streamCallbacks: ChatStreamCallbacks = { - createAssistantMessage: async () => { - // Reset streaming state for new message - streamedContent = ''; - streamedReasoningContent = ''; - - const msg = await DatabaseService.createMessageBranch( - { - children: [], - content: '', - convId, - model: resolvedModel, - role: MessageRole.ASSISTANT, - timestamp: Date.now(), - toolCalls: '', - type: MessageType.TEXT - }, - lastCreatedInFlow - ); - - if (conversationsStore.activeConversation?.id === convId) { - conversationsStore.addMessageToActive(msg); - } - - currentMessageId = msg.id; - lastCreatedInFlow = msg.id; - - return msg; - }, - createToolResultMessage: async ( - toolCallId: string, - content: string, - extras?: DatabaseMessageExtra[], - toolCwd?: string - ) => { - const msg = await DatabaseService.createMessageBranch( - { - children: [], - content, - convId, - extra: extras, - role: MessageRole.TOOL, - timestamp: Date.now(), - toolCallId, - toolCalls: '', - toolCwd, - type: MessageType.TEXT - }, - currentMessageId - ); - - // mirror into the active store and move the node pointer only when this - // conversation is displayed; otherwise persist the node move straight to - // the db for the owning conv so a foreign conv's currNode stays untouched - if (conversationsStore.activeConversation?.id === convId) { - conversationsStore.addMessageToActive(msg); - await conversationsStore.updateCurrentNode(msg.id); - } else { - await DatabaseService.updateCurrentNode(convId, msg.id); - } - - lastCreatedInFlow = msg.id; - - return msg; - }, - onAssistantTurnComplete: async ( - content: string, - reasoningContent: string | undefined, - timings: ChatMessageTimings | undefined, - toolCalls: import('$lib/types/api').ApiChatCompletionToolCall[] | undefined - ) => { - const updateData: Record = { - content, - reasoningContent: reasoningContent || undefined, - timings, - toolCalls: toolCalls ? JSON.stringify(toolCalls) : '' - }; - - if (resolvedModel && !modelPersisted) updateData.model = resolvedModel; - - await DatabaseService.updateMessage(currentMessageId, updateData); - const idx = conversationsStore.findMessageIndex(currentMessageId); - const uiUpdate: Partial = { - content, - reasoningContent: reasoningContent || undefined, - toolCalls: toolCalls ? JSON.stringify(toolCalls) : '' - }; - - if (timings) uiUpdate.timings = timings; - - if (resolvedModel) uiUpdate.model = resolvedModel; - - // touch the active ui array and node pointer only when this conversation - // is displayed; otherwise persist the node move straight to the db so a - // foreign conv's currNode stays untouched - if (conversationsStore.activeConversation?.id === convId) { - conversationsStore.updateMessageAtIndex(idx, uiUpdate); - await conversationsStore.updateCurrentNode(currentMessageId); - } else { - await DatabaseService.updateCurrentNode(convId, currentMessageId); - } - }, - onAttachments: (messageId: string, extras: DatabaseMessageExtra[]) => { - if (!extras.length) return; - - const idx = conversationsStore.findMessageIndex(messageId); - - if (idx === -1) return; - - const msg = conversationsStore.activeMessages[idx]; - const updatedExtras = [...(msg.extra || []), ...extras]; - - conversationsStore.updateMessageAtIndex(idx, { extra: updatedExtras }); - DatabaseService.updateMessage(messageId, { extra: updatedExtras }).catch(console.error); - }, - onChunk: (chunk: string) => { - streamedContent += chunk; - updateStreamingUI(); - this.setChatReasoning(convId, false); - }, - onCompletionId: (id: string) => recordCompletionId(id), - onError: async (error: Error) => { - this.setStreamingActive(false); - - if (isAbortError(error)) { - cleanupStreamingState(); - // If aborted with a pending message (e.g. "Send immediately"), re-send it - const pending = this.consumePendingMessage(convId); - - if (pending) { - this.sendMessage(pending.content, pending.extras); - } - - return; - } - - console.error('Streaming error:', error); - // keep whatever was streamed so far, the message stays in memory and in DB - await this.savePartialResponseIfNeeded(convId); - cleanupStreamingState(); - this.clearPendingMessage(convId); - - const contextInfo = ( - error as Error & { contextInfo?: { n_prompt_tokens: number; n_ctx: number } } - ).contextInfo; - - this.showErrorDialog({ - contextInfo, - message: error.message, - type: error.name === 'TimeoutError' ? ErrorDialogType.TIMEOUT : ErrorDialogType.SERVER - }); - - if (onError) onError(error); - }, - onFlowComplete: (finalTimings?: ChatMessageTimings) => { - if (finalTimings) { - const idx = conversationsStore.findMessageIndex(assistantMessage.id); - - conversationsStore.updateMessageAtIndex(idx, { timings: finalTimings }); - DatabaseService.updateMessage(assistantMessage.id, { - timings: finalTimings - }).catch(console.error); - } - - cleanupStreamingState(); - - if (onComplete) onComplete(streamedContent); - - if (serverStore.isRouterMode) modelsStore.fetchRouterModels().catch(console.error); - - // Pre-encode conversation in KV cache for faster next turn - if (settingsStore.config.preEncodeConversation) { - this.triggerPreEncode( - allMessages, - assistantMessage, - streamedContent, - effectiveModel, - !!settingsStore.config.excludeReasoningFromContext - ); - } - }, - onModel: (modelName: string) => recordModel(modelName), - onReasoningChunk: (chunk: string) => { - streamedReasoningContent += chunk; - // mark streaming state so a stop mid-thinking can persist the partial reasoning - this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); - const idx = conversationsStore.findMessageIndex(currentMessageId); - - conversationsStore.updateMessageAtIndex(idx, { - reasoningContent: streamedReasoningContent - }); - this.setChatReasoning(convId, true); - }, - onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { - const tokensPerSecond = - timings?.predicted_ms && timings?.predicted_n - ? (timings.predicted_n / timings.predicted_ms) * 1000 - : 0; - - this.updateProcessingStateFromTimings( - { - cache_n: timings?.cache_n || 0, - predicted_n: timings?.predicted_n || 0, - predicted_per_second: tokensPerSecond, - prompt_ms: timings?.prompt_ms, - prompt_n: timings?.prompt_n || 0, - prompt_progress: promptProgress - }, - convId - ); - }, - onToolCallsStreaming: (toolCalls) => { - const idx = conversationsStore.findMessageIndex(currentMessageId); - - conversationsStore.updateMessageAtIndex(idx, { - toolCalls: JSON.stringify(toolCalls) - }); - }, - onTurnComplete: (intermediateTimings: ChatMessageTimings) => { - // Update the first assistant message with cumulative agentic timings - const idx = conversationsStore.findMessageIndex(assistantMessage.id); - - conversationsStore.updateMessageAtIndex(idx, { timings: intermediateTimings }); - }, - updateToolResultMessage: async ( - messageId: string, - content: string, - extras?: DatabaseMessageExtra[] - ) => { - // Persist latest content + merged extras; mirror into the active - // store so the chat view sees live updates for streaming tools - // (e.g. exec_shell_command). The existing tool message node - // pointer stays put - the renderer is already scoped to it. - const updates: Partial = { content }; - - if (extras) { - const idx = conversationsStore.findMessageIndex(messageId); - const existing = idx >= 0 ? (conversationsStore.activeMessages[idx]?.extra ?? []) : []; - const merged = [...existing, ...extras]; - - updates.extra = merged; - } - - if (conversationsStore.activeConversation?.id === convId) { - const idx = conversationsStore.findMessageIndex(messageId); - - if (idx >= 0) conversationsStore.updateMessageAtIndex(idx, updates); - } - - await DatabaseService.updateMessage(messageId, updates); - } - }; - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); - - { - const agenticResult = await agenticStore.runAgenticFlow({ - callbacks: streamCallbacks, - conversationId: convId, - flowRootMessageId: assistantMessage.id, - messages: allMessages, - options: { - ...this.getApiOptions(), - ...(effectiveModel ? { model: effectiveModel } : {}) - }, - perChatOverrides, - signal: abortController.signal - }); - - if (agenticResult.handled) { - // Generate LLM based title for new conversations after agentic flow completes - if (firstUserMessageContent) { - await this.generateTitleWithLLM(firstUserMessageContent, streamedContent, convId); - } - - // Check if there's a pending steering message to re-send - const pending = agenticStore.consumePendingSteeringMessage(convId); - - if (pending) { - await this.sendMessage(pending.content, pending.extras); - } - - return; - } - } - - await ChatService.sendMessage( - allMessages, - { - ...this.getApiOptions(), - ...(effectiveModel ? { model: effectiveModel } : {}), - onChunk: streamCallbacks.onChunk, - onComplete: async ( - finalContent?: string, - reasoningContent?: string, - timings?: ChatMessageTimings, - toolCalls?: string - ) => { - const content = streamedContent || finalContent || ''; - const reasoning = streamedReasoningContent || reasoningContent; - const updateData: Record = { - content, - reasoningContent: reasoning || undefined, - timings, - toolCalls: toolCalls || '' - }; - - if (resolvedModel && !modelPersisted) updateData.model = resolvedModel; - - await DatabaseService.updateMessage(currentMessageId, updateData); - const idx = conversationsStore.findMessageIndex(currentMessageId); - const uiUpdate: Partial = { - content, - reasoningContent: reasoning || undefined, - toolCalls: toolCalls || '' - }; - - if (timings) uiUpdate.timings = timings; - - if (resolvedModel) uiUpdate.model = resolvedModel; - - conversationsStore.updateMessageAtIndex(idx, uiUpdate); - await conversationsStore.updateCurrentNode(currentMessageId); - cleanupStreamingState(); - - if (onComplete) await onComplete(content); - - if (serverStore.isRouterMode) modelsStore.fetchRouterModels().catch(console.error); - - // Generate LLM based title for new conversations (avoids stale reference - // issue when user switches conversations while streaming) - if (firstUserMessageContent) { - await this.generateTitleWithLLM(firstUserMessageContent, streamedContent, convId); - } - - // Check if there's a pending message queued during streaming - const pending = this.consumePendingMessage(convId); - - if (pending) { - await this.sendMessage(pending.content, pending.extras); - } - }, - onCompletionId: streamCallbacks.onCompletionId, - onConnectionState: (state: StreamConnectionState) => { - if (convId === conversationsStore.activeConversation?.id) { - this.streamConnectionState = state; - } - }, - onError: streamCallbacks.onError, - onModel: streamCallbacks.onModel, - onReasoningChunk: streamCallbacks.onReasoningChunk, - onTimings: streamCallbacks.onTimings, - stream: true - }, - convId, - abortController.signal - ); - } - - async stopGeneration(): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return; - - await this.stopGenerationForChat(activeConv.id); - } - async stopGenerationForChat(convId: string): Promise { - await this.savePartialResponseIfNeeded(convId); - this.setStreamingActive(false); - // tell the server to stop the generation, not just drop the HTTP socket. without this the - // detached drain keeps producing tokens until eos or max_tokens. use the frozen identity - // captured when the session started, not the live dropdown - const streamStateForStop = this.chatStreamingStates.get(convId); - const modelForStop = streamStateForStop?.model ?? ChatService.getStreamState(convId)?.model; - - void ChatService.cancelServerStream(convId, modelForStop); - // an explicit stop leaves nothing to resume and kills a pending resume retry - ChatService.clearStreamState(convId); - const retryTimer = this.resumeRetryTimers.get(convId); - - if (retryTimer !== undefined) { - clearTimeout(retryTimer); - this.resumeRetryTimers.delete(convId); - } - - this.resumePendingConvs.delete(convId); - this.abortRequest(convId); - this.setChatLoading(convId, false); - this.clearChatStreaming(convId); - this.setProcessingState(convId, null); - this.clearPendingMessage(convId); - } - - private async generateTitleWithLLM( - userContent: string, - assistantContent: string, - convId: string - ): Promise { - const effectiveModel = - serverStore.isRouterMode && modelsStore.selectedModelName - ? modelsStore.selectedModelName - : undefined; - const configValue = settingsStore.config; - const titlePromptTemplate = - typeof configValue.titleGenerationPrompt === 'string' && - configValue.titleGenerationPrompt.trim() - ? configValue.titleGenerationPrompt - : TITLE_GENERATION.DEFAULT_PROMPT; - const titlePrompt = titlePromptTemplate - .replace('{{USER}}', String(userContent || '')) - .replace('{{ASSISTANT}}', String(assistantContent || '')); - const titleMessage: ApiChatMessageData = { - content: titlePrompt, - role: MessageRole.USER - }; - const titleResponse = await ChatService.generateTitle(titleMessage, effectiveModel); - - if (!titleResponse) { - return; - } - - let cleanTitle = titleResponse.trim(); - - cleanTitle = cleanTitle - .replace(TITLE_GENERATION.PREFIX_PATTERN, '') - .replace(TITLE_GENERATION.QUOTE_PATTERN, '') - .trim(); - - if (!cleanTitle || cleanTitle.length < TITLE_GENERATION.MIN_LENGTH) { - const firstLine = userContent.split('\n').find((l) => l.trim().length > 0); - - cleanTitle = firstLine ? firstLine.trim() : TITLE_GENERATION.FALLBACK; - } - - if (cleanTitle && cleanTitle.length >= TITLE_GENERATION.MIN_LENGTH) { - await conversationsStore.updateConversationName(convId, cleanTitle); - } - } - - private async savePartialResponseIfNeeded(convId?: string): Promise { - const conversationId = convId || conversationsStore.activeConversation?.id; - - if (!conversationId) return; - - const streamingState = this.getChatStreamingState(conversationId); - - if (!streamingState) return; - - const messages = - conversationId === conversationsStore.activeConversation?.id - ? conversationsStore.activeMessages - : await conversationsStore.getConversationMessages(conversationId); - - if (!messages.length) return; - - const lastMessage = messages[messages.length - 1]; - - if (lastMessage?.role !== MessageRole.ASSISTANT) return; - - const partialContent = streamingState.response; - const partialReasoning = lastMessage.reasoningContent || ''; - // snapshot the streamed tool calls before clearing so we still know whether - // anything was captured when deciding to skip the DB write below - const hadPartialToolCalls = !!lastMessage.toolCalls?.trim(); - - // nothing to persist when content, reasoning, and streamed tool calls are all empty - // (e.g. stop before any token). otherwise drop the partial tool call and write whatever - // was streamed: incomplete arguments (truncated JSON, missing closing quote) would - // otherwise be re-sent to the server on the next turn and rejected. - if (!partialContent.trim() && !partialReasoning.trim() && !hadPartialToolCalls) return; - - try { - const updateData: { - content?: string; - reasoningContent?: string; - toolCalls?: string; - timings?: ChatMessageTimings; - } = { - toolCalls: '' - }; - - if (partialContent.trim()) updateData.content = partialContent; - - if (partialReasoning.trim()) updateData.reasoningContent = partialReasoning; - - const lastKnownState = this.getProcessingState(conversationId); - - if (lastKnownState) { - updateData.timings = { - cache_n: lastKnownState.cacheTokens || 0, - predicted_ms: - lastKnownState.tokensPerSecond && lastKnownState.tokensDecoded - ? (lastKnownState.tokensDecoded / lastKnownState.tokensPerSecond) * 1000 - : undefined, - predicted_n: lastKnownState.tokensDecoded || 0, - prompt_ms: lastKnownState.promptMs, - prompt_n: lastKnownState.promptTokens || 0 - }; - } - - await DatabaseService.updateMessage(lastMessage.id, updateData); - lastMessage.content = partialContent; - // mirror the drop into the in-memory message so the next request sent via - // sendMessage (queued pending, Send immediately, or manual follow-up) reads - // the cleared value, not whatever the streaming widget had been showing - lastMessage.toolCalls = ''; - - if (updateData.timings) lastMessage.timings = updateData.timings; - } catch (error) { - lastMessage.content = partialContent; - lastMessage.toolCalls = ''; - console.error('Failed to save partial response:', error); - } - } - - async updateMessage(messageId: string, newContent: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return; - - if (this.isChatLoadingInternal(activeConv.id)) await this.stopGeneration(); - - const result = this.getMessageByIdWithRole(messageId, MessageRole.USER); - - if (!result) return; - - const { index: messageIndex, message: messageToUpdate } = result; - const originalContent = messageToUpdate.content; - - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); - const isFirstUserMessage = rootMessage && messageToUpdate.parent === rootMessage.id; - - conversationsStore.updateMessageAtIndex(messageIndex, { content: newContent }); - await DatabaseService.updateMessage(messageId, { content: newContent }); - - if (isFirstUserMessage && newContent.trim()) - await conversationsStore.updateConversationName( - activeConv.id, - generateConversationTitle( - newContent, - Boolean(settingsStore.config.titleGenerationUseFirstLine) - ) - ); - - const messagesToRemove = conversationsStore.activeMessages.slice(messageIndex + 1); - - if (messagesToRemove.length > 0) - await DatabaseService.deleteMessageCascading(activeConv.id, messagesToRemove[0].id); - - conversationsStore.sliceActiveMessages(messageIndex + 1); - conversationsStore.updateConversationTimestamp(); - this.setChatLoading(activeConv.id, true); - this.clearChatStreaming(activeConv.id); - const assistantMessage = await this.createAssistantMessage(); - - conversationsStore.addMessageToActive(assistantMessage); - await conversationsStore.updateCurrentNode(assistantMessage.id); - await this.streamChatCompletion( - conversationsStore.activeMessages.slice(0, -1), - assistantMessage, - undefined, - () => { - conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(messageId), { - content: originalContent - }); - } - ); - } catch (error) { - if (!isAbortError(error)) console.error('Failed to update message:', error); - } - } - - async regenerateMessage(messageId: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv || this.isChatLoadingInternal(activeConv.id)) return; - - this.cancelPreEncode(); - const result = this.getMessageByIdWithRole(messageId, MessageRole.ASSISTANT); - - if (!result) return; - - const { index: messageIndex } = result; - - try { - const messagesToRemove = conversationsStore.activeMessages.slice(messageIndex); - - await DatabaseService.deleteMessageCascading(activeConv.id, messagesToRemove[0].id); - conversationsStore.sliceActiveMessages(messageIndex); - conversationsStore.updateConversationTimestamp(); - this.setChatLoading(activeConv.id, true); - this.clearChatStreaming(activeConv.id); - const parentMessageId = - conversationsStore.activeMessages.length > 0 - ? conversationsStore.activeMessages[conversationsStore.activeMessages.length - 1].id - : undefined; - const assistantMessage = await this.createAssistantMessage(parentMessageId); - - conversationsStore.addMessageToActive(assistantMessage); - await this.streamChatCompletion( - conversationsStore.activeMessages.slice(0, -1), - assistantMessage - ); - } catch (error) { - if (!isAbortError(error)) console.error('Failed to regenerate message:', error); - - this.setChatLoading(activeConv?.id || '', false); - } - } - - async regenerateMessageWithBranching(messageId: string, modelOverride?: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv || this.isChatLoadingInternal(activeConv.id)) return; - - this.cancelPreEncode(); - try { - const idx = conversationsStore.findMessageIndex(messageId); - - if (idx === -1) return; - - const msg = conversationsStore.activeMessages[idx]; - - if (msg.role !== MessageRole.ASSISTANT) return; - - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const parentMessage = findMessageById(allMessages, msg.parent); - - if (!parentMessage) return; - - this.setChatLoading(activeConv.id, true); - this.clearChatStreaming(activeConv.id); - const newAssistantMessage = await DatabaseService.createMessageBranch( - { - children: [], - content: '', - convId: msg.convId, - model: null, - role: msg.role, - timestamp: Date.now(), - toolCalls: '', - type: msg.type - }, - parentMessage.id - ); - - await conversationsStore.updateCurrentNode(newAssistantMessage.id); - conversationsStore.updateConversationTimestamp(); - await conversationsStore.refreshActiveMessages(); - const conversationPath = filterByLeafNodeId( - allMessages, - parentMessage.id, - false - ) as DatabaseMessage[]; - const modelToUse = modelOverride || msg.model || undefined; - - await this.streamChatCompletion( - conversationPath, - newAssistantMessage, - undefined, - undefined, - modelToUse - ); - } catch (error) { - if (!isAbortError(error)) - console.error('Failed to regenerate message with branching:', error); - - this.setChatLoading(activeConv?.id || '', false); - } - } - - async getDeletionInfo(messageId: string): Promise<{ - totalCount: number; - userMessages: number; - assistantMessages: number; - messageTypes: string[]; - }> { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) - return { assistantMessages: 0, messageTypes: [], totalCount: 0, userMessages: 0 }; - - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const messageToDelete = findMessageById(allMessages, messageId); - - // For system messages, don't count descendants as they will be preserved (reparented to root) - if (messageToDelete?.role === MessageRole.SYSTEM) { - const messagesToDelete = allMessages.filter((m) => m.id === messageId); - - let assistantMessages = 0, - userMessages = 0; - - const messageTypes: string[] = []; - - for (const msg of messagesToDelete) { - if (msg.role === MessageRole.USER) { - userMessages++; - - if (!messageTypes.includes('user message')) messageTypes.push('user message'); - } else if (msg.role === MessageRole.ASSISTANT) { - assistantMessages++; - - if (!messageTypes.includes('assistant response')) messageTypes.push('assistant response'); - } - } - - return { assistantMessages, messageTypes, totalCount: 1, userMessages }; - } - - const descendants = findDescendantMessages(allMessages, messageId); - const allToDelete = [messageId, ...descendants]; - const messagesToDelete = allMessages.filter((m) => allToDelete.includes(m.id)); - - let assistantMessages = 0, - userMessages = 0; - - const messageTypes: string[] = []; - - for (const msg of messagesToDelete) { - if (msg.role === MessageRole.USER) { - userMessages++; - - if (!messageTypes.includes('user message')) messageTypes.push('user message'); - } else if (msg.role === MessageRole.ASSISTANT) { - assistantMessages++; - - if (!messageTypes.includes('assistant response')) messageTypes.push('assistant response'); - } - } - - return { assistantMessages, messageTypes, totalCount: allToDelete.length, userMessages }; - } - - async deleteMessage(messageId: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return; - - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const messageToDelete = findMessageById(allMessages, messageId); - - if (!messageToDelete) return; - - const currentPath = filterByLeafNodeId(allMessages, activeConv.currNode || '', false); - const isInCurrentPath = currentPath.some((m) => m.id === messageId); - - if (isInCurrentPath && messageToDelete.parent) { - const siblings = allMessages.filter( - (m) => m.parent === messageToDelete.parent && m.id !== messageId - ); - - if (siblings.length > 0) { - const latestSibling = siblings.reduce((latest, sibling) => - sibling.timestamp > latest.timestamp ? sibling : latest - ); - - await conversationsStore.updateCurrentNode(findLeafNode(allMessages, latestSibling.id)); - } else if (messageToDelete.parent) { - await conversationsStore.updateCurrentNode( - findLeafNode(allMessages, messageToDelete.parent) - ); - } - } - - await DatabaseService.deleteMessageCascading(activeConv.id, messageId); - await conversationsStore.refreshActiveMessages(); - - conversationsStore.updateConversationTimestamp(); - } catch (error) { - console.error('Failed to delete message:', error); - } - } - - /** - * Open a fresh assistant turn anchored at the last tool result of a resolved - * agentic round and let streamChatCompletion route through runAgenticFlow. - * Used by continueAssistantMessage when classifyContinueIntent returns - * next_turn, meaning the target assistant already has its tool_calls paired - * with trailing tool results and the next thing to generate is a brand new - * turn rather than a token level continuation. - */ - private async continueAsNextAgenticTurn(anchorIndex: number): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return; - - const anchor = conversationsStore.activeMessages[anchorIndex]; - - if (!anchor) return; - - this.cancelPreEncode(); - this.setChatLoading(activeConv.id, true); - this.clearChatStreaming(activeConv.id); - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const anchorMessage = findMessageById(allMessages, anchor.id); - - if (!anchorMessage) { - this.setChatLoading(activeConv.id, false); - - return; - } - - const newAssistantMessage = await DatabaseService.createMessageBranch( - { - children: [], - content: '', - convId: activeConv.id, - model: null, - role: MessageRole.ASSISTANT, - timestamp: Date.now(), - toolCalls: '', - type: MessageType.TEXT - }, - anchorMessage.id - ); - - await conversationsStore.updateCurrentNode(newAssistantMessage.id); - conversationsStore.updateConversationTimestamp(); - await conversationsStore.refreshActiveMessages(); - const conversationPath = filterByLeafNodeId( - allMessages, - anchorMessage.id, - false - ) as DatabaseMessage[]; - - await this.streamChatCompletion(conversationPath, newAssistantMessage); - } catch (error) { - if (!isAbortError(error)) console.error('Failed to continue agentic turn:', error); - - this.setChatLoading(activeConv.id, false); - } - } - - async continueAssistantMessage(messageId: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv || this.isChatLoadingInternal(activeConv.id)) return; - - const result = this.getMessageByIdWithRole(messageId, MessageRole.ASSISTANT); - - if (!result) return; - - const { index: idx, message: msg } = result; - // Decide which resume path applies. tool_calls without tool results can - // not be resumed mid sequence by continue_final_message, branch instead. - // tool_calls already paired with tool results need a fresh next turn, - // not a token level continuation of the target assistant. - const intent = classifyContinueIntent(conversationsStore.activeMessages, idx); - - if (intent.kind === ContinueIntentKind.RERUN_TURN) { - return this.regenerateMessageWithBranching(messageId); - } - - if (intent.kind === ContinueIntentKind.NEXT_TURN) { - return this.continueAsNextAgenticTurn(intent.truncateAfter); - } - - try { - this.showErrorDialog(null); - this.setChatLoading(activeConv.id, true); - this.clearChatStreaming(activeConv.id); - - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const dbMessage = findMessageById(allMessages, messageId); - - if (!dbMessage) { - this.setChatLoading(activeConv.id, false); - - return; - } - - const originalContent = dbMessage.content; - const originalReasoning = dbMessage.reasoningContent || ''; - // Hand the persisted DatabaseMessage straight to sendMessage so its - // internal converter preserves tool_calls and extras when present. - // Reconstructing a bare {role, content} here would drop those fields - // and break continue_final_message for messages with tool calls. - const contextWithContinue = conversationsStore.activeMessages.slice(0, idx + 1); - - let appendedContent = ''; - let appendedReasoning = ''; - let hasReceivedContent = false; - - const updateStreamingContent = (fullContent: string) => { - this.setChatStreaming(msg.convId, fullContent, msg.id); - // resolve the row by id on every write, switching to another conv mid continue makes - // this a no op instead of writing positionally into the now displayed conversation - conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { - content: fullContent - }); - }; - const abortController = this.getOrCreateAbortController(msg.convId); - - await ChatService.sendMessage( - contextWithContinue, - { - ...this.getApiOptions(), - continueFinalMessage: true, - onChunk: (chunk: string) => { - appendedContent += chunk; - hasReceivedContent = true; - updateStreamingContent(originalContent + appendedContent); - this.setChatReasoning(msg.convId, false); - }, - onComplete: async ( - finalContent?: string, - reasoningContent?: string, - timings?: ChatMessageTimings - ) => { - const finalAppendedContent = hasReceivedContent ? appendedContent : finalContent || ''; - const finalAppendedReasoning = hasReceivedContent - ? appendedReasoning - : reasoningContent || ''; - const fullContent = originalContent + finalAppendedContent; - const fullReasoning = originalReasoning + finalAppendedReasoning || undefined; - - await DatabaseService.updateMessage(msg.id, { - content: fullContent, - reasoningContent: fullReasoning, - timestamp: Date.now(), - timings - }); - - conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { - content: fullContent, - reasoningContent: fullReasoning, - timestamp: Date.now(), - timings - }); - - conversationsStore.updateConversationTimestamp(msg.convId); - - this.setChatLoading(msg.convId, false); - this.clearChatStreaming(msg.convId); - this.setProcessingState(msg.convId, null); - }, - onCompletionId: (id: string) => { - if (!id) return; - - // refresh the message id so a later skip targets the live slot after a continue - conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { - completionId: id - }); - DatabaseService.updateMessage(msg.id, { completionId: id }).catch(() => {}); - }, - onConnectionState: (state: StreamConnectionState) => { - if (msg.convId === conversationsStore.activeConversation?.id) { - this.streamConnectionState = state; - } - }, - onError: async (error: Error) => { - if (isAbortError(error)) { - if (hasReceivedContent && appendedContent) { - await DatabaseService.updateMessage(msg.id, { - content: originalContent + appendedContent, - reasoningContent: originalReasoning + appendedReasoning || undefined, - timestamp: Date.now() - }); - - conversationsStore.updateMessageAtIndex( - conversationsStore.findMessageIndex(msg.id), - { - content: originalContent + appendedContent, - reasoningContent: originalReasoning + appendedReasoning || undefined, - timestamp: Date.now() - } - ); - } - - this.setChatLoading(msg.convId, false); - this.clearChatStreaming(msg.convId); - this.setProcessingState(msg.convId, null); - - return; - } - - console.error('Continue generation error:', error); - // keep whatever was appended so far, the message stays in memory and in DB - await DatabaseService.updateMessage(msg.id, { - content: originalContent + appendedContent, - reasoningContent: originalReasoning + appendedReasoning || undefined, - timestamp: Date.now() - }); - conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { - content: originalContent + appendedContent, - reasoningContent: originalReasoning + appendedReasoning || undefined, - timestamp: Date.now() - }); - - this.setChatLoading(msg.convId, false); - this.clearChatStreaming(msg.convId); - this.setProcessingState(msg.convId, null); - this.showErrorDialog({ - message: error.message, - type: error.name === 'TimeoutError' ? ErrorDialogType.TIMEOUT : ErrorDialogType.SERVER - }); - }, - onReasoningChunk: (chunk: string) => { - appendedReasoning += chunk; - hasReceivedContent = true; - // mark streaming state so a stop mid-thinking can persist the partial reasoning - this.setChatStreaming(msg.convId, originalContent + appendedContent, msg.id); - conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { - reasoningContent: originalReasoning + appendedReasoning - }); - this.setChatReasoning(msg.convId, true); - }, - onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { - const tokensPerSecond = - timings?.predicted_ms && timings?.predicted_n - ? (timings.predicted_n / timings.predicted_ms) * 1000 - : 0; - - this.updateProcessingStateFromTimings( - { - cache_n: timings?.cache_n || 0, - predicted_n: timings?.predicted_n || 0, - predicted_per_second: tokensPerSecond, - prompt_ms: timings?.prompt_ms, - prompt_n: timings?.prompt_n || 0, - prompt_progress: promptProgress - }, - msg.convId - ); - } - }, - - msg.convId, - abortController.signal - ); - } catch (error) { - if (!isAbortError(error)) console.error('Failed to continue message:', error); - - if (activeConv) this.setChatLoading(activeConv.id, false); - } - } - - async editAssistantMessage( - messageId: string, - newContent: string, - shouldBranch: boolean - ): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv || this.isChatLoadingInternal(activeConv.id)) return; - - const result = this.getMessageByIdWithRole(messageId, MessageRole.ASSISTANT); - - if (!result) return; - - const { index: idx, message: msg } = result; - - try { - if (shouldBranch) { - const newMessage = await DatabaseService.createMessageBranch( - { - children: [], - content: newContent, - convId: msg.convId, - model: msg.model, - role: msg.role, - timestamp: Date.now(), - toolCalls: msg.toolCalls || '', - type: msg.type - }, - msg.parent! - ); - - await conversationsStore.updateCurrentNode(newMessage.id); - } else { - await DatabaseService.updateMessage(msg.id, { content: newContent }); - conversationsStore.updateMessageAtIndex(idx, { content: newContent }); - } - - conversationsStore.updateConversationTimestamp(); - - await conversationsStore.refreshActiveMessages(); - } catch (error) { - console.error('Failed to edit assistant message:', error); - } - } - - async editUserMessagePreserveResponses( - messageId: string, - newContent: string, - newExtras?: DatabaseMessageExtra[] - ): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return; - - const result = this.getMessageByIdWithRole(messageId, MessageRole.USER); - - if (!result) return; - - const { index: idx, message: msg } = result; - - try { - const updateData: Partial = { content: newContent }; - - if (newExtras !== undefined) updateData.extra = JSON.parse(JSON.stringify(newExtras)); - - await DatabaseService.updateMessage(messageId, updateData); - - conversationsStore.updateMessageAtIndex(idx, updateData); - - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); - - if (rootMessage && msg.parent === rootMessage.id && newContent.trim()) { - await conversationsStore.updateConversationName( - activeConv.id, - generateConversationTitle( - newContent, - Boolean(settingsStore.config.titleGenerationUseFirstLine) - ) - ); - } - - conversationsStore.updateConversationTimestamp(); - } catch (error) { - console.error('Failed to edit user message:', error); - } - } - - async editMessageWithBranching( - messageId: string, - newContent: string, - newExtras?: DatabaseMessageExtra[] - ): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv || this.isChatLoadingInternal(activeConv.id)) return; - - let result = this.getMessageByIdWithRole(messageId, MessageRole.USER); - - if (!result) result = this.getMessageByIdWithRole(messageId, MessageRole.SYSTEM); - - if (!result) return; - - const { index: idx, message: msg } = result; - - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); - const isFirstUserMessage = - msg.role === MessageRole.USER && rootMessage && msg.parent === rootMessage.id; - const extrasToUse = - newExtras !== undefined - ? JSON.parse(JSON.stringify(newExtras)) - : msg.extra - ? JSON.parse(JSON.stringify(msg.extra)) - : undefined; - - let messageIdForResponse: string; - - const dbMsg = findMessageById(allMessages, msg.id); - const hasChildren = dbMsg ? dbMsg.children.length > 0 : msg.children.length > 0; - - if (!hasChildren) { - // No responses after this message — update in place instead of branching - const updates: Partial = { - content: newContent, - extra: extrasToUse, - timestamp: Date.now() - }; - - await DatabaseService.updateMessage(msg.id, updates); - conversationsStore.updateMessageAtIndex(idx, updates); - messageIdForResponse = msg.id; - } else { - // Has children — create a new branch as sibling - const parentId = msg.parent || rootMessage?.id; - - if (!parentId) return; - - const newMessage = await DatabaseService.createMessageBranch( - { - children: [], - content: newContent, - convId: msg.convId, - extra: extrasToUse, - model: msg.model, - role: msg.role, - timestamp: Date.now(), - toolCalls: msg.toolCalls || '', - type: msg.type - }, - parentId - ); - - await conversationsStore.updateCurrentNode(newMessage.id); - messageIdForResponse = newMessage.id; - } - - conversationsStore.updateConversationTimestamp(); - - if (isFirstUserMessage && newContent.trim()) - await conversationsStore.updateConversationName( - activeConv.id, - generateConversationTitle( - newContent, - Boolean(settingsStore.config.titleGenerationUseFirstLine) - ) - ); - - await conversationsStore.refreshActiveMessages(); - - if (msg.role === MessageRole.USER) - await this.generateResponseForMessage(messageIdForResponse); - } catch (error) { - console.error('Failed to edit message with branching:', error); - } - } - - private async generateResponseForMessage(userMessageId: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return; - - this.showErrorDialog(null); - this.setChatLoading(activeConv.id, true); - this.clearChatStreaming(activeConv.id); - - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const conversationPath = filterByLeafNodeId( - allMessages, - userMessageId, - false - ) as DatabaseMessage[]; - const assistantMessage = await DatabaseService.createMessageBranch( - { - children: [], - content: '', - convId: activeConv.id, - model: null, - role: MessageRole.ASSISTANT, - timestamp: Date.now(), - toolCalls: '', - type: MessageType.TEXT - }, - userMessageId - ); - - conversationsStore.addMessageToActive(assistantMessage); - - await this.streamChatCompletion(conversationPath, assistantMessage); - } catch (error) { - console.error('Failed to generate response:', error); - this.setChatLoading(activeConv.id, false); - } - } - - private getContextTotal(): number | null { - const activeConvId = this.activeConversationId; - const activeState = activeConvId ? this.getProcessingState(activeConvId) : null; - - if (activeState && typeof activeState.contextTotal === 'number' && activeState.contextTotal > 0) - return activeState.contextTotal; - - if (serverStore.isRouterMode) { - const modelContextSize = modelsStore.selectedModelContextSize; - - if (typeof modelContextSize === 'number' && modelContextSize > 0) { - return modelContextSize; - } - } else { - const propsContextSize = serverStore.contextSize; - - if (typeof propsContextSize === 'number' && propsContextSize > 0) { - return propsContextSize; - } - } - - return null; - } - - updateProcessingStateFromTimings( - timingData: { - prompt_n: number; - prompt_ms?: number; - predicted_n: number; - predicted_per_second: number; - cache_n: number; - prompt_progress?: ChatMessagePromptProgress; - }, - conversationId?: string - ): void { - const processingState = this.parseTimingData(timingData); - - if (processingState === null) { - console.warn('Failed to parse timing data - skipping update'); - - return; - } - - const targetId = conversationId || this.activeConversationId; - - if (targetId) { - this.setProcessingState(targetId, processingState); - } - } - - private parseTimingData(timingData: Record): ApiProcessingState | null { - const cacheTokens = (timingData.cache_n as number) || 0, - predictedTokens = (timingData.predicted_n as number) || 0, - promptMs = (timingData.prompt_ms as number) || undefined, - promptTokens = (timingData.prompt_n as number) || 0, - tokensPerSecond = (timingData.predicted_per_second as number) || 0; - const promptProgress = timingData.prompt_progress as - | { total: number; cache: number; processed: number; time_ms: number } - | undefined; - const contextTotal = this.getContextTotal(); - const currentConfig = settingsStore.config; - const outputTokensMax = currentConfig.max_tokens || -1; - const contextUsed = promptTokens + cacheTokens + predictedTokens, - outputTokensUsed = predictedTokens; - const progressCache = promptProgress?.cache || 0, - progressActualDone = (promptProgress?.processed ?? 0) - progressCache, - progressActualTotal = (promptProgress?.total ?? 0) - progressCache; - const progressPercent = promptProgress - ? Math.round((progressActualDone / progressActualTotal) * 100) - : undefined; - - return { - cacheTokens, - contextTotal, - contextUsed, - hasNextToken: predictedTokens > 0, - outputTokensMax, - outputTokensUsed, - progressPercent, - promptMs, - promptProgress, - promptTokens, - speculative: false, - status: predictedTokens > 0 ? 'generating' : promptProgress ? 'preparing' : 'idle', - temperature: currentConfig.temperature ?? 0.8, - tokensDecoded: predictedTokens, - tokensPerSecond, - tokensRemaining: outputTokensMax - predictedTokens, - topP: currentConfig.top_p ?? 0.95 - }; - } - - restoreProcessingStateFromMessages(messages: DatabaseMessage[], conversationId: string): void { - for (let i = messages.length - 1; i >= 0; i--) { - const message = messages[i]; - - if (message.role === MessageRole.ASSISTANT && message.timings) { - const restoredState = this.parseTimingData({ - cache_n: message.timings.cache_n || 0, - predicted_n: message.timings.predicted_n || 0, - predicted_per_second: - message.timings.predicted_n && message.timings.predicted_ms - ? (message.timings.predicted_n / message.timings.predicted_ms) * 1000 - : 0, - prompt_ms: message.timings.prompt_ms, - prompt_n: message.timings.prompt_n || 0 - }); - - if (restoredState) { - this.setProcessingState(conversationId, restoredState); - - return; - } - } - } - } - - private getApiOptions(): Record { - const currentConfig = settingsStore.config; - const hasValue = (value: unknown): boolean => - value !== undefined && value !== null && value !== ''; - const apiOptions: Record = { stream: true, timings_per_token: true }; - - if (serverStore.isRouterMode) { - const modelName = modelsStore.selectedModelName; - - if (modelName) apiOptions.model = modelName; - } - - if (currentConfig.systemMessage) apiOptions.systemMessage = currentConfig.systemMessage; - - if (currentConfig.disableReasoningParsing) apiOptions.disableReasoningParsing = true; - - if (currentConfig.excludeReasoningFromContext) apiOptions.excludeReasoningFromContext = true; - - // an explicit reasoning choice overrides the server default, DEFAULT sends nothing - const effort = conversationsStore.getReasoningEffort(); - - if (effort !== ReasoningEffort.DEFAULT) { - apiOptions.enableThinking = effort !== ReasoningEffort.OFF; - - if (effort !== ReasoningEffort.OFF) apiOptions.reasoningEffort = effort; - } - - if (hasValue(currentConfig.temperature)) - apiOptions.temperature = Number(currentConfig.temperature); - - if (hasValue(currentConfig.max_tokens)) - apiOptions.max_tokens = Number(currentConfig.max_tokens); - - if (hasValue(currentConfig.dynatemp_range)) - apiOptions.dynatemp_range = Number(currentConfig.dynatemp_range); - - if (hasValue(currentConfig.dynatemp_exponent)) - apiOptions.dynatemp_exponent = Number(currentConfig.dynatemp_exponent); - - if (hasValue(currentConfig.top_k)) apiOptions.top_k = Number(currentConfig.top_k); - - if (hasValue(currentConfig.top_p)) apiOptions.top_p = Number(currentConfig.top_p); - - if (hasValue(currentConfig.min_p)) apiOptions.min_p = Number(currentConfig.min_p); - - if (hasValue(currentConfig.xtc_probability)) - apiOptions.xtc_probability = Number(currentConfig.xtc_probability); - - if (hasValue(currentConfig.xtc_threshold)) - apiOptions.xtc_threshold = Number(currentConfig.xtc_threshold); - - if (hasValue(currentConfig.typ_p)) apiOptions.typ_p = Number(currentConfig.typ_p); - - if (hasValue(currentConfig.repeat_last_n)) - apiOptions.repeat_last_n = Number(currentConfig.repeat_last_n); - - if (hasValue(currentConfig.repeat_penalty)) - apiOptions.repeat_penalty = Number(currentConfig.repeat_penalty); - - if (hasValue(currentConfig.presence_penalty)) - apiOptions.presence_penalty = Number(currentConfig.presence_penalty); - - if (hasValue(currentConfig.frequency_penalty)) - apiOptions.frequency_penalty = Number(currentConfig.frequency_penalty); - - if (hasValue(currentConfig.dry_multiplier)) - apiOptions.dry_multiplier = Number(currentConfig.dry_multiplier); - - if (hasValue(currentConfig.dry_base)) apiOptions.dry_base = Number(currentConfig.dry_base); - - if (hasValue(currentConfig.dry_allowed_length)) - apiOptions.dry_allowed_length = Number(currentConfig.dry_allowed_length); - - if (hasValue(currentConfig.dry_penalty_last_n)) - apiOptions.dry_penalty_last_n = Number(currentConfig.dry_penalty_last_n); - - if (currentConfig.samplers) apiOptions.samplers = currentConfig.samplers; - - if (hasValue(currentConfig.backend_sampling)) - apiOptions.backend_sampling = currentConfig.backend_sampling; - - if (currentConfig.customJson) apiOptions.custom = currentConfig.customJson; - - return apiOptions; - } - - private cancelPreEncode(): void { - if (this.preEncodeAbortController) { - this.preEncodeAbortController.abort(); - this.preEncodeAbortController = null; - } - } - - private async triggerPreEncode( - allMessages: DatabaseMessage[], - assistantMessage: DatabaseMessage, - assistantContent: string, - model?: string | null, - excludeReasoning?: boolean - ): Promise { - this.cancelPreEncode(); - this.preEncodeAbortController = new AbortController(); - - const signal = this.preEncodeAbortController.signal; - - try { - const allIdle = await ChatService.areAllSlotsIdle(model, signal); - - if (!allIdle || signal.aborted) return; - - const messagesWithAssistant: DatabaseMessage[] = [ - ...allMessages, - { ...assistantMessage, content: assistantContent } - ]; - - await ChatService.preEncode(messagesWithAssistant, model, excludeReasoning, signal); - } catch (err) { - if (!isAbortError(err)) { - console.warn('[ChatStore] Pre-encode failed:', err); - } - } - } -} - -export const chatStore = new ChatStore(); diff --git a/tools/ui/src/lib/stores/chat/activity.svelte.ts b/tools/ui/src/lib/stores/chat/activity.svelte.ts new file mode 100644 index 000000000..cd4e0497b --- /dev/null +++ b/tools/ui/src/lib/stores/chat/activity.svelte.ts @@ -0,0 +1,74 @@ +/** + * ChatActivityStore - Conversation activity ledger + * + * Single owner of the "is this conversation doing something" state: + * - `local` - this browser is piping a stream (send, server-stream attach, + * or resume-wait while the owning model loads) + * - `remote` - the backend reports a running session, no local pipe yet + * (global snapshot on mount / visibilitychange) + * + * The union of both drives the sidebar spinners (`loadingConvs`); `local` + * drives the per-conversation loading flags. When a local pipe ends it is + * the authoritative observer of session end, so it also drops the stale + * remote hint in the same call - no cross-owner cleanup, no ghosted + * spinners waiting for the next visibilitychange snapshot. + * + * Composed under chatStore.activity; not exported from the stores barrel. + */ + +import { SvelteSet } from 'svelte/reactivity'; + +export class ChatActivityStore { + /** Convs this browser is piping a stream for (send, attach, resume-wait). */ + private local = new SvelteSet(); + /** Convs the backend reports as having a running session (snapshot sync). */ + private remote = new SvelteSet(); + + /** Convs with any activity, the union the sidebar spinners render. */ + loadingConvs = $derived.by(() => { + const out = new SvelteSet(this.local); + + for (const id of this.remote) out.add(id); + + return Array.from(out); + }); + + /** + * Apply a backend snapshot of running sessions (mount / visibilitychange). + * Diffed so unchanged entries do not re-trigger reactivity. + */ + applyRemoteSnapshot(running: Iterable): void { + const next = new SvelteSet(running); + + for (const id of Array.from(this.remote)) { + if (!next.has(id)) this.remote.delete(id); + } + + for (const id of next) this.remote.add(id); + } + + isLocal(convId: string): boolean { + return this.local.has(convId); + } + + isRemote(convId: string): boolean { + return this.remote.has(convId); + } + + /** + * A local pipe ended for the conv. Also drops the remote hint: the local + * pipe is the authoritative observer of session end, so the sidebar hint + * goes away right away instead of ghosting until the next snapshot. + */ + localEnded(convId: string): void { + this.local.delete(convId); + this.remote.delete(convId); + } + + /** A local pipe (send, attach or resume-wait) started for the conv. */ + markLocal(convId: string): void { + this.local.add(convId); + } +} + +export const chatActivityStore = new ChatActivityStore(); diff --git a/tools/ui/src/lib/stores/context-stats.svelte.ts b/tools/ui/src/lib/stores/chat/context-stats.svelte.ts similarity index 56% rename from tools/ui/src/lib/stores/context-stats.svelte.ts rename to tools/ui/src/lib/stores/chat/context-stats.svelte.ts index 149184563..b5d22cfbd 100644 --- a/tools/ui/src/lib/stores/context-stats.svelte.ts +++ b/tools/ui/src/lib/stores/chat/context-stats.svelte.ts @@ -1,5 +1,5 @@ /** - * contextStatsStore - Context window usage stats for the active conversation + * ContextStatsStore - Context window usage stats for the active conversation * * Combines token usage persisted in message timings metadata with * server-originating data: model context size from /props (modelsStore) @@ -8,12 +8,17 @@ import { MessageRole } from '$lib/enums'; // direct imports between stores, not via the barrel, to avoid circular deps -import { agenticStore } from '$lib/stores/agentic.svelte'; -import { chatStore } from '$lib/stores/chat.svelte'; -import { conversationsStore } from '$lib/stores/conversations.svelte'; -import { modelsStore } from '$lib/stores/models.svelte'; +import { agenticStore } from '$lib/stores/agentic/index.svelte'; +import { chatStore } from '$lib/stores/chat/index.svelte'; +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; import { serverStore } from '$lib/stores/server.svelte'; -import type { ApiProcessingState, ChatMessageTimings, DatabaseMessage } from '$lib/types'; +import type { + ApiProcessingState, + ChatMessageAgenticTimings, + ChatMessageTimings, + DatabaseMessage +} from '$lib/types'; interface LiveStats { freshTokens: number; @@ -22,14 +27,46 @@ interface LiveStats { outputTokens: number; } -function lastAssistantTimings(messages: DatabaseMessage[]): ChatMessageTimings | undefined { - for (let i = messages.length - 1; i >= 0; i--) { - const m = messages[i]; +interface AssistantTimingsSummary { + lastAgenticLlm: ChatMessageAgenticTimings['llm'] | undefined; + lastTimings: ChatMessageTimings | undefined; + cacheTotal: number; + output: number; + outputMs: number; + read: number; +} - if (m.role === MessageRole.ASSISTANT && m.timings) return m.timings; +/** + * One forward pass over the messages computing everything the deriveds + * below need: the last assistant timings (per-turn gauges), the last + * agentic llm totals (cumulative gauge) and the cumulative sums. During + * streaming activeMessages churns every chunk, and each of these used to be + * its own O(n) scan re-run per chunk. + */ +function summarizeAssistantTimings(messages: DatabaseMessage[]): AssistantTimingsSummary { + let lastAgenticLlm: ChatMessageAgenticTimings['llm'] | undefined; + let lastTimings: ChatMessageTimings | undefined; + let read = 0; + let cacheTotal = 0; + let output = 0; + let outputMs = 0; + + for (const m of messages) { + if (m.role !== MessageRole.ASSISTANT || !m.timings) continue; + + lastTimings = m.timings; + + if (m.timings.agentic?.llm?.predicted_n != null) { + lastAgenticLlm = m.timings.agentic.llm; + } + + read += m.timings.prompt_n ?? 0; + cacheTotal += m.timings.cache_n ?? 0; + output += m.timings.predicted_n ?? 0; + outputMs += m.timings.predicted_ms ?? 0; } - return undefined; + return { cacheTotal, lastAgenticLlm, lastTimings, output, outputMs, read }; } function deriveLiveStats(state: ApiProcessingState | null): LiveStats | null { @@ -52,83 +89,14 @@ class ContextStatsStore { // The canonical resolution lives in modelsStore.activeModelId. activeModelId = $derived(modelsStore.activeModelId); - isActiveModelLoaded = $derived( - this.activeModelId !== null && - (!serverStore.isRouterMode || modelsStore.isModelLoaded(this.activeModelId)) + // shared by currentRead/Fresh/Cache/Output and cumulative so a per-chunk + // churn of activeMessages triggers exactly one scan instead of one per + // derived + private assistantTimings = $derived.by(() => + summarizeAssistantTimings(conversationsStore.activeMessages as DatabaseMessage[]) ); - isActiveModelLoading = $derived( - this.activeModelId !== null && modelsStore.isModelOperationInProgress(this.activeModelId) - ); - - contextTotal = $derived.by(() => { - void modelsStore.propsCacheVersion; - - return this.activeModelId ? modelsStore.getModelContextSize(this.activeModelId) : null; - }); - - private liveStats = $derived(deriveLiveStats(chatStore.activeProcessingState)); - - currentRead = $derived.by(() => { - const timings = lastAssistantTimings(conversationsStore.activeMessages as DatabaseMessage[]); - - let read = 0; - - if (timings) { - read = (timings.prompt_n ?? 0) + (timings.cache_n ?? 0); - } - - // live.promptTokens is already the combined reading (prompt + cache), - // so do not also add live.cacheTokens. - if (this.liveStats && this.liveStats.promptTokens > 0) { - read = Math.max(read, this.liveStats.promptTokens); - } - - return read; - }); - - currentFresh = $derived.by(() => { - const timings = lastAssistantTimings(conversationsStore.activeMessages as DatabaseMessage[]); - const fresh = timings?.prompt_n ?? 0; - - return Math.max(fresh, this.liveStats?.freshTokens ?? 0); - }); - - currentCache = $derived.by(() => { - const timings = lastAssistantTimings(conversationsStore.activeMessages as DatabaseMessage[]); - const cached = timings?.cache_n ?? 0; - - if (this.liveStats && this.liveStats.promptTokens > 0) { - return Math.max(cached, this.liveStats.cacheTokens); - } - - return cached; - }); - - currentOutput = $derived.by(() => { - if (this.liveStats && this.liveStats.outputTokens > 0) return this.liveStats.outputTokens; - - const timings = lastAssistantTimings(conversationsStore.activeMessages as DatabaseMessage[]); - - return timings?.predicted_n ?? 0; - }); - - kvTotal = $derived(this.currentRead + this.currentOutput); - - contextUsed = $derived(this.currentRead + this.currentOutput); - - contextAvailable = $derived( - this.contextTotal !== null ? this.contextTotal - this.contextUsed : null - ); - - contextPercent = $derived.by(() => { - if (this.contextTotal === null || this.contextTotal <= 0) return null; - - return Math.round((this.contextUsed / this.contextTotal) * 100); - }); - private cumulative = $derived.by(() => { - const messages = conversationsStore.activeMessages as DatabaseMessage[]; const convId = conversationsStore.activeConversation?.id; // A running agentic flow stamps llm totals on messages only when it // exits, so read its live session totals instead. @@ -147,51 +115,107 @@ class ContextStatsStore { }; } + const { cacheTotal, lastAgenticLlm, output, outputMs, read } = this.assistantTimings; + // Agentic sessions stamp the same agentic.llm totals onto every // assistant message; cache_n is never per-turn so cache_total stays 0. - const agenticMessages = messages.filter( - (m) => m.role === MessageRole.ASSISTANT && m.timings?.agentic?.llm?.predicted_n != null - ); - - if (agenticMessages.length > 0) { - const llm = agenticMessages[agenticMessages.length - 1].timings!.agentic!.llm; - const output = llm.predicted_n ?? 0; - const outputMs = llm.predicted_ms ?? 0; - const averageTokensPerSecond = outputMs > 0 && output > 0 ? (output / outputMs) * 1000 : null; + if (lastAgenticLlm) { + const averageTokensPerSecond = + lastAgenticLlm.predicted_ms > 0 && lastAgenticLlm.predicted_n > 0 + ? (lastAgenticLlm.predicted_n / lastAgenticLlm.predicted_ms) * 1000 + : null; return { averageTokensPerSecond, cacheTotal: 0, - output, - read: llm.prompt_n ?? 0 + output: lastAgenticLlm.predicted_n ?? 0, + read: lastAgenticLlm.prompt_n ?? 0 }; } - let read = 0; - let output = 0; - let outputMs = 0; - let cacheTotal = 0; - - for (const m of messages) { - if (m.role !== MessageRole.ASSISTANT || !m.timings) continue; - - read += m.timings.prompt_n ?? 0; - cacheTotal += m.timings.cache_n ?? 0; - output += m.timings.predicted_n ?? 0; - outputMs += m.timings.predicted_ms ?? 0; - } const averageTokensPerSecond = outputMs > 0 && output > 0 ? (output / outputMs) * 1000 : null; return { averageTokensPerSecond, cacheTotal, output, read }; }); - cumulativeRead = $derived(this.cumulative.read); + averageTokensPerSecond = $derived(this.cumulative.averageTokensPerSecond); - cumulativeOutput = $derived(this.cumulative.output); + contextTotal = $derived.by(() => { + void modelsStore.props.cacheVersion; + + return this.activeModelId ? modelsStore.props.getModelContextSize(this.activeModelId) : null; + }); + + private liveStats = $derived(deriveLiveStats(chatStore.processing.activeState)); + + currentOutput = $derived.by(() => { + if (this.liveStats && this.liveStats.outputTokens > 0) return this.liveStats.outputTokens; + + return this.assistantTimings.lastTimings?.predicted_n ?? 0; + }); + + currentRead = $derived.by(() => { + const timings = this.assistantTimings.lastTimings; + + let read = 0; + + if (timings) { + read = (timings.prompt_n ?? 0) + (timings.cache_n ?? 0); + } + + // live.promptTokens is already the combined reading (prompt + cache), + // so do not also add live.cacheTokens. + if (this.liveStats && this.liveStats.promptTokens > 0) { + read = Math.max(read, this.liveStats.promptTokens); + } + + return read; + }); + + contextUsed = $derived(this.currentRead + this.currentOutput); + + contextAvailable = $derived( + this.contextTotal !== null ? this.contextTotal - this.contextUsed : null + ); + + contextPercent = $derived.by(() => { + if (this.contextTotal === null || this.contextTotal <= 0) return null; + + return Math.round((this.contextUsed / this.contextTotal) * 100); + }); cumulativeCacheTotal = $derived(this.cumulative.cacheTotal); - averageTokensPerSecond = $derived(this.cumulative.averageTokensPerSecond); + cumulativeOutput = $derived(this.cumulative.output); + + cumulativeRead = $derived(this.cumulative.read); + + currentCache = $derived.by(() => { + const cached = this.assistantTimings.lastTimings?.cache_n ?? 0; + + if (this.liveStats && this.liveStats.promptTokens > 0) { + return Math.max(cached, this.liveStats.cacheTokens); + } + + return cached; + }); + + currentFresh = $derived.by(() => { + const fresh = this.assistantTimings.lastTimings?.prompt_n ?? 0; + + return Math.max(fresh, this.liveStats?.freshTokens ?? 0); + }); + + isActiveModelLoaded = $derived( + this.activeModelId !== null && + (!serverStore.isRouterMode || modelsStore.isModelLoaded(this.activeModelId)) + ); + + isActiveModelLoading = $derived( + this.activeModelId !== null && modelsStore.status.isOperationInProgress(this.activeModelId) + ); + + kvTotal = $derived(this.currentRead + this.currentOutput); } export const contextStatsStore = new ContextStatsStore(); diff --git a/tools/ui/src/lib/stores/draft-messages.svelte.ts b/tools/ui/src/lib/stores/chat/drafts.svelte.ts similarity index 76% rename from tools/ui/src/lib/stores/draft-messages.svelte.ts rename to tools/ui/src/lib/stores/chat/drafts.svelte.ts index 235a59122..f480e1efd 100644 --- a/tools/ui/src/lib/stores/draft-messages.svelte.ts +++ b/tools/ui/src/lib/stores/chat/drafts.svelte.ts @@ -1,3 +1,11 @@ +/** + * DraftMessagesStore - Per-conversation input drafts + * + * Keeps in-memory drafts (message text + files) keyed by conversation id, + * plus a dedicated key for the new-chat screen, so the input box restores + * its content when switching conversations. + */ + import { NEW_CHAT_DRAFT_KEY } from '$lib/constants'; interface DraftMessage { @@ -8,6 +16,12 @@ interface DraftMessage { class DraftMessagesStore { private drafts = new Map(); + clearDraftMessage(chatId: string | undefined): void { + const key = chatId ?? NEW_CHAT_DRAFT_KEY; + + this.drafts.delete(key); + } + getDraftMessage(chatId: string | undefined): DraftMessage { const key = chatId ?? NEW_CHAT_DRAFT_KEY; @@ -23,12 +37,6 @@ class DraftMessagesStore { this.drafts.delete(key); } } - - clearDraftMessage(chatId: string | undefined): void { - const key = chatId ?? NEW_CHAT_DRAFT_KEY; - - this.drafts.delete(key); - } } export const draftMessagesStore = new DraftMessagesStore(); diff --git a/tools/ui/src/lib/stores/chat/flows.svelte.ts b/tools/ui/src/lib/stores/chat/flows.svelte.ts new file mode 100644 index 000000000..16c377bb6 --- /dev/null +++ b/tools/ui/src/lib/stores/chat/flows.svelte.ts @@ -0,0 +1,794 @@ +/** + * ChatMessageFlows - Message-level flows for the active conversation + * + * Owns the operations that mutate chat history and (re)stream a response: + * editing, regeneration, continuation and deletion of messages. Created and + * owned by chatStore; the host exposes the streaming core and the + * per-conversation state setters these flows drive. + */ + +import { + ContinueIntentKind, + ErrorDialogType, + MessageRole, + MessageType, + StreamConnectionState +} from '$lib/enums'; +import { ChatService } from '$lib/services/chat.service'; +import { DatabaseService } from '$lib/services/database.service'; +import type { ChatProcessingStore } from '$lib/stores/chat/processing.svelte'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import type { + ChatMessagePromptProgress, + ChatMessageTimings, + DatabaseMessage, + DatabaseMessageExtra, + ErrorDialogState +} from '$lib/types'; +import { + classifyContinueIntent, + filterByLeafNodeId, + findDescendantMessages, + findLeafNode, + findMessageById, + isAbortError +} from '$lib/utils'; + +/** + * The slice of chatStore the flows drive. Kept narrow on purpose so the flows + * cannot reach around the host's full surface; chatStore implements this + * structurally. + */ +export interface ChatFlowsHost { + processing: ChatProcessingStore; + streamConnectionState: StreamConnectionState; + cancelPreEncode(): void; + clearChatStreaming(convId: string, messageId?: string): void; + cleanupStreaming(convId: string): void; + createAssistantMessage(parentId?: string): Promise; + getApiOptions(): Record; + getOrCreateAbortController(convId: string): AbortController; + isChatLoadingInternal(convId: string): boolean; + setChatLoading(convId: string, loading: boolean): void; + setChatReasoning(convId: string, reasoning: boolean): void; + setChatStreaming( + convId: string, + response: string, + messageId: string, + model?: string | null + ): void; + showErrorDialog(state: ErrorDialogState | null): void; + stopGeneration(): Promise; + streamChatCompletion( + allMessages: DatabaseMessage[], + assistantMessage: DatabaseMessage, + onComplete?: (content: string) => Promise, + onError?: (error: Error) => void, + modelOverride?: string | null, + firstUserMessageContent?: string + ): Promise; +} + +export class ChatMessageFlows { + constructor(private host: ChatFlowsHost) {} + + async continueAssistantMessage(messageId: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv || this.host.isChatLoadingInternal(activeConv.id)) return; + + const result = this.getMessageByIdWithRole(messageId, MessageRole.ASSISTANT); + + if (!result) return; + + const { index: idx, message: msg } = result; + // Decide which resume path applies. tool_calls without tool results can + // not be resumed mid sequence by continue_final_message, branch instead. + // tool_calls already paired with tool results need a fresh next turn, + // not a token level continuation of the target assistant. + const intent = classifyContinueIntent(conversationsStore.activeMessages, idx); + + if (intent.kind === ContinueIntentKind.RERUN_TURN) { + return this.regenerateMessageWithBranching(messageId); + } + + if (intent.kind === ContinueIntentKind.NEXT_TURN) { + return this.continueAsNextAgenticTurn(intent.truncateAfter); + } + + try { + this.host.showErrorDialog(null); + this.host.setChatLoading(activeConv.id, true); + this.host.clearChatStreaming(activeConv.id); + + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const dbMessage = findMessageById(allMessages, messageId); + + if (!dbMessage) { + this.host.setChatLoading(activeConv.id, false); + + return; + } + + const originalContent = dbMessage.content; + const originalReasoning = dbMessage.reasoningContent || ''; + // Hand the persisted DatabaseMessage straight to sendMessage so its + // internal converter preserves tool_calls and extras when present. + // Reconstructing a bare {role, content} here would drop those fields + // and break continue_final_message for messages with tool calls. + const contextWithContinue = conversationsStore.activeMessages.slice(0, idx + 1); + + let appendedContent = ''; + let appendedReasoning = ''; + let hasReceivedContent = false; + + const updateStreamingContent = (fullContent: string) => { + this.host.setChatStreaming(msg.convId, fullContent, msg.id); + // resolve the row by id on every write, switching to another conv mid continue makes + // this a no op instead of writing positionally into the now displayed conversation + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { + content: fullContent + }); + }; + const abortController = this.host.getOrCreateAbortController(msg.convId); + + await ChatService.sendMessage( + contextWithContinue, + { + ...this.host.getApiOptions(), + continueFinalMessage: true, + onChunk: (chunk: string) => { + appendedContent += chunk; + hasReceivedContent = true; + updateStreamingContent(originalContent + appendedContent); + this.host.setChatReasoning(msg.convId, false); + }, + onComplete: async ( + finalContent?: string, + reasoningContent?: string, + timings?: ChatMessageTimings + ) => { + const finalAppendedContent = hasReceivedContent ? appendedContent : finalContent || ''; + const finalAppendedReasoning = hasReceivedContent + ? appendedReasoning + : reasoningContent || ''; + const fullContent = originalContent + finalAppendedContent; + const fullReasoning = originalReasoning + finalAppendedReasoning || undefined; + + await DatabaseService.updateMessage(msg.id, { + content: fullContent, + reasoningContent: fullReasoning, + timestamp: Date.now(), + timings + }); + + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { + content: fullContent, + reasoningContent: fullReasoning, + timestamp: Date.now(), + timings + }); + + conversationsStore.updateConversationTimestamp(msg.convId); + + this.host.cleanupStreaming(msg.convId); + }, + onCompletionId: (id: string) => { + if (!id) return; + + // refresh the message id so a later skip targets the live slot after a continue + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { + completionId: id + }); + DatabaseService.updateMessage(msg.id, { completionId: id }).catch(() => {}); + }, + onConnectionState: (state: StreamConnectionState) => { + if (msg.convId === conversationsStore.activeConversation?.id) { + this.host.streamConnectionState = state; + } + }, + onError: async (error: Error) => { + if (isAbortError(error)) { + if (hasReceivedContent && appendedContent) { + await DatabaseService.updateMessage(msg.id, { + content: originalContent + appendedContent, + reasoningContent: originalReasoning + appendedReasoning || undefined, + timestamp: Date.now() + }); + + conversationsStore.updateMessageAtIndex( + conversationsStore.findMessageIndex(msg.id), + { + content: originalContent + appendedContent, + reasoningContent: originalReasoning + appendedReasoning || undefined, + timestamp: Date.now() + } + ); + } + + this.host.cleanupStreaming(msg.convId); + + return; + } + + console.error('Continue generation error:', error); + // keep whatever was appended so far, the message stays in memory and in DB + await DatabaseService.updateMessage(msg.id, { + content: originalContent + appendedContent, + reasoningContent: originalReasoning + appendedReasoning || undefined, + timestamp: Date.now() + }); + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { + content: originalContent + appendedContent, + reasoningContent: originalReasoning + appendedReasoning || undefined, + timestamp: Date.now() + }); + + this.host.cleanupStreaming(msg.convId); + this.host.showErrorDialog({ + message: error.message, + type: error.name === 'TimeoutError' ? ErrorDialogType.TIMEOUT : ErrorDialogType.SERVER + }); + }, + onReasoningChunk: (chunk: string) => { + appendedReasoning += chunk; + hasReceivedContent = true; + // mark streaming state so a stop mid-thinking can persist the partial reasoning + this.host.setChatStreaming(msg.convId, originalContent + appendedContent, msg.id); + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { + reasoningContent: originalReasoning + appendedReasoning + }); + this.host.setChatReasoning(msg.convId, true); + }, + onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { + this.host.processing.applyStreamTimings(timings, promptProgress, msg.convId); + } + }, + + msg.convId, + abortController.signal + ); + } catch (error) { + if (!isAbortError(error)) console.error('Failed to continue message:', error); + + if (activeConv) this.host.setChatLoading(activeConv.id, false); + } + } + + async deleteMessage(messageId: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return; + + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const messageToDelete = findMessageById(allMessages, messageId); + + if (!messageToDelete) return; + + const currentPath = filterByLeafNodeId(allMessages, activeConv.currNode || '', false); + const isInCurrentPath = currentPath.some((m) => m.id === messageId); + + if (isInCurrentPath && messageToDelete.parent) { + const siblings = allMessages.filter( + (m) => m.parent === messageToDelete.parent && m.id !== messageId + ); + + if (siblings.length > 0) { + const latestSibling = siblings.reduce((latest, sibling) => + sibling.timestamp > latest.timestamp ? sibling : latest + ); + + await conversationsStore.updateCurrentNode(findLeafNode(allMessages, latestSibling.id)); + } else if (messageToDelete.parent) { + await conversationsStore.updateCurrentNode( + findLeafNode(allMessages, messageToDelete.parent) + ); + } + } + + await DatabaseService.deleteMessageCascading(activeConv.id, messageId); + await conversationsStore.refreshActiveMessages(); + + conversationsStore.updateConversationTimestamp(); + } catch (error) { + console.error('Failed to delete message:', error); + } + } + + async editAssistantMessage( + messageId: string, + newContent: string, + shouldBranch: boolean + ): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv || this.host.isChatLoadingInternal(activeConv.id)) return; + + const result = this.getMessageByIdWithRole(messageId, MessageRole.ASSISTANT); + + if (!result) return; + + const { index: idx, message: msg } = result; + + try { + if (shouldBranch) { + const newMessage = await DatabaseService.createMessageBranch( + { + children: [], + content: newContent, + convId: msg.convId, + model: msg.model, + role: msg.role, + timestamp: Date.now(), + toolCalls: msg.toolCalls || '', + type: msg.type + }, + msg.parent! + ); + + await conversationsStore.updateCurrentNode(newMessage.id); + } else { + await DatabaseService.updateMessage(msg.id, { content: newContent }); + conversationsStore.updateMessageAtIndex(idx, { content: newContent }); + } + + conversationsStore.updateConversationTimestamp(); + + await conversationsStore.refreshActiveMessages(); + } catch (error) { + console.error('Failed to edit assistant message:', error); + } + } + + async editMessageWithBranching( + messageId: string, + newContent: string, + newExtras?: DatabaseMessageExtra[] + ): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv || this.host.isChatLoadingInternal(activeConv.id)) return; + + let result = this.getMessageByIdWithRole(messageId, MessageRole.USER); + + if (!result) result = this.getMessageByIdWithRole(messageId, MessageRole.SYSTEM); + + if (!result) return; + + const { index: idx, message: msg } = result; + + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); + const isFirstUserMessage = + msg.role === MessageRole.USER && rootMessage && msg.parent === rootMessage.id; + const extrasToUse = + newExtras !== undefined + ? JSON.parse(JSON.stringify(newExtras)) + : msg.extra + ? JSON.parse(JSON.stringify(msg.extra)) + : undefined; + + let messageIdForResponse: string; + + const dbMsg = findMessageById(allMessages, msg.id); + const hasChildren = dbMsg ? dbMsg.children.length > 0 : msg.children.length > 0; + + if (!hasChildren) { + // No responses after this message - update in place instead of branching + const updates: Partial = { + content: newContent, + extra: extrasToUse, + timestamp: Date.now() + }; + + await DatabaseService.updateMessage(msg.id, updates); + conversationsStore.updateMessageAtIndex(idx, updates); + messageIdForResponse = msg.id; + } else { + // Has children - create a new branch as sibling + const parentId = msg.parent || rootMessage?.id; + + if (!parentId) return; + + const newMessage = await DatabaseService.createMessageBranch( + { + children: [], + content: newContent, + convId: msg.convId, + extra: extrasToUse, + model: msg.model, + role: msg.role, + timestamp: Date.now(), + toolCalls: msg.toolCalls || '', + type: msg.type + }, + parentId + ); + + await conversationsStore.updateCurrentNode(newMessage.id); + messageIdForResponse = newMessage.id; + } + + conversationsStore.updateConversationTimestamp(); + + if (isFirstUserMessage && newContent.trim()) + await conversationsStore.applyTitleFromContent(activeConv.id, newContent); + + await conversationsStore.refreshActiveMessages(); + + if (msg.role === MessageRole.USER) + await this.generateResponseForMessage(messageIdForResponse); + } catch (error) { + console.error('Failed to edit message with branching:', error); + } + } + + async editUserMessagePreserveResponses( + messageId: string, + newContent: string, + newExtras?: DatabaseMessageExtra[] + ): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return; + + const result = this.getMessageByIdWithRole(messageId, MessageRole.USER); + + if (!result) return; + + const { index: idx, message: msg } = result; + + try { + const updateData: Partial = { content: newContent }; + + if (newExtras !== undefined) updateData.extra = JSON.parse(JSON.stringify(newExtras)); + + await DatabaseService.updateMessage(messageId, updateData); + + conversationsStore.updateMessageAtIndex(idx, updateData); + + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); + + if (rootMessage && msg.parent === rootMessage.id && newContent.trim()) { + await conversationsStore.applyTitleFromContent(activeConv.id, newContent); + } + + conversationsStore.updateConversationTimestamp(); + } catch (error) { + console.error('Failed to edit user message:', error); + } + } + + async getDeletionInfo(messageId: string): Promise<{ + totalCount: number; + userMessages: number; + assistantMessages: number; + messageTypes: string[]; + }> { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) + return { assistantMessages: 0, messageTypes: [], totalCount: 0, userMessages: 0 }; + + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const messageToDelete = findMessageById(allMessages, messageId); + + // For system messages, don't count descendants as they will be preserved (reparented to root) + if (messageToDelete?.role === MessageRole.SYSTEM) { + const messagesToDelete = allMessages.filter((m) => m.id === messageId); + + let assistantMessages = 0, + userMessages = 0; + + const messageTypes: string[] = []; + + for (const msg of messagesToDelete) { + if (msg.role === MessageRole.USER) { + userMessages++; + + if (!messageTypes.includes('user message')) messageTypes.push('user message'); + } else if (msg.role === MessageRole.ASSISTANT) { + assistantMessages++; + + if (!messageTypes.includes('assistant response')) messageTypes.push('assistant response'); + } + } + + return { assistantMessages, messageTypes, totalCount: 1, userMessages }; + } + + const descendants = findDescendantMessages(allMessages, messageId); + const allToDelete = [messageId, ...descendants]; + const messagesToDelete = allMessages.filter((m) => allToDelete.includes(m.id)); + + let assistantMessages = 0, + userMessages = 0; + + const messageTypes: string[] = []; + + for (const msg of messagesToDelete) { + if (msg.role === MessageRole.USER) { + userMessages++; + + if (!messageTypes.includes('user message')) messageTypes.push('user message'); + } else if (msg.role === MessageRole.ASSISTANT) { + assistantMessages++; + + if (!messageTypes.includes('assistant response')) messageTypes.push('assistant response'); + } + } + + return { assistantMessages, messageTypes, totalCount: allToDelete.length, userMessages }; + } + + async regenerateMessage(messageId: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv || this.host.isChatLoadingInternal(activeConv.id)) return; + + this.host.cancelPreEncode(); + const result = this.getMessageByIdWithRole(messageId, MessageRole.ASSISTANT); + + if (!result) return; + + const { index: messageIndex } = result; + + try { + const messagesToRemove = conversationsStore.activeMessages.slice(messageIndex); + + await DatabaseService.deleteMessageCascading(activeConv.id, messagesToRemove[0].id); + conversationsStore.sliceActiveMessages(messageIndex); + conversationsStore.updateConversationTimestamp(); + this.host.setChatLoading(activeConv.id, true); + this.host.clearChatStreaming(activeConv.id); + const parentMessageId = + conversationsStore.activeMessages.length > 0 + ? conversationsStore.activeMessages[conversationsStore.activeMessages.length - 1].id + : undefined; + const assistantMessage = await this.host.createAssistantMessage(parentMessageId); + + conversationsStore.addMessageToActive(assistantMessage); + await this.host.streamChatCompletion( + conversationsStore.activeMessages.slice(0, -1), + assistantMessage + ); + } catch (error) { + if (!isAbortError(error)) console.error('Failed to regenerate message:', error); + + this.host.setChatLoading(activeConv?.id || '', false); + } + } + + async regenerateMessageWithBranching(messageId: string, modelOverride?: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv || this.host.isChatLoadingInternal(activeConv.id)) return; + + this.host.cancelPreEncode(); + try { + const idx = conversationsStore.findMessageIndex(messageId); + + if (idx === -1) return; + + const msg = conversationsStore.activeMessages[idx]; + + if (msg.role !== MessageRole.ASSISTANT) return; + + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const parentMessage = findMessageById(allMessages, msg.parent); + + if (!parentMessage) return; + + this.host.setChatLoading(activeConv.id, true); + this.host.clearChatStreaming(activeConv.id); + const newAssistantMessage = await DatabaseService.createMessageBranch( + { + children: [], + content: '', + convId: msg.convId, + model: null, + role: msg.role, + timestamp: Date.now(), + toolCalls: '', + type: msg.type + }, + parentMessage.id + ); + + await conversationsStore.updateCurrentNode(newAssistantMessage.id); + conversationsStore.updateConversationTimestamp(); + await conversationsStore.refreshActiveMessages(); + const conversationPath = filterByLeafNodeId( + allMessages, + parentMessage.id, + false + ) as DatabaseMessage[]; + const modelToUse = modelOverride || msg.model || undefined; + + await this.host.streamChatCompletion( + conversationPath, + newAssistantMessage, + undefined, + undefined, + modelToUse + ); + } catch (error) { + if (!isAbortError(error)) + console.error('Failed to regenerate message with branching:', error); + + this.host.setChatLoading(activeConv?.id || '', false); + } + } + + async updateMessage(messageId: string, newContent: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return; + + if (this.host.isChatLoadingInternal(activeConv.id)) await this.host.stopGeneration(); + + const result = this.getMessageByIdWithRole(messageId, MessageRole.USER); + + if (!result) return; + + const { index: messageIndex, message: messageToUpdate } = result; + const originalContent = messageToUpdate.content; + + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); + const isFirstUserMessage = rootMessage && messageToUpdate.parent === rootMessage.id; + + conversationsStore.updateMessageAtIndex(messageIndex, { content: newContent }); + await DatabaseService.updateMessage(messageId, { content: newContent }); + + if (isFirstUserMessage && newContent.trim()) + await conversationsStore.applyTitleFromContent(activeConv.id, newContent); + + const messagesToRemove = conversationsStore.activeMessages.slice(messageIndex + 1); + + if (messagesToRemove.length > 0) + await DatabaseService.deleteMessageCascading(activeConv.id, messagesToRemove[0].id); + + conversationsStore.sliceActiveMessages(messageIndex + 1); + conversationsStore.updateConversationTimestamp(); + this.host.setChatLoading(activeConv.id, true); + this.host.clearChatStreaming(activeConv.id); + const assistantMessage = await this.host.createAssistantMessage(); + + conversationsStore.addMessageToActive(assistantMessage); + await conversationsStore.updateCurrentNode(assistantMessage.id); + await this.host.streamChatCompletion( + conversationsStore.activeMessages.slice(0, -1), + assistantMessage, + undefined, + () => { + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(messageId), { + content: originalContent + }); + } + ); + } catch (error) { + if (!isAbortError(error)) console.error('Failed to update message:', error); + } + } + + /** + * Open a fresh assistant turn anchored at the last tool result of a resolved + * agentic round and let streamChatCompletion route through runAgenticFlow. + * Used by continueAssistantMessage when classifyContinueIntent returns + * next_turn, meaning the target assistant already has its tool_calls paired + * with trailing tool results and the next thing to generate is a brand new + * turn rather than a token level continuation. + */ + private async continueAsNextAgenticTurn(anchorIndex: number): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return; + + const anchor = conversationsStore.activeMessages[anchorIndex]; + + if (!anchor) return; + + this.host.cancelPreEncode(); + this.host.setChatLoading(activeConv.id, true); + this.host.clearChatStreaming(activeConv.id); + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const anchorMessage = findMessageById(allMessages, anchor.id); + + if (!anchorMessage) { + this.host.setChatLoading(activeConv.id, false); + + return; + } + + const newAssistantMessage = await DatabaseService.createMessageBranch( + { + children: [], + content: '', + convId: activeConv.id, + model: null, + role: MessageRole.ASSISTANT, + timestamp: Date.now(), + toolCalls: '', + type: MessageType.TEXT + }, + anchorMessage.id + ); + + await conversationsStore.updateCurrentNode(newAssistantMessage.id); + conversationsStore.updateConversationTimestamp(); + await conversationsStore.refreshActiveMessages(); + const conversationPath = filterByLeafNodeId( + allMessages, + anchorMessage.id, + false + ) as DatabaseMessage[]; + + await this.host.streamChatCompletion(conversationPath, newAssistantMessage); + } catch (error) { + if (!isAbortError(error)) console.error('Failed to continue agentic turn:', error); + + this.host.setChatLoading(activeConv.id, false); + } + } + + private async generateResponseForMessage(userMessageId: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return; + + this.host.showErrorDialog(null); + this.host.setChatLoading(activeConv.id, true); + this.host.clearChatStreaming(activeConv.id); + + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const conversationPath = filterByLeafNodeId( + allMessages, + userMessageId, + false + ) as DatabaseMessage[]; + const assistantMessage = await DatabaseService.createMessageBranch( + { + children: [], + content: '', + convId: activeConv.id, + model: null, + role: MessageRole.ASSISTANT, + timestamp: Date.now(), + toolCalls: '', + type: MessageType.TEXT + }, + userMessageId + ); + + conversationsStore.addMessageToActive(assistantMessage); + + await this.host.streamChatCompletion(conversationPath, assistantMessage); + } catch (error) { + console.error('Failed to generate response:', error); + this.host.setChatLoading(activeConv.id, false); + } + } + + private getMessageByIdWithRole( + messageId: string, + expectedRole?: MessageRole + ): { message: DatabaseMessage; index: number } | null { + const index = conversationsStore.findMessageIndex(messageId); + + if (index === -1) return null; + + const message = conversationsStore.activeMessages[index]; + + if (expectedRole && message.role !== expectedRole) return null; + + return { index, message }; + } +} diff --git a/tools/ui/src/lib/stores/chat/index.svelte.ts b/tools/ui/src/lib/stores/chat/index.svelte.ts new file mode 100644 index 000000000..aab824fd7 --- /dev/null +++ b/tools/ui/src/lib/stores/chat/index.svelte.ts @@ -0,0 +1,1441 @@ +/** + * chatStore - Chat lifecycle, streaming and message operations + * + * Owns the active conversation's chat state: sending messages, streaming + * responses, editing/regeneration flows and per-conversation processing + * activity. Composes the stream manager, message flows, activity ledger and + * processing snapshot; persists through conversationsStore. + * + * Uses ChatService for the API layer and conversationsStore for persistence. + */ + +import { CWD_CLEARED_TEXT, SYSTEM_MESSAGE_PLACEHOLDER, TITLE_GENERATION } from '$lib/constants'; +import { + ErrorDialogType, + MessageRole, + MessageType, + ReasoningEffort, + StreamConnectionState +} from '$lib/enums'; +import { ChatService } from '$lib/services/chat.service'; +import { DatabaseService } from '$lib/services/database.service'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { agenticStore } from '$lib/stores/agentic/index.svelte'; +import { chatActivityStore } from '$lib/stores/chat/activity.svelte'; +import { type ChatFlowsHost, ChatMessageFlows } from '$lib/stores/chat/flows.svelte'; +import { chatProcessingStore } from '$lib/stores/chat/processing.svelte'; +import { type ChatStreamHost, ChatStreamManager } from '$lib/stores/chat/streams.svelte'; +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import { mcpStore } from '$lib/stores/mcp/index.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { serverStore } from '$lib/stores/server.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; +import { toolsStore } from '$lib/stores/tools.svelte'; +import type { + ApiChatMessageData, + ChatMessagePromptProgress, + ChatMessageTimings, + ChatStreamCallbacks, + DatabaseMessage, + DatabaseMessageExtra, + ErrorDialogState +} from '$lib/types'; +import { + findMessageById, + formatCwdMessage, + getConversationModel, + isAbortError, + normalizeModelName +} from '$lib/utils'; +import { SvelteMap } from 'svelte/reactivity'; + +class ChatStore implements ChatStreamHost, ChatFlowsHost { + chatReasoningStates = new SvelteMap(); + chatStreamingStates = new SvelteMap< + string, + { response: string; messageId: string; model?: string | null } + >(); + currentResponse = $state(''); + errorDialogState = $state(null); + // true while the active conversation has a local pipe (send, attach or resume-wait) + isLoading = $derived(this.activity.isLocal(conversationsStore.activeConversation?.id ?? '')); + // true while the active conversation streams reasoning content but no visible content yet + isReasoning = $derived( + this.chatReasoningStates.get(conversationsStore.activeConversation?.id ?? '') ?? false + ); + pendingEditMessageId = $state(null); + // resumable stream connection state for the active conversation + // streaming -> bytes flowing normally, resuming -> waiting on /v1/stream reconnect, lost -> unrecoverable + streamConnectionState = $state(StreamConnectionState.STREAMING); + private abortControllers = new SvelteMap(); + private addFilesHandler: ((files: File[]) => void) | null = $state(null); + // message flows: edit, regenerate, continue, delete + private flows = new ChatMessageFlows(this); + private isEditModeActive = $state(false); + private pendingDraftFiles = $state([]); + private pendingDraftMessage = $state(''); + /** Reactive: queued pending messages for non-agentic streaming */ + private pendingMessages = new SvelteMap< + string, + { content: string; extras?: DatabaseMessageExtra[] } + >(); + private preEncodeAbortController: AbortController | null = null; + + // server-side stream sessions: discovery, attach/replay, resume retry, remote sync + private streams = new ChatStreamManager(this); + + /** Conv activity (local pipe / remote session), composed here. */ + get activity() { + return chatActivityStore; + } + + /** Processing state, composed here so consumers have a single chat scope. */ + get processing() { + return chatProcessingStore; + } + + /** + * Abort the current agentic flow signal without clearing loading state. + * Used by "Send immediately" to force the agentic loop to exit so that + * the pending steering message can be re-sent. + * + * Any tool calls captured mid-stream are dropped before the abort so the + * pending message (or a manual follow-up) does not re-send a half-received + * tool call with invalid JSON arguments to the server. Mirrors what the + * Stop button already does through stopGenerationForChat. + */ + async abortCurrentFlow(convId: string): Promise { + await this.savePartialResponseIfNeeded(convId); + const c = this.abortControllers.get(convId); + + if (c) { + c.abort(); + this.abortControllers.delete(convId); + } + } + + async addMessage( + role: MessageRole, + content: string, + type: MessageType = MessageType.TEXT, + parent: string = '-1', + extras?: DatabaseMessageExtra[], + isSynthetic?: boolean + ): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) throw new Error('No active conversation'); + + let parentId: string | null = null; + + if (parent === '-1') { + const am = conversationsStore.activeMessages; + + if (am.length > 0) parentId = am[am.length - 1].id; + else { + const all = await conversationsStore.getConversationMessages(activeConv.id); + const r = all.find((m) => m.parent === null && m.type === 'root'); + + parentId = r ? r.id : await DatabaseService.createRootMessage(activeConv.id); + } + } else parentId = parent; + + const message = await DatabaseService.createMessageBranch( + { + children: [], + content, + convId: activeConv.id, + extra: extras, + isSynthetic, + role, + timestamp: Date.now(), + toolCalls: '', + type + }, + parentId + ); + + conversationsStore.addMessageToActive(message); + await conversationsStore.updateCurrentNode(message.id); + conversationsStore.updateConversationTimestamp(); + + return message; + } + async addSystemPrompt(): Promise { + let activeConv = conversationsStore.activeConversation; + + if (!activeConv) { + await conversationsStore.createConversation(); + activeConv = conversationsStore.activeConversation; + } + + if (!activeConv) return; + + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); + const rootId = rootMessage + ? rootMessage.id + : await DatabaseService.createRootMessage(activeConv.id); + const existingSystemMessage = allMessages.find( + (m) => m.role === MessageRole.SYSTEM && m.parent === rootId + ); + + if (existingSystemMessage) { + this.pendingEditMessageId = existingSystemMessage.id; + + if (!conversationsStore.activeMessages.some((m) => m.id === existingSystemMessage.id)) + conversationsStore.activeMessages.unshift(existingSystemMessage); + + return; + } + + const am = conversationsStore.activeMessages; + const firstActiveMessage = am.find((m) => m.parent === rootId); + const systemMessage = await DatabaseService.createSystemMessage( + activeConv.id, + SYSTEM_MESSAGE_PLACEHOLDER, + rootId + ); + + if (firstActiveMessage) { + await DatabaseService.updateMessage(firstActiveMessage.id, { + parent: systemMessage.id + }); + await DatabaseService.updateMessage(systemMessage.id, { + children: [firstActiveMessage.id] + }); + const updatedRootChildren = rootMessage + ? rootMessage.children.filter((id: string) => id !== firstActiveMessage.id) + : []; + + await DatabaseService.updateMessage(rootId, { + children: [ + ...updatedRootChildren.filter((id: string) => id !== systemMessage.id), + systemMessage.id + ] + }); + const firstMsgIndex = conversationsStore.findMessageIndex(firstActiveMessage.id); + + if (firstMsgIndex !== -1) + conversationsStore.updateMessageAtIndex(firstMsgIndex, { + parent: systemMessage.id + }); + } + + conversationsStore.activeMessages.unshift(systemMessage); + this.pendingEditMessageId = systemMessage.id; + conversationsStore.updateConversationTimestamp(); + } catch (error) { + console.error('Failed to add system prompt:', error); + } + } + cancelPreEncode(): void { + if (this.preEncodeAbortController) { + this.preEncodeAbortController.abort(); + this.preEncodeAbortController = null; + } + } + + /** + * Resets the loading, streaming and processing state for a conversation + * after a generation ends or errors. Shared by the flows' exit paths. + */ + cleanupStreaming(convId: string): void { + this.setChatLoading(convId, false); + this.clearChatStreaming(convId); + this.processing.setState(convId, null); + } + clearChatStreaming(convId: string, messageId?: string): void { + // session aware: a stale generation must not wipe a newer one's streaming state on the + // same conversation, that would drop the frozen stop identity and stop the wrong session + if (messageId !== undefined) { + const cur = this.chatStreamingStates.get(convId); + + if (cur && cur.messageId !== messageId) return; + } + + this.chatStreamingStates.delete(convId); + + if (convId === conversationsStore.activeConversation?.id) this.currentResponse = ''; + } + clearEditMode(): void { + this.isEditModeActive = false; + this.addFilesHandler = null; + } + + clearPendingEditMessageId(): void { + this.pendingEditMessageId = null; + } + + clearPendingMessage(convId: string): void { + this.pendingMessages.delete(convId); + } + + /** Reset per-view state when (re)mounting the empty chat screen. */ + clearUIState(): void { + this.currentResponse = ''; + } + + consumePendingDraft(): { message: string; files: ChatUploadedFile[] } | null { + if (!this.pendingDraftMessage && this.pendingDraftFiles.length === 0) return null; + + const d = { files: [...this.pendingDraftFiles], message: this.pendingDraftMessage }; + + this.pendingDraftMessage = ''; + this.pendingDraftFiles = []; + + return d; + } + + consumePendingMessage( + convId: string + ): { content: string; extras?: DatabaseMessageExtra[] } | null { + const msg = this.pendingMessages.get(convId); + + if (!msg) return null; + + this.pendingMessages.delete(convId); + + return msg; + } + + async continueAssistantMessage(messageId: string): Promise { + return this.flows.continueAssistantMessage(messageId); + } + + async createAssistantMessage(parentId?: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) throw new Error('No active conversation'); + + return await DatabaseService.createMessageBranch( + { + children: [], + content: '', + convId: activeConv.id, + model: null, + role: MessageRole.ASSISTANT, + timestamp: Date.now(), + toolCalls: '', + type: MessageType.TEXT + }, + parentId || null + ); + } + + async deleteMessage(messageId: string): Promise { + return this.flows.deleteMessage(messageId); + } + + /** + * Server-side stream sessions (discovery, attach/replay, resume retry, + * remote-running snapshot) live in ChatStreamManager. + */ + async discoverActiveStream(convId: string): Promise { + return this.streams.discoverActiveStream(convId); + } + + dismissErrorDialog(): void { + this.errorDialogState = null; + } + + async editAssistantMessage( + messageId: string, + newContent: string, + shouldBranch: boolean + ): Promise { + return this.flows.editAssistantMessage(messageId, newContent, shouldBranch); + } + + async editMessageWithBranching( + messageId: string, + newContent: string, + newExtras?: DatabaseMessageExtra[] + ): Promise { + return this.flows.editMessageWithBranching(messageId, newContent, newExtras); + } + + async editUserMessagePreserveResponses( + messageId: string, + newContent: string, + newExtras?: DatabaseMessageExtra[] + ): Promise { + return this.flows.editUserMessagePreserveResponses(messageId, newContent, newExtras); + } + + getAddFilesHandler(): ((files: File[]) => void) | null { + return this.addFilesHandler; + } + + /** Convs with any activity (local pipe or remote session), sidebar spinners. */ + getAllLoadingChats(): string[] { + return this.activity.loadingConvs; + } + + getApiOptions(): Record { + const currentConfig = settingsStore.config; + const hasValue = (value: unknown): boolean => + value !== undefined && value !== null && value !== ''; + const apiOptions: Record = { stream: true, timings_per_token: true }; + + if (serverStore.isRouterMode) { + const modelName = modelsStore.selectedModelName; + + if (modelName) apiOptions.model = modelName; + } + + if (currentConfig.systemMessage) apiOptions.systemMessage = currentConfig.systemMessage; + + if (currentConfig.disableReasoningParsing) apiOptions.disableReasoningParsing = true; + + if (currentConfig.excludeReasoningFromContext) apiOptions.excludeReasoningFromContext = true; + + // an explicit reasoning choice overrides the server default, DEFAULT sends nothing + const effort = conversationsStore.preferences.getReasoningEffort(); + + if (effort !== ReasoningEffort.DEFAULT) { + apiOptions.enableThinking = effort !== ReasoningEffort.OFF; + + if (effort !== ReasoningEffort.OFF) apiOptions.reasoningEffort = effort; + } + + if (hasValue(currentConfig.temperature)) + apiOptions.temperature = Number(currentConfig.temperature); + + if (hasValue(currentConfig.max_tokens)) + apiOptions.max_tokens = Number(currentConfig.max_tokens); + + if (hasValue(currentConfig.dynatemp_range)) + apiOptions.dynatemp_range = Number(currentConfig.dynatemp_range); + + if (hasValue(currentConfig.dynatemp_exponent)) + apiOptions.dynatemp_exponent = Number(currentConfig.dynatemp_exponent); + + if (hasValue(currentConfig.top_k)) apiOptions.top_k = Number(currentConfig.top_k); + + if (hasValue(currentConfig.top_p)) apiOptions.top_p = Number(currentConfig.top_p); + + if (hasValue(currentConfig.min_p)) apiOptions.min_p = Number(currentConfig.min_p); + + if (hasValue(currentConfig.xtc_probability)) + apiOptions.xtc_probability = Number(currentConfig.xtc_probability); + + if (hasValue(currentConfig.xtc_threshold)) + apiOptions.xtc_threshold = Number(currentConfig.xtc_threshold); + + if (hasValue(currentConfig.typ_p)) apiOptions.typ_p = Number(currentConfig.typ_p); + + if (hasValue(currentConfig.repeat_last_n)) + apiOptions.repeat_last_n = Number(currentConfig.repeat_last_n); + + if (hasValue(currentConfig.repeat_penalty)) + apiOptions.repeat_penalty = Number(currentConfig.repeat_penalty); + + if (hasValue(currentConfig.presence_penalty)) + apiOptions.presence_penalty = Number(currentConfig.presence_penalty); + + if (hasValue(currentConfig.frequency_penalty)) + apiOptions.frequency_penalty = Number(currentConfig.frequency_penalty); + + if (hasValue(currentConfig.dry_multiplier)) + apiOptions.dry_multiplier = Number(currentConfig.dry_multiplier); + + if (hasValue(currentConfig.dry_base)) apiOptions.dry_base = Number(currentConfig.dry_base); + + if (hasValue(currentConfig.dry_allowed_length)) + apiOptions.dry_allowed_length = Number(currentConfig.dry_allowed_length); + + if (hasValue(currentConfig.dry_penalty_last_n)) + apiOptions.dry_penalty_last_n = Number(currentConfig.dry_penalty_last_n); + + if (currentConfig.samplers) apiOptions.samplers = currentConfig.samplers; + + if (hasValue(currentConfig.backend_sampling)) + apiOptions.backend_sampling = currentConfig.backend_sampling; + + if (currentConfig.customJson) apiOptions.custom = currentConfig.customJson; + + return apiOptions; + } + + getChatStreaming(convId: string): { response: string; messageId: string } | undefined { + return this.getChatStreamingState(convId); + } + + async getDeletionInfo(messageId: string): Promise<{ + totalCount: number; + userMessages: number; + assistantMessages: number; + messageTypes: string[]; + }> { + return this.flows.getDeletionInfo(messageId); + } + + getOrCreateAbortController(convId: string): AbortController { + let c = this.abortControllers.get(convId); + + if (!c || c.signal.aborted) { + c = new AbortController(); + this.abortControllers.set(convId, c); + } + + return c; + } + + getPendingMessageContent(convId: string): string | null { + return this.pendingMessages.get(convId)?.content ?? null; + } + + getPendingMessageExtras(convId: string): DatabaseMessageExtra[] | undefined { + return this.pendingMessages.get(convId)?.extras; + } + + getResumeModel(convId: string): string | null { + return this.streams.getResumeModel(convId); + } + + hasPendingDraft(): boolean { + return Boolean(this.pendingDraftMessage) || this.pendingDraftFiles.length > 0; + } + + hasPendingMessage(convId: string): boolean { + return this.pendingMessages.has(convId); + } + + injectPendingMessage(convId: string, content: string, extras?: DatabaseMessageExtra[]): void { + this.pendingMessages.set(convId, { content, extras }); + } + + isChatLoading(convId: string): boolean { + return this.activity.isLocal(convId); + } + + isChatLoadingInternal(convId: string): boolean { + return this.activity.isLocal(convId) || this.chatStreamingStates.has(convId); + } + + isEditing(): boolean { + return this.isEditModeActive; + } + + /** True while the active conversation has a live streaming pipe. */ + isStreaming(): boolean { + return this.chatStreamingStates.has(conversationsStore.activeConversation?.id ?? ''); + } + + /** + * Record a working-directory change into chat history as a synthetic + * user message, so the model sees it on its next turn (the client + * sends the cwd itself via the x-tool-cwd header on tool calls). + * A plain user message is used because some chat templates reject + * tool messages without a preceding tool call. + */ + async recordCwdChange(cwd: string | null): Promise { + const content = cwd + ? formatCwdMessage(cwd, await toolsStore.resolveServerHome()) + : CWD_CLEARED_TEXT; + // Reuse the trailing cwd row when it is already the last message, so + // repeated picks update it in place instead of stacking another row. + const last = conversationsStore.activeMessages[conversationsStore.activeMessages.length - 1]; + + if (last && last.role === MessageRole.USER && last.isSynthetic === true) { + await DatabaseService.updateMessage(last.id, { content, isSynthetic: true }); + conversationsStore.updateMessageAtIndex(conversationsStore.activeMessages.length - 1, { + content, + isSynthetic: true + }); + + return; + } + + await this.addMessage(MessageRole.USER, content, MessageType.TEXT, '-1', undefined, true); + } + + async regenerateMessage(messageId: string): Promise { + return this.flows.regenerateMessage(messageId); + } + + async regenerateMessageWithBranching(messageId: string, modelOverride?: string): Promise { + return this.flows.regenerateMessageWithBranching(messageId, modelOverride); + } + + async removeSystemPromptPlaceholder(messageId: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return false; + + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const systemMessage = findMessageById(allMessages, messageId); + + if (!systemMessage || systemMessage.role !== MessageRole.SYSTEM) return false; + + const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); + + if (!rootMessage) return false; + + if (allMessages.length === 2 && systemMessage.children.length === 0) { + await conversationsStore.deleteConversation(activeConv.id); + + return true; + } + + for (const childId of systemMessage.children) { + await DatabaseService.updateMessage(childId, { parent: rootMessage.id }); + const childIndex = conversationsStore.findMessageIndex(childId); + + if (childIndex !== -1) + conversationsStore.updateMessageAtIndex(childIndex, { parent: rootMessage.id }); + } + await DatabaseService.updateMessage(rootMessage.id, { + children: [ + ...rootMessage.children.filter((id: string) => id !== messageId), + ...systemMessage.children + ] + }); + await DatabaseService.deleteMessage(messageId); + const systemIndex = conversationsStore.findMessageIndex(messageId); + + if (systemIndex !== -1) conversationsStore.activeMessages.splice(systemIndex, 1); + + conversationsStore.updateConversationTimestamp(); + + return false; + } catch (error) { + console.error('Failed to remove system prompt placeholder:', error); + + return false; + } + } + + savePendingDraft(message: string, files: ChatUploadedFile[]): void { + this.pendingDraftMessage = message; + this.pendingDraftFiles = [...files]; + } + async sendMessage(content: string, extras?: DatabaseMessageExtra[]): Promise { + if (!content.trim() && (!extras || extras.length === 0)) return; + + const activeConv = conversationsStore.activeConversation; + + // If agentic loop is running, inject as a steering message instead of starting a new flow + if (activeConv && agenticStore.isRunning(activeConv.id)) { + agenticStore.injectSteeringMessage(activeConv.id, content, extras); + + return; + } + + // If non-agentic streaming is active, queue as a pending message to send after completion + if (activeConv && this.isChatLoadingInternal(activeConv.id)) { + this.injectPendingMessage(activeConv.id, content, extras); + + return; + } + + // Cancel any in-flight pre-encode request + this.cancelPreEncode(); + + // Consume MCP resource attachments - converts them to extras and clears the live store + const resourceExtras = mcpStore.consumeResourceAttachmentsAsExtras(); + const allExtras = resourceExtras.length > 0 ? [...(extras || []), ...resourceExtras] : extras; + + let isNewConversation = false; + + if (!activeConv) { + await conversationsStore.createConversation(); + isNewConversation = true; + } + + const currentConv = conversationsStore.activeConversation; + + if (!currentConv) return; + + this.showErrorDialog(null); + this.setChatLoading(currentConv.id, true); + this.clearChatStreaming(currentConv.id); + try { + let parentIdForUserMessage: string | undefined; + + if (isNewConversation) { + const rootId = await DatabaseService.createRootMessage(currentConv.id); + const currentConfig = settingsStore.config; + const systemPrompt = currentConfig.systemMessage?.toString().trim(); + + let sysOrRootId = rootId; + + if (systemPrompt) { + const systemMessage = await DatabaseService.createSystemMessage( + currentConv.id, + systemPrompt, + rootId + ); + + conversationsStore.addMessageToActive(systemMessage); + sysOrRootId = systemMessage.id; + } + + // Reflect a working directory picked on the new-chat screen into + // chat history before the first user message, so the model sees + // it on its first turn. createConversation() has already threaded + // the pending pick onto the conversation. + if (currentConv.cwd) { + const cwdMessage = await this.addMessage( + MessageRole.USER, + formatCwdMessage(currentConv.cwd, await toolsStore.resolveServerHome()), + MessageType.TEXT, + sysOrRootId, + undefined, + true + ); + + parentIdForUserMessage = cwdMessage.id; + } else { + parentIdForUserMessage = sysOrRootId; + } + } + + const userMessage = await this.addMessage( + MessageRole.USER, + content, + MessageType.TEXT, + parentIdForUserMessage ?? '-1', + allExtras + ); + + if (isNewConversation && content) + await conversationsStore.applyTitleFromContent(currentConv.id, content); + + const assistantMessage = await this.createAssistantMessage(userMessage.id); + + conversationsStore.addMessageToActive(assistantMessage); + await this.streamChatCompletion( + conversationsStore.activeMessages.slice(0, -1), + assistantMessage, + undefined, + undefined, + undefined, + settingsStore.config.titleGenerationUseLLM && isNewConversation ? content : undefined + ); + } catch (error) { + if (isAbortError(error)) { + this.setChatLoading(currentConv.id, false); + + return; + } + + console.error('Failed to send message:', error); + this.setChatLoading(currentConv.id, false); + const dialogType = + error instanceof Error && error.name === 'TimeoutError' + ? ErrorDialogType.TIMEOUT + : ErrorDialogType.SERVER; + const contextInfo = ( + error as Error & { contextInfo?: { n_prompt_tokens: number; n_ctx: number } } + ).contextInfo; + + this.showErrorDialog({ + contextInfo, + message: error instanceof Error ? error.message : 'Unknown error', + type: dialogType + }); + } + } + + setChatLoading(convId: string, loading: boolean): void { + if (loading) { + this.activity.markLocal(convId); + } else { + this.activity.localEnded(convId); + this.setChatReasoning(convId, false); + } + } + + setChatReasoning(convId: string, reasoning: boolean): void { + if (reasoning) this.chatReasoningStates.set(convId, true); + else this.chatReasoningStates.delete(convId); + } + + setChatStreaming( + convId: string, + response: string, + messageId: string, + model?: string | null + ): void { + this.chatStreamingStates.set(convId, { + messageId, + model: model ?? this.chatStreamingStates.get(convId)?.model, + response + }); + + if (convId === conversationsStore.activeConversation?.id) this.currentResponse = response; + } + + setEditModeActive(handler: (files: File[]) => void): void { + this.isEditModeActive = true; + this.addFilesHandler = handler; + } + + showErrorDialog(state: ErrorDialogState | null): void { + this.errorDialogState = state; + } + + async stopGeneration(): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return; + + await this.stopGenerationForChat(activeConv.id); + } + + async stopGenerationForChat(convId: string): Promise { + await this.savePartialResponseIfNeeded(convId); + // tell the server to stop the generation, not just drop the HTTP socket. without this the + // detached drain keeps producing tokens until eos or max_tokens. use the frozen identity + // captured when the session started, not the live dropdown + const streamStateForStop = this.chatStreamingStates.get(convId); + const modelForStop = streamStateForStop?.model ?? ChatService.getStreamState(convId)?.model; + + void ChatService.cancelServerStream(convId, modelForStop); + // an explicit stop leaves nothing to resume and kills a pending resume retry + ChatService.clearStreamState(convId); + this.streams.cancelResumeRetry(convId); + this.abortRequest(convId); + this.setChatLoading(convId, false); + this.clearChatStreaming(convId); + this.processing.setState(convId, null); + this.clearPendingMessage(convId); + } + + async streamChatCompletion( + allMessages: DatabaseMessage[], + assistantMessage: DatabaseMessage, + onComplete?: (content: string) => Promise, + onError?: (error: Error) => void, + modelOverride?: string | null, + firstUserMessageContent?: string + ): Promise { + // the ::model suffix in the stream identity is only for router mode, where it routes to the + // owning child. in single-model mode the identity stays the bare conv id so that attach, stop + // and reattach all agree, regardless of fresh send vs regenerate passing a resolved model + let effectiveModel: string | null | undefined = undefined; + + if (serverStore.isRouterMode) { + const conversationModel = getConversationModel(allMessages); + + effectiveModel = modelOverride || modelsStore.selectedModelName || conversationModel; + } + + if (serverStore.isRouterMode && effectiveModel) { + if (!modelsStore.props.getModelProps(effectiveModel)) + await modelsStore.props.fetchModelProps(effectiveModel); + } + + // Mutable state for the current message being streamed + let currentMessageId = assistantMessage.id; + let streamedContent = ''; + let streamedReasoningContent = ''; + let resolvedModel: string | null = null; + let modelPersisted = false; + + const convId = assistantMessage.convId; + + // Tracks the last message created in this flow. Used as the parent for the next + // turn's assistant message so createAssistantMessage does not have to read + // conversationsStore.activeMessages, which may belong to a different conversation + // after the user navigates while the loop is still running. + let lastCreatedInFlow = currentMessageId; + + // freeze the POST identity from t0 so a stop cancels with the exact session key, + // never a stale or empty model resolved later + this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); + + const recordModel = (modelName: string | null | undefined, persistImmediately = true): void => { + if (!modelName) return; + + const n = normalizeModelName(modelName); + + if (!n || n === resolvedModel) return; + + resolvedModel = n; + const idx = conversationsStore.findMessageIndex(currentMessageId); + + conversationsStore.updateMessageAtIndex(idx, { model: n }); + + if (persistImmediately && !modelPersisted) { + modelPersisted = true; + DatabaseService.updateMessage(currentMessageId, { model: n }).catch(() => { + modelPersisted = false; + resolvedModel = null; + }); + } + }; + + let completionIdRecorded = false; + + const recordCompletionId = (id: string): void => { + if (!id || completionIdRecorded) return; + + completionIdRecorded = true; + const idx = conversationsStore.findMessageIndex(currentMessageId); + + conversationsStore.updateMessageAtIndex(idx, { completionId: id }); + DatabaseService.updateMessage(currentMessageId, { completionId: id }).catch(() => { + completionIdRecorded = false; + }); + }; + const updateStreamingUI = () => { + this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); + const idx = conversationsStore.findMessageIndex(currentMessageId); + + conversationsStore.updateMessageAtIndex(idx, { content: streamedContent }); + }; + const cleanupStreamingState = () => { + this.setChatLoading(convId, false); + this.clearChatStreaming(convId, currentMessageId); + this.processing.setState(convId, null); + }; + + this.processing.setActiveConversation(convId); + const abortController = this.getOrCreateAbortController(convId); + const streamCallbacks: ChatStreamCallbacks = { + createAssistantMessage: async () => { + // Reset streaming state for new message + streamedContent = ''; + streamedReasoningContent = ''; + + const msg = await DatabaseService.createMessageBranch( + { + children: [], + content: '', + convId, + model: resolvedModel, + role: MessageRole.ASSISTANT, + timestamp: Date.now(), + toolCalls: '', + type: MessageType.TEXT + }, + lastCreatedInFlow + ); + + if (conversationsStore.activeConversation?.id === convId) { + conversationsStore.addMessageToActive(msg); + } + + currentMessageId = msg.id; + lastCreatedInFlow = msg.id; + + return msg; + }, + createToolResultMessage: async ( + toolCallId: string, + content: string, + extras?: DatabaseMessageExtra[], + toolCwd?: string + ) => { + const msg = await DatabaseService.createMessageBranch( + { + children: [], + content, + convId, + extra: extras, + role: MessageRole.TOOL, + timestamp: Date.now(), + toolCallId, + toolCalls: '', + toolCwd, + type: MessageType.TEXT + }, + currentMessageId + ); + + // mirror into the active store and move the node pointer only when this + // conversation is displayed; otherwise persist the node move straight to + // the db for the owning conv so a foreign conv's currNode stays untouched + if (conversationsStore.activeConversation?.id === convId) { + conversationsStore.addMessageToActive(msg); + await conversationsStore.updateCurrentNode(msg.id); + } else { + await DatabaseService.updateCurrentNode(convId, msg.id); + } + + lastCreatedInFlow = msg.id; + + return msg; + }, + onAssistantTurnComplete: async ( + content: string, + reasoningContent: string | undefined, + timings: ChatMessageTimings | undefined, + toolCalls: import('$lib/types/api').ApiChatCompletionToolCall[] | undefined + ) => { + const updateData: Record = { + content, + reasoningContent: reasoningContent || undefined, + timings, + toolCalls: toolCalls ? JSON.stringify(toolCalls) : '' + }; + + if (resolvedModel && !modelPersisted) updateData.model = resolvedModel; + + await DatabaseService.updateMessage(currentMessageId, updateData); + const idx = conversationsStore.findMessageIndex(currentMessageId); + const uiUpdate: Partial = { + content, + reasoningContent: reasoningContent || undefined, + toolCalls: toolCalls ? JSON.stringify(toolCalls) : '' + }; + + if (timings) uiUpdate.timings = timings; + + if (resolvedModel) uiUpdate.model = resolvedModel; + + // touch the active ui array and node pointer only when this conversation + // is displayed; otherwise persist the node move straight to the db so a + // foreign conv's currNode stays untouched + if (conversationsStore.activeConversation?.id === convId) { + conversationsStore.updateMessageAtIndex(idx, uiUpdate); + await conversationsStore.updateCurrentNode(currentMessageId); + } else { + await DatabaseService.updateCurrentNode(convId, currentMessageId); + } + }, + onAttachments: (messageId: string, extras: DatabaseMessageExtra[]) => { + if (!extras.length) return; + + const idx = conversationsStore.findMessageIndex(messageId); + + if (idx === -1) return; + + const msg = conversationsStore.activeMessages[idx]; + const updatedExtras = [...(msg.extra || []), ...extras]; + + conversationsStore.updateMessageAtIndex(idx, { extra: updatedExtras }); + DatabaseService.updateMessage(messageId, { extra: updatedExtras }).catch(console.error); + }, + onChunk: (chunk: string) => { + streamedContent += chunk; + updateStreamingUI(); + this.setChatReasoning(convId, false); + }, + onCompletionId: (id: string) => recordCompletionId(id), + onError: async (error: Error) => { + if (isAbortError(error)) { + cleanupStreamingState(); + // If aborted with a pending message (e.g. "Send immediately"), re-send it + const pending = this.consumePendingMessage(convId); + + if (pending) { + this.sendMessage(pending.content, pending.extras); + } + + return; + } + + console.error('Streaming error:', error); + // keep whatever was streamed so far, the message stays in memory and in DB + await this.savePartialResponseIfNeeded(convId); + cleanupStreamingState(); + this.clearPendingMessage(convId); + + const contextInfo = ( + error as Error & { contextInfo?: { n_prompt_tokens: number; n_ctx: number } } + ).contextInfo; + + this.showErrorDialog({ + contextInfo, + message: error.message, + type: error.name === 'TimeoutError' ? ErrorDialogType.TIMEOUT : ErrorDialogType.SERVER + }); + + if (onError) onError(error); + }, + onFlowComplete: (finalTimings?: ChatMessageTimings) => { + if (finalTimings) { + const idx = conversationsStore.findMessageIndex(assistantMessage.id); + + conversationsStore.updateMessageAtIndex(idx, { timings: finalTimings }); + DatabaseService.updateMessage(assistantMessage.id, { + timings: finalTimings + }).catch(console.error); + } + + cleanupStreamingState(); + + if (onComplete) onComplete(streamedContent); + + if (serverStore.isRouterMode) modelsStore.fetchRouterModels().catch(console.error); + + // Pre-encode conversation in KV cache for faster next turn + if (settingsStore.config.preEncodeConversation) { + this.triggerPreEncode( + allMessages, + assistantMessage, + streamedContent, + effectiveModel, + !!settingsStore.config.excludeReasoningFromContext + ); + } + }, + onModel: (modelName: string) => recordModel(modelName), + onReasoningChunk: (chunk: string) => { + streamedReasoningContent += chunk; + // mark streaming state so a stop mid-thinking can persist the partial reasoning + this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); + const idx = conversationsStore.findMessageIndex(currentMessageId); + + conversationsStore.updateMessageAtIndex(idx, { + reasoningContent: streamedReasoningContent + }); + this.setChatReasoning(convId, true); + }, + onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { + this.processing.applyStreamTimings(timings, promptProgress, convId); + }, + onToolCallsStreaming: (toolCalls) => { + const idx = conversationsStore.findMessageIndex(currentMessageId); + + conversationsStore.updateMessageAtIndex(idx, { + toolCalls: JSON.stringify(toolCalls) + }); + }, + onTurnComplete: (intermediateTimings: ChatMessageTimings) => { + // Update the first assistant message with cumulative agentic timings + const idx = conversationsStore.findMessageIndex(assistantMessage.id); + + conversationsStore.updateMessageAtIndex(idx, { timings: intermediateTimings }); + }, + updateToolResultMessage: async ( + messageId: string, + content: string, + extras?: DatabaseMessageExtra[] + ) => { + // Persist latest content + merged extras; mirror into the active + // store so the chat view sees live updates for streaming tools + // (e.g. exec_shell_command). The existing tool message node + // pointer stays put - the renderer is already scoped to it. + const updates: Partial = { content }; + + if (extras) { + const idx = conversationsStore.findMessageIndex(messageId); + const existing = idx >= 0 ? (conversationsStore.activeMessages[idx]?.extra ?? []) : []; + const merged = [...existing, ...extras]; + + updates.extra = merged; + } + + if (conversationsStore.activeConversation?.id === convId) { + const idx = conversationsStore.findMessageIndex(messageId); + + if (idx >= 0) conversationsStore.updateMessageAtIndex(idx, updates); + } + + await DatabaseService.updateMessage(messageId, updates); + } + }; + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); + + { + const agenticResult = await agenticStore.runAgenticFlow({ + callbacks: streamCallbacks, + conversationId: convId, + flowRootMessageId: assistantMessage.id, + messages: allMessages, + options: { + ...this.getApiOptions(), + ...(effectiveModel ? { model: effectiveModel } : {}) + }, + perChatOverrides, + signal: abortController.signal + }); + + if (agenticResult.handled) { + // Generate LLM based title for new conversations after agentic flow completes + if (firstUserMessageContent) { + await this.generateTitleWithLLM(firstUserMessageContent, streamedContent, convId); + } + + // Check if there's a pending steering message to re-send + const pending = agenticStore.consumePendingSteeringMessage(convId); + + if (pending) { + await this.sendMessage(pending.content, pending.extras); + } + + return; + } + } + + await ChatService.sendMessage( + allMessages, + { + ...this.getApiOptions(), + ...(effectiveModel ? { model: effectiveModel } : {}), + onChunk: streamCallbacks.onChunk, + onComplete: async ( + finalContent?: string, + reasoningContent?: string, + timings?: ChatMessageTimings, + toolCalls?: string + ) => { + const content = streamedContent || finalContent || ''; + const reasoning = streamedReasoningContent || reasoningContent; + const updateData: Record = { + content, + reasoningContent: reasoning || undefined, + timings, + toolCalls: toolCalls || '' + }; + + if (resolvedModel && !modelPersisted) updateData.model = resolvedModel; + + await DatabaseService.updateMessage(currentMessageId, updateData); + const idx = conversationsStore.findMessageIndex(currentMessageId); + const uiUpdate: Partial = { + content, + reasoningContent: reasoning || undefined, + toolCalls: toolCalls || '' + }; + + if (timings) uiUpdate.timings = timings; + + if (resolvedModel) uiUpdate.model = resolvedModel; + + conversationsStore.updateMessageAtIndex(idx, uiUpdate); + await conversationsStore.updateCurrentNode(currentMessageId); + cleanupStreamingState(); + + if (onComplete) await onComplete(content); + + if (serverStore.isRouterMode) modelsStore.fetchRouterModels().catch(console.error); + + // Generate LLM based title for new conversations (avoids stale reference + // issue when user switches conversations while streaming) + if (firstUserMessageContent) { + await this.generateTitleWithLLM(firstUserMessageContent, streamedContent, convId); + } + + // Check if there's a pending message queued during streaming + const pending = this.consumePendingMessage(convId); + + if (pending) { + await this.sendMessage(pending.content, pending.extras); + } + }, + onCompletionId: streamCallbacks.onCompletionId, + onConnectionState: (state: StreamConnectionState) => { + if (convId === conversationsStore.activeConversation?.id) { + this.streamConnectionState = state; + } + }, + onError: streamCallbacks.onError, + onModel: streamCallbacks.onModel, + onReasoningChunk: streamCallbacks.onReasoningChunk, + onTimings: streamCallbacks.onTimings, + stream: true + }, + convId, + abortController.signal + ); + } + + syncLoadingStateForChat(convId: string): void { + const s = this.chatStreamingStates.get(convId); + + this.currentResponse = s?.response || ''; + this.processing.setActiveConversation(convId); + + // Sync streaming content to activeMessages so UI displays current content + if (s?.response && s?.messageId) { + const idx = conversationsStore.findMessageIndex(s.messageId); + + if (idx !== -1) { + conversationsStore.updateMessageAtIndex(idx, { content: s.response }); + } + } + } + + async syncRemoteRunningStreams(): Promise { + return this.streams.syncRemoteRunningStreams(); + } + + /** + * Message flows (edit / regenerate / continue / delete) live in + * ChatMessageFlows; these delegate so consumers keep a single entry point. + */ + async updateMessage(messageId: string, newContent: string): Promise { + return this.flows.updateMessage(messageId, newContent); + } + private abortRequest(convId?: string): void { + if (convId) { + const c = this.abortControllers.get(convId); + + if (c) { + c.abort(); + this.abortControllers.delete(convId); + } + } else { + for (const c of this.abortControllers.values()) c.abort(); + this.abortControllers.clear(); + } + } + + private async generateTitleWithLLM( + userContent: string, + assistantContent: string, + convId: string + ): Promise { + const effectiveModel = + serverStore.isRouterMode && modelsStore.selectedModelName + ? modelsStore.selectedModelName + : undefined; + const configValue = settingsStore.config; + const titlePromptTemplate = + typeof configValue.titleGenerationPrompt === 'string' && + configValue.titleGenerationPrompt.trim() + ? configValue.titleGenerationPrompt + : TITLE_GENERATION.DEFAULT_PROMPT; + const titlePrompt = titlePromptTemplate + .replace('{{USER}}', String(userContent || '')) + .replace('{{ASSISTANT}}', String(assistantContent || '')); + const titleMessage: ApiChatMessageData = { + content: titlePrompt, + role: MessageRole.USER + }; + const titleResponse = await ChatService.generateTitle(titleMessage, effectiveModel); + + if (!titleResponse) { + return; + } + + let cleanTitle = titleResponse.trim(); + + cleanTitle = cleanTitle + .replace(TITLE_GENERATION.PREFIX_PATTERN, '') + .replace(TITLE_GENERATION.QUOTE_PATTERN, '') + .trim(); + + if (!cleanTitle || cleanTitle.length < TITLE_GENERATION.MIN_LENGTH) { + const firstLine = userContent.split('\n').find((l) => l.trim().length > 0); + + cleanTitle = firstLine ? firstLine.trim() : TITLE_GENERATION.FALLBACK; + } + + if (cleanTitle && cleanTitle.length >= TITLE_GENERATION.MIN_LENGTH) { + await conversationsStore.updateConversationName(convId, cleanTitle); + } + } + + private getChatStreamingState( + convId: string + ): { response: string; messageId: string } | undefined { + return this.chatStreamingStates.get(convId); + } + + private async savePartialResponseIfNeeded(convId?: string): Promise { + const conversationId = convId || conversationsStore.activeConversation?.id; + + if (!conversationId) return; + + const streamingState = this.getChatStreamingState(conversationId); + + if (!streamingState) return; + + const messages = + conversationId === conversationsStore.activeConversation?.id + ? conversationsStore.activeMessages + : await conversationsStore.getConversationMessages(conversationId); + + if (!messages.length) return; + + const lastMessage = messages[messages.length - 1]; + + if (lastMessage?.role !== MessageRole.ASSISTANT) return; + + const partialContent = streamingState.response; + const partialReasoning = lastMessage.reasoningContent || ''; + // snapshot the streamed tool calls before clearing so we still know whether + // anything was captured when deciding to skip the DB write below + const hadPartialToolCalls = !!lastMessage.toolCalls?.trim(); + + // nothing to persist when content, reasoning, and streamed tool calls are all empty + // (e.g. stop before any token). otherwise drop the partial tool call and write whatever + // was streamed: incomplete arguments (truncated JSON, missing closing quote) would + // otherwise be re-sent to the server on the next turn and rejected. + if (!partialContent.trim() && !partialReasoning.trim() && !hadPartialToolCalls) return; + + try { + const updateData: { + content?: string; + reasoningContent?: string; + toolCalls?: string; + timings?: ChatMessageTimings; + } = { + toolCalls: '' + }; + + if (partialContent.trim()) updateData.content = partialContent; + + if (partialReasoning.trim()) updateData.reasoningContent = partialReasoning; + + const lastKnownState = this.processing.getState(conversationId); + + if (lastKnownState) { + updateData.timings = { + cache_n: lastKnownState.cacheTokens || 0, + predicted_ms: + lastKnownState.tokensPerSecond && lastKnownState.tokensDecoded + ? (lastKnownState.tokensDecoded / lastKnownState.tokensPerSecond) * 1000 + : undefined, + predicted_n: lastKnownState.tokensDecoded || 0, + prompt_ms: lastKnownState.promptMs, + prompt_n: lastKnownState.promptTokens || 0 + }; + } + + await DatabaseService.updateMessage(lastMessage.id, updateData); + lastMessage.content = partialContent; + // mirror the drop into the in-memory message so the next request sent via + // sendMessage (queued pending, Send immediately, or manual follow-up) reads + // the cleared value, not whatever the streaming widget had been showing + lastMessage.toolCalls = ''; + + if (updateData.timings) lastMessage.timings = updateData.timings; + } catch (error) { + lastMessage.content = partialContent; + lastMessage.toolCalls = ''; + console.error('Failed to save partial response:', error); + } + } + + private async triggerPreEncode( + allMessages: DatabaseMessage[], + assistantMessage: DatabaseMessage, + assistantContent: string, + model?: string | null, + excludeReasoning?: boolean + ): Promise { + this.cancelPreEncode(); + this.preEncodeAbortController = new AbortController(); + + const signal = this.preEncodeAbortController.signal; + + try { + const allIdle = await ChatService.areAllSlotsIdle(model, signal); + + if (!allIdle || signal.aborted) return; + + const messagesWithAssistant: DatabaseMessage[] = [ + ...allMessages, + { ...assistantMessage, content: assistantContent } + ]; + + await ChatService.preEncode(messagesWithAssistant, model, excludeReasoning, signal); + } catch (err) { + if (!isAbortError(err)) { + console.warn('[ChatStore] Pre-encode failed:', err); + } + } + } +} + +export const chatStore = new ChatStore(); diff --git a/tools/ui/src/lib/stores/chat/processing.svelte.ts b/tools/ui/src/lib/stores/chat/processing.svelte.ts new file mode 100644 index 000000000..69c1a6925 --- /dev/null +++ b/tools/ui/src/lib/stores/chat/processing.svelte.ts @@ -0,0 +1,188 @@ +/** + * chatProcessingStore - Per-conversation processing state + * + * Owns the live processing snapshot shown while a conversation streams: + * token counts, tokens/sec, prompt progress. Updated from stream timings, + * restored from persisted message timings when a conversation loads. + * + * Composed under chatStore.processing; not exported from the stores barrel. + */ + +import { MessageRole } from '$lib/enums'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { serverStore } from '$lib/stores/server.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; +import type { + ApiProcessingState, + ChatMessagePromptProgress, + ChatMessageTimings, + DatabaseMessage +} from '$lib/types'; +import { SvelteMap } from 'svelte/reactivity'; + +interface ProcessingTimingData { + cache_n: number; + predicted_n: number; + predicted_per_second: number; + prompt_ms?: number; + prompt_n: number; + prompt_progress?: ChatMessagePromptProgress; +} + +export class ChatProcessingStore { + private _activeConversationId = $state(null); + private states = new SvelteMap(); + + /** Processing state of the conversation currently shown in the UI. */ + activeState = $derived( + this._activeConversationId ? (this.states.get(this._activeConversationId) ?? null) : null + ); + + get activeConversationId(): string | null { + return this._activeConversationId; + } + + /** + * Applies a stream timings event (tokens/sec + token counts) to the given + * conversation's processing state. Shared by the chat and continue flows. + */ + applyStreamTimings( + timings?: ChatMessageTimings, + promptProgress?: ChatMessagePromptProgress, + conversationId?: string + ): void { + const tokensPerSecond = + timings?.predicted_ms && timings?.predicted_n + ? (timings.predicted_n / timings.predicted_ms) * 1000 + : 0; + + this.updateFromTimings( + { + cache_n: timings?.cache_n || 0, + predicted_n: timings?.predicted_n || 0, + predicted_per_second: tokensPerSecond, + prompt_ms: timings?.prompt_ms, + prompt_n: timings?.prompt_n || 0, + prompt_progress: promptProgress + }, + conversationId + ); + } + + getConversationIds(): string[] { + return Array.from(this.states.keys()); + } + + getState(conversationId: string): ApiProcessingState | null { + return this.states.get(conversationId) ?? null; + } + + restoreFromMessages(messages: DatabaseMessage[], conversationId: string): void { + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i]; + + if (message.role === MessageRole.ASSISTANT && message.timings) { + this.setState( + conversationId, + this.parseTimingData({ + cache_n: message.timings.cache_n || 0, + predicted_n: message.timings.predicted_n || 0, + predicted_per_second: + message.timings.predicted_n && message.timings.predicted_ms + ? (message.timings.predicted_n / message.timings.predicted_ms) * 1000 + : 0, + prompt_ms: message.timings.prompt_ms, + prompt_n: message.timings.prompt_n || 0 + }) + ); + + return; + } + } + } + + setActiveConversation(conversationId: string | null): void { + this._activeConversationId = conversationId; + } + + /** Passing null clears the state for the conversation. */ + setState(conversationId: string, state: ApiProcessingState | null): void { + if (state === null) this.states.delete(conversationId); + else this.states.set(conversationId, state); + } + + updateFromTimings(timingData: ProcessingTimingData, conversationId?: string): void { + const targetId = conversationId || this._activeConversationId; + + if (targetId) { + this.setState(targetId, this.parseTimingData(timingData)); + } + } + + private getContextTotal(): number | null { + const activeConvId = this._activeConversationId; + const activeState = activeConvId ? this.getState(activeConvId) : null; + + if (activeState && typeof activeState.contextTotal === 'number' && activeState.contextTotal > 0) + return activeState.contextTotal; + + if (serverStore.isRouterMode) { + const modelContextSize = modelsStore.selectedModelContextSize; + + if (typeof modelContextSize === 'number' && modelContextSize > 0) { + return modelContextSize; + } + } else { + const propsContextSize = serverStore.contextSize; + + if (typeof propsContextSize === 'number' && propsContextSize > 0) { + return propsContextSize; + } + } + + return null; + } + + private parseTimingData(timingData: ProcessingTimingData): ApiProcessingState { + const cacheTokens = timingData.cache_n || 0, + predictedTokens = timingData.predicted_n || 0, + promptMs = timingData.prompt_ms || undefined, + promptTokens = timingData.prompt_n || 0, + tokensPerSecond = timingData.predicted_per_second || 0; + const promptProgress = timingData.prompt_progress; + const contextTotal = this.getContextTotal(); + const currentConfig = settingsStore.config; + const outputTokensMax = currentConfig.max_tokens || -1; + const contextUsed = promptTokens + cacheTokens + predictedTokens, + outputTokensUsed = predictedTokens; + const progressCache = promptProgress?.cache || 0, + progressActualDone = (promptProgress?.processed ?? 0) - progressCache, + progressActualTotal = (promptProgress?.total ?? 0) - progressCache; + const progressPercent = promptProgress + ? Math.round((progressActualDone / progressActualTotal) * 100) + : undefined; + + return { + cacheTokens, + contextTotal, + contextUsed, + hasNextToken: predictedTokens > 0, + outputTokensMax, + outputTokensUsed, + progressPercent, + promptMs, + promptProgress, + promptTokens, + speculative: false, + status: predictedTokens > 0 ? 'generating' : promptProgress ? 'preparing' : 'idle', + temperature: currentConfig.temperature ?? 0.8, + tokensDecoded: predictedTokens, + tokensPerSecond, + tokensRemaining: outputTokensMax - predictedTokens, + topP: currentConfig.top_p ?? 0.95 + }; + } +} + +export const chatProcessingStore = new ChatProcessingStore(); diff --git a/tools/ui/src/lib/stores/chat/streams.svelte.ts b/tools/ui/src/lib/stores/chat/streams.svelte.ts new file mode 100644 index 000000000..5abbc81fb --- /dev/null +++ b/tools/ui/src/lib/stores/chat/streams.svelte.ts @@ -0,0 +1,494 @@ +/** + * ChatStreamManager - Server-side stream sessions for conversations + * + * Owns the attach lifecycle for streams that live on the server: discovery, + * replay from byte 0, and resume retry while the owning model loads. The + * remote-running snapshot it produces feeds the chat activity ledger + * (chatStore.activity), which owns the actual running-conv state. Created + * and owned by chatStore; the host exposes the per-conversation state setters. + */ + +import { CONVERSATION_ID_SEPARATOR, STREAM_RESUME_RETRY_MS } from '$lib/constants'; +import { MessageRole, MessageType, StreamConnectionState } from '$lib/enums'; +import { ChatService } from '$lib/services/chat.service'; +import { DatabaseService } from '$lib/services/database.service'; +import type { ChatActivityStore } from '$lib/stores/chat/activity.svelte'; +import type { ChatProcessingStore } from '$lib/stores/chat/processing.svelte'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import type { ApiStreamSession, ChatMessageTimings, DatabaseMessage } from '$lib/types'; +import { streamIdentity } from '$lib/utils'; +import { SvelteMap, SvelteSet } from 'svelte/reactivity'; + +/** + * The slice of chatStore the manager drives. Kept narrow on purpose so the + * manager cannot reach around the host's full surface; chatStore implements + * this structurally. + */ +export interface ChatStreamHost { + activity: ChatActivityStore; + processing: ChatProcessingStore; + chatStreamingStates: SvelteMap< + string, + { response: string; messageId: string; model?: string | null } + >; + streamConnectionState: StreamConnectionState; + getOrCreateAbortController(convId: string): AbortController; + setChatLoading(convId: string, loading: boolean): void; + setChatStreaming( + convId: string, + response: string, + messageId: string, + model?: string | null + ): void; + clearChatStreaming(convId: string, messageId?: string): void; +} + +export class ChatStreamManager { + // in-flight discoverActiveStream guard, keyed by conv id + private discoveringConvs = new SvelteSet(); + // convs whose resume waits on a model load: their loading state belongs to the retry loop, + // so discoverActiveStream must not treat it as a live send and bail + private resumePendingConvs = new SvelteSet(); + // pending resume retry timers while an owning model loads, one per conv + private resumeRetryTimers = new SvelteMap>(); + + /** Kill a pending resume retry, e.g. on explicit stop. */ + cancelResumeRetry(convId: string): void { + const timer = this.resumeRetryTimers.get(convId); + + if (timer !== undefined) { + clearTimeout(timer); + this.resumeRetryTimers.delete(convId); + } + + this.resumePendingConvs.delete(convId); + } + + constructor(private host: ChatStreamHost) {} + + async discoverActiveStream(convId: string): Promise { + if (!convId) return; + + if (this.host.chatStreamingStates.has(convId)) return; + + if (this.host.activity.isLocal(convId) && !this.resumePendingConvs.has(convId)) return; + + // concurrency guard: another discover may already be running for this conv (typical race + // between mount and visibilitychange on tab switch). a second concurrent fetch on the same + // /v1/stream would duplicate every byte into the DB message, this guard bounces it + if (this.discoveringConvs.has(convId)) return; + + this.discoveringConvs.add(convId); + + try { + // the model is frozen at POST time, rebuild the exact conv::model identity from the + // persisted state so the lookup key matches what the server stored. null means a single + // model conv with no ::suffix, only guess from the dropdown with no persisted state + const localState = ChatService.getStreamState(convId); + const streamId = ChatService.resumeStreamIdentity( + convId, + localState, + modelsStore.selectedModelName + ); + // primary path: ask the server which sessions exist for this identity + const serverTarget = await this.probeServerStream(streamId); + + if (serverTarget) { + // pass the full server side identity (may carry a ::model suffix) so the GET routes + // straight to the owning session, no probe or fan out + await this.attachServerStream(convId, serverTarget.conversation_id); + + return; + } + + // fallback: local state remembers an interrupted byte offset for this conv, the server may + // still have a live session matching that identity (we just lost the bytes mid stream). retry + // with the frozen identity, the server probe inside attachServerStream tells us if it exists + if (!localState) { + return; + } + + // quiet status probe first: a full attach flips the loading UI on every try, probing + // keeps the retry loop invisible while the owning model is still loading (503) + const status = await ChatService.probeResumeStatus(streamId); + + if (status === 503) { + // make the wait visible: the empty assistant row persisted at send time renders + // the processing info, whose model load percentage flows from the models feed + this.resumePendingConvs.add(convId); + this.host.setChatLoading(convId, true); + + if (!this.resumeRetryTimers.has(convId)) { + this.resumeRetryTimers.set( + convId, + setTimeout(() => { + this.resumeRetryTimers.delete(convId); + void this.discoverActiveStream(convId); + }, STREAM_RESUME_RETRY_MS) + ); + } + + return; + } + + if (this.resumePendingConvs.delete(convId) && status !== 200) { + // the wait is over without a session to attach, drop the visible loading state + this.host.setChatLoading(convId, false); + } + + if (status === 0) { + // transient network failure, the next mount or visibility change retries + return; + } + + if (status !== 200) { + // the session is gone (stopped, TTL expired), nothing to resume anymore + ChatService.clearStreamState(convId); + + return; + } + + await this.attachServerStream(convId, streamId); + + // if attachServerStream failed (session gone, TTL expired), clear the local state to avoid retrying forever + if (!this.host.chatStreamingStates.has(convId) && !this.host.activity.isLocal(convId)) { + ChatService.clearStreamState(convId); + } + } finally { + this.discoveringConvs.delete(convId); + } + } + + /** + * Model frozen at send time for a stream awaiting resume, from the persisted stream state. + * The load progress indicator targets it after a reload, when the message row has no model + * yet and the dropdown selection may not be restored. + */ + getResumeModel(convId: string): string | null { + return ChatService.getStreamState(convId)?.model ?? null; + } + + /** + * Resync the activity ledger's remote set from the backend. Called by the layout at mount and + * on visibilitychange, no polling. A snapshot semantic: stale entries for sessions that + * finalized while the browser was elsewhere are dropped naturally. + */ + async syncRemoteRunningStreams(): Promise { + // the conversations store loads from IndexedDB asynchronously, the +layout onMount caller + // fires before that finishes. read ids straight from the DB so the result does not depend + // on the store init race, and the sidebar spinners light up at first paint for every conv + // the user owns even if it has not been hydrated into the store yet + let ids: string[]; + + try { + const all = await DatabaseService.getAllConversations(); + + ids = all.map((c) => c.id).filter((id) => !!id); + } catch (e) { + console.warn('syncRemoteRunningStreams DB read failed:', e); + + return; + } + + // only ask about conv ids the user already owns + if (ids.length === 0) { + this.host.activity.applyRemoteSnapshot([]); + + return; + } + + // rebuild the frozen conv::model identity per conv so a session started with a model still + // matches. the server response is mapped back to the bare id below for the sidebar set + const lookupIds = ids.map((id) => + ChatService.resumeStreamIdentity(id, ChatService.getStreamState(id), null) + ); + + let sessions: ApiStreamSession[]; + + try { + sessions = await ChatService.lookupStreamSessions(lookupIds); + } catch (e) { + console.warn('syncRemoteRunningStreams lookup failed:', e); + + return; + } + const running = new SvelteSet(); + + for (const s of sessions) { + if (s && !s.is_done && typeof s.conversation_id === 'string' && s.conversation_id) { + // strip the optional ::model suffix, the sidebar set is keyed by the bare conv id + const sepIdx = s.conversation_id.indexOf(CONVERSATION_ID_SEPARATOR); + const bareId = sepIdx === -1 ? s.conversation_id : s.conversation_id.slice(0, sepIdx); + + running.add(bareId); + } + } + this.host.activity.applyRemoteSnapshot(running); + } + + private async attachServerStream(convId: string, streamId?: string): Promise { + if (!convId) return; + + if (this.host.chatStreamingStates.has(convId)) return; + + // flip the spinner immediately, the user sees activity as soon as the conv becomes active + this.host.setChatLoading(convId, true); + + // only set the active processing conv if we are looking at it, otherwise a background + // attach would steal the indicator from the conv the user is currently viewing + if (convId === conversationsStore.activeConversation?.id) { + this.host.processing.setActiveConversation(convId); + } + + const unlock = () => { + this.host.setChatLoading(convId, false); + this.host.clearChatStreaming(convId); + }; + // fetch the replay stream from byte 0, rebuild the assistant message from scratch. + // resolve the server side identity, fall back to streamIdentity when the caller does not + // pass a streamId. probeServerStream returns the full id (with ::model suffix when present) + const id = streamId || streamIdentity(convId, modelsStore.selectedModelName); + + let response: Response; + + try { + response = await ChatService.fetchStreamReplay(id); + } catch (e) { + console.error(`attachServerStream replay failed for conv ${convId}:`, e); + unlock(); + + return; + } + + // load the target conversation messages by id, not via the active store. when multiple + // attaches run in parallel the active store may reflect another conv and writing through + // its index mixes content across convs (CoT flicker, message bleed). by going through the + // DB we stay isolated, and only mirror into the active store when the attached conv is + // the one currently displayed + let messages: DatabaseMessage[]; + + try { + messages = await DatabaseService.getConversationMessages(convId); + } catch (e) { + console.error('attachServerStream load messages failed:', e); + unlock(); + + return; + } + + // locate the slot to splice into, create a placeholder assistant message if there is none. + // we use the conv-scoped findLastAssistantIdx helpers, they only depend on the array + let targetIdx = this.findLastAssistantIdx(messages); + + if (targetIdx === -1) { + const lastUserIdx = this.findLastUserIdx(messages); + + if (lastUserIdx === -1) { + console.warn( + `attachServerStream: conv ${convId} has no user or assistant message, cannot splice` + ); + unlock(); + + return; + } + + try { + const placeholder = await DatabaseService.createMessageBranch( + { + children: [], + content: '', + convId, + parent: messages[lastUserIdx].id, + role: MessageRole.ASSISTANT, + timestamp: Date.now(), + toolCalls: '', + type: MessageType.TEXT + } as Omit, + messages[lastUserIdx].id + ); + + messages = [...messages, placeholder]; + targetIdx = messages.length - 1; + + // only push into the active store when this conv is the one displayed right now + if (convId === conversationsStore.activeConversation?.id) { + conversationsStore.addMessageToActive(placeholder); + } + } catch (e) { + console.error('attachServerStream placeholder creation failed:', e); + unlock(); + + return; + } + } + + if (targetIdx === -1) { + unlock(); + + return; + } + + const targetMessage = messages[targetIdx]; + const targetMessageId = targetMessage.id; + // when the assistant slot already has content, the running session is a continue or + // another append flow and its buffer holds only the appended deltas. preserve the prefix + // and let the replay add to it. when the slot is empty the session buffer holds the whole + // message so we wipe and rebuild from byte 0 + const existingContent = targetMessage.content ?? ''; + const existingReasoning = targetMessage.reasoningContent ?? ''; + const isAppendMode = existingContent.length > 0; + // helper: write to the active store only when the attached conv is currently displayed. + // the lookup by message id is robust to reordering of activeMessages, two parallel attaches + // can no longer step on each other's indices + const writeActive = (updates: Partial) => { + if (convId !== conversationsStore.activeConversation?.id) { + return; + } + + const liveIdx = conversationsStore.findMessageIndex(targetMessageId); + + if (liveIdx === -1) return; + + conversationsStore.updateMessageAtIndex(liveIdx, updates); + }; + + if (!isAppendMode) { + writeActive({ content: '', reasoningContent: undefined }); + } + + // extract the model suffix, the resume calls in handleStreamResponse must reuse the model + // the session was tagged with, not the live dropdown + const sepIdx = id.indexOf(CONVERSATION_ID_SEPARATOR); + const attachedModel: string | null = sepIdx === -1 ? null : id.slice(sepIdx + 2); + + this.host.setChatStreaming(convId, existingContent, targetMessageId, attachedModel); + const abortController = this.host.getOrCreateAbortController(convId); + + let streamedContent = ''; + let streamedReasoningContent = ''; + + const cleanup = () => { + unlock(); + this.host.processing.setState(convId, null); + }; + + try { + await ChatService.handleStreamResponse( + response, + (chunk: string) => { + streamedContent += chunk; + const displayed = isAppendMode ? existingContent + streamedContent : streamedContent; + + writeActive({ content: displayed }); + this.host.setChatStreaming(convId, displayed, targetMessageId); + }, + async ( + finalContent?: string, + reasoningContent?: string, + timings?: ChatMessageTimings, + toolCalls?: string + ) => { + const streamed = streamedContent || finalContent || ''; + const streamedR = streamedReasoningContent || reasoningContent || ''; + const content = isAppendMode ? existingContent + streamed : streamed; + const reasoning = isAppendMode ? existingReasoning + streamedR : streamedR; + + // the DB write is the source of truth, mirror to the active store only when + // the conv is currently displayed + await DatabaseService.updateMessage(targetMessageId, { + content, + reasoningContent: reasoning || undefined, + timings, + toolCalls: toolCalls || '' + }); + writeActive({ + content, + reasoningContent: reasoning || undefined, + timings + }); + cleanup(); + }, + (err: Error) => { + console.error('attachServerStream pipe error:', err); + cleanup(); + }, + (chunk: string) => { + streamedReasoningContent += chunk; + const displayed = isAppendMode + ? existingReasoning + streamedReasoningContent + : streamedReasoningContent; + + writeActive({ reasoningContent: displayed }); + }, + undefined, + undefined, + undefined, + undefined, + convId, + abortController.signal, + (connState: StreamConnectionState) => { + if (convId === conversationsStore.activeConversation?.id) { + this.host.streamConnectionState = connState; + } + }, + attachedModel + ); + } catch (e) { + console.error('attachServerStream pipe crashed:', e); + cleanup(); + } + } + + private findLastAssistantIdx(messages: DatabaseMessage[]): number { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === MessageRole.ASSISTANT) return i; + } + + return -1; + } + + private findLastUserIdx(messages: DatabaseMessage[]): number { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === MessageRole.USER) return i; + } + + return -1; + } + + /** + * Server side stream discovery, split in three pieces: + * + * probeServerStream(convId) -> hits POST /v1/streams/lookup with the conv id, returns the session to attach + * to or null. Pure read, no side effect, no UI lock. Safe to fire in parallel with anything. + * + * attachServerStream(convId) -> flips the spinner immediately, fetches the replay stream + * from byte 0, finds the assistant slot to splice into (creates a placeholder if the conv has + * no assistant message yet, for cross device or fresh local DB cases), and pipes the SSE bytes + * into the message via handleStreamResponse. + * + * discoverActiveStream(convId) -> probe + attach in one call. Used by callers that do not need + * to overlap the probe with other async work. + * + * The chat page in +page.svelte calls discoverActiveStream once the conversation is active + * (immediately if it already is, after loadConversation settles otherwise), and re-runs it on + * visibilitychange. Attaching only after the conversation is loaded gives the earliest + * possible time to spinner and avoids racing against an empty activeMessages array. + */ + private async probeServerStream(convId: string): Promise { + if (!convId) return null; + + let sessions: ApiStreamSession[]; + + try { + sessions = await ChatService.lookupStreamSessions([convId]); + } catch (e) { + console.warn(`probeServerStream failed for conv ${convId}:`, e); + + return null; + } + + return ChatService.selectActiveStream(sessions); + } +} diff --git a/tools/ui/src/lib/stores/conversations.svelte.ts b/tools/ui/src/lib/stores/conversations/index.svelte.ts similarity index 61% rename from tools/ui/src/lib/stores/conversations.svelte.ts rename to tools/ui/src/lib/stores/conversations/index.svelte.ts index d2184b359..7d6dc326c 100644 --- a/tools/ui/src/lib/stores/conversations.svelte.ts +++ b/tools/ui/src/lib/stores/conversations/index.svelte.ts @@ -1,135 +1,67 @@ /** - * conversationsStore - Reactive State Store for Conversations + * conversationsStore - Conversation lifecycle, persistence and navigation * - * Manages conversation lifecycle, persistence, navigation, and MCP server overrides. - * - * **Architecture & Relationships:** - * - **DatabaseService**: Stateless IndexedDB layer - * - **conversationsStore** (this): Reactive state + business logic - * - **chatStore**: Chat-specific state (streaming, loading) - * - * **Key Responsibilities:** - * - Conversation CRUD (create, load, delete) - * - Message management and tree navigation - * - MCP server per-chat overrides - * - Import/Export functionality - * - Title management with confirmation - * - * @see DatabaseService in services/database.ts for IndexedDB operations + * Owns conversation CRUD, message tree navigation, import/export and title + * management, persisted through DatabaseService. Per-chat options (MCP + * overrides, reasoning effort, cwd) live in ConversationPreferences, + * composed as {@link ConversationsStore.preferences}. */ import { browser } from '$app/environment'; import { goto } from '$app/navigation'; -import { REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY, ROUTES } from '$lib/constants'; -import { MessageRole, ReasoningEffort } from '$lib/enums'; +import { ROUTES } from '$lib/constants'; +import { MessageRole } from '$lib/enums'; import { ConversationTransferService } from '$lib/services/conversation-transfer.service'; import { DatabaseService } from '$lib/services/database.service'; import { MigrationService } from '$lib/services/migration.service'; import { RouterService } from '$lib/services/router.service'; // direct imports between stores, not via the barrel, to avoid circular deps -import { mcpStore } from '$lib/stores/mcp.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; -import type { McpServerOverride } from '$lib/types/database'; +import { + ConversationPreferences, + type ConversationsPreferencesHost +} from '$lib/stores/conversations/preferences.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { filterByLeafNodeId, findLeafNode, generateConversationTitle } from '$lib/utils'; import { SvelteSet } from 'svelte/reactivity'; import { toast } from 'svelte-sonner'; -class ConversationsStore { - /** - * - * - * State - * - * - */ - - /** List of all conversations */ - conversations = $state([]); - +class ConversationsStore implements ConversationsPreferencesHost { /** Currently active conversation */ activeConversation = $state(null); /** Messages in the active conversation (filtered by currNode path) */ activeMessages = $state([]); + /** List of all conversations */ + conversations = $state([]); + /** Whether the store has been initialized */ isInitialized = $state(false); - /** Global (non-conversation-specific) reasoning effort default */ - pendingReasoningEffort = $state(ConversationsStore.loadReasoningEffortDefault()); + /** Per-chat options (MCP overrides, reasoning effort, cwd), composed here. */ + private _preferences = new ConversationPreferences(this); /** - * Working directory picked on the empty new-chat screen, before any - * conversation exists. Consumed by `chatStore.sendMessage()`, which - * records it into chat history as a synthetic message on first send. - * Cleared by `loadConversation` and `clearActiveConversation` so a - * stale pick can't bleed onto an unrelated chat. + * Listeners notified with the ids of conversations that were deleted. + * Lets dependent stores (e.g. agenticStore) drop per-conversation state + * without introducing a circular import back into this store. */ - pendingCwd = $state(null); - - /** Load reasoning effort default from localStorage, DEFAULT defers to the server */ - private static loadReasoningEffortDefault(): ReasoningEffort { - if (typeof globalThis.localStorage === 'undefined') return ReasoningEffort.DEFAULT; - - try { - const raw = localStorage.getItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY); - - return (raw as ReasoningEffort) || ReasoningEffort.DEFAULT; - } catch { - return ReasoningEffort.DEFAULT; - } - } - - /** Persist reasoning effort default to localStorage */ - private saveReasoningEffortDefaults(): void { - if (typeof globalThis.localStorage === 'undefined') return; - - localStorage.setItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY, this.pendingReasoningEffort); - } + private conversationDeletionListeners = new Set<(convIds: string[]) => void>(); /** In-flight init run; shared by concurrent callers, reset on failure to allow retry */ private initPromise: Promise | null = null; /** - * - * - * Lifecycle - * - * + * Memo of the last findMessageIndex() lookup. Streaming calls it once per + * chunk for the same message, so a validated cache hit keeps that O(1) + * instead of a linear scan of activeMessages on every token. */ + private lastMessageIndex: { id: string; index: number } | null = null; - /** - * Initialize the store by loading conversations from database. - * Safe to call multiple times: concurrent callers share a single run, - * and a failed run can be retried by calling again. - */ - init(): Promise { - if (!browser) return Promise.resolve(); - - if (this.initPromise) return this.initPromise; - - this.initPromise = (async () => { - try { - await MigrationService.runAllMigrations(); - await this.loadConversations(); - this.isInitialized = true; - } catch (error) { - console.error('Failed to initialize conversations:', error); - this.initPromise = null; - } - })(); - - return this.initPromise; + get preferences() { + return this._preferences; } - /** - * - * - * Message Array Operations - * - * - */ - /** * Adds a message to the active messages array */ @@ -138,221 +70,37 @@ class ConversationsStore { } /** - * Updates a message at a specific index in active messages + * Applies a field update to a conversation row, mirroring it into both the + * conversations list and the active conversation when it is the target. + * Shared by the rename/pin/preferences flows so no caller can forget to + * mirror one side. */ - updateMessageAtIndex(index: number, updates: Partial): void { - const message = index === -1 ? undefined : this.activeMessages[index]; + applyConversationUpdate(id: string, updates: Partial): void { + const convIndex = this.conversations.findIndex((c) => c.id === id); - if (!message) return; + if (convIndex !== -1) { + const target = this.conversations[convIndex] as unknown as Record; - // Assign field by field rather than replacing the object. Replacing it - // changes the array slot, which invalidates every consumer that merely - // walks the list - notably ChatMessages.displayMessages, which rebuilds - // entries for every message in the conversation. Deep $state proxies make - // per-field writes fine-grained, so only readers of the changed field wake. - const target = message as unknown as Record; - - for (const [key, value] of Object.entries(updates)) { - if (target[key] !== value) { - target[key] = value; + for (const [key, value] of Object.entries(updates)) { + if (target[key] !== value) target[key] = value; } } - } - /** - * Finds the index of a message in active messages - */ - findMessageIndex(messageId: string): number { - return this.activeMessages.findIndex((m) => m.id === messageId); - } - - /** - * Removes messages from active messages starting at an index - */ - sliceActiveMessages(startIndex: number): void { - this.activeMessages = this.activeMessages.slice(0, startIndex); - } - - /** - * Removes a message from active messages by index - */ - removeMessageAtIndex(index: number): DatabaseMessage | undefined { - if (index !== -1) { - return this.activeMessages.splice(index, 1)[0]; - } - - return undefined; - } - - /** - * - * - * Conversation CRUD - * - * - */ - - /** - * Loads all conversations from the database - */ - async loadConversations(): Promise { - const conversations = await DatabaseService.getAllConversations(); - - this.conversations = conversations; - } - - /** - * Creates a new conversation and navigates to it - * @param name - Optional name for the conversation - * @returns The ID of the created conversation - */ - async createConversation(name?: string): Promise { - const conversationName = name || `Chat ${new Date().toLocaleString()}`; - // No MCP override list is seeded: getAllMcpServerOverrides resolves - // servers without a per-conversation override to `mcpServers[i].enabled`, - // and only explicit toggles are stored on the conversation. - // Working directory picked on the new-chat screen gets threaded in - // here too, then cleared so it doesn't bleed onto subsequent new chats. - const conversation = await DatabaseService.createConversation(conversationName, { - cwd: this.pendingCwd ?? undefined, - reasoningEffort: this.pendingReasoningEffort - }); - - this.pendingCwd = null; - - this.conversations = [conversation, ...this.conversations]; - this.activeConversation = conversation; - this.activeMessages = []; - - await goto(RouterService.chat(conversation.id)); - - return conversation.id; - } - - /** - * Loads a specific conversation and its messages - * @param convId - The conversation ID to load - * @returns True if conversation was loaded successfully - */ - async loadConversation(convId: string): Promise { - try { - const conversation = await DatabaseService.getConversation(convId); - - if (!conversation) { - return false; - } - - // Drop any cwd the user drafted on the empty new-chat screen - - // it doesn't belong to this conversation. - this.pendingCwd = null; - - this.activeConversation = conversation; - - if (conversation.currNode) { - const allMessages = await DatabaseService.getConversationMessages(convId); - const filteredMessages = filterByLeafNodeId( - allMessages, - conversation.currNode, - false - ) as DatabaseMessage[]; - - this.activeMessages = filteredMessages; - } else { - const messages = await DatabaseService.getConversationMessages(convId); - - this.activeMessages = messages; - } - - return true; - } catch (error) { - console.error('Failed to load conversation:', error); - - return false; + if (this.activeConversation?.id === id) { + this.activeConversation = { ...this.activeConversation, ...updates }; } } /** - * Clears the active conversation and messages. + * Derives a conversation title from its first message content and applies + * it, honoring the title-generation setting. Shared by every flow that + * edits or creates the first user message. */ - clearActiveConversation(): void { - this.activeConversation = null; - this.activeMessages = []; - // reload defaults so new chats inherit persisted state - this.pendingReasoningEffort = ConversationsStore.loadReasoningEffortDefault(); - this.pendingCwd = null; - } - - /** - * Deletes a conversation and all its messages - * @param convId - The conversation ID to delete - */ - async deleteConversation(convId: string, options?: { deleteWithForks?: boolean }): Promise { - try { - await DatabaseService.deleteConversation(convId, options); - - if (options?.deleteWithForks) { - // Collect all descendants recursively - const idsToRemove = new SvelteSet([convId]); - const queue = [convId]; - - while (queue.length > 0) { - const parentId = queue.pop()!; - - for (const c of this.conversations) { - if (c.forkedFromConversationId === parentId && !idsToRemove.has(c.id)) { - idsToRemove.add(c.id); - queue.push(c.id); - } - } - } - this.conversations = this.conversations.filter((c) => !idsToRemove.has(c.id)); - - if (this.activeConversation && idsToRemove.has(this.activeConversation.id)) { - this.clearActiveConversation(); - await goto(ROUTES.NEW_CHAT); - } - } else { - // Reparent direct children to deleted conv's parent (or promote to top-level) - const deletedConv = this.conversations.find((c) => c.id === convId); - const newParent = deletedConv?.forkedFromConversationId; - - this.conversations = this.conversations - .filter((c) => c.id !== convId) - .map((c) => - c.forkedFromConversationId === convId - ? { ...c, forkedFromConversationId: newParent } - : c - ); - - if (this.activeConversation?.id === convId) { - this.clearActiveConversation(); - await goto(ROUTES.NEW_CHAT); - } - } - } catch (error) { - console.error('Failed to delete conversation:', error); - } - } - - /** - * Deletes all conversations and their messages - */ - async deleteAll(): Promise { - try { - const allConversations = await DatabaseService.getAllConversations(); - - await DatabaseService.bulkDeleteConversations(allConversations.map((c) => c.id)); - - this.clearActiveConversation(); - this.conversations = []; - - toast.success('All conversations deleted'); - - await goto(ROUTES.NEW_CHAT); - } catch (error) { - console.error('Failed to delete all conversations:', error); - toast.error('Failed to delete conversations'); - } + async applyTitleFromContent(convId: string, content: string): Promise { + await this.updateConversationName( + convId, + generateConversationTitle(content, Boolean(settingsStore.config.titleGenerationUseFirstLine)) + ); } /** @@ -387,6 +135,7 @@ class ConversationsStore { await DatabaseService.bulkDeleteConversations([...idsToRemove]); this.conversations = this.conversations.filter((c) => !idsToRemove.has(c.id)); + this.notifyConversationsDeleted([...idsToRemove]); if (activeWasDeleted) { this.clearActiveConversation(); @@ -404,43 +153,6 @@ class ConversationsStore { } } - /** - * Toggles the pinned state of each conversation individually. - * Mixed-pin selections are intentionally not normalised here; the bulk - * action UI surfaces them as a disabled mixed-state instead. - * @param convIds - Conversation IDs to toggle - */ - async bulkToggleConversationPin(convIds: string[]): Promise { - if (convIds.length === 0) return; - - try { - const updates = await DatabaseService.bulkToggleConversationPins(convIds); - const activeId = this.activeConversation?.id; - - if (activeId && updates.has(activeId)) { - this.activeConversation = { - ...this.activeConversation!, - pinned: updates.get(activeId)! - }; - } - - for (let i = 0; i < this.conversations.length; i++) { - const newPinned = updates.get(this.conversations[i].id); - - if (newPinned !== undefined) this.conversations[i].pinned = newPinned; - } - - toast.success( - convIds.length === 1 - ? 'Conversation pin toggled' - : `Updated pin state for ${convIds.length} conversations` - ); - } catch (error) { - console.error('Failed to bulk toggle pin:', error); - toast.error('Failed to update pin state'); - } - } - /** * Bundles the given conversations into a single zip archive and triggers a * browser download (one JSONL file per conversation). @@ -480,418 +192,203 @@ class ConversationsStore { } /** - * - * - * Message Management - * - * + * Toggles the pinned state of each conversation individually. + * Mixed-pin selections are intentionally not normalised here; the bulk + * action UI surfaces them as a disabled mixed-state instead. + * @param convIds - Conversation IDs to toggle */ + async bulkToggleConversationPin(convIds: string[]): Promise { + if (convIds.length === 0) return; - /** - * Refreshes active messages based on currNode after branch navigation. - */ - async refreshActiveMessages(): Promise { - if (!this.activeConversation) return; - - const allMessages = await DatabaseService.getConversationMessages(this.activeConversation.id); - - if (allMessages.length === 0) { - this.activeMessages = []; - - return; - } - - const leafNodeId = - this.activeConversation.currNode || - allMessages.reduce((latest, msg) => (msg.timestamp > latest.timestamp ? msg : latest)).id; - const currentPath = filterByLeafNodeId(allMessages, leafNodeId, false) as DatabaseMessage[]; - - this.activeMessages = currentPath; - } - - /** - * Gets all messages for a specific conversation - * @param convId - The conversation ID - * @returns Array of messages - */ - async getConversationMessages(convId: string): Promise { - return await DatabaseService.getConversationMessages(convId); - } - - /** - * - * - * Title Management - * - * - */ - - /** - * Updates the name of a conversation. - * @param convId - The conversation ID to update - * @param name - The new name for the conversation - */ - async updateConversationName(convId: string, name: string): Promise { try { - await DatabaseService.updateConversation(convId, { name }); + const updates = await DatabaseService.bulkToggleConversationPins(convIds); + const activeId = this.activeConversation?.id; - const convIndex = this.conversations.findIndex((c) => c.id === convId); - - if (convIndex !== -1) { - this.conversations[convIndex].name = name; + if (activeId && updates.has(activeId)) { + this.activeConversation = { + ...this.activeConversation!, + pinned: updates.get(activeId)! + }; } - if (this.activeConversation?.id === convId) { - this.activeConversation = { ...this.activeConversation, name }; - } - } catch (error) { - console.error('Failed to update conversation name:', error); - } - } + for (let i = 0; i < this.conversations.length; i++) { + const newPinned = updates.get(this.conversations[i].id); - /** - * Toggles the pinned status of a conversation. - * @param convId - The conversation ID to toggle - * @returns The new pinned status - */ - async toggleConversationPin(convId: string): Promise { - try { - const newPinnedState = await DatabaseService.toggleConversationPin(convId); - const convIndex = this.conversations.findIndex((c) => c.id === convId); - - if (convIndex !== -1) { - this.conversations[convIndex].pinned = newPinnedState; + if (newPinned !== undefined) this.conversations[i].pinned = newPinned; } - if (this.activeConversation?.id === convId) { - this.activeConversation = { ...this.activeConversation, pinned: newPinnedState }; - } - - return newPinnedState; - } catch (error) { - console.error('Failed to toggle conversation pin:', error); - - return false; - } - } - - /** - * Marks a conversation as recently active: stamps lastModified (persisted) - * and moves it to the top of the list. Only message-activity flows call - * this; metadata updates (rename, pin, settings) do not. - * - * @param convId - Conversation that produced the activity, defaults to the active one - */ - updateConversationTimestamp(convId?: string): void { - const targetId = convId ?? this.activeConversation?.id; - - if (!targetId) return; - - const now = Date.now(); - const chatIndex = this.conversations.findIndex((c) => c.id === targetId); - - if (chatIndex !== -1) { - this.conversations[chatIndex].lastModified = now; - const updatedConv = this.conversations.splice(chatIndex, 1)[0]; - - this.conversations = [updatedConv, ...this.conversations]; - } - - if (this.activeConversation?.id === targetId) { - this.activeConversation = { ...this.activeConversation, lastModified: now }; - } - - DatabaseService.updateConversation(targetId, { lastModified: now }).catch((error) => - console.error('Failed to update conversation timestamp:', error) - ); - } - - /** - * Updates the current node of the active conversation - * @param nodeId - The new current node ID - */ - async updateCurrentNode(nodeId: string): Promise { - if (!this.activeConversation) return; - - await DatabaseService.updateCurrentNode(this.activeConversation.id, nodeId); - this.activeConversation = { ...this.activeConversation, currNode: nodeId }; - } - - /** - * - * - * Branch Navigation - * - * - */ - - /** - * Navigates to a specific sibling branch by updating currNode and refreshing messages. - * @param siblingId - The sibling message ID to navigate to - */ - async navigateToSibling(siblingId: string): Promise { - if (!this.activeConversation) return; - - const allMessages = await DatabaseService.getConversationMessages(this.activeConversation.id); - const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); - const currentFirstUserMessage = this.activeMessages.find( - (m) => m.role === MessageRole.USER && m.parent === rootMessage?.id - ); - const currentLeafNodeId = findLeafNode(allMessages, siblingId); - - await DatabaseService.updateCurrentNode(this.activeConversation.id, currentLeafNodeId); - this.activeConversation = { ...this.activeConversation, currNode: currentLeafNodeId }; - await this.refreshActiveMessages(); - - if (rootMessage && this.activeMessages.length > 0) { - const newFirstUserMessage = this.activeMessages.find( - (m) => m.role === MessageRole.USER && m.parent === rootMessage.id + toast.success( + convIds.length === 1 + ? 'Conversation pin toggled' + : `Updated pin state for ${convIds.length} conversations` ); - - if ( - newFirstUserMessage && - newFirstUserMessage.content.trim() && - (!currentFirstUserMessage || - newFirstUserMessage.id !== currentFirstUserMessage.id || - newFirstUserMessage.content.trim() !== currentFirstUserMessage.content.trim()) - ) { - await this.updateConversationName( - this.activeConversation.id, - generateConversationTitle( - newFirstUserMessage.content, - Boolean(settingsStore.config.titleGenerationUseFirstLine) - ) - ); - } + } catch (error) { + console.error('Failed to bulk toggle pin:', error); + toast.error('Failed to update pin state'); } } /** - * - * - * MCP Server Overrides - * - * + * Clears the active conversation and messages. */ - - /** - * Resolve the default enabled value for a server: its own `enabled` - * flag in `mcpServers`, so the global on/off state lives in one place. - */ - #getDefaultOverride(serverId: string): McpServerOverride | undefined { - const server = mcpStore.getServers().find((s) => s.id === serverId); - - if (!server) return undefined; - - return { enabled: server.enabled, serverId }; + clearActiveConversation(): void { + this.activeConversation = null; + this.activeMessages = []; + // reload defaults so new chats inherit persisted state + this.preferences.resetPending(); } /** - * Gets the effective MCP server override for a specific server. - * A per-conversation override wins when present; a server without one - * resolves to its `mcpServers[i].enabled` default. - * @param serverId - The server ID to check - * @returns The effective override, undefined if no matching server + * Creates a new conversation and navigates to it + * @param name - Optional name for the conversation + * @returns The ID of the created conversation */ - getMcpServerOverride(serverId: string): McpServerOverride | undefined { - const override = this.activeConversation?.mcpServerOverrides?.find( - (o: McpServerOverride) => o.serverId === serverId - ); - - if (override) return override; - - return this.#getDefaultOverride(serverId); - } - - /** - * Gets the effective override list for the current conversation: - * one entry per configured server, resolved per server. The stored - * per-conversation list is sparse and only holds explicit toggles. - */ - getAllMcpServerOverrides(): McpServerOverride[] { - const overrides = this.activeConversation?.mcpServerOverrides; - - return mcpStore.getServers().map((s) => { - const override = overrides?.find((o: McpServerOverride) => o.serverId === s.id); - - return { enabled: override?.enabled ?? s.enabled, serverId: s.id }; + async createConversation(name?: string): Promise { + const conversationName = name || `Chat ${new Date().toLocaleString()}`; + // Working directory and reasoning effort picked on the new-chat screen + // get threaded into the new conversation here, then cleared so they + // don't bleed onto subsequent new chats. + const conversation = await DatabaseService.createConversation(conversationName, { + cwd: this.preferences.pendingCwd ?? undefined, + reasoningEffort: this.preferences.pendingReasoningEffort }); + + this.preferences.pendingCwd = null; + + this.conversations = [conversation, ...this.conversations]; + this.activeConversation = conversation; + this.activeMessages = []; + + await goto(RouterService.chat(conversation.id)); + + return conversation.id; } /** - * Checks if an MCP server is enabled for the active conversation. - * @param serverId - The server ID to check - * @returns True if server is enabled for this conversation + * Deletes all conversations and their messages */ - isMcpServerEnabledForChat(serverId: string): boolean { - const override = this.getMcpServerOverride(serverId); + async deleteAll(): Promise { + try { + const allConversations = await DatabaseService.getAllConversations(); + const allIds = allConversations.map((c) => c.id); - return override?.enabled ?? false; - } + await DatabaseService.bulkDeleteConversations(allIds); - /** - * Sets or removes MCP server override for the active conversation. - * If no conversation exists, persists `enabled` onto `mcpServers[i].enabled` - * (the single source of truth for new-chat defaults). - * @param serverId - The server ID to override - * @param enabled - The enabled state, or undefined to remove per-conversation override - */ - async setMcpServerOverride(serverId: string, enabled: boolean | undefined): Promise { - if (!this.activeConversation) { - if (enabled !== undefined) { - mcpStore.updateServer(serverId, { enabled }); - } + this.clearActiveConversation(); + this.conversations = []; + this.notifyConversationsDeleted(allIds); - return; + toast.success('All conversations deleted'); + + await goto(ROUTES.NEW_CHAT); + } catch (error) { + console.error('Failed to delete all conversations:', error); + toast.error('Failed to delete conversations'); } + } - // Clone to plain objects to avoid Proxy serialization issues with IndexedDB - const currentOverrides = (this.activeConversation.mcpServerOverrides || []).map( - (o: McpServerOverride) => ({ - enabled: o.enabled, - serverId: o.serverId - }) - ); + /** + * Deletes a conversation and all its messages + * @param convId - The conversation ID to delete + */ + async deleteConversation(convId: string, options?: { deleteWithForks?: boolean }): Promise { + try { + await DatabaseService.deleteConversation(convId, options); - let newOverrides: McpServerOverride[]; + if (options?.deleteWithForks) { + // Collect all descendants recursively + const idsToRemove = new SvelteSet([convId]); + const queue = [convId]; - if (enabled === undefined) { - newOverrides = currentOverrides.filter((o: McpServerOverride) => o.serverId !== serverId); - } else { - const existingIndex = currentOverrides.findIndex( - (o: McpServerOverride) => o.serverId === serverId - ); + while (queue.length > 0) { + const parentId = queue.pop()!; - if (existingIndex >= 0) { - newOverrides = [...currentOverrides]; - newOverrides[existingIndex] = { enabled, serverId }; + for (const c of this.conversations) { + if (c.forkedFromConversationId === parentId && !idsToRemove.has(c.id)) { + idsToRemove.add(c.id); + queue.push(c.id); + } + } + } + this.conversations = this.conversations.filter((c) => !idsToRemove.has(c.id)); + + if (this.activeConversation && idsToRemove.has(this.activeConversation.id)) { + this.clearActiveConversation(); + await goto(ROUTES.NEW_CHAT); + } + + this.notifyConversationsDeleted([...idsToRemove]); } else { - newOverrides = [...currentOverrides, { enabled, serverId }]; + // Reparent direct children to deleted conv's parent (or promote to top-level) + const deletedConv = this.conversations.find((c) => c.id === convId); + const newParent = deletedConv?.forkedFromConversationId; + + this.conversations = this.conversations + .filter((c) => c.id !== convId) + .map((c) => + c.forkedFromConversationId === convId + ? { ...c, forkedFromConversationId: newParent } + : c + ); + + if (this.activeConversation?.id === convId) { + this.clearActiveConversation(); + await goto(ROUTES.NEW_CHAT); + } + + this.notifyConversationsDeleted([convId]); } - } - - await DatabaseService.updateConversation(this.activeConversation.id, { - mcpServerOverrides: newOverrides.length > 0 ? newOverrides : undefined - }); - - this.activeConversation = { - ...this.activeConversation, - mcpServerOverrides: newOverrides.length > 0 ? newOverrides : undefined - }; - - const convIndex = this.conversations.findIndex((c) => c.id === this.activeConversation!.id); - - if (convIndex !== -1) { - this.conversations[convIndex].mcpServerOverrides = - newOverrides.length > 0 ? newOverrides : undefined; + } catch (error) { + console.error('Failed to delete conversation:', error); } } /** - * Toggles MCP server enabled state for the active conversation. - * @param serverId - The server ID to toggle + * Downloads a single conversation as a JSONL file, serializing the full message tree. + * @param convId - The conversation ID to download */ - async toggleMcpServerForChat(serverId: string): Promise { - const currentEnabled = this.isMcpServerEnabledForChat(serverId); + async downloadConversation(convId: string): Promise { + const conversation = + this.activeConversation?.id === convId + ? this.activeConversation + : await DatabaseService.getConversation(convId); - await this.setMcpServerOverride(serverId, !currentEnabled); + if (!conversation) return; + + const messages = await DatabaseService.getConversationMessages(convId); + + ConversationTransferService.downloadConversationFile({ conv: conversation, messages }); } /** - * Removes MCP server override for the active conversation. - * @param serverId - The server ID to remove override for - */ - async removeMcpServerOverride(serverId: string): Promise { - await this.setMcpServerOverride(serverId, undefined); - } - - /** - * Gets the effective reasoning effort for the active conversation. - * Returns the conversation override if set, otherwise the global default. - * DEFAULT means no override is sent and the server decides. - */ - getReasoningEffort(): ReasoningEffort { - if (this.activeConversation) { - if (this.activeConversation.reasoningEffort !== undefined) { - return this.activeConversation.reasoningEffort; - } - - // conversations created before the tri-state store an explicit - // opt-out only as thinkingEnabled = false - if (this.activeConversation.thinkingEnabled === false) { - return ReasoningEffort.OFF; - } - } - - return this.pendingReasoningEffort; - } - - /** - * Sets the reasoning effort for the active conversation. - * If no conversation exists, stores the global default. - * @param effort - The effort level ('default' | 'off' | 'low' | 'medium' | 'high' | 'max') - */ - async setReasoningEffort(effort: ReasoningEffort): Promise { - if (!this.activeConversation) { - this.pendingReasoningEffort = effort; - this.saveReasoningEffortDefaults(); - - return; - } - - this.activeConversation = { - ...this.activeConversation, - reasoningEffort: effort - }; - - await DatabaseService.updateConversation(this.activeConversation.id, { - reasoningEffort: effort - }); - - const convIndex = this.conversations.findIndex((c) => c.id === this.activeConversation!.id); - - if (convIndex !== -1) { - this.conversations[convIndex].reasoningEffort = effort; - } - } - - /** - * Sets the working directory for the active conversation. Pass `null` or - * an empty string to clear it, which restores the picker's empty state. + * Finds the index of a message in active messages. * - * On the empty new-chat screen (no active conversation yet), the value - * is buffered into `pendingCwd` so the user can pick before - * sending the first message; `createConversation()` consumes it. - * - * @param value - Absolute server-side path to the working directory, or null to clear + * The last lookup is memoized and reused when it still validates against + * the current array (same id at the same position), which covers the + * streaming hot path where the same message is looked up on every chunk + * while the array itself only mutates by field. Any structural change + * (splice, reassignment, reordering) fails validation and falls back to a + * full scan. */ - async setCwd(value: string | null): Promise { - const trimmed = value?.trim() || undefined; + findMessageIndex(messageId: string): number { + const last = this.lastMessageIndex; + const messages = this.activeMessages; - // No chat yet - buffer for the first chat the user creates. - if (!this.activeConversation) { - this.pendingCwd = trimmed ?? null; - - return; + if ( + last && + last.id === messageId && + last.index >= 0 && + last.index < messages.length && + messages[last.index]?.id === messageId + ) { + return last.index; } - this.activeConversation = { - ...this.activeConversation, - cwd: trimmed - }; + const index = messages.findIndex((m) => m.id === messageId); - await DatabaseService.updateConversation(this.activeConversation.id, { - cwd: trimmed - }); + this.lastMessageIndex = { id: messageId, index }; - const convIndex = this.conversations.findIndex((c) => c.id === this.activeConversation!.id); - - if (convIndex !== -1) { - this.conversations[convIndex].cwd = trimmed; - this.conversations = [...this.conversations]; - } - - this.pendingCwd = null; + return index; } /** @@ -931,28 +428,12 @@ class ConversationsStore { } /** - * - * - * Import & Export - * - * + * Gets all messages for a specific conversation + * @param convId - The conversation ID + * @returns Array of messages */ - - /** - * Downloads a single conversation as a JSONL file, serializing the full message tree. - * @param convId - The conversation ID to download - */ - async downloadConversation(convId: string): Promise { - const conversation = - this.activeConversation?.id === convId - ? this.activeConversation - : await DatabaseService.getConversation(convId); - - if (!conversation) return; - - const messages = await DatabaseService.getConversationMessages(convId); - - ConversationTransferService.downloadConversationFile({ conv: conversation, messages }); + async getConversationMessages(convId: string): Promise { + return await DatabaseService.getConversationMessages(convId); } /** @@ -969,6 +450,280 @@ class ConversationsStore { return result; } + + /** + * Initialize the store by loading conversations from database. + * Safe to call multiple times: concurrent callers share a single run, + * and a failed run can be retried by calling again. + */ + initialize(): Promise { + if (!browser) return Promise.resolve(); + + if (this.initPromise) return this.initPromise; + + this.initPromise = (async () => { + try { + await MigrationService.runAllMigrations(); + await this.loadConversations(); + this.isInitialized = true; + } catch (error) { + console.error('Failed to initialize conversations:', error); + this.initPromise = null; + } + })(); + + return this.initPromise; + } + + /** + * Loads a specific conversation and its messages + * @param convId - The conversation ID to load + * @returns True if conversation was loaded successfully + */ + async loadConversation(convId: string): Promise { + try { + const conversation = await DatabaseService.getConversation(convId); + + if (!conversation) { + return false; + } + + // Drop any cwd the user drafted on the empty new-chat screen - + // it doesn't belong to this conversation. + this.preferences.pendingCwd = null; + + this.activeConversation = conversation; + + if (conversation.currNode) { + const allMessages = await DatabaseService.getConversationMessages(convId); + const filteredMessages = filterByLeafNodeId( + allMessages, + conversation.currNode, + false + ) as DatabaseMessage[]; + + this.activeMessages = filteredMessages; + } else { + const messages = await DatabaseService.getConversationMessages(convId); + + this.activeMessages = messages; + } + + return true; + } catch (error) { + console.error('Failed to load conversation:', error); + + return false; + } + } + + /** + * Loads all conversations from the database + */ + async loadConversations(): Promise { + const conversations = await DatabaseService.getAllConversations(); + + this.conversations = conversations; + } + + /** + * Navigates to a specific sibling branch by updating currNode and refreshing messages. + * @param siblingId - The sibling message ID to navigate to + */ + async navigateToSibling(siblingId: string): Promise { + if (!this.activeConversation) return; + + const allMessages = await DatabaseService.getConversationMessages(this.activeConversation.id); + const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); + const currentFirstUserMessage = this.activeMessages.find( + (m) => m.role === MessageRole.USER && m.parent === rootMessage?.id + ); + const currentLeafNodeId = findLeafNode(allMessages, siblingId); + + await DatabaseService.updateCurrentNode(this.activeConversation.id, currentLeafNodeId); + this.activeConversation = { ...this.activeConversation, currNode: currentLeafNodeId }; + await this.refreshActiveMessages(); + + if (rootMessage && this.activeMessages.length > 0) { + const newFirstUserMessage = this.activeMessages.find( + (m) => m.role === MessageRole.USER && m.parent === rootMessage.id + ); + + if ( + newFirstUserMessage && + newFirstUserMessage.content.trim() && + (!currentFirstUserMessage || + newFirstUserMessage.id !== currentFirstUserMessage.id || + newFirstUserMessage.content.trim() !== currentFirstUserMessage.content.trim()) + ) { + await this.applyTitleFromContent(this.activeConversation.id, newFirstUserMessage.content); + } + } + } + + /** + * Registers a listener invoked with the ids of deleted conversations. + * Returns an unsubscribe function. + */ + onConversationsDeleted(listener: (convIds: string[]) => void): () => void { + this.conversationDeletionListeners.add(listener); + + return () => this.conversationDeletionListeners.delete(listener); + } + + /** + * Refreshes active messages based on currNode after branch navigation. + */ + async refreshActiveMessages(): Promise { + if (!this.activeConversation) return; + + const allMessages = await DatabaseService.getConversationMessages(this.activeConversation.id); + + if (allMessages.length === 0) { + this.activeMessages = []; + + return; + } + + const leafNodeId = + this.activeConversation.currNode || + allMessages.reduce((latest, msg) => (msg.timestamp > latest.timestamp ? msg : latest)).id; + const currentPath = filterByLeafNodeId(allMessages, leafNodeId, false) as DatabaseMessage[]; + + this.activeMessages = currentPath; + } + + /** + * Removes a message from active messages by index + */ + removeMessageAtIndex(index: number): DatabaseMessage | undefined { + if (index !== -1) { + return this.activeMessages.splice(index, 1)[0]; + } + + return undefined; + } + + /** + * Removes messages from active messages starting at an index + */ + sliceActiveMessages(startIndex: number): void { + this.activeMessages = this.activeMessages.slice(0, startIndex); + } + + /** + * Toggles the pinned status of a conversation. + * @param convId - The conversation ID to toggle + * @returns The new pinned status + */ + async toggleConversationPin(convId: string): Promise { + try { + const newPinnedState = await DatabaseService.toggleConversationPin(convId); + + this.applyConversationUpdate(convId, { pinned: newPinnedState }); + + return newPinnedState; + } catch (error) { + console.error('Failed to toggle conversation pin:', error); + + return false; + } + } + + /** + * Updates the name of a conversation. + * @param convId - The conversation ID to update + * @param name - The new name for the conversation + */ + async updateConversationName(convId: string, name: string): Promise { + try { + await DatabaseService.updateConversation(convId, { name }); + + this.applyConversationUpdate(convId, { name }); + } catch (error) { + console.error('Failed to update conversation name:', error); + } + } + + /** + * Marks a conversation as recently active: stamps lastModified (persisted) + * and moves it to the top of the list. Only message-activity flows call + * this; metadata updates (rename, pin, settings) do not. + * + * @param convId - Conversation that produced the activity, defaults to the active one + */ + updateConversationTimestamp(convId?: string): void { + const targetId = convId ?? this.activeConversation?.id; + + if (!targetId) return; + + const now = Date.now(); + const chatIndex = this.conversations.findIndex((c) => c.id === targetId); + + if (chatIndex !== -1) { + this.conversations[chatIndex].lastModified = now; + const updatedConv = this.conversations.splice(chatIndex, 1)[0]; + + this.conversations = [updatedConv, ...this.conversations]; + } + + if (this.activeConversation?.id === targetId) { + this.activeConversation = { ...this.activeConversation, lastModified: now }; + } + + DatabaseService.updateConversation(targetId, { lastModified: now }).catch((error) => + console.error('Failed to update conversation timestamp:', error) + ); + } + + /** + * + * + * Import & Export + * + * + */ + + /** + * Updates the current node of the active conversation + * @param nodeId - The new current node ID + */ + async updateCurrentNode(nodeId: string): Promise { + if (!this.activeConversation) return; + + await DatabaseService.updateCurrentNode(this.activeConversation.id, nodeId); + this.activeConversation = { ...this.activeConversation, currNode: nodeId }; + } + + /** + * Updates a message at a specific index in active messages + */ + updateMessageAtIndex(index: number, updates: Partial): void { + const message = index === -1 ? undefined : this.activeMessages[index]; + + if (!message) return; + + // Assign field by field rather than replacing the object. Replacing it + // changes the array slot, which invalidates every consumer that merely + // walks the list - notably ChatMessages.displayMessages, which rebuilds + // entries for every message in the conversation. Deep $state proxies make + // per-field writes fine-grained, so only readers of the changed field wake. + const target = message as unknown as Record; + + for (const [key, value] of Object.entries(updates)) { + if (target[key] !== value) { + target[key] = value; + } + } + } + + private notifyConversationsDeleted(convIds: string[]): void { + if (convIds.length === 0) return; + + for (const listener of this.conversationDeletionListeners) { + listener(convIds); + } + } } export const conversationsStore = new ConversationsStore(); diff --git a/tools/ui/src/lib/stores/conversations/preferences.svelte.ts b/tools/ui/src/lib/stores/conversations/preferences.svelte.ts new file mode 100644 index 000000000..65a02344b --- /dev/null +++ b/tools/ui/src/lib/stores/conversations/preferences.svelte.ts @@ -0,0 +1,254 @@ +/** + * ConversationPreferences - Per-chat options with global fallback + * + * Owns the options that resolve per conversation: MCP server overrides, + * reasoning effort, and the working directory. Cwd and reasoning effort are + * buffered as pending state and threaded into the next created conversation + * by the host; MCP server overrides edit the sparse `mcpServerOverrides` + * list on the active row (new-chat toggles edit the server's global flag). + * Created and owned by conversationsStore; the host owns the conversation + * rows these options persist onto. + */ + +import { REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY } from '$lib/constants'; +import { ReasoningEffort } from '$lib/enums'; +import { DatabaseService } from '$lib/services/database.service'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { mcpStore } from '$lib/stores/mcp/index.svelte'; +import type { McpServerOverride } from '$lib/types/database'; + +/** Load reasoning effort default from localStorage, DEFAULT defers to the server */ +function loadReasoningEffortDefault(): ReasoningEffort { + if (typeof globalThis.localStorage === 'undefined') return ReasoningEffort.DEFAULT; + + try { + const raw = localStorage.getItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY); + + return (raw as ReasoningEffort) || ReasoningEffort.DEFAULT; + } catch { + return ReasoningEffort.DEFAULT; + } +} + +/** Persist reasoning effort default to localStorage */ +function saveReasoningEffortDefault(effort: ReasoningEffort): void { + if (typeof globalThis.localStorage === 'undefined') return; + + localStorage.setItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY, effort); +} + +/** + * The slice of conversationsStore the preferences read and write. Kept narrow + * on purpose so they cannot reach around the host's full surface; + * conversationsStore implements this structurally. + */ +export interface ConversationsPreferencesHost { + activeConversation: DatabaseConversation | null; + conversations: DatabaseConversation[]; + applyConversationUpdate(id: string, updates: Partial): void; +} + +export class ConversationPreferences { + /** + * Working directory picked on the empty new-chat screen, before any + * conversation exists. Consumed by `chatStore.sendMessage()`, which + * records it into chat history as a synthetic message on first send. + * Cleared by `loadConversation` and `clearActiveConversation` so a + * stale pick can't bleed onto an unrelated chat. + */ + pendingCwd = $state(null); + + /** Global (non-conversation-specific) reasoning effort default */ + pendingReasoningEffort = $state(loadReasoningEffortDefault()); + + constructor(private host: ConversationsPreferencesHost) {} + + /** + * Gets the effective override list for the current conversation: + * one entry per configured server, resolved per server. The stored + * per-conversation list is sparse and only holds explicit toggles. + */ + getAllMcpServerOverrides(): McpServerOverride[] { + const overrides = this.host.activeConversation?.mcpServerOverrides; + + return mcpStore.getServers().map((s) => { + const override = overrides?.find((o: McpServerOverride) => o.serverId === s.id); + + return { enabled: override?.enabled ?? s.enabled, serverId: s.id }; + }); + } + + /** + * Gets the effective MCP server override for a specific server. + * A per-conversation override wins when present; a server without one + * resolves to its `mcpServers[i].enabled` default. + */ + getMcpServerOverride(serverId: string): McpServerOverride | undefined { + const override = this.host.activeConversation?.mcpServerOverrides?.find( + (o: McpServerOverride) => o.serverId === serverId + ); + + if (override) return override; + + return this.getDefaultOverride(serverId); + } + + /** + * Gets the effective reasoning effort for the active conversation. + * Returns the conversation override if set, otherwise the global default. + * DEFAULT means no override is sent and the server decides. + */ + getReasoningEffort(): ReasoningEffort { + if (this.host.activeConversation) { + if (this.host.activeConversation.reasoningEffort !== undefined) { + return this.host.activeConversation.reasoningEffort; + } + + // conversations created before the tri-state store an explicit + // opt-out only as thinkingEnabled = false + if (this.host.activeConversation.thinkingEnabled === false) { + return ReasoningEffort.OFF; + } + } + + return this.pendingReasoningEffort; + } + + /** Checks if an MCP server is enabled for the active conversation. */ + isMcpServerEnabledForChat(serverId: string): boolean { + const override = this.getMcpServerOverride(serverId); + + return override?.enabled ?? false; + } + + /** Removes MCP server override for the active conversation. */ + async removeMcpServerOverride(serverId: string): Promise { + await this.setMcpServerOverride(serverId, undefined); + } + + /** Reload persisted defaults, e.g. when the active conversation is cleared. */ + resetPending(): void { + this.pendingReasoningEffort = loadReasoningEffortDefault(); + this.pendingCwd = null; + } + + /** + * Sets the working directory for the active conversation. Pass `null` or + * an empty string to clear it, which restores the picker's empty state. + * + * On the empty new-chat screen (no active conversation yet), the value + * is buffered into `pendingCwd` so the user can pick before + * sending the first message; `createConversation()` consumes it. + * + * @param value - Absolute server-side path to the working directory, or null to clear + */ + async setCwd(value: string | null): Promise { + const trimmed = value?.trim() || undefined; + + // No chat yet - buffer for the first chat the user creates. + if (!this.host.activeConversation) { + this.pendingCwd = trimmed ?? null; + + return; + } + + this.host.applyConversationUpdate(this.host.activeConversation.id, { + cwd: trimmed + }); + + await DatabaseService.updateConversation(this.host.activeConversation.id, { + cwd: trimmed + }); + + this.pendingCwd = null; + } + + /** + * Sets or removes MCP server override for the active conversation. + * If no conversation exists, persists `enabled` onto `mcpServers[i].enabled` + * (the single source of truth for new-chat defaults). + */ + async setMcpServerOverride(serverId: string, enabled: boolean | undefined): Promise { + if (!this.host.activeConversation) { + if (enabled !== undefined) { + mcpStore.updateServer(serverId, { enabled }); + } + + return; + } + + // Clone to plain objects to avoid Proxy serialization issues with IndexedDB + const currentOverrides = (this.host.activeConversation.mcpServerOverrides || []).map( + (o: McpServerOverride) => ({ + enabled: o.enabled, + serverId: o.serverId + }) + ); + + let newOverrides: McpServerOverride[]; + + if (enabled === undefined) { + newOverrides = currentOverrides.filter((o: McpServerOverride) => o.serverId !== serverId); + } else { + const existingIndex = currentOverrides.findIndex( + (o: McpServerOverride) => o.serverId === serverId + ); + + if (existingIndex >= 0) { + newOverrides = [...currentOverrides]; + newOverrides[existingIndex] = { enabled, serverId }; + } else { + newOverrides = [...currentOverrides, { enabled, serverId }]; + } + } + + await DatabaseService.updateConversation(this.host.activeConversation.id, { + mcpServerOverrides: newOverrides.length > 0 ? newOverrides : undefined + }); + + this.host.applyConversationUpdate(this.host.activeConversation.id, { + mcpServerOverrides: newOverrides.length > 0 ? newOverrides : undefined + }); + } + + /** + * Sets the reasoning effort for the active conversation. + * If no conversation exists, stores the global default. + * @param effort - The effort level ('default' | 'off' | 'low' | 'medium' | 'high' | 'max') + */ + async setReasoningEffort(effort: ReasoningEffort): Promise { + if (!this.host.activeConversation) { + this.pendingReasoningEffort = effort; + saveReasoningEffortDefault(effort); + + return; + } + + this.host.applyConversationUpdate(this.host.activeConversation.id, { + reasoningEffort: effort + }); + + await DatabaseService.updateConversation(this.host.activeConversation.id, { + reasoningEffort: effort + }); + } + + /** Toggles MCP server enabled state for the active conversation. */ + async toggleMcpServerForChat(serverId: string): Promise { + const currentEnabled = this.isMcpServerEnabledForChat(serverId); + + await this.setMcpServerOverride(serverId, !currentEnabled); + } + + /** + * Resolve the default enabled value for a server: its own `enabled` + * flag in `mcpServers`, so the global on/off state lives in one place. + */ + private getDefaultOverride(serverId: string): McpServerOverride | undefined { + const server = mcpStore.getServers().find((s) => s.id === serverId); + + if (!server) return undefined; + + return { enabled: server.enabled, serverId }; + } +} diff --git a/tools/ui/src/lib/stores/device.svelte.ts b/tools/ui/src/lib/stores/device.svelte.ts index 08ce2f205..42aaf4589 100644 --- a/tools/ui/src/lib/stores/device.svelte.ts +++ b/tools/ui/src/lib/stores/device.svelte.ts @@ -34,11 +34,11 @@ class DeviceStore { readonly isIOSDevice: boolean = false; /** The Safari browser app on iOS, excluding other iOS browsers and WKWebViews. */ readonly isIOSSafari: boolean = false; + /** PWA standalone mode: the page was launched from the home screen icon. */ + isStandalone = $state(false); /** Any WKWebView context on iOS: in-app browsers, embedded web views, and the * third-party iOS browsers (all of which share the WKWebView engine). */ readonly isWKWebView: boolean = false; - /** PWA standalone mode: the page was launched from the home screen icon. */ - isStandalone = $state(false); /** OS color scheme preference; the user override lives in settingsStore. */ readonly systemTheme = $state({ isDark: false }); diff --git a/tools/ui/src/lib/stores/index.ts b/tools/ui/src/lib/stores/index.ts index 1aea8a6da..db227158f 100644 --- a/tools/ui/src/lib/stores/index.ts +++ b/tools/ui/src/lib/stores/index.ts @@ -18,34 +18,32 @@ */ // CHAT / MESSAGING -export { chatStore } from './chat.svelte'; +export { chatStore } from './chat/index.svelte'; -export { draftMessagesStore } from './draft-messages.svelte'; - -// AGENTIC (multi-turn tool orchestration) -export { agenticStore } from './agentic.svelte'; - -// CONVERSATIONS -export { conversationsStore } from './conversations.svelte'; +export { draftMessagesStore } from './chat/drafts.svelte'; // CONTEXT STATS (active conversation context window usage) -export { contextStatsStore } from './context-stats.svelte'; +export { contextStatsStore } from './chat/context-stats.svelte'; + +// AGENTIC (multi-turn tool orchestration) +export { agenticStore } from './agentic/index.svelte'; + +// CONVERSATIONS +export { conversationsStore } from './conversations/index.svelte'; // MCP -export { mcpStore } from './mcp.svelte'; - -export { mcpResourceStore } from './mcp-resources.svelte'; +export { mcpStore } from './mcp/index.svelte'; // MODELS -export { modelsStore } from './models.svelte'; +export { modelsStore } from './models/index.svelte'; // SERVER export { serverStore } from './server.svelte'; // SETTINGS / UI PREFERENCES -export { settingsStore } from './settings.svelte'; +export { settingsStore } from './settings/index.svelte'; -export { settingsReferrer } from './settings-referrer.svelte'; +export { settingsReferrer } from './settings/referrer.svelte'; export { permissionsStore } from './permissions.svelte'; diff --git a/tools/ui/src/lib/stores/init.ts b/tools/ui/src/lib/stores/init.ts index 1faa80303..d52c34d0f 100644 --- a/tools/ui/src/lib/stores/init.ts +++ b/tools/ui/src/lib/stores/init.ts @@ -13,9 +13,9 @@ */ // direct imports, not via the barrel, to avoid circular deps -import { conversationsStore } from './conversations.svelte'; +import { conversationsStore } from './conversations/index.svelte'; import { permissionsStore } from './permissions.svelte'; -import { settingsStore } from './settings.svelte'; +import { settingsStore } from './settings/index.svelte'; import { toolsStore } from './tools.svelte'; import { versionStore } from './version.svelte'; import { browser } from '$app/environment'; @@ -33,7 +33,7 @@ export function initStores(): Promise { permissionsStore.initialize(); toolsStore.initialize(); void versionStore.initialize(); - void conversationsStore.init(); + void conversationsStore.initialize(); })(); return startup; diff --git a/tools/ui/src/lib/stores/mcp/health.svelte.ts b/tools/ui/src/lib/stores/mcp/health.svelte.ts new file mode 100644 index 000000000..fffa6ea92 --- /dev/null +++ b/tools/ui/src/lib/stores/mcp/health.svelte.ts @@ -0,0 +1,298 @@ +/** + * MCPHealthCheckManager - Health checks for MCP servers + * + * Owns per-server connectivity probes: connection reuse, capability + * snapshots, and promotion of a successful check to an active connection. + * Created and owned by mcpStore; the host owns the connection registry the + * probes draw from and promote into. + */ + +import { DEFAULT_MCP_CONFIG } from '$lib/constants'; +import { HealthCheckStatus, MCPConnectionPhase, MCPLogLevel } from '$lib/enums'; +import { MCPService } from '$lib/services/mcp.service'; +import type { + ClientCapabilities, + HealthCheckParams, + HealthCheckState, + MCPCapabilitiesInfo, + MCPConnection, + MCPConnectionLog, + MCPServerConfig, + ServerCapabilities +} from '$lib/types'; +import { detectMcpTransportFromUrl } from '$lib/utils'; + +// module-level so the timestamp is not flagged as reactive state by prefer-svelte-reactivity +function createConnectionErrorLog(message: string): MCPConnectionLog { + return { + level: MCPLogLevel.ERROR, + message: `Connection failed: ${message}`, + phase: MCPConnectionPhase.ERROR, + timestamp: new Date() + }; +} + +/** + * The slice of mcpStore the probes drive. Kept narrow on purpose so the + * probes cannot reach around the host's full surface; mcpStore implements + * this structurally. + */ +export interface McpHealthHost { + autoReconnect(serverName: string): Promise; + getExistingConnection(serverId: string): MCPConnection | undefined; + getRequestTimeoutMs(): number; + promoteHealthCheckToConnection(serverId: string, connection: MCPConnection): void; + registerServerConfig(name: string, config: MCPServerConfig): void; + removeConnection(serverId: string): void; +} + +export class MCPHealthCheckManager { + private _checks = $state>({}); + + /** Raw per-server check states, for host-side capability scans. */ + get checks(): Record { + return this._checks; + } + + clear(serverId: string): void { + const { [serverId]: _removed, ...rest } = this._checks; + + this._checks = rest; + } + + constructor(private host: McpHealthHost) {} + + getState(serverId: string): HealthCheckState { + return this._checks[serverId] ?? { status: HealthCheckStatus.IDLE }; + } + + hasState(serverId: string): boolean { + return serverId in this._checks && this._checks[serverId].status !== HealthCheckStatus.IDLE; + } + + /** + * Run a health check for a server. + * If the server already has an active connection, reuses it instead of creating a new one. + * If promoteToActive is true and server is enabled, the connection will be kept + * and promoted to an active connection instead of being disconnected. + */ + async run(server: HealthCheckParams, promoteToActive = false): Promise { + const existingConnection = this.host.getExistingConnection(server.id); + + if (existingConnection) { + // Reuse existing connection - just refresh tools list + try { + const tools = await MCPService.listTools(existingConnection); + const capabilities = this.buildCapabilitiesInfo( + existingConnection.serverCapabilities, + existingConnection.clientCapabilities + ); + + this.setState(server.id, { + capabilities, + connectionTimeMs: existingConnection.connectionTimeMs, + instructions: existingConnection.instructions, + logs: [], + protocolVersion: existingConnection.protocolVersion, + serverInfo: existingConnection.serverInfo, + status: HealthCheckStatus.SUCCESS, + tools: tools.map((tool) => ({ + description: tool.description, + name: tool.name, + title: tool.title + })), + transportType: existingConnection.transportType + }); + + return; + } catch (error) { + console.warn( + `[MCPStore] Failed to reuse connection for ${server.id}, creating new one:`, + error + ); + // Connection may be stale, remove it and create new one + this.host.removeConnection(server.id); + } + } + + const trimmedUrl = server.url.trim(); + const logs: MCPConnectionLog[] = []; + + let currentPhase: MCPConnectionPhase = MCPConnectionPhase.IDLE; + + if (!trimmedUrl) { + this.setState(server.id, { + logs: [], + message: 'Please enter a server URL first.', + status: HealthCheckStatus.ERROR + }); + + return; + } + + this.setState(server.id, { + logs: [], + phase: MCPConnectionPhase.TRANSPORT_CREATING, + status: HealthCheckStatus.CONNECTING + }); + + const timeoutMs = this.host.getRequestTimeoutMs(); + const headers = this.parseHeaders(server.headers); + + try { + const serverConfig: MCPServerConfig = { + handshakeTimeoutMs: DEFAULT_MCP_CONFIG.connectionTimeoutMs, + headers, + requestTimeoutMs: timeoutMs, + transport: detectMcpTransportFromUrl(trimmedUrl), + url: trimmedUrl, + useProxy: server.useProxy + }; + + this.host.registerServerConfig(server.id, serverConfig); + + const connection = await MCPService.connect( + server.id, + serverConfig, + DEFAULT_MCP_CONFIG.clientInfo, + DEFAULT_MCP_CONFIG.capabilities, + (phase, log) => { + currentPhase = phase; + logs.push(log); + this.setState(server.id, { + logs: [...logs], + phase, + status: HealthCheckStatus.CONNECTING + }); + + if (phase === MCPConnectionPhase.DISCONNECTED && promoteToActive) { + console.log( + `[MCPStore][${server.id}] Connection lost during health check, starting auto-reconnect` + ); + this.host.autoReconnect(server.id); + } + } + ); + const tools = connection.tools.map((tool) => ({ + description: tool.description, + name: tool.name, + title: tool.title + })); + const capabilities = this.buildCapabilitiesInfo( + connection.serverCapabilities, + connection.clientCapabilities + ); + + this.setState(server.id, { + capabilities, + connectionTimeMs: connection.connectionTimeMs, + instructions: connection.instructions, + logs, + protocolVersion: connection.protocolVersion, + serverInfo: connection.serverInfo, + status: HealthCheckStatus.SUCCESS, + tools, + transportType: connection.transportType + }); + + if (promoteToActive && server.enabled) { + this.host.promoteHealthCheckToConnection(server.id, connection); + } else { + await MCPService.disconnect(connection); + } + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error occurred'; + + if (logs.at(-1)?.phase !== MCPConnectionPhase.ERROR) { + logs.push(createConnectionErrorLog(message)); + } + + this.setState(server.id, { + logs, + message, + phase: currentPhase, + status: HealthCheckStatus.ERROR + }); + } + } + + async runForServers( + servers: { + id: string; + enabled: boolean; + url: string; + headers?: string; + }[], + skipIfChecked = true, + promoteToActive = false + ): Promise { + const serversToCheck = skipIfChecked + ? servers.filter((s) => !this.hasState(s.id) && s.url.trim()) + : servers.filter((s) => s.url.trim()); + + if (serversToCheck.length === 0) { + return; + } + + const BATCH_SIZE = 5; + + for (let i = 0; i < serversToCheck.length; i += BATCH_SIZE) { + const batch = serversToCheck.slice(i, i + BATCH_SIZE); + + await Promise.allSettled(batch.map((server) => this.run(server, promoteToActive))); + } + } + + /** + * Builds capabilities info from server and client capabilities. + */ + private buildCapabilitiesInfo( + serverCaps?: ServerCapabilities, + clientCaps?: ClientCapabilities + ): MCPCapabilitiesInfo { + return { + client: { + elicitation: clientCaps?.elicitation + ? { form: !!clientCaps.elicitation.form, url: !!clientCaps.elicitation.url } + : undefined, + roots: clientCaps?.roots ? { listChanged: clientCaps.roots.listChanged } : undefined, + sampling: !!clientCaps?.sampling, + tasks: !!clientCaps?.tasks + }, + server: { + completions: !!serverCaps?.completions, + logging: !!serverCaps?.logging, + prompts: serverCaps?.prompts ? { listChanged: serverCaps.prompts.listChanged } : undefined, + resources: serverCaps?.resources + ? { + listChanged: serverCaps.resources.listChanged, + subscribe: serverCaps.resources.subscribe + } + : undefined, + tasks: !!serverCaps?.tasks, + tools: serverCaps?.tools ? { listChanged: serverCaps.tools.listChanged } : undefined + } + }; + } + + private parseHeaders(headersJson?: string): Record | undefined { + if (!headersJson?.trim()) { + return undefined; + } + + try { + const parsed = JSON.parse(headersJson); + + if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) + return parsed as Record; + } catch { + console.warn('[MCPStore] Failed to parse custom headers JSON:', headersJson); + } + + return undefined; + } + + private setState(serverId: string, state: HealthCheckState): void { + this._checks = { ...this._checks, [serverId]: state }; + } +} diff --git a/tools/ui/src/lib/stores/mcp.svelte.ts b/tools/ui/src/lib/stores/mcp/index.svelte.ts similarity index 67% rename from tools/ui/src/lib/stores/mcp.svelte.ts rename to tools/ui/src/lib/stores/mcp/index.svelte.ts index 3e0cb8e1e..ccd53bc9d 100644 --- a/tools/ui/src/lib/stores/mcp.svelte.ts +++ b/tools/ui/src/lib/stores/mcp/index.svelte.ts @@ -1,69 +1,38 @@ /** - * mcpStore - Reactive State Store for MCP Operations + * mcpStore - MCP host: server connections and tool operations * - * Implements the "Host" role in MCP architecture, coordinating multiple server - * connections and providing a unified interface for tool operations. - * - * **Architecture & Relationships:** - * - **MCPService**: Stateless protocol layer (transport, connect, callTool) - * - **mcpStore** (this): Reactive state + business logic - * - * **Key Responsibilities:** - * - Lifecycle management (initialize, shutdown) - * - Multi-server coordination - * - Tool name conflict detection and resolution - * - Automatic tool-to-server routing - * - Health checks - * - * MCP connection state and raw `Tool[]` per server are owned here; the - * OpenAI-compatible wire format for those tools is built in `toolsStore` - * (see {@link toolsStore.mcpEntries} / {@link toolsStore.getEnabledToolsForLLM}). - * - * @see MCPService in services/mcp.service.ts for protocol operations + * Implements the MCP "Host" role, coordinating multiple server connections + * and exposing a unified tool interface: lifecycle, name-conflict detection + * and automatic tool-to-server routing. Owns connection state and raw + * `Tool[]` per server; the OpenAI-compatible wire format is built in + * toolsStore. Composes the health-check manager; uses MCPService for the + * protocol layer. */ import type { ListChangedHandlers } from '@modelcontextprotocol/sdk/types.js'; import { browser } from '$app/environment'; import { SETTINGS_KEYS } from '$lib/constants'; -import { - CACHE, - DEFAULT_MCP_CONFIG, - EXPECTED_THEMED_ICON_PAIR_COUNT, - MCP_ALLOWED_ICON_MIME_TYPES, - MCP_RECONNECT, - MCP_SERVER_ID_PREFIX -} from '$lib/constants'; -import { - ColorMode, - HealthCheckStatus, - MCPConnectionPhase, - MCPLogLevel, - MCPRefType, - UrlProtocol -} from '$lib/enums'; +import { CACHE, DEFAULT_MCP_CONFIG, MCP_RECONNECT, MCP_SERVER_ID_PREFIX } from '$lib/constants'; +import { ColorMode, HealthCheckStatus, MCPConnectionPhase, MCPRefType } from '$lib/enums'; import { MCPService } from '$lib/services/mcp.service'; // direct imports between stores, not via the barrel, to avoid circular deps -import { mcpResourceStore } from '$lib/stores/mcp-resources.svelte'; +import { MCPHealthCheckManager, type McpHealthHost } from '$lib/stores/mcp/health.svelte'; +import { mcpResourceStore } from '$lib/stores/mcp/resources.svelte'; import { serverStore } from '$lib/stores/server.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { - ClientCapabilities, GetPromptResult, HealthCheckParams, HealthCheckState, - MCPCapabilitiesInfo, MCPClientConfig, MCPConnection, - MCPConnectionLog, MCPPromptInfo, MCPResourceAttachment, MCPResourceContent, - MCPResourceIcon, MCPServerConfig, MCPServerDisplayInfo, MCPServerSettingsEntry, MCPToolCall, - ServerCapabilities, ServerStatus, Tool, ToolExecutionResult @@ -72,484 +41,78 @@ import type { DatabaseMessageExtraMcpResource, McpServerOverride } from '$lib/ty import type { SettingsConfigType } from '$lib/types/settings'; import { detectMcpTransportFromUrl, - extractRootDomain, + getMcpIconUrl, + getMcpServerFaviconFallback, + getMcpServerLabel, parseMcpServerSettings, uuid } from '$lib/utils'; import { mode } from 'mode-watcher'; -class MCPStore { - private _isInitializing = $state(false); +class MCPStore implements McpHealthHost { private _error = $state(null); + private _isInitializing = $state(false); private _toolCount = $state(0); - private _connectedServers = $state([]); - private _healthChecks = $state>({}); - - private connections = new Map(); - private toolsIndex = new Map(); - private serverConfigs = new Map(); // Store configs for reconnection - private reconnectingServers = new Set(); // Guard against concurrent reconnections - private configSignature: string | null = null; - private initPromise: Promise | null = null; private activeFlowCount = 0; - get isProxyAvailable(): boolean { - return serverStore.props?.cors_proxy_enabled ?? false; + private configSignature: string | null = null; + private connectedServers = $state([]); + private connections = new Map(); + // health checks: per-server connectivity probes with optional promotion to active connections + private health = new MCPHealthCheckManager(this); + private initPromise: Promise | null = null; + private reconnectingServers = new Set(); // Guard against concurrent reconnections + private serverConfigs = new Map(); // Store configs for reconnection + private serversCache: { raw: unknown; servers: MCPServerSettingsEntry[] } | null = null; + private toolsIndex = new Map(); + + get availableTools(): string[] { + return Array.from(this.toolsIndex.keys()); } - /** - * Generates a unique server ID from an optional ID string or index. - */ - #generateServerId(id: unknown, index: number): string { - if (typeof id === 'string' && id.trim()) { - return id.trim(); - } - - return `${MCP_SERVER_ID_PREFIX}-${index + 1}`; + get connectedServerCount(): number { + return this.connectedServers.length; } - /** - * Parses raw server settings from config into MCPServerSettingsEntry array. - */ - #parseServerSettings(rawServers: unknown): MCPServerSettingsEntry[] { - if (!rawServers) { - return []; - } - - let parsed: unknown; - - if (typeof rawServers === 'string') { - const trimmed = rawServers.trim(); - - if (!trimmed) { - return []; - } - - try { - parsed = JSON.parse(trimmed); - } catch (error) { - console.warn('[MCP] Failed to parse mcpServers JSON:', error); - - return []; - } - } else { - parsed = rawServers; - } - - if (!Array.isArray(parsed)) { - return []; - } - - return parsed.map((entry, index) => { - const url = typeof entry?.url === 'string' ? entry.url.trim() : ''; - const headers = typeof entry?.headers === 'string' ? entry.headers.trim() : undefined; - - return { - displayName: (entry as { displayName?: string })?.displayName, - enabled: Boolean((entry as { enabled?: unknown })?.enabled), - headers: headers || undefined, - id: this.#generateServerId((entry as { id?: unknown })?.id, index), - name: (entry as { name?: string })?.name, - url, - useProxy: Boolean((entry as { useProxy?: unknown })?.useProxy) - } satisfies MCPServerSettingsEntry; - }); - } - - /** - * Request timeout in milliseconds, read live from the global setting - * so a change in Settings applies to every server immediately. - */ - #requestTimeoutMs(): number { - const seconds = - Number(settingsStore.config.mcpRequestTimeoutSeconds) || - DEFAULT_MCP_CONFIG.requestTimeoutSeconds; - - return Math.round(seconds * 1000); - } - - /** - * Builds server configuration from a settings entry. - */ - #buildServerConfig( - entry: MCPServerSettingsEntry, - connectionTimeoutMs = DEFAULT_MCP_CONFIG.connectionTimeoutMs - ): MCPServerConfig | undefined { - if (!entry?.url) { - return undefined; - } - - let headers: Record | undefined; - - if (entry.headers) { - try { - const parsed = JSON.parse(entry.headers); - - if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) - headers = parsed as Record; - } catch { - console.warn('[MCP] Failed to parse custom headers JSON:', entry.headers); - } - } - - return { - handshakeTimeoutMs: connectionTimeoutMs, - headers, - requestTimeoutMs: this.#requestTimeoutMs(), - transport: detectMcpTransportFromUrl(entry.url), - url: entry.url, - useProxy: entry.useProxy - }; - } - - /** - * Checks if a server is enabled for a given chat. - * A per-chat override wins when present; a server without one resolves - * to its own `enabled` flag in `mcpServers`. - */ - #checkServerEnabled( - server: MCPServerSettingsEntry, - perChatOverrides?: McpServerOverride[] - ): boolean { - // Per-chat overrides win when present; missing entries inherit the - // server's own `enabled` flag so partial override lists are not all - // treated as disabled. - const override = perChatOverrides?.find((o) => o.serverId === server.id); - - return override?.enabled ?? server.enabled; - } - - /** - * Builds MCP client configuration from settings. - */ - #buildMcpClientConfig( - cfg: SettingsConfigType, - perChatOverrides?: McpServerOverride[] - ): MCPClientConfig | undefined { - const rawServers = this.#parseServerSettings(cfg.mcpServers); - - if (!rawServers.length) { - return undefined; - } - - const servers: Record = {}; - - for (const [index, entry] of rawServers.entries()) { - if (!this.#checkServerEnabled(entry, perChatOverrides)) continue; - - const normalized = this.#buildServerConfig(entry); - - if (normalized) servers[this.#generateServerId(entry.id, index)] = normalized; - } - - if (Object.keys(servers).length === 0) { - return undefined; - } - - return { - capabilities: DEFAULT_MCP_CONFIG.capabilities, - clientInfo: DEFAULT_MCP_CONFIG.clientInfo, - protocolVersion: DEFAULT_MCP_CONFIG.protocolVersion, - requestTimeoutMs: this.#requestTimeoutMs(), - servers - }; - } - - /** - * Builds capabilities info from server and client capabilities. - */ - #buildCapabilitiesInfo( - serverCaps?: ServerCapabilities, - clientCaps?: ClientCapabilities - ): MCPCapabilitiesInfo { - return { - client: { - elicitation: clientCaps?.elicitation - ? { form: !!clientCaps.elicitation.form, url: !!clientCaps.elicitation.url } - : undefined, - roots: clientCaps?.roots ? { listChanged: clientCaps.roots.listChanged } : undefined, - sampling: !!clientCaps?.sampling, - tasks: !!clientCaps?.tasks - }, - server: { - completions: !!serverCaps?.completions, - logging: !!serverCaps?.logging, - prompts: serverCaps?.prompts ? { listChanged: serverCaps.prompts.listChanged } : undefined, - resources: serverCaps?.resources - ? { - listChanged: serverCaps.resources.listChanged, - subscribe: serverCaps.resources.subscribe - } - : undefined, - tasks: !!serverCaps?.tasks, - tools: serverCaps?.tools ? { listChanged: serverCaps.tools.listChanged } : undefined - } - }; - } - - get isInitializing(): boolean { - return this._isInitializing; - } - - get isInitialized(): boolean { - return this.connections.size > 0; + get connectedServerNames(): string[] { + return this.connectedServers; } get error(): string | null { return this._error; } - get toolCount(): number { - return this._toolCount; - } - - get connectedServerCount(): number { - return this._connectedServers.length; - } - - get connectedServerNames(): string[] { - return this._connectedServers; - } - get isEnabled(): boolean { - const mcpConfig = this.#buildMcpClientConfig(settingsStore.config); + const mcpConfig = this.buildMcpClientConfig(settingsStore.config); return ( mcpConfig !== null && mcpConfig !== undefined && Object.keys(mcpConfig.servers).length > 0 ); } - get availableTools(): string[] { - return Array.from(this.toolsIndex.keys()); + get isInitialized(): boolean { + return this.connections.size > 0; } - private updateState(state: { - isInitializing?: boolean; - error?: string | null; - toolCount?: number; - connectedServers?: string[]; - }): void { - if (state.isInitializing !== undefined) { - this._isInitializing = state.isInitializing; - } - - if (state.error !== undefined) { - this._error = state.error; - } - - if (state.toolCount !== undefined) { - this._toolCount = state.toolCount; - } - - if (state.connectedServers !== undefined) { - this._connectedServers = state.connectedServers; - } + get isInitializing(): boolean { + return this._isInitializing; } - updateHealthCheck(serverId: string, state: HealthCheckState): void { - this._healthChecks = { ...this._healthChecks, [serverId]: state }; + get isProxyAvailable(): boolean { + return serverStore.props?.cors_proxy_enabled ?? false; } - getHealthCheckState(serverId: string): HealthCheckState { - return this._healthChecks[serverId] ?? { status: HealthCheckStatus.IDLE }; + /** Resource state, composed here so consumers have a single MCP scope. */ + get resources() { + return mcpResourceStore; } - hasHealthCheck(serverId: string): boolean { - return ( - serverId in this._healthChecks && - this._healthChecks[serverId].status !== HealthCheckStatus.IDLE - ); + get toolCount(): number { + return this._toolCount; } - clearHealthCheck(serverId: string): void { - const { [serverId]: _removed, ...rest } = this._healthChecks; - - this._healthChecks = rest; - } - - clearAllHealthChecks(): void { - this._healthChecks = {}; - } - - clearError(): void { - this._error = null; - } - - getServers(): MCPServerSettingsEntry[] { - return parseMcpServerSettings(settingsStore.config.mcpServers); - } - - /** - * Get all active MCP connections. - * @returns Map of server names to connections - */ - getConnections(): Map { - return this.connections; - } - - /** - * Resolves the raw label for a server: user-defined display name first, - * then server-reported title or name when the health check succeeded, - * then the configured name (admin baseline or legacy data), then URL. - */ - #serverBaseLabel(server: MCPServerDisplayInfo): string { - if (server.displayName) return server.displayName; - - const healthState = this.getHealthCheckState(server.id); - - if (healthState?.status === HealthCheckStatus.SUCCESS) - return ( - healthState.serverInfo?.title || healthState.serverInfo?.name || server.name || server.url - ); - - return server.name || server.url; - } - - /** - * Returns the display label for a server, suffixed with a positional - * counter when several configured servers resolve to the same base label - * (e.g. two endpoints of the same host reporting an identical name). - * Numbering follows config order, so it is stable across renders. - */ - getServerLabel(server: MCPServerDisplayInfo): string { - const label = this.#serverBaseLabel(server); - const twins = this.getServers().filter((s) => this.#serverBaseLabel(s) === label); - - if (twins.length < 2) return label; - - const position = twins.findIndex((s) => s.id === server.id); - - return position < 0 ? label : `${label} (${position + 1})`; - } - - getServerById(serverId: string): MCPServerSettingsEntry | undefined { - return this.getServers().find((s) => s.id === serverId); - } - - /** - * Get display name for an MCP server by its ID. - * Falls back to the server ID if server is not found. - */ - getServerDisplayName(serverId: string): string { - const server = this.getServerById(serverId); - - return server ? this.getServerLabel(server) : serverId; - } - - /** - * Validates that an icon URI uses a safe scheme (https: or data:). - */ - #isValidIconUri(src: string): boolean { - try { - if (src.startsWith(UrlProtocol.DATA)) return true; - - const url = new URL(src); - - return url.protocol === UrlProtocol.HTTPS; - } catch { - return false; - } - } - - /** - * Selects the best icon URL from an MCP icons array. - * Follows security guidelines from the MCP specification: - * - Only allows https: and data: URIs - * - Filters to supported MIME types - * - * Selection priority: - * 1. Icon matching the current color scheme (dark/light) - * 2. Universal icon (no theme specified); if exactly 2, assumes [0]=light, [1]=dark - * 3. First valid icon as last resort - */ - #getMcpIconUrl(icons: MCPResourceIcon[] | undefined, isDark = false): string | null { - if (!icons?.length) return null; - - const validIcons = icons.filter((icon) => { - if (!icon.src || !this.#isValidIconUri(icon.src)) return false; - - if (icon.mimeType && !MCP_ALLOWED_ICON_MIME_TYPES.has(icon.mimeType)) return false; - - return true; - }); - - if (validIcons.length === 0) return null; - - const preferredTheme = isDark ? ColorMode.DARK : ColorMode.LIGHT; - // 1. Prefer icon explicitly matching the current color scheme - const themedIcon = validIcons.find((icon) => icon.theme === preferredTheme); - - if (themedIcon) return themedIcon.src; - - // 2. Handle universal icons (no theme specified) - const universalIcons = validIcons.filter((icon) => !icon.theme); - - if (universalIcons.length === EXPECTED_THEMED_ICON_PAIR_COUNT) { - // Heuristic: two theme-less icons → assume [0] = light, [1] = dark - return universalIcons[isDark ? 1 : 0].src; - } - - if (universalIcons.length > 0) { - return universalIcons[0].src; - } - - // 3. Last resort: use opposite-theme icon - return validIcons[0].src; - } - - /** - * Get icon URL for an MCP server by its ID. - * Returns the best icon from the MCP server's `icons` array - * (see MCP spec: spec.modelcontextprotocol.io). - * Returns null if no icon is available. - */ - getServerFavicon(serverId: string): string | null { - const server = this.getServerById(serverId); - - if (!server) { - return null; - } - - const isDark = mode.current === ColorMode.DARK; - const healthState = this.getHealthCheckState(serverId); - - if (healthState.status === HealthCheckStatus.SUCCESS && healthState.serverInfo?.icons) { - const mcpIconUrl = this.#getMcpIconUrl(healthState.serverInfo.icons, isDark); - - if (mcpIconUrl) { - return mcpIconUrl; - } - } - - return this.#getServerFaviconFallback(server.url); - } - - /** - * Construct a fallback favicon URL from the MCP server URL. - * e.g. https://mcp.example.com/sse -> https://example.com/favicon.ico - */ - #getServerFaviconFallback(serverUrl: string): string | null { - try { - const url = new URL(serverUrl); - const rootDomain = extractRootDomain(url); - - if (!rootDomain) return null; - - const origin = `${url.protocol}//${rootDomain}`; - const candidates = ['favicon.ico', 'favicon.png']; - - for (const path of candidates) { - const faviconUrl = `${origin}/${path}`; - - if (this.#isValidIconUri(faviconUrl)) { - return faviconUrl; - } - } - } catch { - // Invalid URL, return null - } - - return null; + acquireConnection(): void { + this.activeFlowCount++; } addServer( @@ -571,321 +134,40 @@ class MCPStore { return newServer; } - updateServer(id: string, updates: Partial): void { - const servers = this.getServers(); + /** + * Add a resource as attachment to chat context. + * Automatically fetches content if not cached. + */ + async attachResource(uri: string): Promise { + const resourceInfo = mcpResourceStore.findResourceByUri(uri); - settingsStore.updateConfig( - SETTINGS_KEYS.MCP_SERVERS, - JSON.stringify( - servers.map((server) => (server.id === id ? { ...server, ...updates } : server)) - ) - ); - } + if (!resourceInfo) { + console.error(`[MCPStore] Resource not found: ${uri}`); - removeServer(id: string): void { - const servers = this.getServers(); - - settingsStore.updateConfig( - SETTINGS_KEYS.MCP_SERVERS, - JSON.stringify(servers.filter((s) => s.id !== id)) - ); - this.clearHealthCheck(id); - } - - hasAvailableServers(): boolean { - return parseMcpServerSettings(settingsStore.config.mcpServers).some( - (s) => s.enabled && s.url.trim() - ); - } - hasEnabledServers(perChatOverrides?: McpServerOverride[]): boolean { - return Boolean(this.#buildMcpClientConfig(settingsStore.config, perChatOverrides)); - } - - getEnabledServersForConversation( - perChatOverrides?: McpServerOverride[] - ): MCPServerSettingsEntry[] { - return this.getServers().filter((server) => { - return this.#checkServerEnabled(server, perChatOverrides); - }); - } - - async ensureInitialized(perChatOverrides?: McpServerOverride[]): Promise { - if (!browser) { - return false; + return null; } - const mcpConfig = this.#buildMcpClientConfig(settingsStore.config, perChatOverrides); - const signature = mcpConfig ? JSON.stringify(mcpConfig) : null; - - if (!signature) { - await this.shutdown(); - - return false; + if (mcpResourceStore.isAttached(uri)) { + return null; } - if (this.isInitialized && this.configSignature === signature) { - return true; - } + const attachment = mcpResourceStore.addAttachment(resourceInfo); - if (this.initPromise && this.configSignature === signature) { - return this.initPromise; - } + try { + const content = await this.readResource(uri); - if (this.connections.size > 0 || this.initPromise) await this.shutdown(); - - return this.initialize(signature, mcpConfig!); - } - - private async initialize(signature: string, mcpConfig: MCPClientConfig): Promise { - this.updateState({ error: null, isInitializing: true }); - this.configSignature = signature; - - const serverEntries = Object.entries(mcpConfig.servers); - - if (serverEntries.length === 0) { - this.updateState({ connectedServers: [], isInitializing: false, toolCount: 0 }); - - return false; - } - - this.initPromise = this.doInitialize(signature, mcpConfig, serverEntries); - - return this.initPromise; - } - - private async doInitialize( - signature: string, - mcpConfig: MCPClientConfig, - serverEntries: [string, MCPClientConfig['servers'][string]][] - ): Promise { - const clientInfo = mcpConfig.clientInfo ?? DEFAULT_MCP_CONFIG.clientInfo; - const capabilities = mcpConfig.capabilities ?? DEFAULT_MCP_CONFIG.capabilities; - const results = await Promise.allSettled( - serverEntries.map(async ([name, serverConfig]) => { - // Store config for reconnection - this.serverConfigs.set(name, serverConfig); - - const listChangedHandlers = this.createListChangedHandlers(name); - const connection = await MCPService.connect( - name, - serverConfig, - clientInfo, - capabilities, - (phase) => { - // Handle WebSocket disconnection - if (phase === MCPConnectionPhase.DISCONNECTED) { - console.log(`[MCPStore][${name}] Connection lost, starting auto-reconnect`); - this.autoReconnect(name); - } - }, - listChangedHandlers - ); - - return { connection, name }; - }) - ); - - if (this.configSignature !== signature) { - for (const result of results) { - if (result.status === 'fulfilled') - await MCPService.disconnect(result.value.connection).catch(console.warn); - } - - return false; - } - - for (const result of results) { - if (result.status === 'fulfilled') { - const { connection, name } = result.value; - - this.connections.set(name, connection); - - for (const tool of connection.tools) { - if (this.toolsIndex.has(tool.name)) - console.warn( - `[MCPStore] Tool name conflict: "${tool.name}" exists in "${this.toolsIndex.get(tool.name)}" and "${name}". Using tool from "${name}".` - ); - - this.toolsIndex.set(tool.name, name); - } + if (content) { + mcpResourceStore.updateAttachmentContent(attachment.id, content); } else { - console.error(`[MCPStore] Failed to connect:`, result.reason); + mcpResourceStore.updateAttachmentError(attachment.id, 'Failed to read resource'); } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + + mcpResourceStore.updateAttachmentError(attachment.id, message); } - const successCount = this.connections.size; - - if (successCount === 0 && serverEntries.length > 0) { - this.updateState({ - connectedServers: [], - error: 'All MCP server connections failed', - isInitializing: false, - toolCount: 0 - }); - this.initPromise = null; - - return false; - } - - this.updateState({ - connectedServers: Array.from(this.connections.keys()), - error: null, - isInitializing: false, - toolCount: this.toolsIndex.size - }); - this.initPromise = null; - - return true; - } - - private createListChangedHandlers(serverName: string): ListChangedHandlers { - return { - prompts: { - onChanged: (error: Error | null) => { - if (error) { - console.warn(`[MCPStore][${serverName}] Prompts list changed error:`, error); - - return; - } - } - }, - tools: { - onChanged: (error: Error | null, tools: Tool[] | null) => { - if (error) { - console.warn(`[MCPStore][${serverName}] Tools list changed error:`, error); - - return; - } - - this.handleToolsListChanged(serverName, tools ?? []); - } - } - }; - } - - private handleToolsListChanged(serverName: string, tools: Tool[]): void { - const connection = this.connections.get(serverName); - - if (!connection) { - return; - } - - for (const [toolName, ownerServer] of this.toolsIndex.entries()) { - if (ownerServer === serverName) this.toolsIndex.delete(toolName); - } - - connection.tools = tools; - - for (const tool of tools) { - if (this.toolsIndex.has(tool.name)) - console.warn( - `[MCPStore] Tool name conflict after list change: "${tool.name}" exists in "${this.toolsIndex.get(tool.name)}" and "${serverName}". Using tool from "${serverName}".` - ); - - this.toolsIndex.set(tool.name, serverName); - } - this.updateState({ toolCount: this.toolsIndex.size }); - } - - acquireConnection(): void { - this.activeFlowCount++; - } - - /** - * Release a connection reference. - * By default, keeps connections alive for reuse (shutdownIfUnused=false). - * MCP spec encourages long-lived sessions to avoid reconnection overhead. - */ - async releaseConnection(shutdownIfUnused = false): Promise { - this.activeFlowCount = Math.max(0, this.activeFlowCount - 1); - - if (shutdownIfUnused && this.activeFlowCount === 0) { - await this.shutdown(); - } - } - - getActiveFlowCount(): number { - return this.activeFlowCount; - } - - async shutdown(): Promise { - if (this.initPromise) { - await this.initPromise.catch(() => {}); - this.initPromise = null; - } - - if (this.connections.size === 0) { - return; - } - - await Promise.all( - Array.from(this.connections.values()).map((conn) => - MCPService.disconnect(conn).catch((error) => - console.warn(`[MCPStore] Error disconnecting ${conn.serverName}:`, error) - ) - ) - ); - - this.connections.clear(); - this.toolsIndex.clear(); - this.serverConfigs.clear(); - this.configSignature = null; - this.updateState({ - connectedServers: [], - error: null, - isInitializing: false, - toolCount: 0 - }); - } - - /** - * Immediately reconnect to a server by creating a fresh transport and session. - * Used when a session-expired error (HTTP 404) is detected during tool execution. - * Per MCP spec 2025-11-25: client MUST discard session ID and re-initialize. - * - * Unlike autoReconnect (which uses exponential backoff for connectivity issues), - * this performs a single immediate reconnection attempt since the server is known - * to be reachable (it responded with 404). - */ - private async reconnectServer(serverName: string): Promise { - const serverConfig = this.serverConfigs.get(serverName); - - if (!serverConfig) { - throw new Error(`[MCPStore] No config found for ${serverName}, cannot reconnect`); - } - - // Disconnect stale connection (clears old transport + session ID) - const oldConnection = this.connections.get(serverName); - - if (oldConnection) { - await MCPService.disconnect(oldConnection).catch(console.warn); - this.connections.delete(serverName); - } - - console.log(`[MCPStore][${serverName}] Session expired, reconnecting with fresh session...`); - - const listChangedHandlers = this.createListChangedHandlers(serverName); - const connection = await MCPService.connect( - serverName, - serverConfig, - DEFAULT_MCP_CONFIG.clientInfo, - DEFAULT_MCP_CONFIG.capabilities, - (phase) => { - if (phase === MCPConnectionPhase.DISCONNECTED) { - console.log(`[MCPStore][${serverName}] Connection lost, starting auto-reconnect`); - this.autoReconnect(serverName); - } - }, - listChangedHandlers - ); - - // Replace connection and rebuild tool index for this server - this.connections.set(serverName, connection); - for (const tool of connection.tools) { - this.toolsIndex.set(tool.name, serverName); - } - - console.log(`[MCPStore][${serverName}] Session recovered successfully`); + return mcpResourceStore.getAttachment(attachment.id) ?? null; } /** @@ -901,7 +183,7 @@ class MCPStore { * set inside the phase callback and honoured in the `finally` block after * the guard entry has been removed. */ - private async autoReconnect(serverName: string): Promise { + async autoReconnect(serverName: string): Promise { // Guard against concurrent reconnections if (this.reconnectingServers.has(serverName)) { console.log(`[MCPStore][${serverName}] Reconnection already in progress, skipping`); @@ -968,13 +250,10 @@ class MCPStore { ); const connection = await Promise.race([connectPromise, timeoutPromise]); - // Replace old connection with new one this.connections.set(serverName, connection); // Rebuild tool index for this server - for (const tool of connection.tools) { - this.toolsIndex.set(tool.name, serverName); - } + this.indexServerTools(serverName, connection.tools); console.log(`[MCPStore][${serverName}] Reconnected successfully`); @@ -998,182 +277,68 @@ class MCPStore { } } - getToolNames(): string[] { - return Array.from(this.toolsIndex.keys()); + clearError(): void { + this._error = null; } - hasTool(toolName: string): boolean { - return this.toolsIndex.has(toolName); - } - - getToolServer(toolName: string): string | undefined { - return this.toolsIndex.get(toolName); + clearHealthCheck(serverId: string): void { + this.health.clear(serverId); } /** - * Resolve which configured MCP server owns a given tool name. Looks at - * active connections first (fast path), then falls back to per-server - * health-check data so server-side MCP proxies (where llama-server - * executes MCP tools but the browser does not hold a direct connection) - * still resolve tool names to their owning server. + * Clear all resource attachments. */ - findServerForTool(toolName: string): string | undefined { - const fromIndex = this.toolsIndex.get(toolName); - - if (fromIndex) return fromIndex; - - for (const server of this.getServers()) { - const health = this._healthChecks[server.id]; - - if (!health || health.status !== HealthCheckStatus.SUCCESS) continue; - - if (health.tools.some((tool) => tool.name === toolName)) { - return server.id; - } - } - - return undefined; + clearResourceAttachments(): void { + mcpResourceStore.clearAttachments(); } /** - * Resolve the favicon URL for an MCP server by one of its tool names. - * Returns `null` if the tool is not provided by any configured MCP server, - * or if the owning server has no icon to show. - * Pair with {@link getServerFavicon} for direct server-id lookup. + * Convert current resource attachments to DatabaseMessageExtra[] and clear them. + * Called during message send to persist resources with the user message. */ - getServerFaviconForTool(toolName: string | undefined): string | null { - if (!toolName) return null; + consumeResourceAttachmentsAsExtras(): DatabaseMessageExtraMcpResource[] { + const extras = mcpResourceStore.toMessageExtras(); - const serverId = this.findServerForTool(toolName); - - if (!serverId) return null; - - return this.getServerFavicon(serverId); - } - - hasPromptsSupport(): boolean { - for (const connection of this.connections.values()) { - if (connection.serverCapabilities?.prompts) { - return true; - } + if (extras.length > 0) { + mcpResourceStore.clearAttachments(); } - return false; + return extras; } - /** - * Check if any enabled server with successful health check supports prompts. - * Uses health check state since servers may not have active connections until - * the user actually sends a message or uses prompts. - * @param perChatOverrides - Per-chat server overrides to filter by enabled servers. - * If provided (even empty array), only checks enabled servers. - * If undefined, falls back to each server's own `enabled` flag. - */ - hasPromptsCapability(perChatOverrides?: McpServerOverride[]): boolean { - let enabledServerIds: Set; - - if (perChatOverrides !== undefined) { - enabledServerIds = new Set(perChatOverrides.filter((o) => o.enabled).map((o) => o.serverId)); - } else { - enabledServerIds = new Set( - this.getServers() - .filter((s) => s.enabled) - .map((s) => s.id) - ); - } - - if (enabledServerIds.size === 0) { + async ensureInitialized(perChatOverrides?: McpServerOverride[]): Promise { + if (!browser) { return false; } - for (const [serverId, state] of Object.entries(this._healthChecks)) { - if (!enabledServerIds.has(serverId)) continue; + const mcpConfig = this.buildMcpClientConfig(settingsStore.config, perChatOverrides); + const signature = mcpConfig ? JSON.stringify(mcpConfig) : null; - if ( - state.status === HealthCheckStatus.SUCCESS && - state.capabilities?.server?.prompts !== undefined - ) { - return true; - } + if (!signature) { + await this.shutdown(); + + return false; } - for (const [serverName, connection] of this.connections) { - if (!enabledServerIds.has(serverName)) continue; - - if (connection.serverCapabilities?.prompts) { - return true; - } + if (this.isInitialized && this.configSignature === signature) { + return true; } - return false; - } - - async getAllPrompts(): Promise { - const results: MCPPromptInfo[] = []; - - for (const [serverName, connection] of this.connections) { - if (!connection.serverCapabilities?.prompts) continue; - - const prompts = await MCPService.listPrompts(connection); - - for (const prompt of prompts) { - results.push({ - arguments: prompt.arguments?.map((arg) => ({ - description: arg.description, - name: arg.name, - required: arg.required - })), - description: prompt.description, - name: prompt.name, - serverName, - title: prompt.title - }); - } + if (this.initPromise && this.configSignature === signature) { + return this.initPromise; } - return results; - } + if (this.connections.size > 0 || this.initPromise) await this.shutdown(); - async getPrompt( - serverName: string, - promptName: string, - args?: Record - ): Promise { - const connection = this.connections.get(serverName); - - if (!connection) throw new Error(`Server "${serverName}" not found for prompt "${promptName}"`); - - return MCPService.getPrompt(connection, promptName, args); + return this.initialize(signature, mcpConfig!); } async executeTool(toolCall: MCPToolCall, signal?: AbortSignal): Promise { - const toolName = toolCall.function.name; - const serverName = this.toolsIndex.get(toolName); - - if (!serverName) throw new Error(`Unknown tool: ${toolName}`); - - const connection = this.connections.get(serverName); - - if (!connection) throw new Error(`Server "${serverName}" is not connected`); - - const args = this.parseToolArguments(toolCall.function.arguments); - - try { - return await MCPService.callTool(connection, { arguments: args, name: toolName }, signal); - } catch (error) { - // Session expired (server restarted) - reconnect and retry once - if (MCPService.isSessionExpiredError(error)) { - await this.reconnectServer(serverName); - - const newConnection = this.connections.get(serverName); - - if (!newConnection) throw new Error(`Failed to reconnect to "${serverName}"`); - - return MCPService.callTool(newConnection, { arguments: args, name: toolName }, signal); - } - - throw error; - } + return this.executeToolByName( + toolCall.function.name, + this.parseToolArguments(toolCall.function.arguments), + signal + ); } async executeToolByName( @@ -1206,514 +371,6 @@ class MCPStore { } } - private parseToolArguments(args: string | Record): Record { - if (typeof args === 'string') { - const trimmed = args.trim(); - - if (trimmed === '') { - return {}; - } - - try { - const parsed = JSON.parse(trimmed); - - if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) - throw new Error( - `Tool arguments must be an object, got ${Array.isArray(parsed) ? 'array' : typeof parsed}` - ); - - return parsed as Record; - } catch (error) { - throw new Error(`Failed to parse tool arguments as JSON: ${(error as Error).message}`); - } - } - - if (typeof args === 'object' && args !== null && !Array.isArray(args)) { - return args; - } - - throw new Error(`Invalid tool arguments type: ${typeof args}`); - } - - async getPromptCompletions( - serverName: string, - promptName: string, - argumentName: string, - argumentValue: string - ): Promise<{ values: string[]; total?: number; hasMore?: boolean } | null> { - const connection = this.connections.get(serverName); - - if (!connection) { - console.warn(`[MCPStore] Server "${serverName}" is not connected`); - - return null; - } - - if (!connection.serverCapabilities?.completions) { - return null; - } - - return MCPService.complete( - connection, - { name: promptName, type: MCPRefType.PROMPT }, - { name: argumentName, value: argumentValue } - ); - } - - /** - * Get completions for a resource template argument. - * Uses the MCP Completion API with ref/resource. - */ - async getResourceCompletions( - serverName: string, - uriTemplate: string, - argumentName: string, - argumentValue: string - ): Promise<{ values: string[]; total?: number; hasMore?: boolean } | null> { - const connection = this.connections.get(serverName); - - if (!connection) { - console.warn(`[MCPStore] Server "${serverName}" is not connected`); - - return null; - } - - if (!connection.serverCapabilities?.completions) { - return null; - } - - return MCPService.complete( - connection, - { type: MCPRefType.RESOURCE, uri: uriTemplate }, - { name: argumentName, value: argumentValue } - ); - } - - /** - * Read a resource by an arbitrary URI (e.g., one expanded from a template). - * Unlike readResource(), this does not require the URI to be in the resources list. - */ - async readResourceByUri(serverName: string, uri: string): Promise { - const connection = this.connections.get(serverName); - - if (!connection) { - console.error(`[MCPStore] No connection found for server: ${serverName}`); - - return null; - } - - try { - const result = await MCPService.readResource(connection, uri); - - return result.contents; - } catch (error) { - console.error(`[MCPStore] Failed to read resource ${uri}:`, error); - - return null; - } - } - - private parseHeaders(headersJson?: string): Record | undefined { - if (!headersJson?.trim()) { - return undefined; - } - - try { - const parsed = JSON.parse(headersJson); - - if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) - return parsed as Record; - } catch { - console.warn('[MCPStore] Failed to parse custom headers JSON:', headersJson); - } - - return undefined; - } - - async runHealthChecksForServers( - servers: { - id: string; - enabled: boolean; - url: string; - headers?: string; - }[], - skipIfChecked = true, - promoteToActive = false - ): Promise { - const serversToCheck = skipIfChecked - ? servers.filter((s) => !this.hasHealthCheck(s.id) && s.url.trim()) - : servers.filter((s) => s.url.trim()); - - if (serversToCheck.length === 0) { - return; - } - - const BATCH_SIZE = 5; - - for (let i = 0; i < serversToCheck.length; i += BATCH_SIZE) { - const batch = serversToCheck.slice(i, i + BATCH_SIZE); - - await Promise.allSettled(batch.map((server) => this.runHealthCheck(server, promoteToActive))); - } - } - - /** - * Check if a server already has an active connection that can be reused. - * Returns the existing connection if available. - */ - getExistingConnection(serverId: string): MCPConnection | undefined { - return this.connections.get(serverId); - } - - /** - * Run a health check for a server. - * If the server already has an active connection, reuses it instead of creating a new one. - * If promoteToActive is true and server is enabled, the connection will be kept - * and promoted to an active connection instead of being disconnected. - */ - async runHealthCheck(server: HealthCheckParams, promoteToActive = false): Promise { - // Check if we already have an active connection for this server - const existingConnection = this.connections.get(server.id); - - if (existingConnection) { - // Reuse existing connection - just refresh tools list - try { - const tools = await MCPService.listTools(existingConnection); - const capabilities = this.#buildCapabilitiesInfo( - existingConnection.serverCapabilities, - existingConnection.clientCapabilities - ); - - this.updateHealthCheck(server.id, { - capabilities, - connectionTimeMs: existingConnection.connectionTimeMs, - instructions: existingConnection.instructions, - logs: [], - protocolVersion: existingConnection.protocolVersion, - serverInfo: existingConnection.serverInfo, - status: HealthCheckStatus.SUCCESS, - tools: tools.map((tool) => ({ - description: tool.description, - name: tool.name, - title: tool.title - })), - transportType: existingConnection.transportType - }); - - return; - } catch (error) { - console.warn( - `[MCPStore] Failed to reuse connection for ${server.id}, creating new one:`, - error - ); - // Connection may be stale, remove it and create new one - this.connections.delete(server.id); - } - } - - const trimmedUrl = server.url.trim(); - const logs: MCPConnectionLog[] = []; - - let currentPhase: MCPConnectionPhase = MCPConnectionPhase.IDLE; - - if (!trimmedUrl) { - this.updateHealthCheck(server.id, { - logs: [], - message: 'Please enter a server URL first.', - status: HealthCheckStatus.ERROR - }); - - return; - } - - this.updateHealthCheck(server.id, { - logs: [], - phase: MCPConnectionPhase.TRANSPORT_CREATING, - status: HealthCheckStatus.CONNECTING - }); - - const timeoutMs = this.#requestTimeoutMs(); - const headers = this.parseHeaders(server.headers); - - try { - const serverConfig: MCPServerConfig = { - handshakeTimeoutMs: DEFAULT_MCP_CONFIG.connectionTimeoutMs, - headers, - requestTimeoutMs: timeoutMs, - transport: detectMcpTransportFromUrl(trimmedUrl), - url: trimmedUrl, - useProxy: server.useProxy - }; - - // Store config for reconnection - this.serverConfigs.set(server.id, serverConfig); - - const connection = await MCPService.connect( - server.id, - serverConfig, - DEFAULT_MCP_CONFIG.clientInfo, - DEFAULT_MCP_CONFIG.capabilities, - (phase, log) => { - currentPhase = phase; - logs.push(log); - this.updateHealthCheck(server.id, { - logs: [...logs], - phase, - status: HealthCheckStatus.CONNECTING - }); - - // Handle WebSocket disconnection - if (phase === MCPConnectionPhase.DISCONNECTED && promoteToActive) { - console.log( - `[MCPStore][${server.id}] Connection lost during health check, starting auto-reconnect` - ); - this.autoReconnect(server.id); - } - } - ); - const tools = connection.tools.map((tool) => ({ - description: tool.description, - name: tool.name, - title: tool.title - })); - const capabilities = this.#buildCapabilitiesInfo( - connection.serverCapabilities, - connection.clientCapabilities - ); - - this.updateHealthCheck(server.id, { - capabilities, - connectionTimeMs: connection.connectionTimeMs, - instructions: connection.instructions, - logs, - protocolVersion: connection.protocolVersion, - serverInfo: connection.serverInfo, - status: HealthCheckStatus.SUCCESS, - tools, - transportType: connection.transportType - }); - - // Promote to active connection or disconnect - if (promoteToActive && server.enabled) { - this.promoteHealthCheckToConnection(server.id, connection); - } else { - await MCPService.disconnect(connection); - } - } catch (error) { - const message = error instanceof Error ? error.message : 'Unknown error occurred'; - - if (logs.at(-1)?.phase !== MCPConnectionPhase.ERROR) { - logs.push({ - level: MCPLogLevel.ERROR, - message: `Connection failed: ${message}`, - phase: MCPConnectionPhase.ERROR, - timestamp: new Date() - }); - } - - this.updateHealthCheck(server.id, { - logs, - message, - phase: currentPhase, - status: HealthCheckStatus.ERROR - }); - } - } - - /** - * Promote a health check connection to an active connection. - * This avoids the need to reconnect when the server is needed for agentic flows. - */ - private promoteHealthCheckToConnection(serverId: string, connection: MCPConnection): void { - // Register tools from the connection - for (const tool of connection.tools) { - if (this.toolsIndex.has(tool.name)) { - console.warn( - `[MCPStore] Tool name conflict during promotion: "${tool.name}" exists in "${this.toolsIndex.get(tool.name)}" and "${serverId}". Using tool from "${serverId}".` - ); - } - - this.toolsIndex.set(tool.name, serverId); - } - - // Add to active connections - this.connections.set(serverId, connection); - - // Update state - this.updateState({ - connectedServers: Array.from(this.connections.keys()), - toolCount: this.toolsIndex.size - }); - } - - getServersStatus(): ServerStatus[] { - const statuses: ServerStatus[] = []; - - for (const [name, connection] of this.connections) { - statuses.push({ - error: undefined, - isConnected: true, - name, - toolCount: connection.tools.length - }); - } - - return statuses; - } - - /** - * Get aggregated server instructions from all connected servers. - * Returns an array of { serverName, serverTitle, instructions } objects. - */ - getServerInstructions(): Array<{ - serverName: string; - serverTitle?: string; - instructions: string; - }> { - const results: Array<{ serverName: string; serverTitle?: string; instructions: string }> = []; - - for (const [serverName, connection] of this.connections) { - if (connection.instructions) { - results.push({ - instructions: connection.instructions, - serverName, - serverTitle: connection.serverInfo?.title || connection.serverInfo?.name - }); - } - } - - return results; - } - - /** - * Get server instructions from health check results (for display before active connection). - * Useful for showing instructions in settings UI. - */ - getHealthCheckInstructions(): Array<{ - serverId: string; - serverTitle?: string; - instructions: string; - }> { - const results: Array<{ serverId: string; serverTitle?: string; instructions: string }> = []; - - for (const [serverId, state] of Object.entries(this._healthChecks)) { - if (state.status === HealthCheckStatus.SUCCESS && state.instructions) { - results.push({ - instructions: state.instructions, - serverId, - serverTitle: state.serverInfo?.title || state.serverInfo?.name - }); - } - } - - return results; - } - - /** - * Check if any connected server has instructions. - */ - hasServerInstructions(): boolean { - for (const connection of this.connections.values()) { - if (connection.instructions) { - return true; - } - } - - return false; - } - - /** - * - * - * Resources Operations - * - * - */ - - /** - * Check if any enabled server with successful health check supports resources. - * Uses health check state since servers may not have active connections until - * the user actually sends a message or uses prompts. - * @param perChatOverrides - Per-chat server overrides to filter by enabled servers. - * If provided (even empty array), only checks enabled servers. - * If undefined, falls back to each server's own `enabled` flag. - */ - hasResourcesCapability(perChatOverrides?: McpServerOverride[]): boolean { - let enabledServerIds: Set; - - if (perChatOverrides !== undefined) { - enabledServerIds = new Set(perChatOverrides.filter((o) => o.enabled).map((o) => o.serverId)); - } else { - enabledServerIds = new Set( - this.getServers() - .filter((s) => s.enabled) - .map((s) => s.id) - ); - } - - if (enabledServerIds.size === 0) { - return false; - } - - for (const [serverId, state] of Object.entries(this._healthChecks)) { - if (!enabledServerIds.has(serverId)) continue; - - if ( - state.status === HealthCheckStatus.SUCCESS && - state.capabilities?.server?.resources !== undefined - ) { - return true; - } - } - - for (const [serverName, connection] of this.connections) { - if (!enabledServerIds.has(serverName)) continue; - - if (MCPService.supportsResources(connection)) { - return true; - } - } - - return false; - } - - /** - * Get list of enabled servers that support resources. - * Checks active connections first, then health check state as fallback. - */ - getServersWithResources(): string[] { - const enabledServerIds = new Set( - this.getServers() - .filter((s) => s.enabled) - .map((s) => s.id) - ); - const servers: string[] = []; - - // Check active connections - for (const [name, connection] of this.connections) { - if (!enabledServerIds.has(name)) continue; - - if (MCPService.supportsResources(connection) && !servers.includes(name)) { - servers.push(name); - } - } - - // Also check health check states for servers not yet connected - for (const [serverId, state] of Object.entries(this._healthChecks)) { - if (!enabledServerIds.has(serverId)) continue; - - if ( - !servers.includes(serverId) && - state.status === HealthCheckStatus.SUCCESS && - state.capabilities?.server?.resources !== undefined - ) { - servers.push(serverId); - } - } - - return servers; - } - /** * Fetch resources from all connected servers that support them. * Updates mcpResourceStore with the results. @@ -1793,12 +450,506 @@ class MCPStore { } } + /** + * Resolve which configured MCP server owns a given tool name. Looks at + * active connections first (fast path), then falls back to per-server + * health-check data so server-side MCP proxies (where llama-server + * executes MCP tools but the browser does not hold a direct connection) + * still resolve tool names to their owning server. + */ + findServerForTool(toolName: string): string | undefined { + const fromIndex = this.toolsIndex.get(toolName); + + if (fromIndex) return fromIndex; + + for (const server of this.getServers()) { + const health = this.health.checks[server.id]; + + if (!health || health.status !== HealthCheckStatus.SUCCESS) continue; + + if (health.tools.some((tool) => tool.name === toolName)) { + return server.id; + } + } + + return undefined; + } + getActiveFlowCount(): number { + return this.activeFlowCount; + } + + async getAllPrompts(): Promise { + const results: MCPPromptInfo[] = []; + + for (const [serverName, connection] of this.connections) { + if (!connection.serverCapabilities?.prompts) continue; + + const prompts = await MCPService.listPrompts(connection); + + for (const prompt of prompts) { + results.push({ + arguments: prompt.arguments?.map((arg) => ({ + description: arg.description, + name: arg.name, + required: arg.required + })), + description: prompt.description, + name: prompt.name, + serverName, + title: prompt.title + }); + } + } + + return results; + } + + /** + * Get all active MCP connections. + * @returns Map of server names to connections + */ + getConnections(): Map { + return this.connections; + } + + getEnabledServersForConversation( + perChatOverrides?: McpServerOverride[] + ): MCPServerSettingsEntry[] { + return this.getServers().filter((server) => { + return this.checkServerEnabled(server, perChatOverrides); + }); + } + + /** + * Check if a server already has an active connection that can be reused. + * Returns the existing connection if available. + */ + getExistingConnection(serverId: string): MCPConnection | undefined { + return this.connections.get(serverId); + } + + /** + * Get server instructions from health check results (for display before active connection). + * Useful for showing instructions in settings UI. + */ + getHealthCheckInstructions(): Array<{ + serverId: string; + serverTitle?: string; + instructions: string; + }> { + const results: Array<{ serverId: string; serverTitle?: string; instructions: string }> = []; + + for (const [serverId, state] of Object.entries(this.health.checks)) { + if (state.status === HealthCheckStatus.SUCCESS && state.instructions) { + results.push({ + instructions: state.instructions, + serverId, + serverTitle: state.serverInfo?.title || state.serverInfo?.name + }); + } + } + + return results; + } + + /** + * Health checks live in MCPHealthCheckManager; these delegate so + * consumers keep a single entry point. + */ + getHealthCheckState(serverId: string): HealthCheckState { + return this.health.getState(serverId); + } + + async getPrompt( + serverName: string, + promptName: string, + args?: Record + ): Promise { + const connection = this.connections.get(serverName); + + if (!connection) throw new Error(`Server "${serverName}" not found for prompt "${promptName}"`); + + return MCPService.getPrompt(connection, promptName, args); + } + + async getPromptCompletions( + serverName: string, + promptName: string, + argumentName: string, + argumentValue: string + ): Promise<{ values: string[]; total?: number; hasMore?: boolean } | null> { + const connection = this.connections.get(serverName); + + if (!connection) { + console.warn(`[MCPStore] Server "${serverName}" is not connected`); + + return null; + } + + if (!connection.serverCapabilities?.completions) { + return null; + } + + return MCPService.complete( + connection, + { name: promptName, type: MCPRefType.PROMPT }, + { name: argumentName, value: argumentValue } + ); + } + + /** + * Request timeout in milliseconds, read live from the global setting + * so a change in Settings applies to every server immediately. + */ + getRequestTimeoutMs(): number { + const seconds = + Number(settingsStore.config.mcpRequestTimeoutSeconds) || + DEFAULT_MCP_CONFIG.requestTimeoutSeconds; + + return Math.round(seconds * 1000); + } + + /** + * Get completions for a resource template argument. + * Uses the MCP Completion API with ref/resource. + */ + async getResourceCompletions( + serverName: string, + uriTemplate: string, + argumentName: string, + argumentValue: string + ): Promise<{ values: string[]; total?: number; hasMore?: boolean } | null> { + const connection = this.connections.get(serverName); + + if (!connection) { + console.warn(`[MCPStore] Server "${serverName}" is not connected`); + + return null; + } + + if (!connection.serverCapabilities?.completions) { + return null; + } + + return MCPService.complete( + connection, + { type: MCPRefType.RESOURCE, uri: uriTemplate }, + { name: argumentName, value: argumentValue } + ); + } + + /** + * Get formatted resource context for chat. + */ + getResourceContextForChat(): string { + return mcpResourceStore.formatAttachmentsForContext(); + } + + getServerById(serverId: string): MCPServerSettingsEntry | undefined { + return this.getServers().find((s) => s.id === serverId); + } + + /** + * Get display name for an MCP server by its ID. + * Falls back to the server ID if server is not found. + */ + getServerDisplayName(serverId: string): string { + const server = this.getServerById(serverId); + + return server ? this.getServerLabel(server) : serverId; + } + + /** + * Get icon URL for an MCP server by its ID. + * Returns the best icon from the MCP server's `icons` array + * (see MCP spec: spec.modelcontextprotocol.io). + * Returns null if no icon is available. + */ + getServerFavicon(serverId: string): string | null { + const server = this.getServerById(serverId); + + if (!server) { + return null; + } + + const isDark = mode.current === ColorMode.DARK; + const healthState = this.health.getState(serverId); + + if (healthState.status === HealthCheckStatus.SUCCESS && healthState.serverInfo?.icons) { + const mcpIconUrl = getMcpIconUrl(healthState.serverInfo.icons, isDark); + + if (mcpIconUrl) { + return mcpIconUrl; + } + } + + return getMcpServerFaviconFallback(server.url); + } + + /** + * Resolve the favicon URL for an MCP server by one of its tool names. + * Returns `null` if the tool is not provided by any configured MCP server, + * or if the owning server has no icon to show. + * Pair with {@link getServerFavicon} for direct server-id lookup. + */ + getServerFaviconForTool(toolName: string | undefined): string | null { + if (!toolName) return null; + + const serverId = this.findServerForTool(toolName); + + if (!serverId) return null; + + return this.getServerFavicon(serverId); + } + + /** + * Get aggregated server instructions from all connected servers. + * Returns an array of { serverName, serverTitle, instructions } objects. + */ + getServerInstructions(): Array<{ + serverName: string; + serverTitle?: string; + instructions: string; + }> { + const results: Array<{ serverName: string; serverTitle?: string; instructions: string }> = []; + + for (const [serverName, connection] of this.connections) { + if (connection.instructions) { + results.push({ + instructions: connection.instructions, + serverName, + serverTitle: connection.serverInfo?.title || connection.serverInfo?.name + }); + } + } + + return results; + } + + getServerLabel(server: MCPServerDisplayInfo): string { + return getMcpServerLabel(server, this.getServers(), this.health.checks); + } + + getServers(): MCPServerSettingsEntry[] { + const raw = settingsStore.config.mcpServers; + + // cache the parse: the config string rarely changes and getServers is + // called from hot paths (per-tool display lookups, capability checks) + if (this.serversCache && this.serversCache.raw === raw) { + return this.serversCache.servers; + } + + const servers = parseMcpServerSettings(raw); + + this.serversCache = { raw, servers }; + + return servers; + } + + getServersStatus(): ServerStatus[] { + const statuses: ServerStatus[] = []; + + for (const [name, connection] of this.connections) { + statuses.push({ + error: undefined, + isConnected: true, + name, + toolCount: connection.tools.length + }); + } + + return statuses; + } + + /** + * Get list of enabled servers that support resources. + * Checks active connections first, then health check state as fallback. + */ + getServersWithResources(): string[] { + const enabledServerIds = new Set( + this.getServers() + .filter((s) => s.enabled) + .map((s) => s.id) + ); + const servers: string[] = []; + + for (const [name, connection] of this.connections) { + if (!enabledServerIds.has(name)) continue; + + if (MCPService.supportsResources(connection) && !servers.includes(name)) { + servers.push(name); + } + } + + // Also check health check states for servers not yet connected + for (const [serverId, state] of Object.entries(this.health.checks)) { + if (!enabledServerIds.has(serverId)) continue; + + if ( + !servers.includes(serverId) && + state.status === HealthCheckStatus.SUCCESS && + state.capabilities?.server?.resources !== undefined + ) { + servers.push(serverId); + } + } + + return servers; + } + + getToolNames(): string[] { + return Array.from(this.toolsIndex.keys()); + } + + getToolServer(toolName: string): string | undefined { + return this.toolsIndex.get(toolName); + } + + hasAvailableServers(): boolean { + return parseMcpServerSettings(settingsStore.config.mcpServers).some( + (s) => s.enabled && s.url.trim() + ); + } + + hasEnabledServers(perChatOverrides?: McpServerOverride[]): boolean { + return Boolean(this.buildMcpClientConfig(settingsStore.config, perChatOverrides)); + } + + /** + * Check if any enabled server with successful health check supports prompts. + * Uses health check state since servers may not have active connections until + * the user actually sends a message or uses prompts. + */ + hasPromptsCapability(perChatOverrides?: McpServerOverride[]): boolean { + let enabledServerIds: Set; + + if (perChatOverrides !== undefined) { + enabledServerIds = new Set(perChatOverrides.filter((o) => o.enabled).map((o) => o.serverId)); + } else { + enabledServerIds = new Set( + this.getServers() + .filter((s) => s.enabled) + .map((s) => s.id) + ); + } + + if (enabledServerIds.size === 0) { + return false; + } + + for (const [serverId, state] of Object.entries(this.health.checks)) { + if (!enabledServerIds.has(serverId)) continue; + + if ( + state.status === HealthCheckStatus.SUCCESS && + state.capabilities?.server?.prompts !== undefined + ) { + return true; + } + } + + for (const [serverName, connection] of this.connections) { + if (!enabledServerIds.has(serverName)) continue; + + if (connection.serverCapabilities?.prompts) { + return true; + } + } + + return false; + } + + hasPromptsSupport(): boolean { + for (const connection of this.connections.values()) { + if (connection.serverCapabilities?.prompts) { + return true; + } + } + + return false; + } + + /** + * Check if any enabled server with successful health check supports resources. + * Uses health check state since servers may not have active connections until + * the user actually sends a message or uses prompts. + */ + hasResourcesCapability(perChatOverrides?: McpServerOverride[]): boolean { + let enabledServerIds: Set; + + if (perChatOverrides !== undefined) { + enabledServerIds = new Set(perChatOverrides.filter((o) => o.enabled).map((o) => o.serverId)); + } else { + enabledServerIds = new Set( + this.getServers() + .filter((s) => s.enabled) + .map((s) => s.id) + ); + } + + if (enabledServerIds.size === 0) { + return false; + } + + for (const [serverId, state] of Object.entries(this.health.checks)) { + if (!enabledServerIds.has(serverId)) continue; + + if ( + state.status === HealthCheckStatus.SUCCESS && + state.capabilities?.server?.resources !== undefined + ) { + return true; + } + } + + for (const [serverName, connection] of this.connections) { + if (!enabledServerIds.has(serverName)) continue; + + if (MCPService.supportsResources(connection)) { + return true; + } + } + + return false; + } + + /** + * Check if any connected server has instructions. + */ + hasServerInstructions(): boolean { + for (const connection of this.connections.values()) { + if (connection.instructions) { + return true; + } + } + + return false; + } + + hasTool(toolName: string): boolean { + return this.toolsIndex.has(toolName); + } + + /** + * Promote a health check connection to an active connection. + * This avoids the need to reconnect when the server is needed for agentic flows. + */ + promoteHealthCheckToConnection(serverId: string, connection: MCPConnection): void { + this.indexServerTools(serverId, connection.tools); + + this.connections.set(serverId, connection); + + this.updateState({ + connectedServers: Array.from(this.connections.keys()), + toolCount: this.toolsIndex.size + }); + } + /** * Read resource content from a server. * Caches the result in mcpResourceStore. */ async readResource(uri: string): Promise { - // Check cache first const cached = mcpResourceStore.getCachedContent(uri); if (cached) { @@ -1838,6 +989,120 @@ class MCPStore { } } + /** + * Read a resource by an arbitrary URI (e.g., one expanded from a template). + * Unlike readResource(), this does not require the URI to be in the resources list. + */ + async readResourceByUri(serverName: string, uri: string): Promise { + const connection = this.connections.get(serverName); + + if (!connection) { + console.error(`[MCPStore] No connection found for server: ${serverName}`); + + return null; + } + + try { + const result = await MCPService.readResource(connection, uri); + + return result.contents; + } catch (error) { + console.error(`[MCPStore] Failed to read resource ${uri}:`, error); + + return null; + } + } + + /** Store a server config so auto-reconnect can rebuild the session. */ + registerServerConfig(name: string, config: MCPServerConfig): void { + this.serverConfigs.set(name, config); + } + + /** + * Release a connection reference. + * By default, keeps connections alive for reuse (shutdownIfUnused=false). + * MCP spec encourages long-lived sessions to avoid reconnection overhead. + */ + async releaseConnection(shutdownIfUnused = false): Promise { + this.activeFlowCount = Math.max(0, this.activeFlowCount - 1); + + if (shutdownIfUnused && this.activeFlowCount === 0) { + await this.shutdown(); + } + } + + /** + * Drop a connection without disconnecting, e.g. when a health check finds + * it stale and recreates it. + */ + removeConnection(serverId: string): void { + this.connections.delete(serverId); + } + + /** + * Remove a resource attachment from chat context. + */ + removeResourceAttachment(attachmentId: string): void { + mcpResourceStore.removeAttachment(attachmentId); + } + + removeServer(id: string): void { + const servers = this.getServers(); + + settingsStore.updateConfig( + SETTINGS_KEYS.MCP_SERVERS, + JSON.stringify(servers.filter((s) => s.id !== id)) + ); + this.clearHealthCheck(id); + } + + async runHealthCheck(server: HealthCheckParams, promoteToActive = false): Promise { + return this.health.run(server, promoteToActive); + } + + async runHealthChecksForServers( + servers: { + id: string; + enabled: boolean; + url: string; + headers?: string; + }[], + skipIfChecked = true, + promoteToActive = false + ): Promise { + return this.health.runForServers(servers, skipIfChecked, promoteToActive); + } + + async shutdown(): Promise { + if (this.initPromise) { + await this.initPromise.catch(() => {}); + this.initPromise = null; + } + + if (this.connections.size === 0) { + return; + } + + await Promise.all( + Array.from(this.connections.values()).map((conn) => + MCPService.disconnect(conn).catch((error) => + console.warn(`[MCPStore] Error disconnecting ${conn.serverName}:`, error) + ) + ) + ); + + this.connections.clear(); + this.toolsIndex.clear(); + this.serverConfigs.clear(); + this.configSignature = null; + this.updateState({ + connectedServers: [], + error: null, + isInitializing: false, + toolCount: 0 + }); + } + /** * Subscribe to resource updates. */ @@ -1906,78 +1171,366 @@ class MCPStore { } } + updateServer(id: string, updates: Partial): void { + const servers = this.getServers(); + + settingsStore.updateConfig( + SETTINGS_KEYS.MCP_SERVERS, + JSON.stringify( + servers.map((server) => (server.id === id ? { ...server, ...updates } : server)) + ) + ); + } + /** - * Add a resource as attachment to chat context. - * Automatically fetches content if not cached. + * Builds MCP client configuration from settings. */ - async attachResource(uri: string): Promise { - const resourceInfo = mcpResourceStore.findResourceByUri(uri); + private buildMcpClientConfig( + cfg: SettingsConfigType, + perChatOverrides?: McpServerOverride[] + ): MCPClientConfig | undefined { + const rawServers = parseMcpServerSettings(cfg.mcpServers); - if (!resourceInfo) { - console.error(`[MCPStore] Resource not found: ${uri}`); - - return null; + if (!rawServers.length) { + return undefined; } - // Check if already attached - if (mcpResourceStore.isAttached(uri)) { - return null; + const servers: Record = {}; + + for (const [index, entry] of rawServers.entries()) { + if (!this.checkServerEnabled(entry, perChatOverrides)) continue; + + const normalized = this.buildServerConfig(entry); + + if (normalized) servers[this.generateServerId(entry.id, index)] = normalized; } - // Add attachment (initially loading) - const attachment = mcpResourceStore.addAttachment(resourceInfo); + if (Object.keys(servers).length === 0) { + return undefined; + } - // Fetch content - try { - const content = await this.readResource(uri); + return { + capabilities: DEFAULT_MCP_CONFIG.capabilities, + clientInfo: DEFAULT_MCP_CONFIG.clientInfo, + protocolVersion: DEFAULT_MCP_CONFIG.protocolVersion, + requestTimeoutMs: this.getRequestTimeoutMs(), + servers + }; + } - if (content) { - mcpResourceStore.updateAttachmentContent(attachment.id, content); - } else { - mcpResourceStore.updateAttachmentError(attachment.id, 'Failed to read resource'); + /** + * Builds server configuration from a settings entry. + */ + private buildServerConfig( + entry: MCPServerSettingsEntry, + connectionTimeoutMs = DEFAULT_MCP_CONFIG.connectionTimeoutMs + ): MCPServerConfig | undefined { + if (!entry?.url) { + return undefined; + } + + let headers: Record | undefined; + + if (entry.headers) { + try { + const parsed = JSON.parse(entry.headers); + + if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) + headers = parsed as Record; + } catch { + console.warn('[MCP] Failed to parse custom headers JSON:', entry.headers); } - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - - mcpResourceStore.updateAttachmentError(attachment.id, message); } - return mcpResourceStore.getAttachment(attachment.id) ?? null; + return { + handshakeTimeoutMs: connectionTimeoutMs, + headers, + requestTimeoutMs: this.getRequestTimeoutMs(), + transport: detectMcpTransportFromUrl(entry.url), + url: entry.url, + useProxy: entry.useProxy + }; } /** - * Remove a resource attachment from chat context. + * Checks if a server is enabled for a given chat. + * A per-chat override wins when present; a server without one resolves + * to its own `enabled` flag in `mcpServers`. */ - removeResourceAttachment(attachmentId: string): void { - mcpResourceStore.removeAttachment(attachmentId); + private checkServerEnabled( + server: MCPServerSettingsEntry, + perChatOverrides?: McpServerOverride[] + ): boolean { + const override = perChatOverrides?.find((o) => o.serverId === server.id); + + return override?.enabled ?? server.enabled; } - /** - * Clear all resource attachments. - */ - clearResourceAttachments(): void { - mcpResourceStore.clearAttachments(); + private createListChangedHandlers(serverName: string): ListChangedHandlers { + return { + prompts: { + onChanged: (error: Error | null) => { + if (error) { + console.warn(`[MCPStore][${serverName}] Prompts list changed error:`, error); + + return; + } + } + }, + tools: { + onChanged: (error: Error | null, tools: Tool[] | null) => { + if (error) { + console.warn(`[MCPStore][${serverName}] Tools list changed error:`, error); + + return; + } + + this.handleToolsListChanged(serverName, tools ?? []); + } + } + }; } - /** - * Get formatted resource context for chat. - */ - getResourceContextForChat(): string { - return mcpResourceStore.formatAttachmentsForContext(); - } + private async doInitialize( + signature: string, + mcpConfig: MCPClientConfig, + serverEntries: [string, MCPClientConfig['servers'][string]][] + ): Promise { + const clientInfo = mcpConfig.clientInfo ?? DEFAULT_MCP_CONFIG.clientInfo; + const capabilities = mcpConfig.capabilities ?? DEFAULT_MCP_CONFIG.capabilities; + const results = await Promise.allSettled( + serverEntries.map(async ([name, serverConfig]) => { + this.serverConfigs.set(name, serverConfig); - /** - * Convert current resource attachments to DatabaseMessageExtra[] and clear them. - * Called during message send to persist resources with the user message. - */ - consumeResourceAttachmentsAsExtras(): DatabaseMessageExtraMcpResource[] { - const extras = mcpResourceStore.toMessageExtras(); + const listChangedHandlers = this.createListChangedHandlers(name); + const connection = await MCPService.connect( + name, + serverConfig, + clientInfo, + capabilities, + (phase) => { + if (phase === MCPConnectionPhase.DISCONNECTED) { + console.log(`[MCPStore][${name}] Connection lost, starting auto-reconnect`); + this.autoReconnect(name); + } + }, + listChangedHandlers + ); - if (extras.length > 0) { - mcpResourceStore.clearAttachments(); + return { connection, name }; + }) + ); + + if (this.configSignature !== signature) { + for (const result of results) { + if (result.status === 'fulfilled') + await MCPService.disconnect(result.value.connection).catch(console.warn); + } + + return false; } - return extras; + for (const result of results) { + if (result.status === 'fulfilled') { + const { connection, name } = result.value; + + this.connections.set(name, connection); + + this.indexServerTools(name, connection.tools); + } else { + console.error(`[MCPStore] Failed to connect:`, result.reason); + } + } + + const successCount = this.connections.size; + + if (successCount === 0 && serverEntries.length > 0) { + this.updateState({ + connectedServers: [], + error: 'All MCP server connections failed', + isInitializing: false, + toolCount: 0 + }); + this.initPromise = null; + + return false; + } + + this.updateState({ + connectedServers: Array.from(this.connections.keys()), + error: null, + isInitializing: false, + toolCount: this.toolsIndex.size + }); + this.initPromise = null; + + return true; + } + + /** + * Generates a unique server ID from an optional ID string or index. + */ + private generateServerId(id: unknown, index: number): string { + if (typeof id === 'string' && id.trim()) { + return id.trim(); + } + + return `${MCP_SERVER_ID_PREFIX}-${index + 1}`; + } + + private handleToolsListChanged(serverName: string, tools: Tool[]): void { + const connection = this.connections.get(serverName); + + if (!connection) { + return; + } + + for (const [toolName, ownerServer] of this.toolsIndex.entries()) { + if (ownerServer === serverName) this.toolsIndex.delete(toolName); + } + + connection.tools = tools; + + for (const tool of tools) { + if (this.toolsIndex.has(tool.name)) + console.warn( + `[MCPStore] Tool name conflict after list change: "${tool.name}" exists in "${this.toolsIndex.get(tool.name)}" and "${serverName}". Using tool from "${serverName}".` + ); + + this.toolsIndex.set(tool.name, serverName); + } + this.updateState({ toolCount: this.toolsIndex.size }); + } + + /** + * Registers the tools exposed by a server into the global name->server index, + * warning on conflicts. Shared by connect, reconnect and auto-reconnect. + */ + private indexServerTools(serverName: string, tools: Tool[]): void { + for (const tool of tools) { + if (this.toolsIndex.has(tool.name)) + console.warn( + `[MCPStore] Tool name conflict: "${tool.name}" exists in "${this.toolsIndex.get(tool.name)}" and "${serverName}". Using tool from "${serverName}".` + ); + + this.toolsIndex.set(tool.name, serverName); + } + } + + private async initialize(signature: string, mcpConfig: MCPClientConfig): Promise { + this.updateState({ error: null, isInitializing: true }); + this.configSignature = signature; + + const serverEntries = Object.entries(mcpConfig.servers); + + if (serverEntries.length === 0) { + this.updateState({ connectedServers: [], isInitializing: false, toolCount: 0 }); + + return false; + } + + this.initPromise = this.doInitialize(signature, mcpConfig, serverEntries); + + return this.initPromise; + } + + private parseToolArguments(args: string | Record): Record { + if (typeof args === 'string') { + const trimmed = args.trim(); + + if (trimmed === '') { + return {}; + } + + try { + const parsed = JSON.parse(trimmed); + + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) + throw new Error( + `Tool arguments must be an object, got ${Array.isArray(parsed) ? 'array' : typeof parsed}` + ); + + return parsed as Record; + } catch (error) { + throw new Error(`Failed to parse tool arguments as JSON: ${(error as Error).message}`); + } + } + + if (typeof args === 'object' && args !== null && !Array.isArray(args)) { + return args; + } + + throw new Error(`Invalid tool arguments type: ${typeof args}`); + } + + /** + * Immediately reconnect to a server by creating a fresh transport and session. + * Used when a session-expired error (HTTP 404) is detected during tool execution. + * Per MCP spec 2025-11-25: client MUST discard session ID and re-initialize. + * + * Unlike autoReconnect (which uses exponential backoff for connectivity issues), + * this performs a single immediate reconnection attempt since the server is known + * to be reachable (it responded with 404). + */ + private async reconnectServer(serverName: string): Promise { + const serverConfig = this.serverConfigs.get(serverName); + + if (!serverConfig) { + throw new Error(`[MCPStore] No config found for ${serverName}, cannot reconnect`); + } + + // Disconnect stale connection (clears old transport + session ID) + const oldConnection = this.connections.get(serverName); + + if (oldConnection) { + await MCPService.disconnect(oldConnection).catch(console.warn); + this.connections.delete(serverName); + } + + console.log(`[MCPStore][${serverName}] Session expired, reconnecting with fresh session...`); + + const listChangedHandlers = this.createListChangedHandlers(serverName); + const connection = await MCPService.connect( + serverName, + serverConfig, + DEFAULT_MCP_CONFIG.clientInfo, + DEFAULT_MCP_CONFIG.capabilities, + (phase) => { + if (phase === MCPConnectionPhase.DISCONNECTED) { + console.log(`[MCPStore][${serverName}] Connection lost, starting auto-reconnect`); + this.autoReconnect(serverName); + } + }, + listChangedHandlers + ); + + this.connections.set(serverName, connection); + this.indexServerTools(serverName, connection.tools); + + console.log(`[MCPStore][${serverName}] Session recovered successfully`); + } + + private updateState(state: { + isInitializing?: boolean; + error?: string | null; + toolCount?: number; + connectedServers?: string[]; + }): void { + if (state.isInitializing !== undefined) { + this._isInitializing = state.isInitializing; + } + + if (state.error !== undefined) { + this._error = state.error; + } + + if (state.toolCount !== undefined) { + this._toolCount = state.toolCount; + } + + if (state.connectedServers !== undefined) { + this.connectedServers = state.connectedServers; + } } } diff --git a/tools/ui/src/lib/stores/mcp-resources.svelte.ts b/tools/ui/src/lib/stores/mcp/resources.svelte.ts similarity index 96% rename from tools/ui/src/lib/stores/mcp-resources.svelte.ts rename to tools/ui/src/lib/stores/mcp/resources.svelte.ts index b68def89f..79ff2c209 100644 --- a/tools/ui/src/lib/stores/mcp-resources.svelte.ts +++ b/tools/ui/src/lib/stores/mcp/resources.svelte.ts @@ -38,32 +38,40 @@ function generateAttachmentId(): string { } class MCPResourceStore { - private _serverResources = $state>(new SvelteMap()); - private _cachedResources = $state>(new SvelteMap()); - private _subscriptions = $state>(new SvelteMap()); private _attachments = $state([]); + private _cachedResources = $state>(new SvelteMap()); private _isLoading = $state(false); + private _serverResources = $state>(new SvelteMap()); + private _subscriptions = $state>(new SvelteMap()); - get serverResources(): Map { - return this._serverResources; - } - - get cachedResources(): Map { - return this._cachedResources; - } - - get subscriptions(): Map { - return this._subscriptions; + get attachmentCount(): number { + return this._attachments.length; } get attachments(): MCPResourceAttachment[] { return this._attachments; } + get cachedResources(): Map { + return this._cachedResources; + } + + get hasAttachments(): boolean { + return this._attachments.length > 0; + } + get isLoading(): boolean { return this._isLoading; } + get serverResources(): Map { + return this._serverResources; + } + + get subscriptions(): Map { + return this._subscriptions; + } + get totalResourceCount(): number { let count = 0; @@ -84,86 +92,183 @@ class MCPResourceStore { return count; } - get attachmentCount(): number { - return this._attachments.length; - } + /** + * Add a resource attachment to the current chat context + */ + addAttachment(resource: MCPResourceInfo): MCPResourceAttachment { + const attachment: MCPResourceAttachment = { + id: generateAttachmentId(), + loading: true, + resource + }; - get hasAttachments(): boolean { - return this._attachments.length > 0; + this._attachments = [...this._attachments, attachment]; + console.log(`[MCPResources] Added attachment: ${resource.uri}`); + + return attachment; } /** - * - * - * Server Resources Management - * - * + * Register a subscription for a resource */ - - /** - * Set resources for a server (called after listResources) - */ - setServerResources( - serverName: string, - resources: MCPResource[], - templates: MCPResourceTemplate[] - ): void { - this._serverResources.set(serverName, { - error: undefined, - lastFetched: new Date(), - loading: false, - resources, + addSubscription(uri: string, serverName: string): void { + this._subscriptions.set(uri, { serverName, - templates + subscribedAt: new Date(), + uri }); - console.log( - `[MCPResources][${serverName}] Set ${resources.length} resources, ${templates.length} templates` - ); - } - /** - * Set loading state for a server's resources - */ - setServerLoading(serverName: string, loading: boolean): void { - const existing = this._serverResources.get(serverName); + const cached = this._cachedResources.get(uri); - if (existing) { - this._serverResources.set(serverName, { ...existing, loading }); - } else { - this._serverResources.set(serverName, { - error: undefined, - loading, - resources: [], - serverName, - templates: [] - }); + if (cached) { + this._cachedResources.set(uri, { ...cached, subscribed: true }); } + + console.log(`[MCPResources] Added subscription: ${uri}`); } /** - * Set error state for a server's resources + * Cache resource content after reading */ - setServerError(serverName: string, error: string): void { - const existing = this._serverResources.get(serverName); + cacheResourceContent(resource: MCPResourceInfo, content: MCPResourceContent[]): void { + // Enforce cache size limit + if (this._cachedResources.size >= MCP_RESOURCE_CACHE.MAX_ENTRIES) { + const oldestKey = this._cachedResources.keys().next().value; - if (existing) { - this._serverResources.set(serverName, { ...existing, error, loading: false }); - } else { - this._serverResources.set(serverName, { - error, - loading: false, - resources: [], - serverName, - templates: [] - }); + if (oldestKey) { + this._cachedResources.delete(oldestKey); + } } + + this._cachedResources.set(resource.uri, { + content, + fetchedAt: new Date(), + resource, + subscribed: this._subscriptions.has(resource.uri) + }); + console.log(`[MCPResources] Cached content for: ${resource.uri}`); } /** - * Get resources for a specific server + * Clear all state (e.g., on full reset) */ - getServerResources(serverName: string): MCPServerResources | undefined { - return this._serverResources.get(serverName); + clear(): void { + this._serverResources.clear(); + this._cachedResources.clear(); + this._subscriptions.clear(); + this._attachments = []; + this._isLoading = false; + console.log(`[MCPResources] Cleared all state`); + } + + /** + * Clear all attachments + */ + clearAttachments(): void { + this._attachments = []; + console.log(`[MCPResources] Cleared all attachments`); + } + + /** + * Clear all cached content + */ + clearCache(): void { + this._cachedResources.clear(); + console.log(`[MCPResources] Cleared all cached content`); + } + + /** + * Clear resources for a server (e.g., when disconnected) + */ + clearServerResources(serverName: string): void { + this._serverResources.delete(serverName); + + for (const [uri, cached] of this._cachedResources) { + if (cached.resource.serverName === serverName) { + this._cachedResources.delete(uri); + } + } + + for (const [uri, sub] of this._subscriptions) { + if (sub.serverName === serverName) { + this._subscriptions.delete(uri); + } + } + + console.log(`[MCPResources][${serverName}] Cleared all resources`); + } + + /** + * Find resource info by URI across all servers + */ + findResourceByUri(uri: string): MCPResourceInfo | undefined { + const normalizedUri = normalizeResourceUri(uri); + + for (const [serverName, serverRes] of this._serverResources) { + const resource = + serverRes.resources.find((r) => r.uri === uri) ?? + serverRes.resources.find((r) => normalizeResourceUri(r.uri) === normalizedUri); + + if (resource) { + return { + annotations: resource.annotations, + description: resource.description, + icons: resource.icons, + mimeType: resource.mimeType, + name: resource.name, + serverName, + title: resource.title, + uri: resource.uri + }; + } + } + + return undefined; + } + + /** + * Find server name for a resource URI + */ + findServerForUri(uri: string): string | undefined { + for (const [serverName, serverRes] of this._serverResources) { + if (serverRes.resources.some((r) => r.uri === uri)) { + return serverName; + } + } + + return undefined; + } + + /** + * Get resource content as text for chat context + * Formats content for inclusion in LLM prompts + */ + formatAttachmentsForContext(): string { + if (this._attachments.length === 0) return ''; + + const parts: string[] = []; + + for (const attachment of this._attachments) { + if (attachment.error) continue; + + if (!attachment.content || attachment.content.length === 0) continue; + + const resourceName = attachment.resource.title || attachment.resource.name; + const serverName = attachment.resource.serverName; + + for (const content of attachment.content) { + if ('text' in content && content.text) { + parts.push(`\n\n--- Resource: ${resourceName} (from ${serverName}) ---\n${content.text}`); + } else if ('blob' in content && content.blob) { + // For binary content, just note it exists + parts.push( + `\n\n--- Resource: ${resourceName} (from ${serverName}) ---\n[${BINARY_CONTENT_LABEL}: ${content.mimeType || RESOURCE_UNKNOWN_TYPE}]` + ); + } + } + } + + return parts.join(''); } /** @@ -215,57 +320,10 @@ class MCPResourceStore { } /** - * Clear resources for a server (e.g., when disconnected) + * Get attachment by ID */ - clearServerResources(serverName: string): void { - this._serverResources.delete(serverName); - - // Also clear cached content for this server's resources - for (const [uri, cached] of this._cachedResources) { - if (cached.resource.serverName === serverName) { - this._cachedResources.delete(uri); - } - } - - // Clear subscriptions for this server - for (const [uri, sub] of this._subscriptions) { - if (sub.serverName === serverName) { - this._subscriptions.delete(uri); - } - } - - console.log(`[MCPResources][${serverName}] Cleared all resources`); - } - - /** - * - * - * Resource Content Caching - * - * - */ - - /** - * Cache resource content after reading - */ - cacheResourceContent(resource: MCPResourceInfo, content: MCPResourceContent[]): void { - // Enforce cache size limit - if (this._cachedResources.size >= MCP_RESOURCE_CACHE.MAX_ENTRIES) { - // Remove oldest entry - const oldestKey = this._cachedResources.keys().next().value; - - if (oldestKey) { - this._cachedResources.delete(oldestKey); - } - } - - this._cachedResources.set(resource.uri, { - content, - fetchedAt: new Date(), - resource, - subscribed: this._subscriptions.has(resource.uri) - }); - console.log(`[MCPResources] Cached content for: ${resource.uri}`); + getAttachment(attachmentId: string): MCPResourceAttachment | undefined { + return this._attachments.find((att) => att.id === attachmentId); } /** @@ -276,7 +334,6 @@ class MCPResourceStore { if (!cached) return undefined; - // Check if cache is still valid const age = Date.now() - cached.fetchedAt.getTime(); if (age > MCP_RESOURCE_CACHE.TTL_MS && !cached.subscribed) { @@ -290,100 +347,22 @@ class MCPResourceStore { } /** - * Invalidate cached content for a resource (e.g., on update notification) + * Get resources for a specific server */ - invalidateCache(uri: string): void { - this._cachedResources.delete(uri); - console.log(`[MCPResources] Invalidated cache for: ${uri}`); - } - - /** - * Clear all cached content - */ - clearCache(): void { - this._cachedResources.clear(); - console.log(`[MCPResources] Cleared all cached content`); - } - - /** - * - * - * Subscriptions - * - * - */ - - /** - * Register a subscription for a resource - */ - addSubscription(uri: string, serverName: string): void { - this._subscriptions.set(uri, { - serverName, - subscribedAt: new Date(), - uri - }); - - // Update cached resource if exists - const cached = this._cachedResources.get(uri); - - if (cached) { - this._cachedResources.set(uri, { ...cached, subscribed: true }); - } - - console.log(`[MCPResources] Added subscription: ${uri}`); - } - - /** - * Remove a subscription for a resource - */ - removeSubscription(uri: string): void { - this._subscriptions.delete(uri); - - // Update cached resource if exists - const cached = this._cachedResources.get(uri); - - if (cached) { - this._cachedResources.set(uri, { ...cached, subscribed: false }); - } - - console.log(`[MCPResources] Removed subscription: ${uri}`); - } - - /** - * Check if a resource is subscribed - */ - isSubscribed(uri: string): boolean { - return this._subscriptions.has(uri); - } - - /** - * Handle resource update notification - */ - handleResourceUpdate(uri: string): void { - // Invalidate cache so next read gets fresh content - this.invalidateCache(uri); - - // Update subscription last update time - const sub = this._subscriptions.get(uri); - - if (sub) { - this._subscriptions.set(uri, { ...sub, lastUpdate: new Date() }); - } - - console.log(`[MCPResources] Resource updated: ${uri}`); + getServerResources(serverName: string): MCPServerResources | undefined { + return this._serverResources.get(serverName); } /** * Handle resources list changed notification */ handleResourcesListChanged(serverName: string): void { - // Mark server resources as needing refresh const existing = this._serverResources.get(serverName); if (existing) { this._serverResources.set(serverName, { ...existing, - lastFetched: undefined // Mark as stale + lastFetched: undefined }); } @@ -399,60 +378,27 @@ class MCPResourceStore { */ /** - * Add a resource attachment to the current chat context + * Handle resource update notification */ - addAttachment(resource: MCPResourceInfo): MCPResourceAttachment { - const attachment: MCPResourceAttachment = { - id: generateAttachmentId(), - loading: true, - resource - }; + handleResourceUpdate(uri: string): void { + // Invalidate cache so next read gets fresh content + this.invalidateCache(uri); - this._attachments = [...this._attachments, attachment]; - console.log(`[MCPResources] Added attachment: ${resource.uri}`); + const sub = this._subscriptions.get(uri); - return attachment; + if (sub) { + this._subscriptions.set(uri, { ...sub, lastUpdate: new Date() }); + } + + console.log(`[MCPResources] Resource updated: ${uri}`); } /** - * Update attachment with fetched content + * Invalidate cached content for a resource (e.g., on update notification) */ - updateAttachmentContent(attachmentId: string, content: MCPResourceContent[]): void { - this._attachments = this._attachments.map((att) => - att.id === attachmentId ? { ...att, content, error: undefined, loading: false } : att - ); - } - - /** - * Update attachment with error - */ - updateAttachmentError(attachmentId: string, error: string): void { - this._attachments = this._attachments.map((att) => - att.id === attachmentId ? { ...att, error, loading: false } : att - ); - } - - /** - * Remove an attachment - */ - removeAttachment(attachmentId: string): void { - this._attachments = this._attachments.filter((att) => att.id !== attachmentId); - console.log(`[MCPResources] Removed attachment: ${attachmentId}`); - } - - /** - * Clear all attachments - */ - clearAttachments(): void { - this._attachments = []; - console.log(`[MCPResources] Cleared all attachments`); - } - - /** - * Get attachment by ID - */ - getAttachment(attachmentId: string): MCPResourceAttachment | undefined { - return this._attachments.find((att) => att.id === attachmentId); + invalidateCache(uri: string): void { + this._cachedResources.delete(uri); + console.log(`[MCPResources] Invalidated cache for: ${uri}`); } /** @@ -467,12 +413,34 @@ class MCPResourceStore { } /** - * - * - * Utility Methods - * - * + * Check if a resource is subscribed */ + isSubscribed(uri: string): boolean { + return this._subscriptions.has(uri); + } + + /** + * Remove an attachment + */ + removeAttachment(attachmentId: string): void { + this._attachments = this._attachments.filter((att) => att.id !== attachmentId); + console.log(`[MCPResources] Removed attachment: ${attachmentId}`); + } + + /** + * Remove a subscription for a resource + */ + removeSubscription(uri: string): void { + this._subscriptions.delete(uri); + + const cached = this._cachedResources.get(uri); + + if (cached) { + this._cachedResources.set(uri, { ...cached, subscribed: false }); + } + + console.log(`[MCPResources] Removed subscription: ${uri}`); + } /** * Set global loading state @@ -482,88 +450,62 @@ class MCPResourceStore { } /** - * Find resource info by URI across all servers + * Set error state for a server's resources */ - findResourceByUri(uri: string): MCPResourceInfo | undefined { - const normalizedUri = normalizeResourceUri(uri); + setServerError(serverName: string, error: string): void { + const existing = this._serverResources.get(serverName); - for (const [serverName, serverRes] of this._serverResources) { - const resource = - serverRes.resources.find((r) => r.uri === uri) ?? - serverRes.resources.find((r) => normalizeResourceUri(r.uri) === normalizedUri); - - if (resource) { - return { - annotations: resource.annotations, - description: resource.description, - icons: resource.icons, - mimeType: resource.mimeType, - name: resource.name, - serverName, - title: resource.title, - uri: resource.uri - }; - } + if (existing) { + this._serverResources.set(serverName, { ...existing, error, loading: false }); + } else { + this._serverResources.set(serverName, { + error, + loading: false, + resources: [], + serverName, + templates: [] + }); } - - return undefined; } /** - * Find server name for a resource URI + * Set loading state for a server's resources */ - findServerForUri(uri: string): string | undefined { - for (const [serverName, serverRes] of this._serverResources) { - if (serverRes.resources.some((r) => r.uri === uri)) { - return serverName; - } - } + setServerLoading(serverName: string, loading: boolean): void { + const existing = this._serverResources.get(serverName); - return undefined; + if (existing) { + this._serverResources.set(serverName, { ...existing, loading }); + } else { + this._serverResources.set(serverName, { + error: undefined, + loading, + resources: [], + serverName, + templates: [] + }); + } } /** - * Clear all state (e.g., on full reset) + * Set resources for a server (called after listResources) */ - clear(): void { - this._serverResources.clear(); - this._cachedResources.clear(); - this._subscriptions.clear(); - this._attachments = []; - this._isLoading = false; - console.log(`[MCPResources] Cleared all state`); - } - - /** - * Get resource content as text for chat context - * Formats content for inclusion in LLM prompts - */ - formatAttachmentsForContext(): string { - if (this._attachments.length === 0) return ''; - - const parts: string[] = []; - - for (const attachment of this._attachments) { - if (attachment.error) continue; - - if (!attachment.content || attachment.content.length === 0) continue; - - const resourceName = attachment.resource.title || attachment.resource.name; - const serverName = attachment.resource.serverName; - - for (const content of attachment.content) { - if ('text' in content && content.text) { - parts.push(`\n\n--- Resource: ${resourceName} (from ${serverName}) ---\n${content.text}`); - } else if ('blob' in content && content.blob) { - // For binary content, just note it exists - parts.push( - `\n\n--- Resource: ${resourceName} (from ${serverName}) ---\n[${BINARY_CONTENT_LABEL}: ${content.mimeType || RESOURCE_UNKNOWN_TYPE}]` - ); - } - } - } - - return parts.join(''); + setServerResources( + serverName: string, + resources: MCPResource[], + templates: MCPResourceTemplate[] + ): void { + this._serverResources.set(serverName, { + error: undefined, + lastFetched: new Date(), + loading: false, + resources, + serverName, + templates + }); + console.log( + `[MCPResources][${serverName}] Set ${resources.length} resources, ${templates.length} templates` + ); } /** @@ -605,6 +547,24 @@ class MCPResourceStore { return extras; } + + /** + * Update attachment with fetched content + */ + updateAttachmentContent(attachmentId: string, content: MCPResourceContent[]): void { + this._attachments = this._attachments.map((att) => + att.id === attachmentId ? { ...att, content, error: undefined, loading: false } : att + ); + } + + /** + * Update attachment with error + */ + updateAttachmentError(attachmentId: string, error: string): void { + this._attachments = this._attachments.map((att) => + att.id === attachmentId ? { ...att, error, loading: false } : att + ); + } } export const mcpResourceStore = new MCPResourceStore(); diff --git a/tools/ui/src/lib/stores/models.svelte.ts b/tools/ui/src/lib/stores/models.svelte.ts deleted file mode 100644 index c741d144c..000000000 --- a/tools/ui/src/lib/stores/models.svelte.ts +++ /dev/null @@ -1,1077 +0,0 @@ -import { FAVORITE_MODELS_LOCALSTORAGE_KEY, MODEL_PROPS_CACHE } from '$lib/constants'; -import { - FileTypeCategory, - ModelModality, - ServerModelsSseEventType, - ServerModelStatus -} from '$lib/enums'; -import { ModelsService } from '$lib/services/models.service'; -import { PropsService } from '$lib/services/props.service'; -// direct imports between stores, not via the barrel, to avoid circular deps -import { conversationsStore } from '$lib/stores/conversations.svelte'; -import { serverStore } from '$lib/stores/server.svelte'; -// deep imports, not the '$lib/utils' barrel: it re-exports modules that reach back -// into the stores, and going through it here would read a half-built module -import { TTLCache } from '$lib/utils/cache-ttl'; -import { - detectThinkingSupport, - detectThinkingSupportWithReason -} from '$lib/utils/chat-template-thinking-detector'; -import { getConversationModel } from '$lib/utils/conversation-utils'; -import { SvelteMap, SvelteSet } from 'svelte/reactivity'; -import { toast } from 'svelte-sonner'; - -/** - * modelsStore - Reactive store for model management in both MODEL and ROUTER modes. - * - * **Architecture & Relationships:** - * - **ModelsService**: Stateless service for model API communication - * - **PropsService**: Stateless service for props/modalities fetching - * - **modelsStore** (this class): Reactive store for model state - * - **conversationsStore**: Tracks which conversations use which models - * - * **API Inconsistency Workaround:** - * In MODEL mode, `/props` returns modalities for the single model. - * In ROUTER mode, `/props` has no modalities — must use `/props?model=` per model. - * This store normalizes this behavior so consumers don't need to know the server mode. - */ -class ModelsStore { - /** - * - * - * State - * - * - */ - - models = $state([]); - routerModels = $state([]); - loading = $state(false); - updating = $state(false); - error = $state(null); - selectedModelId = $state(null); - selectedModelName = $state(null); - - // Dedup concurrent fetch() callers — all awaiters share the same inflight promise. - // Without this, ?model= URL handler races an in-progress fetch and sees an empty list. - private inflightFetch: Promise | null = null; - - private modelUsage = $state>>(new Map()); - private modelLoadingStates = new SvelteMap(); - - // /models/sse feed state, the single source of truth for status and load progress - private statusAbort: AbortController | null = null; - private statusReaderActive = false; - private loadProgress = new SvelteMap(); - private statusWaiters = new Map< - string, - { target: ServerModelStatus; resolve: () => void; reject: (e: Error) => void } - >(); - - favoriteModelIds = $state>(this.loadFavoritesFromStorage()); - - /** - * Model-specific props cache with TTL. - * Key: modelId, Value: props data including modalities. - * TTL: 10 minutes — props don't change frequently. - */ - private modelPropsCache = new TTLCache({ - maxEntries: MODEL_PROPS_CACHE.MAX_ENTRIES, - ttlMs: MODEL_PROPS_CACHE.TTL_MS - }); - private modelPropsFetching = $state>(new Set()); - - /** - * Version counter for props cache — used to trigger reactivity when props are updated. - */ - propsCacheVersion = $state(0); - - /** - * - * - * Computed Getters - * - * - */ - - get selectedModel(): ModelOption | null { - if (!this.selectedModelId) return null; - - return this.models.find((m) => m.id === this.selectedModelId) ?? null; - } - - get loadedModelIds(): string[] { - return this.routerModels - .filter( - (m) => - m.status.value === ServerModelStatus.LOADED || - m.status.value === ServerModelStatus.SLEEPING - ) - .map((m) => m.id); - } - - get loadingModelIds(): string[] { - return Array.from(this.modelLoadingStates.entries()) - .filter(([, loading]) => loading) - .map(([id]) => id); - } - - /** - * Get model name in MODEL mode (single model). - * Extracts from model_path or model_alias from server props. - * In ROUTER mode, returns null (model is per-conversation). - */ - get singleModelName(): string | null { - if (serverStore.isRouterMode) return null; - - const props = serverStore.props; - - if (props?.model_alias) return props.model_alias; - - if (!props?.model_path) return null; - - return props.model_path.split(/(\\|\/)/).pop() || null; - } - - /** - * Model the active conversation view resolves to. Router mode: the user's - * selection first, then the conversation's own model. Otherwise the single - * served model, from the models list or the server props as a fallback. - */ - get activeModelId(): string | null { - if (!serverStore.isRouterMode) { - return this.models.length > 0 ? this.models[0].model : this.singleModelName; - } - - if (this.selectedModelId) { - const selected = this.models.find((m) => m.id === this.selectedModelId); - - if (selected) return selected.model; - } - - const conversationModel = getConversationModel(conversationsStore.activeMessages); - - if (conversationModel) { - const model = this.models.find((m) => m.model === conversationModel); - - if (model) return model.model; - } - - return null; - } - - get selectedModelContextSize(): number | null { - if (!this.selectedModelName) return null; - - return this.getModelContextSize(this.selectedModelName); - } - - /** - * - * - * Modalities - * - * - */ - - getModelModalities(modelId: string): ModelModalities | null { - if (!serverStore.isRouterMode && serverStore.props?.modalities) { - return this.buildModalities(serverStore.props.modalities); - } - - const model = this.models.find((m) => m.model === modelId || m.id === modelId); - - if (model?.modalities) { - return model.modalities; - } - - const props = this.modelPropsCache.get(modelId); - - if (props?.modalities) { - return this.buildModalities(props.modalities); - } - - return null; - } - - modelSupportsVision(modelId: string): boolean { - return this.getModelModalities(modelId)?.vision ?? false; - } - - modelSupportsAudio(modelId: string): boolean { - return this.getModelModalities(modelId)?.audio ?? false; - } - - modelSupportsVideo(modelId: string): boolean { - return this.getModelModalities(modelId)?.video ?? false; - } - - getModelModalitiesArray(modelId: string): ModelModality[] { - const modalities = this.getModelModalities(modelId); - - if (!modalities) return []; - - const result: ModelModality[] = []; - - if (modalities.vision) result.push(ModelModality.VISION); - - if (modalities.audio) result.push(ModelModality.AUDIO); - - if (modalities.video) result.push(ModelModality.VIDEO); - - return result; - } - - getModelProps(modelId: string): ApiLlamaCppServerProps | null { - return this.modelPropsCache.get(modelId); - } - - getModelContextSize(modelId: string): number | null { - const props = this.getModelProps(modelId); - const nCtx = props?.default_generation_settings?.n_ctx; - - return typeof nCtx === 'number' ? nCtx : null; - } - - isModelPropsFetching(modelId: string): boolean { - return this.modelPropsFetching.has(modelId); - } - - /** - * - * - * Status Queries - * - * - */ - - isModelLoaded(modelId: string): boolean { - const model = this.routerModels.find((m) => m.id === modelId); - - return ( - model?.status.value === ServerModelStatus.LOADED || - model?.status.value === ServerModelStatus.SLEEPING - ); - } - - isModelOperationInProgress(modelId: string): boolean { - return this.modelLoadingStates.get(modelId) ?? false; - } - - getModelStatus(modelId: string): ServerModelStatus | null { - const model = this.routerModels.find((m) => m.id === modelId); - - return model?.status.value ?? null; - } - - getModelUsage(modelId: string): SvelteSet { - return this.modelUsage.get(modelId) ?? new SvelteSet(); - } - - isModelInUse(modelId: string): boolean { - const usage = this.modelUsage.get(modelId); - - return usage !== undefined && usage.size > 0; - } - // - // Thinking Support Detection - // - - /** - * Whether the selected model's chat template supports thinking/reasoning. - * Uses heuristic detection on the model's chat_template from /props. - * - * - MODEL mode: the global /props already describes the single loaded model, - * so its chat_template is used directly and no per-model cache is involved - * - ROUTER mode: fetches /props?model= for the selected model (cached), - * triggering an async fetch if not yet cached - */ - get supportsThinking(): boolean { - if (!serverStore.isRouterMode) { - return detectThinkingSupport(serverStore.props?.chat_template ?? ''); - } - - const modelId = this.selectedModelName; - - if (!modelId) return false; - - if (!this.modelPropsCache.get(modelId)) { - this.fetchModelProps(modelId); - } - - const props = this.getModelProps(modelId); - - return detectThinkingSupport(props?.chat_template ?? ''); - } - - /** - * Check if a specific model supports thinking. - * In MODEL mode the global /props describes the single loaded model. - * In ROUTER mode, fetches model props if not cached. - */ - checkModelSupportsThinking(modelId: string): boolean { - if (!serverStore.isRouterMode) { - return detectThinkingSupport(serverStore.props?.chat_template ?? ''); - } - - if (!modelId) return false; - - if (!this.modelPropsCache.get(modelId)) { - this.fetchModelProps(modelId); - } - - const props = this.getModelProps(modelId); - - return detectThinkingSupport(props?.chat_template ?? ''); - } - - /** - * Detailed thinking support detection result with reason for debugging/UI. - */ - get thinkingSupportDetails(): { supported: boolean; reason: string } { - if (!serverStore.isRouterMode) { - return detectThinkingSupportWithReason(serverStore.props?.chat_template ?? ''); - } - - const modelId = this.selectedModelName; - - if (!modelId) { - return { reason: 'No model selected', supported: false }; - } - - if (!this.modelPropsCache.get(modelId)) { - this.fetchModelProps(modelId); - } - - const props = this.getModelProps(modelId); - - return detectThinkingSupportWithReason(props?.chat_template ?? ''); - } - - /** - * - * - * Data Fetching - * - * - */ - - /** - * Fetch list of models from server and detect server role. - * Also fetches modalities for MODEL mode (single model). - */ - async fetch(force = false): Promise { - if (this.inflightFetch) return this.inflightFetch; - - if (this.models.length > 0 && !force) return; - - this.inflightFetch = this.runFetch(); - try { - await this.inflightFetch; - } finally { - this.inflightFetch = null; - } - } - - private async runFetch(): Promise { - this.loading = true; - this.error = null; - - try { - if (!serverStore.props) { - await serverStore.fetch(); - } - - const router = serverStore.isRouterMode; - - if (router) { - const response = await ModelsService.listRouter(); - - this.routerModels = response.data; - this.models = this.buildModelOptions(response); - - await this.fetchModalitiesForLoadedModels(); - - const visible = this.getVisibleModels(); - - if (visible.length === 1 && this.isModelLoaded(visible[0].model)) { - this.selectModelById(visible[0].id); - } - } else { - this.models = await this.fetchModelModeInternal(); - } - } catch (error) { - this.models = []; - this.error = error instanceof Error ? error.message : 'Failed to load models'; - - throw error; - } finally { - this.loading = false; - } - } - - /** Fetch models in MODEL mode (single model, standard OpenAI-compatible). */ - private async fetchModelModeInternal(): Promise { - const response = await ModelsService.list(); - - return this.buildModelOptions(response); - } - - /** - * Build ModelOption[] from an API response. - * Both MODEL and ROUTER modes share the same mapping logic; - * they differ only in which endpoint is called. - */ - private buildModelOptions( - response: ApiModelListResponse | ApiRouterModelsListResponse - ): ModelOption[] { - return response.data.map((item: ApiModelDataEntry, index: number) => { - const details = response.models?.[index]; - const rawCapabilities = Array.isArray(details?.capabilities) ? details?.capabilities : []; - const displayNameSource = - details?.name && details.name.trim().length > 0 ? details.name : item.id; - const modelId = details?.model || item.id; - - return { - aliases: item.aliases ?? [], - capabilities: rawCapabilities.filter((value: unknown): value is string => Boolean(value)), - description: details?.description, - details: details?.details, - id: item.id, - meta: item.meta ?? null, - modalities: this.buildArchitectureModalities(item.architecture), - model: modelId, - name: this.toDisplayName(displayNameSource), - parsedId: ModelsService.parseModelId(modelId), - tags: item.tags ?? [] - }; - }); - } - - /** - * Fetch router models with full metadata (ROUTER mode only). - * No-op in router mode — fetch() already calls listRouter() internally. - * Kept for API compatibility (e.g. handleOpenChange dropdown open handler). - */ - async fetchRouterModels(): Promise { - if (!serverStore.isRouterMode) return; - - try { - const response = await ModelsService.listRouter(); - - this.routerModels = response.data; - await this.fetchModalitiesForLoadedModels(); - - const visible = this.getVisibleModels(); - - if (visible.length === 1 && this.isModelLoaded(visible[0].model)) { - this.selectModelById(visible[0].id); - } - } catch (error) { - console.warn('Failed to fetch router models:', error); - this.routerModels = []; - } - } - - /** - * Fetch props for a specific model from /props endpoint. - * Uses caching to avoid redundant requests. - * - * In ROUTER mode, this only fetches props if the model is loaded, - * since unloaded models return 400 from /props endpoint. - * - * @param modelId - Model identifier to fetch props for - * @returns Props data or null if fetch failed or model not loaded - */ - async fetchModelProps(modelId: string): Promise { - const cached = this.modelPropsCache.get(modelId); - - if (cached) return cached; - - if (serverStore.isRouterMode && !this.isModelLoaded(modelId)) { - return null; - } - - if (this.modelPropsFetching.has(modelId)) return null; - - this.modelPropsFetching.add(modelId); - - try { - const props = await PropsService.fetchForModel(modelId); - - this.modelPropsCache.set(modelId, props); - this.propsCacheVersion++; - - return props; - } catch (error) { - console.warn(`Failed to fetch props for model ${modelId}:`, error); - - return null; - } finally { - this.modelPropsFetching.delete(modelId); - } - } - - /** Fetch modalities for all loaded models from /props endpoint. */ - async fetchModalitiesForLoadedModels(): Promise { - const loadedModelIds = this.loadedModelIds; - - if (loadedModelIds.length === 0) return; - - const propsPromises = loadedModelIds.map((modelId) => this.fetchModelProps(modelId)); - - try { - const results = await Promise.all(propsPromises); - - this.models = this.models.map((model) => { - const modelIndex = loadedModelIds.indexOf(model.model); - - if (modelIndex === -1) return model; - - const props = results[modelIndex]; - - if (!props?.modalities) return model; - - return { ...model, modalities: this.buildModalities(props.modalities) }; - }); - - this.propsCacheVersion++; - } catch (error) { - console.warn('Failed to fetch modalities for loaded models:', error); - } - } - - /** - * Update modalities for a specific model. - * Called when a model is loaded or when we need fresh modality data. - */ - async updateModelModalities(modelId: string): Promise { - const props = await this.fetchModelProps(modelId); - - if (!props?.modalities) return; - - this.models = this.models.map((model) => - model.model === modelId - ? { ...model, modalities: this.buildModalities(props.modalities!) } - : model - ); - - this.propsCacheVersion++; - } - - /** - * Filter to models visible in the UI (ui !== false). - */ - private getVisibleModels(): ModelOption[] { - return this.models.filter((option) => this.getModelProps(option.model)?.ui !== false); - } - - /** - * Gets the model name from the last assistant message in the active conversation. - * Used by both the chat page and settings page to maintain model consistency. - */ - getModelFromLastAssistantResponse(): string | null { - const messages = conversationsStore.activeMessages; - - if (!messages || messages.length === 0) return null; - - for (let i = messages.length - 1; i >= 0; i--) { - if (messages[i].model) { - return messages[i].model; - } - } - - return null; - } - - /** - * Auto-selects the model from the last assistant response if available and loaded. - * Returns true if a model was selected, false otherwise. - */ - async selectModelFromLastAssistantResponse(): Promise { - const lastModel = this.getModelFromLastAssistantResponse(); - - if (!lastModel || this.selectedModelName === lastModel) return false; - - const matchingModel = this.models.find((option) => option.model === lastModel); - - if (!matchingModel || !this.isModelLoaded(lastModel)) return false; - - try { - await this.selectModelById(matchingModel.id); - console.log(`[modelsStore] Automatically selected model: ${lastModel} from last message`); - - return true; - } catch (error) { - console.warn('[modelsStore] Failed to automatically select model from last message:', error); - - return false; - } - } - - /** - * Auto-selects the first available model if none is selected. - * Prioritizes: - * 1. Model from active conversation's last assistant response (if loaded) - * 2. Model from active conversation's last assistant response (if not loaded) - * 3. First loaded model (not from active conversation) - * 4. A favorite model - * 5. First available model - */ - async ensureFirstModelSelected(): Promise { - if (this.selectedModelName) return; - - const availableModels = this.getVisibleModels(); - - if (availableModels.length === 0) return; - - // Try to select model from last assistant response first - const lastModel = this.getModelFromLastAssistantResponse(); - - if (lastModel) { - const lastModelOption = availableModels.find((m) => m.model === lastModel); - - if (lastModelOption) { - await this.selectModelById(lastModelOption.id); - - if (this.isModelLoaded(lastModel)) { - await this.fetchModelProps(lastModel); - } - - return; - } - } - - // Try a loaded model first - const loadedModel = availableModels.find((m) => this.isModelLoaded(m.model)); - - if (loadedModel) { - await this.selectModelById(loadedModel.id); - await this.fetchModelProps(loadedModel.model); - - return; - } - - // Try loading a favorite model - const favorite = this.favoriteModelIds.values().next()?.value; - - if (favorite) { - await this.selectModelById(favorite); - - return; - } - - // Fall back to the first available model - await this.selectModelById(availableModels[0].id); - } - - /** - * - * - * Model Selection - * - * - */ - - async selectModelById(modelId: string): Promise { - if (!modelId || this.updating) return; - - if (this.selectedModelId === modelId) return; - - const option = this.models.find((model) => model.id === modelId); - - if (!option) throw new Error('Selected model is not available'); - - this.updating = true; - this.error = null; - - try { - this.selectedModelId = option.id; - this.selectedModelName = option.model; - } finally { - this.updating = false; - } - } - - /** - * Select a model by its model name (used for syncing with conversation model). - */ - selectModelByName(modelName: string): void { - const option = this.models.find((model) => model.model === modelName); - - if (option) { - this.selectedModelId = option.id; - this.selectedModelName = option.model; - } - } - - clearSelection(): void { - this.selectedModelId = null; - this.selectedModelName = null; - } - - findModelByName(modelName: string): ModelOption | null { - return ( - this.models.find( - (model) => - model.model === modelName || model.id === modelName || model.aliases?.includes(modelName) - ) ?? null - ); - } - - findModelById(modelId: string): ModelOption | null { - return this.models.find((model) => model.id === modelId) ?? null; - } - - hasModel(modelName: string): boolean { - return this.models.some((model) => model.model === modelName); - } - - /** - * - * - * Loading / Unloading Models - * - * - */ - - // reconnect delay after the feed drops or the server is not ready yet - /** - * Open the /models/sse feed and keep it live with auto reconnect. - * Idempotent and router mode only. The feed drives status and progress, - * so it replaces any post-operation polling. - */ - subscribeStatus(): void { - if (this.statusReaderActive) return; - - if (!serverStore.isRouterMode) return; - - this.statusReaderActive = true; - this.statusAbort = new AbortController(); - void this.runStatusReader(this.statusAbort.signal); - } - - /** - * Close the /models/sse feed and drop transient progress. - */ - unsubscribeStatus(): void { - this.statusReaderActive = false; - this.statusAbort?.abort(); - this.statusAbort = null; - this.loadProgress.clear(); - } - - /** - * Current load progress for a model, or null when not loading. - */ - getLoadProgress(modelId: string): ModelLoadProgress | null { - return this.loadProgress.get(modelId) ?? null; - } - - /** - * Read the feed and reconnect until unsubscribed. - */ - private async runStatusReader(signal: AbortSignal): Promise { - await ModelsService.watchModelEvents(signal, (event) => this.applyStatusEvent(event)); - } - - /** - * Route one feed record by event kind. Only the status_* events carry a - * status payload, models_reload triggers a list refresh, model_remove drops - * the row, download_* belong to the download surface, not here. - */ - private applyStatusEvent(event: ApiModelsSseEvent): void { - switch (event.event) { - case ServerModelsSseEventType.STATUS_CHANGE: - case ServerModelsSseEventType.MODEL_STATUS: - case ServerModelsSseEventType.STATUS_UPDATE: - this.applyModelStatus(event); - - break; - case ServerModelsSseEventType.MODELS_RELOAD: - void this.fetchRouterModels(); - - break; - case ServerModelsSseEventType.MODEL_REMOVE: - this.removeRouterModel(event.model); - - break; - case ServerModelsSseEventType.DOWNLOAD_PROGRESS: - break; - } - } - - /** - * Apply a status envelope: update the model row, track or clear progress, - * settle any pending load or unload awaiter. - */ - private applyModelStatus(event: ApiModelsSseEvent): void { - const model = event.model; - const data = event.data; - - if (!model || !data?.status) return; - - const status = data.status; - - this.setRouterModelStatus(model, status); - - if (status === ServerModelStatus.LOADING) { - if (data.progress) this.loadProgress.set(model, data.progress); - } else { - this.loadProgress.delete(model); - } - - if (status === ServerModelStatus.LOADED) { - void this.updateModelModalities(model); - } - - const failed = - status === ServerModelStatus.FAILED || - (status === ServerModelStatus.UNLOADED && (data.exit_code ?? 0) !== 0); - - if (failed) { - this.rejectStatus(model, new Error(`Model failed: ${this.toDisplayName(model)}`)); - - return; - } - - this.settleStatus(model, status); - } - - /** - * Drop a model row reported gone by the feed and settle its awaiters. - */ - private removeRouterModel(modelId: string): void { - if (this.routerModels.findIndex((m) => m.id === modelId) === -1) return; - - this.routerModels = this.routerModels.filter((m) => m.id !== modelId); - this.loadProgress.delete(modelId); - this.rejectStatus(modelId, new Error(`Model removed: ${this.toDisplayName(modelId)}`)); - } - - /** - * Update one model row status in place, reassigning to trigger reactivity. - */ - private setRouterModelStatus(modelId: string, status: ServerModelStatus): void { - const idx = this.routerModels.findIndex((m) => m.id === modelId); - - if (idx === -1) return; - - const current = this.routerModels[idx]; - - if (current.status.value === status) return; - - const next = [...this.routerModels]; - - next[idx] = { ...current, status: { ...current.status, value: status } }; - this.routerModels = next; - } - - /** - * Register an awaiter that resolves when the feed reports target status. - * One operation runs per model at a time, so one awaiter per model is kept. - */ - private waitForStatus(modelId: string, target: ServerModelStatus): Promise { - return new Promise((resolve, reject) => { - this.statusWaiters.set(modelId, { reject, resolve, target }); - }); - } - - /** - * Resolve and drop the awaiter when the model reaches its target status. - */ - private settleStatus(modelId: string, status: ServerModelStatus): void { - const waiter = this.statusWaiters.get(modelId); - - if (waiter && waiter.target === status) { - this.statusWaiters.delete(modelId); - waiter.resolve(); - } - } - - /** - * Reject and drop the awaiter for a model. - */ - private rejectStatus(modelId: string, error: Error): void { - const waiter = this.statusWaiters.get(modelId); - - if (waiter) { - this.statusWaiters.delete(modelId); - waiter.reject(error); - } - } - - async loadModel(modelId: string): Promise { - if (this.isModelLoaded(modelId)) return; - - if (this.modelLoadingStates.get(modelId)) return; - - this.modelLoadingStates.set(modelId, true); - this.error = null; - - // the feed drives completion, so it must be live before the request - this.subscribeStatus(); - - const reachedLoaded = this.waitForStatus(modelId, ServerModelStatus.LOADED); - - reachedLoaded.catch(() => {}); - - try { - await ModelsService.load(modelId); - await reachedLoaded; - toast.success(`Model loaded: ${this.toDisplayName(modelId)}`); - } catch (error) { - this.rejectStatus(modelId, error instanceof Error ? error : new Error('load failed')); - this.error = error instanceof Error ? error.message : 'Failed to load model'; - toast.error(`Failed to load model: ${this.toDisplayName(modelId)}`); - - throw error; - } finally { - this.modelLoadingStates.set(modelId, false); - } - } - - async unloadModel(modelId: string): Promise { - if (!this.isModelLoaded(modelId)) return; - - if (this.modelLoadingStates.get(modelId)) return; - - this.modelLoadingStates.set(modelId, true); - this.error = null; - - this.subscribeStatus(); - - const reachedUnloaded = this.waitForStatus(modelId, ServerModelStatus.UNLOADED); - - reachedUnloaded.catch(() => {}); - - try { - await ModelsService.unload(modelId); - await reachedUnloaded; - toast.info(`Model unloaded: ${this.toDisplayName(modelId)}`); - } catch (error) { - this.rejectStatus(modelId, error instanceof Error ? error : new Error('unload failed')); - this.error = error instanceof Error ? error.message : 'Failed to unload model'; - toast.error(`Failed to unload model: ${this.toDisplayName(modelId)}`); - - throw error; - } finally { - this.modelLoadingStates.set(modelId, false); - } - } - - async ensureModelLoaded(modelId: string): Promise { - if (this.isModelLoaded(modelId)) return; - - await this.loadModel(modelId); - } - - /** - * - * - * Favorites - * - * - */ - - isFavorite(modelId: string): boolean { - return this.favoriteModelIds.has(modelId); - } - - toggleFavorite(modelId: string): void { - const next = new SvelteSet(this.favoriteModelIds); - - if (next.has(modelId)) { - next.delete(modelId); - } else { - next.add(modelId); - } - - this.favoriteModelIds = next; - - try { - localStorage.setItem(FAVORITE_MODELS_LOCALSTORAGE_KEY, JSON.stringify([...next])); - } catch { - toast.error('Failed to save favorite models to local storage'); - } - } - - private loadFavoritesFromStorage(): Set { - try { - const raw = localStorage.getItem(FAVORITE_MODELS_LOCALSTORAGE_KEY); - - return raw ? new Set(JSON.parse(raw) as string[]) : new Set(); - } catch { - toast.error('Failed to load favorite models from local storage'); - - return new Set(); - } - } - - /** - * - * - * Utilities - * - * - */ - - private toDisplayName(id: string): string { - const segments = id.split(/\\|\//); - const candidate = segments.pop(); - - return candidate && candidate.trim().length > 0 ? candidate : id; - } - - private buildModalities( - modalities: NonNullable - ): ModelModalities { - return { - audio: modalities.audio ?? false, - video: modalities.video ?? false, - vision: modalities.vision ?? false - }; - } - - /** Map the router modalities, the only source available while a model is not loaded. */ - private buildArchitectureModalities( - architecture: ApiModelDataEntry['architecture'] - ): ModelModalities | undefined { - if (!architecture) return undefined; - - const inputs = architecture.input_modalities; - - return { - audio: inputs.includes(FileTypeCategory.AUDIO), - video: inputs.includes(FileTypeCategory.VIDEO), - vision: inputs.includes(FileTypeCategory.IMAGE) - }; - } - - clear(): void { - this.unsubscribeStatus(); - this.statusWaiters.forEach((waiter) => waiter.reject(new Error('Models store cleared'))); - this.statusWaiters.clear(); - this.models = []; - this.routerModels = []; - this.loading = false; - this.updating = false; - this.error = null; - this.selectedModelId = null; - this.selectedModelName = null; - this.modelUsage.clear(); - this.modelLoadingStates.clear(); - this.modelPropsCache.clear(); - this.modelPropsFetching.clear(); - } - - /** - * Prune expired entries from caches. - * Call periodically for proactive memory cleanup. - */ - pruneExpiredCache(): number { - return this.modelPropsCache.prune(); - } -} - -export const modelsStore = new ModelsStore(); diff --git a/tools/ui/src/lib/stores/models/index.svelte.ts b/tools/ui/src/lib/stores/models/index.svelte.ts new file mode 100644 index 000000000..90d6fe76b --- /dev/null +++ b/tools/ui/src/lib/stores/models/index.svelte.ts @@ -0,0 +1,451 @@ +/** + * modelsStore - Model management for MODEL and ROUTER modes + * + * Owns model lists, selection, favorites and load/unload state. Composes the + * per-model props cache (modalities, thinking detection) as + * {@link ModelsStore.props} and the /models/sse status feed as + * {@link ModelsStore.status}; tracks which conversations use which models. + */ + +import { FAVORITE_MODELS_LOCALSTORAGE_KEY } from '$lib/constants'; +import { ServerModelStatus } from '$lib/enums'; +import { ModelsService } from '$lib/services/models.service'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import { type ModelPropsHost, ModelPropsManager } from '$lib/stores/models/props.svelte'; +import { type ModelStatusHost, ModelStatusManager } from '$lib/stores/models/status.svelte'; +import { serverStore } from '$lib/stores/server.svelte'; +import { getConversationModel } from '$lib/utils/conversation-utils'; +import { SvelteSet } from 'svelte/reactivity'; +import { toast } from 'svelte-sonner'; + +class ModelsStore implements ModelPropsHost, ModelStatusHost { + error = $state(null); + favoriteModelIds = $state>(this.loadFavoritesFromStorage()); + loading = $state(false); + models = $state([]); + routerModels = $state([]); + selectedModelId = $state(null); + selectedModelName = $state(null); + + updating = $state(false); + + /** Per-model props cache, modalities and thinking detection, composed here. */ + private _props = new ModelPropsManager(this); + + /** Load/unload operations and the /models/sse status feed, composed here. */ + private _status = new ModelStatusManager(this); + + // Dedup concurrent fetch() callers — all awaiters share the same inflight promise. + // Without this, ?model= URL handler races an in-progress fetch and sees an empty list. + private inflightFetch: Promise | null = null; + + /** + * Model the active conversation view resolves to. Router mode: the user's + * selection first, then the conversation's own model. Otherwise the single + * served model, from the models list or the server props as a fallback. + */ + get activeModelId(): string | null { + if (!serverStore.isRouterMode) { + return this.models.length > 0 ? this.models[0].model : this.singleModelName; + } + + if (this.selectedModelId) { + const selected = this.models.find((m) => m.id === this.selectedModelId); + + if (selected) return selected.model; + } + + const conversationModel = getConversationModel(conversationsStore.activeMessages); + + if (conversationModel) { + const model = this.models.find((m) => m.model === conversationModel); + + if (model) return model.model; + } + + return null; + } + + get loadedModelIds(): string[] { + return this.routerModels + .filter( + (m) => + m.status.value === ServerModelStatus.LOADED || + m.status.value === ServerModelStatus.SLEEPING + ) + .map((m) => m.id); + } + + get props() { + return this._props; + } + + get selectedModel(): ModelOption | null { + if (!this.selectedModelId) return null; + + return this.models.find((m) => m.id === this.selectedModelId) ?? null; + } + + get selectedModelContextSize(): number | null { + if (!this.selectedModelName) return null; + + return this.props.getModelContextSize(this.selectedModelName); + } + + /** + * Get model name in MODEL mode (single model). + * Extracts from model_path or model_alias from server props. + * In ROUTER mode, returns null (model is per-conversation). + */ + get singleModelName(): string | null { + if (serverStore.isRouterMode) return null; + + const props = serverStore.props; + + if (props?.model_alias) return props.model_alias; + + if (!props?.model_path) return null; + + return props.model_path.split(/(\\|\/)/).pop() || null; + } + + get status() { + return this._status; + } + + clearSelection(): void { + this.selectedModelId = null; + this.selectedModelName = null; + } + + /** + * Auto-selects the first available model if none is selected. + * Prioritizes: + * 1. Model from active conversation's last assistant response (if loaded) + * 2. Model from active conversation's last assistant response (if not loaded) + * 3. First loaded model (not from active conversation) + * 4. A favorite model + * 5. First available model + */ + async ensureFirstModelSelected(): Promise { + if (this.selectedModelName) return; + + const availableModels = this.getVisibleModels(); + + if (availableModels.length === 0) return; + + // Try to select model from last assistant response first + const lastModel = this.getModelFromLastAssistantResponse(); + + if (lastModel) { + const lastModelOption = availableModels.find((m) => m.model === lastModel); + + if (lastModelOption) { + await this.selectModelById(lastModelOption.id); + + if (this.isModelLoaded(lastModel)) { + await this.props.fetchModelProps(lastModel); + } + + return; + } + } + + // Try a loaded model first + const loadedModel = availableModels.find((m) => this.isModelLoaded(m.model)); + + if (loadedModel) { + await this.selectModelById(loadedModel.id); + await this.props.fetchModelProps(loadedModel.model); + + return; + } + + // Try loading a favorite model + const favorite = this.favoriteModelIds.values().next()?.value; + + if (favorite) { + await this.selectModelById(favorite); + + return; + } + + // Fall back to the first available model + await this.selectModelById(availableModels[0].id); + } + + /** + * Fetch list of models from server and detect server role. + * Also fetches modalities for MODEL mode (single model). + */ + async fetch(force = false): Promise { + if (this.inflightFetch) return this.inflightFetch; + + if (this.models.length > 0 && !force) return; + + this.inflightFetch = this.runFetch(); + try { + await this.inflightFetch; + } finally { + this.inflightFetch = null; + } + } + + /** + * Fetch router models with full metadata (ROUTER mode only). + * No-op in router mode — fetch() already calls listRouter() internally. + * Kept for API compatibility (e.g. handleOpenChange dropdown open handler). + */ + async fetchRouterModels(): Promise { + if (!serverStore.isRouterMode) return; + + try { + const response = await ModelsService.listRouter(); + + this.routerModels = response.data; + await this.props.fetchModalitiesForLoadedModels(); + + const visible = this.getVisibleModels(); + + if (visible.length === 1 && this.isModelLoaded(visible[0].model)) { + this.selectModelById(visible[0].id); + } + } catch (error) { + console.warn('Failed to fetch router models:', error); + this.routerModels = []; + } + } + + findModelById(modelId: string): ModelOption | null { + return this.models.find((model) => model.id === modelId) ?? null; + } + + findModelByName(modelName: string): ModelOption | null { + return ( + this.models.find( + (model) => + model.model === modelName || model.id === modelName || model.aliases?.includes(modelName) + ) ?? null + ); + } + + /** + * Gets the model name from the last assistant message in the active conversation. + * Used by both the chat page and settings page to maintain model consistency. + */ + getModelFromLastAssistantResponse(): string | null { + const messages = conversationsStore.activeMessages; + + if (!messages || messages.length === 0) return null; + + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].model) { + return messages[i].model; + } + } + + return null; + } + + getModelStatus(modelId: string): ServerModelStatus | null { + const model = this.routerModels.find((m) => m.id === modelId); + + return model?.status.value ?? null; + } + + hasModel(modelName: string): boolean { + return this.models.some((model) => model.model === modelName); + } + + isFavorite(modelId: string): boolean { + return this.favoriteModelIds.has(modelId); + } + + isModelLoaded(modelId: string): boolean { + const model = this.routerModels.find((m) => m.id === modelId); + + return ( + model?.status.value === ServerModelStatus.LOADED || + model?.status.value === ServerModelStatus.SLEEPING + ); + } + + async selectModelById(modelId: string): Promise { + if (!modelId || this.updating) return; + + if (this.selectedModelId === modelId) return; + + const option = this.models.find((model) => model.id === modelId); + + if (!option) throw new Error('Selected model is not available'); + + this.updating = true; + this.error = null; + + try { + this.selectedModelId = option.id; + this.selectedModelName = option.model; + } finally { + this.updating = false; + } + } + + /** + * Select a model by its model name (used for syncing with conversation model). + */ + selectModelByName(modelName: string): void { + const option = this.models.find((model) => model.model === modelName); + + if (option) { + this.selectedModelId = option.id; + this.selectedModelName = option.model; + } + } + + /** + * Auto-selects the model from the last assistant response if available and loaded. + * Returns true if a model was selected, false otherwise. + */ + async selectModelFromLastAssistantResponse(): Promise { + const lastModel = this.getModelFromLastAssistantResponse(); + + if (!lastModel || this.selectedModelName === lastModel) return false; + + const matchingModel = this.models.find((option) => option.model === lastModel); + + if (!matchingModel || !this.isModelLoaded(lastModel)) return false; + + try { + await this.selectModelById(matchingModel.id); + console.log(`[modelsStore] Automatically selected model: ${lastModel} from last message`); + + return true; + } catch (error) { + console.warn('[modelsStore] Failed to automatically select model from last message:', error); + + return false; + } + } + + toDisplayName(id: string): string { + const segments = id.split(/\\|\//); + const candidate = segments.pop(); + + return candidate && candidate.trim().length > 0 ? candidate : id; + } + + toggleFavorite(modelId: string): void { + const next = new SvelteSet(this.favoriteModelIds); + + if (next.has(modelId)) { + next.delete(modelId); + } else { + next.add(modelId); + } + + this.favoriteModelIds = next; + + try { + localStorage.setItem(FAVORITE_MODELS_LOCALSTORAGE_KEY, JSON.stringify([...next])); + } catch { + toast.error('Failed to save favorite models to local storage'); + } + } + + /** + * Build ModelOption[] from an API response. + * Both MODEL and ROUTER modes share the same mapping logic; + * they differ only in which endpoint is called. + */ + private buildModelOptions( + response: ApiModelListResponse | ApiRouterModelsListResponse + ): ModelOption[] { + return response.data.map((item: ApiModelDataEntry, index: number) => { + const details = response.models?.[index]; + const rawCapabilities = Array.isArray(details?.capabilities) ? details?.capabilities : []; + const displayNameSource = + details?.name && details.name.trim().length > 0 ? details.name : item.id; + const modelId = details?.model || item.id; + + return { + aliases: item.aliases ?? [], + capabilities: rawCapabilities.filter((value: unknown): value is string => Boolean(value)), + description: details?.description, + details: details?.details, + id: item.id, + meta: item.meta ?? null, + modalities: this.props.buildArchitectureModalities(item.architecture), + model: modelId, + name: this.toDisplayName(displayNameSource), + parsedId: ModelsService.parseModelId(modelId), + tags: item.tags ?? [] + }; + }); + } + + /** Fetch models in MODEL mode (single model, standard OpenAI-compatible). */ + private async fetchModelModeInternal(): Promise { + const response = await ModelsService.list(); + + return this.buildModelOptions(response); + } + + /** + * Filter to models visible in the UI (ui !== false). + */ + private getVisibleModels(): ModelOption[] { + return this.models.filter((option) => this.props.getModelProps(option.model)?.ui !== false); + } + + private loadFavoritesFromStorage(): Set { + try { + const raw = localStorage.getItem(FAVORITE_MODELS_LOCALSTORAGE_KEY); + + return raw ? new Set(JSON.parse(raw) as string[]) : new Set(); + } catch { + toast.error('Failed to load favorite models from local storage'); + + return new Set(); + } + } + + private async runFetch(): Promise { + this.loading = true; + this.error = null; + + try { + if (!serverStore.props) { + await serverStore.fetch(); + } + + const router = serverStore.isRouterMode; + + if (router) { + const response = await ModelsService.listRouter(); + + this.routerModels = response.data; + this.models = this.buildModelOptions(response); + + await this.props.fetchModalitiesForLoadedModels(); + + const visible = this.getVisibleModels(); + + if (visible.length === 1 && this.isModelLoaded(visible[0].model)) { + this.selectModelById(visible[0].id); + } + } else { + this.models = await this.fetchModelModeInternal(); + } + } catch (error) { + this.models = []; + this.error = error instanceof Error ? error.message : 'Failed to load models'; + + throw error; + } finally { + this.loading = false; + } + } +} + +export const modelsStore = new ModelsStore(); diff --git a/tools/ui/src/lib/stores/models/props.svelte.ts b/tools/ui/src/lib/stores/models/props.svelte.ts new file mode 100644 index 000000000..9d2d817ac --- /dev/null +++ b/tools/ui/src/lib/stores/models/props.svelte.ts @@ -0,0 +1,273 @@ +/** + * ModelPropsManager - Per-model props cache, modalities and thinking detection + * + * Owns the /props?model= cache with TTL, the modality views over it, + * and chat-template thinking detection. Created and owned by modelsStore; + * the host owns the model lists that fetched modalities are mirrored onto. + * + * **API Inconsistency Workaround:** + * In MODEL mode, `/props` returns modalities for the single model. + * In ROUTER mode, `/props` has no modalities - must use `/props?model=` per model. + */ + +import { MODEL_PROPS_CACHE } from '$lib/constants'; +import { FileTypeCategory, ModelModality } from '$lib/enums'; +import { PropsService } from '$lib/services/props.service'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { serverStore } from '$lib/stores/server.svelte'; +// deep imports, not the '$lib/utils' barrel: it re-exports modules that reach back +// into the stores, and going through it here would read a half-built module +import { TTLCache } from '$lib/utils/cache-ttl'; +import { detectThinkingSupport } from '$lib/utils/chat-template-thinking-detector'; +import { SvelteSet } from 'svelte/reactivity'; + +/** + * The slice of modelsStore the manager reads. Kept narrow on purpose so it + * cannot reach around the host's full surface; modelsStore implements this + * structurally. + */ +export interface ModelPropsHost { + /** Model rows the manager mirrors fetched modalities onto. */ + models: ModelOption[]; + readonly selectedModelName: string | null; + readonly loadedModelIds: string[]; + isModelLoaded(modelId: string): boolean; +} + +export class ModelPropsManager { + /** Version counter for the cache - bumped on writes so $derived consumers recompute. */ + cacheVersion = $state(0); + /** + * Model-specific props cache with TTL. + * Key: modelId, Value: props data including modalities. + */ + private cache = new TTLCache({ + maxEntries: MODEL_PROPS_CACHE.MAX_ENTRIES, + ttlMs: MODEL_PROPS_CACHE.TTL_MS + }); + private fetching = new SvelteSet(); + + /** + * Whether the selected model's chat template supports thinking/reasoning. + * Uses heuristic detection on the model's chat_template from /props. + * + * - MODEL mode: the global /props already describes the single loaded model, + * so its chat_template is used directly and no per-model cache is involved + * - ROUTER mode: fetches /props?model= for the selected model (cached), + * triggering an async fetch if not yet cached + */ + get supportsThinking(): boolean { + if (!serverStore.isRouterMode) { + return detectThinkingSupport(serverStore.props?.chat_template ?? ''); + } + + const modelId = this.host.selectedModelName; + + if (!modelId) return false; + + if (!this.cache.get(modelId)) { + this.fetchModelProps(modelId); + } + + const props = this.getModelProps(modelId); + + return detectThinkingSupport(props?.chat_template ?? ''); + } + + /** Map the router modalities, the only source available while a model is not loaded. */ + buildArchitectureModalities( + architecture: ApiModelDataEntry['architecture'] + ): ModelModalities | undefined { + if (!architecture) return undefined; + + const inputs = architecture.input_modalities; + + return { + audio: inputs.includes(FileTypeCategory.AUDIO), + video: inputs.includes(FileTypeCategory.VIDEO), + vision: inputs.includes(FileTypeCategory.IMAGE) + }; + } + + /** + * Check if a specific model supports thinking. + * In MODEL mode the global /props describes the single loaded model. + * In ROUTER mode, fetches model props if not cached. + */ + checkModelSupportsThinking(modelId: string): boolean { + if (!serverStore.isRouterMode) { + return detectThinkingSupport(serverStore.props?.chat_template ?? ''); + } + + if (!modelId) return false; + + if (!this.cache.get(modelId)) { + this.fetchModelProps(modelId); + } + + const props = this.getModelProps(modelId); + + return detectThinkingSupport(props?.chat_template ?? ''); + } + + constructor(private host: ModelPropsHost) {} + + /** Fetch modalities for all loaded models from /props endpoint. */ + async fetchModalitiesForLoadedModels(): Promise { + const loadedModelIds = this.host.loadedModelIds; + + if (loadedModelIds.length === 0) return; + + const propsPromises = loadedModelIds.map((modelId) => this.fetchModelProps(modelId)); + + try { + const results = await Promise.all(propsPromises); + + this.host.models = this.host.models.map((model) => { + const modelIndex = loadedModelIds.indexOf(model.model); + + if (modelIndex === -1) return model; + + const props = results[modelIndex]; + + if (!props?.modalities) return model; + + return { ...model, modalities: this.buildModalities(props.modalities) }; + }); + + this.cacheVersion++; + } catch (error) { + console.warn('Failed to fetch modalities for loaded models:', error); + } + } + + /** + * Fetch props for a specific model from /props endpoint. + * Uses caching to avoid redundant requests. + * + * In ROUTER mode, this only fetches props if the model is loaded, + * since unloaded models return 400 from /props endpoint. + * + * @param modelId - Model identifier to fetch props for + * @returns Props data or null if fetch failed or model not loaded + */ + async fetchModelProps(modelId: string): Promise { + const cached = this.cache.get(modelId); + + if (cached) return cached; + + if (serverStore.isRouterMode && !this.host.isModelLoaded(modelId)) { + return null; + } + + if (this.fetching.has(modelId)) return null; + + this.fetching.add(modelId); + + try { + const props = await PropsService.fetchForModel(modelId); + + this.cache.set(modelId, props); + this.cacheVersion++; + + return props; + } catch (error) { + console.warn(`Failed to fetch props for model ${modelId}:`, error); + + return null; + } finally { + this.fetching.delete(modelId); + } + } + + getModelContextSize(modelId: string): number | null { + const props = this.getModelProps(modelId); + const nCtx = props?.default_generation_settings?.n_ctx; + + return typeof nCtx === 'number' ? nCtx : null; + } + + getModelModalities(modelId: string): ModelModalities | null { + if (!serverStore.isRouterMode && serverStore.props?.modalities) { + return this.buildModalities(serverStore.props.modalities); + } + + const model = this.host.models.find((m) => m.model === modelId || m.id === modelId); + + if (model?.modalities) { + return model.modalities; + } + + const props = this.cache.get(modelId); + + if (props?.modalities) { + return this.buildModalities(props.modalities); + } + + return null; + } + + getModelModalitiesArray(modelId: string): ModelModality[] { + const modalities = this.getModelModalities(modelId); + + if (!modalities) return []; + + const result: ModelModality[] = []; + + if (modalities.vision) result.push(ModelModality.VISION); + + if (modalities.audio) result.push(ModelModality.AUDIO); + + if (modalities.video) result.push(ModelModality.VIDEO); + + return result; + } + + getModelProps(modelId: string): ApiLlamaCppServerProps | null { + return this.cache.get(modelId); + } + + isModelPropsFetching(modelId: string): boolean { + return this.fetching.has(modelId); + } + + modelSupportsAudio(modelId: string): boolean { + return this.getModelModalities(modelId)?.audio ?? false; + } + + modelSupportsVideo(modelId: string): boolean { + return this.getModelModalities(modelId)?.video ?? false; + } + + modelSupportsVision(modelId: string): boolean { + return this.getModelModalities(modelId)?.vision ?? false; + } + + /** + * Update modalities for a specific model. + * Called when a model is loaded or when we need fresh modality data. + */ + async updateModelModalities(modelId: string): Promise { + const props = await this.fetchModelProps(modelId); + + if (!props?.modalities) return; + + this.host.models = this.host.models.map((model) => + model.model === modelId + ? { ...model, modalities: this.buildModalities(props.modalities!) } + : model + ); + + this.cacheVersion++; + } + + private buildModalities( + modalities: NonNullable + ): ModelModalities { + return { + audio: modalities.audio ?? false, + video: modalities.video ?? false, + vision: modalities.vision ?? false + }; + } +} diff --git a/tools/ui/src/lib/stores/models/status.svelte.ts b/tools/ui/src/lib/stores/models/status.svelte.ts new file mode 100644 index 000000000..d0160aa4d --- /dev/null +++ b/tools/ui/src/lib/stores/models/status.svelte.ts @@ -0,0 +1,278 @@ +/** + * ModelStatusManager - Model load/unload operations and the /models/sse feed + * + * Owns the status feed subscription, load progress tracking, and the + * awaiters that settle load/unload operations. The feed drives status and + * progress, so it replaces any post-operation polling. Created and owned by + * modelsStore; the host owns the router model rows the feed updates. + */ + +import { ServerModelsSseEventType, ServerModelStatus } from '$lib/enums'; +import { ModelsService } from '$lib/services/models.service'; +import type { ModelPropsManager } from '$lib/stores/models/props.svelte'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { serverStore } from '$lib/stores/server.svelte'; +import { SvelteMap } from 'svelte/reactivity'; +import { toast } from 'svelte-sonner'; + +/** + * The slice of modelsStore the manager drives. Kept narrow on purpose so it + * cannot reach around the host's full surface; modelsStore implements this + * structurally. + */ +export interface ModelStatusHost { + error: string | null; + readonly props: ModelPropsManager; + /** Router model rows the status feed updates. */ + routerModels: ApiModelDataEntry[]; + fetchRouterModels(): Promise; + isModelLoaded(modelId: string): boolean; + toDisplayName(id: string): string; +} + +export class ModelStatusManager { + private loadingStates = new SvelteMap(); + private loadProgress = new SvelteMap(); + // /models/sse feed state, the single source of truth for status and load progress + private statusAbort: AbortController | null = null; + private statusReaderActive = false; + private statusWaiters = new SvelteMap< + string, + { target: ServerModelStatus; resolve: () => void; reject: (e: Error) => void } + >(); + + constructor(private host: ModelStatusHost) {} + + async ensureLoaded(modelId: string): Promise { + if (this.host.isModelLoaded(modelId)) return; + + await this.load(modelId); + } + + /** + * Current load progress for a model, or null when not loading. + */ + getLoadProgress(modelId: string): ModelLoadProgress | null { + return this.loadProgress.get(modelId) ?? null; + } + + isOperationInProgress(modelId: string): boolean { + return this.loadingStates.get(modelId) ?? false; + } + + async load(modelId: string): Promise { + if (this.host.isModelLoaded(modelId)) return; + + if (this.loadingStates.get(modelId)) return; + + this.loadingStates.set(modelId, true); + this.host.error = null; + + // the feed drives completion, so it must be live before the request + this.subscribe(); + + const reachedLoaded = this.waitForStatus(modelId, ServerModelStatus.LOADED); + + reachedLoaded.catch(() => {}); + + try { + await ModelsService.load(modelId); + await reachedLoaded; + toast.success(`Model loaded: ${this.host.toDisplayName(modelId)}`); + } catch (error) { + this.rejectStatus(modelId, error instanceof Error ? error : new Error('load failed')); + this.host.error = error instanceof Error ? error.message : 'Failed to load model'; + toast.error(`Failed to load model: ${this.host.toDisplayName(modelId)}`); + + throw error; + } finally { + this.loadingStates.set(modelId, false); + } + } + + /** + * Open the /models/sse feed and keep it live with auto reconnect. + * Idempotent and router mode only. + */ + subscribe(): void { + if (this.statusReaderActive) return; + + if (!serverStore.isRouterMode) return; + + this.statusReaderActive = true; + this.statusAbort = new AbortController(); + void this.runStatusReader(this.statusAbort.signal); + } + + async unload(modelId: string): Promise { + if (!this.host.isModelLoaded(modelId)) return; + + if (this.loadingStates.get(modelId)) return; + + this.loadingStates.set(modelId, true); + this.host.error = null; + + this.subscribe(); + + const reachedUnloaded = this.waitForStatus(modelId, ServerModelStatus.UNLOADED); + + reachedUnloaded.catch(() => {}); + + try { + await ModelsService.unload(modelId); + await reachedUnloaded; + toast.info(`Model unloaded: ${this.host.toDisplayName(modelId)}`); + } catch (error) { + this.rejectStatus(modelId, error instanceof Error ? error : new Error('unload failed')); + this.host.error = error instanceof Error ? error.message : 'Failed to unload model'; + toast.error(`Failed to unload model: ${this.host.toDisplayName(modelId)}`); + + throw error; + } finally { + this.loadingStates.set(modelId, false); + } + } + + /** + * Close the /models/sse feed and drop transient progress. + */ + unsubscribe(): void { + this.statusReaderActive = false; + this.statusAbort?.abort(); + this.statusAbort = null; + this.loadProgress.clear(); + } + + /** + * Apply a status envelope: update the model row, track or clear progress, + * settle any pending load or unload awaiter. + */ + private applyModelStatus(event: ApiModelsSseEvent): void { + const model = event.model; + const data = event.data; + + if (!model || !data?.status) return; + + const status = data.status; + + this.setRouterModelStatus(model, status); + + if (status === ServerModelStatus.LOADING) { + if (data.progress) this.loadProgress.set(model, data.progress); + } else { + this.loadProgress.delete(model); + } + + if (status === ServerModelStatus.LOADED) { + void this.host.props.updateModelModalities(model); + } + + const failed = + status === ServerModelStatus.FAILED || + (status === ServerModelStatus.UNLOADED && (data.exit_code ?? 0) !== 0); + + if (failed) { + this.rejectStatus(model, new Error(`Model failed: ${this.host.toDisplayName(model)}`)); + + return; + } + + this.settleStatus(model, status); + } + + /** + * Route one feed record by event kind. Only the status_* events carry a + * status payload, models_reload triggers a list refresh, model_remove drops + * the row, download_* belong to the download surface, not here. + */ + private applyStatusEvent(event: ApiModelsSseEvent): void { + switch (event.event) { + case ServerModelsSseEventType.STATUS_CHANGE: + case ServerModelsSseEventType.MODEL_STATUS: + case ServerModelsSseEventType.STATUS_UPDATE: + this.applyModelStatus(event); + + break; + case ServerModelsSseEventType.MODELS_RELOAD: + void this.host.fetchRouterModels(); + + break; + case ServerModelsSseEventType.MODEL_REMOVE: + this.removeRouterModel(event.model); + + break; + case ServerModelsSseEventType.DOWNLOAD_PROGRESS: + break; + } + } + + /** + * Reject and drop the awaiter for a model. + */ + private rejectStatus(modelId: string, error: Error): void { + const waiter = this.statusWaiters.get(modelId); + + if (waiter) { + this.statusWaiters.delete(modelId); + waiter.reject(error); + } + } + + /** + * Drop a model row reported gone by the feed and settle its awaiters. + */ + private removeRouterModel(modelId: string): void { + if (this.host.routerModels.findIndex((m) => m.id === modelId) === -1) return; + + this.host.routerModels = this.host.routerModels.filter((m) => m.id !== modelId); + this.loadProgress.delete(modelId); + this.rejectStatus(modelId, new Error(`Model removed: ${this.host.toDisplayName(modelId)}`)); + } + + /** + * Read the feed and reconnect until unsubscribed. + */ + private async runStatusReader(signal: AbortSignal): Promise { + await ModelsService.watchModelEvents(signal, (event) => this.applyStatusEvent(event)); + } + + /** + * Update one model row status in place, reassigning to trigger reactivity. + */ + private setRouterModelStatus(modelId: string, status: ServerModelStatus): void { + const idx = this.host.routerModels.findIndex((m) => m.id === modelId); + + if (idx === -1) return; + + const current = this.host.routerModels[idx]; + + if (current.status.value === status) return; + + const next = [...this.host.routerModels]; + + next[idx] = { ...current, status: { ...current.status, value: status } }; + this.host.routerModels = next; + } + + /** + * Resolve and drop the awaiter when the model reaches its target status. + */ + private settleStatus(modelId: string, status: ServerModelStatus): void { + const waiter = this.statusWaiters.get(modelId); + + if (waiter && waiter.target === status) { + this.statusWaiters.delete(modelId); + waiter.resolve(); + } + } + + /** + * Register an awaiter that resolves when the feed reports target status. + * One operation runs per model at a time, so one awaiter per model is kept. + */ + private waitForStatus(modelId: string, target: ServerModelStatus): Promise { + return new Promise((resolve, reject) => { + this.statusWaiters.set(modelId, { reject, resolve, target }); + }); + } +} diff --git a/tools/ui/src/lib/stores/permissions.svelte.ts b/tools/ui/src/lib/stores/permissions.svelte.ts index 3e83538e9..f4eae4b7e 100644 --- a/tools/ui/src/lib/stores/permissions.svelte.ts +++ b/tools/ui/src/lib/stores/permissions.svelte.ts @@ -1,3 +1,11 @@ +/** + * permissionsStore - Allowed tool permissions + * + * Owns the set of tools the user has permanently allowed, persisted to + * localStorage. The agentic loop's permission gates consult it to run a + * tool without prompting. + */ + import { browser } from '$app/environment'; import { ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY } from '$lib/constants'; import { SvelteSet } from 'svelte/reactivity'; @@ -5,6 +13,24 @@ import { SvelteSet } from 'svelte/reactivity'; class PermissionsStore { private _tools = $state(new SvelteSet()); + get tools(): ReadonlySet { + return this._tools; + } + + allowTool(key: string): void { + this._tools.add(key); + this.persist(); + } + + allowTools(keys: string[]): void { + for (const key of keys) this._tools.add(key); + this.persist(); + } + + hasTool(key: string): boolean { + return this._tools.has(key); + } + /** * Load persisted permissions. Called by initStores() after migrations * have run. @@ -29,30 +55,12 @@ class PermissionsStore { } } - get tools(): ReadonlySet { - return this._tools; - } - - hasTool(key: string): boolean { - return this._tools.has(key); - } - - allowTool(key: string): void { - this._tools.add(key); - this._persist(); - } - - allowTools(keys: string[]): void { - for (const key of keys) this._tools.add(key); - this._persist(); - } - revokeTool(key: string): void { this._tools.delete(key); - this._persist(); + this.persist(); } - private _persist(): void { + private persist(): void { try { localStorage.setItem(ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY, JSON.stringify([...this._tools])); } catch (err) { diff --git a/tools/ui/src/lib/stores/server.svelte.ts b/tools/ui/src/lib/stores/server.svelte.ts index 7de5850b9..e145e2891 100644 --- a/tools/ui/src/lib/stores/server.svelte.ts +++ b/tools/ui/src/lib/stores/server.svelte.ts @@ -1,79 +1,57 @@ +/** + * serverStore - Server connection state, configuration and role detection + * + * Owns the connection state and properties fetched from /props, plus MODEL + * vs ROUTER role detection and server-wide generation defaults. Uses + * PropsService for the /props fetch. + */ + import { ServerRole } from '$lib/enums'; import { PropsService } from '$lib/services/props.service'; import { ApiError } from '$lib/utils'; const LOADING_RETRY_INTERVAL_MS = 1000; -/** - * serverStore - Server connection state, configuration, and role detection - * - * This store manages the server connection state and properties fetched from `/props`. - * It provides reactive state for server configuration and role detection. - * - * **Architecture & Relationships:** - * - **PropsService**: Stateless service for fetching `/props` data - * - **serverStore** (this class): Reactive store for server state - * - **modelsStore**: Independent store for model management (uses PropsService directly) - * - * **Key Features:** - * - **Server State**: Connection status, loading, error handling - * - **Role Detection**: MODEL (single model) vs ROUTER (multi-model) - * - **Default Params**: Server-wide generation defaults - */ class ServerStore { - /** - * - * - * State - * - * - */ - - props = $state(null); - loading = $state(false); error = $state(null); - status = $state(null); + loading = $state(false); + props = $state(null); role = $state(null); + status = $state(null); private fetchPromise: Promise | null = null; private retryTimer: ReturnType | null = null; - /** - * - * - * Getters - * - * - */ - - get defaultParams(): ApiLlamaCppServerProps['default_generation_settings']['params'] | null { - return this.props?.default_generation_settings?.params || null; - } - get contextSize(): number | null { const nCtx = this.props?.default_generation_settings?.n_ctx; return typeof nCtx === 'number' ? nCtx : null; } - get uiSettings(): Record | undefined { - return this.props?.ui_settings ?? this.props?.webui_settings; - } - - get isRouterMode(): boolean { - return this.role === ServerRole.ROUTER; + get defaultParams(): ApiLlamaCppServerProps['default_generation_settings']['params'] | null { + return this.props?.default_generation_settings?.params || null; } get isModelMode(): boolean { return this.role === ServerRole.MODEL; } - /** - * - * - * Data Handling - * - * - */ + get isRouterMode(): boolean { + return this.role === ServerRole.ROUTER; + } + + get uiSettings(): Record | undefined { + return this.props?.ui_settings ?? this.props?.webui_settings; + } + + clear(): void { + this.clearRetryTimer(); + this.props = null; + this.error = null; + this.status = null; + this.loading = false; + this.role = null; + this.fetchPromise = null; + } /** * @param background - Set by the automatic "still loading" poll. Skips the @@ -124,14 +102,20 @@ class ServerStore { await fetchPromise; } - clear(): void { - this.clearRetryTimer(); - this.props = null; - this.error = null; - this.status = null; - this.loading = false; - this.role = null; - this.fetchPromise = null; + private clearRetryTimer(): void { + if (this.retryTimer) { + clearTimeout(this.retryTimer); + this.retryTimer = null; + } + } + + private detectRole(props: ApiLlamaCppServerProps): void { + const newRole = props?.role === ServerRole.ROUTER ? ServerRole.ROUTER : ServerRole.MODEL; + + if (this.role !== newRole) { + this.role = newRole; + console.info(`Server running in ${newRole === ServerRole.ROUTER ? 'ROUTER' : 'MODEL'} mode`); + } } private scheduleRetry(): void { @@ -142,30 +126,6 @@ class ServerStore { this.fetch({ background: true }); }, LOADING_RETRY_INTERVAL_MS); } - - private clearRetryTimer(): void { - if (this.retryTimer) { - clearTimeout(this.retryTimer); - this.retryTimer = null; - } - } - - /** - * - * - * Utilities - * - * - */ - - private detectRole(props: ApiLlamaCppServerProps): void { - const newRole = props?.role === ServerRole.ROUTER ? ServerRole.ROUTER : ServerRole.MODEL; - - if (this.role !== newRole) { - this.role = newRole; - console.info(`Server running in ${newRole === ServerRole.ROUTER ? 'ROUTER' : 'MODEL'} mode`); - } - } } export const serverStore = new ServerStore(); diff --git a/tools/ui/src/lib/stores/settings.svelte.ts b/tools/ui/src/lib/stores/settings/index.svelte.ts similarity index 89% rename from tools/ui/src/lib/stores/settings.svelte.ts rename to tools/ui/src/lib/stores/settings/index.svelte.ts index f23f6953a..0373ade42 100644 --- a/tools/ui/src/lib/stores/settings.svelte.ts +++ b/tools/ui/src/lib/stores/settings/index.svelte.ts @@ -1,34 +1,10 @@ /** * settingsStore - Application configuration and theme management * - * This store manages all application settings including AI model parameters, UI preferences, - * and theme configuration. It provides persistent storage through localStorage with reactive - * state management using Svelte 5 runes. - * - * **Architecture & Relationships:** - * - **settingsStore** (this class): Configuration state management - * - Manages AI model parameters (temperature, max tokens, etc.) - * - Handles theme switching and persistence - * - Provides localStorage synchronization - * - Offers reactive configuration access - * - * - **ChatService**: Reads model parameters for API requests - * - **UI Components**: Subscribe to theme and configuration changes - * - * **Key Features:** - * - **Model Parameters**: Temperature, max tokens, top-p, top-k, repeat penalty - * - **Theme Management**: Auto, light, dark theme switching - * - **Persistence**: Automatic localStorage synchronization - * - **Reactive State**: Svelte 5 runes for automatic UI updates - * - **Default Handling**: Graceful fallback to defaults for missing settings - * - **Batch Updates**: Efficient multi-setting updates - * - **Reset Functionality**: Restore defaults for individual or all settings - * - * **Configuration Categories:** - * - Generation parameters (temperature, tokens, sampling) - * - UI preferences (theme, display options) - * - System settings (model selection, prompts) - * - Advanced options (seed, penalties, context handling) + * Owns generation parameters, UI preferences and theme, persisted to + * localStorage with Svelte 5 runes. Applies the admin's server ui_settings + * as defaults on first visit; sampling parameters sync with the server via + * ParameterSyncService. */ import { browser } from '$app/environment'; @@ -53,14 +29,6 @@ import { import { setMode } from 'mode-watcher'; class SettingsStore { - /** - * - * - * State - * - * - */ - config = $state({ ...SETTING_CONFIG_DEFAULT }); isInitialized = $state(false); userOverrides = $state>(new Set()); @@ -69,29 +37,182 @@ class SettingsStore { // application of server ui_settings defaults for new users. private isFirstVisit = false; + canSyncParameter(key: string): boolean { + return ParameterSyncService.canSyncParameter(key); + } /** - * - * - * Utilities (private helpers) - * - * + * Clear all user overrides (for debugging) */ - - /** - * Helper method to get server defaults with null safety - * Centralizes the pattern of getting and extracting server defaults - */ - private getServerDefaults(): Record { - return ParameterSyncService.extractServerDefaults(serverStore.defaultParams); + clearAllUserOverrides(): void { + this.userOverrides.clear(); + this.saveConfig(); + console.log('Cleared all user overrides'); } /** - * - * - * Lifecycle - * - * + * Export all settings as a versioned JSON-compatible object. + * The export captures the full config (excluding sensitive values like API key) + * and user overrides. Sensitive fields are filtered out for security by default. + * @param includeSensitiveData - If true, include sensitive fields (apiKey, MCP server headers) in export */ + exportSettings(includeSensitiveData: boolean = false): SettingsExportType { + // Build config excluding sensitive data unless user opts in + const configToExport: Record = + includeSensitiveData + ? { ...this.config } + : Object.fromEntries(Object.entries(this.config).filter(([key]) => key !== 'apiKey')); + + // Handle MCP servers: exclude custom headers unless user opts in + if ('mcpServers' in configToExport && !includeSensitiveData) { + try { + const mcpServers = JSON.parse(configToExport.mcpServers as string) as Array< + Record + >; + const safeServers = mcpServers.map((server) => { + delete server.headers; + + return server; + }); + + configToExport.mcpServers = JSON.stringify(safeServers); + } catch { + // If parsing fails, just exclude the entire mcpServers field + delete (configToExport as Record).mcpServers; + } + } + + return { + config: configToExport, + timestamp: Date.now(), + userOverrides: Array.from(this.userOverrides), + version: 1 + }; + } + + /** + * Reset all parameters to their default values (from props) + * This is used by the "Reset to Default" functionality + * Prioritizes Server defaults from /props, falls back to UI defaults + */ + forceSyncWithServerDefaults(): void { + const propsDefaults = this.getServerDefaults(); + const uiSettings = serverStore.uiSettings; + + for (const key of ParameterSyncService.getSyncableParameterKeys()) { + if (uiSettings && key in uiSettings) { + // UI setting from admin config: write actual value + setConfigValue(this.config, key, uiSettings[key]); + } else if (propsDefaults[key] !== undefined) { + // sampling param: clear it, let server decide + setConfigValue(this.config, key, ''); + } else if (key in SETTING_CONFIG_DEFAULT) { + setConfigValue(this.config, key, getConfigValue(SETTING_CONFIG_DEFAULT, key)); + } + + this.userOverrides.delete(key); + } + + // Non-syncable keys: reset is a full return to the instance state, the + // admin baseline value when defined, the factory default otherwise. + for (const key of Object.keys(SETTING_CONFIG_DEFAULT)) { + if (ParameterSyncService.canSyncParameter(key)) { + continue; + } + + const value = + uiSettings && key in uiSettings && uiSettings[key] !== undefined + ? uiSettings[key] + : getConfigValue(SETTING_CONFIG_DEFAULT, key); + + setConfigValue(this.config, key, value); + + if (key === SETTINGS_KEYS.THEME) { + setMode(value as ColorMode); + } + + this.userOverrides.delete(key); + } + + this.saveConfig(); + } + + /** + * Get the entire configuration object + * @returns The complete configuration object + */ + getAllConfig(): SettingsConfigType { + return { ...this.config }; + } + + /** + * Get a specific configuration value + * @param key - The configuration key to get + * @returns The configuration value + */ + getConfig(key: K): SettingsConfigType[K] { + return this.config[key]; + } + + /** + * Get diff between current settings and server defaults + */ + getParameterDiff() { + const serverDefaults = this.getServerDefaults(); + + if (Object.keys(serverDefaults).length === 0) return {}; + + const configAsRecord = configToParameterRecord( + this.config, + ParameterSyncService.getSyncableParameterKeys() + ); + + return ParameterSyncService.createParameterDiff(configAsRecord, serverDefaults); + } + + /** + * Get parameter information including source for a specific parameter + */ + getParameterInfo(key: string) { + const propsDefaults = this.getServerDefaults(); + const currentValue = getConfigValue(this.config, key); + + return ParameterSyncService.getParameterInfo( + key, + currentValue ?? '', + propsDefaults, + this.userOverrides + ); + } + + /** + * Import settings from a previously exported object. + * Restores config (including theme) and user overrides. + * @param data - The exported settings object + */ + importSettings(data: SettingsExportType): void { + if (!browser) return; + + if (!data || !data.config) { + throw new Error('Invalid settings data: missing config'); + } + + // Restore config (theme is included in config) + this.config = { + ...SETTING_CONFIG_DEFAULT, + ...data.config + }; + + // Restore user overrides (derived state — may be stale if server defaults differ) + this.userOverrides = new Set(data.userOverrides ?? []); + + // Persist to localStorage + this.saveConfig(); + + // Apply theme for immediate visual feedback + setMode(this.config[SETTINGS_KEYS.THEME] as ColorMode); + + console.log('Settings imported successfully'); + } /** * Initialize the settings store by loading from localStorage. @@ -111,6 +232,201 @@ class SettingsStore { } } + /** + * Reset all settings to defaults. + */ + resetAll() { + this.resetConfig(); + + this.resetTheme(); + } + + /** + * Reset configuration to defaults + */ + resetConfig() { + this.config = { ...SETTING_CONFIG_DEFAULT }; + + this.saveConfig(); + } + + /** + * Reset a parameter to Server default (or UI default if no Server default) + */ + resetParameterToServerDefault(key: string): void { + const serverDefaults = this.getServerDefaults(); + const uiSettings = serverStore.uiSettings; + + if (uiSettings && key in uiSettings) { + // UI setting from admin config: write actual value + setConfigValue(this.config, key, uiSettings[key]); + } else if (serverDefaults[key] !== undefined) { + // sampling param known by server: clear it, let server decide + setConfigValue(this.config, key, ''); + } else if (key in SETTING_CONFIG_DEFAULT) { + setConfigValue(this.config, key, getConfigValue(SETTING_CONFIG_DEFAULT, key)); + } + + this.userOverrides.delete(key); + this.saveConfig(); + } + + /** + * Reset theme to default value. + * Theme is now stored inside the config object. + */ + resetTheme() { + this.updateConfig(SETTINGS_KEYS.THEME, SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.THEME]); + + setMode(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.THEME] as ColorMode); + } + + /** + * Initialize settings with props defaults when server properties are first loaded + * This sets up the default values from /props endpoint + */ + syncWithServerDefaults(): void { + const propsDefaults = this.getServerDefaults(); + + if (Object.keys(propsDefaults).length === 0) return; + + const uiSettings = serverStore.uiSettings; + const uiSettingsKeys = new Set(uiSettings ? Object.keys(uiSettings) : []); + + for (const [key, propsValue] of Object.entries(propsDefaults)) { + const currentValue = getConfigValue(this.config, key); + const normalizedCurrent = normalizeFloatingPoint(currentValue); + const normalizedDefault = normalizeFloatingPoint(propsValue); + + // if user value matches server, it's not a real override + if (normalizedCurrent === normalizedDefault) { + this.userOverrides.delete(key); + + if (!uiSettingsKeys.has(key) && getConfigValue(SETTING_CONFIG_DEFAULT, key) === undefined) { + setConfigValue(this.config, key, undefined); + } + } + } + + // UI settings are the admin's defaults for new users: applied once on + // the first visit, never on later loads, so the user's config can + // diverge. "Reset to Default" is the explicit way back to the baseline. + // A first visit config carries factory values only, so a key that + // already diverges here was set by the user before the baseline could + // be reached, through the API key splash, and stays theirs. + if (uiSettings && this.isFirstVisit) { + this.isFirstVisit = false; + + for (const [key, value] of Object.entries(uiSettings)) { + if (value === undefined || this.userOverrides.has(key)) continue; + + if (getConfigValue(this.config, key) !== getConfigValue(SETTING_CONFIG_DEFAULT, key)) { + continue; + } + + setConfigValue(this.config, key, value); + + // theme lives in mode-watcher, not just in config -> propagate + if (key === SETTINGS_KEYS.THEME) { + setMode(value as ColorMode); + } + } + } + + this.saveConfig(); + console.log('User overrides after sync:', Array.from(this.userOverrides)); + } + + /** + * Update a specific configuration setting + * @param key - The configuration key to update + * @param value - The new value for the configuration key + */ + updateConfig(key: K, value: SettingsConfigType[K]): void { + this.config[key] = value; + + if (ParameterSyncService.canSyncParameter(key as string)) { + const propsDefaults = this.getServerDefaults(); + const propsDefault = propsDefaults[key as string]; + + if (propsDefault !== undefined) { + const normalizedValue = normalizeFloatingPoint(value); + const normalizedDefault = normalizeFloatingPoint(propsDefault); + + if (normalizedValue === normalizedDefault) { + this.userOverrides.delete(key as string); + } else { + this.userOverrides.add(key as string); + } + } + } + + this.saveConfig(); + } + + /** + * + * + * Import / Export + * + * + */ + + /** + * Update multiple configuration settings at once + * @param updates - Object containing the configuration updates + */ + updateMultipleConfig(updates: Partial) { + Object.assign(this.config, updates); + + const propsDefaults = this.getServerDefaults(); + + for (const [key, value] of Object.entries(updates)) { + if (ParameterSyncService.canSyncParameter(key)) { + const propsDefault = propsDefaults[key]; + + if (propsDefault !== undefined) { + const normalizedValue = normalizeFloatingPoint(value); + const normalizedDefault = normalizeFloatingPoint(propsDefault); + + if (normalizedValue === normalizedDefault) { + this.userOverrides.delete(key); + } else { + this.userOverrides.add(key); + } + } + } + } + + this.saveConfig(); + } + + /** + * Update the theme setting. + * @param newTheme - The new theme value + */ + updateTheme(newTheme: string) { + this.updateConfig(SETTINGS_KEYS.THEME, newTheme); + + setMode(newTheme as ColorMode); + } + + /** + * + * + * Utilities (private helpers) + * + * + */ + + /** + * Helper method to get server defaults with null safety + * Centralizes the pattern of getting and extracting server defaults + */ + private getServerDefaults(): Record { + return ParameterSyncService.extractServerDefaults(serverStore.defaultParams); + } + /** * Load configuration from localStorage * Returns default values for missing keys to prevent breaking changes @@ -171,69 +487,6 @@ class SettingsStore { setMode(legacyTheme as ColorMode); } } - /** - * - * - * Config Updates - * - * - */ - - /** - * Update a specific configuration setting - * @param key - The configuration key to update - * @param value - The new value for the configuration key - */ - updateConfig(key: K, value: SettingsConfigType[K]): void { - this.config[key] = value; - - if (ParameterSyncService.canSyncParameter(key as string)) { - const propsDefaults = this.getServerDefaults(); - const propsDefault = propsDefaults[key as string]; - - if (propsDefault !== undefined) { - const normalizedValue = normalizeFloatingPoint(value); - const normalizedDefault = normalizeFloatingPoint(propsDefault); - - if (normalizedValue === normalizedDefault) { - this.userOverrides.delete(key as string); - } else { - this.userOverrides.add(key as string); - } - } - } - - this.saveConfig(); - } - - /** - * Update multiple configuration settings at once - * @param updates - Object containing the configuration updates - */ - updateMultipleConfig(updates: Partial) { - Object.assign(this.config, updates); - - const propsDefaults = this.getServerDefaults(); - - for (const [key, value] of Object.entries(updates)) { - if (ParameterSyncService.canSyncParameter(key)) { - const propsDefault = propsDefaults[key]; - - if (propsDefault !== undefined) { - const normalizedValue = normalizeFloatingPoint(value); - const normalizedDefault = normalizeFloatingPoint(propsDefault); - - if (normalizedValue === normalizedDefault) { - this.userOverrides.delete(key); - } else { - this.userOverrides.add(key); - } - } - } - } - - this.saveConfig(); - } /** * Save the current configuration to localStorage @@ -252,331 +505,6 @@ class SettingsStore { console.error('Failed to save config to localStorage:', error); } } - - /** - * Update the theme setting. - * @param newTheme - The new theme value - */ - updateTheme(newTheme: string) { - this.updateConfig(SETTINGS_KEYS.THEME, newTheme); - - setMode(newTheme as ColorMode); - } - - /** - * - * - * Reset - * - * - */ - - /** - * Reset configuration to defaults - */ - resetConfig() { - this.config = { ...SETTING_CONFIG_DEFAULT }; - - this.saveConfig(); - } - - /** - * Reset theme to default value. - * Theme is now stored inside the config object. - */ - resetTheme() { - this.updateConfig(SETTINGS_KEYS.THEME, SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.THEME]); - - setMode(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.THEME] as ColorMode); - } - - /** - * Reset all settings to defaults. - */ - resetAll() { - this.resetConfig(); - - this.resetTheme(); - } - - /** - * Reset a parameter to Server default (or UI default if no Server default) - */ - resetParameterToServerDefault(key: string): void { - const serverDefaults = this.getServerDefaults(); - const uiSettings = serverStore.uiSettings; - - if (uiSettings && key in uiSettings) { - // UI setting from admin config: write actual value - setConfigValue(this.config, key, uiSettings[key]); - } else if (serverDefaults[key] !== undefined) { - // sampling param known by server: clear it, let server decide - setConfigValue(this.config, key, ''); - } else if (key in SETTING_CONFIG_DEFAULT) { - setConfigValue(this.config, key, getConfigValue(SETTING_CONFIG_DEFAULT, key)); - } - - this.userOverrides.delete(key); - this.saveConfig(); - } - - /** - * - * - * Server Sync - * - * - */ - - /** - * Initialize settings with props defaults when server properties are first loaded - * This sets up the default values from /props endpoint - */ - syncWithServerDefaults(): void { - const propsDefaults = this.getServerDefaults(); - - if (Object.keys(propsDefaults).length === 0) return; - - const uiSettings = serverStore.uiSettings; - const uiSettingsKeys = new Set(uiSettings ? Object.keys(uiSettings) : []); - - for (const [key, propsValue] of Object.entries(propsDefaults)) { - const currentValue = getConfigValue(this.config, key); - const normalizedCurrent = normalizeFloatingPoint(currentValue); - const normalizedDefault = normalizeFloatingPoint(propsValue); - - // if user value matches server, it's not a real override - if (normalizedCurrent === normalizedDefault) { - this.userOverrides.delete(key); - - if (!uiSettingsKeys.has(key) && getConfigValue(SETTING_CONFIG_DEFAULT, key) === undefined) { - setConfigValue(this.config, key, undefined); - } - } - } - - // UI settings are the admin's defaults for new users: applied once on - // the first visit, never on later loads, so the user's config can - // diverge. "Reset to Default" is the explicit way back to the baseline. - // A first visit config carries factory values only, so a key that - // already diverges here was set by the user before the baseline could - // be reached, through the API key splash, and stays theirs. - if (uiSettings && this.isFirstVisit) { - this.isFirstVisit = false; - - for (const [key, value] of Object.entries(uiSettings)) { - if (value === undefined || this.userOverrides.has(key)) continue; - - if (getConfigValue(this.config, key) !== getConfigValue(SETTING_CONFIG_DEFAULT, key)) { - continue; - } - - setConfigValue(this.config, key, value); - - // theme lives in mode-watcher, not just in config -> propagate - if (key === SETTINGS_KEYS.THEME) { - setMode(value as ColorMode); - } - } - } - - this.saveConfig(); - console.log('User overrides after sync:', Array.from(this.userOverrides)); - } - - /** - * Reset all parameters to their default values (from props) - * This is used by the "Reset to Default" functionality - * Prioritizes Server defaults from /props, falls back to UI defaults - */ - forceSyncWithServerDefaults(): void { - const propsDefaults = this.getServerDefaults(); - const uiSettings = serverStore.uiSettings; - - for (const key of ParameterSyncService.getSyncableParameterKeys()) { - if (uiSettings && key in uiSettings) { - // UI setting from admin config: write actual value - setConfigValue(this.config, key, uiSettings[key]); - } else if (propsDefaults[key] !== undefined) { - // sampling param: clear it, let server decide - setConfigValue(this.config, key, ''); - } else if (key in SETTING_CONFIG_DEFAULT) { - setConfigValue(this.config, key, getConfigValue(SETTING_CONFIG_DEFAULT, key)); - } - - this.userOverrides.delete(key); - } - - // Non-syncable keys: reset is a full return to the instance state, the - // admin baseline value when defined, the factory default otherwise. - for (const key of Object.keys(SETTING_CONFIG_DEFAULT)) { - if (ParameterSyncService.canSyncParameter(key)) { - continue; - } - - const value = - uiSettings && key in uiSettings && uiSettings[key] !== undefined - ? uiSettings[key] - : getConfigValue(SETTING_CONFIG_DEFAULT, key); - - setConfigValue(this.config, key, value); - - if (key === SETTINGS_KEYS.THEME) { - setMode(value as ColorMode); - } - - this.userOverrides.delete(key); - } - - this.saveConfig(); - } - - /** - * - * - * Utilities - * - * - */ - - /** - * Get a specific configuration value - * @param key - The configuration key to get - * @returns The configuration value - */ - getConfig(key: K): SettingsConfigType[K] { - return this.config[key]; - } - - /** - * Get the entire configuration object - * @returns The complete configuration object - */ - getAllConfig(): SettingsConfigType { - return { ...this.config }; - } - - canSyncParameter(key: string): boolean { - return ParameterSyncService.canSyncParameter(key); - } - - /** - * Get parameter information including source for a specific parameter - */ - getParameterInfo(key: string) { - const propsDefaults = this.getServerDefaults(); - const currentValue = getConfigValue(this.config, key); - - return ParameterSyncService.getParameterInfo( - key, - currentValue ?? '', - propsDefaults, - this.userOverrides - ); - } - - /** - * Get diff between current settings and server defaults - */ - getParameterDiff() { - const serverDefaults = this.getServerDefaults(); - - if (Object.keys(serverDefaults).length === 0) return {}; - - const configAsRecord = configToParameterRecord( - this.config, - ParameterSyncService.getSyncableParameterKeys() - ); - - return ParameterSyncService.createParameterDiff(configAsRecord, serverDefaults); - } - - /** - * Clear all user overrides (for debugging) - */ - clearAllUserOverrides(): void { - this.userOverrides.clear(); - this.saveConfig(); - console.log('Cleared all user overrides'); - } - - /** - * - * - * Import / Export - * - * - */ - - /** - * Export all settings as a versioned JSON-compatible object. - * The export captures the full config (excluding sensitive values like API key) - * and user overrides. Sensitive fields are filtered out for security by default. - * @param includeSensitiveData - If true, include sensitive fields (apiKey, MCP server headers) in export - */ - exportSettings(includeSensitiveData: boolean = false): SettingsExportType { - // Build config excluding sensitive data unless user opts in - const configToExport: Record = - includeSensitiveData - ? { ...this.config } - : Object.fromEntries(Object.entries(this.config).filter(([key]) => key !== 'apiKey')); - - // Handle MCP servers: exclude custom headers unless user opts in - if ('mcpServers' in configToExport && !includeSensitiveData) { - try { - const mcpServers = JSON.parse(configToExport.mcpServers as string) as Array< - Record - >; - const safeServers = mcpServers.map((server) => { - delete server.headers; - - return server; - }); - - configToExport.mcpServers = JSON.stringify(safeServers); - } catch { - // If parsing fails, just exclude the entire mcpServers field - delete (configToExport as Record).mcpServers; - } - } - - return { - config: configToExport, - timestamp: Date.now(), - userOverrides: Array.from(this.userOverrides), - version: 1 - }; - } - - /** - * Import settings from a previously exported object. - * Restores config (including theme) and user overrides. - * @param data - The exported settings object - */ - importSettings(data: SettingsExportType): void { - if (!browser) return; - - if (!data || !data.config) { - throw new Error('Invalid settings data: missing config'); - } - - // Restore config (theme is included in config) - this.config = { - ...SETTING_CONFIG_DEFAULT, - ...data.config - }; - - // Restore user overrides (derived state — may be stale if server defaults differ) - this.userOverrides = new Set(data.userOverrides ?? []); - - // Persist to localStorage - this.saveConfig(); - - // Apply theme for immediate visual feedback - setMode(this.config[SETTINGS_KEYS.THEME] as ColorMode); - - console.log('Settings imported successfully'); - } } export const settingsStore = new SettingsStore(); diff --git a/tools/ui/src/lib/stores/settings-referrer.svelte.ts b/tools/ui/src/lib/stores/settings/referrer.svelte.ts similarity index 50% rename from tools/ui/src/lib/stores/settings-referrer.svelte.ts rename to tools/ui/src/lib/stores/settings/referrer.svelte.ts index 297a0d6a4..9679049df 100644 --- a/tools/ui/src/lib/stores/settings-referrer.svelte.ts +++ b/tools/ui/src/lib/stores/settings/referrer.svelte.ts @@ -1,3 +1,10 @@ +/** + * settingsReferrer - Remembers the settings route to return to after exit + * + * Tracks the last settings section the user was on so the app can return + * there after a fallback exit. Standalone reactive value, no host. + */ + import { SETTINGS_FALLBACK_EXIT_ROUTE } from '$lib/constants'; let _url = $state(SETTINGS_FALLBACK_EXIT_ROUTE); diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.ts index 9f044c83e..e255b8a43 100644 --- a/tools/ui/src/lib/stores/tools.svelte.ts +++ b/tools/ui/src/lib/stores/tools.svelte.ts @@ -1,3 +1,12 @@ +/** + * toolsStore - Tool registry and enablement + * + * Owns the server tool listing (with working-directory resolution), built-in + * browser tools, MCP tools and per-tool enablement, exposed as a unified + * tool set for the LLM and the tools UI. Consumed by the agentic loop and + * the chat flows. + */ + import { browser } from '$app/environment'; import { buildBrowserInfoToolDefinition, @@ -18,9 +27,9 @@ import { } from '$lib/enums'; import { ToolsService } from '$lib/services/tools.service'; // direct imports between stores, not via the barrel, to avoid circular deps -import { mcpStore } from '$lib/stores/mcp.svelte'; -import { modelsStore } from '$lib/stores/models.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { mcpStore } from '$lib/stores/mcp/index.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { OpenAIToolDefinition, ToolEntry, ToolGroup } from '$lib/types'; import { buildSandboxToolDefinition } from '$lib/utils'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; @@ -28,273 +37,18 @@ import { SvelteMap, SvelteSet } from 'svelte/reactivity'; /** Stable selection identity for a tool, shared by the disabled set and the permission store */ class ToolsStore { - private _serverTools = $state([]); - private _loading = $state(false); - private _error = $state(null); private _disabledTools = $state(new SvelteSet()); + private _error = $state(null); + private _loading = $state(false); + private _serverHome = $state(undefined); + private _serverTools = $state([]); + private _toolsEndpointUnreachable = $state(false); // server tools that resolve their paths against the working directory, // as declared by the server in its `/tools` listing - private _cwdAwareTools = $state(new SvelteSet()); - private _toolsEndpointUnreachable = $state(false); - private _serverHome = $state(undefined); + private cwdAwareTools = $state(new SvelteSet()); - /** - * Load persisted disabled tools and fetch the builtin tool list. - * Called by initStores() after migrations have run. - */ - initialize(): void { - // browser-only init: skip on SSR to avoid localStorage/fetch side effects - if (!browser) return; - - try { - const stored = localStorage.getItem(DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY); - - if (stored) { - const parsed = JSON.parse(stored); - - if (Array.isArray(parsed)) { - for (const key of parsed) { - if (typeof key === 'string') this._disabledTools.add(key); - } - } - } - } catch (err) { - console.error('[ToolsStore] Failed to load disabled tools from localStorage:', err); - } - - this.fetchServerTools(); - } - - private persistDisabledTools(): void { - try { - localStorage.setItem( - DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY, - JSON.stringify([...this._disabledTools]) - ); - } catch { - // ignore storage errors - } - } - - private toolKey(source: ToolSource, name: string, serverId?: string): string { - switch (source) { - case ToolSource.MCP: - return serverId ? `mcp-${serverId}:${name}` : `mcp:${name}`; - case ToolSource.CUSTOM: - return `custom:${name}`; - case ToolSource.BROWSER: - return `browser:${name}`; - default: - return `server:${name}`; - } - } - - private inferTypeFromDefault(value: unknown): string | undefined { - if (typeof value === 'string') return 'string'; - - if (typeof value === 'boolean') return 'boolean'; - - if (typeof value === 'number') return Number.isInteger(value) ? 'integer' : 'number'; - - if (Array.isArray(value)) return 'array'; - - if (value !== null && typeof value === 'object') return 'object'; - - return undefined; - } - - /** - * Recursively normalize a JSON Schema object: infers `type` from `default` - * for properties / items that omit it, and descends into nested `properties` - * and `items`. Returns a new object -- does not mutate the input. - */ - private normalizeJsonSchema(schema: Record): Record { - if (!schema || typeof schema !== 'object') return schema; - - const normalized: Record = { ...schema }; - - if (normalized.properties && typeof normalized.properties === 'object') { - const props = normalized.properties as Record>; - const normalizedProps: Record> = {}; - - for (const [key, prop] of Object.entries(props)) { - if (!prop || typeof prop !== 'object') { - normalizedProps[key] = prop; - - continue; - } - - const normalizedProp: Record = { ...prop }; - - if (!normalizedProp.type && normalizedProp.default !== undefined) { - const inferred = this.inferTypeFromDefault(normalizedProp.default); - - if (inferred) normalizedProp.type = inferred; - } - - if (normalizedProp.properties) { - Object.assign( - normalizedProp, - this.normalizeJsonSchema(normalizedProp as Record) - ); - } - - if (normalizedProp.items && typeof normalizedProp.items === 'object') { - normalizedProp.items = this.normalizeJsonSchema( - normalizedProp.items as Record - ); - } - - normalizedProps[key] = normalizedProp; - } - normalized.properties = normalizedProps; - } - - return normalized; - } - - private mcpDefinition( - name: string, - description: string | undefined, - schema?: Record - ): OpenAIToolDefinition { - return { - function: { - description, - name, - parameters: schema ?? { properties: {}, required: [], type: JsonSchemaType.OBJECT } - }, - type: ToolCallType.FUNCTION - }; - } - - get serverTools(): OpenAIToolDefinition[] { - return this._serverTools; - } - - get serverHome(): string | null { - return this._serverHome ?? null; - } - - get mcpTools(): OpenAIToolDefinition[] { - return this.mcpEntries().map((e) => e.definition); - } - - get browserTools(): OpenAIToolDefinition[] { - const tools: OpenAIToolDefinition[] = [buildGetDatetimeToolDefinition()]; - - if (settingsStore.config.jsSandboxEnabled) { - tools.push(buildSandboxToolDefinition(!!settingsStore.config.symbolicMathEnabled)); - } - - const readMedia = this.readMediaTool(); - - if (readMedia) tools.push(readMedia); - - // provide browser's get_info tool if server doesn't provide one - if (!this.hasServerTool(BuiltInTool.SERVER_GET_INFO)) { - tools.push(buildBrowserInfoToolDefinition()); - } - - return tools; - } - - private hasServerTool(name: BuiltInTool): boolean { - return this._serverTools.some((def) => def.function.name === name); - } - - /** - * `read_media` runs in the browser on top of the server's `read_file`, so it - * exists only when that tool is served and the active model can perceive the - * bytes. The server cannot make this call - it does not know which model the - * conversation uses. - */ - private readMediaTool(): OpenAIToolDefinition | null { - if (!this.hasServerTool(BuiltInTool.SERVER_READ_FILE)) return null; - - const model = modelsStore.selectedModelName ?? modelsStore.models[0]?.model ?? ''; - - if (!model) return null; - - const vision = modelsStore.modelSupportsVision(model); - const audio = modelsStore.modelSupportsAudio(model); - - if (!vision && !audio) return null; - - return buildReadMediaToolDefinition(vision, audio); - } - - get customTools(): OpenAIToolDefinition[] { - const raw = settingsStore.config.customJson; - - if (!raw || typeof raw !== 'string') return []; - - try { - const parsed = JSON.parse(raw); - - if (!Array.isArray(parsed)) return []; - - return parsed.filter( - (t: unknown): t is OpenAIToolDefinition => - typeof t === 'object' && - t !== null && - 'type' in t && - (t as OpenAIToolDefinition).type === 'function' && - 'function' in t && - typeof (t as OpenAIToolDefinition).function?.name === 'string' - ); - } catch { - return []; - } - } - - /** Normalize MCP tools from live connections when available, fall back to health check data */ - private mcpEntries(): { - serverId: string; - serverName: string; - definition: OpenAIToolDefinition; - }[] { - const out: { serverId: string; serverName: string; definition: OpenAIToolDefinition }[] = []; - const connections = mcpStore.getConnections(); - - if (connections.size > 0) { - for (const [serverId, connection] of connections) { - const serverName = mcpStore.getServerDisplayName(serverId); - - for (const tool of connection.tools) { - const rawSchema = (tool.inputSchema as Record) ?? { - properties: {}, - required: [], - type: JsonSchemaType.OBJECT - }; - - out.push({ - definition: { - function: { - description: tool.description, - name: tool.name, - parameters: this.normalizeJsonSchema(rawSchema) - }, - type: ToolCallType.FUNCTION - }, - serverId, - serverName - }); - } - } - } else { - for (const { serverId, serverName, tools } of this.getMcpToolsFromHealthChecks()) { - for (const tool of tools) { - out.push({ - definition: this.mcpDefinition(tool.name, tool.description), - serverId, - serverName - }); - } - } - } - - return out; + get allToolDefinitions(): OpenAIToolDefinition[] { + return this.allTools.map((t) => t.definition); } /** Canonical flat list of tool entries with source metadata and stable keys, deduped by key */ @@ -353,6 +107,97 @@ class ToolsStore { return entries; } + get browserTools(): OpenAIToolDefinition[] { + const tools: OpenAIToolDefinition[] = [buildGetDatetimeToolDefinition()]; + + if (settingsStore.config.jsSandboxEnabled) { + tools.push(buildSandboxToolDefinition(!!settingsStore.config.symbolicMathEnabled)); + } + + const readMedia = this.readMediaTool(); + + if (readMedia) tools.push(readMedia); + + // provide browser's get_info tool if server doesn't provide one + if (!this.hasServerTool(BuiltInTool.SERVER_GET_INFO)) { + tools.push(buildBrowserInfoToolDefinition()); + } + + return tools; + } + + get customTools(): OpenAIToolDefinition[] { + const raw = settingsStore.config.customJson; + + if (!raw || typeof raw !== 'string') return []; + + try { + const parsed = JSON.parse(raw); + + if (!Array.isArray(parsed)) return []; + + return parsed.filter( + (t: unknown): t is OpenAIToolDefinition => + typeof t === 'object' && + t !== null && + 'type' in t && + (t as OpenAIToolDefinition).type === 'function' && + 'function' in t && + typeof (t as OpenAIToolDefinition).function?.name === 'string' + ); + } catch { + return []; + } + } + + get disabledTools(): SvelteSet { + return this._disabledTools; + } + + get error(): string | null { + return this._error; + } + + /** + * Check if a working directory is worth setting: at least one server tool + * that reads it is both served and left enabled by the user. + */ + get hasEnabledCwdTools(): boolean { + return this._serverTools.some((def) => { + const name = def.function.name; + + return ( + this.cwdAwareTools.has(name) && + !this._disabledTools.has(this.toolKey(ToolSource.SERVER, name)) + ); + }); + } + + /** Check if there are any enabled tools available (server, MCP, or custom) */ + get hasEnabledTools(): boolean { + return this.getEnabledToolsForLLM().length > 0; + } + + get isToolsEndpointUnreachable(): boolean { + return this._toolsEndpointUnreachable; + } + + get loading(): boolean { + return this._loading; + } + + get mcpTools(): OpenAIToolDefinition[] { + return this.mcpEntries().map((e) => e.definition); + } + + get serverHome(): string | null { + return this._serverHome ?? null; + } + + get serverTools(): OpenAIToolDefinition[] { + return this._serverTools; + } + /** Tools grouped by category for tree display, derived from the canonical entries */ get toolGroups(): ToolGroup[] { const groups: ToolGroup[] = []; @@ -382,16 +227,47 @@ class ToolsStore { return groups; } - private groupLabel(entry: ToolEntry): string { - switch (entry.source) { - case ToolSource.MCP: - return entry.serverName ?? ''; - case ToolSource.CUSTOM: - return TOOL_GROUP_LABELS[ToolSource.CUSTOM]; - case ToolSource.BROWSER: - return TOOL_GROUP_LABELS[ToolSource.BROWSER]; - default: - return TOOL_GROUP_LABELS[ToolSource.SERVER]; + /** Enable all tools belonging to a specific MCP server */ + enableAllToolsForServer(serverId: string): void { + const connection = mcpStore.getConnections().get(serverId); + + if (!connection) return; + + for (const tool of connection.tools) { + this._disabledTools.delete(this.toolKey(ToolSource.MCP, tool.name, serverId)); + } + this.persistDisabledTools(); + } + + async fetchServerTools(): Promise { + if (this._loading) return; + + this._loading = true; + this._error = null; + this._toolsEndpointUnreachable = false; + + try { + const toolInfos = await ToolsService.list(); + + this._serverTools = toolInfos.map((info) => info.definition); + this.cwdAwareTools = new SvelteSet( + toolInfos.filter((info) => info.uses_cwd).map((info) => info.tool) + ); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : String(err); + + this._error = errorMessage; + + // 403 from /tools means the server was started without --tools + // TODO: check status code instead of relying on message + if (errorMessage.includes('this feature is disabled')) { + this._toolsEndpointUnreachable = true; + console.info('[ToolsStore] Server tools are disabled on the server'); + } else { + console.error('[ToolsStore] Failed to fetch server tools:', err); + } + } finally { + this._loading = false; } } @@ -430,112 +306,9 @@ class ToolsStore { return result; } - get allToolDefinitions(): OpenAIToolDefinition[] { - return this.allTools.map((t) => t.definition); - } - - get loading(): boolean { - return this._loading; - } - - get error(): string | null { - return this._error; - } - - get isToolsEndpointUnreachable(): boolean { - return this._toolsEndpointUnreachable; - } - - get disabledTools(): SvelteSet { - return this._disabledTools; - } - - isToolEnabled(key: string): boolean { - return !this._disabledTools.has(key); - } - - toggleTool(key: string): void { - if (this._disabledTools.has(key)) { - this._disabledTools.delete(key); - } else { - this._disabledTools.add(key); - } - - this.persistDisabledTools(); - } - - setToolEnabled(key: string, enabled: boolean): void { - if (enabled) { - this._disabledTools.delete(key); - } else { - this._disabledTools.add(key); - } - } - - /** Enable all tools belonging to a specific MCP server */ - enableAllToolsForServer(serverId: string): void { - const connection = mcpStore.getConnections().get(serverId); - - if (!connection) return; - - for (const tool of connection.tools) { - this._disabledTools.delete(this.toolKey(ToolSource.MCP, tool.name, serverId)); - } - this.persistDisabledTools(); - } - - toggleGroup(group: ToolGroup): void { - const allEnabled = group.tools.every((t) => this.isToolEnabled(t.key)); - const target = !allEnabled; - - for (const tool of group.tools) { - if (target) this._disabledTools.delete(tool.key); - else this._disabledTools.add(tool.key); - } - this.persistDisabledTools(); - } - - isGroupFullyEnabled(group: ToolGroup): boolean { - return group.tools.length > 0 && group.tools.every((t) => this.isToolEnabled(t.key)); - } - - /** Get MCP tools from health check data, used when live connections aren't established yet */ - private getMcpToolsFromHealthChecks(): { - serverId: string; - serverName: string; - tools: { name: string; description?: string }[]; - }[] { - const result: ReturnType = []; - - for (const server of mcpStore.getServers()) { - if (!server.enabled) continue; - - const health = mcpStore.getHealthCheckState(server.id); - - if (health.status === HealthCheckStatus.SUCCESS && health.tools.length > 0) { - result.push({ - serverId: server.id, - serverName: mcpStore.getServerLabel(server), - tools: health.tools - }); - } - } - - return result; - } - - /** First canonical entry matching a tool name, runtime tool calls resolve by name */ - private findEntryByName(toolName: string): ToolEntry | null { - for (const entry of this.allTools) { - if (entry.definition.function.name === toolName) return entry; - } - - return null; - } - - /** Determine the source of a tool by its name */ - getToolSource(toolName: string): ToolSource | null { - return this.findEntryByName(toolName)?.source ?? null; + /** Permission key for a tool name, identical to the selection key */ + getPermissionKey(toolName: string): string | null { + return this.findEntryByName(toolName)?.key ?? null; } /** Get the display label for the server that owns a given tool */ @@ -555,61 +328,44 @@ class ToolsStore { return ''; } - /** Permission key for a tool name, identical to the selection key */ - getPermissionKey(toolName: string): string | null { - return this.findEntryByName(toolName)?.key ?? null; - } - - /** Check if there are any enabled tools available (server, MCP, or custom) */ - get hasEnabledTools(): boolean { - return this.getEnabledToolsForLLM().length > 0; + /** Determine the source of a tool by its name */ + getToolSource(toolName: string): ToolSource | null { + return this.findEntryByName(toolName)?.source ?? null; } /** - * Check if a working directory is worth setting: at least one server tool - * that reads it is both served and left enabled by the user. + * Load persisted disabled tools and fetch the builtin tool list. + * Called by initStores() after migrations have run. */ - get hasEnabledCwdTools(): boolean { - return this._serverTools.some((def) => { - const name = def.function.name; - - return ( - this._cwdAwareTools.has(name) && - !this._disabledTools.has(this.toolKey(ToolSource.SERVER, name)) - ); - }); - } - - async fetchServerTools(): Promise { - if (this._loading) return; - - this._loading = true; - this._error = null; - this._toolsEndpointUnreachable = false; + initialize(): void { + // browser-only init: skip on SSR to avoid localStorage/fetch side effects + if (!browser) return; try { - const toolInfos = await ToolsService.list(); + const stored = localStorage.getItem(DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY); - this._serverTools = toolInfos.map((info) => info.definition); - this._cwdAwareTools = new SvelteSet( - toolInfos.filter((info) => info.uses_cwd).map((info) => info.tool) - ); - } catch (err) { - const errorMessage = err instanceof Error ? err.message : String(err); + if (stored) { + const parsed = JSON.parse(stored); - this._error = errorMessage; - - // 403 from /tools means the server was started without --tools - // TODO: check status code instead of relying on message - if (errorMessage.includes('this feature is disabled')) { - this._toolsEndpointUnreachable = true; - console.info('[ToolsStore] Server tools are disabled on the server'); - } else { - console.error('[ToolsStore] Failed to fetch server tools:', err); + if (Array.isArray(parsed)) { + for (const key of parsed) { + if (typeof key === 'string') this._disabledTools.add(key); + } + } } - } finally { - this._loading = false; + } catch (err) { + console.error('[ToolsStore] Failed to load disabled tools from localStorage:', err); } + + this.fetchServerTools(); + } + + isGroupFullyEnabled(group: ToolGroup): boolean { + return group.tools.length > 0 && group.tools.every((t) => this.isToolEnabled(t.key)); + } + + isToolEnabled(key: string): boolean { + return !this._disabledTools.has(key); } /** @@ -637,6 +393,259 @@ class ToolsStore { return this._serverHome; } + + setToolEnabled(key: string, enabled: boolean): void { + if (enabled) { + this._disabledTools.delete(key); + } else { + this._disabledTools.add(key); + } + } + + toggleGroup(group: ToolGroup): void { + const allEnabled = group.tools.every((t) => this.isToolEnabled(t.key)); + const target = !allEnabled; + + for (const tool of group.tools) { + if (target) this._disabledTools.delete(tool.key); + else this._disabledTools.add(tool.key); + } + this.persistDisabledTools(); + } + + toggleTool(key: string): void { + if (this._disabledTools.has(key)) { + this._disabledTools.delete(key); + } else { + this._disabledTools.add(key); + } + + this.persistDisabledTools(); + } + + /** First canonical entry matching a tool name, runtime tool calls resolve by name */ + private findEntryByName(toolName: string): ToolEntry | null { + for (const entry of this.allTools) { + if (entry.definition.function.name === toolName) return entry; + } + + return null; + } + + /** Get MCP tools from health check data, used when live connections aren't established yet */ + private getMcpToolsFromHealthChecks(): { + serverId: string; + serverName: string; + tools: { name: string; description?: string }[]; + }[] { + const result: ReturnType = []; + + for (const server of mcpStore.getServers()) { + if (!server.enabled) continue; + + const health = mcpStore.getHealthCheckState(server.id); + + if (health.status === HealthCheckStatus.SUCCESS && health.tools.length > 0) { + result.push({ + serverId: server.id, + serverName: mcpStore.getServerLabel(server), + tools: health.tools + }); + } + } + + return result; + } + + private groupLabel(entry: ToolEntry): string { + switch (entry.source) { + case ToolSource.MCP: + return entry.serverName ?? ''; + case ToolSource.CUSTOM: + return TOOL_GROUP_LABELS[ToolSource.CUSTOM]; + case ToolSource.BROWSER: + return TOOL_GROUP_LABELS[ToolSource.BROWSER]; + default: + return TOOL_GROUP_LABELS[ToolSource.SERVER]; + } + } + + private hasServerTool(name: BuiltInTool): boolean { + return this._serverTools.some((def) => def.function.name === name); + } + + private inferTypeFromDefault(value: unknown): string | undefined { + if (typeof value === 'string') return 'string'; + + if (typeof value === 'boolean') return 'boolean'; + + if (typeof value === 'number') return Number.isInteger(value) ? 'integer' : 'number'; + + if (Array.isArray(value)) return 'array'; + + if (value !== null && typeof value === 'object') return 'object'; + + return undefined; + } + + private mcpDefinition( + name: string, + description: string | undefined, + schema?: Record + ): OpenAIToolDefinition { + return { + function: { + description, + name, + parameters: schema ?? { properties: {}, required: [], type: JsonSchemaType.OBJECT } + }, + type: ToolCallType.FUNCTION + }; + } + + /** Normalize MCP tools from live connections when available, fall back to health check data */ + private mcpEntries(): { + serverId: string; + serverName: string; + definition: OpenAIToolDefinition; + }[] { + const out: { serverId: string; serverName: string; definition: OpenAIToolDefinition }[] = []; + const connections = mcpStore.getConnections(); + + if (connections.size > 0) { + for (const [serverId, connection] of connections) { + const serverName = mcpStore.getServerDisplayName(serverId); + + for (const tool of connection.tools) { + const rawSchema = (tool.inputSchema as Record) ?? { + properties: {}, + required: [], + type: JsonSchemaType.OBJECT + }; + + out.push({ + definition: { + function: { + description: tool.description, + name: tool.name, + parameters: this.normalizeJsonSchema(rawSchema) + }, + type: ToolCallType.FUNCTION + }, + serverId, + serverName + }); + } + } + } else { + for (const { serverId, serverName, tools } of this.getMcpToolsFromHealthChecks()) { + for (const tool of tools) { + out.push({ + definition: this.mcpDefinition(tool.name, tool.description), + serverId, + serverName + }); + } + } + } + + return out; + } + + /** + * Recursively normalize a JSON Schema object: infers `type` from `default` + * for properties / items that omit it, and descends into nested `properties` + * and `items`. Returns a new object -- does not mutate the input. + */ + private normalizeJsonSchema(schema: Record): Record { + if (!schema || typeof schema !== 'object') return schema; + + const normalized: Record = { ...schema }; + + if (normalized.properties && typeof normalized.properties === 'object') { + const props = normalized.properties as Record>; + const normalizedProps: Record> = {}; + + for (const [key, prop] of Object.entries(props)) { + if (!prop || typeof prop !== 'object') { + normalizedProps[key] = prop; + + continue; + } + + const normalizedProp: Record = { ...prop }; + + if (!normalizedProp.type && normalizedProp.default !== undefined) { + const inferred = this.inferTypeFromDefault(normalizedProp.default); + + if (inferred) normalizedProp.type = inferred; + } + + if (normalizedProp.properties) { + Object.assign( + normalizedProp, + this.normalizeJsonSchema(normalizedProp as Record) + ); + } + + if (normalizedProp.items && typeof normalizedProp.items === 'object') { + normalizedProp.items = this.normalizeJsonSchema( + normalizedProp.items as Record + ); + } + + normalizedProps[key] = normalizedProp; + } + normalized.properties = normalizedProps; + } + + return normalized; + } + + private persistDisabledTools(): void { + try { + localStorage.setItem( + DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY, + JSON.stringify([...this._disabledTools]) + ); + } catch { + // ignore storage errors + } + } + + /** + * `read_media` runs in the browser on top of the server's `read_file`, so it + * exists only when that tool is served and the active model can perceive the + * bytes. The server cannot make this call - it does not know which model the + * conversation uses. + */ + private readMediaTool(): OpenAIToolDefinition | null { + if (!this.hasServerTool(BuiltInTool.SERVER_READ_FILE)) return null; + + const model = modelsStore.selectedModelName ?? modelsStore.models[0]?.model ?? ''; + + if (!model) return null; + + const vision = modelsStore.props.modelSupportsVision(model); + const audio = modelsStore.props.modelSupportsAudio(model); + + if (!vision && !audio) return null; + + return buildReadMediaToolDefinition(vision, audio); + } + + private toolKey(source: ToolSource, name: string, serverId?: string): string { + switch (source) { + case ToolSource.MCP: + return serverId ? `mcp-${serverId}:${name}` : `mcp:${name}`; + case ToolSource.CUSTOM: + return `custom:${name}`; + case ToolSource.BROWSER: + return `browser:${name}`; + default: + return `server:${name}`; + } + } } export const toolsStore = new ToolsStore(); diff --git a/tools/ui/src/lib/types/agentic.d.ts b/tools/ui/src/lib/types/agentic.d.ts index e7c6d34e1..1a604476b 100644 --- a/tools/ui/src/lib/types/agentic.d.ts +++ b/tools/ui/src/lib/types/agentic.d.ts @@ -205,7 +205,7 @@ export interface AgenticSection { /** ID of the model-side tool call (matches tool_calls[i].id). Lets * downstream consumers correlate a section with the agentic loop's * currently-executing tool, e.g. to drive live-streaming UI state - * by matching against agenticStore.executingToolCallId. */ + * by matching against agenticStore.getExecutingToolCallId. */ toolCallId?: string; wasInterrupted?: boolean; } diff --git a/tools/ui/src/lib/utils/api-fetch.ts b/tools/ui/src/lib/utils/api-fetch.ts index 65e1129de..205920004 100644 --- a/tools/ui/src/lib/utils/api-fetch.ts +++ b/tools/ui/src/lib/utils/api-fetch.ts @@ -1,7 +1,6 @@ import { getAuthHeaders, getJsonHeaders } from './api-headers'; import { base } from '$app/paths'; -import { ERROR_MESSAGES, HTTP_CODE_TO_STRING } from '$lib/constants'; -import { UrlProtocol } from '$lib/enums'; +import { API_ABSOLUTE_URL_PROTOCOLS, ERROR_MESSAGES, HTTP_CODE_TO_STRING } from '$lib/constants'; /** * API Fetch Utilities @@ -63,10 +62,8 @@ export async function apiFetch(path: string, options: ApiFetchOptions = {}): const { authOnly = false, headers: customHeaders, ...fetchOptions } = options; const baseHeaders = authOnly ? getAuthHeaders() : getJsonHeaders(); const headers = { ...baseHeaders, ...customHeaders }; - const url = - path.startsWith(UrlProtocol.HTTP) || path.startsWith(UrlProtocol.HTTPS) - ? path - : `${base}${path}`; + // absolute URLs with an allowed protocol pass through untouched; relative paths get the base prefix + const url = API_ABSOLUTE_URL_PROTOCOLS.some((p) => path.startsWith(p)) ? path : `${base}${path}`; let response; @@ -117,28 +114,7 @@ export async function apiFetchWithParams( } } - const { authOnly = false, headers: customHeaders, ...fetchOptions } = options; - const baseHeaders = authOnly ? getAuthHeaders() : getJsonHeaders(); - const headers = { ...baseHeaders, ...customHeaders }; - - let response; - - try { - response = await fetch(url.toString(), { - ...fetchOptions, - headers - }); - } catch (e) { - throw new Error(beautifyNetworkError(e)); - } - - if (!response.ok) { - const errorMessage = await parseErrorMessage(response); - - throw new ApiError(errorMessage, response.status); - } - - return response.json() as Promise; + return apiFetch(url.toString(), options); } /** diff --git a/tools/ui/src/lib/utils/api-headers.ts b/tools/ui/src/lib/utils/api-headers.ts index 4b2b19d44..49d56d061 100644 --- a/tools/ui/src/lib/utils/api-headers.ts +++ b/tools/ui/src/lib/utils/api-headers.ts @@ -1,7 +1,7 @@ import { redactValue } from './redact'; import { CORS_PROXY, HEADERS } from '$lib/constants'; import { MimeTypeApplication } from '$lib/enums'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; /** * Get authorization headers for API requests diff --git a/tools/ui/src/lib/utils/api-key-validation.ts b/tools/ui/src/lib/utils/api-key-validation.ts index 8cde154fd..187199afc 100644 --- a/tools/ui/src/lib/utils/api-key-validation.ts +++ b/tools/ui/src/lib/utils/api-key-validation.ts @@ -3,7 +3,7 @@ import { browser } from '$app/environment'; import { base } from '$app/paths'; import { HEADERS } from '$lib/constants'; import { MimeTypeApplication } from '$lib/enums'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; /** * Validates API key by making a request to the server props endpoint diff --git a/tools/ui/src/lib/utils/audio-recording.ts b/tools/ui/src/lib/utils/audio-recording.ts index 1241d6e05..4cfe17378 100644 --- a/tools/ui/src/lib/utils/audio-recording.ts +++ b/tools/ui/src/lib/utils/audio-recording.ts @@ -14,10 +14,37 @@ import { MimeTypeAudio } from '$lib/enums'; * - Proper cleanup and resource management */ export class AudioRecorder { - private mediaRecorder: MediaRecorder | null = null; private audioChunks: Blob[] = []; - private stream: MediaStream | null = null; + private mediaRecorder: MediaRecorder | null = null; private recordingState: boolean = false; + private stream: MediaStream | null = null; + + cancelRecording(): void { + const recorder = this.mediaRecorder; + const stream = this.stream; + + this.mediaRecorder = null; + this.audioChunks = []; + this.stream = null; + this.recordingState = false; + + if (recorder && recorder.state !== 'inactive') { + // Drop the original handlers so the pending stop event does not touch the instance + recorder.onstop = null; + recorder.onerror = null; + recorder.stop(); + } + + if (stream) { + for (const track of stream.getTracks()) { + track.stop(); + } + } + } + + isRecording(): boolean { + return this.recordingState; + } async startRecording(): Promise { try { @@ -90,33 +117,6 @@ export class AudioRecorder { }); } - isRecording(): boolean { - return this.recordingState; - } - - cancelRecording(): void { - const recorder = this.mediaRecorder; - const stream = this.stream; - - this.mediaRecorder = null; - this.audioChunks = []; - this.stream = null; - this.recordingState = false; - - if (recorder && recorder.state !== 'inactive') { - // Drop the original handlers so the pending stop event does not touch the instance - recorder.onstop = null; - recorder.onerror = null; - recorder.stop(); - } - - if (stream) { - for (const track of stream.getTracks()) { - track.stop(); - } - } - } - private initializeRecorder(stream: MediaStream): void { const options: MediaRecorderOptions = {}; diff --git a/tools/ui/src/lib/utils/cache-ttl.ts b/tools/ui/src/lib/utils/cache-ttl.ts index bb0100755..bec40989c 100644 --- a/tools/ui/src/lib/utils/cache-ttl.ts +++ b/tools/ui/src/lib/utils/cache-ttl.ts @@ -31,9 +31,29 @@ interface CacheEntry { export class TTLCache { private cache = new Map>(); - private readonly ttlMs: number; private readonly maxEntries: number; private readonly onEvict?: (key: string, value: unknown) => void; + private readonly ttlMs: number; + + /** + * Get the number of entries (including potentially expired ones). + */ + get size(): number { + return this.cache.size; + } + + /** + * Clear all entries from cache. + */ + clear(): void { + if (this.onEvict) { + for (const [key, entry] of this.cache) { + this.onEvict(key, entry.value); + } + } + + this.cache.clear(); + } constructor(options: TTLCacheOptions = {}) { this.ttlMs = options.ttlMs ?? CACHE.DEFAULT_TTL_MS; @@ -41,6 +61,19 @@ export class TTLCache { this.onEvict = options.onEvict; } + /** + * Delete a specific key from cache. + */ + delete(key: K): boolean { + const entry = this.cache.get(key); + + if (entry && this.onEvict) { + this.onEvict(key, entry.value); + } + + return this.cache.delete(key); + } + /** * Get a value from cache. Returns null if expired or not found. */ @@ -61,25 +94,6 @@ export class TTLCache { return entry.value; } - /** - * Set a value in cache with TTL. - */ - set(key: K, value: V, customTtlMs?: number): void { - // Evict oldest entries if at capacity - if (this.cache.size >= this.maxEntries && !this.cache.has(key)) { - this.evictOldest(); - } - - const ttl = customTtlMs ?? this.ttlMs; - const now = Date.now(); - - this.cache.set(key, { - expiresAt: now + ttl, - lastAccessed: now, - value - }); - } - /** * Check if key exists and is not expired. */ @@ -98,36 +112,19 @@ export class TTLCache { } /** - * Delete a specific key from cache. + * Get all valid (non-expired) keys. */ - delete(key: K): boolean { - const entry = this.cache.get(key); + keys(): K[] { + const now = Date.now(); + const validKeys: K[] = []; - if (entry && this.onEvict) { - this.onEvict(key, entry.value); - } - - return this.cache.delete(key); - } - - /** - * Clear all entries from cache. - */ - clear(): void { - if (this.onEvict) { - for (const [key, entry] of this.cache) { - this.onEvict(key, entry.value); + for (const [key, entry] of this.cache) { + if (now <= entry.expiresAt) { + validKeys.push(key); } } - this.cache.clear(); - } - - /** - * Get the number of entries (including potentially expired ones). - */ - get size(): number { - return this.cache.size; + return validKeys; } /** @@ -150,38 +147,22 @@ export class TTLCache { } /** - * Get all valid (non-expired) keys. + * Set a value in cache with TTL. */ - keys(): K[] { + set(key: K, value: V, customTtlMs?: number): void { + // Evict oldest entries if at capacity + if (this.cache.size >= this.maxEntries && !this.cache.has(key)) { + this.evictOldest(); + } + + const ttl = customTtlMs ?? this.ttlMs; const now = Date.now(); - const validKeys: K[] = []; - for (const [key, entry] of this.cache) { - if (now <= entry.expiresAt) { - validKeys.push(key); - } - } - - return validKeys; - } - - /** - * Evict the oldest (least recently accessed) entry. - */ - private evictOldest(): void { - let oldestKey: K | null = null; - let oldestTime = Infinity; - - for (const [key, entry] of this.cache) { - if (entry.lastAccessed < oldestTime) { - oldestTime = entry.lastAccessed; - oldestKey = key; - } - } - - if (oldestKey !== null) { - this.delete(oldestKey); - } + this.cache.set(key, { + expiresAt: now + ttl, + lastAccessed: now, + value + }); } /** @@ -205,6 +186,25 @@ export class TTLCache { return true; } + + /** + * Evict the oldest (least recently accessed) entry. + */ + private evictOldest(): void { + let oldestKey: K | null = null; + let oldestTime = Infinity; + + for (const [key, entry] of this.cache) { + if (entry.lastAccessed < oldestTime) { + oldestTime = entry.lastAccessed; + oldestKey = key; + } + } + + if (oldestKey !== null) { + this.delete(oldestKey); + } + } } /** @@ -213,14 +213,26 @@ export class TTLCache { */ export class ReactiveTTLMap { private entries = $state>>(new Map()); - private readonly ttlMs: number; private readonly maxEntries: number; + private readonly ttlMs: number; + + get size(): number { + return this.entries.size; + } + + clear(): void { + this.entries.clear(); + } constructor(options: TTLCacheOptions = {}) { this.ttlMs = options.ttlMs ?? CACHE.DEFAULT_TTL_MS; this.maxEntries = options.maxEntries ?? CACHE.DEFAULT_MAX_ENTRIES; } + delete(key: K): boolean { + return this.entries.delete(key); + } + get(key: K): V | null { const entry = this.entries.get(key); @@ -237,21 +249,6 @@ export class ReactiveTTLMap { return entry.value; } - set(key: K, value: V, customTtlMs?: number): void { - if (this.entries.size >= this.maxEntries && !this.entries.has(key)) { - this.evictOldest(); - } - - const ttl = customTtlMs ?? this.ttlMs; - const now = Date.now(); - - this.entries.set(key, { - expiresAt: now + ttl, - lastAccessed: now, - value - }); - } - has(key: K): boolean { const entry = this.entries.get(key); @@ -266,18 +263,6 @@ export class ReactiveTTLMap { return true; } - delete(key: K): boolean { - return this.entries.delete(key); - } - - clear(): void { - this.entries.clear(); - } - - get size(): number { - return this.entries.size; - } - prune(): number { const now = Date.now(); @@ -293,6 +278,21 @@ export class ReactiveTTLMap { return pruned; } + set(key: K, value: V, customTtlMs?: number): void { + if (this.entries.size >= this.maxEntries && !this.entries.has(key)) { + this.evictOldest(); + } + + const ttl = customTtlMs ?? this.ttlMs; + const now = Date.now(); + + this.entries.set(key, { + expiresAt: now + ttl, + lastAccessed: now, + value + }); + } + private evictOldest(): void { let oldestKey: K | null = null; let oldestTime = Infinity; diff --git a/tools/ui/src/lib/utils/chat-form-input-rich-tokenizer.ts b/tools/ui/src/lib/utils/chat-form-input-rich-tokenizer.ts index c09afd018..626b10b29 100644 --- a/tools/ui/src/lib/utils/chat-form-input-rich-tokenizer.ts +++ b/tools/ui/src/lib/utils/chat-form-input-rich-tokenizer.ts @@ -38,7 +38,7 @@ import { SETTINGS_KEYS } from '$lib/constants'; import { BooleanString, ChatFormInputRichTokenKind } from '$lib/enums'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { toolsStore } from '$lib/stores/tools.svelte'; import type { ChatFormInputRichToken } from '$lib/types/chat-form-input-rich'; diff --git a/tools/ui/src/lib/utils/convert-files-to-extra.ts b/tools/ui/src/lib/utils/convert-files-to-extra.ts index e348f25fe..735e91c44 100644 --- a/tools/ui/src/lib/utils/convert-files-to-extra.ts +++ b/tools/ui/src/lib/utils/convert-files-to-extra.ts @@ -4,8 +4,8 @@ import { isLikelyTextFile, readFileAsText } from './text-files'; import { isWebpMimeType, webpBase64UrlToPngDataURL } from './webp-to-png'; import { SETTINGS_KEYS } from '$lib/constants'; import { AttachmentType, FileTypeCategory, SpecialFileType } from '$lib/enums'; -import { modelsStore } from '$lib/stores/models.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { ChatUploadedFile, DatabaseMessageExtra, FileProcessingResult } from '$lib/types'; import { getFileTypeCategory } from '$lib/utils'; import { toast } from 'svelte-sonner'; @@ -112,7 +112,7 @@ export async function parseFilesToMessageExtras( const currentConfig = settingsStore.config; // Use per-model vision check for router mode const hasVisionSupport = activeModelId - ? modelsStore.modelSupportsVision(activeModelId) + ? modelsStore.props.modelSupportsVision(activeModelId) : false; // Force PDF-to-text for non-vision models diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.ts index 239f9f572..079cdc871 100644 --- a/tools/ui/src/lib/utils/index.ts +++ b/tools/ui/src/lib/utils/index.ts @@ -130,7 +130,7 @@ export { getImageErrorFallbackHtml } from './image-error-fallback'; // SSE-with-JSON stream iterator (used by server tool streaming, decoupled // from chat.service.ts which embeds its own SSE parser for resume support) -export { parseSseJsonStream } from './sse'; +export { extractSseDataPayload, parseSseJsonStream, splitSseRecords } from './sse'; // Stream session identity (conversation-id based) export { streamIdentity } from './stream-identity'; @@ -150,7 +150,10 @@ export { getResourceIcon, getResourceTextContent, getResourceBlobContent, - downloadResourceContent + downloadResourceContent, + getMcpIconUrl, + getMcpServerFaviconFallback, + getMcpServerLabel } from './mcp'; // URI Template utilities diff --git a/tools/ui/src/lib/utils/mcp.ts b/tools/ui/src/lib/utils/mcp.ts index 61d5f8a9a..c60a59e80 100644 --- a/tools/ui/src/lib/utils/mcp.ts +++ b/tools/ui/src/lib/utils/mcp.ts @@ -1,3 +1,4 @@ +import { extractRootDomain } from './url'; import { AlertTriangle, Code, @@ -12,8 +13,10 @@ import { CODE_FILE_EXTENSION_REGEX, DEFAULT_RESOURCE_FILENAME, DISPLAY_NAME_SEPARATOR_REGEX, + EXPECTED_THEMED_ICON_PAIR_COUNT, FILE_EXTENSION_REGEX, IMAGE_FILE_EXTENSION_REGEX, + MCP_ALLOWED_ICON_MIME_TYPES, MCP_SERVER_ID_PREFIX, MCP_SSE, MIME_TYPE_PREFIXES, @@ -24,8 +27,22 @@ import { TEXT_FILE_EXTENSION_REGEX, URI_PATTERNS } from '$lib/constants'; -import { MCPLogLevel, MCPTransportType, MimeTypeText, UrlProtocol } from '$lib/enums'; -import type { MCPResourceContent, MCPResourceInfo, MCPServerSettingsEntry } from '$lib/types'; +import { + ColorMode, + HealthCheckStatus, + MCPLogLevel, + MCPTransportType, + MimeTypeText, + UrlProtocol +} from '$lib/enums'; +import type { + HealthCheckState, + MCPResourceContent, + MCPResourceIcon, + MCPResourceInfo, + MCPServerDisplayInfo, + MCPServerSettingsEntry +} from '$lib/types'; import type { MimeTypeUnion } from '$lib/types/common'; import type { Component } from 'svelte'; @@ -316,3 +333,132 @@ export function downloadResourceContent( document.body.removeChild(a); URL.revokeObjectURL(url); } + +/** + * Validates that an icon URI uses a safe scheme (https: or data:). + */ +function isValidMcpIconUri(src: string): boolean { + try { + if (src.startsWith(UrlProtocol.DATA)) return true; + + const url = new URL(src); + + return url.protocol === UrlProtocol.HTTPS; + } catch { + return false; + } +} + +/** + * Selects the best icon URL from an MCP icons array. + * Follows security guidelines from the MCP specification: + * - Only allows https: and data: URIs + * - Filters to supported MIME types + * + * Selection priority: + * 1. Icon matching the current color scheme (dark/light) + * 2. Universal icon (no theme specified); if exactly 2, assumes [0]=light, [1]=dark + * 3. First valid icon as last resort + */ +export function getMcpIconUrl(icons: MCPResourceIcon[] | undefined, isDark = false): string | null { + if (!icons?.length) return null; + + const validIcons = icons.filter((icon) => { + if (!icon.src || !isValidMcpIconUri(icon.src)) return false; + + if (icon.mimeType && !MCP_ALLOWED_ICON_MIME_TYPES.has(icon.mimeType)) return false; + + return true; + }); + + if (validIcons.length === 0) return null; + + const preferredTheme = isDark ? ColorMode.DARK : ColorMode.LIGHT; + // 1. Prefer icon explicitly matching the current color scheme + const themedIcon = validIcons.find((icon) => icon.theme === preferredTheme); + + if (themedIcon) return themedIcon.src; + + // 2. Handle universal icons (no theme specified) + const universalIcons = validIcons.filter((icon) => !icon.theme); + + if (universalIcons.length === EXPECTED_THEMED_ICON_PAIR_COUNT) { + // Heuristic: two theme-less icons → assume [0] = light, [1] = dark + return universalIcons[isDark ? 1 : 0].src; + } + + if (universalIcons.length > 0) { + return universalIcons[0].src; + } + + // 3. Last resort: use opposite-theme icon + return validIcons[0].src; +} + +/** + * Construct a fallback favicon URL from the MCP server URL. + * e.g. https://mcp.example.com/sse -> https://example.com/favicon.ico + */ +export function getMcpServerFaviconFallback(serverUrl: string): string | null { + try { + const url = new URL(serverUrl); + const rootDomain = extractRootDomain(url); + + if (!rootDomain) return null; + + const origin = `${url.protocol}//${rootDomain}`; + const candidates = ['favicon.ico', 'favicon.png']; + + for (const path of candidates) { + const faviconUrl = `${origin}/${path}`; + + if (isValidMcpIconUri(faviconUrl)) { + return faviconUrl; + } + } + } catch { + // Invalid URL, return null + } + + return null; +} + +/** + * Resolves the raw label for a server: user-defined display name first, + * then server-reported title or name when the health check succeeded, + * then the configured name (admin baseline or legacy data), then URL. + */ +function getMcpServerBaseLabel( + server: MCPServerDisplayInfo, + healthState?: HealthCheckState +): string { + if (server.displayName) return server.displayName; + + if (healthState?.status === HealthCheckStatus.SUCCESS) + return ( + healthState.serverInfo?.title || healthState.serverInfo?.name || server.name || server.url + ); + + return server.name || server.url; +} + +/** + * Returns the display label for a server, suffixed with a positional + * counter when several configured servers resolve to the same base label + * (e.g. two endpoints of the same host reporting an identical name). + * Numbering follows config order, so it is stable across renders. + */ +export function getMcpServerLabel( + server: MCPServerDisplayInfo, + servers: MCPServerDisplayInfo[], + healthChecks: Record +): string { + const label = getMcpServerBaseLabel(server, healthChecks[server.id]); + const twins = servers.filter((s) => getMcpServerBaseLabel(s, healthChecks[s.id]) === label); + + if (twins.length < 2) return label; + + const position = twins.findIndex((s) => s.id === server.id); + + return position < 0 ? label : `${label} (${position + 1})`; +} diff --git a/tools/ui/src/lib/utils/process-uploaded-files.ts b/tools/ui/src/lib/utils/process-uploaded-files.ts index 49bdd2412..e71371345 100644 --- a/tools/ui/src/lib/utils/process-uploaded-files.ts +++ b/tools/ui/src/lib/utils/process-uploaded-files.ts @@ -4,8 +4,8 @@ import { isSvgMimeType, svgBase64UrlToPngDataURL } from './svg-to-png'; import { isWebpMimeType, webpBase64UrlToPngDataURL } from './webp-to-png'; import { SETTINGS_KEYS } from '$lib/constants'; import { FileTypeCategory } from '$lib/enums'; -import { modelsStore } from '$lib/stores/models.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { getFileTypeCategory } from '$lib/utils'; import { toast } from 'svelte-sonner'; @@ -108,7 +108,7 @@ export async function processFilesToChatUploaded( // Show suggestion toast if vision model is available but PDF as image is disabled const hasVisionSupport = activeModelId - ? modelsStore.modelSupportsVision(activeModelId) + ? modelsStore.props.modelSupportsVision(activeModelId) : false; const currentConfig = settingsStore.config; diff --git a/tools/ui/src/lib/utils/source-history.ts b/tools/ui/src/lib/utils/source-history.ts index 32995ae03..6228ae7e4 100644 --- a/tools/ui/src/lib/utils/source-history.ts +++ b/tools/ui/src/lib/utils/source-history.ts @@ -12,9 +12,9 @@ export interface SourceHistoryEntry { } export class SourceHistory { - private undoStack: SourceHistoryEntry[] = []; - private redoStack: SourceHistoryEntry[] = []; private lastPush = 0; + private redoStack: SourceHistoryEntry[] = []; + private undoStack: SourceHistoryEntry[] = []; constructor( private limit = 100, @@ -32,17 +32,6 @@ export class SourceHistory { this.redoStack = []; } - undo(current: SourceHistoryEntry): SourceHistoryEntry | null { - const entry = this.undoStack.pop(); - - if (!entry) return null; - - this.redoStack.push(current); - this.lastPush = 0; // the next edit after an undo starts a new group - - return entry; - } - redo(current: SourceHistoryEntry): SourceHistoryEntry | null { const entry = this.redoStack.pop(); @@ -53,4 +42,15 @@ export class SourceHistory { return entry; } + + undo(current: SourceHistoryEntry): SourceHistoryEntry | null { + const entry = this.undoStack.pop(); + + if (!entry) return null; + + this.redoStack.push(current); + this.lastPush = 0; // the next edit after an undo starts a new group + + return entry; + } } diff --git a/tools/ui/src/lib/utils/sse.ts b/tools/ui/src/lib/utils/sse.ts index 41d9a1152..c984e77ee 100644 --- a/tools/ui/src/lib/utils/sse.ts +++ b/tools/ui/src/lib/utils/sse.ts @@ -25,6 +25,30 @@ export interface SseJsonEvent { data: T; } +/** + * Splits a raw SSE byte buffer into complete records on the blank-line + * boundary, returning the leftover partial record separately. Shared by the + * record-based consumers (parseSseJsonStream, models.service). + */ +export function splitSseRecords(buffer: string): { records: string[]; rest: string } { + const parts = buffer.split(SSE_RECORD_SEPARATOR); + + return { records: parts.slice(0, -1), rest: parts[parts.length - 1] ?? '' }; +} + +/** + * Extracts the joined `data:` payload from one SSE record (the data lines + * concatenated with a newline), or an empty string when the record carries + * no data lines. Used by models.service to parse status envelopes. + */ +export function extractSseDataPayload(record: string): string { + return record + .split(SSE_LINE_SEPARATOR) + .filter((line) => line.startsWith(SSE_DATA_PREFIX)) + .map((line) => line.slice(SSE_DATA_PREFIX.length).trim()) + .join(SSE_LINE_SEPARATOR); +} + export async function* parseSseJsonStream( response: Response, signal?: AbortSignal @@ -46,9 +70,9 @@ export async function* parseSseJsonStream( if (done) break; buffer += decoder.decode(value, { stream: true }); - const records = buffer.split(SSE_RECORD_SEPARATOR); + const { records, rest } = splitSseRecords(buffer); - buffer = records.pop() ?? ''; + buffer = rest; for (const record of records) { if (!record) continue; diff --git a/tools/ui/src/routes/(chat)/+page.svelte b/tools/ui/src/routes/(chat)/+page.svelte index 224d264c4..de8574e35 100644 --- a/tools/ui/src/routes/(chat)/+page.svelte +++ b/tools/ui/src/routes/(chat)/+page.svelte @@ -47,8 +47,8 @@ serverStore.isRouterMode && !modelsStore.isModelLoaded(model.id) ) { - modelsStore - .loadModel(model.id) + modelsStore.status + .load(model.id) .catch((error) => console.error('Failed to load model:', error)); } } catch (error) { @@ -77,7 +77,7 @@ onMount(async () => { if (!conversationsStore.isInitialized) { - await conversationsStore.init(); + await conversationsStore.initialize(); } conversationsStore.clearActiveConversation(); diff --git a/tools/ui/src/routes/+layout.svelte b/tools/ui/src/routes/+layout.svelte index 8314cd2a2..f87bbe26a 100644 --- a/tools/ui/src/routes/+layout.svelte +++ b/tools/ui/src/routes/+layout.svelte @@ -216,11 +216,11 @@ if (!serverStore.isRouterMode) return; untrack(() => { - modelsStore.subscribeStatus(); + modelsStore.status.subscribe(); }); return () => { - modelsStore.unsubscribeStatus(); + modelsStore.status.unsubscribe(); }; }); diff --git a/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts b/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts index 0b06d57a5..b4d6df453 100644 --- a/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts +++ b/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts @@ -14,7 +14,7 @@ import { perfState } from './components/agentic-perf-state.svelte'; import AgenticPerfWrapper from './components/AgenticPerfWrapper.svelte'; import ChatMessagesPerfWrapper from './components/ChatMessagesPerfWrapper.svelte'; import { MessageRole } from '$lib/enums'; -import { conversationsStore } from '$lib/stores/conversations.svelte'; +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; import type { DatabaseMessage } from '$lib/types'; import { tick } from 'svelte'; import { describe, it } from 'vitest'; diff --git a/tools/ui/tests/client/apikey-splash.svelte.test.ts b/tools/ui/tests/client/apikey-splash.svelte.test.ts index bad7f6ccb..b2705dd8c 100644 --- a/tools/ui/tests/client/apikey-splash.svelte.test.ts +++ b/tools/ui/tests/client/apikey-splash.svelte.test.ts @@ -1,5 +1,5 @@ import { CONFIG_LOCALSTORAGE_KEY } from '$lib/constants'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { validateApiKey } from '$lib/utils/api-key-validation'; import { beforeEach, describe, expect, it } from 'vitest'; diff --git a/tools/ui/tests/client/chat-form-enter-code-block.svelte.test.ts b/tools/ui/tests/client/chat-form-enter-code-block.svelte.test.ts index 485dc3965..3454170b6 100644 --- a/tools/ui/tests/client/chat-form-enter-code-block.svelte.test.ts +++ b/tools/ui/tests/client/chat-form-enter-code-block.svelte.test.ts @@ -7,7 +7,7 @@ import ChatFormTestWrapper from './components/ChatFormTestWrapper.svelte'; import { SETTINGS_KEYS } from '$lib/constants'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { tick } from 'svelte'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { userEvent } from 'vitest/browser'; diff --git a/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte b/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte index 504f68597..ab5cc38bc 100644 --- a/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte +++ b/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte @@ -4,7 +4,7 @@ // toolMessages array) rather than a single message subtree. import ChatMessages from '$lib/components/app/chat/ChatMessages/ChatMessages.svelte'; import * as Tooltip from '$lib/components/ui/tooltip'; - import { conversationsStore } from '$lib/stores/conversations.svelte'; + import { conversationsStore } from '$lib/stores/conversations/index.svelte'; diff --git a/tools/ui/tests/client/mcp-display-name.svelte.test.ts b/tools/ui/tests/client/mcp-display-name.svelte.test.ts index f17e08cf1..7db0ffd42 100644 --- a/tools/ui/tests/client/mcp-display-name.svelte.test.ts +++ b/tools/ui/tests/client/mcp-display-name.svelte.test.ts @@ -1,6 +1,6 @@ import { McpServerForm } from '$lib/components/app/mcp'; -import { mcpStore } from '$lib/stores/mcp.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { mcpStore } from '$lib/stores/mcp/index.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { beforeEach, describe, expect, it } from 'vitest'; import { render } from 'vitest-browser-svelte'; diff --git a/tools/ui/tests/client/sandbox.service.svelte.test.ts b/tools/ui/tests/client/sandbox.service.svelte.test.ts index 7c0d7926f..547e3ac1f 100644 --- a/tools/ui/tests/client/sandbox.service.svelte.test.ts +++ b/tools/ui/tests/client/sandbox.service.svelte.test.ts @@ -10,7 +10,7 @@ const run = (code: string, timeoutMs?: number) => describe('sandbox service', () => { beforeEach(async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.config = { ...settingsStore.config, diff --git a/tools/ui/tests/client/settings-registry-invariants.svelte.test.ts b/tools/ui/tests/client/settings-registry-invariants.svelte.test.ts index 45af7e0d1..0ed6996b5 100644 --- a/tools/ui/tests/client/settings-registry-invariants.svelte.test.ts +++ b/tools/ui/tests/client/settings-registry-invariants.svelte.test.ts @@ -1,7 +1,7 @@ import { CONFIG_LOCALSTORAGE_KEY, SETTING_CONFIG_DEFAULT } from '$lib/constants'; import { ParameterSyncService } from '$lib/services/parameter-sync.service'; import { serverStore } from '$lib/stores/server.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { SettingsConfigType } from '$lib/types'; import { beforeEach, describe, expect, it } from 'vitest'; diff --git a/tools/ui/tests/client/settings-render-keys-migration.svelte.test.ts b/tools/ui/tests/client/settings-render-keys-migration.svelte.test.ts index 32f4ff3dd..ce65aeb70 100644 --- a/tools/ui/tests/client/settings-render-keys-migration.svelte.test.ts +++ b/tools/ui/tests/client/settings-render-keys-migration.svelte.test.ts @@ -6,7 +6,7 @@ import { CONFIG_LOCALSTORAGE_KEY } from '$lib/constants'; import { MigrationService } from '$lib/services/migration.service'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { beforeEach, describe, expect, it } from 'vitest'; const RENDER_KEYS_MIGRATION_ID = 'render-keys-unfold-v1'; diff --git a/tools/ui/tests/client/ui-settings-sync.svelte.test.ts b/tools/ui/tests/client/ui-settings-sync.svelte.test.ts index 6dca891c8..ca9268e2e 100644 --- a/tools/ui/tests/client/ui-settings-sync.svelte.test.ts +++ b/tools/ui/tests/client/ui-settings-sync.svelte.test.ts @@ -1,6 +1,6 @@ import { CONFIG_LOCALSTORAGE_KEY } from '$lib/constants'; import { serverStore } from '$lib/stores/server.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { beforeEach, describe, expect, it } from 'vitest'; function mockProps(uiSettings: Record) { diff --git a/tools/ui/tests/client/update-message-in-place.svelte.test.ts b/tools/ui/tests/client/update-message-in-place.svelte.test.ts index 65298b44b..ea3b65d0c 100644 --- a/tools/ui/tests/client/update-message-in-place.svelte.test.ts +++ b/tools/ui/tests/client/update-message-in-place.svelte.test.ts @@ -8,7 +8,7 @@ // -> 3.07ms at 40). Mutating in place keeps it flat. import { MessageRole } from '$lib/enums'; -import { conversationsStore } from '$lib/stores/conversations.svelte'; +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; import type { DatabaseMessage } from '$lib/types'; import { describe, expect, it } from 'vitest'; diff --git a/tools/ui/tests/stories/ChatMessage.stories.svelte b/tools/ui/tests/stories/ChatMessage.stories.svelte index 84fee2ea1..e9bf7a6f6 100644 --- a/tools/ui/tests/stories/ChatMessage.stories.svelte +++ b/tools/ui/tests/stories/ChatMessage.stories.svelte @@ -105,7 +105,7 @@ message: userMessage }} play={async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.updateConfig('showRawOutputSwitch', false); }} @@ -118,7 +118,7 @@ message: assistantMessage }} play={async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.updateConfig('showRawOutputSwitch', false); }} @@ -131,7 +131,7 @@ message: assistantWithReasoning }} play={async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.updateConfig('showRawOutputSwitch', false); }} @@ -144,7 +144,7 @@ message: rawOutputMessage }} play={async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.updateConfig('showRawOutputSwitch', true); }} @@ -157,7 +157,7 @@ }} asChild play={async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.updateConfig('showRawOutputSwitch', false); // Phase 1: Stream reasoning content in chunks @@ -213,11 +213,11 @@ message: processingMessage }} play={async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.updateConfig('showRawOutputSwitch', false); // Import the chat store to simulate loading state - const { chatStore } = await import('$lib/stores/chat.svelte'); + const { chatStore } = await import('$lib/stores/chat/index.svelte'); // Set loading state to true to trigger the processing UI chatStore.isLoading = true; diff --git a/tools/ui/tests/stories/ModelsSelector.stories.svelte b/tools/ui/tests/stories/ModelsSelector.stories.svelte index d63300cb2..7018d09e7 100644 --- a/tools/ui/tests/stories/ModelsSelector.stories.svelte +++ b/tools/ui/tests/stories/ModelsSelector.stories.svelte @@ -4,7 +4,7 @@ import ModelsSelectorOption from '$lib/components/app/models/ModelsSelectorOption.svelte'; import type { GroupedModelOptions, ModelItem } from '$lib/components/app/models/utils'; import { ServerModelStatus } from '$lib/enums'; - import { modelsStore } from '$lib/stores/models.svelte'; + import { modelsStore } from '$lib/stores/models/index.svelte'; const { Story } = defineMeta({ parameters: { diff --git a/tools/ui/tests/stories/SidebarNavigation.stories.svelte b/tools/ui/tests/stories/SidebarNavigation.stories.svelte index 635992601..ddaa90485 100644 --- a/tools/ui/tests/stories/SidebarNavigation.stories.svelte +++ b/tools/ui/tests/stories/SidebarNavigation.stories.svelte @@ -53,7 +53,7 @@ asChild name="Default" play={async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); waitFor(() => setTimeout(() => { @@ -71,7 +71,7 @@ asChild name="SearchActive" play={async ({ userEvent }) => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); waitFor(() => setTimeout(() => { @@ -98,7 +98,7 @@ name="Empty" play={async () => { // Mock empty conversations store - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.conversations = []; }} diff --git a/tools/ui/tests/stories/fixtures/storybook-mocks.ts b/tools/ui/tests/stories/fixtures/storybook-mocks.ts index 736674690..ac9fb63cd 100644 --- a/tools/ui/tests/stories/fixtures/storybook-mocks.ts +++ b/tools/ui/tests/stories/fixtures/storybook-mocks.ts @@ -1,4 +1,4 @@ -import { modelsStore } from '$lib/stores/models.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; import { serverStore } from '$lib/stores/server.svelte'; /** diff --git a/tools/ui/tests/unit/chat-activity.test.ts b/tools/ui/tests/unit/chat-activity.test.ts new file mode 100644 index 000000000..051648ead --- /dev/null +++ b/tools/ui/tests/unit/chat-activity.test.ts @@ -0,0 +1,77 @@ +import { ChatActivityStore } from '$lib/stores/chat/activity.svelte'; +import { beforeEach, describe, expect, it } from 'vitest'; + +describe('ChatActivityStore', () => { + let store: ChatActivityStore; + + beforeEach(() => { + store = new ChatActivityStore(); + }); + + it('starts with no local or remote activity', () => { + expect(store.loadingConvs).toEqual([]); + expect(store.isLocal('a')).toBe(false); + expect(store.isRemote('a')).toBe(false); + }); + + it('markLocal adds a conv to the local set and the loading union', () => { + store.markLocal('a'); + + expect(store.isLocal('a')).toBe(true); + expect(store.isRemote('a')).toBe(false); + expect(store.loadingConvs).toEqual(['a']); + }); + + it('localEnded removes a local conv', () => { + store.markLocal('a'); + store.localEnded('a'); + + expect(store.isLocal('a')).toBe(false); + expect(store.loadingConvs).toEqual([]); + }); + + it('localEnded also drops a stale remote hint for the same conv', () => { + store.markLocal('a'); + store.applyRemoteSnapshot(['a']); + expect(store.isRemote('a')).toBe(true); + + store.localEnded('a'); + + expect(store.isLocal('a')).toBe(false); + expect(store.isRemote('a')).toBe(false); + expect(store.loadingConvs).toEqual([]); + }); + + it('applyRemoteSnapshot adds remote convs and unions them with local', () => { + store.markLocal('local'); + store.applyRemoteSnapshot(['remote']); + + expect(store.isRemote('remote')).toBe(true); + expect(store.loadingConvs).toEqual(['local', 'remote']); + }); + + it('applyRemoteSnapshot removes remote convs missing from the snapshot', () => { + store.applyRemoteSnapshot(['a', 'b']); + store.applyRemoteSnapshot(['a']); + + expect(store.isRemote('a')).toBe(true); + expect(store.isRemote('b')).toBe(false); + expect(store.loadingConvs).toEqual(['a']); + }); + + it('applyRemoteSnapshot keeps local convs absent from the snapshot', () => { + store.markLocal('local'); + store.applyRemoteSnapshot(['remote']); + store.applyRemoteSnapshot([]); + + expect(store.isLocal('local')).toBe(true); + expect(store.loadingConvs).toEqual(['local']); + }); + + it('loadingConvs does not duplicate a conv that is both local and remote', () => { + store.markLocal('a'); + store.applyRemoteSnapshot(['a']); + + expect(store.loadingConvs).toEqual(['a']); + }); +}); diff --git a/tools/ui/tests/unit/mcp-override-fallback.test.ts b/tools/ui/tests/unit/mcp-override-fallback.test.ts index 47d6ac253..12ed6e4c4 100644 --- a/tools/ui/tests/unit/mcp-override-fallback.test.ts +++ b/tools/ui/tests/unit/mcp-override-fallback.test.ts @@ -46,7 +46,7 @@ describe('conversationsStore MCP override resolution', () => { // The settings store constructor bails in node env (no `browser`), // so seed the config directly. The shape mirrors what `loadConfig` // would build from localStorage. - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); const raw = localStorage.getItem(CONFIG_LOCALSTORAGE_KEY) ?? '{}'; const saved = JSON.parse(raw) as Record; @@ -73,77 +73,77 @@ describe('conversationsStore MCP override resolution', () => { } it('inherits server.enabled when no conversation is active', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.activeConversation = null; - expect(conversationsStore.isMcpServerEnabledForChat('alpha')).toBe(false); - expect(conversationsStore.isMcpServerEnabledForChat('bravo')).toBe(true); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('alpha')).toBe(false); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('bravo')).toBe(true); }); it('inherits server.enabled on a newly created chat with no overrides', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.activeConversation = makeConversation(); // Empty override list: must fall back to global server.enabled, not all-off. - expect(conversationsStore.isMcpServerEnabledForChat('alpha')).toBe(false); - expect(conversationsStore.isMcpServerEnabledForChat('bravo')).toBe(true); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('alpha')).toBe(false); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('bravo')).toBe(true); }); it('inherits server.enabled on a newly created chat when overrides is undefined', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.activeConversation = makeConversation(undefined); - expect(conversationsStore.isMcpServerEnabledForChat('alpha')).toBe(false); - expect(conversationsStore.isMcpServerEnabledForChat('bravo')).toBe(true); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('alpha')).toBe(false); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('bravo')).toBe(true); }); it('uses explicit per-chat overrides, with defaults for non-overridden servers', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); // Override flips bravo off for this chat, alpha keeps its global default. conversationsStore.activeConversation = makeConversation([ { enabled: false, serverId: 'bravo' } ]); - expect(conversationsStore.isMcpServerEnabledForChat('alpha')).toBe(false); - expect(conversationsStore.isMcpServerEnabledForChat('bravo')).toBe(false); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('alpha')).toBe(false); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('bravo')).toBe(false); }); it('getAllMcpServerOverrides returns a complete list merged from defaults', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.activeConversation = makeConversation([ { enabled: true, serverId: 'alpha' } ]); - expect(conversationsStore.getAllMcpServerOverrides()).toEqual([ + expect(conversationsStore.preferences.getAllMcpServerOverrides()).toEqual([ { enabled: true, serverId: 'alpha' }, { enabled: true, serverId: 'bravo' } ]); }); it('getAllMcpServerOverrides falls back to defaults when there are no explicit overrides', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.activeConversation = makeConversation(); - expect(conversationsStore.getAllMcpServerOverrides()).toEqual([ + expect(conversationsStore.preferences.getAllMcpServerOverrides()).toEqual([ { enabled: false, serverId: 'alpha' }, { enabled: true, serverId: 'bravo' } ]); }); it('getMcpServerOverride returns the global default when the server has no explicit override', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.activeConversation = makeConversation([ { enabled: true, serverId: 'alpha' } ]); - expect(conversationsStore.getMcpServerOverride('bravo')).toEqual({ + expect(conversationsStore.preferences.getMcpServerOverride('bravo')).toEqual({ enabled: true, serverId: 'bravo' }); diff --git a/tools/ui/tests/unit/stream-resume.test.ts b/tools/ui/tests/unit/stream-resume.test.ts index 43d89272e..ce4eee9aa 100644 --- a/tools/ui/tests/unit/stream-resume.test.ts +++ b/tools/ui/tests/unit/stream-resume.test.ts @@ -92,6 +92,67 @@ describe('ChatService stream resume', () => { expect(ChatService.getStreamState('conv-a')!.model).toBe('model-y'); }); + describe('throttled saves (per-chunk path)', () => { + // unique conversation ids: the throttle tracker is module state and + // outlives beforeEach's localStorage.clear() + let counter = 0; + + const freshConv = () => `conv-throttle-${++counter}`; + + it('writes immediately when no write was recorded for the conversation', () => { + const conv = freshConv(); + + ChatService.saveStreamStateThrottled(conv, 100); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(100); + }); + + it('holds a save pending when it lands inside the interval, flush forces it out', () => { + const conv = freshConv(); + + ChatService.saveStreamStateThrottled(conv, 100); + ChatService.saveStreamStateThrottled(conv, 200); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(100); + + ChatService.flushStreamState(conv); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(200); + }); + + it('flush is a no-op when nothing is pending', () => { + const conv = freshConv(); + + ChatService.saveStreamStateThrottled(conv, 100); + ChatService.flushStreamState(conv); + ChatService.flushStreamState(conv); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(100); + }); + + it('an immediate save resets the throttle window', () => { + const conv = freshConv(); + + ChatService.saveStreamStateThrottled(conv, 100); + ChatService.saveStreamState(conv, 150); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(150); + + ChatService.saveStreamStateThrottled(conv, 200); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(150); + + ChatService.flushStreamState(conv); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(200); + }); + + it('clearStreamState drops the pending throttled state', () => { + const conv = freshConv(); + + ChatService.saveStreamStateThrottled(conv, 100); + ChatService.saveStreamStateThrottled(conv, 200); + ChatService.clearStreamState(conv); + expect(ChatService.getStreamState(conv)).toBeNull(); + + ChatService.flushStreamState(conv); + expect(ChatService.getStreamState(conv)).toBeNull(); + }); + }); + describe('resumeStreamIdentity', () => { it('appends the persisted model so the resume key matches the frozen POST identity', () => { ChatService.saveStreamState('conv-a', 10, 'model-x'); From 6b4fa88a6ce2429958ea4ee7c0223928e0979fa2 Mon Sep 17 00:00:00 2001 From: lhez Date: Thu, 20 Aug 2026 10:52:07 -0700 Subject: [PATCH 19/44] opencl: fix local size for norm (#27339) --- ggml/src/ggml-opencl/ggml-opencl.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index fbf7dadb9..d169f3389 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -12830,7 +12830,10 @@ static void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - const int nth = MIN(64, ne00); + int nth = 1; + while (nth < ne00 && nth < 64) { + nth *= 2; + } cl_kernel kernel = backend_ctx->kernel_norm; From 6503355df0eb4f65875012523263c302fe0088c1 Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Thu, 20 Aug 2026 10:58:35 -0700 Subject: [PATCH 20/44] opencl: fix q6_K flat mul_mat for Adreno A6x/A7x GPUs with older E031 compilers (#26476) * opencl: decline KV-convert flash_attn variants on Adreno A7X (compiler SIGSEGV) The Adreno 740 (A7X) compiler E031.41 crashes inside clBuildProgram when building the flash_attn programs whose KV path is mixed-type or dequantized: flash_attn_f32_f16, flash_attn_f32_q8_0, flash_attn_f32_q4_0. It is a driver crash rather than a compile-error return, so build_program_from_source_ex() cannot catch it. The uniform f32 and f16 programs build correctly. Decline the three KV-convert variants on the A7X in supports_op so they never lazy-compile; those attention layers run on the CPU backend instead. Same idiom as the existing Intel DK=512 and X1E carve-outs. test-backend-ops FLASH_ATTN_EXT on the 740: 226 OK / 0 FAIL, previously exit 139. Other parts are unaffected - the gate is dead code there. * opencl: fix q6_K flat mul_mat on older Adreno E031 compilers, gated kernel_mul_mv_q6_K_f32_flat produces ~10x-wrong output on the older Adreno E031 compilers while q4_K and q5_K are correct. Four codegen defects, each confirmed on-device against the CPU reference: 1. 64-bit ulong arithmetic is miscompiled, so every weight and scale read hit the wrong address - the primary cause, and why q5_K (int offsets) was unaffected. The block index is computed in int and widened only inside the pointer expression. 2. The vectorized dequant (int4/float4 bit-ops, convert_*4, dot()) is miscompiled; the 6-bit weights are reconstructed and the dot done scalar. 3. vload4 of the f32 activations is miscompiled; replaced by a scalar-indexed load. 4. The accumulation is miscompiled unless a side effect forces the partial sums to materialize. A printf under a guard the compiler cannot prove false acts as a zero-cost optimizer barrier; its placement is load-bearing. The defect tracks the compiler, not the GPU generation: it reproduces on E031.38 (Adreno 642L) and E031.41 (Adreno 740) and is fixed by E031.45 (Adreno 619), so the workarounds are gated on the compiler version. Where they are not needed they cost real throughput - 42.4 -> 35.1 GFLOPS on an Adreno 840 q6_K GEMV. The explicit compiler-type check is required, not redundant: newer_than_or_same() is false for every non-E031 compiler, so negating it alone would enable the workarounds on E17 and DX. test-backend-ops MUL_MAT is 919/919 on the Adreno 740, 642L, 619, 840 and 850; the 740 and 642L were 909/919 before. The 642L additionally needs the A6X per-kernel-program support to reach these tests at all. --- ggml/src/ggml-opencl/ggml-opencl.cpp | 42 +++++++- .../kernels/mul_mv_q6_k_f32_flat.cl | 96 ++++++++++++++++++- 2 files changed, 136 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index d169f3389..49cd9fd35 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -582,6 +582,8 @@ struct ggml_backend_opencl_context { bool adreno_use_bin_kernels; get_adreno_bin_kernel_func_t get_adreno_bin_kernel_func = nullptr; ggml_cl_compiler_version adreno_cl_compiler_version; + // The q6_K flat mul_mat codegen workarounds are needed by old E031 compilers only. + bool q6_k_flat_old_compiler; std::string kernel_compile_opts; // cached for lazy-compiled kernels. @@ -1931,8 +1933,14 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { #else const std::string kernel_src = read_file("mul_mv_q6_k_f32_flat.cl"); #endif + // The codegen workarounds in this kernel are a measured 13-20% loss on + // compilers that do not need them, so only the affected ones build them; + // everyone else gets the original source. + const std::string q6k_opts = backend_ctx->q6_k_flat_old_compiler + ? compile_opts + " -DADRENO_OLD_COMPILER=1" + : compile_opts; cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), q6k_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q6_K_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); @@ -5917,6 +5925,16 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) { (backend_ctx->adreno_cl_compiler_version.type == E031 && backend_ctx->adreno_cl_compiler_version.major >= 47) || (backend_ctx->adreno_cl_compiler_version.type == DX && backend_ctx->adreno_cl_compiler_version.major >= 17); + // The q6_K flat mul_mat miscompile is a defect of the older E031 compilers, not a + // property of any GPU generation: it reproduces on E031.38 (Adreno 642L) and E031.41 + // (Adreno 740) and is fixed by E031.45 (Adreno 619). Gate on the compiler so parts + // that do not need the workarounds do not pay for them. The explicit type check is + // required: newer_than_or_same() is false for every non-E031 compiler, so negating it + // alone would enable the workarounds on E17/DX. + backend_ctx->q6_k_flat_old_compiler = + backend_ctx->adreno_cl_compiler_version.type == E031 && + !backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 45, 0, 0); + size_t ext_str_size; clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size); char *ext_buffer = (char *)alloca(ext_str_size + 1); @@ -7496,6 +7514,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16; const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32; + const bool is_f32_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0 && op->type == GGML_TYPE_F32 && dk % 32 == 0 && dv % 32 == 0; @@ -7503,6 +7522,21 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te v->type == GGML_TYPE_Q4_0 && op->type == GGML_TYPE_F32 && dk % 32 == 0 && dv % 32 == 0; + // A7X (Adreno 740, compiler E031.41) SIGSEGVs inside clBuildProgram + // building the flash_attn programs whose KV path is mixed-type or + // dequantized — f32_f16, q8_0, q4_0 (reproduced at DK=40 and DK=64; it + // is DK-independent). It is a driver crash, not codegen-wrong-output, so + // it cannot be caught in-process (fatal=false only handles clean compile + // errors). The uniform f16_f16 / f32_f32 programs compile fine on this + // compiler, so decline only the KV-convert variants; ggml then runs + // those (f16-KV / quant-KV) attention layers on the CPU backend. + // Negative compiler carve-out, same idiom as the Intel DK=512 decline + // below and the X1E driver-quirk guards. + if (backend_ctx && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && + (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) { + return false; + } + // Asymmetric KV: host-dequants both sides to F32, uses f32 kernel. auto is_kv_type_ok = [](ggml_type t) { return t == GGML_TYPE_F16 || t == GGML_TYPE_F32 || @@ -20583,6 +20617,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3)); + // The optimizer-barrier arg exists only in the ADRENO_OLD_COMPILER build of + // this kernel; conformant compilers get the original 17-arg signature. + if (backend_ctx->q6_k_flat_old_compiler) { + cl_uchar q6k_mask = 0xFF; // never 0xFE in prod; see the kernel note + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_uchar), &q6k_mask)); + } #else kernel = backend_ctx->kernel_mul_mv_q6_K_f32; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl index 57b90c05a..2cca5335d 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl @@ -28,6 +28,13 @@ #define QK_K 256 +// ADRENO_OLD_COMPILER is defined by the host (-D) only for the Adreno E031 +// compilers older than E031.45, which miscompile several constructs this kernel +// used (confirmed on E031.38 and E031.41; E031.45 is clean). Every other +// compiler -- newer E031, E17, DX, Intel, and every non-Adreno device that +// builds this program -- takes the #else branches, which are the original +// source: the workarounds below cost ~13% on the q6_K flat n=1 GEMV where they +// are not needed. inline float block_q_6_K_dot_y_flat( global uchar * blk_ql, global uchar * blk_qh, @@ -37,6 +44,9 @@ inline float block_q_6_K_dot_y_flat( int ip, int is, int l0, +#if defined(ADRENO_OLD_COMPILER) + int dbg, +#endif float4 y0, float4 y1, float4 y2, @@ -48,10 +58,40 @@ inline float block_q_6_K_dot_y_flat( global uchar * q1 = blk_ql + ib*128 + q_offset_l; global uchar * q2 = q1 + QK_K/8; global uchar * qh = blk_qh + ib*64 + q_offset_h; - global char * sc = blk_scales + ib*16 + is; float dall = blk_d[ib]; +#if defined(ADRENO_OLD_COMPILER) + // The vectorized dequant (int4/float4 bit-ops, convert_*4, dot()) and vload4 + // are miscompiled here -> garbage weights. Reconstruct the 6-bit weights and + // take the dot product scalar. q4_K/q5_K flat already use scalar paths, which + // is why q6_K was the only flat GEMV that failed. + // Scales are SIGNED int8; read as uchar and sign-extend arithmetically so the + // result does not depend on whether the compiler treats `char` as signed. + global uchar * sc = (global uchar *)(blk_scales + ib*16 + is); + + int s0 = (int)sc[0] - 256*(sc[0] >> 7); + int s2 = (int)sc[2] - 256*(sc[2] >> 7); + int s4 = (int)sc[4] - 256*(sc[4] >> 7); + int s6 = (int)sc[6] - 256*(sc[6] >> 7); + + // one 6-bit weight: low/high nibble of a ql byte OR'd with a 2-bit qh plane + // (plane p in {0,1,2,3} selects qh bits 2p..2p+1) placed at bits 4-5, minus 32. + #define Q6W(qb, sh, hb, p) ((float)((((int)(qb) >> (sh)) & 15) | ((((int)(hb) >> (2*(p))) & 3) << 4)) - 32.f) + + float d0 = y0.s0*Q6W(q1[0],0,qh[0],0) + y0.s1*Q6W(q1[1],0,qh[1],0) + y0.s2*Q6W(q1[2],0,qh[2],0) + y0.s3*Q6W(q1[3],0,qh[3],0); + float d1 = y1.s0*Q6W(q2[0],0,qh[0],1) + y1.s1*Q6W(q2[1],0,qh[1],1) + y1.s2*Q6W(q2[2],0,qh[2],1) + y1.s3*Q6W(q2[3],0,qh[3],1); + float d2 = y2.s0*Q6W(q1[0],4,qh[0],2) + y2.s1*Q6W(q1[1],4,qh[1],2) + y2.s2*Q6W(q1[2],4,qh[2],2) + y2.s3*Q6W(q1[3],4,qh[3],2); + float d3 = y3.s0*Q6W(q2[0],4,qh[0],3) + y3.s1*Q6W(q2[1],4,qh[1],3) + y3.s2*Q6W(q2[2],4,qh[2],3) + y3.s3*Q6W(q2[3],4,qh[3],3); + #undef Q6W + + if (dbg) printf("HELPER dall=%f s=[%d %d %d %d] d=[%f %f %f %f] ql0=%d qh0=%d y00=%f\n", + dall, s0, s2, s4, s6, d0, d1, d2, d3, (int)q1[0], (int)qh[0], y0.s0); + + return dall * (d0 * s0 + d1 * s2 + d2 * s4 + d3 * s6); +#else + global char * sc = blk_scales + ib*16 + is; + // Vectorized loads: 3 uchar4 weight loads instead of 12 scalar byte reads. // q_offset_l/h are 4-aligned, so these are aligned vector loads. uchar4 q1v = vload4(0, q1); @@ -72,6 +112,7 @@ inline float block_q_6_K_dot_y_flat( return dall * (dot(y0, w0) * sc[0] + dot(y1, w1) * sc[2] + dot(y2, w2) * sc[4] + dot(y3, w3) * sc[6]); +#endif } #undef N_DST @@ -113,6 +154,11 @@ kernel void kernel_mul_mv_q6_K_f32_flat( int ne1, int r2, int r3 +#if defined(ADRENO_OLD_COMPILER) + , + uchar q6k_mask // runtime 0xFF; the host passes it so the compiler cannot + // constant-fold the printf guards below into nothing +#endif ) { src1 = (global float*)((global char*)src1 + offset1); dst = (global float*)((global char*)dst + offsetd); @@ -128,6 +174,22 @@ kernel void kernel_mul_mv_q6_K_f32_flat( int first_row = (N_SIMDGROUP * r0 + get_sub_group_id()) * N_DST; +#if defined(ADRENO_OLD_COMPILER) + // 64-bit `ulong` integer arithmetic is miscompiled here -> the base-pointer byte + // offsets came out wrong, so EVERY weight/scale read hit the wrong address. This + // was the primary cause of the q6_K flat failure (q5_K uses int offsets and is + // unaffected). Compute the block index in `int` and widen to `ulong` only inside + // the pointer expression: the byte offset stays 64-bit, but there is no ulong + // arithmetic chain to miscompile. The int index would overflow past ~2^31 blocks, + // which no realistic weight reaches -- but that is a narrowing, so keep it off the + // conformant path, which retains full ulong arithmetic. + int offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + + global uchar * blk_ql = (global uchar *) src0_ql + (ulong)offset_src0 * 128; + global uchar * blk_qh = (global uchar *) src0_qh + (ulong)offset_src0 * 64; + global char * blk_scales = (global char *) src0_s + (ulong)offset_src0 * 16; + global half * blk_d = (global half *) src0_d + offset_src0; +#else ulong offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); ulong offset_src0_ql = offset_src0 * 128; ulong offset_src0_qh = offset_src0 * 64; @@ -138,6 +200,7 @@ kernel void kernel_mul_mv_q6_K_f32_flat( global uchar * blk_qh = (global uchar *) src0_qh + offset_src0_qh; global char * blk_scales = (global char *) src0_s + offset_src0_s; global half * blk_d = (global half *) src0_d + offset_src0_d; +#endif global float * yy = (global float *) src1 + r1*ne10 + im*ne00*ne1; int tid = get_sub_group_local_id()%(N_SIMDWIDTH/BLOCK_STRIDE); // within-super-block part, 0..15 @@ -155,24 +218,55 @@ kernel void kernel_mul_mv_q6_K_f32_flat( for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { global float * y = yy + ib * QK_K + 128*ip + l0; +#if defined(ADRENO_OLD_COMPILER) + // vload4 of f32 is miscompiled here; index the lanes scalar instead. + float4 y0 = (float4)(y[ 0], y[ 1], y[ 2], y[ 3]); + float4 y1 = (float4)(y[32], y[33], y[34], y[35]); + float4 y2 = (float4)(y[64], y[65], y[66], y[67]); + float4 y3 = (float4)(y[96], y[97], y[98], y[99]); +#else float4 y0 = vload4(0, y + 0); float4 y1 = vload4(0, y + 32); float4 y2 = vload4(0, y + 64); float4 y3 = vload4(0, y + 96); +#endif for (int row = 0; row < N_DST; row++) { if (first_row + row < ne01) { +#if defined(ADRENO_OLD_COMPILER) + int dbg = (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ib==0 && + ne00==256 && ne01==16 && get_sub_group_local_id()==0) ? 1 : 0; + sumf[row] += block_q_6_K_dot_y_flat( + blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb, + ib, ip, is, l0, dbg, y0, y1, y2, y3); +#else sumf[row] += block_q_6_K_dot_y_flat( blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb, ib, ip, is, l0, y0, y1, y2, y3); +#endif } } } +#if defined(ADRENO_OLD_COMPILER) + // Optimizer barrier. This compiler drops the sumf partials unless a side effect + // forces them to materialize. q6k_mask is a kernel arg the compiler cannot prove + // is never 0xFE (the host always passes 0xFF), so the printf survives compilation + // but never executes. FRAGILE: the exact set and placement of these guarded + // printfs is load-bearing on E031.41 -- removing any one re-breaks q6_K. + if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && ne00==256 && ne01==16 && get_sub_group_local_id()<16) { + printf("Q6KLANE lane=%d ip=%d il=%d is=%d l0=%d sumf0=%f\n", + get_sub_group_local_id(), ip, il, is, l0, sumf[0]); + } +#endif for (int row = 0; row < N_DST; row++) { float tot = sub_group_reduce_add(sumf[row]); if (get_sub_group_local_id() == 0 && first_row + row < ne01) { dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot; +#if defined(ADRENO_OLD_COMPILER) + if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ne00==256 && ne01==16) + printf("Q6KTOT tot=%f\n", tot); +#endif } } } From a30273376ef669023334fc20ad02ae4ed8196a65 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 21:31:29 +0300 Subject: [PATCH 21/44] metal : clamp K extent in tensor API mat-mat kernel for K not a multiple of 32 (#27450) The Tensor API mat-mat path of kernel_mul_mm (GGML_METAL_HAS_TENSOR) fed a static K=32 tile to the matmul2d op on every iteration. On the last, partial K tile (ne00 % 32 != 0) the src1 slice extends past the K extent of the tensor, and the op reads those out-of-bounds elements (undefined behavior per the MSL specification, section 2.22.2). Depending on stale memory contents, this corrupted the result or produced NaN. Make the matmul2d op use dynamic_extent for K, and clamp the K extent of both operand tensor views to the remaining valid K range (min(32, K - loop_k)) per iteration, so the op reads exactly the valid K range on every iteration (mirroring the tail handling of the MPP matmul2d examples). On K-aligned inputs the clamp degenerates to the full 32-wide tile: the only difference from the static-K op is that the dynamic-K op derives K from the operand extents and edge-checks the tile against the tensor extents (a handful of integer ops per iteration). Add test-backend-ops MUL_MAT cases with K not a multiple of 32 to exercise the unaligned K path. Assisted-by: pi:llama.cpp/Qwen3.8-27B --- ggml/src/ggml-metal/ggml-metal.metal | 15 +++++++++++---- tests/test-backend-ops.cpp | 8 ++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 949931c8d..27f97b5e0 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -10365,9 +10365,12 @@ kernel void kernel_mul_mm( auto tB = tensor(ptrB, dextents(K, N), array({1, strideB})); // Configure matmul operation + // note: K is dynamic_extent (clamped to the valid range in PHASE 2), since a static + // N_MM_NK_TOTAL K tile would read src1 out of bounds when K % N_MM_NK_TOTAL != 0 + // ref: https://github.com/ggml-org/llama.cpp/pull/27064 mpp::tensor_ops::matmul2d< mpp::tensor_ops::matmul2d_descriptor( - NRB, NRA, N_MM_NK_TOTAL, false, true, true, + NRB, NRA, static_cast(dynamic_extent), false, true, true, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), execution_simdgroups> mm; @@ -10419,10 +10422,14 @@ kernel void kernel_mul_mm( threadgroup_barrier(mem_flags::mem_threadgroup); // === PHASE 2: Tensor matmul === - auto mA = tA.slice(0, 0); - auto mB = tB.slice(loop_k, rb); + // Clamp the K extent of both operand tensors to the remaining valid K range so + // the dynamic-K op never reads past the K extent of src1 (or the staged A tile). + const int kExt = min(N_MM_NK_TOTAL, K - loop_k); - mm.run(mB, mA, cT); + auto tAv = tensor(sa, dextents(kExt, NRA), array({1, N_MM_NK_TOTAL})); + auto tBv = tensor(ptrB + loop_k + rb * strideB, dextents(kExt, N - rb), array({1, strideB})); + + mm.run(tBv, tAv, cT); threadgroup_barrier(mem_flags::mem_threadgroup); } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 17098825b..89b954a7d 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9298,6 +9298,14 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 6, 4096, 5120, {1, 1}, {1, 1})); + // K not a multiple of 32 + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 65, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 80, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 32, 80, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 32, 80, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 588, {1, 1}, {1, 1})); // 14*14*3, e.g. conv_2d im2col + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 80, {4, 1}, {1, 1})); + #if 0 // test the mat-mat path for Metal for (int k = 1; k < 512; ++k) { From 0e1d9185c5fe82e905d1f5ae6b2e5dcd607a8dfd Mon Sep 17 00:00:00 2001 From: Eve <139727413+netrunnereve@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:01:32 +0000 Subject: [PATCH 22/44] ci: use shell script to check cmake pkg (#27414) * use regular script to build cmake pkg * use old grep without perl --- .github/workflows/build-cmake-pkg.yml | 36 ++++++++++++--------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-cmake-pkg.yml b/.github/workflows/build-cmake-pkg.yml index 0e4069ce3..c44fba2c6 100644 --- a/.github/workflows/build-cmake-pkg.yml +++ b/.github/workflows/build-cmake-pkg.yml @@ -27,30 +27,26 @@ jobs: cmake --install build --prefix "$PREFIX" --config Release export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake - tclsh <<'EOF' - set build(commit) [string trim [exec git rev-parse --short HEAD]] - set build(number) [string trim [exec git rev-list --count HEAD]] + build_commit=$(git rev-parse --short HEAD | xargs) + build_number=$(git rev-list --count HEAD | xargs) - set cmakelists [read [open "CMakeLists.txt" r]] - regexp {set\(LLAMA_VERSION_MAJOR\s+(\d+)\)} $cmakelists -> major - regexp {set\(LLAMA_VERSION_MINOR\s+(\d+)\)} $cmakelists -> minor - regexp {set\(LLAMA_VERSION_PATCH\s+(\d+)\)} $cmakelists -> patch - set build(version) "$major.$minor.$patch" + major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$") + minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$") + patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$") + build_version="$major.$minor.$patch" - set llamaconfig [read [open "$env(LLAMA_CONFIG)" r]] - set checks [list "set\\(LLAMA_VERSION \\s+$build(version)\\)" \ - "set\\(LLAMA_BUILD_COMMIT\\s+$build(commit)\\)" \ - "set\\(LLAMA_BUILD_NUMBER\\s+$build(number)\\)"] + checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)" + "set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)" + "set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)") - puts -nonewline "Checking llama-config.cmake version... " - foreach check $checks { - if {![regexp -expanded -- $check $llamaconfig]} { - puts "\"$check\" failed!" + for check in "${checks[@]}"; do + if ! grep -qE "$check" "$LLAMA_CONFIG"; then + echo "Checking llama-config.cmake version... \"$check\" failed!" exit 1 - } - } - puts "success." - EOF + fi + done + + echo "Checking llama-config.cmake version... success." cd examples/simple-cmake-pkg cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake From 749f688fcaa4c472ec034b08cb8a907c45cfaa02 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Fri, 21 Aug 2026 00:36:57 +0200 Subject: [PATCH 23/44] ggml: support ggml_rope_set_offset on opencl, sycl, wgpu, hexagon (#27345) * ggml: support ggml_rope_set_offset on opencl, sycl, wgpu, hexagon * rm inplace optimization --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 5 +- ggml/src/ggml-hexagon/htp/rope-ops.c | 22 ++-- ggml/src/ggml-opencl/ggml-opencl.cpp | 11 +- ggml/src/ggml-opencl/kernels/rope.cl | 92 +++++++++-------- ggml/src/ggml-sycl/ggml-sycl.cpp | 2 - ggml/src/ggml-sycl/rope.cpp | 106 +++++++++++--------- ggml/src/ggml-webgpu/ggml-webgpu.cpp | 8 +- ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl | 18 ++-- 8 files changed, 152 insertions(+), 112 deletions(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index b262a73d9..e8a5009b3 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -3180,8 +3180,9 @@ static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * s static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { const int32_t * op_params = &op->op_params[0]; - if (op_params[15] != 0) { - return false; // FIXME: support ggml_rope_set_offset + // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems) + if (op_params[15] % 32 != 0) { + return false; } int mode = op_params[2]; diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 5bc7d74f5..6c6898249 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -53,6 +53,7 @@ struct htp_rope_context { int32_t n_dims; + int32_t n_offs; int32_t mode; int32_t n_ctx_orig; int32_t sections[4]; @@ -405,32 +406,40 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict static void inline rope_basic_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { + const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op) #pragma unroll(4) for (uint32_t i = 0; i < nr; i++) { float * d = (float *) (dst + i * rctx->dst_row_size_aligned); float * s = (float *) (src + i * rctx->src0_row_size_aligned); - hvx_rope_f32_aa(d, s, rctx->n_dims, theta_cache); + hvx_rope_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache); // fill the remain channels with data from src tensor - if (rctx->n_dims < ne0) { - hvx_copy_f32_uu((uint8_t *)(d + rctx->n_dims), (uint8_t *)(s + rctx->n_dims), ne0 - rctx->n_dims); + if (n_offs > 0) { + hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs); + } + if (n_offs + rctx->n_dims < ne0) { + hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims); } } } static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { + const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op) #pragma unroll(4) for (uint32_t i = 0; i < nr; i++) { float * d = (float *) (dst + i * rctx->dst_row_size_aligned); float * s = (float *) (src + i * rctx->src0_row_size_aligned); - hvx_rope_neox_f32_aa(d, s, rctx->n_dims, theta_cache); + hvx_rope_neox_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache); // fill the remain channels with data from src tensor - if (rctx->n_dims < ne0) { - hvx_copy_f32_uu((uint8_t *)(d + rctx->n_dims), (uint8_t *)(s + rctx->n_dims), ne0 - rctx->n_dims); + if (n_offs > 0) { + hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs); + } + if (n_offs + rctx->n_dims < ne0) { + hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims); } } } @@ -673,6 +682,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { rctx.n_dims = ((const int32_t *) op_params)[1]; rctx.mode = ((const int32_t *) op_params)[2]; rctx.n_ctx_orig = ((const int32_t *) op_params)[4]; + rctx.n_offs = ((const int32_t *) op_params)[15]; memcpy(&rctx.freq_base, (int32_t *) op_params + 5, sizeof(float)); memcpy(&rctx.freq_scale, (int32_t *) op_params + 6, sizeof(float)); diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 49cd9fd35..26f952a17 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -7434,9 +7434,6 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_OP_DIAG_MASK_INF: return op->ne[3] == 1; case GGML_OP_ROPE: { - if (((const int32_t *) op->op_params)[15] != 0) { - return false; // FIXME: support ggml_rope_set_offset - } const int mode = ((const int32_t *) op->op_params)[2]; const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; const bool is_vision = mode == GGML_ROPE_TYPE_VISION; @@ -23910,6 +23907,7 @@ static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const const int n_dims = ((int *) dst->op_params)[1]; const int mode = ((int *) dst->op_params)[2]; const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; + const int n_offs = ((int32_t *) dst->op_params)[15]; float freq_base; float freq_scale; @@ -23938,6 +23936,7 @@ static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const if (is_vision) { GGML_ASSERT(n_dims == ne00/2); + GGML_ASSERT(n_offs == 0); // offset not supported for vision, as the rotated pairs span the whole row } cl_kernel kernel; @@ -24029,6 +24028,12 @@ static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const if (is_mrope && !is_vision) { CL_CHECK(clSetKernelArg(kernel, 34, sizeof(int), &is_imrope)); } + // norm and neox have n_offs after beta_slow, mrope has it after is_imrope + if (!is_mrope && !is_vision) { + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(int), &n_offs)); + } else if (is_mrope && !is_vision) { + CL_CHECK(clSetKernelArg(kernel, 35, sizeof(int), &n_offs)); + } size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; diff --git a/ggml/src/ggml-opencl/kernels/rope.cl b/ggml/src/ggml-opencl/kernels/rope.cl index 82f4cd874..27fdbbbc4 100644 --- a/ggml/src/ggml-opencl/kernels/rope.cl +++ b/ggml/src/ggml-opencl/kernels/rope.cl @@ -75,7 +75,8 @@ kernel void kernel_rope_norm_f32( float ext_factor, float attn_factor, float beta_fast, - float beta_slow + float beta_slow, + int n_offs ) { src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); @@ -94,14 +95,15 @@ kernel void kernel_rope_norm_f32( float inv_ndims = -1.f/n_dims; for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) { - if (i0 < n_dims) { - int ic = i0/2; + if (i0 >= n_offs && i0 < n_offs + n_dims) { + int iw = i0 - n_offs; // relative idx + int ic = iw/2; - float theta = theta_base * pow(freq_base, inv_ndims*i0); + float theta = theta_base * pow(freq_base, inv_ndims*iw); float freq_factor = src2 != src0 ? src2[ic] : 1.0f; - float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor); + float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor); global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); @@ -154,7 +156,8 @@ kernel void kernel_rope_norm_f16( float ext_factor, float attn_factor, float beta_fast, - float beta_slow + float beta_slow, + int n_offs ) { src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); @@ -173,14 +176,15 @@ kernel void kernel_rope_norm_f16( float inv_ndims = -1.f/n_dims; for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) { - if (i0 < n_dims) { - int ic = i0/2; + if (i0 >= n_offs && i0 < n_offs + n_dims) { + int iw = i0 - n_offs; // relative idx + int ic = iw/2; - float theta = theta_base * pow(freq_base, inv_ndims*i0); + float theta = theta_base * pow(freq_base, inv_ndims*iw); float freq_factor = src2 != src0 ? src2[ic] : 1.0f; - float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor); + float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor); global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); @@ -233,7 +237,8 @@ kernel void kernel_rope_neox_f32( float ext_factor, float attn_factor, float beta_fast, - float beta_slow + float beta_slow, + int n_offs ) { src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); @@ -252,17 +257,18 @@ kernel void kernel_rope_neox_f32( float inv_ndims = -1.f/n_dims; for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) { - if (i0 < n_dims) { - int ic = i0/2; + if (i0 >= n_offs && i0 < n_offs + n_dims) { + int iw = i0 - n_offs; // relative idx + int ic = iw/2; - const float theta = theta_base * pow(freq_base, inv_ndims*i0); + const float theta = theta_base * pow(freq_base, inv_ndims*iw); const float freq_factor = src2 != src0 ? src2[ic] : 1.0f; - float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor); + float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor); - global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00); - global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0); + global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00); + global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + (n_offs + ic)*nb0); const float x0 = src[0]; const float x1 = src[n_dims/2]; @@ -312,7 +318,8 @@ kernel void kernel_rope_neox_f16( float ext_factor, float attn_factor, float beta_fast, - float beta_slow + float beta_slow, + int n_offs ) { src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); @@ -331,17 +338,18 @@ kernel void kernel_rope_neox_f16( float inv_ndims = -1.f/n_dims; for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) { - if (i0 < n_dims) { - int ic = i0/2; + if (i0 >= n_offs && i0 < n_offs + n_dims) { + int iw = i0 - n_offs; // relative idx + int ic = iw/2; - const float theta = theta_base * pow(freq_base, inv_ndims*i0); + const float theta = theta_base * pow(freq_base, inv_ndims*iw); const float freq_factor = src2 != src0 ? src2[ic] : 1.0f; - float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor); + float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor); - global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00); - global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0); + global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00); + global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + (n_offs + ic)*nb0); const float x0 = src[0]; const float x1 = src[n_dims/2]; @@ -393,7 +401,8 @@ kernel void kernel_rope_multi_f32( float beta_fast, float beta_slow, int4 sections, - int is_imrope + int is_imrope, + int n_offs ) { src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); @@ -414,10 +423,11 @@ kernel void kernel_rope_multi_f32( float inv_ndims = -1.f/n_dims; for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) { - if (i0 < n_dims) { - int ic = i0/2; + if (i0 >= n_offs && i0 < n_offs + n_dims) { + int iw = i0 - n_offs; // relative idx + int ic = iw/2; - const int sector = (i0 / 2) % sect_dims; + const int sector = ic % sect_dims; float theta_base = 0.0f; if (is_imrope) { @@ -445,14 +455,14 @@ kernel void kernel_rope_multi_f32( } } - const float theta = theta_base * pow(freq_base, inv_ndims*i0); + const float theta = theta_base * pow(freq_base, inv_ndims*iw); const float freq_factor = src2 != src0 ? src2[ic] : 1.0f; - float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor); + float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor); - global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00); - global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0); + global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00); + global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + (n_offs + ic)*nb0); const float x0 = src[0]; const float x1 = src[n_dims/2]; @@ -504,7 +514,8 @@ kernel void kernel_rope_multi_f16( float beta_fast, float beta_slow, int4 sections, - int is_imrope + int is_imrope, + int n_offs ) { src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); @@ -525,10 +536,11 @@ kernel void kernel_rope_multi_f16( float inv_ndims = -1.f/n_dims; for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) { - if (i0 < n_dims) { - int ic = i0/2; + if (i0 >= n_offs && i0 < n_offs + n_dims) { + int iw = i0 - n_offs; // relative idx + int ic = iw/2; - const int sector = (i0 / 2) % sect_dims; + const int sector = ic % sect_dims; float theta_base = 0.0f; if (is_imrope) { @@ -556,14 +568,14 @@ kernel void kernel_rope_multi_f16( } } - const float theta = theta_base * pow(freq_base, inv_ndims*i0); + const float theta = theta_base * pow(freq_base, inv_ndims*iw); const float freq_factor = src2 != src0 ? src2[ic] : 1.0f; - float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor); + float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor); - global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00); - global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0); + global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00); + global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + (n_offs + ic)*nb0); const float x0 = src[0]; const float x1 = src[n_dims/2]; diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index c7434a6bd..57aae9011 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6242,8 +6242,6 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons } case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: - // FIXME: support ggml_rope_set_offset - return ((const int32_t *) op->op_params)[15] == 0; case GGML_OP_IM2COL: case GGML_OP_IM2COL_3D: case GGML_OP_UPSCALE: diff --git a/ggml/src/ggml-sycl/rope.cpp b/ggml/src/ggml-sycl/rope.cpp index 9d83a1e9f..b6d22559d 100644 --- a/ggml/src/ggml-sycl/rope.cpp +++ b/ggml/src/ggml-sycl/rope.cpp @@ -41,7 +41,7 @@ template static void rope_norm(const T *x, D *dst, const int ne00, const int ne01, const int ne02, const int s01, const int s02, const int s03, const int s1, const int s2, const int s3, - const int n_dims, const int32_t *pos, + const int n_dims, const int n_offs, const int32_t *pos, const float freq_scale, const float ext_factor, const float attn_factor, const rope_corr_dims corr_dims, const float theta_scale, const float *freq_factors, @@ -78,19 +78,21 @@ static void rope_norm(const T *x, D *dst, const int ne00, const int ne01, ggml_sycl_memcpy_1<4>(dst + idst, &v); } }; - if (i0 >= n_dims) { + if (i0 < n_offs || i0 >= n_offs + n_dims) { store_coaelsced(x[ix + 0], x[ix + 1]); return; } - const float theta_base = pos[i2] * dpct::pow(theta_scale, i0 / 2.0f); + const int iw = i0 - n_offs; // relative idx - const float freq_factor = has_ff ? freq_factors[i0 / 2] : 1.0f; + const float theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f); + + const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f; float cos_theta; float sin_theta; - rope_yarn(theta_base / freq_factor, freq_scale, corr_dims, i0, + rope_yarn(theta_base / freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta); const float x0 = x[ix + 0]; @@ -104,7 +106,7 @@ template static void rope_neox(const T *x, D *dst, const int ne00, const int ne01, const int ne02, const int s01, const int s02, const int s03, const int s1, const int s2, const int s3, - const int n_dims, const int32_t *pos, + const int n_dims, const int n_offs, const int32_t *pos, const float freq_scale, const float ext_factor, const float attn_factor, const rope_corr_dims corr_dims, const float theta_scale, const float *freq_factors, @@ -132,35 +134,38 @@ static void rope_neox(const T *x, D *dst, const int ne00, const int ne01, idst += row_indices[i2] * set_rows_stride; } - if (i0 >= n_dims) { + if (i0 < n_offs || i0 >= n_offs + n_dims) { dst[idst + i0 / 2 + 0] = ggml_sycl_cast(x[ix + i0 / 2 + 0]); dst[idst + i0 / 2 + 1] = ggml_sycl_cast(x[ix + i0 / 2 + 1]); return; } - const float theta_base = pos[i2] * dpct::pow(theta_scale, i0 / 2.0f); + const int iw = i0 - n_offs; // relative idx - const float freq_factor = has_ff ? freq_factors[i0 / 2] : 1.0f; + const float theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f); + + const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f; float cos_theta; float sin_theta; - rope_yarn(theta_base / freq_factor, freq_scale, corr_dims, i0, + rope_yarn(theta_base / freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta); - const float x0 = x[ix + 0]; - const float x1 = x[ix + n_dims / 2]; + // idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2 + const float x0 = x[ix + n_offs / 2 + 0]; + const float x1 = x[ix + n_offs / 2 + n_dims / 2]; - dst[idst + 0] = ggml_sycl_cast(x0 * cos_theta - x1 * sin_theta); - dst[idst + n_dims / 2] = ggml_sycl_cast(x0 * sin_theta + x1 * cos_theta); + dst[idst + n_offs / 2 + 0] = ggml_sycl_cast(x0 * cos_theta - x1 * sin_theta); + dst[idst + n_offs / 2 + n_dims / 2] = ggml_sycl_cast(x0 * sin_theta + x1 * cos_theta); } template static void rope_multi(const T *x, T *dst, const int ne00, const int ne01, const int ne02, const int s01, const int s02, const int s03, const int s1, const int s2, const int s3, - const int n_dims, const int32_t *pos, + const int n_dims, const int n_offs, const int32_t *pos, const float freq_scale, const float ext_factor, const float attn_factor, const rope_corr_dims corr_dims, const float theta_scale, const float *freq_factors, @@ -183,54 +188,57 @@ static void rope_multi(const T *x, T *dst, const int ne00, const int ne01, int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3; const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03; - if (i0 >= n_dims) { + if (i0 < n_offs || i0 >= n_offs + n_dims) { dst[idst + i0 / 2 + 0] = x[ix + i0 / 2 + 0]; dst[idst + i0 / 2 + 1] = x[ix + i0 / 2 + 1]; return; } + const int iw = i0 - n_offs; // relative idx + const int sect_dims = sections.v[0] + sections.v[1] + sections.v[2] + sections.v[3]; const int sec_w = sections.v[1] + sections.v[0]; - const int sector = (i0 / 2) % sect_dims; + const int sector = (iw / 2) % sect_dims; float theta_base = 0.0; if (is_imrope) { if (sector % 3 == 1 && sector < 3 * sections.v[1]) { // h - theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, i0 / 2.0f); + theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, iw / 2.0f); } else if (sector % 3 == 2 && sector < 3 * sections.v[2]) { // w - theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, i0 / 2.0f); + theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, iw / 2.0f); } else if (sector % 3 == 0 && sector < 3 * sections.v[0]) { // t - theta_base = pos[i2] * dpct::pow(theta_scale, i0 / 2.0f); + theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f); } else { - theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, i0 / 2.0f); + theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, iw / 2.0f); } } else { if (sector < sections.v[0]) { - theta_base = pos[i2] * dpct::pow(theta_scale, i0 / 2.0f); + theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f); } else if (sector >= sections.v[0] && sector < sec_w) { - theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, i0 / 2.0f); + theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, iw / 2.0f); } else if (sector >= sec_w && sector < sec_w + sections.v[2]) { - theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, i0 / 2.0f); + theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, iw / 2.0f); } else if (sector >= sec_w + sections.v[2]) { - theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, i0 / 2.0f); + theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, iw / 2.0f); } } - const float freq_factor = has_ff ? freq_factors[i0 / 2] : 1.0f; + const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f; float cos_theta; float sin_theta; - rope_yarn(theta_base / freq_factor, freq_scale, corr_dims, i0, + rope_yarn(theta_base / freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta); - const float x0 = x[ix + 0]; - const float x1 = x[ix + n_dims / 2]; + // idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2 + const float x0 = x[ix + n_offs / 2 + 0]; + const float x1 = x[ix + n_offs / 2 + n_dims / 2]; - dst[idst + 0] = x0 * cos_theta - x1 * sin_theta; - dst[idst + n_dims / 2] = x0 * sin_theta + x1 * cos_theta; + dst[idst + n_offs / 2 + 0] = x0 * cos_theta - x1 * sin_theta; + dst[idst + n_offs / 2 + n_dims / 2] = x0 * sin_theta + x1 * cos_theta; } template @@ -293,7 +301,7 @@ static void rope_norm_sycl(const T *x, D *dst, const int ne00, const int ne01, const int ne02, const int s01, const int s02, const int s03, const int s1, const int s2, const int s3, const int n_dims, - const int nr, const int32_t *pos, const float freq_scale, + const int n_offs, const int nr, const int32_t *pos, const float freq_scale, const float freq_base, const float ext_factor, const float attn_factor, const rope_corr_dims corr_dims, const float *freq_factors, const int64_t *row_indices, @@ -313,7 +321,7 @@ rope_norm_sycl(const T *x, D *dst, const int ne00, const int ne01, GGML_UNUSED(item_ct1); rope_norm( x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, - pos, freq_scale, ext_factor, attn_factor, corr_dims, + n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride); }); } else { @@ -323,7 +331,7 @@ rope_norm_sycl(const T *x, D *dst, const int ne00, const int ne01, GGML_UNUSED(item_ct1); rope_norm( x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, - pos, freq_scale, ext_factor, attn_factor, corr_dims, + n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride); }); } @@ -334,7 +342,7 @@ static void rope_neox_sycl(const T *x, D *dst, const int ne00, const int ne01, const int ne02, const int s01, const int s02, const int s03, const int s1, const int s2, const int s3, const int n_dims, - const int nr, const int32_t *pos, const float freq_scale, + const int n_offs, const int nr, const int32_t *pos, const float freq_scale, const float freq_base, const float ext_factor, const float attn_factor, const rope_corr_dims corr_dims, const float *freq_factors, const int64_t *row_indices, @@ -354,7 +362,7 @@ rope_neox_sycl(const T *x, D *dst, const int ne00, const int ne01, GGML_UNUSED(item_ct1); rope_neox( x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, - pos, freq_scale, ext_factor, attn_factor, corr_dims, + n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride); }); } else { @@ -364,7 +372,7 @@ rope_neox_sycl(const T *x, D *dst, const int ne00, const int ne01, GGML_UNUSED(item_ct1); rope_neox( x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, - pos, freq_scale, ext_factor, attn_factor, corr_dims, + n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride); }); } @@ -375,7 +383,7 @@ static void rope_multi_sycl(const T *x, T *dst, const int ne00, const int ne01, const int ne02, const int s01, const int s02, const int s03, const int s1, const int s2, const int s3, const int n_dims, - const int nr, const int32_t *pos, const float freq_scale, + const int n_offs, const int nr, const int32_t *pos, const float freq_scale, const float freq_base, const float ext_factor, const float attn_factor, const rope_corr_dims corr_dims, const float *freq_factors, const mrope_sections sections, @@ -395,7 +403,7 @@ rope_multi_sycl(const T *x, T *dst, const int ne00, const int ne01, GGML_UNUSED(item_ct1); rope_multi( x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, - pos, freq_scale, ext_factor, attn_factor, corr_dims, + n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope); }); } else { @@ -405,7 +413,7 @@ rope_multi_sycl(const T *x, T *dst, const int ne00, const int ne01, GGML_UNUSED(item_ct1); rope_multi( x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, - pos, freq_scale, ext_factor, attn_factor, corr_dims, + n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope); }); } @@ -497,6 +505,7 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst, const int n_dims = ((int32_t *)dst->op_params)[1]; const int mode = ((int32_t *)dst->op_params)[2]; const int n_ctx_orig = ((int32_t *)dst->op_params)[4]; + const int n_offs = ((int32_t *)dst->op_params)[15]; mrope_sections sections; float freq_base; @@ -526,6 +535,7 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst, if (is_vision) { GGML_ASSERT(n_dims == ne00 / 2); + GGML_ASSERT(n_offs == 0); // offset not supported for vision, as the rotated pairs span the whole row } const int32_t *pos = (const int32_t *)src1_d; @@ -545,19 +555,19 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst, if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) { rope_neox_sycl( (const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01, - s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base, + s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, set_rows_stride, stream); } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) { rope_neox_sycl( (const float *)src0_d, (sycl::half *)dst_d, ne00, ne01, ne02, - s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, + s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, set_rows_stride, stream); } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) { rope_neox_sycl( (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01, - ne02, s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, + ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, set_rows_stride, stream); } else { @@ -568,13 +578,13 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst, if (src0->type == GGML_TYPE_F32) { rope_multi_sycl((const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01, s02, s03, s1, s2, - s3, n_dims, nr, pos, freq_scale, freq_base, + s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope, stream); } else if (src0->type == GGML_TYPE_F16) { rope_multi_sycl( (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01, - ne02, s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, + ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope, stream); } else { @@ -602,19 +612,19 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst, if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) { rope_norm_sycl( (const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01, - s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base, + s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, set_rows_stride, stream); } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) { rope_norm_sycl( (const float *)src0_d, (sycl::half *)dst_d, ne00, ne01, ne02, - s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, + s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, set_rows_stride, stream); } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) { rope_norm_sycl( (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01, - ne02, s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, + ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, set_rows_stride, stream); } else { diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index 4367f9a61..2434848a5 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -2714,6 +2714,7 @@ static webgpu_encoded_op ggml_webgpu_rope(webgpu_context & ctx, const int n_dims = ((int32_t *) dst->op_params)[1]; const int mode = ((int32_t *) dst->op_params)[2]; const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; + const int n_offs = ((int32_t *) dst->op_params)[15]; float freq_base; float freq_scale; @@ -2762,7 +2763,8 @@ static webgpu_encoded_op ggml_webgpu_rope(webgpu_context & ctx, (uint32_t) sections[0], (uint32_t) sections[1], (uint32_t) sections[2], - (uint32_t) sections[3] + (uint32_t) sections[3], + (uint32_t) n_offs }; std::vector entries = { ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src0), @@ -4472,9 +4474,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const supports_op = (op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32) && ggml_is_contiguous_rows(src0); break; case GGML_OP_ROPE: - // FIXME: support ggml_rope_set_offset - supports_op = - (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && ((const int32_t *) op->op_params)[15] == 0; + supports_op = op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16; break; case GGML_OP_GLU: switch (ggml_get_glu_op(op)) { diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl index 1c874e142..6ff53088c 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl @@ -38,7 +38,8 @@ struct Params { sections0: u32, sections1: u32, sections2: u32, - sections3: u32 + sections3: u32, + n_offs: u32 }; @group(0) @binding(0) @@ -126,7 +127,8 @@ fn rope_yarn(theta_extrap: f32, i: u32) -> vec2 { fn pair_base(i0: u32, div_2: bool) -> u32 { if (div_2) { - return i0 / 2; + // first channel of the rotated pair: n_offs + (i0 - n_offs)/2 + return i0 / 2 + params.n_offs / 2; } else { return i0; } @@ -165,20 +167,22 @@ fn main(@builtin(global_invocation_id) gid: vec3) { let i_src_row = params.offset_src0 + i3 * params.stride_src03 + i2 * params.stride_src02 + i1 * params.stride_src01; let i_dst_row = params.offset_dst + i3 * params.stride_dst3 + i2 * params.stride_dst2 + i1 * params.stride_dst1; - if (i0 >= params.n_dims && !is_vision) { + if ((i0 < params.n_offs || i0 >= params.n_offs + params.n_dims) && !is_vision) { let i_src = i_src_row + i0; let i_dst = i_dst_row + i0; rotate(i_dst, i_dst + 1, f32(src0[i_src]), f32(src0[i_src + 1])); return; } + let iw = i0 - params.n_offs; // relative idx + var theta_base_mult: u32 = 0; - var theta_scale_pwr: u32 = i0 / 2; + var theta_scale_pwr: u32 = iw / 2; if (is_mrope) { let sect_dims = params.sections0 + params.sections1 + params.sections2 + params.sections3; let sec_w = params.sections1 + params.sections0; let sec_e = params.sections2 + sec_w; - let sector = (i0 / 2) % sect_dims; + let sector = (iw / 2) % sect_dims; if (is_imrope) { if (sector % 3 == 1 && sector < 3 * params.sections1) { theta_base_mult = 1; @@ -203,7 +207,7 @@ fn main(@builtin(global_invocation_id) gid: vec3) { } else if (sector >= sec_e) { if (is_vision) { theta_scale_pwr = sector - sec_e; - theta_scale_pwr = (i0 / 2) % sec_e; + theta_scale_pwr = (iw / 2) % sec_e; } theta_base_mult = 3; } else if (is_vision) { @@ -212,7 +216,7 @@ fn main(@builtin(global_invocation_id) gid: vec3) { } } let theta_base = f32(src1[params.offset_src1 + i2 + params.ne2 * theta_base_mult]) * pow(params.theta_scale, f32(theta_scale_pwr)); - let thetas = rope_yarn(theta_base/freq_factor(i0), i0); + let thetas = rope_yarn(theta_base/freq_factor(iw), iw); let i_src = i_src_row + pair_base(i0, is_neox || is_mrope || is_vision); let i_dst = i_dst_row + pair_base(i0, is_neox || is_mrope || is_vision); From a298422da78eb75e440a7de0ca408af64d323d93 Mon Sep 17 00:00:00 2001 From: vk <89937361+itsvedantkumar@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:06:59 +0530 Subject: [PATCH 24/44] docs: fix typos in ET.md (#27457) --- docs/backend/ET.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/backend/ET.md b/docs/backend/ET.md index 8d9ba12c8..8ebc15fb7 100644 --- a/docs/backend/ET.md +++ b/docs/backend/ET.md @@ -116,7 +116,7 @@ in inline assembler. Most kernels are very naive with lots of low hanging fruits left: > [!IMPORTANT] -> Several assembly instructions emmited by the compiler are not implemented +> Several assembly instructions emitted by the compiler are not implemented > in hardware and software emulation in firmware is not ready yet. > Eventually firmware will transparently trap unimplemented instructions > and will emulate them inside exception handler. Until then, kernel @@ -138,12 +138,12 @@ Most kernels are very naive with lots of low hanging fruits left: > kernel build process. Feel free to take ideas/code from there or try linking > it in. -Before commiting any changes to operations and/or kernels, don't forget +Before committing any changes to operations and/or kernels, don't forget to update supported ops reports (instructions at `docs/ops.md`). When logging is enabled (e.g. by setting `--log-file` cli param), each compute kernel run outputs a line with -pipe-delimited key-value pairs containing kernel level performance infomation. +pipe-delimited key-value pairs containing kernel level performance information. Line is prefixed with `ET_PERF`: ``` @@ -160,7 +160,7 @@ to `GGML_ET_PROFILE/et_runtime_trace.json` and `GGML_ET_PROFILE/kernel_map` on e ### Uberkernel -The in-knernel implementaiton of device dispatch/kernel fusion. The ET SDK has a non-trivial op-to-op gap. `Uberkernel` (name taken from the original Esperanto AI's compiler) +The in-kernel implementation of device dispatch/kernel fusion. The ET SDK has a non-trivial op-to-op gap. `Uberkernel` (name taken from the original Esperanto AI's compiler) dispatches multiple already existing kernel implementations with device side synchronization. Due to the processor's design, there is no natural memory visibility horizon between sub-kernel invocations. This makes uberkernel much more difficult to develop and debug. Currently Uberkerel is hidden begind the `GGML_ET_UBERKERNEL` environment variable and is disabled by default. Setting it to 1 enables it and provides significant performance improvements but is only From b2e5e9b28b2484fbf94b543432ece638996a8b97 Mon Sep 17 00:00:00 2001 From: Chris Danis Date: Fri, 21 Aug 2026 01:13:58 -0400 Subject: [PATCH 25/44] TP: enable tensor split for LFM2/LFM2MOE (#26993) Assisted-by: deepseek-v4-flash --- src/llama-arch.cpp | 2 -- src/llama-model.cpp | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 408954401..c9b504c33 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -1062,8 +1062,6 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: case LLM_ARCH_GRANITE_HYBRID: - case LLM_ARCH_LFM2: - case LLM_ARCH_LFM2MOE: case LLM_ARCH_MINIMAX_01: case LLM_ARCH_MINIMAX_M2: case LLM_ARCH_MINIMAX_M3: diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 3759c8625..d7874e0a9 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -487,6 +487,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight"); } if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) { + if (ud->model->arch == LLM_ARCH_LFM2 || ud->model->arch == LLM_ARCH_LFM2MOE) { + // the LFM2 shortconv block runs fully mirrored, so its conv state must be mirrored too + return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED, ""); + } return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight"); } if (std::regex_match(tensor_name, pattern_ssm_conv1d)) { From 9e96cf77ffd4ebd05bad82932906ec3f59ed54ce Mon Sep 17 00:00:00 2001 From: Neo Zhang Date: Fri, 21 Aug 2026 13:14:54 +0800 Subject: [PATCH 26/44] sycl : fix load model with mlock issue (#27250) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 57aae9011..837242879 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -1517,8 +1517,13 @@ static ggml_backend_buffer_t ggml_backend_sycl_host_buffer_type_alloc_buffer(ggm } static size_t ggml_backend_sycl_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { - ggml_backend_sycl_device_context * dev_ctx = (ggml_backend_sycl_device_context *) buft->device->context; - return dpct::dev_mgr::instance().get_device(dev_ctx->device).get_max_mem_alloc_size(); + + if (g_ggml_sycl_enable_host_pinned_mem) { + ggml_backend_sycl_device_context * dev_ctx = (ggml_backend_sycl_device_context *) buft->device->context; + return dpct::dev_mgr::instance().get_device(dev_ctx->device).get_max_mem_alloc_size(); + } else { + return SIZE_MAX; + } } ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type() { From 6602dd338941d1fd562e6ca3934acc5602c15a9d Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Fri, 21 Aug 2026 07:15:40 +0200 Subject: [PATCH 27/44] sycl: fix multiple warnings in compiling sycl backend (#26713) * Update norm.cpp * Update helper.hpp * Update im2col.cpp * Update fattn-mkl.cpp * Update element_wise.cpp * Update fattn-mkl.cpp * Update set_rows.cpp * Update element_wise.cpp * Update ggml-sycl.cpp * Update ggml-sycl.cpp * Update ggml-sycl.cpp * Update ggml-sycl.cpp * Update ggml-sycl.cpp * Update norm.cpp * Update CMakeLists.txt * Update CMakeLists.txt * Update CMakeLists.txt * Update ggml-sycl.cpp --- ggml/src/ggml-cpu/CMakeLists.txt | 9 +++++---- ggml/src/ggml-sycl/dpct/helper.hpp | 2 +- ggml/src/ggml-sycl/element_wise.cpp | 8 ++++---- ggml/src/ggml-sycl/fattn-mkl.cpp | 9 ++++----- ggml/src/ggml-sycl/ggml-sycl.cpp | 20 +++++++++++--------- ggml/src/ggml-sycl/im2col.cpp | 4 ++-- ggml/src/ggml-sycl/norm.cpp | 8 -------- ggml/src/ggml-sycl/set_rows.cpp | 2 +- 8 files changed, 28 insertions(+), 34 deletions(-) diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index a6cc49586..32e1e7aa1 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -737,8 +737,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name) set_target_properties(${GGML_CPU_NAME} PROPERTIES COMPILE_FLAGS "-msimd128") endif() - if (CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") - # The compiler automatically enables "-ffast-math" which can cause NaNs in tests due to "-fassociative-math" - target_compile_options(${GGML_CPU_NAME} PRIVATE "-fno-associative-math") - endif() + if (CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") + # The compiler automatically enables "-ffast-math" which can cause NaNs in tests due to "-fassociative-math" + target_compile_options(${GGML_CPU_NAME} PRIVATE "$<$,$>:$<$:/clang:>-fno-associative-math>") + endif() + endfunction() diff --git a/ggml/src/ggml-sycl/dpct/helper.hpp b/ggml/src/ggml-sycl/dpct/helper.hpp index 664b8e969..85af4cab6 100644 --- a/ggml/src/ggml-sycl/dpct/helper.hpp +++ b/ggml/src/ggml-sycl/dpct/helper.hpp @@ -62,7 +62,7 @@ #define DPCT_UNUSED(x) (void)(x) -inline void _abort(const char * str) { +[[noreturn]] inline void _abort(const char * str) { std::cerr << str << std::endl; std::abort(); } diff --git a/ggml/src/ggml-sycl/element_wise.cpp b/ggml/src/ggml-sycl/element_wise.cpp index 8619ed6f4..95914873e 100644 --- a/ggml/src/ggml-sycl/element_wise.cpp +++ b/ggml/src/ggml-sycl/element_wise.cpp @@ -10,7 +10,7 @@ (ITEM.get_local_range(IDX) * ITEM.get_group(IDX) + ITEM.get_local_id(IDX)) static void acc_f32(const char * x, const char * y, float * dst, const int64_t ne, - const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, @@ -455,7 +455,7 @@ static void unary_mul_sycl(const T * x, const T * g, T * dst, const int64_t k, c namespace ggml_sycl_detail { static void acc_f32_sycl(const char *x, const char *y, float *dst, const int64_t n_elements, - const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, @@ -466,7 +466,7 @@ static void acc_f32_sycl(const char *x, const char *y, float *dst, sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE)), [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { acc_f32(x, y, dst, n_elements, - ne0, ne1, ne2, ne3, + ne0, ne1, ne2, nb00, nb01, nb02, nb03, ne10, ne11, ne12, ne13, nb10, nb11, nb12, nb13, @@ -970,7 +970,7 @@ static inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor const int64_t offset = (int64_t) ((const int32_t *) dst->op_params)[3] / (int64_t) sizeof(float); ggml_sycl_detail::acc_f32_sycl(src0_d, src1_d, dst_d, ggml_nelements(dst), - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], diff --git a/ggml/src/ggml-sycl/fattn-mkl.cpp b/ggml/src/ggml-sycl/fattn-mkl.cpp index fc22b7bdb..2d164a084 100644 --- a/ggml/src/ggml-sycl/fattn-mkl.cpp +++ b/ggml/src/ggml-sycl/fattn-mkl.cpp @@ -43,7 +43,7 @@ static void mkl_fa_pack_q_fp16( dpct::queue_ptr stream, sycl::half * __restrict dst, const float * __restrict q_src, - int n_queries, int n_query_rows, int DKQ, + int n_queries, int DKQ, int gqa_ratio, int kvh_base_head, float q_scale, int64_t q_row_stride, int64_t q_head_stride, int64_t wg_size) { @@ -121,7 +121,7 @@ static void mkl_fa_online_softmax_chunk( float * __restrict VKQ_accum, int q0, int q_rows, int n_queries, int DV, int chunk_size, int chunk_start, - int kvh_head, int gqa_ratio, + int kvh_head, const sycl::half * mask_data, int64_t mask_head_stride, int64_t mask_row_stride, int mask_n_heads, float logit_softcap, int64_t wg_size) { @@ -473,7 +473,6 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * MKL_ACCUM(dequant_time_us, t_deq); // --- Resolve mask pointers --- - const sycl::half * mask_data = nullptr; int64_t mask_head_stride = 0; int64_t mask_row_stride = 0; int mask_n_heads = 0; @@ -547,7 +546,7 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * // 1. Pack all GQA Q heads into fp16 (full n_query_rows) mkl_fa_pack_q_fp16(stream, Q_head_f16_ptr, Q_batch, - n_queries, n_query_rows, DKQ, + n_queries, DKQ, gqa_ratio, kvh_base_head, q_scale, q_row_stride, q_head_stride, wg_size); @@ -605,7 +604,7 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * KQ_max_ptr, KQ_sum_ptr, VKQ_accum_ptr, q0, q_rows, n_queries, DV, this_chunk, chunk_start, - kvh_base_head, gqa_ratio, + kvh_base_head, mask_batch, mask_head_stride, mask_row_stride, mask_n_heads, logit_softcap, wg_size); diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 837242879..7ebdce7fb 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -921,16 +921,16 @@ ggml_backend_sycl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, void * dev_ptr; if (use_usm_system) { - GGML_SYCL_DEBUG("[SYCL] allocating %lu Bytes with USM system\n", size); + GGML_SYCL_DEBUG("[SYCL] allocating %zu Bytes with USM system\n", size); dev_ptr = (void *)aligned_malloc_host(alignment, aligned_size); if (!dev_ptr) { - GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on host\n", __func__, size); + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size); return nullptr; } } else { SYCL_CHECK(CHECK_TRY_ERROR(dev_ptr = (void *)ggml_sycl_malloc_device(size, *stream))); if (!dev_ptr) { - GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device\n", __func__, size); + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on device\n", __func__, size); return nullptr; } } @@ -1177,7 +1177,7 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer, SYCL_CHECK(CHECK_TRY_ERROR(buf = (char *)ggml_sycl_malloc_device(size, *stream))); if (!buf) { char err_buf[1024]; - snprintf(err_buf, 1023, "%s: can't allocate %lu Bytes of memory on device\n", __func__, size); + snprintf(err_buf, 1023, "%s: can't allocate %zu Bytes of memory on device\n", __func__, size); throw std::runtime_error(err_buf); } // set padding to 0 to avoid possible NaN values @@ -1651,7 +1651,7 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool { SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *)ggml_sycl_malloc_device(look_ahead_size, *qptr))); if (!ptr) { - GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device/GPU\n", __func__, look_ahead_size); + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on device/GPU\n", __func__, look_ahead_size); return nullptr; } @@ -1663,7 +1663,7 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool { (uint32_t)(max_size/1024/1024), (uint32_t)(g_sycl_pool_size[id]/1024/1024), (uint32_t)(size/1024/1024)); #endif - // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg look_ahead_size=%lu, return %p\n", look_ahead_size, ptr); + // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg look_ahead_size=%zu, return %p\n", look_ahead_size, ptr); return ptr; } @@ -1843,7 +1843,7 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool { SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr))); if (!ptr) { - GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on host\n", __func__, size); + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size); return nullptr; } pool_size += size; @@ -2779,9 +2779,9 @@ inline void ggml_sycl_op_mul_mat_sycl( const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get(); { +#if GGML_SYCL_DNNL const int64_t gemm_flops = (int64_t)row_diff * src1_ncols * ne10; const bool use_mkl_direct = gemm_flops < 256 * 256 * 256; -#if GGML_SYCL_DNNL if (g_ggml_sycl_enable_dnn && !use_mkl_direct) { DnnlGemmWrapper::row_gemm(ctx, row_diff, src1_ncols, ne10, src0_ddf_i, DnnlGemmWrapper::to_dt(), src1_ddf1_i, DnnlGemmWrapper::to_dt(), @@ -3518,7 +3518,9 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons float * dst_ddf = static_cast(dst->data); const sycl::half * src1_f16 = static_cast(src1->data); +#if GGML_SYCL_DNNL const size_t type_size_src0 = ggml_type_size(src0->type); +#endif const size_t type_size_src1 = ggml_type_size(src1->type); bool is_src0_cont_2 = ggml_is_contiguous_2(src0); @@ -3535,6 +3537,7 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_nc_sycl", dst, /*num_src=*/2, " : converting src1 to fp16"); +#if GGML_SYCL_DNNL // iterate tensor dims and find the slowest moving dim and stride int last_dim=0; int last_str=0; @@ -3554,7 +3557,6 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons } } -#if GGML_SYCL_DNNL // oneDNN handles strided data and does not need overhead of ggml_get_to_fp16_nc_sycl const int64_t ne_src1 = src1->nb[last_str] * src1->ne[last_dim] / type_size_src1; src1_f16_alloc.alloc(ne_src1); diff --git a/ggml/src/ggml-sycl/im2col.cpp b/ggml/src/ggml-sycl/im2col.cpp index 7bf3584fb..e66616759 100644 --- a/ggml/src/ggml-sycl/im2col.cpp +++ b/ggml/src/ggml-sycl/im2col.cpp @@ -85,7 +85,7 @@ static void im2col_sycl(const float * x, */ stream->parallel_for(sycl::nd_range<3>(block_nums * sycl::range<3>(1, 1, MIN(IC_KH_KW, SYCL_IM2COL_BLOCK_SIZE)), sycl::range<3>(1, 1, MIN(IC_KH_KW, SYCL_IM2COL_BLOCK_SIZE))), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3>) { im2col_kernel(x, dst, IC, IW, IH, OH, OW, KW, KH, IC_IH_IW, IH_IW, N_OH, KH_KW, IC_KH_KW, s0, s1, p0, p1, d0, d1); }); @@ -271,7 +271,7 @@ static void im2col_3d_sycl(const float * src, */ stream->parallel_for(sycl::nd_range<3>(block_nums * sycl::range<3>(1, 1, MIN(IC_KD_KH_KW, SYCL_IM2COL_BLOCK_SIZE)), sycl::range<3>(1, 1, MIN(IC_KD_KH_KW, SYCL_IM2COL_BLOCK_SIZE))), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3>) { im2col_3d_kernel(src, dst, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW, OH_OW, KD_KH_KW, ID_IH_IW, KH_KW, IH_IW, IC_ID_IH_IW, IC_KD_KH_KW, OW_KD_KH_KW, OD_OH_OW_IC_KD_KH_KW, OH_OW_IC_KD_KH_KW, OW_IC_KD_KH_KW, N_OD_OH, OD_OH, diff --git a/ggml/src/ggml-sycl/norm.cpp b/ggml/src/ggml-sycl/norm.cpp index 682a9f51e..f98a7a954 100644 --- a/ggml/src/ggml-sycl/norm.cpp +++ b/ggml/src/ggml-sycl/norm.cpp @@ -7,9 +7,6 @@ static void norm_f32(const float* x, float* dst, const int ncols, const int64_t dst_stride_col, const int64_t dst_stride_row, const int64_t dst_stride_channel, const int64_t dst_stride_sample, const float eps, const sycl::nd_item<3>& item_ct1, sycl::float2* s_sum, int block_size) { - const int nrows = item_ct1.get_group_range(2); - const int nchannels = item_ct1.get_group_range(1); - const int nthreads = item_ct1.get_local_range(2); const int sample = item_ct1.get_group(0); const int channel = item_ct1.get_group(1); @@ -155,9 +152,6 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols, const float* mul = nullptr, const int64_t mul_stride_row = 0, const int64_t mul_stride_channel = 0, const int64_t mul_stride_sample = 0, const int mul_nrows = 0, const int mul_nchannels = 0, const int mul_nsamples = 0) { - const int nrows = item_ct1.get_group_range(2); - const int nchannels = item_ct1.get_group_range(1); - const int sample = item_ct1.get_group(0); const int channel = item_ct1.get_group(1); const int row = item_ct1.get_group(2); @@ -225,8 +219,6 @@ static void l2_norm_f32(const float * x, float * dst, const int ncols, const int64_t src_stride_sample, const int64_t dst_stride_col, const int64_t dst_stride_row, const int64_t dst_stride_channel, const int64_t dst_stride_sample, const float eps, const sycl::nd_item<3>& item_ct1, float* s_sum, const int block_size) { - const int nrows = item_ct1.get_group_range(2); - const int nchannels = item_ct1.get_group_range(1); const int row = item_ct1.get_group(2); const int channel = item_ct1.get_group(1); diff --git a/ggml/src/ggml-sycl/set_rows.cpp b/ggml/src/ggml-sycl/set_rows.cpp index 52a0bcb6e..5f8d881a2 100644 --- a/ggml/src/ggml-sycl/set_rows.cpp +++ b/ggml/src/ggml-sycl/set_rows.cpp @@ -291,7 +291,7 @@ static void set_rows_sycl( stream->parallel_for( sycl::nd_range<1>(grid_size * block_size, block_size), - [=](sycl::nd_item<1> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { k_set_rows( src0_d, src1_d, dst_d, ne00, ne01, ne02, From 1cb3f5eb41d51fc98ac6b1d16ff199c427edd68d Mon Sep 17 00:00:00 2001 From: HumerousGorgon <31957201+HumerousGorgon@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:16:29 +0800 Subject: [PATCH 28/44] sycl: Update gate logic for Alchemist GPUs regarding OneDNN features. (#26635) * feat: updated gating logic of fattn-onednn.cpp * verified device types * Update ggml/src/ggml-sycl/fattn-onednn.cpp Accepted recommendations to add bmg_g31 arch. Co-authored-by: Neo Zhang * Improved SPDA gate, added documentation. * Added arch var to reworked gate, fixing build errors. * Fix trailing whitespaces. --------- Co-authored-by: Neo Zhang --- ggml/src/ggml-sycl/fattn-onednn.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml-sycl/fattn-onednn.cpp b/ggml/src/ggml-sycl/fattn-onednn.cpp index fd17a25d5..a50129519 100644 --- a/ggml/src/ggml-sycl/fattn-onednn.cpp +++ b/ggml/src/ggml-sycl/fattn-onednn.cpp @@ -21,14 +21,6 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst) { if (!g_ggml_sycl_fa_onednn) { return false; } - // Battlemage (Xe2) only, for now. On other Intel archs oneDNN's fused SDPA returns wrong results - // for some shapes (e.g. head_dim=64 on Arc / xe_hpg) -- an oneDNN bug tracked upstream at - // https://github.com/uxlfoundation/oneDNN/issues/5510. Remove this hardware limitation once that - // is fixed; until then non-BMG archs fall back to the existing FA kernel. - const gpu_arch arch = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.arch; - if (arch != gpu_arch::intel_gpu_bmg_g21 && arch != gpu_arch::intel_gpu_bmg_g31) { - return false; - } const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; const ggml_tensor * V = dst->src[2]; @@ -60,6 +52,17 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst) { } } } + // This is the improved SPDA gate. Rather than gating Alchemist GPUs from all SPDA features, we instead target only the failing shapes. + // If the GPU being assessed isn't in the grouping below, it has full access to all SPDA shapes. Otherwise, if it's an Alchemist GPU, we block only the shapes with head sizes that fail. + // It is much easier to compare the device to a small list of failing cases than to define all the passing ones. + const gpu_arch arch = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.arch; + bool support_spda = !(arch == gpu_arch::intel_gpu_dg2_g10 || + arch == gpu_arch::intel_gpu_dg2_g11 || + arch == gpu_arch::intel_gpu_dg2_g12); + + if (!support_spda && K->ne[0] == 64) { + return false; + } // Optional KV-length ceiling (GGML_SYCL_FA_ONEDNN_MAX_KV, 0 = unlimited). Escape hatch: // very long sequences make the fused SDPA slow enough to risk the xe driver watchdog on // some stacks; past the cap we fall back to the native FA kernel instead. From cd26896c19e6775b29a86908b5f049bbaec73305 Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Thu, 20 Aug 2026 22:30:17 -0700 Subject: [PATCH 29/44] opencl: keep the vocab-scale K-quant lm_head on the CPU for Adreno A7X (compiler issue workaround) (#26440) * opencl: keep the vocab-scale K-quant lm_head on the CPU on the Adreno A7X * opencl: revise comments --------- Co-authored-by: Li He --- ggml/src/ggml-opencl/ggml-opencl.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 26f952a17..84f854cc2 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -7393,6 +7393,19 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || op->src[0]->type == GGML_TYPE_Q6_K) { + // The E031.41 compiler (usually with A7x) miscompiles the flat K-quant + // GEMV kernels (kernel_mul_mv_q*_K_f32_flat) and makes lm_head run much + // slower than it should. So, make it fallback to CPU to preserve performance + // for this compiler series. + static const char * a7x_lmhead_env = getenv("GGML_OPENCL_A7X_LMHEAD_CPU"); + static const bool a7x_lmhead_cpu = (a7x_lmhead_env == nullptr || a7x_lmhead_env[0] != '0'); + if (a7x_lmhead_cpu && + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && + (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || + op->src[0]->type == GGML_TYPE_Q6_K) && + op->src[0]->ne[1] >= 32768) { // vocab-scale weight; no FFN/attn weight is this tall + return false; + } return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); } else if (op->src[0]->type == GGML_TYPE_Q8_0) { return op->src[1]->type == GGML_TYPE_F32; From 9e89a196b8141c8bec0ad4c0bfd0cfc73bb8cdea Mon Sep 17 00:00:00 2001 From: Todd Malsbary Date: Fri, 21 Aug 2026 00:23:02 -0700 Subject: [PATCH 30/44] sycl : Add Q5_K ESIMD kernel (#26376) * Add DMMV Q4_K and Q6_K ESIMD kernels Configure cmake build with -DGGML_SYCL_ESIMD=ON to enable. Signed-off-by: Todd Malsbary * Refactor ESIMD kernels to share common code Signed-off-by: Todd Malsbary * Move control of ESIMD from compile to runtime Signed-off-by: Todd Malsbary * Use ESIMD by default when available Signed-off-by: Todd Malsbary * Fix possible error when using ESIMD by default While not an issue in the current version, this will become an issue when additional QK ESIMD kernels are added (such as Q2_K). Signed-off-by: Todd Malsbary * Add explicit unroll to ESIMD kernels Signed-off-by: Todd Malsbary * Tidy up ESIMD kernels a bit Signed-off-by: Todd Malsbary * Add DMMV Q5_K ESIMD kernel Signed-off-by: Todd Malsbary * Remove redundant copyright notice Signed-off-by: Todd Malsbary --------- Signed-off-by: Todd Malsbary --- ggml/src/ggml-sycl/dmmv.cpp | 27 ++++++- ggml/src/ggml-sycl/esimd.hpp | 134 ++++++++++++++++++++++++++++--- ggml/src/ggml-sycl/ggml-sycl.cpp | 1 + 3 files changed, 149 insertions(+), 13 deletions(-) diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cpp index d8da0a16b..fdcadbf91 100644 --- a/ggml/src/ggml-sycl/dmmv.cpp +++ b/ggml/src/ggml-sycl/dmmv.cpp @@ -1955,6 +1955,23 @@ static void dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(const void *vx, const }); } +static void dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(const void *vx, const float *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int workgroups = (nrows + 1) / 2; + stream->submit([&](sycl::handler &h) { + sycl::local_accessor lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h); + h.parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), + [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { + dequantize_mul_mat_vec_reorder_esimd( + vx, y, dst, ncols, nrows, lmem, it); + }); + }); +} + static void dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(const void *vx, const float *y, float *dst, const int ncols, const int nrows, @@ -2134,7 +2151,15 @@ void ggml_sycl_op_dequantize_mul_mat_vec( case GGML_TYPE_Q5_K: if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { - dequantize_mul_mat_vec_q5_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + if (g_ggml_sycl_enable_esimd) { + dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } + else +#endif + { + dequantize_mul_mat_vec_q5_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } } else { dequantize_mul_mat_vec_q5_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } diff --git a/ggml/src/ggml-sycl/esimd.hpp b/ggml/src/ggml-sycl/esimd.hpp index d7609b11f..04e596ed3 100644 --- a/ggml/src/ggml-sycl/esimd.hpp +++ b/ggml/src/ggml-sycl/esimd.hpp @@ -1,15 +1,3 @@ -// -// MIT license -// Copyright (C) 2026 Intel Corporation -// SPDX-License-Identifier: MIT -// - -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// - #ifndef GGML_SYCL_ESIMD_HPP #define GGML_SYCL_ESIMD_HPP @@ -287,6 +275,128 @@ template <> struct esimd_reorder_q_traits { } }; +// --------------------------------------------------------------------------- +// Q5_K, SOA reorder layout produced by reorder_qw_q5_k: +// [qs: nb*(QK_K/2)] [qh: nb*(QK_K/8)] [scales: nb*K_SCALE_SIZE] [dm: nb*sizeof(half2)] +// with nb = nrows*num_blocks_per_row. +// +// Identical to Q4_K except each 4-bit quant gains a 5th (high) bit from qh: +// output chunk c (0..7) adds 16 when bit c of qh[l] is set, where qh[l] indexes +// the same 32 bytes for every chunk (matches dequantize_row_q5_K). +// --------------------------------------------------------------------------- +template <> struct esimd_reorder_q_traits { + struct ptrs { + const uint8_t * qs; + const uint8_t * qh; + const uint8_t * scales; + const sycl::half * dm; + }; + + static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) { + const uint8_t * qs = (const uint8_t *) vx; + const uint8_t * qh = qs + nb * (QK_K / 2); + const uint8_t * scales = qh + nb * (QK_K / 8); + const sycl::half * dm = (const sycl::half *) (scales + nb * K_SCALE_SIZE); + return { qs, qh, scales, dm }; + } + + // extract bit `bit` (0..7) of each lane and move it to bit position 4, + // e.g. for the 4-bit base quant's 5th (high) bit. `bit` is always a + // compile-time-known unrolled loop constant at call sites, so this folds + // to a single mask (bit==4), mask+left-shift (bit<4), or mask+right-shift + // (bit>4) instead of the shift+mask+shift a naive `(qh>>bit & 1) << 4` emits. + static ESIMD_INLINE sycl::ext::intel::esimd::simd extract_bit_to_pos4( + sycl::ext::intel::esimd::simd qh, int bit) { + using namespace sycl::ext::intel::esimd; + simd masked = convert(qh & simd((uint8_t) (1u << bit))); + if (bit < 4) { + return masked << simd((uint16_t) (4 - bit)); + } else if (bit > 4) { + return masked >> simd((uint16_t) (bit - 4)); + } + return masked; + } + + static ESIMD_INLINE void mac_pair( + const ptrs & pa, size_t bia, + const ptrs & pb, size_t bib, bool has_b, + sycl::ext::intel::esimd::simd & y_vec, + sycl::ext::intel::esimd::simd & acc_a, + sycl::ext::intel::esimd::simd & acc_b) { + using namespace sycl::ext::intel::esimd; + + simd qs_a = block_load(pa.qs + bia * (QK_K / 2)); + simd qs_b = 0; + simd qh_a = block_load(pa.qh + bia * (QK_K / 8)); + simd qh_b = 0; + simd scales_a = block_load(pa.scales + bia * K_SCALE_SIZE); + simd scales_b = 0; + + const float dall_a = (float) pa.dm[bia * 2 + 0]; + const float dmin_a = (float) pa.dm[bia * 2 + 1]; + float dall_b = 0.0f; + float dmin_b = 0.0f; + if (has_b) { + qs_b = block_load(pb.qs + bib * (QK_K / 2)); + qh_b = block_load(pb.qh + bib * (QK_K / 8)); + scales_b = block_load(pb.scales + bib * K_SCALE_SIZE); + dall_b = (float) pb.dm[bib * 2 + 0]; + dmin_b = (float) pb.dm[bib * 2 + 1]; + } + + simd scale_f_a, min_f_a, scale_f_b, min_f_b; + unpack_scale_min_k4(scales_a, dall_a, dmin_a, scale_f_a, min_f_a); + unpack_scale_min_k4(scales_b, dall_b, dmin_b, scale_f_b, min_f_b); + + simd qs_lo_a = qs_a & simd(0x0F); + simd qs_hi_a = qs_a >> simd(4); + simd qs_lo_b = qs_b & simd(0x0F); + simd qs_hi_b = qs_b >> simd(4); + +#pragma unroll + for (int sb = 0; sb < 8; sb += 2) { + const int q_offset = sb * 16; + simd y_lo = y_vec.select<32, 1>(sb * 32); + simd y_hi = y_vec.select<32, 1>((sb + 1) * 32); + + const float scale_a_lo = scale_f_a[sb]; + const float scale_a_hi = scale_f_a[sb + 1]; + const float min_a_lo = min_f_a[sb]; + const float min_a_hi = min_f_a[sb + 1]; + const float scale_b_lo = scale_f_b[sb]; + const float scale_b_hi = scale_f_b[sb + 1]; + const float min_b_lo = min_f_b[sb]; + const float min_b_hi = min_f_b[sb + 1]; + + simd qa_lo_u8 = qs_lo_a.select<32, 1>(q_offset); + simd qa_hi_u8 = qs_hi_a.select<32, 1>(q_offset); + simd qb_lo_u8 = qs_lo_b.select<32, 1>(q_offset); + simd qb_hi_u8 = qs_hi_b.select<32, 1>(q_offset); + simd qa_lo = convert(qa_lo_u8); + simd qa_hi = convert(qa_hi_u8); + simd qb_lo = convert(qb_lo_u8); + simd qb_hi = convert(qb_hi_u8); + + // add the 5th bit: chunk sb uses qh bit sb, chunk sb+1 uses qh bit sb+1; + // qh always indexes the same 32 bytes regardless of chunk + qa_lo += extract_bit_to_pos4(qh_a, sb); + qa_hi += extract_bit_to_pos4(qh_a, sb + 1); + qb_lo += extract_bit_to_pos4(qh_b, sb); + qb_hi += extract_bit_to_pos4(qh_b, sb + 1); + + simd deq_a_lo = convert(qa_lo) * scale_a_lo + min_a_lo; + simd deq_a_hi = convert(qa_hi) * scale_a_hi + min_a_hi; + simd deq_b_lo = convert(qb_lo) * scale_b_lo + min_b_lo; + simd deq_b_hi = convert(qb_hi) * scale_b_hi + min_b_hi; + + acc_a += y_lo * deq_a_lo; + acc_b += y_lo * deq_b_lo; + acc_a += y_hi * deq_a_hi; + acc_b += y_hi * deq_b_hi; + } + } +}; + // --------------------------------------------------------------------------- // Q6_K, SOA reorder layout: // [ql: nb*(QK_K/2)] [qh: nb*(QK_K/4)] [scales(int8): nb*(QK_K/16)] [d: nb*half] diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 7ebdce7fb..de56ea5b9 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -3811,6 +3811,7 @@ static bool ggml_sycl_supports_reorder_esimd(enum ggml_type type) { switch (type) { case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: return true; default: From 5fff128451d7603857597ee1fc18ac1dfb90f148 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 21 Aug 2026 10:29:17 +0300 Subject: [PATCH 31/44] test : make the FA V-is-view-of-K case a test case parameter (#27394) Resolve the TODO in test_flash_attn_ext: the branch that creates V as a sub-view of K (MLA-based models) was hardcoded for the 576/512 head shapes. Add a v_is_view_of_k test case parameter (default false) and select the sub-view branch on it; the existing 576/512 (DeepSeek MLA) cases now pass it explicitly, so the test coverage is unchanged. Also add more V-is-sub-view-of-K cases: the 320/256 (Mistral4 MLA) and 192/128 head shapes, and full views with equal head sizes (128/128 F16, 64/64 q8_0). Assisted-by: pi:llama.cpp/Qwen3.8-27B --- tests/test-backend-ops.cpp | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 89b954a7d..8e3b273a1 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -7085,9 +7085,10 @@ struct test_flash_attn_ext : public test_case { const ggml_type type_V; std::array permute; const bool kv_view; // create K/V as views of a larger buffer (like a KV cache) + const bool v_is_view_of_k; std::string vars() override { - return VARS_TO_STR15(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view); + return VARS_TO_STR16(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view, v_is_view_of_k); } double max_nmse_err() override { @@ -7104,9 +7105,9 @@ struct test_flash_attn_ext : public test_case { test_flash_attn_ext(int64_t hsk = 128, int64_t hsv = 128, int64_t nh = 32, std::array nr23 = {1, 1}, int64_t kv = 96, int64_t nb = 8, bool mask = true, bool sinks = false, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32, ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array permute = {0, 1, 2, 3}, - bool kv_view = true) + bool kv_view = true, bool v_is_view_of_k = false) : hsk(hsk), hsv(hsv), nh(nh), nr23(nr23), kv(kv), nb(nb), mask(mask), sinks(sinks), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec), - type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view) {} + type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view), v_is_view_of_k(v_is_view_of_k) {} ggml_tensor * build_graph(ggml_context * ctx) override { const int64_t hsk_padded = GGML_PAD(hsk, ggml_blck_size(type_K)); @@ -7138,14 +7139,14 @@ struct test_flash_attn_ext : public test_case { ggml_set_name(k, "k"); ggml_tensor * v = nullptr; - if (type_K == type_V && hsk_padded == 576 && hsv_padded == 512) { - // TODO: this branch should become a separate test case parameter instead of hardcoding this for these head shapes - - // in this branch, the V cache is sub-view of the K cache. this is used by some MLA-based models + if (v_is_view_of_k) { + // the V cache is a sub-view of the K cache. this is used by some MLA-based models // for more info: // - https://github.com/ggml-org/llama.cpp/pull/13435 // - https://github.com/ggml-org/llama.cpp/pull/18953#issuecomment-3774948392 // - https://github.com/ggml-org/llama.cpp/pull/18986 + GGML_ASSERT(type_K == type_V && hsv_padded <= hsk_padded); + v = ggml_view_4d(ctx, k, hsv_padded, kv, nh, nr23[1], k->nb[1], k->nb[2], k->nb[3], 0); } else { v = create_permuted(type_V, hsv_padded, kv, nh, nr23[1], kv_view); // the V tensor is usually a view of the V cache @@ -9906,12 +9907,14 @@ static std::vector> make_test_cases_eval() { if (hsk != 128 && prec == GGML_PREC_DEFAULT) continue; for (ggml_type type_KV : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) { if (type_KV != GGML_TYPE_F16 && hsk != 64 && hsk != 72) continue; + // DeepSeek MLA: the V cache is a sub-view of the K cache + const bool v_is_view_of_k = hsk == 576; test_cases.emplace_back(new test_flash_attn_ext( - hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV)); + hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV, {0, 1, 2, 3}, true, v_is_view_of_k)); // run fewer test cases permuted if (mask == true && max_bias == 0.0f && logit_softcap == 0 && kv == 512) { test_cases.emplace_back(new test_flash_attn_ext( - hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV, {0, 2, 1, 3})); + hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV, {0, 2, 1, 3}, true, v_is_view_of_k)); } } } @@ -9950,11 +9953,16 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1025, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3})); test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 16384, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); - // MLA shape (V is a view of K) with quantized KV - // (the test harness builds V as a view of K for this shape; see build_graph) - test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); - test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); - test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + // MLA shape: the V cache is a sub-view of the K cache, with quantized KV + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); + + // more V-is-sub-view-of-K cases: other head shapes, and full views with equal head sizes + test_cases.emplace_back(new test_flash_attn_ext(320, 256, 1, {32, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); + test_cases.emplace_back(new test_flash_attn_ext(192, 128, 4, {8, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 8, {4, 1}, 512, 8, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); // large-KV F16 cases (Qwen3.6-27B geometry and a llama-class control): the upstream matrix // stops at kv=1024, blind to long-context FA bugs (e.g. the oneDNN SDPA ordering race on BMG). From ff14356e0caf6988f61f1f15f9dfe7d5ab398271 Mon Sep 17 00:00:00 2001 From: Todd Malsbary Date: Fri, 21 Aug 2026 01:01:40 -0700 Subject: [PATCH 32/44] sycl : add Q2_K reordered MMVQ and ESIMD kernels (#26336) * Add DMMV Q4_K and Q6_K ESIMD kernels Configure cmake build with -DGGML_SYCL_ESIMD=ON to enable. Signed-off-by: Todd Malsbary * Refactor ESIMD kernels to share common code Signed-off-by: Todd Malsbary * Move control of ESIMD from compile to runtime Signed-off-by: Todd Malsbary * Use ESIMD by default when available Signed-off-by: Todd Malsbary * Fix possible error when using ESIMD by default While not an issue in the current version, this will become an issue when additional QK ESIMD kernels are added (such as Q2_K). Signed-off-by: Todd Malsbary * Add explicit unroll to ESIMD kernels Signed-off-by: Todd Malsbary * Tidy up ESIMD kernels a bit Signed-off-by: Todd Malsbary * Add a reordered Q2_K MMVQ kernel Signed-off-by: Todd Malsbary * Add DMMV Q2_K ESIMD kernel Signed-off-by: Todd Malsbary --------- Signed-off-by: Todd Malsbary --- ggml/src/ggml-sycl/convert.cpp | 25 ++++++++- ggml/src/ggml-sycl/dequantize.hpp | 41 +++++++++++++++ ggml/src/ggml-sycl/dmmv.cpp | 27 +++++++++- ggml/src/ggml-sycl/esimd.hpp | 87 +++++++++++++++++++++++++++++++ ggml/src/ggml-sycl/ggml-sycl.cpp | 2 + ggml/src/ggml-sycl/mmvq.cpp | 75 +++++++++++++++++++++++++- ggml/src/ggml-sycl/quants.hpp | 23 ++++++++ ggml/src/ggml-sycl/vecdotq.hpp | 33 ++++++++++++ 8 files changed, 309 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp index 9ec927695..b660b56ab 100644 --- a/ggml/src/ggml-sycl/convert.cpp +++ b/ggml/src/ggml-sycl/convert.cpp @@ -76,6 +76,19 @@ static void dequantize_row_q2_K_sycl(const void *vx, dst_t *y, const int64_t k, #endif } +template +static void dequantize_row_q2_K_sycl_reorder(const void *vx, dst_t *y, const int64_t k, + dpct::queue_ptr stream) { + const int64_t nb = k / QK_K; + + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_q2_K_reorder(vx, y, item_ct1, nb); + }); +} + template static void dequantize_row_q3_K_sycl(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { @@ -667,7 +680,11 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { return dequantize_block_sycl; } case GGML_TYPE_Q2_K: - return dequantize_row_q2_K_sycl; + if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + return dequantize_row_q2_K_sycl_reorder; + } else { + return dequantize_row_q2_K_sycl; + } case GGML_TYPE_Q3_K: if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { return dequantize_row_q3_K_sycl_reorder; @@ -753,7 +770,11 @@ to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) { return dequantize_block_sycl; } case GGML_TYPE_Q2_K: - return dequantize_row_q2_K_sycl; + if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + return dequantize_row_q2_K_sycl_reorder; + } else { + return dequantize_row_q2_K_sycl; + } case GGML_TYPE_Q3_K: if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { return dequantize_row_q3_K_sycl_reorder; diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp index 876ba1b44..1b13e0f1a 100644 --- a/ggml/src/ggml-sycl/dequantize.hpp +++ b/ggml/src/ggml-sycl/dequantize.hpp @@ -943,6 +943,47 @@ static void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restri } +template +static void dequantize_block_q2_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, + const sycl::nd_item<3> & item_ct1, int64_t n_blocks) { +#if QK_K == 256 + const int64_t i = item_ct1.get_group(2); + if (i >= n_blocks) { + return; + } + + const uint8_t * base = static_cast(vx); + const size_t qs_offset = i * (QK_K / 4); + const size_t scales_offset = n_blocks * (QK_K / 4) + i * (QK_K / 16); + const size_t dm_offset = n_blocks * (QK_K / 4) + n_blocks * (QK_K / 16) + i * sizeof(ggml_half2); + + const uint8_t * qs = base + qs_offset; + const uint8_t * scales = base + scales_offset; + const ggml_half2 * dm = reinterpret_cast(base + dm_offset); + + const int64_t tid = item_ct1.get_local_id(2); + const int64_t n = tid / 32; + const int64_t l = tid - 32 * n; + const int64_t is = 8 * n + l / 16; + + const uint8_t q = qs[32 * n + l]; + dst_t * y = yy + i * QK_K + 128 * n; + + const float dall = (*dm)[0]; + const float dmin = (*dm)[1]; + y[l+ 0] = dall * (scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (scales[is+0] >> 4); + y[l+32] = dall * (scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (scales[is+2] >> 4); + y[l+64] = dall * (scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (scales[is+4] >> 4); + y[l+96] = dall * (scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (scales[is+6] >> 4); +#else + GGML_UNUSED(vx); + GGML_UNUSED(yy); + GGML_UNUSED(item_ct1); + GGML_UNUSED(n_blocks); + GGML_ABORT("Q2_K reorder dequantize not supported for QK_K != 256"); +#endif +} + template static void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy, const sycl::nd_item<3> &item_ct1) { diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cpp index fdcadbf91..d47d6831a 100644 --- a/ggml/src/ggml-sycl/dmmv.cpp +++ b/ggml/src/ggml-sycl/dmmv.cpp @@ -1921,6 +1921,23 @@ ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd( } } +static void dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(const void *vx, const float *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int workgroups = (nrows + 1) / 2; + stream->submit([&](sycl::handler &h) { + sycl::local_accessor lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h); + h.parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), + [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { + dequantize_mul_mat_vec_reorder_esimd( + vx, y, dst, ncols, nrows, lmem, it); + }); + }); +} + static void dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(const void *vx, const float *y, float *dst, const int ncols, const int nrows, @@ -2111,7 +2128,15 @@ void ggml_sycl_op_dequantize_mul_mat_vec( case GGML_TYPE_Q2_K: if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { - dequantize_mul_mat_vec_q2_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + if (g_ggml_sycl_enable_esimd) { + dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } + else +#endif + { + dequantize_mul_mat_vec_q2_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } } else { dequantize_mul_mat_vec_q2_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } diff --git a/ggml/src/ggml-sycl/esimd.hpp b/ggml/src/ggml-sycl/esimd.hpp index 04e596ed3..0485ff0ce 100644 --- a/ggml/src/ggml-sycl/esimd.hpp +++ b/ggml/src/ggml-sycl/esimd.hpp @@ -61,6 +61,93 @@ static ESIMD_INLINE void unpack_scale_min_k4( min_f = convert(m) * (-dmin); } +// --------------------------------------------------------------------------- +// Q2_K, SOA reorder layout produced by reorder_qw_q2_k: +// [qs: nb*(QK_K/4)] [scales: nb*(QK_K/16)] [dm: nb*sizeof(half2)] +// with nb = nrows*num_blocks_per_row. +// +// 2 bits per weight. The 8 output chunks of 32 (matching dequantize_row_q2_K) +// map to super-chunk s (0..7): byte base 32*(s/4) into the 64-byte qs array, +// bit shift 2*(s%4); the low 16 lanes use scales[2s], the high 16 use +// scales[2s+1], with dl = d*(sc & 0xF), ml = dmin*(sc >> 4), deq = dl*q - ml. +// --------------------------------------------------------------------------- +template <> struct esimd_reorder_q_traits { + struct ptrs { + const uint8_t * qs; + const uint8_t * scales; + const sycl::half * dm; + }; + + static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) { + const uint8_t * qs = (const uint8_t *) vx; + const uint8_t * scales = qs + nb * (QK_K / 4); + const sycl::half * dm = (const sycl::half *) (scales + nb * (QK_K / 16)); + return { qs, scales, dm }; + } + + static ESIMD_INLINE void mac_pair( + const ptrs & pa, size_t bia, + const ptrs & pb, size_t bib, bool has_b, + sycl::ext::intel::esimd::simd & y_vec, + sycl::ext::intel::esimd::simd & acc_a, + sycl::ext::intel::esimd::simd & acc_b) { + using namespace sycl::ext::intel::esimd; + + simd qs_a = block_load(pa.qs + bia * (QK_K / 4)); + simd qs_b = 0; + simd scales_a = block_load(pa.scales + bia * (QK_K / 16)); + simd scales_b = 0; + + const float dall_a = (float) pa.dm[bia * 2 + 0]; + const float dmin_a = (float) pa.dm[bia * 2 + 1]; + float dall_b = 0.0f; + float dmin_b = 0.0f; + if (has_b) { + qs_b = block_load(pb.qs + bib * (QK_K / 4)); + scales_b = block_load(pb.scales + bib * (QK_K / 16)); + dall_b = (float) pb.dm[bib * 2 + 0]; + dmin_b = (float) pb.dm[bib * 2 + 1]; + } + + // per-chunk scale (d * (sc & 0xF)) and min (-dmin * (sc >> 4)), all 16 codes; + // min carries the negation so the dequant epilogue adds (matches Q4_K/Q5_K) + simd scale_f_a = convert(scales_a & simd(0x0F)) * dall_a; + simd min_f_a = convert(scales_a >> simd(4)) * (-dmin_a); + simd scale_f_b = convert(scales_b & simd(0x0F)) * dall_b; + simd min_f_b = convert(scales_b >> simd(4)) * (-dmin_b); + +#pragma unroll + for (int s = 0; s < 8; ++s) { + const int byte_base = 32 * (s / 4); + const uint8_t shift = (uint8_t) (2 * (s % 4)); + simd y_s = y_vec.select<32, 1>(s * 32); + + simd qa = (qs_a.select<32, 1>(byte_base) >> shift) & simd(3); + simd qb = (qs_b.select<32, 1>(byte_base) >> shift) & simd(3); + + const float scale_a_lo = scale_f_a[2 * s + 0]; + const float scale_a_hi = scale_f_a[2 * s + 1]; + const float min_a_lo = min_f_a[2 * s + 0]; + const float min_a_hi = min_f_a[2 * s + 1]; + const float scale_b_lo = scale_f_b[2 * s + 0]; + const float scale_b_hi = scale_f_b[2 * s + 1]; + const float min_b_lo = min_f_b[2 * s + 0]; + const float min_b_hi = min_f_b[2 * s + 1]; + + simd scale_vec_a = splat_lo_hi(scale_a_lo, scale_a_hi); + simd min_vec_a = splat_lo_hi(min_a_lo, min_a_hi); + simd scale_vec_b = splat_lo_hi(scale_b_lo, scale_b_hi); + simd min_vec_b = splat_lo_hi(min_b_lo, min_b_hi); + + simd deq_a = convert(qa) * scale_vec_a + min_vec_a; + simd deq_b = convert(qb) * scale_vec_b + min_vec_b; + + acc_a += y_s * deq_a; + acc_b += y_s * deq_b; + } + } +}; + // --------------------------------------------------------------------------- // Q3_K, SOA reorder layout produced by reorder_qw_q3_k: // [qs: nb*(QK_K/4)] [hmask: nb*(QK_K/8)] [scales: nb*12] [d: nb*sizeof(half)] diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index de56ea5b9..3f82020f4 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -3796,6 +3796,7 @@ inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) { case GGML_TYPE_Q1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: @@ -3809,6 +3810,7 @@ inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) { static bool ggml_sycl_supports_reorder_esimd(enum ggml_type type) { #ifdef GGML_SYCL_DMMV_HAS_ESIMD switch (type) { + case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 123b2a2f0..bfccb4b08 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -1401,6 +1401,64 @@ static void mul_mat_vec_q2_K_q8_1_sycl_switch_ncols( } } +static void reorder_mul_mat_vec_q2_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + + // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel. + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder>(vx, vy, dst, ncols, nrows, + nd_item); + }); + }); +} + +template +static void reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst>( + vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, nd_item); + }); + }); +} + +static void reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_q2_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for Q2_K reorder multi-col MMVQ", ncols_dst); + } +} + static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, @@ -2297,7 +2355,21 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens } break; case GGML_TYPE_Q2_K: - if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); + reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y_bytes, stride_col_dst, stream); + return; + } else { + GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q2_k_q8_1_sycl\n"); + reorder_mul_mat_vec_q2_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } + } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { const int stride_col_y = src1_padded_col_size / QK8_1; const int stride_col_dst = dst->ne[0]; GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); @@ -2306,6 +2378,7 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens src1_ncols, stride_col_y, stride_col_dst, stream); return; } else if (i == 0 || src1_ncols == 1) { + GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_K_q8_1_sycl\n"); mul_mat_vec_q2_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); } break; diff --git a/ggml/src/ggml-sycl/quants.hpp b/ggml/src/ggml-sycl/quants.hpp index 95287f175..a26a6ce6e 100644 --- a/ggml/src/ggml-sycl/quants.hpp +++ b/ggml/src/ggml-sycl/quants.hpp @@ -58,6 +58,29 @@ template <> struct block_q_t { static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } }; +template <> struct block_q_t { + struct traits { + static constexpr uint32_t qk = QK_K; + static constexpr uint32_t qi = QI2_K; + static constexpr uint32_t qr = QR2_K; + static constexpr uint32_t vdr_mmvq = 1; + }; + + // Reordered layout: [qs (QK_K/4 per block)] [scales (QK_K/16 per block)] [dm] + static constexpr std::pair get_block_offset(const int block_index, const int /* n_blocks */) { + return { block_index * (QK_K / 4), 0 }; + } + + static constexpr std::pair get_d_offset(int nrows, int ncols, const int block_index) { + auto nblocks = (nrows * (ncols / QK_K)); + auto total_qs_bytes = nblocks * (QK_K / 4); + return { total_qs_bytes + block_index * (QK_K / 16), + total_qs_bytes + nblocks * (QK_K / 16) + block_index * sizeof(ggml_half2) }; + } + + static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } +}; + template <> struct block_q_t { struct traits { static constexpr uint32_t qk = QK_K; diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp index c11a6e8f9..3ad4cee93 100644 --- a/ggml/src/ggml-sycl/vecdotq.hpp +++ b/ggml/src/ggml-sycl/vecdotq.hpp @@ -429,6 +429,39 @@ template <> struct reorder_vec_dot_q_sycl { } }; +template <> struct reorder_vec_dot_q_sycl { + static constexpr ggml_type gtype = GGML_TYPE_Q2_K; + + using q2_k_block = ggml_sycl_reordered::block_q_t; + using q2_k_traits = typename q2_k_block::traits; + + __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair ibx_offset, + const std::pair d_offset, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + const uint8_t * base = static_cast(vbq); + const uint8_t * qs = base + ibx_offset.first; + const uint8_t * scales = base + d_offset.first; + const ggml_half2 * dm = reinterpret_cast(base + d_offset.second); + + const int bq8_offset = QR2_K * (iqs / QI8_1); + const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1 / 2); + + const int v = get_int_from_uint8_aligned(qs, iqs); + + int u[QR2_K]; + float d8[QR2_K]; + +#pragma unroll + for (int i = 0; i < QR2_K; ++i) { + const int8_t * quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1; + u[i] = get_int_from_int8_aligned(quant_base_ptr, iqs % QI8_1); + d8[i] = (*(q8_1_ds + bq8_offset + i))[0]; + } + + return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales + scale_offset, *dm, d8); + } +}; + template <> struct reorder_vec_dot_q_sycl { static constexpr ggml_type gtype = GGML_TYPE_Q3_K; From 62b22690602665a0f34ce5f722915839f4e4913d Mon Sep 17 00:00:00 2001 From: Charles Xu Date: Fri, 21 Aug 2026 10:33:30 +0200 Subject: [PATCH 33/44] kleidiai : add SME2 F32 GEMV kernel support (#26891) --- ggml/src/ggml-cpu/CMakeLists.txt | 3 ++ ggml/src/ggml-cpu/kleidiai/kernels.cpp | 48 ++++++++++++++++--------- ggml/src/ggml-cpu/kleidiai/kleidiai.cpp | 48 ++++++++++++++++++------- 3 files changed, 70 insertions(+), 29 deletions(-) diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 32e1e7aa1..e16ac996a 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -639,6 +639,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name) ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/ ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/ ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/ + ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/ ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/) set(ARCH_FLAGS_TEMP "${ARCH_FLAGS}") @@ -701,6 +702,8 @@ function(ggml_add_cpu_backend_variant_impl tag_name) ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa_asm.S ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa.c ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa_asm.S + ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla.c + ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla_asm.S ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_bf16p2vlx2_f32_sme.c ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.c ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_f16pmrx2_f32_neon.c diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.cpp b/ggml/src/ggml-cpu/kleidiai/kernels.cpp index 3c31ab9d3..70b519f29 100644 --- a/ggml/src/ggml-cpu/kleidiai/kernels.cpp +++ b/ggml/src/ggml-cpu/kleidiai/kernels.cpp @@ -23,6 +23,7 @@ #include "kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.h" #include "kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.h" #include "kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa.h" +#include "kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla.h" #include "kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa.h" #include "kai_lhs_pack_bf16p2vlx2_f32_sme.h" @@ -76,6 +77,21 @@ static inline void kernel_run_fn10(size_t m, size_t n, size_t k, size_t /*bl*/, Fn(m, n, k, lhs, rhs, dst, dst_stride_row, dst_stride_col, clamp_min, clamp_max); } +template +static inline void kernel_run_lhs_stride_fn10(size_t m, + size_t n, + size_t k, + size_t lhs_stride, + const void * lhs, + const void * rhs, + void * dst, + size_t dst_stride_row, + size_t dst_stride_col, + float clamp_min, + float clamp_max) { + Fn(m, n, k, lhs, lhs_stride, rhs, dst, dst_stride_row, dst_stride_col, clamp_min, clamp_max); +} + template static inline void kernel_run_float_fn10(size_t m, size_t n, size_t k, size_t /*bl*/, const void* lhs, const void* rhs, void* dst, @@ -947,25 +963,25 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .packed_size_ex = */ &lhs_ps_fn5, /* .pack_func_ex = */ &lhs_pack_void_fn9, }, - /* SME GEMV */ + /* SME2 GEMV */ { - /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, - /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, - /* .get_mr = */ kai_get_mr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, - /* .get_nr = */ kai_get_nr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, - /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, - /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, - /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, - /* .get_lhs_offset_ex = */ nullptr, - /* .get_rhs_packed_offset_ex = */ nullptr, - /* .run_kernel_ex = */ nullptr, + /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_mr = */ kai_get_m_step_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_nr = */ kai_get_nr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_lhs_offset_ex = */ &kernel_offs_fn2, + /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, + /* .run_kernel_ex = */ &kernel_run_lhs_stride_fn10, }, /* .gemv_lhs_info = */ { - /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f32p2vlx1_f32_sme, - /* .get_packed_offset_ex = */ &lhs_offs_fn5, - /* .packed_size_ex = */ &lhs_ps_fn5, - /* .pack_func_ex = */ &lhs_pack_void_fn9, + /* .get_offset = */ nullptr, + /* .get_packed_offset_ex = */ nullptr, + /* .packed_size_ex = */ nullptr, + /* .pack_func_ex = */ nullptr, }, /* .rhs_info = */ { /* .packed_stride = */ nullptr, diff --git a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp index 2266c1689..6729ae842 100644 --- a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +++ b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp @@ -696,6 +696,15 @@ class tensor_traits : public ggml::cpu::tensor_traits { } if (op->src[0]->type == GGML_TYPE_F32) { + ggml_kleidiai_kernels * primary = kernel_chain[0]; + kernel_info * gemv_kernel = primary ? &primary->gemv : nullptr; + if (is_gemv && op->src[1]->nb[0] == (int64_t) sizeof(float) && gemv_kernel && + gemv_kernel->get_lhs_offset_ex && gemv_kernel->get_rhs_packed_offset_ex && + gemv_kernel->run_kernel_ex && gemv_kernel->get_dst_offset) { + size = 0; + return true; + } + size_t cursor = 0; bool any_slot = false; @@ -811,15 +820,28 @@ class tensor_traits : public ggml::cpu::tensor_traits { return false; } - kernel_info * kernel = &kernels->gemm; + const size_t k = ne00; + const size_t m = ne11; + const size_t n = ne01; + const bool use_gemv = m == 1 && src1->nb[0] == (int64_t) sizeof(float) && + kernels->gemv.get_lhs_offset_ex && + kernels->gemv.get_rhs_packed_offset_ex && + kernels->gemv.run_kernel_ex && + kernels->gemv.get_dst_offset; + + kernel_info * kernel = use_gemv ? &kernels->gemv : &kernels->gemm; lhs_packing_info * lhs_info = &kernels->gemm_lhs_info; - if (!kernel || !lhs_info || !lhs_info->get_offset || !lhs_info->get_packed_offset_ex || - !lhs_info->packed_size_ex || !lhs_info->pack_func_ex || + if (!kernel || !kernel->get_lhs_offset_ex || !kernel->get_rhs_packed_offset_ex || !kernel->run_kernel_ex || !kernel->get_dst_offset) { return false; } + if (!use_gemv && (!lhs_info || !lhs_info->get_offset || !lhs_info->get_packed_offset_ex || + !lhs_info->packed_size_ex || !lhs_info->pack_func_ex)) { + return false; + } + const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data); const bool has_header = kleidiai_is_weight_header_valid(header); @@ -832,16 +854,14 @@ class tensor_traits : public ggml::cpu::tensor_traits { const int nth = params->nth > 0 ? params->nth : 1; const int ith = params->ith; - const size_t k = ne00; - const size_t m = ne11; - const size_t n = ne01; - const size_t mr = kernel->get_mr(); const size_t kr = kernel->get_kr(); const size_t sr = kernel->get_sr(); - const size_t lhs_packed_size = lhs_info->packed_size_ex(m, k, 0, mr, kr, sr); - GGML_ASSERT(lhs_packed_size <= params->wsize); + const size_t lhs_packed_size = use_gemv ? 0 : lhs_info->packed_size_ex(m, k, 0, mr, kr, sr); + if (!use_gemv) { + GGML_ASSERT(lhs_packed_size <= params->wsize); + } uint8_t * lhs_packed = static_cast(params->wdata); const size_t dst_stride = dst->nb[1]; @@ -853,7 +873,7 @@ class tensor_traits : public ggml::cpu::tensor_traits { const uint8_t * lhs_batch_base = static_cast(src1->data) + batch_idx * src1->nb[2]; uint8_t * dst_batch_base = static_cast(dst->data) + batch_idx * dst->nb[2]; - { + if (!use_gemv) { const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)mr); int64_t max_threads = mr ? (m_roundup_mr / (int64_t)mr) : nth; max_threads = std::max(1, max_threads); @@ -903,15 +923,17 @@ class tensor_traits : public ggml::cpu::tensor_traits { const size_t n_to_process = std::min(chunk_cols, n - n_start); if (n_to_process > 0) { - const size_t lhs_packed_offset = lhs_info->get_packed_offset_ex(0, k, 0, mr, kr, sr); + const size_t lhs_offset = use_gemv ? kernel->get_lhs_offset_ex(0, k, 0) + : lhs_info->get_packed_offset_ex(0, k, 0, mr, kr, sr); const size_t rhs_packed_offset = kernel->get_rhs_packed_offset_ex(n_start, k, 0); const size_t dst_offset = kernel->get_dst_offset(0, n_start, dst_stride); - const void * lhs_ptr = lhs_packed + lhs_packed_offset; + const void * lhs_ptr = use_gemv ? lhs_batch_base + lhs_offset + : lhs_packed + lhs_offset; const void * rhs_ptr = rhs_base + rhs_packed_offset; float * dst_ptr = reinterpret_cast(dst_batch_base + dst_offset); - kernel->run_kernel_ex(m, n_to_process, k, 0, + kernel->run_kernel_ex(m, n_to_process, k, use_gemv ? src1->nb[1] : 0, lhs_ptr, rhs_ptr, dst_ptr, From 17197474510622a3b4ea7d0909d70b606f542b96 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 21 Aug 2026 11:33:40 +0300 Subject: [PATCH 34/44] ci : release clean-up (#27477) --- .github/workflows/make-release.yml | 12 ++++++------ .github/workflows/release.yml | 1 + scripts/make-release-desc.sh | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml index 78d7caa7a..95e0f3475 100644 --- a/.github/workflows/make-release.yml +++ b/.github/workflows/make-release.yml @@ -62,15 +62,15 @@ jobs: GITHUB_TOKEN: ${{ github.token }} with: tag_name: ${{ steps.checks.outputs.version }} - # TODO: remove the prerelease flag once the semantic versioning workflow is ready - # ref: https://github.com/ggml-org/ggml/discussions/1579 - prerelease: true + prerelease: false + # TODO: enrich the body of the release with more information body: | - > [!NOTE] - > Semantic versioning is still work in progress. - > More info can be found in https://github.com/ggml-org/ggml/discussions/1579 + ## Overview + + New version has been released. ${{ steps.desc.outputs.nightly }} + **More info:** [dist : releases and versioning of ggml-org projects](https://github.com/ggml-org/ggml/discussions/1579) ## ${{ steps.desc.outputs.changelog_title }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 61b2f5485..a3c67604e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1688,6 +1688,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ steps.tag.outputs.name }} + prerelease: true body: |
diff --git a/scripts/make-release-desc.sh b/scripts/make-release-desc.sh index 100f855b3..f1e4566e4 100755 --- a/scripts/make-release-desc.sh +++ b/scripts/make-release-desc.sh @@ -52,10 +52,10 @@ PREV="$( { git tag --list; echo "${VERSION}"; } \ if [[ -n "${PREV}" ]]; then CHANGELOG="$(git log --oneline "${PREV}..${RELEASE_COMMIT}")" - CHANGELOG_TITLE="Change log since ${PREV}" + CHANGELOG_TITLE="Changelog since ${PREV}" else CHANGELOG="(no previous release tag found)" - CHANGELOG_TITLE="Change log" + CHANGELOG_TITLE="Changelog" fi # Nightly release: the b* tag pointing at the release commit (|| true: no match is not an error) From e467c2ff6174835b4079d0acf123c5ca64615088 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 21 Aug 2026 13:20:44 +0300 Subject: [PATCH 35/44] ci : add nightly-tag.txt to make-release (#27485) As agreed in ggml discussion #1579, the official semver releases now include a nightly-tag.txt asset containing the tag of the corresponding nightly release (e.g. b10485). The Web UI assets are published to the HF bucket under the nightly tag, so this makes them discoverable for each official release. - make-release-desc.sh: expose the resolved nightly tag as a nightly_tag output - make-release.yml: create nightly-tag.txt from that tag, upload it as a release asset (skipped on dry-run), mention it in the release body and in the dry-run summary Assisted-by: pi:llama.cpp/Qwen3.8-27B --- .github/workflows/make-release.yml | 37 ++++++++++++++++++++++++++++++ scripts/make-release-desc.sh | 4 +++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml index 95e0f3475..451ec261f 100644 --- a/.github/workflows/make-release.yml +++ b/.github/workflows/make-release.yml @@ -55,6 +55,20 @@ jobs: env: GITHUB_REPOSITORY: ${{ github.repository }} + - name: Create nightly-tag.txt + id: nightly_tag_file + run: | + NIGHTLY_TAG="${{ steps.desc.outputs.nightly_tag }}" + if [[ -z "${NIGHTLY_TAG}" ]]; then + echo "Warning: no nightly tag found for the release commit - nightly-tag.txt will not be created" + echo "create=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "${NIGHTLY_TAG}" > nightly-tag.txt + echo "create=true" >> "$GITHUB_OUTPUT" + echo "nightly-tag.txt:" + cat nightly-tag.txt + - name: Create release if: ${{ github.event.inputs.dry_run == 'false' }} uses: ggml-org/action-create-release@v1 @@ -70,18 +84,41 @@ jobs: New version has been released. ${{ steps.desc.outputs.nightly }} + + **Web UI:** the `nightly-tag.txt` asset contains the tag of the corresponding nightly release + **More info:** [dist : releases and versioning of ggml-org projects](https://github.com/ggml-org/ggml/discussions/1579) ## ${{ steps.desc.outputs.changelog_title }} ${{ steps.desc.outputs.changelog }} + - name: Upload nightly-tag.txt + if: ${{ github.event.inputs.dry_run == 'false' && steps.nightly_tag_file.outputs.create == 'true' }} + uses: actions/github-script@v8 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const fs = require('fs'); + const release_id = '${{ steps.create_release.outputs.id }}'; + console.log('uploadReleaseAsset', 'nightly-tag.txt'); + await github.rest.repos.uploadReleaseAsset({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: release_id, + name: 'nightly-tag.txt', + data: await fs.readFileSync('./nightly-tag.txt') + }); + - name: Dry run summary if: ${{ github.event.inputs.dry_run == 'true' }} run: | if [[ "${{ steps.checks.outputs.checks_passed }}" == "true" ]]; then echo "Dry run complete - all checks passed." echo "Would have created tag: ${{ steps.checks.outputs.version }}" + if [[ -n "${{ steps.desc.outputs.nightly_tag }}" ]]; then + echo "Would have uploaded nightly-tag.txt: ${{ steps.desc.outputs.nightly_tag }}" + fi else echo "::error::Dry run found release check failures. A release tag would not be created." exit 1 diff --git a/scripts/make-release-desc.sh b/scripts/make-release-desc.sh index f1e4566e4..59aa67cba 100755 --- a/scripts/make-release-desc.sh +++ b/scripts/make-release-desc.sh @@ -15,7 +15,8 @@ # tag exists. # # Env (when running in GitHub Actions): -# GITHUB_OUTPUT: previous_tag, changelog_title, changelog and nightly are written here +# GITHUB_OUTPUT: previous_tag, changelog_title, changelog, nightly and nightly_tag +# are written here # GITHUB_REPOSITORY: owner/repo, used to build the nightly release URL (skipped when unset) set -euo pipefail @@ -80,6 +81,7 @@ if [[ -n "${GITHUB_OUTPUT:-}" ]]; then echo "previous_tag=${PREV}" echo "changelog_title=${CHANGELOG_TITLE}" echo "nightly=${NIGHTLY}" + echo "nightly_tag=${NIGHTLY_TAG}" echo "changelog< Date: Fri, 21 Aug 2026 12:30:03 +0200 Subject: [PATCH 36/44] ui: Settings navigation cleanup (#27241) * ui : rework the settings registry into ordered raw-data sections SETTINGS_REGISTRY becomes an ordered SettingsSectionEntry[] array; the array order is the sidebar display order. Section titles, color mode options and title radio options are declared inline in their section or entry. Entries gain showInUi; MCP servers, the system-message toggle and the title LLM flag become hidden entries of their own section. Derived values (config defaults, help info, chat sections, numeric field lists, syncable parameters) are still derived here; they move to their actual consumers in follow-up commits. * ui : extract settings localStorage persistence into SettingsService Stateless load/save of the settings config and user-override keys, plus the legacy theme key migration. Business logic (default merging, mobile sendOnEnter default, applying the migrated theme) stays in the store. * ui : move the settings exit route into ROUTES SETTINGS_FALLBACK_EXIT_ROUTE is just a route, so it lives with the other routes as ROUTES.SETTINGS_EXIT. * ui : derive the syncable parameter list in the parameter sync service The syncable parameter mapping is only consumed by the sync service, so derive it there from the registry instead of exporting it from the constants file. * ui : restore isPrivate for API key masking * ui : clean up settings registry and router fetch guard Drop the per-entry section field (duplicates the parent slug and is never read) and guard the router model fetch on fields?.length so the Tools/Import-Export pages with empty fields are excluded again. Assisted-by: pi * ui : merge sampling and penalties settings into one section Assisted-by: pi --- .../settings/SettingsChat/SettingsChat.svelte | 10 +- tools/ui/src/lib/constants/index.ts | 2 +- .../ui/src/lib/constants/routes.constants.ts | 14 +- ...try.constants.ts => settings.constants.ts} | 1005 ++++++++--------- tools/ui/src/lib/services/index.ts | 10 + .../lib/services/parameter-sync.service.ts | 16 +- tools/ui/src/lib/services/settings.service.ts | 76 ++ .../src/lib/stores/settings/index.svelte.ts | 76 +- .../lib/stores/settings/referrer.svelte.ts | 4 +- tools/ui/src/lib/types/settings.d.ts | 4 +- tools/ui/src/routes/settings/+layout.svelte | 4 +- 11 files changed, 610 insertions(+), 611 deletions(-) rename tools/ui/src/lib/constants/{settings-registry.constants.ts => settings.constants.ts} (74%) create mode 100644 tools/ui/src/lib/services/settings.service.ts diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte index 4233039ef..97ff30ba7 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte @@ -15,7 +15,7 @@ NUMERIC_FIELDS, POSITIVE_INTEGER_FIELDS, SETTINGS_CHAT_SECTIONS, - SETTINGS_SECTION_TITLES + SETTINGS_SECTION_SLUGS } from '$lib/constants'; import { ColorMode } from '$lib/enums/ui.enums'; import { RouterService } from '$lib/services/router.service'; @@ -46,7 +46,7 @@ let fetchInitiated = false; $effect(() => { - if (serverStore.isRouterMode && currentSection.fields && !fetchInitiated) { + if (serverStore.isRouterMode && currentSection.fields?.length && !fetchInitiated) { fetchInitiated = true; void modelsStore @@ -148,9 +148,9 @@

{currentSection.title}

- {#if currentSection.title === SETTINGS_SECTION_TITLES.TOOLS} + {#if currentSection.slug === SETTINGS_SECTION_SLUGS.TOOLS} - {:else if currentSection.title === SETTINGS_SECTION_TITLES.IMPORT_EXPORT} + {:else if currentSection.slug === SETTINGS_SECTION_SLUGS.IMPORT_EXPORT} {:else if currentSection.fields}
@@ -161,7 +161,7 @@ onThemeChange={handleThemeChange} /> - {#if currentSection.title === SETTINGS_SECTION_TITLES.GENERAL} + {#if currentSection.slug === SETTINGS_SECTION_SLUGS.GENERAL}