From 645ca2834bc16e7eab112a91aeb282ebb913f935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 19 Aug 2026 15:44:16 +0200 Subject: [PATCH 01/36] ci : re-enable release dependency for sycl (#27385) --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 01de72a8ff..f4e8bd2997 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1579,14 +1579,14 @@ jobs: - windows - windows-cpu - windows-cuda - #- windows-sycl + - windows-sycl - windows-rocm - windows-openvino #- ubuntu-22-rocm - ubuntu-cpu - ubuntu-vulkan - ubuntu-24-openvino - #- ubuntu-24-sycl + - ubuntu-24-sycl - android-arm64 - macos-cpu - ios-xcode From 2e92ecd0247d25f09797f8fdb044a166522fc05d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 19 Aug 2026 16:05:50 +0200 Subject: [PATCH 02/36] models : remove duplicate metadata load (#27378) --- src/models/deepseek32.cpp | 2 -- src/models/glm-dsa.cpp | 2 -- src/models/glm4-moe.cpp | 2 -- 3 files changed, 6 deletions(-) diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp index 08555a8016..2b82a780c4 100644 --- a/src/models/deepseek32.cpp +++ b/src/models/deepseek32.cpp @@ -10,8 +10,6 @@ void llama_model_deepseek32::load_arch_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false); // MoE parameters - ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert); - ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp index 803ef76747..93a1448b46 100644 --- a/src/models/glm-dsa.cpp +++ b/src/models/glm-dsa.cpp @@ -32,8 +32,6 @@ void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false); // MoE parameters - ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert); - ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cpp index d60e47ddf0..8cde66978c 100644 --- a/src/models/glm4-moe.cpp +++ b/src/models/glm4-moe.cpp @@ -6,8 +6,6 @@ void llama_model_glm4_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false); // MoE parameters - ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert); - ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); From 01ac3ad761e2a1076b6186254f41c17a2d42a31c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 19 Aug 2026 16:40:47 +0200 Subject: [PATCH 03/36] ci : add release attestation url (#27389) --- .github/workflows/release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f4e8bd2997..20c9690869 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1665,6 +1665,7 @@ jobs: tar -czvf release/llama-${{ steps.tag.outputs.name }}-ui.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./ui-dist . - name: Attest release artifacts + id: attest uses: actions/attest@v4 with: subject-path: 'release/*' @@ -1696,6 +1697,9 @@ jobs: **Website:** - + **Attestations:** + - <${{ steps.attest.outputs.attestation-url }}> + **macOS/iOS:** - [macOS Apple Silicon (arm64)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-arm64.tar.gz) - macOS Apple Silicon (arm64, KleidiAI enabled) [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23780) From 6cc504a2e90d5e41fb37e98b1da879f72d4d9f31 Mon Sep 17 00:00:00 2001 From: Titaniumtown Date: Wed, 19 Aug 2026 07:46:01 -0700 Subject: [PATCH 04/36] sycl: report zero devices instead of aborting when the host has none (#27291) Prevents crash when not even performing SYCL compute, for instance when trying to run `llama-quantize`. --- ggml/src/ggml-sycl/ggml-sycl.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 92c26839fc..c7434a6bdb 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -109,7 +109,14 @@ int g_ggml_sycl_enable_host_pinned_mem = 1; static ggml_sycl_device_info ggml_sycl_init() { ggml_sycl_device_info info = {}; - info.device_count = dpct::dev_mgr::instance().device_count(); + // Do not hard crash when there exists no SYCL devices. + // We want to allow the user to use non-SYCL tools when SYCL is compiled (such as llama-quantize) + try { + info.device_count = dpct::dev_mgr::instance().device_count(); + } catch (sycl::exception const &exc) { + GGML_LOG_INFO("%s: no SYCL device available: %s\n", __func__, exc.what()); + info.device_count = 0; + } if (info.device_count == 0) { GGML_LOG_ERROR("%s: failed to initialize: %s\n", GGML_SYCL_NAME, __func__); return info; From 7221e24f579efbb5fd2a2ced1fffaef1aff2fe0d Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 19 Aug 2026 08:53:31 -0600 Subject: [PATCH 05/36] model : GraniteSWAForCausalLM / GraniteMoeSWAForCausalLM (#25505) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(convert): Add conversion for GraniteSWAForCausalLM Branch: GraniteSWAForCausalLM AI-usage: full (Bob, OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart * feat(llama): Add granite_swa support Branch: GraniteSWAForCausalLM AI-usage: full (Bob, OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart * feat(conversion): Add conversion infra for rope_pattern array NOTE: There is other work also targeting this, so this may be removed depending on merge order. Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart * fix(conversion): Fix SWA pattern logic and support for non-rope layers Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart * feat(conversion): Add support for GraniteMoeSWA Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart * feat: Add llama_hparams::has_rope and arch constants NOTE: This shadows the work done for Granite Speech https://github.com/ggml-org/llama.cpp/pull/25107 Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart * feat: Add support for per-layer rope determination Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart * style: Fix failing flake8 for extra newlines Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * test: Write out SLIDING_WINDOW_PATTERN in llama-model-saver Branch: GraniteSWAForCausalLM AI-usage: full (OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart * fix(convert): Fix missing registration for GraniteMoeSWAForCausalLM Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * fix: Load MoE params as optional Branch: GraniteSWAForCausalLM AI-usage: draft (OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart * feat: Handle MoE params in conversion branch: GraniteSWAForCausalLM AI-usage: full (OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart * style: Remove unnecessary newline AI-usage: none Signed-off-by: Gabe Goodhart * fix: Remove unnecessary tensor additions to GRANITE architecture Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * fix: Correctly handle naming for ffn gate inp Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * fix: Always default hparams.rope_pattern to 1s This isn't strictly necessary, but it will allow other models to rely on hparams.has_rope(il) without needting to prepopulate. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * feat: Move to has_rope for all granite model architectures Now that we have a proper hparam for this, it's better to use it and not require a hacky fallback in the hparam method itself. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * feat: No hacky rope_finetuned fallback in has_rope Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * fix: Fully remove rope hparam filling in granitemoe There are no granitemoe models that use NoPE (it's not actually used in the layer building below), so this was just dead code. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * fix: Save out rope_pattern in model-saver Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * fix: Set hparams.rope_finetuned for round trip Since the value is _read_ from rope_finetuned, we need to persist it when the model is saved with the saver. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * fix: Code review cleanup Signed-off-by: Gabe Goodhart Co-authored-by: Sigbjørn Skjæret Co-authored-by: Sigbjørn Skjæret * refactor: Keep gate/up fused for MoE path Branch: GraniteSWAForCausalLM AI-usage: full (Claude + Sonnet 5) Signed-off-by: Gabe Goodhart * fix: Skip GRANITE_SWA in model saver https://github.com/ggml-org/llama.cpp/pull/25505#discussion_r3773175651 Keeping is_swa_impl in the saver can break other models. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * add sliding window pattern for model in test * style: Fix indentation Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * fix: Fix \r\n Thanks Claude! Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart * feat: Keep shared expert fused Branch: GraniteSWAForCausalLM AI-usage: full (Claude + Sonnet 5) Signed-off-by: Gabe Goodhart * style: More indentation fixes Signed-off-by: Gabe Goodhart Co-authored-by: Sigbjørn Skjæret --------- Signed-off-by: Gabe Goodhart Co-authored-by: Sigbjørn Skjæret --- conversion/__init__.py | 2 + conversion/granite.py | 102 +++++++++++ gguf-py/gguf/constants.py | 28 +++ gguf-py/gguf/gguf_writer.py | 3 + gguf-py/gguf/tensor_mapping.py | 1 + src/llama-arch.cpp | 3 + src/llama-arch.h | 3 + src/llama-hparams.cpp | 6 +- src/llama-hparams.h | 4 + src/llama-model-saver.cpp | 2 + src/llama-model.cpp | 4 + src/models/granite-hybrid.cpp | 8 +- src/models/granite-moe.cpp | 5 - src/models/granite-swa.cpp | 319 +++++++++++++++++++++++++++++++++ src/models/granite-switch.cpp | 7 +- src/models/granite.cpp | 8 +- src/models/models.h | 28 +++ tests/test-llama-archs.cpp | 2 +- 18 files changed, 517 insertions(+), 18 deletions(-) create mode 100644 src/models/granite-swa.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index 3232a1050b..5ae6ad819f 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -109,6 +109,8 @@ TEXT_MODEL_MAP: dict[str, str] = { "GraniteSwitchForCausalLM": "granite", "GraniteSpeechForConditionalGeneration": "granite", "GraniteSpeechPlusForConditionalGeneration": "granite", + "GraniteSWAForCausalLM": "granite", + "GraniteMoeSWAForCausalLM": "granite", "Grok1ForCausalLM": "grok", "GrokForCausalLM": "grok", "GroveMoeForCausalLM": "grovemoe", diff --git a/conversion/granite.py b/conversion/granite.py index 5f1e3e8472..796d37cca2 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -74,6 +74,108 @@ class GraniteModel(LlamaModel): return super().filter_tensors(item) +@ModelBase.register("GraniteSWAForCausalLM") +class GraniteSWAModel(GraniteModel): + """Conversion for IBM's GraniteSWAForCausalLM (interleaved sliding window attention)""" + model_arch = gguf.MODEL_ARCH.GRANITE_SWA + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + if name.endswith("sinks"): + name += ".weight" + + return super().filter_tensors((name, gen)) + + def set_gguf_parameters(self): + """GraniteSWA uses Granite parameters plus sliding window configuration.""" + super().set_gguf_parameters() + + # Add sliding_window from config + sliding_window = self.hparams.get("sliding_window", 128) + self.gguf_writer.add_sliding_window(sliding_window) + logger.info("gguf: (granite_swa) sliding_window = %s", sliding_window) + + # Derive sliding_window_pattern from layer_types + if layer_types := self.hparams.get("layer_types"): + is_swa = [t == "sliding_attention" for t in layer_types] + self.gguf_writer.add_sliding_window_pattern(is_swa) + logger.info("gguf: (granite_swa) sliding_window_pattern = %d SWA layers / %d total", + sum(is_swa), len(is_swa)) + else: + # Fall back to period-based pattern: i % 4 != 0 + # This matches the transformers default pattern + n_layers = self.block_count + is_swa = [i % 4 != 0 for i in range(n_layers)] + self.gguf_writer.add_sliding_window_pattern(is_swa) + logger.info("gguf: (granite_swa) sliding_window_pattern (inferred) = %d SWA layers / %d total", + sum(is_swa), n_layers) + + # Add rope_pattern from no_rope_layers + if no_rope_layers := self.hparams.get("no_rope_layers"): + # Convert 1/0 to bool (1 = use RoPE, 0 = NoPE) + rope_pattern = [bool(x) for x in no_rope_layers] + self.gguf_writer.add_rope_pattern(rope_pattern) + logger.info("gguf: (granite_swa) rope_pattern = %d RoPE layers / %d total", + sum(rope_pattern), len(rope_pattern)) + + +@ModelBase.register("GraniteMoeSWAForCausalLM") +class GraniteMoeSWAModel(GraniteSWAModel): + """Conversion for IBM's GraniteMoeSWAForCausalLM (unified dense + MoE with iSWA)""" + model_arch = gguf.MODEL_ARCH.GRANITE_SWA + + def set_gguf_parameters(self): + super().set_gguf_parameters() + if shared_intermediate_size := self.hparams.get("shared_intermediate_size"): + self.gguf_writer.add_expert_shared_feed_forward_length(shared_intermediate_size) + logger.info("gguf: (granitemoewa) shared_intermediate_size = %s", shared_intermediate_size) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + """Split merged MoE tensors (gate+up) following standard MoE pattern.""" + + # Handle expert FFN tensors (merged gate+up) - swash format: experts.gate_up_proj + # Kept fused since inference (build_moe_ffn) supports a single gate_up_exps + # tensor for the routed experts. + if name.endswith("block_sparse_moe.experts.gate_up_proj"): + ffn_dim = self.hparams["intermediate_size"] + assert data_torch.shape[-2] == 2 * ffn_dim, f"Merged FFN tensor size must be 2 * intermediate_size, got {data_torch.shape[-2]}" + yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_UP_EXP, bid), bid) + return + + # Handle expert FFN down projection - swash format: experts.down_proj + if name.endswith("block_sparse_moe.experts.down_proj"): + yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_EXP, bid), bid) + return + + # Handle expert FFN tensors (merged gate+up) - standard granite format: input_linear.weight + # Kept fused since inference (build_moe_ffn) supports a single gate_up_exps + # tensor for the routed experts. + if name.endswith("block_sparse_moe.input_linear.weight"): + ffn_dim = self.hparams["intermediate_size"] + assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * intermediate_size" + yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_UP_EXP, bid), bid) + return + + # Handle shared expert FFN tensors (if present) - kept fused since + # inference (build_ffn) supports a single ffn_up_shexp tensor with + # LLM_FFN_SWIGLU for the shared expert. + if name.endswith("shared_mlp.input_linear.weight"): + ffn_dim = self.hparams.get("shared_intermediate_size", self.hparams["intermediate_size"]) + assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * shared_intermediate_size" + yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), bid) + return + + # Handle shared expert output (if present) + if name.endswith("shared_mlp.output_linear.weight"): + yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, bid), bid) + return + + # Pass through to parent for all other tensors (including sinks) + yield from super().modify_tensors(data_torch, name, bid) + + @ModelBase.register("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM") @ModelBase.example("ibm-granite/granite-3.1-3b-a800m-instruct") class GraniteMoeModel(GraniteModel): diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index d043c9b6ec..fad8d1fd8c 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -208,6 +208,7 @@ class Keys: SHARED_KV_LAYERS = "{arch}.attention.shared_kv_layers" SLIDING_WINDOW_PATTERN = "{arch}.attention.sliding_window_pattern" TEMPERATURE_SCALE = "{arch}.attention.temperature_scale" + ROPE_PATTERN = "{arch}.attention.rope_pattern" class Indexer: HEAD_COUNT = "{arch}.attention.indexer.head_count" @@ -549,6 +550,7 @@ class MODEL_ARCH(IntEnum): GRANITE_MOE = auto() GRANITE_HYBRID = auto() GRANITE_SWITCH = auto() + GRANITE_SWA = auto() CHAMELEON = auto() WAVTOKENIZER_DEC = auto() PLM = auto() @@ -1265,6 +1267,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.GRANITE_MOE: "granitemoe", MODEL_ARCH.GRANITE_HYBRID: "granitehybrid", MODEL_ARCH.GRANITE_SWITCH: "graniteswitch", + MODEL_ARCH.GRANITE_SWA: "granite_swa", MODEL_ARCH.CHAMELEON: "chameleon", MODEL_ARCH.WAVTOKENIZER_DEC: "wavtokenizer-dec", MODEL_ARCH.PLM: "plm", @@ -4152,6 +4155,31 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.GRANITE_SWA: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_SINKS, + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + # MoE (GraniteMoeSWA) + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_GATE_UP_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + # Shared expert - gate+up kept fused in FFN_UP_SHEXP (LLM_FFN_SWIGLU) + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + ], MODEL_ARCH.CHAMELEON: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 9e0914fd86..16ae9f999d 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -824,6 +824,9 @@ class GGUFWriter: else: self.add_array(key, value) + def add_rope_pattern(self, value: Sequence[bool]) -> None: + self.add_array(Keys.Attention.ROPE_PATTERN.format(arch=self.arch), value) + def add_dense_features_dims(self, dense:str, in_f:int, out_f:int) -> None: self.add_uint32(Keys.LLM.DENSE_FEAT_IN_SIZE.format(arch=self.arch, dense=dense), in_f) self.add_uint32(Keys.LLM.DENSE_FEAT_OUT_SIZE.format(arch=self.arch, dense=dense), out_f) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 3292942b41..a0571ccd32 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -458,6 +458,7 @@ class TensorNameMap: "transformer.decoder_layer.{bid}.router", # Grok "transformer.blocks.{bid}.ffn.router.layer", # dbrx "model.layers.{bid}.block_sparse_moe.router.layer", # granitemoe + "model.layers.{bid}.block_sparse_moe.router", # granite_swa "model.layers.{bid}.feed_forward.router", # llama4 jamba "encoder.layers.{bid}.mlp.router.layer", # nomic-bert-moe "model.layers.{bid}.mlp.router", # openai-moe diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 5b88bde14d..955c2d7965 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -102,6 +102,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_GRANITE_MOE, "granitemoe" }, { LLM_ARCH_GRANITE_HYBRID, "granitehybrid" }, { LLM_ARCH_GRANITE_SWITCH, "graniteswitch" }, + { LLM_ARCH_GRANITE_SWA, "granite_swa" }, { LLM_ARCH_CHAMELEON, "chameleon" }, { LLM_ARCH_WAVTOKENIZER_DEC, "wavtokenizer-dec" }, { LLM_ARCH_PLM, "plm" }, @@ -261,6 +262,8 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" }, { LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" }, { LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, "%s.attention.sliding_window_pattern" }, + { LLM_KV_ATTENTION_ROPE_PATTERN, "%s.attention.rope_pattern" }, + { LLM_KV_ATTENTION_SCALE, "%s.attention.scale" }, { LLM_KV_ATTENTION_OUTPUT_SCALE, "%s.attention.output_scale" }, { LLM_KV_ATTENTION_VALUE_SCALE, "%s.attention.value_scale" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 8042120a25..48fe051a99 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -107,6 +107,7 @@ enum llm_arch { LLM_ARCH_GRANITE_MOE, LLM_ARCH_GRANITE_HYBRID, LLM_ARCH_GRANITE_SWITCH, + LLM_ARCH_GRANITE_SWA, LLM_ARCH_CHAMELEON, LLM_ARCH_WAVTOKENIZER_DEC, LLM_ARCH_PLM, @@ -267,6 +268,8 @@ enum llm_kv { LLM_KV_ATTENTION_SLIDING_WINDOW, LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, LLM_KV_ATTENTION_SCALE, + LLM_KV_ATTENTION_ROPE_PATTERN, + LLM_KV_ATTENTION_OUTPUT_SCALE, LLM_KV_ATTENTION_VALUE_SCALE, LLM_KV_ATTENTION_TEMPERATURE_LENGTH, diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index e3f0cf0ede..cbe31134ff 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -291,7 +291,11 @@ bool llama_hparams::has_rope(uint32_t il) const { return false; } - return true; + if (il < n_layer_all) { + return rope_pattern[il] != 0; + } + + GGML_ABORT("%s: il (%u) out of bounds (n_layer_all: %u)\n", __func__, il, n_layer_all); } uint32_t llama_hparams::n_layer() const { diff --git a/src/llama-hparams.h b/src/llama-hparams.h index e91ce1cc3c..f6af36436b 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -144,6 +144,10 @@ struct llama_hparams { std::array rope_sections; + // Per-layer RoPE enable flags (1 = use RoPE, 0 = NoPE) + // by default, all layers use RoPE (controlled by rope_finetuned) + std::array rope_pattern; + // Sliding Window Attention (SWA) llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE; // the size of the sliding window (0 - no SWA) diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index be9524d404..b9e0a60094 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -30,6 +30,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) { case LLM_ARCH_MUSE_GLIMMER: case LLM_ARCH_MELLUM: case LLM_ARCH_LAGUNA: + case LLM_ARCH_GRANITE_SWA: return false; default: return true; @@ -272,6 +273,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, hparams.n_lora_value_res_mix); add_kv(LLM_KV_ATTENTION_GATE_LORA_RANK, hparams.n_lora_gate); add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts); + add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, true); add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); // add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???); add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 0d74a2135b..3759c86259 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -246,6 +246,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_minicpm(params); case LLM_ARCH_GRANITE_HYBRID: return new llama_model_granite_hybrid(params); + case LLM_ARCH_GRANITE_SWA: + return new llama_model_granite_swa(params); case LLM_ARCH_CHAMELEON: return new llama_model_chameleon(params); case LLM_ARCH_WAVTOKENIZER_DEC: @@ -1157,6 +1159,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0); std::fill(hparams.rope_sections.begin(), hparams.rope_sections.end(), 0); + std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), 1); std::fill(hparams.is_swa_impl.begin(), hparams.is_swa_impl.end(), 0); std::fill(hparams.is_recr_impl.begin(), hparams.is_recr_impl.end(), llm_arch_is_recurrent(ml.get_arch()) ? 1 : 0); std::fill(hparams.is_indexer_full_impl.begin(), hparams.is_indexer_full_impl.end(), 0); @@ -2639,6 +2642,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_GRANITE_MOE: case LLM_ARCH_GRANITE_HYBRID: case LLM_ARCH_GRANITE_SWITCH: + case LLM_ARCH_GRANITE_SWA: case LLM_ARCH_CHAMELEON: case LLM_ARCH_BAILINGMOE: case LLM_ARCH_BAILINGMOE3: diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cpp index eb23095aec..8a8f7e19ff 100644 --- a/src/models/granite-hybrid.cpp +++ b/src/models/granite-hybrid.cpp @@ -16,7 +16,8 @@ void llama_model_granite_hybrid::load_arch_hparams(llama_model_loader & ml) { // Granite uses rope_finetuned as a switch for rope, so default to true bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); - hparams.rope_finetuned = rope_finetuned; + hparams.rope_finetuned = rope_finetuned; // needed for round trip save + std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned); // A layer is recurrent IFF the n_head_kv value is set to 0 for (uint32_t i = 0; i < hparams.n_layer(); ++i) { @@ -147,7 +148,7 @@ llama_model_granite_hybrid::graph::graph(const llama_model & model, const llm_gr // Positional embeddings populated if rope enabled ggml_tensor * inp_pos = nullptr; - if (hparams.rope_finetuned) { + if (hparams.has_rope(0)) { inp_pos = build_inp_pos(); } @@ -206,8 +207,7 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_attention_layer(ggml_tens const int il) { auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); - const bool use_rope = hparams.rope_finetuned; - if (use_rope) { + if (hparams.has_rope(il)) { ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); diff --git a/src/models/granite-moe.cpp b/src/models/granite-moe.cpp index 115263c418..09be49393e 100644 --- a/src/models/granite-moe.cpp +++ b/src/models/granite-moe.cpp @@ -7,11 +7,6 @@ void llama_model_granite_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false); - // Granite uses rope_finetuned as a switch for rope, so default to true - bool rope_finetuned = true; - ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); - hparams.rope_finetuned = rope_finetuned; - switch (hparams.n_layer()) { case 32: type = LLM_TYPE_3B; break; case 40: type = LLM_TYPE_3B; break; diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp new file mode 100644 index 0000000000..3aa2b63b23 --- /dev/null +++ b/src/models/granite-swa.cpp @@ -0,0 +1,319 @@ +#include "models.h" + +#include + +void llama_model_granite_swa::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); + ml.get_key(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale, false); + ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); + ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false); + + // MoE expert configuration + ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false); + ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false); + + // iSWA configuration + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + + // Granite4 Vision uses array deepstack_mapping + ml.get_arr(LLM_KV_DEEPSTACK_MAPPING, hparams.deepstack_mapping_arr, false); + + // Count the unique deepstack input indices + std::unordered_set unique_deepstack_idxs; + for (const auto val : hparams.deepstack_mapping_arr) { + if (val >= 0) { + unique_deepstack_idxs.insert(val); + } + } + hparams.n_deepstack_layers = unique_deepstack_idxs.size(); + + // Ensure all values are valid (avoid overflow attacks) + for (const auto val : unique_deepstack_idxs) { + if (val > hparams.n_deepstack_layers) { + std::stringstream ss; + ss << "Invalid deepstack index: " << val << " > " << hparams.n_deepstack_layers; + throw std::runtime_error(ss.str()); + } + } + + // Per-layer RoPE pattern (optional) + ml.get_arr(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, false); + + switch (hparams.n_layer()) { + case 32: type = LLM_TYPE_3B; break; + case 40: type = LLM_TYPE_3B; break; + // Add additional layer/vocab/etc checks here for other model sizes + default: type = LLM_TYPE_UNKNOWN; + } + + // For Granite MoE Shared + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, /* required */ false); +} + +void llama_model_granite_swa::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + // output + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); + + // optional bias tensors + layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); + + // Per-layer attention sinks for iSWA + layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) { + layer.rope_long = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0)); + layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0)); + } + else { + layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0)); + } + + if (n_expert == 0) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + + // optional MLP bias + layer.ffn_gate_b = create_tensor(tn(LLM_TENSOR_FFN_GATE, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); + layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); + layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); + } else { + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0); + create_tensor_gate_up_exps(layer, i, n_embd, n_ff, n_expert, 0); + + // For Granite MoE Shared - gate+up kept fused in ffn_up_shexp (see LLM_FFN_SWIGLU below) + if (hparams.n_ff_shexp > 0) { + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, 2*hparams.n_ff_shexp}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {hparams.n_ff_shexp, n_embd}, 0); + } + } + } +} + +std::unique_ptr llama_model_granite_swa::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_granite_swa::graph::graph( + const llama_model & model, + const llm_graph_params & params) + : llm_graph_context(params) { + + const int64_t n_embd_head = hparams.n_embd_head_v(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + GGML_ASSERT(n_embd_head == n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - built only if rope enabled + ggml_tensor * inp_pos = build_inp_pos(); + auto * inp_attn = build_attn_inp_kv_iswa(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { + + // Granite Vision 4.1 deepstack: inject the projector stream that + // targets decoder layer `il` before the decoder runs. + // NOTE: skip the first deepstack layer since that's inpL + const auto & deepstack_emb_idx = hparams.deepstack_mapping_arr[il]; + if (il > 0 && deepstack_emb_idx >= 0) { + ggml_tensor * ds = ggml_view_2d(ctx0, + res->t_inp_embd, n_embd, n_tokens, + res->t_inp_embd->nb[1], + deepstack_emb_idx * n_embd * sizeof(float)); + inpL = ggml_add(ctx0, inpL, ds); + cb(inpL, "deepstack_in", il); + } + + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + cur = build_attention_layer( + cur, inp_pos, inp_attn, + model, n_embd_head, il); + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + // ffn + cur = build_layer_ffn(cur, inpSA, model, il); + + // input for next layer + inpL = cur; + } + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur, model.output_s); + + // For Granite architectures - scale logits + cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_logit_scale); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} + +ggml_tensor * llama_model_granite_swa::graph::build_attention_layer( + ggml_tensor * cur, + ggml_tensor * inp_pos, + llm_graph_input_attn_kv_iswa * inp_attn, + const llama_model & model, + const int64_t n_embd_head, + const int il) { + + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); + + const bool use_rope = hparams.has_rope(il); + if (use_rope) { + ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; + + // Pass layer.attn_sinks to build_attn for sink-based attention modulation + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, model.layers[il].attn_sinks, nullptr, kq_scale, il); + cb(cur, "attn_out", il); + return cur; +} + +ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn( + ggml_tensor * cur, + ggml_tensor * inpSA, + const llama_model & model, + const int il) { + + // For Granite architectures - scale residual + if (hparams.f_residual_scale) { + cur = ggml_scale(ctx0, cur, hparams.f_residual_scale); + } + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network (non-MoE) + if (model.layers[il].ffn_gate_inp == nullptr) { + + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + } else { + // MoE branch + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + ggml_tensor * moe_out = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, true, + hparams.expert_weights_scale, + LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, + il, + nullptr, model.layers[il].ffn_gate_up_exps); + cb(moe_out, "ffn_moe_out", il); + + // For Granite MoE Shared - gate+up kept fused in ffn_up_shexp + if (hparams.n_ff_shexp > 0) { + ggml_tensor * ffn_shexp = build_ffn(cur, + model.layers[il].ffn_up_shexp, NULL, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down_shexp, NULL, NULL, + NULL, + LLM_FFN_SWIGLU, LLM_FFN_SEQ, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } else { + cur = moe_out; + } + } + + // For Granite architectures - scale residual + if (hparams.f_residual_scale) { + cur = ggml_scale(ctx0, cur, hparams.f_residual_scale); + } + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + return cur; +} diff --git a/src/models/granite-switch.cpp b/src/models/granite-switch.cpp index 80f6b86edc..7c9a901c8a 100644 --- a/src/models/granite-switch.cpp +++ b/src/models/granite-switch.cpp @@ -11,7 +11,8 @@ void llama_model_granite_switch::load_arch_hparams(llama_model_loader & ml) { bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); - hparams.rope_finetuned = rope_finetuned; + hparams.rope_finetuned = rope_finetuned; // needed for round trip save + std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned); switch (hparams.n_layer()) { case 40: type = hparams.n_embd == 4096 ? LLM_TYPE_8B : LLM_TYPE_3B; break; @@ -254,7 +255,7 @@ llama_model_granite_switch::graph::graph( cb(inpL, "inp_embd", -1); ggml_tensor * inp_pos = nullptr; - if (hparams.rope_finetuned) { + if (hparams.has_rope(0)) { inp_pos = build_inp_pos(); } auto * inp_attn = build_attn_inp_kv(); @@ -361,7 +362,7 @@ ggml_tensor * llama_model_granite_switch::graph::build_attention_layer( Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - if (hparams.rope_finetuned) { + if (hparams.has_rope(il)) { ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, diff --git a/src/models/granite.cpp b/src/models/granite.cpp index 4a75c5ff3c..9e9f97e94d 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -33,7 +33,8 @@ void llama_model_granite::load_arch_hparams(llama_model_loader & ml) { // Granite uses rope_finetuned as a switch for rope, so default to true bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); - hparams.rope_finetuned = rope_finetuned; + hparams.rope_finetuned = rope_finetuned; // needed for round trip save + std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned); switch (hparams.n_layer()) { case 32: type = LLM_TYPE_3B; break; @@ -127,7 +128,7 @@ llama_model_granite::graph::graph( // inp_pos - built only if rope enabled ggml_tensor * inp_pos = nullptr; - if (hparams.rope_finetuned) { + if (hparams.has_rope(0)) { inp_pos = build_inp_pos(); } auto * inp_attn = build_attn_inp_kv(); @@ -203,8 +204,7 @@ ggml_tensor * llama_model_granite::graph::build_attention_layer( auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); - const bool use_rope = hparams.rope_finetuned; - if (use_rope) { + if (hparams.has_rope(il)) { ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); Qcur = ggml_rope_ext( ctx0, Qcur, inp_pos, rope_factors, diff --git a/src/models/models.h b/src/models/models.h index 180b30a46d..1dd30dfd16 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1719,6 +1719,34 @@ struct llama_model_granite_hybrid : public llama_model_base { }; +struct llama_model_granite_swa : public llama_model_base { + llama_model_granite_swa(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + + private: + ggml_tensor * build_attention_layer( + ggml_tensor * cur, + ggml_tensor * inp_pos, + llm_graph_input_attn_kv_iswa * inp_attn, + const llama_model & model, + const int64_t n_embd_head, + const int il); + + ggml_tensor * build_layer_ffn( + ggml_tensor * cur, + ggml_tensor * inpSA, + const llama_model & model, + const int il); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_chameleon : public llama_model_base { llama_model_chameleon(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 032ac35376..4eb3763cbe 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -197,7 +197,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f); // SWA pattern: every 5th layer is full attention (matches E2B layer_types) ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5)); - } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_MUSE_GLIMMER) { + } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA) { std::vector pattern; pattern.reserve(n_layer); for (uint32_t il = 0; il < n_layer; il++) { From 3e7344670adf63ce28527a4d42f2d71eca27c41e Mon Sep 17 00:00:00 2001 From: Niklas Wenzel Date: Wed, 19 Aug 2026 17:05:48 +0200 Subject: [PATCH 06/36] Revert "common: share thread pools when `n_threads` differ (#27138)" (#27337) * Revert "common: share thread pools when `n_threads` differ (#27138)" This reverts commit 04b569142da23d91beca090a99098d592d3f3c80. Co-authored-by: Max Krasnyansky * common: add comment about inability to share threadpool --------- Co-authored-by: Max Krasnyansky --- common/common.cpp | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 0f2f01ad0e..25ca838dff 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1750,18 +1750,6 @@ struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const commo return tpp; } -namespace { - -bool can_share_threadpool(const ggml_threadpool_params & tpp1, const ggml_threadpool_params & tpp2) { - // n_threads does not matter -> we'll use what's larger - ggml_threadpool_params tpp_comparison = tpp1; - tpp_comparison.n_threads = tpp2.n_threads; - - return ggml_threadpool_params_match(&tpp_comparison, &tpp2); -} - -} // namespace - common_threadpools::~common_threadpools() { if (!free_fn) { return; @@ -1790,9 +1778,9 @@ void common_threadpools::init(llama_context * ctx, const common_params & params) struct ggml_threadpool_params tpp = ggml_threadpool_params_from_cpu_params(params.cpuparams); - if (can_share_threadpool(tpp, tpp_batch)) { - tpp.n_threads = std::max(tpp.n_threads, tpp_batch.n_threads); - } else { + // each pool needs to match the respective n_threads exactly + // see: https://github.com/ggml-org/llama.cpp/pull/27138#issuecomment-5332307332 + if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) { threadpool_batch = ggml_threadpool_new_fn(&tpp_batch); if (!threadpool_batch) { COM_WRN("batch threadpool create failed : n_threads %d\n", tpp_batch.n_threads); From b95502ba9aa0eb73a2f4fc8878d7fbe6a847a0b9 Mon Sep 17 00:00:00 2001 From: Jetson Tan Date: Wed, 19 Aug 2026 23:43:10 +0800 Subject: [PATCH 07/36] vulkan: add null checks in ggml_vk_queue_command_pools_cleanup (#27353) * Guard against null queue pointers. --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 62ef7bb8a2..6c60ac0dc2 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -3384,10 +3384,10 @@ static void ggml_vk_queue_command_pools_cleanup(vk_device& device) { // Arbitrary frequency to cleanup/reuse command buffers static constexpr uint32_t cleanup_frequency = 10; - if (device->compute_queue->cmd_pool.buffers_in_use() >= cleanup_frequency) { + if (device->compute_queue && device->compute_queue->cmd_pool.buffers_in_use() >= cleanup_frequency) { ggml_vk_command_pool_cleanup(device, device->compute_queue->cmd_pool); } - if (device->transfer_queue->cmd_pool.buffers_in_use() >= cleanup_frequency) { + if (device->transfer_queue && device->transfer_queue->cmd_pool.buffers_in_use() >= cleanup_frequency) { ggml_vk_command_pool_cleanup(device, device->transfer_queue->cmd_pool); } } From dc72703fc69698b1ea68ece8d2dd8a96e6a4e1fe Mon Sep 17 00:00:00 2001 From: Nathanw1014 <67372905+Nathanw1014@users.noreply.github.com> Date: Thu, 20 Aug 2026 01:14:15 +0930 Subject: [PATCH 08/36] vulkan : dequant q8_0 KV once in coopmat1 (#25494) * vulkan : dequant q8_0 KV once in coopmat1 Assisted-by: Claude (Opus 4.8) * vulkan : fall back instead of aborting when FA scratch exceeds maxStorageBufferRange * vulkan : require KV-cache layout in FA dequant path Assisted-by: Claude (Opus 4.8) * vulkan : skip FA dequant path on coopmat2 Assisted-by: Claude (Opus 4.8) * tests : add contiguously-allocated quant K/V FA tests Assisted-by: Claude (Opus 4.8) * vulkan : trim comments * vulkan : tighten permutation checks for FA path * vulkan : set prealloc_x_need_sync after the FA dispatch * vulkan : exclude Intel Xe1 from FA dequant path --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 78 +++++++++++++++++-- .../vulkan-shaders/dequant_q8_0.comp | 11 +++ .../vulkan-shaders/vulkan-shaders-gen.cpp | 4 + tests/test-backend-ops.cpp | 18 +++-- 4 files changed, 101 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 6c60ac0dc2..f6cbaecb7f 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -913,6 +913,7 @@ struct vk_device_struct { vk_pipeline pipeline_quantize_q8_1_x4; vk_pipeline pipeline_dequant[GGML_TYPE_COUNT]; + vk_pipeline pipeline_dequant_transpose[GGML_TYPE_COUNT]; // fused dequant+transpose for FA quant-KV vk_pipeline pipeline_dequant_mul_mat_vec_f32_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols]; vk_pipeline pipeline_dequant_mul_mat_vec_f16_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols]; vk_pipeline pipeline_dequant_mul_mat_vec_id_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT]; @@ -5391,6 +5392,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q5_0], "dequant_q5_0", dequant_q5_0_len, dequant_q5_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q5_1], "dequant_q5_1", dequant_q5_1_len, dequant_q5_1_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q8_0], "dequant_q8_0", dequant_q8_0_len, dequant_q8_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant_transpose[GGML_TYPE_Q8_0], "dequant_q8_0_transpose", dequant_q8_0_transpose_len, dequant_q8_0_transpose_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q2_K], "dequant_q2_k", dequant_q2_k_len, dequant_q2_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ2_0], "dequant_tq2_0", dequant_tq2_0_len, dequant_tq2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q3_K], "dequant_q3_k", dequant_q3_k_len, dequant_q3_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); @@ -10823,9 +10825,32 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx const bool f32acc = !ctx->device->fp16 || dst->op_params[3] == GGML_PREC_F32 || k->type == GGML_TYPE_BF16; + // dequant K/V once into an f16 scratch, reordered KV layout so FA can read without a stride + auto is_dense_kv_cache = [](const ggml_tensor * t) { + return t->nb[0] == ggml_type_size(t->type) && + t->nb[2] == ggml_row_size(t->type, t->ne[0]) && + t->nb[1] == t->nb[2] * t->ne[2] && + t->nb[3] == t->nb[1] * t->ne[1]; + }; + const bool k_quant = k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_BF16 && k->type != GGML_TYPE_F32; + const bool v_quant = v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_BF16 && v->type != GGML_TYPE_F32; + const bool use_dequant_kv = k_quant && v_quant && neq1 >= 64 && + is_dense_kv_cache(k) && is_dense_kv_cache(v) && + (uint64_t)ggml_nelements(k) * sizeof(ggml_fp16_t) <= ctx->device->properties.limits.maxStorageBufferRange && + (uint64_t)ggml_nelements(v) * sizeof(ggml_fp16_t) <= ctx->device->properties.limits.maxStorageBufferRange && + ctx->device->pipeline_dequant_transpose[k->type] != nullptr && + ctx->device->pipeline_dequant_transpose[v->type] != nullptr && + // coopmat2 path does not benefit from the f16 scratch + !ctx->device->coopmat2 && + // Intel Xe1 regresses, see PR 25494 + (ctx->device->vendor_id != VK_VENDOR_ID_INTEL || + (ctx->device->coopmat_support && ctx->device->architecture != vk_device_architecture::INTEL_XE1)); + const ggml_type k_type_eff = use_dequant_kv ? GGML_TYPE_F16 : k->type; + const ggml_type v_type_eff = use_dequant_kv ? GGML_TYPE_F16 : v->type; + // For scalar/coopmat1 FA, we can use the "large" size to accommodate qga. // For coopmat2 FA, we always use the small size (which is still pretty large for gqa). - vk_fa_tuning_params tuning_params = get_fa_tuning_params(ctx->device, HSK, HSV, 512, KV, k->type, v->type, f32acc); + vk_fa_tuning_params tuning_params = get_fa_tuning_params(ctx->device, HSK, HSV, 512, KV, k_type_eff, v_type_eff, f32acc); const uint32_t max_gqa = std::min(tuning_params.block_rows, 32u); if (N <= 8 && qk_ratio > 1 && qk_ratio <= max_gqa && @@ -10838,7 +10863,7 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx workgroups_y /= gqa_ratio; } - tuning_params = get_fa_tuning_params(ctx->device, HSK, HSV, N, KV, k->type, v->type, f32acc); + tuning_params = get_fa_tuning_params(ctx->device, HSK, HSV, N, KV, k_type_eff, v_type_eff, f32acc); const uint32_t q_stride = (uint32_t)(nbq1 / ggml_type_size(q->type)); uint32_t k_stride = (uint32_t)(nbk1 / ggml_type_size(k->type)); @@ -10852,6 +10877,17 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx v_stride /= 4; } + uint32_t nbk2_eff = (uint32_t)nbk2, nbk3_eff = (uint32_t)nbk3; + uint32_t nbv2_eff = (uint32_t)nbv2, nbv3_eff = (uint32_t)nbv3; + if (use_dequant_kv) { + k_stride = HSK; + v_stride = HSV; + nbk2_eff = (uint32_t)((uint64_t)HSK * KV * sizeof(ggml_fp16_t)); + nbk3_eff = (uint32_t)((uint64_t)HSK * KV * nek2 * sizeof(ggml_fp16_t)); + nbv2_eff = (uint32_t)((uint64_t)HSV * KV * sizeof(ggml_fp16_t)); + nbv3_eff = (uint32_t)((uint64_t)HSV * KV * nev2 * sizeof(ggml_fp16_t)); + } + const uint32_t alignment = tuning_params.block_cols; bool aligned = (KV % alignment) == 0 && // the "aligned" shader variant will forcibly align strides, for performance @@ -10878,7 +10914,7 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx bool use_mask_opt = mask && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16 && (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK > 256 || HSV > 256); vk_fa_pipeline_state fa_pipeline_state = get_fa_pipeline_state(ctx->device, tuning_params, HSK, HSV, aligned, f32acc, - mask != nullptr, use_mask_opt, logit_softcap != 0, k->type, v->type); + mask != nullptr, use_mask_opt, logit_softcap != 0, k_type_eff, v_type_eff); vk_pipeline pipeline = nullptr; @@ -10982,6 +11018,34 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx vk_subbuffer sinks_buf = sinks ? ggml_vk_tensor_subbuffer(ctx, sinks) : q_buf; vk_subbuffer mask_opt_buf = use_mask_opt ? ggml_vk_subbuffer(ctx, ctx->prealloc_y, 0) : q_buf; + if (use_dequant_kv) { + const uint64_t fp = sizeof(ggml_fp16_t); + const uint64_t k_f16_sz = (uint64_t)ggml_nelements(k) * fp; + const uint64_t v_f16_sz = (uint64_t)ggml_nelements(v) * fp; + if (ctx->prealloc_size_x < k_f16_sz + v_f16_sz) { + ctx->prealloc_size_x = k_f16_sz + v_f16_sz; + ggml_vk_preallocate_buffers(ctx, subctx); + } + vk_pipeline tr_k = ctx->device->pipeline_dequant_transpose[k->type]; + vk_pipeline tr_v = ctx->device->pipeline_dequant_transpose[v->type]; + ggml_pipeline_request_descriptor_sets(ctx, tr_k, 1); + ggml_pipeline_request_descriptor_sets(ctx, tr_v, 1); + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + vk_subbuffer k_dst = vk_subbuffer{ ctx->prealloc_x, 0, k_f16_sz }; + vk_subbuffer v_dst = vk_subbuffer{ ctx->prealloc_x, k_f16_sz, v_f16_sz }; + const uint32_t k_nel = (uint32_t)ggml_nelements(k); + const uint32_t v_nel = (uint32_t)ggml_nelements(v); + { const std::vector pc = { (uint32_t)HSK, (uint32_t)nek2, (uint32_t)KV, 0, k_nel }; + ggml_vk_dispatch_pipeline(ctx, subctx, tr_k, { k_buf, k_dst }, pc, { k_nel, 1, 1 }); } + { const std::vector pc = { (uint32_t)HSV, (uint32_t)nev2, (uint32_t)KV, 0, v_nel }; + ggml_vk_dispatch_pipeline(ctx, subctx, tr_v, { v_buf, v_dst }, pc, { v_nel, 1, 1 }); } + ggml_vk_sync_buffers(ctx, subctx); + k_buf = k_dst; + v_buf = v_dst; + } + uint32_t mask_n_head_log2 = ((sinks != nullptr) << 24) | n_head_log2; if (use_mask_opt) @@ -11011,8 +11075,8 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx (uint32_t)nev2, (uint32_t)nev3, nem1, nem2, nem3, q_stride, (uint32_t)nbq2, (uint32_t)nbq3, - k_stride, (uint32_t)nbk2, (uint32_t)nbk3, - v_stride, (uint32_t)nbv2, (uint32_t)nbv3, + k_stride, nbk2_eff, nbk3_eff, + v_stride, nbv2_eff, nbv3_eff, scale, max_bias, logit_softcap, mask_n_head_log2, m0, m1, gqa_ratio, split_kv, split_k }; @@ -11054,6 +11118,10 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx {q_buf, k_buf, v_buf, mask_buf, sinks_buf, dst_buf, mask_opt_buf}, pc, { workgroups_x, workgroups_y, workgroups_z }); } + + if (use_dequant_kv) { + ctx->prealloc_x_need_sync = true; + } } static vk_conv_shapes ggml_vk_conv_select_shape(ggml_backend_vk_context * ctx, uint32_t K, uint32_t NPQ) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp index 10844ddf78..3b3fbbe899 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp @@ -18,7 +18,18 @@ void main() { return; } +#ifdef DEQUANT_TRANSPOSE + // read [HS, NH, KV, NS], write [HS, KV, NH, NS] + const uint HS = p.M, NH = p.K, KVn = p.stride_a; + const uint e0 = ib * 32; + const uint b_idx = (e0 % HS) + + ((e0 / (HS * NH)) % KVn) * HS + + ((e0 / HS) % NH) * (HS * KVn) + + (e0 / (HS * NH * KVn)) * (HS * KVn * NH) + + 16 * il; +#else const uint b_idx = 1024*i + 32*ir + 16*il; +#endif const float d = float(data_a[ib].d); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index fbc2ea3ca2..caa0c889a4 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -780,6 +780,10 @@ void process_shaders() { if (tname != "f16" && tname != "bf16") { string_to_spv("dequant_" + tname, "dequant_" + tname + ".comp", merge_maps(base_dict, {{data_a_key, "1"}, {"D_TYPE", "float16_t"}})); } + // Fused dequant+transpose variant for FA quant-KV (per-head-contiguous f16 scratch). + if (tname == "q8_0") { + string_to_spv("dequant_" + tname + "_transpose", "dequant_" + tname + ".comp", merge_maps(base_dict, {{data_a_key, "1"}, {"D_TYPE", "float16_t"}, {"DEQUANT_TRANSPOSE", "1"}})); + } shader = (tname == "f32" || tname == "f16" || tname == "bf16") ? "get_rows.comp" : "get_rows_quant.comp"; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 2a1851b59d..c9946c7ae4 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -7084,9 +7084,10 @@ struct test_flash_attn_ext : public test_case { const ggml_type type_K; const ggml_type type_V; std::array permute; + const bool kv_view; // create K/V as views of a larger buffer (like a KV cache) std::string vars() override { - return VARS_TO_STR14(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute); + return VARS_TO_STR15(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view); } double max_nmse_err() override { @@ -7102,9 +7103,10 @@ struct test_flash_attn_ext : public test_case { test_flash_attn_ext(int64_t hsk = 128, int64_t hsv = 128, int64_t nh = 32, std::array nr23 = {1, 1}, int64_t kv = 96, int64_t nb = 8, bool mask = true, bool sinks = false, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32, - ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array permute = {0, 1, 2, 3}) + ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array permute = {0, 1, 2, 3}, + bool kv_view = true) : hsk(hsk), hsv(hsv), nh(nh), nr23(nr23), kv(kv), nb(nb), mask(mask), sinks(sinks), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec), - type_K(type_K), type_V(type_V), permute(permute) {} + type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view) {} ggml_tensor * build_graph(ggml_context * ctx) override { const int64_t hsk_padded = GGML_PAD(hsk, ggml_blck_size(type_K)); @@ -7132,7 +7134,7 @@ struct test_flash_attn_ext : public test_case { ggml_tensor * q = create_permuted(GGML_TYPE_F32, hsk_padded, nb, nh*nr23[0], nr23[1], false); ggml_set_name(q, "q"); - ggml_tensor * k = create_permuted(type_K, hsk_padded, kv, nh, nr23[1], true); // the K tensor is usually a view of the K cache + ggml_tensor * k = create_permuted(type_K, hsk_padded, kv, nh, nr23[1], kv_view); // the K tensor is usually a view of the K cache ggml_set_name(k, "k"); ggml_tensor * v = nullptr; @@ -7146,7 +7148,7 @@ struct test_flash_attn_ext : public test_case { // - https://github.com/ggml-org/llama.cpp/pull/18986 v = ggml_view_4d(ctx, k, hsv_padded, kv, nh, nr23[1], k->nb[1], k->nb[2], k->nb[3], 0); } else { - v = create_permuted(type_V, hsv_padded, kv, nh, nr23[1], true); // the V tensor is usually a view of the V cache + v = create_permuted(type_V, hsv_padded, kv, nh, nr23[1], kv_view); // the V tensor is usually a view of the V cache } ggml_set_name(v, "v"); @@ -9941,6 +9943,12 @@ static std::vector> make_test_cases_eval() { GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); } + // dense-allocated (non-view) quant K/V at batch >= 64, in cache and native layouts + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, false)); + test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, { 10, 5, 4, 3})); test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, {30000, 1, 1, 1})); test_cases.emplace_back(new test_cross_entropy_loss_back(GGML_TYPE_F32, { 10, 5, 4, 3})); From ee0ea03adf9dd16959ad3095d7f0d04bd51318f4 Mon Sep 17 00:00:00 2001 From: s0mecode <213953308+s0mecode@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:44:42 +0400 Subject: [PATCH 09/36] server : make models endpoints private when authentication is enabled (#26347) * server : make models endpoints private when authentication is enabled * tests : fix models endpoint auth --- tools/server/server-http.cpp | 2 -- tools/server/server.cpp | 4 ++-- tools/server/tests/unit/test_router.py | 16 +++++++++------- tools/server/tests/unit/test_security.py | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index b11dc09d0a..2ec137aa07 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -198,8 +198,6 @@ bool server_http_context::init(const common_params & params) { std::unordered_set endpoints { "/health", "/v1/health", - "/models", - "/v1/models", }; endpoints.insert(frontend_paths.begin(), frontend_paths.end()); return endpoints; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 77722b9a61..01cc6633a3 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -235,8 +235,8 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_http.get ("/metrics", ex_wrapper(routes.get_metrics)); ctx_http.get ("/props", ex_wrapper(routes.get_props)); ctx_http.post("/props", ex_wrapper(routes.post_props)); - ctx_http.get ("/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check) - ctx_http.get ("/v1/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check) + ctx_http.get ("/models", ex_wrapper(routes.get_models)); + ctx_http.get ("/v1/models", ex_wrapper(routes.get_models)); ctx_http.post("/completion", ex_wrapper(routes.post_completions)); // legacy ctx_http.post("/completions", ex_wrapper(routes.post_completions)); ctx_http.post("/v1/completions", ex_wrapper(routes.post_completions_oai)); diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.py index 0e1467de37..96eb87978f 100644 --- a/tools/server/tests/unit/test_router.py +++ b/tools/server/tests/unit/test_router.py @@ -63,14 +63,16 @@ def test_router_chat_completion_stream(model: str, success: bool): assert content == "" -def _get_model_ids(is_reload: bool) -> set[str]: - res = server.make_request("GET", "/models" + ("?reload=1" if is_reload else "")) +def _get_model_ids(is_reload: bool, headers: dict | None = None) -> set[str]: + res = server.make_request( + "GET", "/models" + ("?reload=1" if is_reload else ""), headers=headers + ) assert res.status_code == 200 return {item["id"] for item in res.body.get("data", [])} -def _get_model_status(model_id: str) -> str: - res = server.make_request("GET", "/models") +def _get_model_status(model_id: str, headers: dict | None = None) -> str: + res = server.make_request("GET", "/models", headers=headers) assert res.status_code == 200 for item in res.body.get("data", []): if item.get("id") == model_id or item.get("model") == model_id: @@ -78,11 +80,11 @@ def _get_model_status(model_id: str) -> str: raise AssertionError(f"Model {model_id} not found in /models response") -def _wait_for_model_status(model_id: str, desired: set[str], timeout: int = 60) -> str: +def _wait_for_model_status(model_id: str, desired: set[str], timeout: int = 60, headers: dict | None = None) -> str: deadline = time.time() + timeout last_status = None while time.time() < deadline: - last_status = _get_model_status(model_id) + last_status = _get_model_status(model_id, headers=headers) if last_status in desired: return last_status time.sleep(0.01) @@ -100,7 +102,7 @@ def _load_model_and_wait( assert load_res.status_code == 200 assert isinstance(load_res.body, dict) assert load_res.body.get("success") is True - _wait_for_model_status(model_id, {"loaded"}, timeout=timeout) + _wait_for_model_status(model_id, {"loaded"}, timeout=timeout, headers=headers) def test_router_unload_model(): diff --git a/tools/server/tests/unit/test_security.py b/tools/server/tests/unit/test_security.py index ac0544575b..36fc439f9b 100644 --- a/tools/server/tests/unit/test_security.py +++ b/tools/server/tests/unit/test_security.py @@ -15,7 +15,7 @@ def create_server(): server.api_key = TEST_API_KEY -@pytest.mark.parametrize("endpoint", ["/health", "/models"]) +@pytest.mark.parametrize("endpoint", ["/health"]) def test_access_public_endpoint(endpoint: str): global server server.start() From 947fd9bb2bdeaa72e9dd74b6aa3b5d68f03f3d6a Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Wed, 19 Aug 2026 20:48:09 +0200 Subject: [PATCH 10/36] server: refactor sleep handling, allow access /metrics during sleep (#27376) * add cached responses * refactor on_sleeping_state * allow accessing metrics during sleep * metrics task should not reset timer * updated docs * fix * fix get_res_model_info * add test * fix a race condition * split metrics and slots tasks / results * should_reset_buckets --- tools/server/README-dev.md | 30 ++ tools/server/README.md | 1 + tools/server/server-context.cpp | 388 ++++++++++++++++---------- tools/server/server-context.h | 13 +- tools/server/server-queue.cpp | 33 ++- tools/server/server-queue.h | 19 +- tools/server/server-task.cpp | 7 +- tools/server/server-task.h | 24 +- tools/server/tests/unit/test_sleep.py | 88 ++++++ 9 files changed, 422 insertions(+), 181 deletions(-) diff --git a/tools/server/README-dev.md b/tools/server/README-dev.md index 94fbbde80d..0f42b2ee16 100644 --- a/tools/server/README-dev.md +++ b/tools/server/README-dev.md @@ -291,6 +291,36 @@ The flow for downloading a new model: - If a stop request comes in, the router asks the child process to stop (same mechanism as running a model in child process) - Otherwise, upon completion, we call `load_models()` to refresh the list of models +### Sleep mode + +Sleep mode was initially introduced in PR [#18228](https://github.com/ggml-org/llama.cpp/pull/18228). The main idea is to have: +- `server_queue` keeping track of the idle timeout +- When the timeout is detected, `server_queue` signals to `server_context_impl` that it should go into sleep +- `server_context_impl` frees all `llama_context` and `mtmd_context` + +Compared to simply exiting the whole process, this approach allows accessing some read-only endpoints during sleep, while also handling wakeup-on-request. Any inference request will wake the server up. + +Call stack on entering sleeping: +- `server_queue::start_loop` (main thread) sees no task for `idle_sleep_ms` --> `sleeping = true` +- `cb0(true)` --> `server_routes::update_cached_responses` + - snapshots `/props`, `/models` and metrics; the model is still alive here +- `cb1(true)` --> `server_context_impl::handle_sleeping_state` + - `callback_state(SERVER_STATE_SLEEPING)` --> reported to router in child mode + - `destroy()` --> frees `llama_context` and `mtmd_context` +- `condition_tasks.wait` until `req_stop_sleeping` + +Call stack on waking up: +- `server_res_generator` constructor (HTTP thread) --> `server_queue::wait_until_no_sleep` + - sets `req_stop_sleeping = true`, then waits until `sleeping == false` +- `server_queue::start_loop` (main thread) wakes up +- `cb1(false)` --> `server_context_impl::handle_sleeping_state` + - `load_model()`, which then emits `callback_state(SERVER_STATE_READY)` +- `cb0(false)` --> `server_routes::update_cached_responses` + - nothing to do, the cache is only read during sleep +- `sleeping = false` --> `notify_all` unblocks the HTTP thread, the request is handled as usual + +Endpoints created with `create_response(true)` (`/health`, `/props`, `/models`, `/metrics`) skip `wait_until_no_sleep`, so they answer from the cached responses instead of waking the server. + ### Notable Related PRs - Initial server implementation: https://github.com/ggml-org/llama.cpp/pull/1443 diff --git a/tools/server/README.md b/tools/server/README.md index 78274967db..b63a0e6dac 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -2071,6 +2071,7 @@ Note that the following endpoints are exempt from being considered as incoming t - `GET /health` - `GET /props` - `GET /models` +- `GET /metrics` ## More examples diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 842e4203cd..21ff783941 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -818,6 +818,14 @@ public: } } + server_metrics get_metrics() const { + return metrics; + } + + void reset_metrics_bucket() { + metrics.reset_bucket(); + } + private: // note: accessing these fields outside of this class is not thread-safe // use server_context methods instead @@ -898,6 +906,10 @@ private: void handle_sleeping_state(bool new_state) { GGML_ASSERT(sleeping != new_state); if (new_state) { + if (callback_state) { + callback_state(SERVER_STATE_SLEEPING, {}); + // note: for sleeping == false, event is emitted by load_model() + } SRV_INF("%s", "server is entering sleeping state\n"); destroy(); } else { @@ -2290,8 +2302,8 @@ private: // returns false to decline the task, it is offered again after the decode is done bool process_single_task(server_task && task, bool is_yielding) { - // while yielding, an encode / decode is running and only accessing metrics is safe - if (is_yielding && task.type != SERVER_TASK_TYPE_METRICS) { + // while yielding, an encode / decode is running and only reading the server state is safe + if (is_yielding && task.type != SERVER_TASK_TYPE_METRICS && task.type != SERVER_TASK_TYPE_SLOT_GET) { SRV_DBG("decoding, decline task, id_task = %d\n", task.id); return false; } @@ -2417,28 +2429,17 @@ private: } break; case SERVER_TASK_TYPE_METRICS: { - json slots_data = json::array(); - - int n_idle_slots = 0; int n_processing_slots = 0; for (server_slot & slot : slots) { - json slot_data = slot.to_json(slots_debug == 0); - if (slot.is_processing()) { n_processing_slots++; - } else { - n_idle_slots++; } - - slots_data.push_back(slot_data); } - SRV_DBG("n_idle_slots = %d, n_processing_slots = %d\n", n_idle_slots, n_processing_slots); + SRV_DBG("n_processing_slots = %d\n", n_processing_slots); auto res = std::make_unique(); res->id = task.id; - res->slots_data = std::move(slots_data); - res->n_idle_slots = n_idle_slots; res->n_processing_slots = n_processing_slots; res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size(); res->metrics = metrics; @@ -2446,6 +2447,28 @@ private: if (task.metrics_reset_bucket) { metrics.reset_bucket(); } + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SLOT_GET: + { + json slots_data = json::array(); + + int n_idle_slots = 0; + + for (server_slot & slot : slots) { + if (!slot.is_processing()) { + n_idle_slots++; + } + + slots_data.push_back(slot.to_json(slots_debug == 0)); + } + SRV_DBG("n_idle_slots = %d\n", n_idle_slots); + + auto res = std::make_unique(); + res->id = task.id; + res->slots_data = std::move(slots_data); + res->n_idle_slots = n_idle_slots; + queue_results.send(std::move(res)); } break; case SERVER_TASK_TYPE_SLOT_SAVE: @@ -4142,12 +4165,6 @@ struct server_res_generator : server_res_spipe { void server_context::set_state_callback(server_state_callback_t callback) { impl->callback_state = std::move(callback); - impl->queue_tasks.on_sleeping_state([this](bool sleeping) { - if (sleeping) { - impl->callback_state(SERVER_STATE_SLEEPING, {}); - } - // for sleeping == false, event is emitted by load_model() - }); } // @@ -4431,6 +4448,119 @@ server_routes::server_routes(const common_params & params, server_context & ctx_ queue_tasks(ctx_server.impl->queue_tasks), queue_results(ctx_server.impl->queue_results) { init_routes(); + + // note: this must be registered before load_model() + // so that on sleep phase, the callback is called before ctx is destroyed + queue_tasks.on_sleeping_state([this](bool is_sleeping) { + update_cached_responses(is_sleeping); + }); +} + +static json get_res_model_info(const server_context_meta & meta) { + // note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep + + return { + {"id", meta.model_name}, + {"aliases", meta.model_aliases}, + {"tags", meta.model_tags}, + {"object", "model"}, + {"created", std::time(0)}, + {"owned_by", "llamacpp"}, + {"meta", { + {"vocab_type", meta.model_vocab_type}, + {"n_vocab", meta.model_vocab_n_tokens}, + {"n_ctx", meta.slot_n_ctx}, + {"n_ctx_train", meta.model_n_ctx_train}, + {"n_embd", meta.model_n_embd_inp}, + {"n_params", meta.model_n_params}, + {"size", meta.model_size}, + {"ftype", meta.model_ftype}, + }}, + }; +} + +static json get_res_models(const server_context_meta & meta) { + // note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep + + return { + {"models", { + { + {"name", meta.model_name}, + {"model", meta.model_name}, + {"modified_at", ""}, + {"size", ""}, + {"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash + {"type", "model"}, + {"description", ""}, + {"tags", {""}}, + {"capabilities", meta.has_mtmd ? json({"completion","multimodal"}) : json({"completion"})}, + {"parameters", ""}, + {"details", { + {"parent_model", ""}, + {"format", "gguf"}, + {"family", ""}, + {"families", {""}}, + {"parameter_size", ""}, + {"quantization_level", ""} + }} + } + }}, + {"object", "list"}, + {"data", { + get_res_model_info(meta), + }} + }; +} + +static json get_res_props(const server_context_meta & meta, const common_params & params, bool is_sleeping) { + // note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep + + task_params tparams; + tparams.sampling = params.sampling; + json default_generation_settings_for_props = json { + { "params", tparams.to_json(true) }, + { "n_ctx", meta.slot_n_ctx }, + }; + + std::string tmpl_default = common_chat_templates_source(meta.chat_params.tmpls.get(), ""); + std::string tmpl_tools = common_chat_templates_source(meta.chat_params.tmpls.get(), "tool_use"); + + json props = { + { "default_generation_settings", default_generation_settings_for_props }, + { "total_slots", params.n_parallel }, + { "model_alias", meta.model_name }, + { "model_ftype", meta.model_ftype }, + { "model_path", meta.model_path }, + { "modalities", json { + {"vision", meta.has_inp_image}, + {"video", meta.has_inp_video}, + {"audio", meta.has_inp_audio}, + } }, + { "media_marker", get_media_marker() }, + { "endpoint_slots", params.endpoint_slots }, + { "endpoint_props", params.endpoint_props }, + { "endpoint_metrics", params.endpoint_metrics }, + { "ui", params.ui }, + { "ui_settings", meta.json_ui_settings }, + { "chat_template", tmpl_default }, + { "chat_template_caps", meta.chat_template_caps }, + { "bos_token", meta.bos_token_str }, + { "eos_token", meta.eos_token_str }, + { "build_info", meta.build_info }, + { "is_sleeping", is_sleeping }, + { "cors_proxy_enabled", params.ui_mcp_proxy }, + }; + if (params.use_jinja) { + if (!tmpl_tools.empty()) { + props["chat_template_tool_use"] = tmpl_tools; + } + } + + return props; +} + +json server_routes::get_model_info() const { + return get_res_model_info(*meta); } void server_routes::init_routes() { @@ -4451,41 +4581,64 @@ void server_routes::init_routes() { }; this->get_metrics = [this](const server_http_req & req) { - auto res = create_response(); + auto res = create_response(true); if (!params.endpoint_metrics) { res->error(format_error_response("This server does not support metrics endpoint. Start it with `--metrics`", ERROR_TYPE_NOT_SUPPORTED)); return res; } - // request slots data using task queue - { - server_task task(SERVER_TASK_TYPE_METRICS); - task.id = res->rd.get_new_id(); + // render response using cached_metrics + auto use_cached_metrics = [&]() { + std::unique_lock lock(mutex_cache); + res->headers["Process-Start-Time-Unix"] = std::to_string(cached_metrics.t_start); + server_task_result_metrics tmp; + tmp.metrics = cached_metrics; + res->content_type = "text/plain; version=0.0.4"; + res->status = 200; + res->data = tmp.to_metrics(); // the gauges are averaged over the window between two scrapes - task.metrics_reset_bucket = true; - res->rd.post_task(std::move(task), true); // high-priority task + cached_metrics.reset_bucket(); + should_reset_buckets = true; + }; + + if (queue_tasks.is_sleeping()) { + use_cached_metrics(); + + } else { + // request slots data using task queue + { + server_task task(SERVER_TASK_TYPE_METRICS); + task.id = res->rd.get_new_id(); + // the gauges are averaged over the window between two scrapes + task.metrics_reset_bucket = true; + res->rd.post_task(std::move(task), true); // high-priority task + } + + // a task posted right before sleeping is never processed, do not wait for it + auto result = res->rd.next([&]{ + return req.should_stop() || queue_tasks.is_sleeping(); + }); + if (!result) { + if (!req.should_stop()) { + use_cached_metrics(); + } + return res; + } + + if (result->is_error()) { + res->error(result->to_json()); + return res; + } + + auto res_task = dynamic_cast(result.get()); + GGML_ASSERT(res_task != nullptr); + + res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->metrics.t_start); + res->content_type = "text/plain; version=0.0.4"; + res->status = 200; + res->data = res_task->to_metrics(); } - // get the result - auto result = res->rd.next(req.should_stop); - if (!result) { - // connection was closed - GGML_ASSERT(req.should_stop()); - return res; - } - - if (result->is_error()) { - res->error(result->to_json()); - return res; - } - - auto res_task = dynamic_cast(result.get()); - GGML_ASSERT(res_task != nullptr); - - res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->metrics.t_start); - res->content_type = "text/plain; version=0.0.4"; - res->status = 200; - res->data = res_task->to_metrics(); return res; }; @@ -4498,7 +4651,7 @@ void server_routes::init_routes() { // request slots data using task queue { - server_task task(SERVER_TASK_TYPE_METRICS); + server_task task(SERVER_TASK_TYPE_SLOT_GET); task.id = res->rd.get_new_id(); res->rd.post_task(std::move(task), true); // high-priority task } @@ -4516,7 +4669,7 @@ void server_routes::init_routes() { return res; } - auto * res_task = dynamic_cast(result.get()); + auto * res_task = dynamic_cast(result.get()); GGML_ASSERT(res_task != nullptr); // optionally return "fail_on_no_slot" error @@ -4566,53 +4719,13 @@ void server_routes::init_routes() { this->get_props = [this](const server_http_req &) { auto res = create_response(true); - - // this endpoint can be accessed during sleeping - // the next LOC is to avoid someone accidentally use ctx_server - bool ctx_server; // do NOT delete this line - GGML_UNUSED(ctx_server); - - task_params tparams; - tparams.sampling = params.sampling; - json default_generation_settings_for_props = json { - { "params", tparams.to_json(true) }, - { "n_ctx", meta->slot_n_ctx }, - }; - - std::string tmpl_default = common_chat_templates_source(meta->chat_params.tmpls.get(), ""); - std::string tmpl_tools = common_chat_templates_source(meta->chat_params.tmpls.get(), "tool_use"); - - json props = { - { "default_generation_settings", default_generation_settings_for_props }, - { "total_slots", params.n_parallel }, - { "model_alias", meta->model_name }, - { "model_ftype", meta->model_ftype }, - { "model_path", meta->model_path }, - { "modalities", json { - {"vision", meta->has_inp_image}, - {"video", meta->has_inp_video}, - {"audio", meta->has_inp_audio}, - } }, - { "media_marker", get_media_marker() }, - { "endpoint_slots", params.endpoint_slots }, - { "endpoint_props", params.endpoint_props }, - { "endpoint_metrics", params.endpoint_metrics }, - { "ui", params.ui }, - { "ui_settings", meta->json_ui_settings }, - { "chat_template", tmpl_default }, - { "chat_template_caps", meta->chat_template_caps }, - { "bos_token", meta->bos_token_str }, - { "eos_token", meta->eos_token_str }, - { "build_info", meta->build_info }, - { "is_sleeping", queue_tasks.is_sleeping() }, - { "cors_proxy_enabled", params.ui_mcp_proxy }, - }; - if (params.use_jinja) { - if (!tmpl_tools.empty()) { - props["chat_template_tool_use"] = tmpl_tools; - } + // note: do NOT use ctx_server here, this endpoint must be accessible during sleep + if (queue_tasks.is_sleeping()) { + std::unique_lock lock(mutex_cache); + res->ok(cached_props); + } else { + res->ok(get_res_props(*meta, params, false)); } - res->ok(props); return res; }; @@ -4874,42 +4987,13 @@ void server_routes::init_routes() { this->get_models = [this](const server_http_req &) { auto res = create_response(true); - - // this endpoint can be accessed during sleeping - // the next LOC is to avoid someone accidentally use ctx_server - bool ctx_server; // do NOT delete this line - GGML_UNUSED(ctx_server); - - json models = { - {"models", { - { - {"name", meta->model_name}, - {"model", meta->model_name}, - {"modified_at", ""}, - {"size", ""}, - {"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash - {"type", "model"}, - {"description", ""}, - {"tags", {""}}, - {"capabilities", meta->has_mtmd ? json({"completion","multimodal"}) : json({"completion"})}, - {"parameters", ""}, - {"details", { - {"parent_model", ""}, - {"format", "gguf"}, - {"family", ""}, - {"families", {""}}, - {"parameter_size", ""}, - {"quantization_level", ""} - }} - } - }}, - {"object", "list"}, - {"data", { - get_model_info(), - }} - }; - - res->ok(models); + // note: do NOT use ctx_server here, this endpoint must be accessible during sleep + if (queue_tasks.is_sleeping()) { + std::unique_lock lock(mutex_cache); + res->ok(cached_models); + } else { + res->ok(get_res_models(*meta)); + } return res; }; @@ -5119,27 +5203,6 @@ void server_routes::init_routes() { }; } -json server_routes::get_model_info() const { - return json { - {"id", meta->model_name}, - {"aliases", meta->model_aliases}, - {"tags", meta->model_tags}, - {"object", "model"}, - {"created", std::time(0)}, - {"owned_by", "llamacpp"}, - {"meta", { - {"vocab_type", meta->model_vocab_type}, - {"n_vocab", meta->model_vocab_n_tokens}, - {"n_ctx", meta->slot_n_ctx}, - {"n_ctx_train", meta->model_n_ctx_train}, - {"n_embd", meta->model_n_embd_inp}, - {"n_params", meta->model_n_params}, - {"size", meta->model_size}, - {"ftype", meta->model_ftype}, - }}, - }; -} - std::unique_ptr server_routes::handle_slots_save(const server_http_req & req, int id_slot) { auto res = create_response(); const json request_data = json::parse(req.body); @@ -5388,3 +5451,24 @@ std::unique_ptr server_routes::handle_count_tokens(const l res->ok(response); return res; } + +void server_routes::update_cached_responses(bool is_sleeping) { + // caller is task_queue, so ctx_server can be accessed without holding locks + std::unique_lock lock(mutex_cache); + + if (is_sleeping) { + cached_models = get_res_models(*meta); + cached_props = get_res_props(*meta, params, true); + cached_metrics = ctx_server.get_metrics(); + + should_reset_buckets = false; + + SRV_DBG("%s\n", "cached responses updated"); + + } else if (should_reset_buckets) { + // a scrape during sleep already reported these buckets + ctx_server.reset_metrics_bucket(); + + should_reset_buckets = false; + } +} diff --git a/tools/server/server-context.h b/tools/server/server-context.h index f9ab1132b1..764df0e085 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -8,6 +8,7 @@ #include #include +#include #include struct server_context_impl; // private implementation @@ -174,9 +175,19 @@ private: std::unique_ptr meta; const common_params & params; - const server_context_impl & ctx_server; + server_context_impl & ctx_server; server_queue & queue_tasks; server_response & queue_results; std::unique_ptr create_response(bool bypass_sleep = false); + + // cached responses, to be used during sleep + std::mutex mutex_cache; + json cached_models = nullptr; + json cached_props = nullptr; + server_metrics cached_metrics; + // set when a scrape during sleep already reported the throughput buckets + bool should_reset_buckets = false; + // call right before sleep to update the cached responses + void update_cached_responses(bool is_sleeping); }; diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp index 2bcc9bd8f2..78169e9a5d 100644 --- a/tools/server/server-queue.cpp +++ b/tools/server/server-queue.cpp @@ -3,6 +3,7 @@ #include "log.h" +#include #include #include @@ -20,6 +21,10 @@ // server_queue // +static bool task_resets_idle_timer(server_task_type type) { + return type != SERVER_TASK_TYPE_METRICS; +} + int server_queue::post(server_task && task, bool front) { std::unique_lock lock(mutex_tasks); GGML_ASSERT(task.id != -1); @@ -27,20 +32,24 @@ int server_queue::post(server_task && task, bool front) { if (task.type == SERVER_TASK_TYPE_CANCEL) { cleanup_pending_task(task.id_target); } - const int task_id = task.id; + const int task_id = task.id; + const bool reset_timer = task_resets_idle_timer(task.type); QUE_DBG("new task, id = %d, front = %d\n", task_id, front); if (front) { queue_tasks.push_front(std::move(task)); } else { queue_tasks.push_back(std::move(task)); } - time_last_task = ggml_time_ms(); + if (reset_timer) { + time_last_task = ggml_time_ms(); + } condition_tasks.notify_one(); return task_id; } int server_queue::post(std::vector && tasks, bool front) { std::unique_lock lock(mutex_tasks); + bool reset_timer = false; for (auto & task : tasks) { if (task.id == -1) { task.id = id++; @@ -49,6 +58,7 @@ int server_queue::post(std::vector && tasks, bool front) { if (task.type == SERVER_TASK_TYPE_CANCEL) { cleanup_pending_task(task.id_target); } + reset_timer |= task_resets_idle_timer(task.type); QUE_DBG("new task, id = %d/%d, front = %d\n", task.id, (int) tasks.size(), front); if (front) { queue_tasks.push_front(std::move(task)); @@ -56,7 +66,9 @@ int server_queue::post(std::vector && tasks, bool front) { queue_tasks.push_back(std::move(task)); } } - time_last_task = ggml_time_ms(); + if (reset_timer) { + time_last_task = ggml_time_ms(); + } condition_tasks.notify_one(); return 0; } @@ -294,11 +306,14 @@ void server_queue::start_loop(int64_t idle_sleep_ms) { QUE_DBG("%s", "update slots\n"); // this will run the main inference process for all slots + const int64_t t_update_slots = ggml_time_ms(); callback_update_slots(); { // update_slots() may take a while to finish, we need to make sure it's not counted as idle + // shift instead of reset, so that non-task_resets_idle_timer tasks do not delay the sleep std::unique_lock lock(mutex_tasks); - time_last_task = ggml_time_ms(); + const int64_t now = ggml_time_ms(); + time_last_task = std::min(now, time_last_task + (now - t_update_slots)); } QUE_DBG("%s", "waiting for new tasks\n"); @@ -312,7 +327,10 @@ void server_queue::start_loop(int64_t idle_sleep_ms) { if (should_sleep()) { QUE_INF("%s", "entering sleeping state\n"); sleeping = true; - callback_sleeping_state(true); + // Call order cb0 -> cb1 -> cb{N} + for (auto & cb : callback_sleeping_state) { + cb(true); + } req_stop_sleeping = false; // wait until we are requested to exit sleeping state condition_tasks.wait(lock, [&]{ @@ -323,7 +341,10 @@ void server_queue::start_loop(int64_t idle_sleep_ms) { } QUE_INF("%s", "exiting sleeping state\n"); req_stop_sleeping = false; - callback_sleeping_state(false); + // Call order cb{N} -> cb1 -> cb0 + for (size_t i = callback_sleeping_state.size(); i > 0; i--) { + callback_sleeping_state[i - 1](false); + } sleeping = false; time_last_task = ggml_time_ms(); condition_tasks.notify_all(); // notify wait_until_no_sleep() diff --git a/tools/server/server-queue.h b/tools/server/server-queue.h index 52d30095c1..e17733a743 100644 --- a/tools/server/server-queue.h +++ b/tools/server/server-queue.h @@ -44,7 +44,7 @@ private: // callback functions std::function callback_new_task; std::function callback_update_slots; - std::function callback_sleeping_state; + std::vector> callback_sleeping_state; public: ~server_queue() { worker_stop(); } @@ -86,6 +86,7 @@ public: * * Sleeping procedure (disabled if idle_sleep_ms < 0): * - If there is no task after idle_sleep_ms, enter sleeping state + * note: metrics tasks are processed as usual, but do not reset the idle timer * - Call callback_sleeping_state(true) * - Wait until req_stop_sleeping is set to true * - Call callback_sleeping_state(false) @@ -127,18 +128,12 @@ public: } // Register callback for sleeping state change; multiple callbacks are allowed - // note: when entering sleeping state, the callback is called AFTER sleeping is set to true - // when leaving sleeping state, the callback is called BEFORE sleeping is set to false + // for example: register order cb0, cb1, cb2 + // entering sleep: queue.sleeping = true --> cb0(true) --> cb1(true) --> cb2(true) + // leaving sleep: cb2(false) --> cb1(false) --> cb0(false) --> queue.sleeping = false + // note: caller will hold mutex_tasks while calling the callbacks void on_sleeping_state(std::function callback) { - if (callback_sleeping_state) { - auto prev_callback = std::move(callback_sleeping_state); - callback_sleeping_state = [prev_callback, callback](bool sleeping) { - prev_callback(sleeping); - callback(sleeping); - }; - } else { - callback_sleeping_state = std::move(callback); - } + callback_sleeping_state.push_back(std::move(callback)); } private: diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 64afbc5edf..258cdcf8fb 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1512,10 +1512,15 @@ json server_task_result_error::to_json() { // // server_task_result_metrics // -json server_task_result_metrics::to_json() { +json server_task_result_slots::to_json() { return slots_data; } +json server_task_result_metrics::to_json() { + // not used, /metrics renders prometheus text via to_metrics() + return json{}; +} + // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names std::string server_task_result_metrics::to_metrics() { const std::vector counters = { diff --git a/tools/server/server-task.h b/tools/server/server-task.h index b6da4d4bd6..25ff015122 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -22,6 +22,7 @@ enum server_task_type { SERVER_TASK_TYPE_CONTROL, SERVER_TASK_TYPE_NEXT_RESPONSE, SERVER_TASK_TYPE_METRICS, + SERVER_TASK_TYPE_SLOT_GET, SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE, @@ -489,22 +490,16 @@ struct server_task_result_error : server_task_result { virtual json to_json() override; }; +// used by /metrics API struct server_task_result_metrics : server_task_result { // these are immediate stats, not accumulated (server_metrics is cumulative) - int n_idle_slots; - int n_processing_slots; - int n_tasks_deferred; + int n_processing_slots = 0; + int n_tasks_deferred = 0; server_metrics metrics; - // while we can also use std::vector this requires copying the slot object which can be quite messy - // therefore, we use json to temporarily store the slot.to_json() result - json slots_data = json::array(); - - // used by /slots API virtual json to_json() override; - // used by /metrics API struct metric_item { std::string name; std::string description; @@ -513,6 +508,17 @@ struct server_task_result_metrics : server_task_result { std::string to_metrics(); }; +// used by /slots API +struct server_task_result_slots : server_task_result { + int n_idle_slots = 0; + + // while we can also use std::vector this requires copying the slot object which can be quite messy + // therefore, we use json to temporarily store the slot.to_json() result + json slots_data = json::array(); + + virtual json to_json() override; +}; + struct server_task_result_slot_save_load : server_task_result { std::string filename; bool is_save; // true = save, false = load diff --git a/tools/server/tests/unit/test_sleep.py b/tools/server/tests/unit/test_sleep.py index 3374165e83..515f7077d3 100644 --- a/tools/server/tests/unit/test_sleep.py +++ b/tools/server/tests/unit/test_sleep.py @@ -11,6 +11,35 @@ def create_server(): server = ServerPreset.tinyllama2() +def is_sleeping(server: ServerProcess) -> bool: + res = server.make_request("GET", "/props") + assert res.status_code == 200 + return res.body["is_sleeping"] + + +def wait_for_sleep(server: ServerProcess, timeout: float = 10.0): + start = time.time() + while time.time() - start < timeout: + if is_sleeping(server): + return + time.sleep(0.1) + raise TimeoutError("server did not go to sleep") + + +def fetch_metrics(server: ServerProcess) -> str: + res = server.make_request("GET", "/metrics") + assert res.status_code == 200 + assert isinstance(res.body, str) + return res.body + + +def get_metric(text: str, name: str) -> float: + prefix = f"llamacpp:{name} " + values = [ln for ln in text.splitlines() if ln.startswith(prefix)] + assert len(values) == 1, f"{name} not found in metrics" + return float(values[0][len(prefix):]) + + def test_server_sleep(): global server server.sleep_idle_seconds = 1 @@ -25,6 +54,10 @@ def test_server_sleep(): res = server.make_request("GET", "/props") assert res.status_code == 200 assert res.body["is_sleeping"] == True + res = server.make_request("GET", "/models") + assert res.status_code == 200 + assert len(res.body["data"]) == 1 + assert res.body["data"][0]["id"] == server.model_alias # make a generation request to wake up the server res = server.make_request("POST", "/completion", data={ @@ -37,3 +70,58 @@ def test_server_sleep(): res = server.make_request("GET", "/props") assert res.status_code == 200 assert res.body["is_sleeping"] == False + + +def test_server_sleep_read_only_endpoints(): + global server + server.sleep_idle_seconds = 1 + server.server_metrics = True + server.start() + + res = server.make_request("POST", "/completion", data={ + "n_predict": 4, + "prompt": "Hello", + }) + assert res.status_code == 200 + + # the first scrape resets the throughput buckets, so that the second one reports + # the same zero rates as the snapshot taken on entering sleep + fetch_metrics(server) + metrics_awake = fetch_metrics(server) + assert get_metric(metrics_awake, "tokens_predicted_total") > 0 + + wait_for_sleep(server) + + # during sleep, metrics are served from the snapshot taken right before sleeping + assert fetch_metrics(server) == metrics_awake + + # scraping /metrics must not wake the server up + assert is_sleeping(server) + + +def test_server_sleep_metrics_buckets(): + global server + server.sleep_idle_seconds = 1 + server.server_metrics = True + server.start() + + res = server.make_request("POST", "/completion", data={ + "n_predict": 8, + "prompt": "Hello", + }) + assert res.status_code == 200 + + wait_for_sleep(server) + + # the first scrape reports the throughput of the last generation + assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") > 0 + + # nothing runs while sleeping, so the next scrapes report an empty window + assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0 + assert is_sleeping(server) + + # waking up must not report the buckets again + res = server.make_request("POST", "/tokenize", data={"content": "Hello"}) + assert res.status_code == 200 + assert is_sleeping(server) == False + assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0 From cd644c39545aac3dca63261f99a9bfc35956cb25 Mon Sep 17 00:00:00 2001 From: Pascal Date: Wed, 19 Aug 2026 22:03:13 +0200 Subject: [PATCH 11/36] ggml-cpu: gate __fp16 on __ARM_FP16_FORMAT_IEEE (#26860) * ggml-cpu: gate __fp16 on __ARM_FP16_FORMAT_IEEE __ARM_NEON only signals NEON availability. The __fp16 type also needs the IEEE half format, implied on AArch64 but selected with -mfp16-format=ieee on 32 bit Arm, where the compiler otherwise rejects the type. The guard keeps every toolchain that provides the type on the same code and sends that one configuration to the generic lookup path. * ggml-cpu: gate the NEON+FMA block on __ARM_FP16_FORMAT_IEEE Both halves of the F16 section dereference __fp16, so armv7 with neon-vfpv4 hits the same unknown type error. Without the IEEE format the configuration now falls back to the scalar path. Address review from @JonathanC-ARM --- ggml/src/ggml-cpu/simd-mappings.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cpu/simd-mappings.h b/ggml/src/ggml-cpu/simd-mappings.h index fca5119e1a..10ce4bfc59 100644 --- a/ggml/src/ggml-cpu/simd-mappings.h +++ b/ggml/src/ggml-cpu/simd-mappings.h @@ -29,13 +29,15 @@ extern "C" { // FP16 to FP32 conversion // 16-bit float -// on Arm, we use __fp16 +// on Arm, we use __fp16, which requires the IEEE fp16 format: implied on +// AArch64, selected by -mfp16-format=ieee on 32 bit Arm, where the compiler +// may otherwise reject the type // on x86, we use uint16_t // // for old CUDA compilers (<= 11), we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/10616 // for MUSA compilers , we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/11843 // -#if defined(__ARM_NEON) && !(defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11) && !defined(__MUSACC__) +#if defined(__ARM_NEON) && defined(__ARM_FP16_FORMAT_IEEE) && !(defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11) && !defined(__MUSACC__) #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) neon_compute_fp16_to_fp32(x) #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) neon_compute_fp32_to_fp16(x) @@ -326,7 +328,7 @@ inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) { #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE #endif -#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA) +#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA) && defined(__ARM_FP16_FORMAT_IEEE) #define GGML_SIMD From b062ba735e5e15817f6a1dbdb84717ee3097f619 Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Wed, 19 Aug 2026 13:35:17 -0700 Subject: [PATCH 12/36] opencl: port fused ssm_scan kernel (Mamba-2, d_state in {128, 256}) to GPU (#26439) * opencl: port fused ssm_scan kernel (Mamba-2, d_state in {128, 256}) Fold the fused per-token SSM_SCAN recurrent step from opencl/gdn-qwen36-35b onto the unified base. Previously SSM_SCAN fell back to CPU here; now scalar-A Mamba-2 with d_state in {128,256}, all-f32, runs on GPU. Other shapes (incl. Mamba-1 element-wise A) still fall back. test-backend-ops -o SSM_SCAN passes on Adreno X2-90. opt-out via GGML_OPENCL_DISABLE_SSM_SCAN=1. * opencl: cleanup * opencl: require K == 1 --------- Co-authored-by: Li He --- ggml/src/ggml-opencl/CMakeLists.txt | 1 + ggml/src/ggml-opencl/ggml-opencl.cpp | 143 +++++++++++++++ ggml/src/ggml-opencl/kernels/ssm_scan.cl | 216 +++++++++++++++++++++++ 3 files changed, 360 insertions(+) create mode 100644 ggml/src/ggml-opencl/kernels/ssm_scan.cl diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 1dc7071771..72334d5ce2 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -202,6 +202,7 @@ set(GGML_OPENCL_KERNELS sqr sqrt ssm_conv + ssm_scan gated_delta_net sub sum_rows diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 733fab1c34..fa4702a855 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -866,6 +866,9 @@ struct ggml_backend_opencl_context { // [size_idx][kda][tgpp] where size_idx: 0=S_V=16, 1=32, 2=64, 3=128; kda: 0 or 1. // tgpp 0 = TG variant (COLS_PER_LANE_GROUP=1), tgpp 1 = prefill variant (COLS_PER_LANE_GROUP=4). cl_kernel kernel_gated_delta_net_f32[4][2][2] = {}; + cl_kernel kernel_ssm_scan_f32_mamba2_d128 = nullptr; + cl_kernel kernel_ssm_scan_f32_mamba2_d256 = nullptr; + cl_kernel kernel_timestep_embedding; cl_kernel kernel_gemv_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns_bin; cl_kernel kernel_gemm_moe_q8_0_f32_ns; @@ -3154,6 +3157,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // ssm_scan (Mamba-2 fused per-token recurrent step; d_state in {128, 256}) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "ssm_scan.cl.h" + }; +#else + const std::string kernel_src = read_file("ssm_scan.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d128 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d128", &err), err)); + CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d256 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d256", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // gated_delta_net: one kernel per (S_V, KDA, tgpp) triple. { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -7301,6 +7322,23 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32); case GGML_OP_SSM_CONV: return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32); + case GGML_OP_SSM_SCAN: { + // Mamba-2 fused per-token scan. Requires src3->ne[0] == 1 (scalar + // A per head); d_state in {128, 256}; all sources f32. Falls back + // to CPU otherwise (incl. Mamba-1 element-wise A). + for (int i = 0; i < 6; ++i) { + if (op->src[i]->type != GGML_TYPE_F32) { + return false; + } + } + if (op->type != GGML_TYPE_F32) { + return false; + } + const int K = ggml_get_op_params_i32(op, 0); + const int d_state = (int) op->src[0]->ne[0]; + const bool is_mamba2 = (op->src[3]->ne[0] == 1); + return is_mamba2 && (d_state == 128 || d_state == 256) && (K == 1); + } case GGML_OP_GATED_DELTA_NET: { // Match the Vulkan backend: only F32 -> F32, S_v in {16, 32, 64, 128}. @@ -12260,6 +12298,103 @@ static void ggml_cl_mean(ggml_backend_t backend, const ggml_tensor * src0, const backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } +static void ggml_cl_ssm_scan(ggml_backend_t backend, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; // s + const ggml_tensor * src1 = dst->src[1]; // x + const ggml_tensor * src2 = dst->src[2]; // dt + const ggml_tensor * src3 = dst->src[3]; // A + const ggml_tensor * src4 = dst->src[4]; // B + const ggml_tensor * src5 = dst->src[5]; // C + const ggml_tensor * src6 = dst->src[6]; // ids + + GGML_ASSERT(src0 && src1 && src2 && src3 && src4 && src5 && src6 && dst); + + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + + ggml_tensor_extra_cl * e0 = (ggml_tensor_extra_cl *) src0->extra; + ggml_tensor_extra_cl * e1 = (ggml_tensor_extra_cl *) src1->extra; + ggml_tensor_extra_cl * e2 = (ggml_tensor_extra_cl *) src2->extra; + ggml_tensor_extra_cl * e3 = (ggml_tensor_extra_cl *) src3->extra; + ggml_tensor_extra_cl * e4 = (ggml_tensor_extra_cl *) src4->extra; + ggml_tensor_extra_cl * e5 = (ggml_tensor_extra_cl *) src5->extra; + ggml_tensor_extra_cl * e6 = (ggml_tensor_extra_cl *) src6->extra; + ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *) dst->extra; + + cl_ulong o0 = e0->offset + src0->view_offs; + cl_ulong o1 = e1->offset + src1->view_offs; + cl_ulong o2 = e2->offset + src2->view_offs; + cl_ulong o3 = e3->offset + src3->view_offs; + cl_ulong o4 = e4->offset + src4->view_offs; + cl_ulong o5 = e5->offset + src5->view_offs; + cl_ulong o6 = e6->offset + src6->view_offs; + cl_ulong od = ed->offset + dst->view_offs; + + const int d_state = (int) src0->ne[0]; + const int head_dim = (int) src0->ne[1]; + const int n_head = (int) src1->ne[1]; + const int n_group = (int) src4->ne[1]; + const int n_tokens = (int) src1->ne[2]; + const int n_seqs = (int) src1->ne[3]; + + // Mirror CPU ref: s_off = ggml_nelements(src1) * sizeof(float) + const cl_ulong s_off_bytes = (cl_ulong) ggml_nelements(src1) * sizeof(float); + + cl_kernel kernel = (d_state == 128) + ? backend_ctx->kernel_ssm_scan_f32_mamba2_d128 + : backend_ctx->kernel_ssm_scan_f32_mamba2_d256; + GGML_ASSERT(kernel != nullptr); + + cl_ulong s0_nb2 = src0->nb[2]; + cl_ulong s0_nb3 = src0->nb[3]; + cl_ulong x_nb2 = src1->nb[2]; + cl_ulong x_nb3 = src1->nb[3]; + cl_ulong dt_nb1 = src2->nb[1]; + cl_ulong dt_nb2 = src2->nb[2]; + cl_ulong A_nb1 = src3->nb[1]; + cl_ulong B_nb2 = src4->nb[2]; + cl_ulong B_nb3 = src4->nb[3]; + cl_ulong C_nb2 = src5->nb[2]; + cl_ulong C_nb3 = src5->nb[3]; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &e0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &o0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &e1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &o1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &e2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &o2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &e3->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &o3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &e4->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &o4)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), &e5->data_device)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &o5)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_mem), &e6->data_device)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &o6)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_mem), &ed->data_device)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &od)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &s0_nb2)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &s0_nb3)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &x_nb2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &x_nb3)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &dt_nb1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &dt_nb2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &A_nb1)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &B_nb2)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &B_nb3)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(cl_ulong), &C_nb2)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &C_nb3)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &s_off_bytes)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(int), &head_dim)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &n_group)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(int), &n_tokens)); + + size_t global_work_size[] = { (size_t)n_head * head_dim * 64, (size_t)n_seqs, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + static void ggml_cl_ssm_conv(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -24746,6 +24881,14 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_ssm_conv; break; + case GGML_OP_SSM_SCAN: + if (!any_on_device) { + return false; + } + // SSM_SCAN has 7 source tensors, so it cannot use the standard + // (src0, src1, dst) func signature. Dispatch directly and return. + ggml_cl_ssm_scan(backend, tensor); + return true; case GGML_OP_GATED_DELTA_NET: if (!any_on_device) { return false; diff --git a/ggml/src/ggml-opencl/kernels/ssm_scan.cl b/ggml/src/ggml-opencl/kernels/ssm_scan.cl new file mode 100644 index 0000000000..37698d123f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/ssm_scan.cl @@ -0,0 +1,216 @@ +// Mamba2 fused SSM scan kernel. One workgroup per (head, dim, seq); WG size = +// 64 threads. Each thread owns c_factor = d_state/64 state elements in +// private registers; the state stays resident across the n_tokens t-loop +// +// References: +// ggml/src/ggml-cuda/ssm-scan.cu:117 ssm_scan_f32_group +// ggml/src/ggml-cpu/ops.cpp:9368 ggml_compute_forward_ssm_scan_f32 + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_khr_subgroups +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#if defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#else +#define REQD_SUBGROUP_SIZE_64 +#endif + +inline float softplus_f32(float x) { + return (x <= 20.0f) ? log(1.0f + exp(x)) : x; +} + +// d_state = 128 (most Mamba-2 models, e.g. mamba2-2.7B, Codestral-Mamba). +// WG = 64 threads, each holds 2 state elements (tid and tid+64). +REQD_SUBGROUP_SIZE_64 +kernel void kernel_ssm_scan_f32_mamba2_d128( + global const char * src0_base, ulong src0_off, + global const char * src1_base, ulong src1_off, + global const char * src2_base, ulong src2_off, + global const char * src3_base, ulong src3_off, + global const char * src4_base, ulong src4_off, + global const char * src5_base, ulong src5_off, + global const char * src6_base, ulong src6_off, + global char * dst_base, ulong dst_off, + ulong s0_nb2, ulong s0_nb3, + ulong x_nb2, ulong x_nb3, + ulong dt_nb1, ulong dt_nb2, + ulong A_nb1, + ulong B_nb2, ulong B_nb3, + ulong C_nb2, ulong C_nb3, + ulong s_off_bytes, + int head_dim, int n_head, int n_group, int n_tokens +) { + const int d_state = 128; + + const int tid = (int) get_local_id(0); + const int wg_x = (int) get_group_id(0); + const int seq_id = (int) get_group_id(1); + + const int head_id = wg_x / head_dim; + const int dim_id = wg_x - head_id * head_dim; + const int g = head_id / (n_head / n_group); + + src0_base += src0_off; + src1_base += src1_off; + src2_base += src2_off; + src3_base += src3_off; + src4_base += src4_off; + src5_base += src5_off; + src6_base += src6_off; + dst_base += dst_off; + + const int seq_slot = ((global const int *) src6_base)[seq_id]; + + const ulong state_base_off = (ulong)seq_slot * s0_nb3 + (ulong)head_id * s0_nb2 + + (ulong)dim_id * d_state * sizeof(float); + global const float * s0_warp = (global const float *)(src0_base + state_base_off); + const ulong state_out_off = (ulong)seq_id * s0_nb3 + (ulong)head_id * s0_nb2 + + (ulong)dim_id * d_state * sizeof(float); + global float * s_warp = (global float *)(dst_base + s_off_bytes + state_out_off); + + global const char * x_seq = src1_base + (ulong)seq_id * x_nb3; + global const char * dt_seq = src2_base + (ulong)seq_id * dt_nb2; + global const char * B_seq = src4_base + (ulong)seq_id * B_nb3 + (ulong)g * d_state * sizeof(float); + global const char * C_seq = src5_base + (ulong)seq_id * C_nb3 + (ulong)g * d_state * sizeof(float); + + const ulong y_dim_total = (ulong)n_head * head_dim; + global float * y_seq = (global float *)dst_base + + (ulong)seq_id * (ulong)n_tokens * y_dim_total; + + const float A_val = ((global const float *)src3_base)[(ulong)head_id * A_nb1 / sizeof(float)]; + + // c_factor = 2: each thread owns 2 state elements (tid and tid+64). + float state0 = s0_warp[tid]; + float state1 = s0_warp[tid + 64]; + + for (int t = 0; t < n_tokens; ++t) { + const float dt_h = ((global const float *)(dt_seq + (ulong)t * dt_nb1))[head_id]; + const float dt_softplus = softplus_f32(dt_h); + const float dA = exp(dt_softplus * A_val); + const float x_val = ((global const float *)(x_seq + (ulong)t * x_nb2))[(ulong)head_id * head_dim + dim_id]; + const float x_dt = x_val * dt_softplus; + + const float B0 = ((global const float *)(B_seq + (ulong)t * B_nb2))[tid]; + const float B1 = ((global const float *)(B_seq + (ulong)t * B_nb2))[tid + 64]; + const float C0 = ((global const float *)(C_seq + (ulong)t * C_nb2))[tid]; + const float C1 = ((global const float *)(C_seq + (ulong)t * C_nb2))[tid + 64]; + + state0 = state0 * dA + B0 * x_dt; + state1 = state1 * dA + B1 * x_dt; + const float partial = state0 * C0 + state1 * C1; + + const float sum = sub_group_reduce_add(partial); + if (tid == 0) { + y_seq[(ulong)t * y_dim_total + (ulong)head_id * head_dim + dim_id] = sum; + } + } + + s_warp[tid] = state0; + s_warp[tid + 64] = state1; +} + +// d_state = 256 (Falcon-H1). WG = 64 threads, each holds 4 state elements. +REQD_SUBGROUP_SIZE_64 +kernel void kernel_ssm_scan_f32_mamba2_d256( + global const char * src0_base, ulong src0_off, + global const char * src1_base, ulong src1_off, + global const char * src2_base, ulong src2_off, + global const char * src3_base, ulong src3_off, + global const char * src4_base, ulong src4_off, + global const char * src5_base, ulong src5_off, + global const char * src6_base, ulong src6_off, + global char * dst_base, ulong dst_off, + ulong s0_nb2, ulong s0_nb3, + ulong x_nb2, ulong x_nb3, + ulong dt_nb1, ulong dt_nb2, + ulong A_nb1, + ulong B_nb2, ulong B_nb3, + ulong C_nb2, ulong C_nb3, + ulong s_off_bytes, + int head_dim, int n_head, int n_group, int n_tokens +) { + const int d_state = 256; + + const int tid = (int) get_local_id(0); + const int wg_x = (int) get_group_id(0); + const int seq_id = (int) get_group_id(1); + + const int head_id = wg_x / head_dim; + const int dim_id = wg_x - head_id * head_dim; + const int g = head_id / (n_head / n_group); + + src0_base += src0_off; + src1_base += src1_off; + src2_base += src2_off; + src3_base += src3_off; + src4_base += src4_off; + src5_base += src5_off; + src6_base += src6_off; + dst_base += dst_off; + + const int seq_slot = ((global const int *) src6_base)[seq_id]; + + const ulong state_base_off = (ulong)seq_slot * s0_nb3 + (ulong)head_id * s0_nb2 + + (ulong)dim_id * d_state * sizeof(float); + global const float * s0_warp = (global const float *)(src0_base + state_base_off); + const ulong state_out_off = (ulong)seq_id * s0_nb3 + (ulong)head_id * s0_nb2 + + (ulong)dim_id * d_state * sizeof(float); + global float * s_warp = (global float *)(dst_base + s_off_bytes + state_out_off); + + global const char * x_seq = src1_base + (ulong)seq_id * x_nb3; + global const char * dt_seq = src2_base + (ulong)seq_id * dt_nb2; + global const char * B_seq = src4_base + (ulong)seq_id * B_nb3 + (ulong)g * d_state * sizeof(float); + global const char * C_seq = src5_base + (ulong)seq_id * C_nb3 + (ulong)g * d_state * sizeof(float); + + const ulong y_dim_total = (ulong)n_head * head_dim; + global float * y_seq = (global float *)dst_base + + (ulong)seq_id * (ulong)n_tokens * y_dim_total; + + const float A_val = ((global const float *)src3_base)[(ulong)head_id * A_nb1 / sizeof(float)]; + + // c_factor = 4: each thread owns 4 state elements. + float state0 = s0_warp[tid]; + float state1 = s0_warp[tid + 64]; + float state2 = s0_warp[tid + 128]; + float state3 = s0_warp[tid + 192]; + + for (int t = 0; t < n_tokens; ++t) { + const float dt_h = ((global const float *)(dt_seq + (ulong)t * dt_nb1))[head_id]; + const float dt_softplus = softplus_f32(dt_h); + const float dA = exp(dt_softplus * A_val); + const float x_val = ((global const float *)(x_seq + (ulong)t * x_nb2))[(ulong)head_id * head_dim + dim_id]; + const float x_dt = x_val * dt_softplus; + + global const float * B_t = (global const float *)(B_seq + (ulong)t * B_nb2); + global const float * C_t = (global const float *)(C_seq + (ulong)t * C_nb2); + + const float B0 = B_t[tid]; + const float B1 = B_t[tid + 64]; + const float B2 = B_t[tid + 128]; + const float B3 = B_t[tid + 192]; + const float C0 = C_t[tid]; + const float C1 = C_t[tid + 64]; + const float C2 = C_t[tid + 128]; + const float C3 = C_t[tid + 192]; + + state0 = state0 * dA + B0 * x_dt; + state1 = state1 * dA + B1 * x_dt; + state2 = state2 * dA + B2 * x_dt; + state3 = state3 * dA + B3 * x_dt; + const float partial = state0 * C0 + state1 * C1 + state2 * C2 + state3 * C3; + + const float sum = sub_group_reduce_add(partial); + if (tid == 0) { + y_seq[(ulong)t * y_dim_total + (ulong)head_id * head_dim + dim_id] = sum; + } + } + + s_warp[tid] = state0; + s_warp[tid + 64] = state1; + s_warp[tid + 128] = state2; + s_warp[tid + 192] = state3; +} From 990e3bfee333dad19499722cfa4102a3478d7a8b Mon Sep 17 00:00:00 2001 From: Yiwei Shao <44545837+njsyw1997@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:42:57 -0700 Subject: [PATCH 13/36] hexagon: fix FA HMX queue ordering and pack the rescale D matrices (#27042) * hexagon: fix FA HMX queue ordering in the pipelined path * hexagon: double buffer D matrix, store diagonal tile only * format code * align the indentation --- ggml/src/ggml-hexagon/htp/flash-attn-ops.c | 76 ++++++++++++---------- ggml/src/ggml-hexagon/htp/flash-attn-ops.h | 19 ++++-- 2 files changed, 57 insertions(+), 38 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c index fe78718c61..8176562904 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c @@ -132,8 +132,8 @@ struct hmx_fa_context { __fp16 * vtcm_v_tiles[2]; // V tiles (column-major, double-buffered) __fp16 * vtcm_s_tiles[2]; // S = QK^T [g_br, Bc] (double-buffered) __fp16 * vtcm_p_tiles[2]; // P = softmax(S) [g_br, Bc] - __fp16 * vtcm_d_tiles; // Diagonal rescale [g_br, g_br] - __fp16 * vtcm_d_inv_l; // Diagonal rescale (1/l) [g_br, g_br] + __fp16 * vtcm_d_tiles[2]; // Diagonal rescale, g_br/32 packed diagonal tiles (double-buffered) + __fp16 * vtcm_d_inv_l; // Diagonal rescale (1/l), same packed layout HVX_Vector * vtcm_m_vec; // Row max [g_br] HVX_Vector * vtcm_l_vec; // Row sum [g_br] HVX_Vector * vtcm_s_rowmax; // Softmax intermediate [g_br] @@ -782,13 +782,14 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { } } - // Initialize vtcm_d_tiles and vtcm_d_inv_l to 0 + // Zero the whole rescale region: vtcm_d_tiles[0], the optional vtcm_d_tiles[1] + // and vtcm_d_inv_l are equal-sized and allocated back to back, so one run covers + // them all. The scatter only ever writes the diagonal, ignore the rest. const size_t d_bytes_per_t = hex_align_up(d_tile_bytes / n, 128); const size_t d_start = i * d_bytes_per_t; const size_t d_end = hex_smin(d_start + d_bytes_per_t, d_tile_bytes); if (d_start < d_tile_bytes) { - hvx_splat_u8_a((char *) factx->vtcm_d_tiles + d_start, 0, d_end - d_start); - hvx_splat_u8_a((char *) factx->vtcm_d_inv_l + d_start, 0, d_end - d_start); + hvx_splat_u8_a((char *) factx->vtcm_d_tiles[0] + d_start, 0, d_end - d_start); } } @@ -1432,17 +1433,19 @@ static inline void fa_softmax_impl( const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16)); HVX_Vector v_exp_m_diff = exp_m_diff_f16; + __fp16 * const d_tiles_out = factx->vtcm_d_tiles[args->buf_idx]; + size_t t0 = r_vec_idx * 2; if (t0 < args->n_row_tiles) { const HVX_Vector v_content = v_exp_m_diff; - __fp16 * out_base = factx->vtcm_d_tiles + t0 * (args->n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; + __fp16 * out_base = d_tiles_out + t0 * HMX_FP16_TILE_N_ELMS; Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content); } size_t t1 = r_vec_idx * 2 + 1; if (t1 < args->n_row_tiles) { const HVX_Vector v_content = Q6_V_vror_VR(v_exp_m_diff, 64); - __fp16 * out_base = factx->vtcm_d_tiles + t1 * (args->n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; + __fp16 * out_base = d_tiles_out + t1 * HMX_FP16_TILE_N_ELMS; Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content); } } @@ -1506,7 +1509,7 @@ static __attribute__((noinline)) void fa_build_d_diag_inv_l(struct hmx_fa_contex v_content = Q6_V_vror_VR(v_content, 64); } - __fp16 * out_base = factx->vtcm_d_inv_l + i * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; + __fp16 * out_base = factx->vtcm_d_inv_l + i * HMX_FP16_TILE_N_ELMS; Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content); } } @@ -1615,7 +1618,7 @@ static void hmx_fa_o_update_worker(void * data) { const size_t o_stride = n_row_tiles_g_br * HMX_FP16_TILE_N_ELMS; const size_t v_stride = n_tiles_per_bc * HMX_FP16_TILE_N_ELMS; for (size_t r = 0; r < n_row_tiles; ++r) { - const __fp16 * d_diag = d_tiles + r * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; + const __fp16 * d_diag = d_tiles + r * HMX_FP16_TILE_N_ELMS; const __fp16 * p_tile_in = p_tiles + (r * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS; const __fp16 * o_rc = o_prev + r * HMX_FP16_TILE_N_ELMS; const __fp16 * v_tile_in = v_tiles; @@ -1654,7 +1657,7 @@ static void hmx_fa_o_norm_worker(void * data) { asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales)); const size_t o_stride = n_row_tiles_g_br * HMX_FP16_TILE_N_ELMS; for (size_t r = 0; r < n_row_tiles; ++r) { - const __fp16 * d_diag = d_tiles + r * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; + const __fp16 * d_diag = d_tiles + r * HMX_FP16_TILE_N_ELMS; const __fp16 * o_rc = o_prev + r * HMX_FP16_TILE_N_ELMS; __fp16 * o_out = o_curr + r * DV_tiles * HMX_FP16_TILE_N_ELMS; @@ -1882,7 +1885,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { factx.vtcm_s_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_s_tiles[1], pipeline); factx.vtcm_p_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_p_tiles[0]); factx.vtcm_p_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_p_tiles[1], pipeline); - factx.vtcm_d_tiles = VTCM_LAYOUT_PTR(__fp16, base, L.off_d_tiles); + factx.vtcm_d_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_d_tiles[0]); + factx.vtcm_d_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_d_tiles[1], pipeline); factx.vtcm_d_inv_l = VTCM_LAYOUT_PTR(__fp16, base, L.off_d_inv_l); factx.vtcm_m_vec = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_m_vec); factx.vtcm_l_vec = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_l_vec); @@ -2039,7 +2043,30 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { } } - // ---- 3. Pop and run K-prep for next block & push next QK-dot ---- + // ---- 3. Start HMX O update for block kv_blk - 1 (reads P[1 - buf_idx], V[1 - buf_idx], D) ---- + // O update relys on the previous block's P and V tiles. + // O update MUST be pushed before the next block's QK-dot: hmx_queue_pop() retires the + // oldest descriptor, so push order alone decides which pop waits for which job. + // If OU went in after QK(i+1), the pop below would retire QK(i+1) and leave + // OU(i-1) in flight into the next iteration, where V-prep overwrites V[prev_buf]. + if (kv_blk > 0) { + const size_t prev_buf = 1 - buf_idx; + ou_job[prev_buf].o_curr = o_tile_curr; + ou_job[prev_buf].o_prev = o_tile_prev; + ou_job[prev_buf].p_tiles = factx.vtcm_p_tiles[prev_buf]; + ou_job[prev_buf].v_tiles = factx.vtcm_v_tiles[prev_buf]; + ou_job[prev_buf].d_tiles = factx.vtcm_d_tiles[prev_buf]; + ou_job[prev_buf].hmx_scales = factx.vtcm_hmx_scales_id; + ou_job[prev_buf].n_row_tiles = n_row_tiles; + ou_job[prev_buf].n_col_tiles = + hmx_ceil_div(hex_smin(Bc, nek1 - (kv_blk - 1) * Bc), HMX_FP16_TILE_N_COLS); + ou_job[prev_buf].n_row_tiles_g_br = n_row_tiles_g_br; + ou_job[prev_buf].n_tiles_per_bc = n_tiles_per_bc; + ou_job[prev_buf].DV = DV; + hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job[prev_buf])); + } + + // ---- 4. Pop and run K-prep for next block & push next QK-dot ---- if (kv_blk + 1 < factx.n_kv_blocks) { const uint32_t next_start = (kv_blk + 1) * Bc; const uint32_t next_rows = hex_smin(Bc, nek1 - next_start); @@ -2059,10 +2086,10 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job[next_buf])); } - // ---- 4. Wait for current block's QK-dot to finish ---- + // ---- 5. Wait for current block's QK-dot to finish ---- hmx_queue_pop(hmx_q); - // ---- 5. Phase 2: softmax + build_D ---- + // ---- 6. Phase 2: softmax + build_D ---- fa_softmax_args_t sargs; memset(&sargs, 0, sizeof(sargs)); sargs.factx = &factx; @@ -2085,23 +2112,6 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride; sargs.slopes = factx.vtcm_slopes; - // Start HMX O update for block kv_blk - 1 (reads P[1 - buf_idx], V[1 - buf_idx]) - if (kv_blk > 0) { - const size_t prev_buf = 1 - buf_idx; - ou_job[prev_buf].o_curr = o_tile_curr; - ou_job[prev_buf].o_prev = o_tile_prev; - ou_job[prev_buf].p_tiles = factx.vtcm_p_tiles[prev_buf]; - ou_job[prev_buf].v_tiles = factx.vtcm_v_tiles[prev_buf]; - ou_job[prev_buf].d_tiles = factx.vtcm_d_tiles; - ou_job[prev_buf].hmx_scales = factx.vtcm_hmx_scales_id; - ou_job[prev_buf].n_row_tiles = n_row_tiles; - ou_job[prev_buf].n_col_tiles = hmx_ceil_div(hex_smin(Bc, nek1 - (kv_blk - 1) * Bc), HMX_FP16_TILE_N_COLS); - ou_job[prev_buf].n_row_tiles_g_br = n_row_tiles_g_br; - ou_job[prev_buf].n_tiles_per_bc = n_tiles_per_bc; - ou_job[prev_buf].DV = DV; - hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job[prev_buf])); - } - // Run Softmax on HVX (blocking call) fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br); @@ -2128,7 +2138,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { ou_job[0].o_prev = o_tile_prev; ou_job[0].p_tiles = factx.vtcm_p_tiles[1 - buf_idx]; ou_job[0].v_tiles = factx.vtcm_v_tiles[1 - buf_idx]; - ou_job[0].d_tiles = factx.vtcm_d_tiles; + ou_job[0].d_tiles = factx.vtcm_d_tiles[1 - buf_idx]; ou_job[0].hmx_scales = factx.vtcm_hmx_scales_id; ou_job[0].n_row_tiles = n_row_tiles; ou_job[0].n_col_tiles = last_cols; @@ -2232,7 +2242,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { ou_job.o_prev = o_tile_prev; ou_job.p_tiles = factx.vtcm_p_tiles[0]; ou_job.v_tiles = factx.vtcm_v_tiles[0]; - ou_job.d_tiles = factx.vtcm_d_tiles; + ou_job.d_tiles = factx.vtcm_d_tiles[0]; ou_job.hmx_scales = factx.vtcm_hmx_scales_id; ou_job.n_row_tiles = n_row_tiles; ou_job.n_col_tiles = n_col_tiles; diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h index efe5ce5481..c4d1906316 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h @@ -109,7 +109,7 @@ struct hmx_fa_vtcm_layout { size_t off_v_tiles[2]; size_t off_s_tiles[2]; size_t off_p_tiles[2]; - size_t off_d_tiles; + size_t off_d_tiles[2]; size_t off_d_inv_l; size_t off_m_vec; size_t off_l_vec; @@ -125,7 +125,7 @@ struct hmx_fa_vtcm_layout { size_t q_tile_bytes; size_t o_tile_bytes; size_t s_tile_bytes; // S and P tiles (same size) - size_t d_tile_bytes; + size_t d_tile_bytes; // d_tiles[0..1] + d_inv_l, allocated back to back size_t m_line_bytes; // one mask row size_t m_buf_slot_bytes; // one dma_cache slot = align_up(Br * m_line_bytes, 4096) size_t col_vec_bytes; @@ -149,7 +149,12 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, const size_t k_tile_size = hex_align_up(Bc * DK * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); const size_t v_tile_size = hex_align_up(Bc * DV * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); const size_t s_tile_size = hex_align_up(g_br * Bc * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); - const size_t d_tile_size = hex_align_up(g_br * g_br * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); + + // The rescale matrices are diagonal: the HMX kernels only ever load the g_br/32 + // tiles that sit on the diagonal, so store just those, packed back to back with + // a stride of one tile. The old [g_br, g_br] square layout allocated g_br/32 + // times more than it used, which is also why a second D buffer was unaffordable. + const size_t d_tile_size = (g_br / HMX_FP16_TILE_N_ROWS) * HTP_FA_HMX_TILE_SIZE; const size_t q_dma_size = hex_align_up(g_br * DK * (is_q_fp32 ? sizeof(float) : sizeof(__fp16)), 128); const size_t k_dma_size = hex_align_up(Bc * hex_round_up(DK * sizeof(__fp16), 128), 128); @@ -167,7 +172,8 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, VTCM_LAYOUT_ALLOC(off, off_q_tiles, q_tile_size); VTCM_LAYOUT_ALLOC(off, off_o_tiles[0], o_tile_size); VTCM_LAYOUT_ALLOC(off, off_o_tiles[1], o_tile_size); - VTCM_LAYOUT_ALLOC(off, off_d_tiles, d_tile_size); + VTCM_LAYOUT_ALLOC(off, off_d_tiles[0], d_tile_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_d_tiles[1], d_tile_size, pipeline); VTCM_LAYOUT_ALLOC(off, off_d_inv_l, d_tile_size); // Group B & C share start offset (Group B tiles must be 2KB aligned) @@ -213,7 +219,10 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, L->o_tile_bytes = o_tile_size; L->col_vec_bytes = col_vec_size; L->s_tile_bytes = s_tile_size; - L->d_tile_bytes = d_tile_size; + // Measured from the actual offsets rather than assumed to be N * d_tile_size, so + // that inserting a region between them (or adding padding to VTCM_LAYOUT_ALLOC) + // cannot silently leave the tail of the run unzeroed. + L->d_tile_bytes = (L->off_d_inv_l + d_tile_size) - L->off_d_tiles[0]; L->m_line_bytes = m_line_size; L->m_buf_slot_bytes = m_buf_slot; L->row_buf_stride = row_vec_size / 128; From d59d455fd8ea09e5a2e87ce2a9d668267ffb5ccd Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Wed, 19 Aug 2026 14:53:27 -0700 Subject: [PATCH 14/36] tensor-split meta backend fixes (#26502) * backend: propagate buffer usage in meta backend * ggml-meta: make sure to call init_tensor for all new tensors * meta: remove explicit check for meta backend in ggml_backend_meta_get_split_state I can't seem to reproduce the original failure in the latest code. --- ggml/src/ggml-backend-impl.h | 1 + ggml/src/ggml-backend-meta.cpp | 20 ++++++++++++++++++-- ggml/src/ggml-backend.cpp | 2 ++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h index 9c56ec30c5..40cea024c3 100644 --- a/ggml/src/ggml-backend-impl.h +++ b/ggml/src/ggml-backend-impl.h @@ -83,6 +83,7 @@ extern "C" { GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers); GGML_API bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer); GGML_API void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); + GGML_API void ggml_backend_meta_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); // // Backend (meta) diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 7654ea1f30..775ae99267 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -1118,7 +1118,6 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( } static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync) { - GGML_ASSERT(ggml_backend_buffer_is_meta(tensor->buffer)); ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) tensor->buffer->context; return ggml_backend_meta_get_split_state(buf_ctx->get_simple_tensor_container(tensor), tensor, assume_sync); } @@ -1178,7 +1177,15 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m t_ij->flags = tensor->flags; memcpy(t_ij->op_params, tensor->op_params, sizeof(tensor->op_params)); ggml_set_name(t_ij, tensor->name); + t_ij->buffer = simple_buf; + if (simple_buf) { + // the backend that owns the buffer will set .extra + ggml_backend_buffer_init_tensor(simple_buf, t_ij); + } else { + t_ij->extra = tensor->extra; + } + t_ij->view_src = tensor->view_src; t_ij->view_offs = tensor->view_offs; if (t_ij->view_src != nullptr && ggml_backend_buffer_is_meta(t_ij->view_src->buffer)) { @@ -1209,7 +1216,6 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m t_ij->data = (char *) ggml_backend_buffer_get_base(simple_buf) + size_t(tensor->data) - size_t(ggml_backend_buffer_get_base(tensor->buffer)); } - t_ij->extra = tensor->extra; for (int i = 0; i < GGML_MAX_SRC; i++) { t_ij->src[i] = tensor->src[i]; if (tensor->src[i] == tensor) { @@ -1502,6 +1508,16 @@ bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) { return buf != nullptr && buf->iface.free_buffer == ggml_backend_meta_buffer_iface.free_buffer; } +void ggml_backend_meta_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) { + GGML_ASSERT(ggml_backend_buffer_is_meta(buffer)); + ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context; + for (size_t i = 0; i < buf_ctx->bufs.size(); i++) { + if (buf_ctx->bufs[i]) { + ggml_backend_buffer_set_usage(buf_ctx->bufs[i].get(), usage); + } + } +} + static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft); diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index f6fb91798c..d5ba5b5cea 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -182,6 +182,8 @@ void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backe // FIXME: add a generic callback to the buffer interface if (ggml_backend_buffer_is_multi_buffer(buffer)) { ggml_backend_multi_buffer_set_usage(buffer, usage); + } else if (ggml_backend_buffer_is_meta(buffer)) { + ggml_backend_meta_buffer_set_usage(buffer, usage); } } From 9ee9fc04c136ef2ae729bfc60d18961b23c13ddf Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Wed, 19 Aug 2026 20:40:19 -0700 Subject: [PATCH 15/36] opencl: make the MoE expert scatter deterministic (#26464) --- ggml/src/ggml-opencl/ggml-opencl.cpp | 48 +++++++++--- .../ggml-opencl/kernels/moe_sort_by_expert.cl | 73 +++++++++++++++++++ 2 files changed, 110 insertions(+), 11 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index fa4702a855..fbf7dadb90 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -895,6 +895,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) q4_0 MoE prefill GEMM cl_kernel kernel_moe_reorder_b; cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter; + cl_kernel kernel_moe_scatter_stable = nullptr; // deterministic slot assignment cl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sum cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat; cl_kernel kernel_mul_mv_id_q8_0_f32, kernel_mul_mv_id_q8_0_f32_flat; @@ -4463,6 +4464,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err)); CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err)); CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_scatter_stable = clCreateKernel(prog, "kernel_moe_scatter_stable", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -20863,18 +20865,42 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, size_t fill_local_size[] = {64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, fill_global_size, fill_local_size, src); - // Scatter - kernel = backend_ctx->kernel_moe_scatter; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &original_router_buf)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &post_router_buf)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &emap_buf)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tile_offset_buf)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &slot_counter_buf)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne21)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne20)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne02)); + // Scatter. The deterministic variant is the default: kernel_moe_scatter derives + // each token's slot from an atomic counter, so the packing inside an expert - and + // with it the output of the ragged prefill GEMM - changes from run to run. Set + // GGML_OPENCL_MOE_STABLE_SCATTER=0 to restore the atomic version. + static const bool stable_scatter = []{ + const char * e = getenv("GGML_OPENCL_MOE_STABLE_SCATTER"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, histogram_global_size, histogram_local_size, src); + if (stable_scatter) { + kernel = backend_ctx->kernel_moe_scatter_stable; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &original_router_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &post_router_buf)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &emap_buf)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tile_offset_buf)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne21)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + + // one workgroup (one wave) per expert; each ranks its own tokens + size_t scatter_global_size[] = {64, (size_t)ne02}; + size_t scatter_local_size[] = {64, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, scatter_global_size, scatter_local_size, src); + } else { + kernel = backend_ctx->kernel_moe_scatter; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &original_router_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &post_router_buf)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &emap_buf)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tile_offset_buf)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &slot_counter_buf)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne21)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne02)); + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, histogram_global_size, histogram_local_size, src); + } // [MOE_TILES] env-gated padding probe: read back total_tiles (= Sum_e // ceil(k_e/n_tile_size)) and compare to the ideal tile count for the real diff --git a/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl b/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl index d9703429b1..d52d11aa56 100644 --- a/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +++ b/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl @@ -68,6 +68,79 @@ __kernel void kernel_moe_scatter( emap[tile_idx] = val; } +// Deterministic replacement for kernel_moe_scatter. +// +// kernel_moe_scatter takes each token's slot from atomic_inc(slot_counter[expert]), +// so the token -> slot packing inside an expert depends on which work-item wins the +// atomic and changes from run to run. The ragged prefill GEMM path is sensitive to +// that packing (the non-ragged path is not, since its padded slots alias slot 0 and +// are overwritten last), which makes MoE prompt processing non-reproducible: the same +// binary on the same prompt returns one of several outputs. +// +// Here the slot is the token's rank in flat (n, k) order among the tokens routed to +// the same expert - a fixed function of the routing input. One workgroup per expert +// walks the flat routing list in blocks of 64 and ranks its own tokens with a +// workgroup scan, carrying a running count between blocks. Cost is one pass over the +// routing list per expert; the list is a few KiB and stays in cache. +__kernel void kernel_moe_scatter_stable( + __global const int * input, + __global int * post_router, + __global ushort * emap, + __global const int * tile_offset, + int N, + int topK, + uint n_experts +) { + const int e = get_group_id(1); + const int lid = get_local_id(0); + const int M = N * topK; + + __local int scan[64]; + __local int running; + + if (lid == 0) { + running = 0; + } + barrier(CLK_LOCAL_MEM_FENCE); + + for (int base = 0; base < M; base += 64) { + const int j = base + lid; + + int pred = 0; + if (j < M) { + const int n = j / topK; + const int k = j - n * topK; + pred = (input[n * (int)n_experts + k] == e) ? 1 : 0; + } + + scan[lid] = pred; + barrier(CLK_LOCAL_MEM_FENCE); + + // Hillis-Steele inclusive scan over the 64 lanes + for (int off = 1; off < 64; off <<= 1) { + int add = (lid >= off) ? scan[lid - off] : 0; + barrier(CLK_LOCAL_MEM_FENCE); + scan[lid] += add; + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (pred) { + const int local_slot = running + (scan[lid] - 1); // exclusive rank + const int tile_idx = tile_offset[e] + (local_slot >> 5); + const int lane = local_slot & 31; + + post_router[tile_idx * 32 + lane] = j; + emap[tile_idx] = (ushort)e; + } + + barrier(CLK_LOCAL_MEM_FENCE); + if (lid == 63) { + running += scan[63]; + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} + __kernel void kernel_moe_fill( __global int * post_router, __global int * total_tiles, From 2cfdb5fc08a81f6c95519fc5a60a86943a05c94c Mon Sep 17 00:00:00 2001 From: Markus Tavenrath Date: Thu, 20 Aug 2026 08:52:28 +0200 Subject: [PATCH 16/36] vulkan : add source groups for shaders (#26666) --- ggml/src/ggml-vulkan/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ggml/src/ggml-vulkan/CMakeLists.txt b/ggml/src/ggml-vulkan/CMakeLists.txt index 1dc6a145de..e733ad5cc9 100644 --- a/ggml/src/ggml-vulkan/CMakeLists.txt +++ b/ggml/src/ggml-vulkan/CMakeLists.txt @@ -200,8 +200,11 @@ if (Vulkan_FOUND) set (_ggml_vk_header "${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan-shaders.hpp") set (_ggml_vk_input_dir "${CMAKE_CURRENT_SOURCE_DIR}/vulkan-shaders") set (_ggml_vk_output_dir "${CMAKE_CURRENT_BINARY_DIR}/vulkan-shaders.spv") + set (_ggml_vk_generated_shader_files ${_ggml_vk_header}) file(GLOB _ggml_vk_shader_files CONFIGURE_DEPENDS "${_ggml_vk_input_dir}/*.comp") + set_source_files_properties(${_ggml_vk_shader_files} PROPERTIES HEADER_FILE_ONLY TRUE) + target_sources(ggml-vulkan PRIVATE ${_ggml_vk_shader_files}) # Because external projects do not provide source-level tracking, # the vulkan-shaders-gen sources need to be explicitly added to @@ -241,8 +244,11 @@ if (Vulkan_FOUND) COMMENT "Generate vulkan shaders for ${file}" ) target_sources(ggml-vulkan PRIVATE ${_ggml_vk_target_cpp}) + list(APPEND _ggml_vk_generated_shader_files ${_ggml_vk_target_cpp}) endforeach() + source_group("Vulkan shaders" FILES ${_ggml_vk_shader_files}) + source_group("Generated Vulkan shaders" FILES ${_ggml_vk_generated_shader_files}) else() message(WARNING "Vulkan not found") endif() From f466cfa38fac99e80a2aa4b58b3203b33872fe9c Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 10:00:16 +0300 Subject: [PATCH 17/36] spec : avoid binding reference to null pointer (#27404) --- common/speculative.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/speculative.cpp b/common/speculative.cpp index ae55e357d5..89e9b2782c 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2649,6 +2649,10 @@ void common_speculative_draft(common_speculative * spec) { for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) dparams.size(); ++seq_id) { auto & dp = dparams[seq_id]; + if (!dp.drafting) { + continue; + } + auto & result = *dp.result; // a new draft has been sampled From 929d47a39163d67a2808413eaf8916097c4bb53c Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 10:00:35 +0300 Subject: [PATCH 18/36] graph : create V as a view of K in the k_iswa build_attn (#27392) build_attn with the llm_graph_input_attn_k_iswa input was using the cached K tensor itself as V. Create V as a view of K (the first v_cur->ne[0] elements of each row), like the other K-only build_attn overloads. The deepseek4 MTP call site now passes the kv tensor as v_cur. Assisted-by: pi:llama.cpp/Qwen3.8-27B --- src/llama-graph.cpp | 4 +--- src/models/deepseek4.cpp | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 1896758c5d..5212e19a25 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -3099,8 +3099,6 @@ ggml_tensor * llm_graph_context::build_attn( int il) const { const bool is_swa = hparams.is_swa(il); - GGML_UNUSED(v_cur); - auto * k_rot = is_swa ? inp->self_k_rot_swa : inp->self_k_rot; if (k_rot) { @@ -3133,7 +3131,7 @@ ggml_tensor * llm_graph_context::build_attn( // MLA-style attention: the cached K is used as V ggml_tensor * q = q_cur; ggml_tensor * k = mctx_cur->get_k(ctx0, il); - ggml_tensor * v = k; + ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0); ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il); cb(cur, "kqv_out", il); diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 89cd461765..366ca2e546 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -1225,7 +1225,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl( if (inp_mtp) { out = build_attn(inp_mtp, nullptr, nullptr, nullptr, - q, kv, nullptr, + q, kv, kv, nullptr, layer.attn_sinks, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); cb(out, "attn_raw", il); From d9b6be07d0864ab09417b17ba36f9788087dd22c Mon Sep 17 00:00:00 2001 From: Alexander Heisler <126129661+heislera763@users.noreply.github.com> Date: Thu, 20 Aug 2026 03:27:51 -0400 Subject: [PATCH 19/36] ggml-cuda: provide static workspace for cuBLAS handles (#26574) * provide static workspace for cuBLAS handles * account for concurrent streams when using GGML_CUDA_GRAPH_OPT * drop cublas_handle overloads and remove direct cublasSetStream calls * Update ggml/src/ggml-cuda/common.cuh --------- Co-authored-by: Oliver Simons --- ggml/src/ggml-cuda/common.cuh | 29 ++++++++++++++++++----------- ggml/src/ggml-cuda/ggml-cuda.cu | 19 +++++++++++-------- ggml/src/ggml-cuda/out-prod.cu | 2 -- ggml/src/ggml-cuda/solve_tri.cu | 8 +++----- ggml/src/ggml-cuda/ssm-scan.cu | 1 - 5 files changed, 32 insertions(+), 27 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index d27d8acb1d..14dd1098c9 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1418,7 +1418,9 @@ struct ggml_backend_cuda_context { cudaEvent_t copy_event = nullptr; cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } }; - cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr}; + cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {nullptr}; + void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {nullptr}; + size_t cublas_workspace_sizes[GGML_CUDA_MAX_DEVICES] = {0}; int curr_stream_no = 0; @@ -1495,17 +1497,22 @@ struct ggml_backend_cuda_context { ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; } - cublasHandle_t cublas_handle(int device) { - if (cublas_handles[device] == nullptr) { - ggml_cuda_set_device(device); - CUBLAS_CHECK(cublasCreate(&cublas_handles[device])); - CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device], CUBLAS_TF32_TENSOR_OP_MATH)); - } - return cublas_handles[device]; - } - cublasHandle_t cublas_handle() { - return cublas_handle(device); + if (cublas_handles[device][curr_stream_no] == nullptr) { + ggml_cuda_set_device(device); + CUBLAS_CHECK(cublasCreate(&cublas_handles[device][curr_stream_no])); + CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device][curr_stream_no], CUBLAS_TF32_TENSOR_OP_MATH)); + CUBLAS_CHECK(cublasSetStream(cublas_handles[device][curr_stream_no], stream())); +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUBLAS_VER_MAJOR > 11 || (CUBLAS_VER_MAJOR == 11 && CUBLAS_VER_MINOR >= 2)) + if (cublas_workspace_sizes[device] == 0) { + const int cc = ggml_cuda_info().devices[device].cc; + cublas_workspace_sizes[device] = (cc >= GGML_CUDA_CC_HOPPER) ? 32 * 1024 * 1024 : 4 * 1024 * 1024; + } + CUDA_CHECK(cudaMalloc(&cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); + CUBLAS_CHECK(cublasSetWorkspace(cublas_handles[device][curr_stream_no], cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); +#endif + } + return cublas_handles[device][curr_stream_no]; } // pool diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 3b2a0ea851..a8a1c09ca3 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -711,9 +711,12 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() { if (streams[i][j] != nullptr) { CUDA_CHECK(cudaStreamDestroy(streams[i][j])); } - } - if (cublas_handles[i] != nullptr) { - CUBLAS_CHECK(cublasDestroy(cublas_handles[i])); + if (cublas_handles[i][j] != nullptr) { + CUBLAS_CHECK(cublasDestroy(cublas_handles[i][j])); + } + if (cublas_workspaces[i][j] != nullptr) { + CUDA_CHECK(cudaFree(cublas_workspaces[i][j])); + } } } } @@ -1416,7 +1419,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const const int64_t ne_dst = ggml_nelements(dst); cudaStream_t main_stream = ctx.stream(); - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(), main_stream)); + cublasHandle_t cublas_h = ctx.cublas_handle(); const size_t src0_ts = ggml_type_size(src0->type); GGML_ASSERT(nb00 == src0_ts); @@ -1539,14 +1542,14 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const // probably because the internal kernel selection logic is suboptimal. if (compute_type == GGML_TYPE_F32 && ne12 == 1 && ne13 == 1) { CUBLAS_CHECK( - cublasSgemm(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasSgemm(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, (const float *) alpha, (const float *) src0_ptr, s01, (const float *) src1_ptr, s11, (const float *) beta, (float *) dst_ptr, ne0)); } else if (ne12 == 1 && ne13 == 1) { CUBLAS_CHECK( - cublasGemmEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, src0_ptr, cu_data_type_a, s01, src1_ptr, cu_data_type_b, s11, @@ -1561,7 +1564,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const // there is no broadcast and src0, src1 are contiguous across dims 2, 3 // use cublasGemmStridedBatchedEx CUBLAS_CHECK( - cublasGemmStridedBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmStridedBatchedEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, src0_ptr, cu_data_type_a, s01, sma, // strideA src1_ptr, cu_data_type_b, s11, smb, // strideB @@ -1599,7 +1602,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const CUDA_CHECK(cudaGetLastError()); CUBLAS_CHECK( - cublasGemmBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmBatchedEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, (const void **) (ptrs_src.get() + 0*ne23), cu_data_type_a, s01, (const void **) (ptrs_src.get() + 1*ne23), cu_data_type_b, s11, diff --git a/ggml/src/ggml-cuda/out-prod.cu b/ggml/src/ggml-cuda/out-prod.cu index 46b9f3a67e..c46e0455de 100644 --- a/ggml/src/ggml-cuda/out-prod.cu +++ b/ggml/src/ggml-cuda/out-prod.cu @@ -54,8 +54,6 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const float alpha = 1.0f; const float beta = 0.0f; - CUBLAS_CHECK(cublasSetStream(handle, stream)); - const int64_t lda = nb01 / sizeof(float); const int64_t ldc = nb1 / sizeof(float); diff --git a/ggml/src/ggml-cuda/solve_tri.cu b/ggml/src/ggml-cuda/solve_tri.cu index 07ca33f513..d96783420a 100644 --- a/ggml/src/ggml-cuda/solve_tri.cu +++ b/ggml/src/ggml-cuda/solve_tri.cu @@ -65,15 +65,13 @@ static void solve_tri_f32_cublas(ggml_backend_cuda_context & ctx, get_batch_pointers<<<(total_batches + 255) / 256, 256, 0, stream>>>(A, X, A_ptrs_dev, X_ptrs_dev, ne02, total_batches, s02, s03, s2, s3); - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream)); - // Yes, this is necessary, without this we get RMSE errors - CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(id), CUBLAS_DEFAULT_MATH)); - CUBLAS_CHECK(cublasStrsmBatched(ctx.cublas_handle(id), CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N, + CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(), CUBLAS_DEFAULT_MATH)); + CUBLAS_CHECK(cublasStrsmBatched(ctx.cublas_handle(), CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N, CUBLAS_DIAG_NON_UNIT, k, n, &alpha, A_ptrs_dev, n, X_ptrs_dev, k, total_batches)); // revert to standard mode from common.cuh - CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(id), CUBLAS_TF32_TENSOR_OP_MATH)); + CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(), CUBLAS_TF32_TENSOR_OP_MATH)); GGML_UNUSED_VARS(s12, s13); } diff --git a/ggml/src/ggml-cuda/ssm-scan.cu b/ggml/src/ggml-cuda/ssm-scan.cu index ef342f01f1..40cb38dee7 100644 --- a/ggml/src/ggml-cuda/ssm-scan.cu +++ b/ggml/src/ggml-cuda/ssm-scan.cu @@ -632,7 +632,6 @@ static void ssm_scan_ssd_f32_cuda( // Step 3: chunked SSD loop // Per chunk: pre_matmul (incl. M) + 4 cuBLAS (CB, Y, S@C, state update) + scale_state cublasHandle_t handle = ctx.cublas_handle(); - CUBLAS_CHECK(cublasSetStream(handle, stream)); const float alpha_one = 1.0f; const float beta_zero = 0.0f; const float beta_one = 1.0f; From a3b1effcda84caeb180427b1346d0212841418f6 Mon Sep 17 00:00:00 2001 From: Rock Chen Date: Thu, 20 Aug 2026 15:35:28 +0800 Subject: [PATCH 20/36] convert: fix get block count error for Nemotron 3 Ultra (#27101) * convert: fix get block count error for Nemotron Signed-off-by: Rock Chen * fix this in NemotronHModel.__init__ instead. This reverts commit ca689cbc8792ba69ea1fd5d8b3ae0485c47536ec. --------- Signed-off-by: Rock Chen --- conversion/nemotron.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/conversion/nemotron.py b/conversion/nemotron.py index 3e37c7b469..e5d1671851 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -207,7 +207,9 @@ class NemotronHModel(GraniteHybridModel): # calling the parent __init__. This is because the parent constructor # uses self.model_arch to build the tensor name map, and all MoE-specific # mappings would be missed if it were called with the default non-MoE arch. - hparams = ModelBase.load_hparams(args[0], self.is_mistral_format) + hparams = kwargs.pop("hparams", None) + if hparams is None: + hparams = ModelBase.load_hparams(args[0], self.is_mistral_format) has_moe_params = ( "num_experts_per_tok" in hparams or (isinstance(hparams.get("llm_config"), dict) and "num_experts_per_tok" in hparams["llm_config"]) @@ -215,8 +217,11 @@ class NemotronHModel(GraniteHybridModel): if has_moe_params: self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE self.is_moe = True + layers_block_type = hparams.get("layers_block_type") + if layers_block_type is not None: + hparams["num_hidden_layers"] = len(layers_block_type) - super().__init__(*args, **kwargs) + super().__init__(*args, hparams=hparams, **kwargs) # Save the top-level head_dim for later self.head_dim = self.hparams.get("head_dim", self.hparams.get("attention_head_dim")) From 849798132173c3c511dffe3a03c3c760d707b05f Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Thu, 20 Aug 2026 10:42:33 +0200 Subject: [PATCH 21/36] ggml: fix backend split scheduler race condition (#26040) * ggml: fix backend split scheduler race condition splits without input were running concurrently with other splits, while potentially reusing memory the other split is accessing * only sync when split has no inputs --- ggml/src/ggml-backend.cpp | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index d5ba5b5cea..e519bdf50a 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -1601,11 +1601,23 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s std::vector ids; std::vector used_ids; + int prev_backend_id = -1; + for (int split_id = 0; split_id < sched->n_splits; split_id++) { struct ggml_backend_sched_split * split = &splits[split_id]; int split_backend_id = split->backend_id; ggml_backend_t split_backend = sched->backends[split_backend_id]; + // ensure the previous split's async work has completed before we start + // this split, the allocator may have reused buffer regions across splits + if (split->n_inputs == 0 && prev_backend_id >= 0 && prev_backend_id != split_backend_id) { + if (sched->events[prev_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_synchronize(sched->events[prev_backend_id][sched->cur_copy]); + } else { + ggml_backend_synchronize(sched->backends[prev_backend_id]); + } + } + // copy the input tensors to the split backend for (int input_id = 0; input_id < split->n_inputs; input_id++) { ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]); @@ -1768,12 +1780,12 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s } } - // record the event of this copy - if (split->n_inputs > 0) { - if (sched->events[split_backend_id][sched->cur_copy] != NULL) { - ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend); - } + // record the event of this split + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend); } + + prev_backend_id = split_backend_id; } return GGML_STATUS_SUCCESS; From f20395dae59ba30ab0a10e0e0b0db6eeb8e8a282 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 13:35:15 +0300 Subject: [PATCH 22/36] Revert "tensor-split meta backend fixes (#26502)" (#27433) This reverts commit d59d455fd8ea09e5a2e87ce2a9d668267ffb5ccd. --- ggml/src/ggml-backend-impl.h | 1 - ggml/src/ggml-backend-meta.cpp | 20 ++------------------ ggml/src/ggml-backend.cpp | 2 -- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h index 40cea024c3..9c56ec30c5 100644 --- a/ggml/src/ggml-backend-impl.h +++ b/ggml/src/ggml-backend-impl.h @@ -83,7 +83,6 @@ extern "C" { GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers); GGML_API bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer); GGML_API void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); - GGML_API void ggml_backend_meta_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); // // Backend (meta) diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 775ae99267..7654ea1f30 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -1118,6 +1118,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( } static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync) { + GGML_ASSERT(ggml_backend_buffer_is_meta(tensor->buffer)); ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) tensor->buffer->context; return ggml_backend_meta_get_split_state(buf_ctx->get_simple_tensor_container(tensor), tensor, assume_sync); } @@ -1177,15 +1178,7 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m t_ij->flags = tensor->flags; memcpy(t_ij->op_params, tensor->op_params, sizeof(tensor->op_params)); ggml_set_name(t_ij, tensor->name); - t_ij->buffer = simple_buf; - if (simple_buf) { - // the backend that owns the buffer will set .extra - ggml_backend_buffer_init_tensor(simple_buf, t_ij); - } else { - t_ij->extra = tensor->extra; - } - t_ij->view_src = tensor->view_src; t_ij->view_offs = tensor->view_offs; if (t_ij->view_src != nullptr && ggml_backend_buffer_is_meta(t_ij->view_src->buffer)) { @@ -1216,6 +1209,7 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m t_ij->data = (char *) ggml_backend_buffer_get_base(simple_buf) + size_t(tensor->data) - size_t(ggml_backend_buffer_get_base(tensor->buffer)); } + t_ij->extra = tensor->extra; for (int i = 0; i < GGML_MAX_SRC; i++) { t_ij->src[i] = tensor->src[i]; if (tensor->src[i] == tensor) { @@ -1508,16 +1502,6 @@ bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) { return buf != nullptr && buf->iface.free_buffer == ggml_backend_meta_buffer_iface.free_buffer; } -void ggml_backend_meta_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) { - GGML_ASSERT(ggml_backend_buffer_is_meta(buffer)); - ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context; - for (size_t i = 0; i < buf_ctx->bufs.size(); i++) { - if (buf_ctx->bufs[i]) { - ggml_backend_buffer_set_usage(buf_ctx->bufs[i].get(), usage); - } - } -} - static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft); diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index e519bdf50a..3d6310f3ff 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -182,8 +182,6 @@ void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backe // FIXME: add a generic callback to the buffer interface if (ggml_backend_buffer_is_multi_buffer(buffer)) { ggml_backend_multi_buffer_set_usage(buffer, usage); - } else if (ggml_backend_buffer_is_meta(buffer)) { - ggml_backend_meta_buffer_set_usage(buffer, usage); } } From 70aff25250075bf23b533c207b55168a4f926350 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 13:43:59 +0300 Subject: [PATCH 23/36] metal : dequantize quantized KV to F16 before flash attention (#27390) * metal: dequantize q8_0 KV to f16 before flash attention Add a preprocessing pass for GGML_OP_FLASH_ATTN_EXT on the Metal backend: when the KV cache is quantized (Q8_0 for now), dequantize K and V into a contiguous F16 scratch buffer and run the existing F16 flash attention kernels on it, instead of the in-kernel dequantization path. - new kernel kernel_flash_attn_ext_dequant_to_f16: one thread per quant block (K then V), stride-aware so permuted KV is supported; instantiated for Q8_0 (extending to Q4_0/Q4_1/Q5_0/Q5_1 is one instantiation + one gate case) - the gate is type-only: dequantize whenever the KV is quantized, regardless of head sizes, GQA ratio or n_kv; the attention kernels themselves are untouched - the F16 copies live in the op's own scratch allocation (ggml_metal_op_flash_attn_ext_extra_dequant_f16); the KV pad kernel reads the dequantized buffers when the path is active - the FA pipeline getters gain a use_f16_kv flag selecting the existing f16 kernels and contiguous strides - ref: https://github.com/ggml-org/llama.cpp/pull/25556 Verification (M2 Ultra): - test-backend-ops test -o FLASH_ATTN_EXT: 4798/4798 pass, including the new q8_0 eval cases (decode/prompt, permuted, sinks+ALiBi+softcap, kv=113 pad path, kv=16384) - llama-perplexity on Qwen2.5-0.5B with -ctk q8_0 -ctv q8_0 matches the f16 KV reference (PPL 1.0008 vs 1.0008) Assisted-by: pi:llama.cpp/Qwen3.8-27B * metal : launch the FA KV dequant kernel separately for K and V Simplify kernel_flash_attn_ext_dequant_to_f16: it now dequantizes a single tensor (its own ne/nb and dst) with no is_v branching, and the op dispatches it twice with the same pipeline - once for K and once for V. The kargs struct shrinks to a single ne/nb set plus nblocks. Assisted-by: pi:llama.cpp/Qwen3.8-27B * metal : dequantize q4_0, q4_1, q5_0 and q5_1 KV to f16 before flash attention The dequant pass now covers all quantized KV types supported by the Metal flash attention kernels. The dequant kernel, kargs, scratch allocation and dispatch are type-generic, so each type is one kernel instantiation plus one gate case. Assisted-by: pi:llama.cpp/Qwen3.8-27B * metal : skip the redundant V dequant when V is a view of K In MLA-based models, the V of the FA op is a view of K (the first ne20 elements of each K row); the dequantized V is then a view of the dequantized K, so skip the second dequant dispatch, do not reserve the V scratch region, and let the pad and attention kernels read V from the K F16 buffer with K's strides. The detection follows the CUDA backend: V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs)) Also fix the FA pipeline getters: ns10/ns20 are function constants baked into the kernels and must be the actual K/V row widths as seen by the kernel. The dispatch now passes them explicitly (nb11_attn/nb10_attn, nb21_attn/nb20_attn) instead of the getters assuming contiguous F16 KV (ns20 = dv), which was wrong when V is read from K with K's row pitch (e.g. 576 vs 512). New test cases: 576/512 q8_0 (MLA shape, V is a view of K) at kv=113 (KV pad), nb=1 (vec) and nb=64 (non-vec). Assisted-by: pi:llama.cpp/Qwen3.8-27B * test : remove backend-specific wording from test-backend-ops comments Assisted-by: pi:llama.cpp/Qwen3.8-27B * pi : avoid backend mentions in test-backend-ops comments Assisted-by: pi:llama.cpp/Qwen3.8-27B * metal : rename the FA dequant_f16 identifiers to kv_f16 Assisted-by: pi:llama.cpp/Qwen3.8-27B * cont : clean-up * cont : remove TODO --- .pi/gg/SYSTEM.md | 1 + ggml/src/ggml-metal/ggml-metal-device.cpp | 37 ++- ggml/src/ggml-metal/ggml-metal-device.h | 14 +- ggml/src/ggml-metal/ggml-metal-impl.h | 12 + ggml/src/ggml-metal/ggml-metal-ops.cpp | 269 ++++++++++++++++++---- ggml/src/ggml-metal/ggml-metal-ops.h | 1 + ggml/src/ggml-metal/ggml-metal.cpp | 1 + ggml/src/ggml-metal/ggml-metal.metal | 47 ++++ tests/test-backend-ops.cpp | 29 +++ 9 files changed, 358 insertions(+), 53 deletions(-) diff --git a/.pi/gg/SYSTEM.md b/.pi/gg/SYSTEM.md index d39afbe033..6a757c8694 100644 --- a/.pi/gg/SYSTEM.md +++ b/.pi/gg/SYSTEM.md @@ -9,6 +9,7 @@ General: Coding: - When in doubt, always refer to the CONTRIBUTING.md file of the project +- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments - When referencing issues or PRs in comments, use the format: - C/C++ code: `// ref: ` - Other (CMake, etc.): `# ref: ` diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 953c757558..52043696eb 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -1409,6 +1409,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_p return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16( + ggml_metal_library_t lib, + const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + char base[256]; + + snprintf(base, 256, "kernel_flash_attn_ext_kv_%s_f16", ggml_type_name(op->src[1]->type)); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, base); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, base, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk( ggml_metal_library_t lib, const struct ggml_tensor * op, @@ -1460,7 +1477,10 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext( bool has_bias, bool has_scap, bool has_kvpad, - int32_t nsg) { + int32_t nsg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); char base[256]; @@ -1469,15 +1489,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext( const int32_t dk = (int32_t) op->src[1]->ne[0]; const int32_t dv = (int32_t) op->src[2]->ne[0]; - const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0]; - const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0]; + const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type); // do bounds checks for the mask? const bool bc_mask = op->src[3] && (op->src[3]->ne[1] % 8 != 0); snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d", "flash_attn_ext", - ggml_type_name(op->src[1]->type), + type, dk, dv); @@ -1526,7 +1545,10 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v bool has_scap, bool has_kvpad, int32_t nsg, - int32_t nwg) { + int32_t nwg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); char base[256]; @@ -1535,12 +1557,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v const int32_t dk = (int32_t) op->src[1]->ne[0]; const int32_t dv = (int32_t) op->src[2]->ne[0]; - const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0]; - const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0]; + const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type); snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d", "flash_attn_ext_vec", - ggml_type_name(op->src[1]->type), + type, dk, dv); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 7e1deeaa21..b7d4660588 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -176,6 +176,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_mask, int32_t ncpsg); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16( + ggml_metal_library_t lib, + const struct ggml_tensor * op); + struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk( ggml_metal_library_t lib, const struct ggml_tensor * op, @@ -190,7 +194,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_bias, bool has_scap, bool has_kvpad, - int32_t nsg); + int32_t nsg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec( ggml_metal_library_t lib, @@ -201,7 +208,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_scap, bool has_kvpad, int32_t nsg, - int32_t nwg); + int32_t nwg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_reduce( ggml_metal_library_t lib, diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 05ea7470ee..f0b7799791 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -345,6 +345,18 @@ typedef struct { bool inplace; } ggml_metal_kargs_rope; +typedef struct { + int32_t ne0; + int32_t ne1; + int32_t ne2; + int32_t ne3; + uint64_t nb0; + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; + int32_t nblocks; +} ggml_metal_kargs_flash_attn_ext_kv_f16; + typedef struct { int32_t ne11; int32_t ne_12_2; // assume K and V are same shape diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index d8435e9577..2dde14d8dc 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2801,6 +2801,44 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) { return (ne01 < 20) && (ne00 % 32 == 0); } +// ref: https://github.com/ggml-org/llama.cpp/pull/27390 +// dequantize the quantized KV cache to F16 before running the F16 flash attention kernels +static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + switch (op->src[1]->type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + return true; + default: + return false; + } +} + +// in some models (e.g. MLA-based), V is a view of K (the first ne20 elements of each K row); +// the dequantized V is then a view of the dequantized K and does not need its own dequant or scratch +// - ref: https://github.com/ggml-org/llama.cpp/pull/13435 +static bool ggml_metal_op_flash_attn_ext_v_is_view_of_k(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + const ggml_tensor * K = op->src[1]; + const ggml_tensor * V = op->src[2]; + + return V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs)); +} + +// size of the F16 dequantized K tensor; the dequantized V tensor follows it in the same scratch buffer +static size_t ggml_metal_op_flash_attn_ext_kv_f16_k_size(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); + + return GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne10*ne11*ne12*ne13, 16); +} + size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); @@ -2816,6 +2854,18 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { size_t res = 0; const bool has_mask = op->src[3] != nullptr; + const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op); + + // when the KV is dequantized to F16, the pad kernel copies the tail chunk from the F16 scratch buffer + // note: when V is a view of K, the dequantized V is read from the dequantized K with K's row stride + const bool v_is_view_of_k = use_kv_f16 && ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + uint64_t nb11_pad = nb11; + uint64_t nb21_pad = nb21; + + if (use_kv_f16) { + nb11_pad = sizeof(ggml_fp16_t)*ne10; + nb21_pad = sizeof(ggml_fp16_t)*(v_is_view_of_k ? ne10 : ne20); + } // note: the non-vec kernel requires more extra memory, so always reserve for it GGML_ASSERT(OP_FLASH_ATTN_EXT_NCPSG >= OP_FLASH_ATTN_EXT_VEC_NCPSG); @@ -2828,8 +2878,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { if (has_kvpad) { res += OP_FLASH_ATTN_EXT_VEC_NCPSG*( - nb11*ne12*ne13 + - nb21*ne22*ne23 + + nb11_pad*ne12*ne13 + + nb21_pad*ne22*ne23 + (has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0)); } } else { @@ -2838,8 +2888,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { if (has_kvpad) { res += OP_FLASH_ATTN_EXT_NCPSG*( - nb11*ne12*ne13 + - nb21*ne22*ne23 + + nb11_pad*ne12*ne13 + + nb21_pad*ne22*ne23 + (has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0)); } } @@ -2915,6 +2965,28 @@ size_t ggml_metal_op_flash_attn_ext_extra_tmp(const ggml_tensor * op) { return res; } +size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { + return 0; + } + + GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); + + const size_t k_size = ggml_metal_op_flash_attn_ext_kv_f16_k_size(op); + + // when V is a view of K, the dequantized V is a view of the dequantized K + const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + if (v_is_view_of_k) { + return k_size; + } + + const size_t v_size = GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne20*ne21*ne22*ne23, 16); + + return k_size + v_size; +} + int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -2989,6 +3061,111 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_buffer_id bid_tmp = bid_blk; bid_tmp.offs += ggml_metal_op_flash_attn_ext_extra_blk(op); + ggml_metal_buffer_id bid_kv_f16 = bid_tmp; + bid_kv_f16.offs += ggml_metal_op_flash_attn_ext_extra_tmp(op); + + const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op); + + ggml_metal_buffer_id bid_k = bid_src1; + ggml_metal_buffer_id bid_v = bid_src2; + + uint64_t nb10_attn = nb10; + uint64_t nb11_attn = nb11; + uint64_t nb12_attn = nb12; + uint64_t nb13_attn = nb13; + uint64_t nb20_attn = nb20; + uint64_t nb21_attn = nb21; + uint64_t nb22_attn = nb22; + uint64_t nb23_attn = nb23; + + if (use_kv_f16) { + assert(ggml_metal_op_flash_attn_ext_extra_kv_f16(op) != 0); + + const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + + const int64_t nblocks1_64 = (ne10/ggml_blck_size(op->src[1]->type))*(int64_t) ne11*ne12*ne13; + GGML_ASSERT(nblocks1_64 <= INT32_MAX); + const int32_t nblocks1 = nblocks1_64; + + ggml_metal_buffer_id bid_v_f16 = bid_kv_f16; + bid_v_f16.offs += ggml_metal_op_flash_attn_ext_kv_f16_k_size(op); + + auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16(lib, op); + const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline0), 256); + + // K + ggml_metal_kargs_flash_attn_ext_kv_f16 args_k = { + /*.ne0 =*/ ne10, + /*.ne1 =*/ ne11, + /*.ne2 =*/ ne12, + /*.ne3 =*/ ne13, + /*.nb0 =*/ nb10, + /*.nb1 =*/ nb11, + /*.nb2 =*/ nb12, + /*.nb3 =*/ nb13, + /*.nblocks =*/ nblocks1, + }; + + ggml_metal_encoder_set_pipeline(enc, pipeline0); + ggml_metal_encoder_set_bytes (enc, &args_k, sizeof(args_k), 0); + ggml_metal_encoder_set_buffer (enc, bid_src1, 1); + ggml_metal_encoder_set_buffer (enc, bid_kv_f16, 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, (nblocks1 + nth - 1)/nth, 1, 1, nth, 1, 1); + + // V (skip when V is a view of K: the dequantized V is a view of the dequantized K) + if (!v_is_view_of_k) { + const int64_t nblocks2_64 = (ne20/ggml_blck_size(op->src[2]->type))*(int64_t) ne21*ne22*ne23; + GGML_ASSERT(nblocks2_64 <= INT32_MAX); + const int32_t nblocks2 = nblocks2_64; + + ggml_metal_kargs_flash_attn_ext_kv_f16 args_v = { + /*.ne0 =*/ ne20, + /*.ne1 =*/ ne21, + /*.ne2 =*/ ne22, + /*.ne3 =*/ ne23, + /*.nb0 =*/ nb20, + /*.nb1 =*/ nb21, + /*.nb2 =*/ nb22, + /*.nb3 =*/ nb23, + /*.nblocks =*/ nblocks2, + }; + + ggml_metal_encoder_set_pipeline(enc, pipeline0); + ggml_metal_encoder_set_bytes (enc, &args_v, sizeof(args_v), 0); + ggml_metal_encoder_set_buffer (enc, bid_src2, 1); + ggml_metal_encoder_set_buffer (enc, bid_v_f16, 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, (nblocks2 + nth - 1)/nth, 1, 1, nth, 1, 1); + } + + // the pad and attention kernels read the dequantized KV + ggml_metal_op_concurrency_reset(ctx); + + bid_k = bid_kv_f16; + bid_v = v_is_view_of_k ? bid_k : bid_v_f16; + + // contiguous F16 layout of the dequantized K + nb10_attn = sizeof(ggml_fp16_t); + nb11_attn = nb10_attn*ne10; + nb12_attn = nb11_attn*ne11; + nb13_attn = nb12_attn*ne12; + + // if V is a view of K, the dequantized V is read from the dequantized K with K's strides + if (v_is_view_of_k) { + nb20_attn = nb10_attn; + nb21_attn = nb11_attn; + nb22_attn = nb12_attn; + nb23_attn = nb13_attn; + } else { + // contiguous F16 layout of the dequantized V + nb20_attn = sizeof(ggml_fp16_t); + nb21_attn = nb20_attn*ne20; + nb22_attn = nb21_attn*ne21; + nb23_attn = nb22_attn*ne22; + } + } + if (!ggml_metal_op_flash_attn_ext_use_vec(op)) { // half8x8 kernel const int nqptg = OP_FLASH_ATTN_EXT_NQPSG; // queries per threadgroup @@ -3009,12 +3186,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ne11, /*.ne_12_2 =*/ne12, /*.ne_12_3 =*/ne13, - /*.nb11 =*/nb11, - /*.nb12 =*/nb12, - /*.nb13 =*/nb13, - /*.nb21 =*/nb21, - /*.nb22 =*/nb22, - /*.nb23 =*/nb23, + /*.nb11 =*/nb11_attn, + /*.nb12 =*/nb12_attn, + /*.nb13 =*/nb13_attn, + /*.nb21 =*/nb21_attn, + /*.nb22 =*/nb22_attn, + /*.nb23 =*/nb23_attn, /*.ne31 =*/ne31, /*.ne32 =*/ne32, /*.ne33 =*/ne33, @@ -3027,8 +3204,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_pipeline(enc, pipeline0); ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0); - ggml_metal_encoder_set_buffer (enc, bid_src1, 1); - ggml_metal_encoder_set_buffer (enc, bid_src2, 2); + ggml_metal_encoder_set_buffer (enc, bid_k, 1); + ggml_metal_encoder_set_buffer (enc, bid_v, 2); ggml_metal_encoder_set_buffer (enc, bid_src3, 3); ggml_metal_encoder_set_buffer (enc, bid_pad, 4); @@ -3073,7 +3250,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_op_concurrency_reset(ctx); } - const int is_q = ggml_is_quantized(op->src[1]->type) ? 1 : 0; + const int is_q = !use_kv_f16 && ggml_is_quantized(op->src[1]->type) ? 1 : 0; // 2*(2*ncpsg) // ncpsg soft_max values + ncpsg mask values @@ -3104,6 +3281,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { const size_t smem = FATTN_SMEM(nsg); + const int32_t ns10 = nb11_attn/nb10_attn; + const int32_t ns20 = nb21_attn/nb20_attn; + ggml_metal_kargs_flash_attn_ext args = { /*.ne01 =*/ ne01, /*.ne02 =*/ ne02, @@ -3114,14 +3294,14 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ ne11, /*.ne_12_2 =*/ ne12, /*.ne_12_3 =*/ ne13, - /*.ns10 =*/ int32_t(nb11/nb10), - /*.nb11 =*/ nb11, - /*.nb12 =*/ nb12, - /*.nb13 =*/ nb13, - /*.ns20 =*/ int32_t(nb21/nb20), - /*.nb21 =*/ nb21, - /*.nb22 =*/ nb22, - /*.nb23 =*/ nb23, + /*.ns10 =*/ ns10, + /*.nb11 =*/ nb11_attn, + /*.nb12 =*/ nb12_attn, + /*.nb13 =*/ nb13_attn, + /*.ns20 =*/ ns20, + /*.nb21 =*/ nb21_attn, + /*.nb22 =*/ nb22_attn, + /*.nb23 =*/ nb23_attn, /*.ne31 =*/ ne31, /*.ne32 =*/ ne32, /*.ne33 =*/ ne33, @@ -3139,13 +3319,13 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.logit_softcap =*/ logit_softcap, }; - auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg); + auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, use_kv_f16, ns10, ns20); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, bid_src0, 1); - ggml_metal_encoder_set_buffer (enc, bid_src1, 2); - ggml_metal_encoder_set_buffer (enc, bid_src2, 3); + ggml_metal_encoder_set_buffer (enc, bid_k, 2); + ggml_metal_encoder_set_buffer (enc, bid_v, 3); ggml_metal_encoder_set_buffer (enc, bid_src3, 4); ggml_metal_encoder_set_buffer (enc, bid_src4, 5); ggml_metal_encoder_set_buffer (enc, bid_pad, 6); @@ -3177,12 +3357,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ne11, /*.ne_12_2 =*/ne12, /*.ne_12_3 =*/ne13, - /*.nb11 =*/nb11, - /*.nb12 =*/nb12, - /*.nb13 =*/nb13, - /*.nb21 =*/nb21, - /*.nb22 =*/nb22, - /*.nb23 =*/nb23, + /*.nb11 =*/nb11_attn, + /*.nb12 =*/nb12_attn, + /*.nb13 =*/nb13_attn, + /*.nb21 =*/nb21_attn, + /*.nb22 =*/nb22_attn, + /*.nb23 =*/nb23_attn, /*.ne31 =*/ne31, /*.ne32 =*/ne32, /*.ne33 =*/ne33, @@ -3195,8 +3375,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_pipeline(enc, pipeline0); ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0); - ggml_metal_encoder_set_buffer (enc, bid_src1, 1); - ggml_metal_encoder_set_buffer (enc, bid_src2, 2); + ggml_metal_encoder_set_buffer (enc, bid_k, 1); + ggml_metal_encoder_set_buffer (enc, bid_v, 2); ggml_metal_encoder_set_buffer (enc, bid_src3, 3); ggml_metal_encoder_set_buffer (enc, bid_pad, 4); @@ -3242,6 +3422,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { } } + const int32_t ns10 = nb11_attn/nb10_attn; + const int32_t ns20 = nb21_attn/nb20_attn; + ggml_metal_kargs_flash_attn_ext_vec args = { /*.ne01 =*/ ne01, /*.ne02 =*/ ne02, @@ -3252,14 +3435,14 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ ne11, /*.ne_12_2 =*/ ne12, /*.ne_12_3 =*/ ne13, - /*.ns10 =*/ int32_t(nb11/nb10), - /*.nb11 =*/ nb11, - /*.nb12 =*/ nb12, - /*.nb13 =*/ nb13, - /*.ns20 =*/ int32_t(nb21/nb20), - /*.nb21 =*/ nb21, - /*.nb22 =*/ nb22, - /*.nb23 =*/ nb23, + /*.ns10 =*/ ns10, + /*.nb11 =*/ nb11_attn, + /*.nb12 =*/ nb12_attn, + /*.nb13 =*/ nb13_attn, + /*.ns20 =*/ ns20, + /*.nb21 =*/ nb21_attn, + /*.nb22 =*/ nb22_attn, + /*.nb23 =*/ nb23_attn, /*.ne31 =*/ ne31, /*.ne32 =*/ ne32, /*.ne33 =*/ ne33, @@ -3277,15 +3460,15 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.logit_softcap =*/ logit_softcap, }; - auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg); + auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg, use_kv_f16, ns10, ns20); GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, bid_src0, 1); - ggml_metal_encoder_set_buffer (enc, bid_src1, 2); - ggml_metal_encoder_set_buffer (enc, bid_src2, 3); + ggml_metal_encoder_set_buffer (enc, bid_k, 2); + ggml_metal_encoder_set_buffer (enc, bid_v, 3); ggml_metal_encoder_set_buffer (enc, bid_src3, 4); ggml_metal_encoder_set_buffer (enc, bid_src4, 5); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index b03b59e0bd..159a628d04 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -42,6 +42,7 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_pad(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_blk(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_tmp(const struct ggml_tensor * op); +size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const struct ggml_tensor * op); int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx); int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx); diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index ef3c92f271..0e8d409e0b 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -225,6 +225,7 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_ res += ggml_metal_op_flash_attn_ext_extra_pad(tensor); res += ggml_metal_op_flash_attn_ext_extra_blk(tensor); res += ggml_metal_op_flash_attn_ext_extra_tmp(tensor); + res += ggml_metal_op_flash_attn_ext_extra_kv_f16(tensor); } break; case GGML_OP_CUMSUM: case GGML_OP_ARGSORT: diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 0537fa4cf8..949931c8dc 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -6318,6 +6318,53 @@ template [[host_name("kernel_fwht_f32_128")]] kernel kernel_fwht_t kernel_fwht_f template [[host_name("kernel_fwht_f32_256")]] kernel kernel_fwht_t kernel_fwht_f32<256>; template [[host_name("kernel_fwht_f32_512")]] kernel kernel_fwht_t kernel_fwht_f32<512>; +// dequantize a quantized KV cache tensor to contiguous F16 before running the F16 flash attention kernels +// - one thread per block; dispatched separately for K and V +// - ref: https://github.com/ggml-org/llama.cpp/pull/27390 +template < + typename block_t, + short QK, + void (*deq_t4x4)(device const block_t *, short, thread float4x4 &)> +kernel void kernel_flash_attn_ext_kv_f16( + constant ggml_metal_kargs_flash_attn_ext_kv_f16 & args, + device const char * x, + device half * x_dst, + uint gid [[thread_position_in_grid]]) { + if (gid >= (uint) args.nblocks) { + return; + } + + const uint nb = args.ne0/QK; + const uint i0 = gid%nb; + uint ib = gid/nb; + const uint i1 = ib%args.ne1; + ib /= args.ne1; + const uint i2 = ib%args.ne2; + const uint i3 = ib/args.ne2; + + const uint64_t offs = i0*args.nb0 + i1*args.nb1 + i2*args.nb2 + i3*args.nb3; + + device const block_t * src = (device const block_t *) (x + offs); + device half4 * dst = (device half4 *) x_dst + (QK/4)*gid; + + for (short i = 0; i < QK/16; ++i) { + float4x4 reg; + deq_t4x4(src, i, reg); + dst[4*i + 0] = (half4) reg[0]; + dst[4*i + 1] = (half4) reg[1]; + dst[4*i + 2] = (half4) reg[2]; + dst[4*i + 3] = (half4) reg[3]; + } +} + +typedef decltype(kernel_flash_attn_ext_kv_f16) kernel_flash_attn_ext_kv_f16_t; + +template [[host_name("kernel_flash_attn_ext_kv_q4_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q4_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q5_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q5_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q8_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; + constant bool FC_flash_attn_ext_pad_has_mask [[function_constant(FC_FLASH_ATTN_EXT_PAD + 0)]]; constant int32_t FC_flash_attn_ext_pad_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_PAD + 25)]]; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c9946c7ae4..17098825bc 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9934,6 +9934,20 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q2_0)); test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_F16)); + // q8_0 KV cases: decode and prompt batches, KV pad, permuted KV, feature flags, and long context + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3})); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 2}, 1025, 1, true, true, 8, 30, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1025, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3})); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 16384, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + + // MLA shape (V is a view of K) with quantized KV + // (the test harness builds V as a view of K for this shape; see build_graph) + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + // large-KV F16 cases (Qwen3.6-27B geometry and a llama-class control): the upstream matrix // stops at kv=1024, blind to long-context FA bugs (e.g. the oneDNN SDPA ordering race on BMG). for (int64_t kv : { 4096, 16384 }) { @@ -10325,6 +10339,21 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + // q8_0 KV cases with long context (decode and prompt) + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 128, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 2048, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + for (int kv : { 4096, 8192, 16384, }) { for (int hs : { 64, 128, }) { for (int nr : { 1, 4, }) { From dc64a1620e8ae6f01fb33f09470b7160f69fd0f2 Mon Sep 17 00:00:00 2001 From: Aldehir Rojas Date: Thu, 20 Aug 2026 06:59:03 -0500 Subject: [PATCH 24/36] common : gracefully fallback on unsupported regex patterns in JSON schema (#26939) --- common/json-schema-to-grammar.cpp | 142 +++++++++++++++++++------- tests/test-json-schema-to-grammar.cpp | 64 ++++++++++++ 2 files changed, 171 insertions(+), 35 deletions(-) diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index b18607cd65..955b4e014b 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -278,7 +278,9 @@ static std::unordered_map GRAMMAR_LITERAL_ESCAPES = { {'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}, {'\\', "\\\\"} }; -static std::unordered_set NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'}; +static const int MAX_PATTERN_DEPTH = 100; + +static std::unordered_set NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?', '^', '$'}; static std::unordered_set ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'^', '$', '.', '[', ']', '(', ')', '|', '{', '}', '*', '+', '?'}; static std::string replacePattern(const std::string & input, const std::regex & regex, const std::function & replacement) { @@ -309,6 +311,32 @@ static std::string format_literal(const std::string & literal) { std::string gbnf_format_literal(const std::string & literal) { return format_literal(literal); } +static size_t gbnf_escape_length(const std::string & pattern, size_t pos) { + if (pos + 1 >= pattern.length() || pattern[pos] != '\\') { + return 0; + } + size_t n_hex = 0; + switch (pattern[pos + 1]) { + case 'x': n_hex = 2; break; + case 'u': n_hex = 4; break; + case 'U': n_hex = 8; break; + case 't': case 'r': case 'n': case '\\': case '"': case '[': case ']': + return 2; + default: + return 0; + } + if (pos + 2 + n_hex > pattern.length()) { + return 0; + } + for (size_t i = pos + 2; i < pos + 2 + n_hex; i++) { + char h = pattern[i]; + if (!((h >= '0' && h <= '9') || (h >= 'a' && h <= 'f') || (h >= 'A' && h <= 'F'))) { + return 0; + } + } + return 2 + n_hex; +} + class common_schema_converter { private: friend class common_schema_info; @@ -345,16 +373,42 @@ private: return string_join(rules, " | "); } + // thrown when the pattern is a valid regex with no grammar equivalent + struct unsupported_pattern : public std::runtime_error { + using std::runtime_error::runtime_error; + }; + + // thrown when the pattern is not a valid regex + struct invalid_pattern : public std::runtime_error { + using std::runtime_error::runtime_error; + }; + std::string _visit_pattern(const std::string & pattern, const std::string & name) { - if (!(pattern.front() == '^' && pattern.back() == '$')) { - _errors.push_back("Pattern must start with '^' and end with '$'"); + auto rules_snapshot = _rules; + try { + return _pattern_to_rule(pattern, name); + } catch (const unsupported_pattern & err) { + // revert rules + _rules = std::move(rules_snapshot); + _warnings.push_back("pattern " + pattern + " is not supported (" + err.what() + "), accepting any string"); + return _add_rule(name, _add_primitive("string", PRIMITIVE_RULES.at("string"))); + } catch (const invalid_pattern & err) { + _rules = std::move(rules_snapshot); + _errors.push_back("Invalid pattern " + pattern + ": " + err.what()); return ""; } + } + + std::string _pattern_to_rule(const std::string & pattern, const std::string & name) { + if (pattern.length() < 2 || pattern.front() != '^' || pattern.back() != '$') { + throw unsupported_pattern("not anchored with '^' and '$'"); + } std::string sub_pattern = pattern.substr(1, pattern.length() - 2); std::unordered_map sub_rule_ids; size_t i = 0; size_t length = sub_pattern.length(); + int paren_depth = 0; using literal_or_rule = std::pair; auto to_rule = [&](const literal_or_rule & ls) { @@ -363,7 +417,6 @@ private: return is_literal ? "\"" + s + "\"" : s; }; std::function transform = [&]() -> literal_or_rule { - size_t start = i; std::vector seq; auto get_dot = [&]() { @@ -420,43 +473,42 @@ private: if (i + 1 < length && sub_pattern[i + 1] == ':') { i += 2; // skip "?:" for non-capturing group, treat as regular group } else { - // lookahead/lookbehind (?=, ?!, ?<=, ? 0) { - if (sub_pattern[i] == '\\' && i + 1 < length) { - i += 2; // skip escaped character - } else { - if (sub_pattern[i] == '(') depth++; - else if (sub_pattern[i] == ')') depth--; - i++; - } - } - continue; + // lookaround, named group, inline flags, ... + throw unsupported_pattern("unsupported group syntax"); } } + paren_depth++; + if (paren_depth > MAX_PATTERN_DEPTH) { + throw unsupported_pattern("pattern nesting too deep"); + } seq.emplace_back("(" + to_rule(transform()) + ")", false); } else if (c == ')') { i++; - if (start > 0 && sub_pattern[start - 1] != '(' && (start < 2 || sub_pattern[start - 2] != '?' || sub_pattern[start - 1] != ':')) { - _errors.push_back("Unbalanced parentheses"); + if (paren_depth == 0) { + throw invalid_pattern("unbalanced parentheses"); } + paren_depth--; return join_seq(); + } else if (c == '^' || c == '$') { + throw unsupported_pattern("anchor inside the pattern"); } else if (c == '[') { std::string square_brackets = std::string(1, c); i++; while (i < length && sub_pattern[i] != ']') { if (sub_pattern[i] == '\\') { - square_brackets += sub_pattern.substr(i, 2); - i += 2; + auto escape_length = gbnf_escape_length(sub_pattern, i); + if (escape_length == 0) { + throw unsupported_pattern("unsupported escape in character class: " + sub_pattern.substr(i, 2)); + } + square_brackets += sub_pattern.substr(i, escape_length); + i += escape_length; } else { square_brackets += sub_pattern[i]; i++; } } if (i >= length) { - _errors.push_back("Unbalanced square brackets"); + throw invalid_pattern("unterminated character class"); } square_brackets += ']'; i++; @@ -465,6 +517,9 @@ private: seq.emplace_back("|", false); i++; } else if (c == '*' || c == '+' || c == '?') { + if (seq.empty()) { + throw invalid_pattern("nothing to repeat"); + } seq.back() = std::make_pair(to_rule(seq.back()) + c, false); i++; } else if (c == '{') { @@ -475,18 +530,19 @@ private: i++; } if (i >= length) { - _errors.push_back("Unbalanced curly brackets"); + throw unsupported_pattern("unterminated curly brackets"); } curly_brackets += '}'; i++; auto nums = string_split(curly_brackets.substr(1, curly_brackets.length() - 2), ","); int min_times = 0; int max_times = std::numeric_limits::max(); + if (nums.size() != 1 && nums.size() != 2) { + throw unsupported_pattern("wrong number of values in curly brackets"); + } try { if (nums.size() == 1) { min_times = max_times = std::stoi(nums[0]); - } else if (nums.size() != 2) { - _errors.push_back("Wrong number of values in curly brackets"); } else { if (!nums[0].empty()) { min_times = std::stoi(nums[0]); @@ -495,9 +551,11 @@ private: max_times = std::stoi(nums[1]); } } - } catch (const std::invalid_argument & e) { - _errors.push_back("Invalid number in curly brackets"); - return std::make_pair("", false); + } catch (const std::logic_error &) { + throw unsupported_pattern("invalid number in curly brackets"); + } + if (seq.empty()) { + throw invalid_pattern("nothing to repeat"); } auto &last = seq.back(); auto &sub = last.first; @@ -523,15 +581,22 @@ private: return NON_LITERAL_SET.find(c) != NON_LITERAL_SET.end(); }; while (i < length) { - if (sub_pattern[i] == '\\' && i < length - 1) { + if (sub_pattern[i] == '\\') { + if (i == length - 1) { + throw invalid_pattern("trailing backslash"); + } char next = sub_pattern[i + 1]; if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.find(next) != ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.end()) { i++; literal += sub_pattern[i]; i++; } else { - literal += sub_pattern.substr(i, 2); - i += 2; + auto escape_length = gbnf_escape_length(sub_pattern, i); + if (escape_length == 0) { + throw unsupported_pattern("unsupported escape: " + sub_pattern.substr(i, 2)); + } + literal += sub_pattern.substr(i, escape_length); + i += escape_length; } } else if (sub_pattern[i] == '"') { literal += "\\\""; @@ -544,14 +609,21 @@ private: break; } } - if (!literal.empty()) { - seq.emplace_back(literal, true); + if (literal.empty()) { // nothing was consumed, ex. a stray ']' or '}' + throw unsupported_pattern(std::string("unsupported character: ") + c); } + seq.emplace_back(literal, true); } } return join_seq(); }; - return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\""); + + auto rule = to_rule(transform()); + if (paren_depth != 0) { + throw invalid_pattern("unbalanced parentheses"); + } + + return _add_rule(name, "\"\\\"\" (" + rule + ") \"\\\"\""); } /* diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index f095274cd1..74b57cf1b6 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -1564,6 +1564,70 @@ int main() { space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", }); + + run({ + SUCCESS, + "unanchored regexp", + R"""({ + "type": "string", + "pattern": "[0-9]+" + })""", + R"""( + char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) + root ::= string + space ::= | " " | "\n"{1,2} [ \t]{0,20} + string ::= "\"" char* "\"" + )""", + }); + + // the rules of the partial conversion (here "root-0") must not leak into the grammar + run({ + SUCCESS, + "regexp with unsupported shorthand", + R"""({ + "type": "string", + "pattern": "^[0-9]{3}\\w$" + })""", + R"""( + char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) + root ::= string + space ::= | " " | "\n"{1,2} [ \t]{0,20} + string ::= "\"" char* "\"" + )""", + }); + + // a regexp that is invalid under any flavor is still an error + run({ + FAILURE, + "regexp with unbalanced parentheses", + R"""({ + "type": "string", + "pattern": "^(a$" + })""", + "" + }); + + // only the property with the bad pattern degrades + run({ + SUCCESS, + "unsupported regexp in a property", + R"""({ + "type": "object", + "properties": { + "a": { "type": "string", "pattern": "^[a-z\\-]+$" } + }, + "required": ["a"], + "additionalProperties": false + })""", + R"""( + a ::= string + a-kv ::= "\"a\"" space ":" space a + char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) + root ::= "{" space a-kv space "}" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + string ::= "\"" char* "\"" + )""", + }); } if (getenv("LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR")) { From 2b5621094ef383cdcd8428ef6d22efe5df976532 Mon Sep 17 00:00:00 2001 From: Pranesh Gonegandla Date: Thu, 20 Aug 2026 12:36:21 +0000 Subject: [PATCH 25/36] CUDA: adding switch points per HW and quant type to tune the mvq->MMQ decode crossover (#26079) * CUDA: runtime GGML_CUDA_MMVQ_MAX to tune the mvq->MMQ decode crossover Add a runtime override of the mul_mat_vec_q -> MMQ batch crossover (default MMVQ_MAX_BATCH_SIZE). Lowering it routes batches above the threshold from the CUDA-core vector kernel to the int8 MMQ tensor-core path, which is faster once quantized decode becomes compute-bound at B>1 (measured +23-41% at B=8 on RTX 5090 for Q4_K dense, no low-batch loss). The value is parsed once and clamped to [1, MMVQ_MAX_BATCH_SIZE], since mul_mat_vec_q asserts ncols_dst <= that; invalid input warns and falls back to the default. The override is applied consistently in both the mul_mat_vec_q and MUL_MAT_ID dispatch paths. Default behavior unchanged. * Added Blackwell specific switch point, to reduce dependence on runtime env var. * Add per-HW switch point values for DGX Spark and removing runtime env var * Adding switch points for Ada, tested on RTX 4090 * Modifying DGX Spark numbers based on latest run and adding some comments and small functional changes relating to MoE * Reverting an unnecessary conditional * Update ggml/src/ggml-cuda/mmvq.cu --------- Co-authored-by: praneshgo <227579474+praneshgo@users.noreply.github.com> Co-authored-by: Oliver Simons --- ggml/src/ggml-cuda/mmvq.cu | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index c999238045..9705348098 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -290,6 +290,42 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { if (!ggml_is_quantized(type)) { return false; } + // k-quants cost more to decode and mvq redoes that per column, so MMQ wins sooner. + // Only list quant-types MMQ supports, others would fall back to cuBLAS. + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ADA_LOVELACE) { + switch (type) { // tuned on RTX 4090 + case GGML_TYPE_Q2_K: + return ne11 <= 4; + case GGML_TYPE_Q3_K: + return ne11 <= 6; + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return ne11 <= 7; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_BLACKWELL) { + switch (type) { // tuned on RTX 5090 + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return ne11 <= 5; + case GGML_TYPE_Q6_K: + return ne11 <= 7; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) { + switch (type) { // tuned on DGX Spark GB10 + case GGML_TYPE_Q2_K: + return ne11 <= 6; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } if (GGML_CUDA_CC_IS_CDNA(cc)) { if (GGML_CUDA_CC_IS_CDNA1(cc)) { switch (type) { From 8a832e4bf3284ec145bace0cbb8991bd97e1e144 Mon Sep 17 00:00:00 2001 From: Aritro Bandyopadhyay <71339004+AriBandyo@users.noreply.github.com> Date: Thu, 20 Aug 2026 06:37:14 -0600 Subject: [PATCH 26/36] server : fix --docker-repo being treated as router mode (#27416) --- tools/server/server.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 01cc6633a3..230b578e07 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -133,7 +133,8 @@ int llama_server(common_params & params, int argc, char ** argv) { // router server never loads a model and must not touch the GPU const bool is_router_server = params.model.path.empty() - && params.model.hf_repo.empty(); + && params.model.hf_repo.empty() + && params.model.docker_repo.empty(); // skip device enumeration so the CUDA primary context stays uncreated common_params_print_info(params, !is_router_server); From 9855ad69d38c6b8ca9e1e552646ea2566fd932d9 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Thu, 20 Aug 2026 15:22:16 +0200 Subject: [PATCH 27/36] server: (router) lazy-load startup_models after main setup (#27424) * server: (router) lazy-load startup_models after main setup * only allow is_first_load to populate it * nits * nits 2 --- tools/server/README.md | 2 +- tools/server/server-models.cpp | 70 +++++++++++++++++----------------- tools/server/server-models.h | 7 ++++ tools/server/server.cpp | 12 ++++++ 4 files changed, 55 insertions(+), 36 deletions(-) diff --git a/tools/server/README.md b/tools/server/README.md index b63a0e6dac..f5d747eee8 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1757,7 +1757,7 @@ The precedence rule for preset options is as follows: 3. **Global options** defined in the preset file (`[*]`) We also offer additional options that are exclusive to presets (these aren't treated as command-line arguments): -- `load-on-startup` (boolean): Controls whether the model loads automatically when the server starts +- `load-on-startup` (boolean): Controls whether the model loads automatically when the server starts. Only applies at startup: if the model list is reloaded later (for example after editing the preset file), a newly added model is listed but not loaded - `stop-timeout` (int, seconds): After requested unload, wait for this many seconds before forcing termination (default: 10) - `dedup-cache-models` (boolean): When the preset uses `hf-repo` pointing to a model that is already downloaded, hide the corresponding cached model entry from `GET /models` (the preset entry remains visible). Set it in the `[*]` section to apply to all presets. diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 35b9355700..d605451941 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -672,24 +672,26 @@ void server_models::load_models() { apply_hidden(); log_available_models(); - std::vector models_to_load; - for (const auto & [name, inst] : mapping) { - std::string val; - if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) { - models_to_load.push_back(name); + // skipped on reload, see startup_models + if (startup_models.has_value()) { + std::vector models_to_load; + for (const auto & [name, inst] : mapping) { + std::string val; + if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) { + models_to_load.push_back(name); + } } - } - if ((int)models_to_load.size() > base_params.models_max) { - throw std::runtime_error(string_format( - "number of models to load on startup (%zu) exceeds models_max (%d)", - models_to_load.size(), base_params.models_max)); + if ((int)models_to_load.size() > base_params.models_max) { + throw std::runtime_error(string_format( + "number of models to load on startup (%zu) exceeds models_max (%d)", + models_to_load.size(), base_params.models_max)); + } + + // to be lazy-loaded after main() setup phase is completed + startup_models = std::move(models_to_load); } lk.unlock(); - for (const auto & name : models_to_load) { - SRV_INF("(startup) loading model %s\n", name.c_str()); - load(name); - } } else { // RELOAD: diff the new preset list against the current mapping and reconcile is_reloading = true; @@ -819,8 +821,8 @@ void server_models::load_models() { inst.meta.update_caps(); } - // add models that are new in this reload - std::vector newly_added; + // add models that are new in this reload, load-on-startup is not honored here since a + // reload never spawns an instance for (const auto & [name, preset] : final_presets) { if (mapping.find(name) == mapping.end()) { server_model_meta meta{ @@ -841,42 +843,40 @@ void server_models::load_models() { // /* need_download */ false, }; add_model(std::move(meta)); - newly_added.push_back(name); } } apply_stop_timeout(); apply_hidden(); - // clear reload flag before unlocking for autoload - load() blocks on !is_reloading, - // so clearing it here (while still locked) prevents a deadlock in the autoload calls below + // clear reload flag under the lock, this releases the load() calls waiting on !is_reloading is_reloading = false; cv.notify_all(); log_available_models(); - // collect autoload candidates while still under the lock - std::vector to_autoload; - for (const auto & name : newly_added) { - auto it = mapping.find(name); - if (it != mapping.end()) { - std::string val; - if (it->second.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) { - to_autoload.push_back(name); - } - } - } - lk.unlock(); - for (const auto & name : to_autoload) { - SRV_INF("(reload) loading new model %s\n", name.c_str()); - load(name); - } notify_sse("models_reload", "*"); } } +void server_models::load_startup_models() { + std::vector to_load; + { + std::lock_guard lk(mutex); + if (!startup_models.has_value()) { + return; // already drained + } + to_load = std::move(*startup_models); + startup_models.reset(); + } + for (const auto & name : to_load) { + SRV_INF("(startup) loading model %s\n", name.c_str()); + load(name); + } +} + void server_models::update_meta(const std::string & name, const server_model_meta & meta) { std::lock_guard lk(mutex); auto it = mapping.find(name); diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 79b231cba8..5cbb6a801e 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -136,6 +136,10 @@ private: // if true, the next get_meta() will trigger a reload of model list bool need_reload = false; + // models marked with load-on-startup, unset once load_startup_models() drains it + // no value means the startup phase is over, so a reload must not queue anything + std::optional> startup_models{std::in_place}; + // conv_id -> model name that currently serves its stream session, lets the resumable stream // routes go straight to the owning child instead of polling every one. populated when // proxy_request forwards a POST carrying an X-Conversation-Id. best effort: a stale entry just @@ -231,6 +235,9 @@ public: // - if a model is not running, it will be added or updated according to the source void load_models(); + // lazy-load startup_models, to be called after main() setup phase + void load_startup_models(); + // check if a model instance exists (thread-safe) bool has_model(const std::string & name); diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 230b578e07..5fe2729ba1 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -424,6 +424,18 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_http.stop(); }; + try { + models_routes->models.load_startup_models(); + } catch (const std::exception & e) { + SRV_ERR("failed to load models on startup: %s\n", e.what()); + ctx_http.stop(); + if (ctx_http.thread.joinable()) { + ctx_http.thread.join(); + } + clean_up(); + return 1; + } + } else { // setup clean up function, to be called before exit clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() { From bf0040e15fd5b716262658f4d652c9cee959cf91 Mon Sep 17 00:00:00 2001 From: Oliver Simons Date: Thu, 20 Aug 2026 15:42:26 +0200 Subject: [PATCH 28/36] CI: Use LLVM's OpenMP over MSVC_DEBUG_non_redist on Windows (#26678) * CI: Use LLVM's OpenMP over MSFT_DEBUG_non_redist on Windows Currently, we ship the non-redist debug version of microsoft's libomp. This PR changes this to official LLVM's release, also packaging the license as needed. * Remove LLVM SHA from job name to increase legibility * Add temp validations to CI * Revert "Add temp validations to CI" This reverts commit eef97c88b5bac280803ebb3c3b7bb09f89b0fd88. * Build OpenMP in CI * Make OpenMP fetch self-contained in cmake and cache in CI * Robustify Licens-packaging 1. Ship OpenMP license, not LLVM's. 2. Invalidate cache also on checksum of the license * Remove stale reference in docs/build.md * No longer package base license in release This was scope-creep * Add explanatory comment to OpenMP license * Remove arm64 smoke Forgot this during conflict resolution during rebase of c54c0e9cf6030a5a54ce8bdd81b3e146d9787d42 * Remove GGML_OPENMP_FETCH_CACHE_DIR as requested by @CISC * whitespace changes --- .github/workflows/build-cpu.yml | 5 +- .github/workflows/release.yml | 3 +- cmake/arm64-windows-llvm.cmake | 1 + docs/build.md | 3 +- ggml/CMakeLists.txt | 1 + ggml/src/CMakeLists.txt | 118 +++++++++++++++++++++++++++- ggml/src/ggml-cpu/CMakeLists.txt | 2 +- ggml/src/ggml-zendnn/CMakeLists.txt | 4 +- 8 files changed, 128 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml index 2016a57f87..a63ffe94b4 100644 --- a/.github/workflows/build-cpu.yml +++ b/.github/workflows/build-cpu.yml @@ -119,6 +119,7 @@ jobs: ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256 windows: + name: windows / ${{ matrix.build }} runs-on: windows-2025 env: @@ -130,13 +131,13 @@ jobs: include: - build: 'x64-cpu-static' arch: 'x64' - defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF' + defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF' - build: 'x64-openblas' arch: 'x64' defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"' - build: 'arm64' arch: 'arm64' - defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON' + defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON' steps: - name: Clone diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 20c9690869..61b2f5485d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -681,6 +681,7 @@ jobs: name: llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip windows-cpu: + name: windows-cpu / ${{ matrix.arch }} needs: [check-release] if: ${{ needs.check-release.outputs.should_release == 'true' }} @@ -728,6 +729,7 @@ jobs: -DGGML_BACKEND_DL=ON ^ -DGGML_CPU_ALL_VARIANTS=${{ matrix.arch == 'x64' && 'ON' || 'OFF' }} ^ -DGGML_OPENMP=ON ^ + -DGGML_OPENMP_FETCH=ON ^ ${{ env.CMAKE_ARGS }} cmake --build build --config Release @@ -739,7 +741,6 @@ jobs: - name: Pack artifacts id: pack_artifacts run: | - Copy-Item "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Redist\MSVC\14.51.36231\debug_nonredist\${{ matrix.arch }}\Microsoft.VC145.OpenMP.LLVM\libomp140.${{ matrix.arch == 'x64' && 'x86_64' || 'aarch64' }}.dll" .\build\bin\Release\ 7z a -snl llama-bin-win-cpu-${{ matrix.arch }}.zip .\build\bin\Release\* - name: Upload artifacts diff --git a/cmake/arm64-windows-llvm.cmake b/cmake/arm64-windows-llvm.cmake index 8023796800..cdba4e7494 100644 --- a/cmake/arm64-windows-llvm.cmake +++ b/cmake/arm64-windows-llvm.cmake @@ -8,6 +8,7 @@ set( CMAKE_CXX_COMPILER clang++ ) set( CMAKE_C_COMPILER_TARGET ${target} ) set( CMAKE_CXX_COMPILER_TARGET ${target} ) +set( CMAKE_ASM_COMPILER_TARGET ${target} ) set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" ) set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" ) diff --git a/docs/build.md b/docs/build.md index ca086a0be1..45fe7f17a2 100644 --- a/docs/build.md +++ b/docs/build.md @@ -72,9 +72,10 @@ cmake --build build --config Release - Please remember to always use a Developer Command Prompt / PowerShell for VS2022 for git, build, test - For Windows on ARM (arm64, WoA) build with: ```bash - cmake --preset arm64-windows-llvm-release -D GGML_OPENMP=OFF + cmake --preset arm64-windows-llvm-release -D GGML_OPENMP_FETCH=ON cmake --build build-arm64-windows-llvm-release ``` + `GGML_OPENMP_FETCH` downloads the official LLVM OpenMP runtime and requires Clang, 7-Zip and network access during configuration. CMake selects the runtime from the target architecture, so this also works when cross-compiling for WoA from x64. The extracted header, import library, DLL and OpenMP license are placed under `build/_deps`. The build copies `libomp.dll` and `LICENSE-LLVM-OpenMP` to the runtime output directory and installs them together. Omit the option to use CMake's normal OpenMP detection, or pass `-D GGML_OPENMP=OFF` to disable OpenMP. For building with ninja generator and clang compiler as default: -set path:set LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\x64;C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.41.34120\lib\x64\uwp;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\x64 ```bash diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index b7110fa12b..9d807d5c83 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -243,6 +243,7 @@ set (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING "ggml: metal minimum macOS version") set (GGML_METAL_STD "" CACHE STRING "ggml: metal standard version (-std flag)") option(GGML_OPENMP "ggml: use OpenMP" ON) +option(GGML_OPENMP_FETCH "ggml: fetch LLVM OpenMP" OFF) option(GGML_RPC "ggml: use RPC" OFF) option(GGML_SYCL "ggml: use SYCL" OFF) option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF) diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index 82e9480c2f..96535b49fa 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -222,9 +222,123 @@ if (GGML_SCHED_NO_REALLOC) target_compile_definitions(ggml-base PUBLIC GGML_SCHED_NO_REALLOC) endif() -if (GGML_OPENMP) +if (GGML_OPENMP_FETCH) + if (NOT GGML_OPENMP) + message(FATAL_ERROR "GGML_OPENMP_FETCH requires GGML_OPENMP") + elseif (NOT WIN32 OR NOT (CMAKE_C_COMPILER_ID MATCHES "Clang")) + message(FATAL_ERROR "GGML_OPENMP_FETCH currently requires Clang on Windows") + endif() + + set(GGML_OPENMP_LLVM_VERSION "20.1.8") + string(REGEX MATCH "^[0-9]+" GGML_OPENMP_LLVM_VERSION_MAJOR "${GGML_OPENMP_LLVM_VERSION}") + string(REGEX MATCH "^[0-9]+" GGML_OPENMP_COMPILER_VERSION_MAJOR "${CMAKE_C_COMPILER_VERSION}") + if (NOT GGML_OPENMP_COMPILER_VERSION_MAJOR STREQUAL GGML_OPENMP_LLVM_VERSION_MAJOR) + message(FATAL_ERROR "LLVM OpenMP ${GGML_OPENMP_LLVM_VERSION} requires Clang ${GGML_OPENMP_LLVM_VERSION_MAJOR}.x") + endif() + + string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" GGML_OPENMP_SYSTEM_PROCESSOR) + if (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(amd64|x86_64)$") + set(GGML_OPENMP_ARCH "x64") + set(GGML_OPENMP_INSTALLER_SUFFIX "win64") + set(GGML_OPENMP_INSTALLER_SHA256 "3197846a2b19063687dd56e93e34cd941e3548d907f23a6131571321bdf9fe7b") + elseif (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") + set(GGML_OPENMP_ARCH "arm64") + set(GGML_OPENMP_INSTALLER_SUFFIX "woa64") + set(GGML_OPENMP_INSTALLER_SHA256 "7c4ac97eb2ae6b960ca5f9caf3ff6124c8d2a18cc07a7840a4d2ea15537bad8e") + else() + message(FATAL_ERROR "GGML_OPENMP_FETCH does not support ${CMAKE_SYSTEM_PROCESSOR}") + endif() + + set(GGML_OPENMP_CACHE_DIR "${CMAKE_BINARY_DIR}/_deps") + set(GGML_OPENMP_ROOT "${GGML_OPENMP_CACHE_DIR}/llvm-openmp-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_ARCH}") + set(GGML_OPENMP_LIBRARY "${GGML_OPENMP_ROOT}/lib/libomp.lib") + set(GGML_OPENMP_RUNTIME "${GGML_OPENMP_ROOT}/bin/libomp.dll") + set(GGML_OPENMP_HEADER "${GGML_OPENMP_ROOT}/include/omp.h") + set(GGML_OPENMP_LICENSE "${GGML_OPENMP_ROOT}/LICENSE.TXT") + set(GGML_OPENMP_LICENSE_SHA256 "fdad1758a9e1f9d5a81e18879b3406772115edc92c24bfa36b70c654f325e8e4") + + if (NOT EXISTS "${GGML_OPENMP_LIBRARY}" OR NOT EXISTS "${GGML_OPENMP_RUNTIME}" OR NOT EXISTS "${GGML_OPENMP_HEADER}") + find_program(GGML_OPENMP_7Z NAMES 7z 7zz 7za) + if (NOT GGML_OPENMP_7Z) + message(FATAL_ERROR "GGML_OPENMP_FETCH requires 7-Zip to extract the LLVM installer") + endif() + + set(GGML_OPENMP_INSTALLER "${GGML_OPENMP_ROOT}/LLVM-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_INSTALLER_SUFFIX}.exe") + set(GGML_OPENMP_EXTRACT_DIR "${GGML_OPENMP_ROOT}/extract") + set(GGML_OPENMP_INSTALLER_URL "https://github.com/llvm/llvm-project/releases/download/llvmorg-${GGML_OPENMP_LLVM_VERSION}/LLVM-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_INSTALLER_SUFFIX}.exe") + + file(MAKE_DIRECTORY "${GGML_OPENMP_EXTRACT_DIR}") + file(DOWNLOAD "${GGML_OPENMP_INSTALLER_URL}" "${GGML_OPENMP_INSTALLER}" + EXPECTED_HASH "SHA256=${GGML_OPENMP_INSTALLER_SHA256}" + SHOW_PROGRESS + STATUS GGML_OPENMP_DOWNLOAD_STATUS) + list(GET GGML_OPENMP_DOWNLOAD_STATUS 0 GGML_OPENMP_DOWNLOAD_RESULT) + if (NOT GGML_OPENMP_DOWNLOAD_RESULT EQUAL 0) + list(GET GGML_OPENMP_DOWNLOAD_STATUS 1 GGML_OPENMP_DOWNLOAD_ERROR) + message(FATAL_ERROR "Failed to download LLVM OpenMP: ${GGML_OPENMP_DOWNLOAD_ERROR}") + endif() + + execute_process( + COMMAND "${GGML_OPENMP_7Z}" e -y "-o${GGML_OPENMP_EXTRACT_DIR}" "${GGML_OPENMP_INSTALLER}" -r libomp.lib libomp.dll omp.h + RESULT_VARIABLE GGML_OPENMP_EXTRACT_RESULT + OUTPUT_QUIET) + if (NOT GGML_OPENMP_EXTRACT_RESULT EQUAL 0 OR + NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/libomp.lib" OR + NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/libomp.dll" OR + NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/omp.h") + message(FATAL_ERROR "Failed to extract libomp from ${GGML_OPENMP_INSTALLER}") + endif() + + file(MAKE_DIRECTORY "${GGML_OPENMP_ROOT}/lib" "${GGML_OPENMP_ROOT}/bin" "${GGML_OPENMP_ROOT}/include") + file(COPY "${GGML_OPENMP_EXTRACT_DIR}/libomp.lib" DESTINATION "${GGML_OPENMP_ROOT}/lib") + file(COPY "${GGML_OPENMP_EXTRACT_DIR}/libomp.dll" DESTINATION "${GGML_OPENMP_ROOT}/bin") + file(COPY "${GGML_OPENMP_EXTRACT_DIR}/omp.h" DESTINATION "${GGML_OPENMP_ROOT}/include") + file(REMOVE_RECURSE "${GGML_OPENMP_INSTALLER}" "${GGML_OPENMP_EXTRACT_DIR}") + endif() + + # The NSIS installer embeds LLVM's general license in its UI but does not install it as a file; use OpenMP's license to include its additional notices. + if (EXISTS "${GGML_OPENMP_LICENSE}") + file(SHA256 "${GGML_OPENMP_LICENSE}" GGML_OPENMP_LICENSE_ACTUAL_SHA256) + endif() + if (NOT GGML_OPENMP_LICENSE_ACTUAL_SHA256 STREQUAL GGML_OPENMP_LICENSE_SHA256) + file(DOWNLOAD "https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-${GGML_OPENMP_LLVM_VERSION}/openmp/LICENSE.TXT" "${GGML_OPENMP_LICENSE}" + EXPECTED_HASH "SHA256=${GGML_OPENMP_LICENSE_SHA256}") + endif() + + if (COMMAND license_add_file) + license_add_file("LLVM OpenMP" "${GGML_OPENMP_LICENSE}") + endif() + + add_library(ggml-openmp-c INTERFACE) + target_compile_options(ggml-openmp-c INTERFACE "$<$:-fopenmp=libomp>") + target_include_directories(ggml-openmp-c SYSTEM INTERFACE "${GGML_OPENMP_ROOT}/include") + target_link_libraries(ggml-openmp-c INTERFACE "${GGML_OPENMP_LIBRARY}") + + add_library(ggml-openmp-cxx INTERFACE) + target_compile_options(ggml-openmp-cxx INTERFACE "$<$:-fopenmp=libomp>") + target_include_directories(ggml-openmp-cxx SYSTEM INTERFACE "${GGML_OPENMP_ROOT}/include") + target_link_libraries(ggml-openmp-cxx INTERFACE "${GGML_OPENMP_LIBRARY}") + + set(GGML_OPENMP_RUNTIME_OUTPUT_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}") + if (CMAKE_CONFIGURATION_TYPES) + string(APPEND GGML_OPENMP_RUNTIME_OUTPUT_DIR "/$") + endif() + add_custom_target(ggml-openmp-runtime ALL + COMMAND ${CMAKE_COMMAND} -E make_directory "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${GGML_OPENMP_RUNTIME}" "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}/libomp.dll" + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${GGML_OPENMP_LICENSE}" "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}/LICENSE-LLVM-OpenMP") + add_dependencies(ggml-base ggml-openmp-runtime) + install(FILES "${GGML_OPENMP_RUNTIME}" DESTINATION ${CMAKE_INSTALL_BINDIR}) + install(FILES "${GGML_OPENMP_LICENSE}" DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME LICENSE-LLVM-OpenMP) + + set(GGML_OPENMP_TARGET_C ggml-openmp-c) + set(GGML_OPENMP_TARGET_CXX ggml-openmp-cxx) + set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "") +elseif (GGML_OPENMP) find_package(OpenMP) if (OpenMP_FOUND) + set(GGML_OPENMP_TARGET_C OpenMP::OpenMP_C) + set(GGML_OPENMP_TARGET_CXX OpenMP::OpenMP_CXX) set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "") else() set(GGML_OPENMP_ENABLED "OFF" CACHE INTERNAL "") @@ -236,7 +350,7 @@ endif() if (GGML_OPENMP_ENABLED) target_compile_definitions(ggml-base PRIVATE GGML_USE_OPENMP) - target_link_libraries(ggml-base PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX) + target_link_libraries(ggml-base PRIVATE ${GGML_OPENMP_TARGET_C} ${GGML_OPENMP_TARGET_CXX}) endif() add_library(ggml diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 836bae4d05..a6cc49586b 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -74,7 +74,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name) if (GGML_OPENMP_ENABLED) target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_OPENMP) - target_link_libraries(${GGML_CPU_NAME} PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX) + target_link_libraries(${GGML_CPU_NAME} PRIVATE ${GGML_OPENMP_TARGET_C} ${GGML_OPENMP_TARGET_CXX}) endif() if (GGML_LLAMAFILE) diff --git a/ggml/src/ggml-zendnn/CMakeLists.txt b/ggml/src/ggml-zendnn/CMakeLists.txt index 87d721f6d7..6e393d6b66 100644 --- a/ggml/src/ggml-zendnn/CMakeLists.txt +++ b/ggml/src/ggml-zendnn/CMakeLists.txt @@ -86,6 +86,6 @@ endif() target_link_libraries(ggml-zendnn PRIVATE m pthread) -if (GGML_OPENMP) - target_link_libraries(ggml-zendnn PRIVATE OpenMP::OpenMP_CXX) +if (GGML_OPENMP_ENABLED) + target_link_libraries(ggml-zendnn PRIVATE ${GGML_OPENMP_TARGET_CXX}) endif() From 63b64a50a37600243c7dfbc4bbb92bd360a11ff7 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 17:00:54 +0300 Subject: [PATCH 29/36] metal : dequant kv cache only for large batches (#27438) --- ggml/src/ggml-metal/ggml-metal-ops.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 2dde14d8dc..8311544b39 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2806,6 +2806,13 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) { static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); + // depending on compute/bandwidth ratio, dequant to f16 kv is not always beneficial + // ref: https://github.com/ggml-org/llama.cpp/pull/27390#issuecomment-5355152767 + // TODO: tune per device + if (op->src[0]->ne[1] < 32) { + return false; + } + switch (op->src[1]->type) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -2968,9 +2975,10 @@ size_t ggml_metal_op_flash_attn_ext_extra_tmp(const ggml_tensor * op) { size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); - if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { - return 0; - } + // note: always reserve the temp buffer to avoid graph reallocations + //if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { + // return 0; + //} GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); From 78ec4c378031811671d1c76a067acbee4f4c56ce Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Thu, 20 Aug 2026 09:18:11 -0500 Subject: [PATCH 30/36] vulkan: FA MMQ should use fp32 for Q quantization calculations (#27413) Codex found that qd could be a denorm and 1/qd would overflow. --- .../ggml-vulkan/vulkan-shaders/flash_attn.comp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp index 6c264c7861..0c1b6d0673 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp @@ -121,13 +121,13 @@ void main() { const uint buf_ib = r * qf_stride + d / 8; const uint buf_iqs = d % 8; - FLOAT_TYPEV4 vals = is_in_bounds ? FLOAT_TYPEV4(data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale) : FLOAT_TYPEV4(0.0f); - const FLOAT_TYPEV4 abs_vals = abs(vals); + vec4 vals = is_in_bounds ? data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale : vec4(0.0f); + const vec4 abs_vals = abs(vals); - const FLOAT_TYPE thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w)); - const FLOAT_TYPE amax = subgroupClusteredMax(thread_max, 8); - const FLOAT_TYPE qd = amax / FLOAT_TYPE(127.0); - const FLOAT_TYPE qd_inv = qd != FLOAT_TYPE(0.0) ? FLOAT_TYPE(1.0) / qd : FLOAT_TYPE(0.0); + const float thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w)); + const float amax = subgroupClusteredMax(thread_max, 8); + const float qd = amax / 127.0f; + const float qd_inv = qd != 0.0f ? 1.0f / qd : 0.0f; vals = round(vals * qd_inv); Qf[buf_ib].qs[buf_iqs] = pack32(i8vec4(vals)); @@ -136,11 +136,11 @@ void main() { // the row-sum scaled by qd, used in k_dot_correction. if (FaTypeK == FA_TYPE_Q8_0) { if (buf_iqs == 0) { - Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0); + Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0f); } } else { - const FLOAT_TYPE thread_sum = vals.x + vals.y + vals.z + vals.w; - const FLOAT_TYPE sum = subgroupClusteredAdd(thread_sum, 8); + const float thread_sum = vals.x + vals.y + vals.z + vals.w; + const float sum = subgroupClusteredAdd(thread_sum, 8); if (buf_iqs == 0) { Qf[buf_ib].ds = FLOAT_TYPEV2(qd, sum * qd); From 07822bddf80d73f1168e592c52e69caaff820f9c Mon Sep 17 00:00:00 2001 From: Tarek Dakhran Date: Thu, 20 Aug 2026 16:36:57 +0200 Subject: [PATCH 31/36] model : support DSpark for LFM2 models (#27383) --- conversion/__init__.py | 1 + conversion/qwen.py | 15 ++++++++++++++- src/llama-arch.cpp | 2 ++ src/models/lfm2.cpp | 25 +++++++++++++++++-------- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/conversion/__init__.py b/conversion/__init__.py index 5ae6ad819f..4b8817ead4 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -57,6 +57,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "Qwen3DSparkModel": "qwen", "DSparkDraftModel": "qwen", "DSparkSpeculator": "qwen", + "Lfm2DSparkDraftModel": "qwen", "DeepseekV4ForCausalLM": "deepseek", "DeepseekV4DSparkModel": "deepseek", "DistilBertForMaskedLM": "bert", diff --git a/conversion/qwen.py b/conversion/qwen.py index 26b10452b6..3553657639 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -709,7 +709,7 @@ class DFlashModel(Qwen3Model): yield from super().modify_tensors(data_torch, name, bid) -@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator") +@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator", "Lfm2DSparkDraftModel") @ModelBase.example("satgeze/Qwen3.6-27B-DSpark") class DSparkModel(DFlashModel): # DSpark = DFlash + a semi-autoregressive Markov head. @@ -759,6 +759,13 @@ class DSparkModel(DFlashModel): return None return super().filter_tensors(item) + _ROPE_PERMUTE_SUFFIXES = ( + "self_attn.q_proj.weight", + "self_attn.k_proj.weight", + "self_attn.q_norm.weight", + "self_attn.k_norm.weight", + ) + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if name == "model.d2t": self._d2t = data_torch @@ -767,6 +774,12 @@ class DSparkModel(DFlashModel): if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith(("embed_tokens.weight", "lm_head.weight")): return + # interleaved-rope checkpoints (rope_is_neox_style = false) -> NeoX layout: per head, even dims first then odd + if not self.hparams.get("rope_is_neox_style", True) and name.endswith(self._ROPE_PERMUTE_SUFFIXES): + head_dim = self.hparams["head_dim"] + shape = data_torch.shape + data_torch = data_torch.reshape(-1, head_dim // 2, 2, *shape[1:]).transpose(1, 2).reshape(shape) + yield from super().modify_tensors(data_torch, name, bid) def prepare_tensors(self): diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 955c2d7965..4089544016 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -1032,6 +1032,8 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) { case LLM_ARCH_DEEPSEEK4: case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: + case LLM_ARCH_LFM2: + case LLM_ARCH_LFM2MOE: return true; default: return false; diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp index 70e837d6eb..9a42955570 100644 --- a/src/models/lfm2.cpp +++ b/src/models/lfm2.cpp @@ -2,6 +2,8 @@ #include "../llama-memory-hybrid-iswa.h" #include "../llama-memory-hybrid.h" +#include + void llama_model_lfm2::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -202,15 +204,20 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ } GGML_ASSERT(bx->ne[0] > conv->ne[0]); - // last d_conv columns is a new conv state - auto * new_conv = ggml_view_3d(ctx0, bx, conv->ne[0], bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2], - (bx->ne[0] - conv->ne[0]) * ggml_element_size(bx)); - GGML_ASSERT(ggml_are_same_shape(conv, new_conv)); + // write conv states: slot 0 = the final state, slot s = the state s tokens back (partial rollback) + const int64_t K = hparams.causal_attn && cparams.n_rs_seq > 0 ? (int64_t) cparams.n_rs_seq + 1 : 1; + const int64_t n_written = std::min(n_seq_tokens, K); + const auto mem_size = mctx_cur->get_size(); + const size_t row_size = ggml_row_size(conv_state->type, (int64_t) d_conv * n_embd); - // write new conv conv state - ggml_build_forward_expand(gf, ggml_cpy(ctx0, new_conv, - ggml_view_1d(ctx0, conv_state, ggml_nelements(new_conv), - kv_head * d_conv * n_embd * ggml_element_size(new_conv)))); + for (int64_t slot = 0; slot < n_written; ++slot) { + auto * conv_snap = ggml_view_3d(ctx0, bx, d_conv, bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2], + (bx->ne[0] - d_conv - slot) * ggml_element_size(bx)); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_snap, + ggml_view_2d(ctx0, conv_state, (int64_t) d_conv * n_embd, n_seqs, + conv_state->nb[1], + ((size_t) slot * mem_size + kv_head) * row_size))); + } auto * conv_kernel = model.layers[il].shortconv.conv; auto * conv_out = ggml_ssm_conv(ctx0, bx, conv_kernel); @@ -242,6 +249,8 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = cur; + const bool is_moe_layer = il >= static_cast(hparams.n_layer_dense_lead); auto * prev_cur = cur; From 681c29d36a13be54d317ee147b272da9163dbef3 Mon Sep 17 00:00:00 2001 From: John-Henry Lim <42513874+Interpause@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:45:37 +0800 Subject: [PATCH 32/36] mtmd: add --mmproj-device argument (#23255) * feat: add --mmproj-device arg & backwards compatible MTMD_BACKEND_DEVICE env var * feat: load mmproj device backend immediately, add -mmdev shortflag * fix: its a pointer now get the name * clean up * gen docs * nits --------- Co-authored-by: Xuan Son Nguyen --- common/arg.cpp | 20 ++++++++++++++++++++ common/common.h | 7 ++++--- tools/cli/README.md | 1 + tools/mtmd/clip.cpp | 11 +++++------ tools/mtmd/clip.h | 1 + tools/mtmd/debug/mtmd-debug.cpp | 1 + tools/mtmd/mtmd-cli.cpp | 1 + tools/mtmd/mtmd.cpp | 2 ++ tools/mtmd/mtmd.h | 1 + tools/server/README.md | 5 +++-- tools/server/server-context.cpp | 1 + tools/tts/tts.cpp | 1 + 12 files changed, 41 insertions(+), 11 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 6f5fe377d5..0a479c6aaa 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2595,6 +2595,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.mmproj_use_gpu = value; } ).set_examples(mmproj_examples).set_env("LLAMA_ARG_MMPROJ_OFFLOAD")); + add_opt(common_arg( + // note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet + {"-mmdev", "--mmproj-device"}, "DEVICE", + "device to use for multimodal projector (none = don't offload, default: auto)\n" + "use --list-devices to see a list of available devices", + [](common_params & params, const std::string & value) { + if (value == "none") { + params.mmproj_use_gpu = false; + params.mmproj_device = nullptr; + return; + } + auto devices = parse_device_list(value); + // parse_device_list pushes nullptr at back so devices is length 2 for single device. + if (devices.size() > 2) { + throw std::invalid_argument("only one device may be specified for mmproj"); + } + params.mmproj_use_gpu = true; + params.mmproj_device = devices.front(); + } + ).set_examples(mmproj_examples).set_env("MTMD_BACKEND_DEVICE")); // no LLAMA_ARG_ prefix for backward compatibility reason add_opt(common_arg( {"--image", "--audio", "--video"}, "FILE", "path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files\n", diff --git a/common/common.h b/common/common.h index d8a16897b8..de49dac9f6 100644 --- a/common/common.h +++ b/common/common.h @@ -581,9 +581,10 @@ struct common_params { // multimodal models (see tools/mtmd) struct common_params_model mmproj; - bool mmproj_use_gpu = true; // use GPU for multimodal model - bool no_mmproj = false; // explicitly disable multimodal model - std::vector image; // path to image file(s) ; TODO: change the name to "media" + bool mmproj_use_gpu = true; // use GPU for multimodal model + ggml_backend_dev_t mmproj_device = nullptr; // GPU device to use for multimodal model + bool no_mmproj = false; // explicitly disable multimodal model + std::vector image; // path to image file(s) ; TODO: change the name to "media" int image_min_tokens = -1; int image_max_tokens = -1; int mtmd_batch_max_tokens = 1024; diff --git a/tools/cli/README.md b/tools/cli/README.md index b3543ed4da..c9cbacafcd 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -162,6 +162,7 @@ | `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md
(env: LLAMA_ARG_MMPROJ_URL) | | `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)
(env: LLAMA_ARG_MMPROJ_AUTO) | | `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)
(env: LLAMA_ARG_MMPROJ_OFFLOAD) | +| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: auto)
use --list-devices to see a list of available devices
(env: MTMD_BACKEND_DEVICE) | | `--image, --audio, --video FILE` | path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files | | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MIN_TOKENS) | | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MAX_TOKENS) | diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index b9dd5e8452..45e33042dd 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -186,14 +186,13 @@ struct clip_ctx { throw std::runtime_error("failed to initialize CPU backend"); } if (ctx_params.use_gpu) { - auto * backend_name = std::getenv("MTMD_BACKEND_DEVICE"); - if (backend_name != nullptr) { - backend = ggml_backend_init_by_name(backend_name, nullptr); + if (ctx_params.device != nullptr) { + backend = ggml_backend_dev_init(ctx_params.device, nullptr); if (!backend) { - LOG_WRN("%s: Warning: Failed to initialize \"%s\" backend, falling back to default GPU backend\n", __func__, backend_name); + throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n", + __func__, ggml_backend_dev_name(ctx_params.device))); } - } - if (!backend) { + } else { backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr); backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr); } diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index a5b7137752..e07f258156 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -48,6 +48,7 @@ enum clip_flash_attn_type { struct clip_context_params { bool use_gpu; + ggml_backend_dev_t device; enum clip_flash_attn_type flash_attn_type; int image_min_tokens; int image_max_tokens; diff --git a/tools/mtmd/debug/mtmd-debug.cpp b/tools/mtmd/debug/mtmd-debug.cpp index b88a16f0f8..2719dae9b2 100644 --- a/tools/mtmd/debug/mtmd-debug.cpp +++ b/tools/mtmd/debug/mtmd-debug.cpp @@ -84,6 +84,7 @@ int main(int argc, char ** argv) { const char * clip_path = params.mmproj.path.c_str(); mtmd_context_params mparams = mtmd_context_params_default(); mparams.use_gpu = params.mmproj_use_gpu; + mparams.device = params.mmproj_device; mparams.print_timings = true; mparams.n_threads = params.cpuparams.n_threads; mparams.flash_attn_type = params.flash_attn_type; diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index 07b45b6440..f6c787fdb6 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -154,6 +154,7 @@ struct mtmd_cli_context { const char * clip_path = params.mmproj.path.c_str(); mtmd_context_params mparams = mtmd_context_params_default(); mparams.use_gpu = params.mmproj_use_gpu; + mparams.device = params.mmproj_device; mparams.print_timings = true; mparams.n_threads = params.cpuparams.n_threads; mparams.flash_attn_type = params.flash_attn_type; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 0f5cb8c7ae..95f17f7af8 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -456,6 +456,7 @@ static clip_flash_attn_type mtmd_get_clip_flash_attn_type(enum llama_flash_attn_ mtmd_context_params mtmd_context_params_default() { mtmd_context_params params { /* use_gpu */ true, + /* device */ nullptr, /* print_timings */ true, /* n_threads */ 4, /* image_marker */ nullptr, @@ -564,6 +565,7 @@ struct mtmd_context { clip_context_params ctx_clip_params { /* use_gpu */ ctx_params.use_gpu, + /* device */ ctx_params.device, /* flash_attn_type */ mtmd_get_clip_flash_attn_type(ctx_params.flash_attn_type), /* image_min_tokens */ ctx_params.image_min_tokens, /* image_max_tokens */ ctx_params.image_max_tokens, diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index ef4f99c0b6..ef88efd316 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -89,6 +89,7 @@ typedef bool (*mtmd_progress_callback)(float progress, void * user_data); struct mtmd_context_params { bool use_gpu; + ggml_backend_dev_t device; bool print_timings; int n_threads; const char * image_marker; // deprecated, use media_marker instead diff --git a/tools/server/README.md b/tools/server/README.md index f5d747eee8..93736c3edf 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -178,6 +178,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md
(env: LLAMA_ARG_MMPROJ_URL) | | `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)
(env: LLAMA_ARG_MMPROJ_AUTO) | | `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)
(env: LLAMA_ARG_MMPROJ_OFFLOAD) | +| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: auto)
use --list-devices to see a list of available devices
(env: MTMD_BACKEND_DEVICE) | | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MIN_TOKENS) | | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MAX_TOKENS) | | `--mtmd-batch-max-tokens N` | maximum number of image tokens per batch when encoding images (default: 1024)
(env: LLAMA_ARG_MTMD_BATCH_MAX_TOKENS) | @@ -196,11 +197,11 @@ For the full list of features, please refer to [server's changelog](https://gith | `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG) | | `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG_FILE) | | `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)
(env: LLAMA_ARG_UI_MCP_PROXY) | -| `--tools TOOL1,TOOL2,...` | experimental: whether to enable server tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | +| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | | `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)
available options:
'docker:', 'podman:': spin up a new container and reuse it for all invocations, clean up on server exit
'docker-container:', 'podman-container:': use an existing container by ID, won't stop on server exit
'ssh:': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required

(env: LLAMA_ARG_TOOLS_RUNTIME) | | `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_CONFIG) | | `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_JSON) | -| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all server tools - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_AGENT) | +| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_AGENT) | | `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)
(env: LLAMA_ARG_UI) | | `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)
(env: LLAMA_ARG_EMBEDDINGS) | | `--rerank, --reranking` | enable reranking endpoint on server (default: disabled)
(env: LLAMA_ARG_RERANKING) | diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 21ff783941..1293c86402 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -998,6 +998,7 @@ private: mtmd_context_params mparams = mtmd_context_params_default(); if (has_mmproj) { mparams.use_gpu = params_base.mmproj_use_gpu; + mparams.device = params_base.mmproj_device; mparams.print_timings = false; mparams.n_threads = params_base.cpuparams.n_threads; mparams.flash_attn_type = params_base.flash_attn_type; diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp index fd7522f8de..368123baf5 100644 --- a/tools/tts/tts.cpp +++ b/tools/tts/tts.cpp @@ -86,6 +86,7 @@ int main(int argc, char ** argv) { mtmd_context_params mtmd_params = mtmd_context_params_default(); mtmd_params.use_gpu = params.mmproj_use_gpu; + mtmd_params.device = params.mmproj_device; mtmd::context_ptr mctx(mtmd_init_from_file(params.mmproj.path.c_str(), model, mtmd_params)); if (!mctx) { LOG_ERR("failed to load mmproj %s\n", params.mmproj.path.c_str()); From 521a64cd01979bb5b1a466152c576a9d809b068d Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Thu, 20 Aug 2026 19:02:04 +0200 Subject: [PATCH 33/36] ui: Stores split refactor (#27240) * ui: Extract server stream lifecycle from chatStore into ChatStreamManager Discovery, attach/replay, resume retry and the remote-running snapshot formed a cohesive cluster inside chatStore. It now lives in chat-streams.svelte.ts as ChatStreamManager, owned by chatStore, which keeps the public entry points as delegates so components are unchanged. chatStore: 2877 -> 2418 lines. * ui: Extract user interaction gates from agenticStore into AgenticGates Tool permission requests, turn-limit continue prompts and queued steering messages are the state the loop waits on between turns. They had no coupling to session state, so they now live in agentic-gates.svelte.ts; agenticStore keeps delegates so components are unchanged. agenticStore: 1196 -> 1073 lines. * ui: Compose MCP resources under mcpStore.resources Resource state was a second import scope next to mcpStore. Consumers now go through mcpStore.resources, so the MCP surface is one store; mcp-resources.svelte.ts stays a separate file owned by mcpStore. * ui: Reorganize stores into domain namespaces * fix: Update stale doc comments * ui: Consolidate conv running-state into a chat activity ledger Running-state was split across chatStore.chatLoadingStates (local pipes), ChatStreamManager.remoteRunningConvs (backend sessions) and attachingConvs (attach lifecycle), unioned by hand in getAllLoadingChats and cross-cleaned by setChatLoading calling streams.clearRemoteRunning - the 'spinner ghosts until tab toggle' workaround. chatActivityStore now owns both sets with one transition per event: markLocal / localEnded (local pipe end also drops the stale remote hint, no cross-owner call) / applyRemoteSnapshot (diffed). The sidebar reads chatStore.activity.loadingConvs through the unchanged getAllLoadingChats entry point. Consequences: - isStreamingActive and its five manual writers are gone; isStreaming() now reports whether the active conversation has a live streaming pipe, which is what all four consumers (assistant row, stop action, context gauge, chat screen) actually check - isLoading/isReasoning become derived from the per-conv maps plus the active conversation, dropping the manual resync in syncLoadingStateForChat and clearUIState - attachingConvs and the last-attach coordination disappear from ChatStreamManager - getAllStreamingChats (no consumers) is removed * ui: Give store collaborators narrow host interfaces Collaborators took 'host: typeof ', i.e. the store's entire public surface, which is how chatStore's streamChatCompletion, createAssistantMessage, getApiOptions and setStreamingActive got widened to public. Replace with per-collaborator interfaces carrying only the members each one drives: - ChatStreamHost (chat/streams) - activity, processing, streaming states, abort controller, loading/streaming setters - ChatFlowsHost (chat/flows) - streaming core, message creation, per-conv state setters - McpHealthHost (mcp/health) - connection registry + reconnection - ModelPropsHost / ModelStatusHost (models) - model rows, feed updates; the managers write modalities/status back onto the host's rows, so those members stay writable - ConversationsPreferencesHost (conversations) - the active row and the conversation list The store classes now declare 'implements ' so the contract is visible at the class level, and the 'import type { }' back references in the collaborators disappear entirely - the host contract is local to each collaborator file, and collaborators can no longer reach around their slice. Members stay public (structural typing), but the collaborator side is now compiler-enforced. * test: Chat Activity store test * refactor: Cleanup * chore: Remove legacy architecture docs * ui: Memoize findMessageIndex for the streaming hot path Streaming looks up the same message index on every chunk, a linear scan of activeMessages each time. Cache the last lookup and reuse it after validating the id still sits at the same position (O(1)); any structural change to the array fails validation and falls back to a full scan. * ui: Throttle per-chunk stream state writes to localStorage saveStreamState ran JSON.stringify + a synchronous localStorage.setItem on every decoded chunk of the stream. The read loop now goes through a new saveStreamStateThrottled (one write per conversation per 500ms, latest value held pending); the public saveStreamState keeps its immediate-write contract for stream start and pre-fetch, and also resets the throttle window. A pending offset is force-flushed at resume boundaries (resumeStream reads the offset back from localStorage), on visibilitychange->hidden and on pagehide, so a reload always finds a usable offset. The resume offset only needs to be roughly current since the server retransmits from a line boundary and the client discards its partial line. Adds unit tests for the throttled/flush/clear interplay. * ui: Compute context gauge timing stats in one pass currentRead/Fresh/Cache/Output were separate deriveds, each running a full reverse scan of activeMessages for the last assistant timings, and cumulative ran its own forward scan plus an agentic filter - 4-5 O(n) passes per chunk while streaming. Replace with a single summarizeAssistantTimings() pass (last assistant timings, last agentic llm totals and the cumulative sums) feeding a shared derived snapshot. Semantics unchanged, including the live-stats overrides and the agentic llm-totals branch. * agentic : clear session state when a conversation is deleted Every conversation that ran an agentic flow left an AgenticSession in the store forever; clearSession was never called. conversationsStore now notifies deletion listeners and agenticStore drops the matching sessions, avoiding a circular import back into conversationsStore. * chat : extract ChatService.normalizeMessagesForApi The DB->API message normalization (convert + drop empty system messages) was duplicated in sendMessage, preEncode and the agentic flow. Extract it into one shared method and call it from all three. * sse : share record splitting and data extraction splitSseRecords and extractSseDataPayload centralize the record-boundary splitting and data: line extraction used by parseSseJsonStream and the models status feed. chat.service keeps its own line-based parser for resume support. * api : delegate apiFetchWithParams to apiFetch apiFetchWithParams duplicated apiFetch's headers/fetch/error handling body-for-body; it only differs in URL construction. Build the URL and delegate. * chat flows : dedupe title, timings and cleanup handling - conversationsStore.applyTitleFromContent centralizes the title-from-first- message logic duplicated in 5 places - ChatProcessingStore.applyStreamTimings centralizes the onTimings handler shared by the chat and continue flows - host.cleanupStreaming centralizes the loading/streaming/processing reset repeated across the continue flow's exit paths * conversations : centralize conversation update mirroring rename, pin, mcp override, reasoning effort and cwd all repeated the same write-DB-then-mirror-into-list-and-active dance. A single applyConversationUpdate(id, updates) on the host collapses all five and removes the forgot-to-mirror-one-field bug class. Drops the redundant array reassignment in setCwd (deep field assignment is reactive). * mcp : dedupe tool execution, server parsing and tool indexing - executeTool delegates to executeToolByName (only diff was argument parsing) - drop the private #parseServerSettings copy; use parseMcpServerSettings - cache getServers() keyed on the raw config value (hot path) - indexServerTools() unifies the three identical toolsIndex rebuild loops Assisted-by: Claude * mcp : share cursor pagination and tool indexing - MCPService.paginate() collapses the identical do-while loops in listAllResources and listAllResourceTemplates - promoteHealthCheckToConnection now uses indexServerTools like the other connect paths Assisted-by: Claude * database : share message parent-child bookkeeping - addChildToParent() dedups the append-to-children update in createMessageBranch and createSystemMessage - removeChildFromParent() dedups the remove-from-children cleanup in deleteMessage and deleteMessageCascading - bulkAdd the cloned messages when forking a conversation instead of one add per message Assisted-by: Claude * chore: Lint/format * fix: `pagehide` event from `window` * refactor: Api Fetch util * docs : rewrite architecture sections in README Update the high-level diagram, routes, hooks, stores, services and data flow tables to match the current UI structure (mcp/settings/search routes, agentic/tools/mcp stores, MCPService/ToolsService/SandboxService, /tools API). Fix stale architectural patterns for per-conversation state and modality validation. * chore : add ESLint rule for blank lines between accessors Enforce a blank line between consecutive class accessors. The core padding-line-between-statements rule does not cover class members, so a local rule is needed. * refactor : reorder store members and unify naming Order store class members as public fields, private fields, constructor, getters, public methods, then private methods. Normalize private naming to the `private` keyword (drop `#` and the `_` prefix where there is no matching public getter). Rename conversationsStore.init() to initialize() to match the other stores. * refactor : prefix lookup methods with get in agentic and chat stores Unify bare-name lookup methods with the get* prefix used across the other stores (mcp, models, tools, settings). Renames currentTurn, totalToolCalls, lastError, streamingToolCall, executingToolCallId, pendingPermissionRequest, pendingContinueRequest, pendingSteeringMessageContent, pendingSteeringMessageExtras in the agentic store and pendingMessageContent, pendingMessageExtras in the chat store. Updates the two consuming components and a doc comment. * refactor: Clean up comments in stores' and services' code * chore : add ESLint rule for class member ordering Enforce structural order (public fields -> private fields -> constructor -> getters -> setters -> public methods -> private methods) with alphabetical sorting within each group via perfectionist/sort-classes. Dependency detection keeps Svelte $derived fields in a valid dependency order instead of alphabetizing them, since Svelte rejects forward references. Assisted-by: Claude * refactor : reorder class members to match new ESLint rule Apply the sort-classes rule across stores, services, hooks and utils. Pure reordering - verified no logic changes by comparing sorted line multisets before/after. All tests and svelte-check pass. --- tools/ui/README.md | 166 +- .../high-level-architecture-simplified.md | 145 - .../architecture/high-level-architecture.md | 373 --- tools/ui/docs/flows/chat-flow.md | 228 -- tools/ui/docs/flows/conversations-flow.md | 183 -- .../flows/data-flow-simplified-model-mode.md | 45 - .../flows/data-flow-simplified-router-mode.md | 77 - tools/ui/docs/flows/database-flow.md | 174 - tools/ui/docs/flows/mcp-flow.md | 226 -- tools/ui/docs/flows/models-flow.md | 181 -- tools/ui/docs/flows/server-flow.md | 76 - tools/ui/docs/flows/settings-flow.md | 156 - tools/ui/eslint.config.js | 84 +- .../ChatAttachmentsPreview.svelte | 2 +- .../app/chat/ChatForm/ChatForm.svelte | 12 +- .../ChatFormActionAddMcpServersSubmenu.svelte | 4 +- .../ChatFormActionAddSheet.svelte | 10 +- .../ChatFormActionModels.svelte | 12 +- .../ChatFormActions/ChatFormActions.svelte | 6 +- .../ChatFormContextGauge.svelte | 6 +- .../ChatForm/ChatFormMcpResourcesList.svelte | 6 +- .../ChatFormPickerMcpPrompts.svelte | 2 +- .../ChatMessageAssistant.svelte | 2 +- .../ChatMessageAssistantModel.svelte | 2 +- .../ChatMessageAgenticContent.svelte | 8 +- .../app/chat/ChatMessages/ChatMessages.svelte | 12 +- .../dialogs/DialogMcpResourcesBrowser.svelte | 18 +- .../app/dialogs/DialogMcpServerAddNew.svelte | 2 +- .../app/dialogs/DialogModelInformation.svelte | 4 +- .../app/mcp/McpActiveServersAvatars.svelte | 4 +- .../McpResourcesBrowser.svelte | 6 +- .../app/models/ModelsSelectorDropdown.svelte | 6 +- .../app/models/ModelsSelectorOption.svelte | 12 +- .../app/models/ModelsSelectorSheet.svelte | 4 +- .../settings/SettingsChat/SettingsChat.svelte | 2 +- .../SettingsChat/SettingsChatFields.svelte | 4 +- .../app/settings/SettingsMcpServers.svelte | 8 +- .../constants/attachment-menu.constants.ts | 2 +- tools/ui/src/lib/constants/cache.constants.ts | 10 - tools/ui/src/lib/constants/url.constants.ts | 6 + .../src/lib/hooks/use-auto-scroll.svelte.ts | 178 +- .../use-chat-screen-active-model.svelte.ts | 10 +- .../src/lib/hooks/use-context-gauge.svelte.ts | 8 +- .../lib/hooks/use-models-selector.svelte.ts | 8 +- .../lib/hooks/use-processing-state.svelte.ts | 2 +- .../lib/hooks/use-reasoning-menu.svelte.ts | 11 +- .../src/lib/hooks/use-tools-panel.svelte.ts | 4 +- tools/ui/src/lib/services/chat.service.ts | 1975 ++++++------ .../services/conversation-transfer.service.ts | 346 +- tools/ui/src/lib/services/database.service.ts | 765 +++-- tools/ui/src/lib/services/index.ts | 28 +- tools/ui/src/lib/services/mcp.service.ts | 1426 ++++---- .../ui/src/lib/services/migration.service.ts | 21 +- tools/ui/src/lib/services/models.service.ts | 267 +- .../lib/services/parameter-sync.service.ts | 176 +- tools/ui/src/lib/services/props.service.ts | 16 +- .../ui/src/lib/services/read-media.service.ts | 9 +- tools/ui/src/lib/services/router.service.ts | 7 + tools/ui/src/lib/services/sandbox-harness.ts | 7 + tools/ui/src/lib/services/sandbox.service.ts | 10 +- tools/ui/src/lib/services/tools.service.ts | 25 +- .../ui/src/lib/stores/agentic/gates.svelte.ts | 208 ++ .../index.svelte.ts} | 489 +-- tools/ui/src/lib/stores/chat.svelte.ts | 2868 ----------------- .../ui/src/lib/stores/chat/activity.svelte.ts | 74 + .../stores/{ => chat}/context-stats.svelte.ts | 248 +- .../drafts.svelte.ts} | 20 +- tools/ui/src/lib/stores/chat/flows.svelte.ts | 794 +++++ tools/ui/src/lib/stores/chat/index.svelte.ts | 1441 +++++++++ .../src/lib/stores/chat/processing.svelte.ts | 188 ++ .../ui/src/lib/stores/chat/streams.svelte.ts | 494 +++ .../index.svelte.ts} | 1195 +++---- .../conversations/preferences.svelte.ts | 254 ++ tools/ui/src/lib/stores/device.svelte.ts | 4 +- tools/ui/src/lib/stores/index.ts | 28 +- tools/ui/src/lib/stores/init.ts | 6 +- tools/ui/src/lib/stores/mcp/health.svelte.ts | 298 ++ .../{mcp.svelte.ts => mcp/index.svelte.ts} | 2547 ++++++--------- .../resources.svelte.ts} | 616 ++-- tools/ui/src/lib/stores/models.svelte.ts | 1077 ------- .../ui/src/lib/stores/models/index.svelte.ts | 451 +++ .../ui/src/lib/stores/models/props.svelte.ts | 273 ++ .../ui/src/lib/stores/models/status.svelte.ts | 278 ++ tools/ui/src/lib/stores/permissions.svelte.ts | 48 +- tools/ui/src/lib/stores/server.svelte.ts | 128 +- .../index.svelte.ts} | 810 +++-- .../referrer.svelte.ts} | 7 + tools/ui/src/lib/stores/tools.svelte.ts | 863 ++--- tools/ui/src/lib/types/agentic.d.ts | 2 +- tools/ui/src/lib/utils/api-fetch.ts | 32 +- tools/ui/src/lib/utils/api-headers.ts | 2 +- tools/ui/src/lib/utils/api-key-validation.ts | 2 +- tools/ui/src/lib/utils/audio-recording.ts | 58 +- tools/ui/src/lib/utils/cache-ttl.ts | 204 +- .../utils/chat-form-input-rich-tokenizer.ts | 2 +- .../src/lib/utils/convert-files-to-extra.ts | 6 +- tools/ui/src/lib/utils/index.ts | 7 +- tools/ui/src/lib/utils/mcp.ts | 150 +- .../src/lib/utils/process-uploaded-files.ts | 6 +- tools/ui/src/lib/utils/source-history.ts | 26 +- tools/ui/src/lib/utils/sse.ts | 28 +- tools/ui/src/routes/(chat)/+page.svelte | 6 +- tools/ui/src/routes/+layout.svelte | 4 +- .../client/agentic-stream.perf.svelte.test.ts | 2 +- .../tests/client/apikey-splash.svelte.test.ts | 2 +- .../chat-form-enter-code-block.svelte.test.ts | 2 +- .../components/ChatMessagesPerfWrapper.svelte | 2 +- .../client/mcp-display-name.svelte.test.ts | 4 +- .../client/sandbox.service.svelte.test.ts | 2 +- ...ettings-registry-invariants.svelte.test.ts | 2 +- ...tings-render-keys-migration.svelte.test.ts | 2 +- .../client/ui-settings-sync.svelte.test.ts | 2 +- .../update-message-in-place.svelte.test.ts | 2 +- .../tests/stories/ChatMessage.stories.svelte | 14 +- .../stories/ModelsSelector.stories.svelte | 2 +- .../stories/SidebarNavigation.stories.svelte | 6 +- .../tests/stories/fixtures/storybook-mocks.ts | 2 +- tools/ui/tests/unit/chat-activity.test.ts | 77 + .../tests/unit/mcp-override-fallback.test.ts | 38 +- tools/ui/tests/unit/stream-resume.test.ts | 61 + 120 files changed, 11220 insertions(+), 12829 deletions(-) delete mode 100644 tools/ui/docs/architecture/high-level-architecture-simplified.md delete mode 100644 tools/ui/docs/architecture/high-level-architecture.md delete mode 100644 tools/ui/docs/flows/chat-flow.md delete mode 100644 tools/ui/docs/flows/conversations-flow.md delete mode 100644 tools/ui/docs/flows/data-flow-simplified-model-mode.md delete mode 100644 tools/ui/docs/flows/data-flow-simplified-router-mode.md delete mode 100644 tools/ui/docs/flows/database-flow.md delete mode 100644 tools/ui/docs/flows/mcp-flow.md delete mode 100644 tools/ui/docs/flows/models-flow.md delete mode 100644 tools/ui/docs/flows/server-flow.md delete mode 100644 tools/ui/docs/flows/settings-flow.md create mode 100644 tools/ui/src/lib/stores/agentic/gates.svelte.ts rename tools/ui/src/lib/stores/{agentic.svelte.ts => agentic/index.svelte.ts} (77%) delete mode 100644 tools/ui/src/lib/stores/chat.svelte.ts create mode 100644 tools/ui/src/lib/stores/chat/activity.svelte.ts rename tools/ui/src/lib/stores/{ => chat}/context-stats.svelte.ts (56%) rename tools/ui/src/lib/stores/{draft-messages.svelte.ts => chat/drafts.svelte.ts} (76%) create mode 100644 tools/ui/src/lib/stores/chat/flows.svelte.ts create mode 100644 tools/ui/src/lib/stores/chat/index.svelte.ts create mode 100644 tools/ui/src/lib/stores/chat/processing.svelte.ts create mode 100644 tools/ui/src/lib/stores/chat/streams.svelte.ts rename tools/ui/src/lib/stores/{conversations.svelte.ts => conversations/index.svelte.ts} (61%) create mode 100644 tools/ui/src/lib/stores/conversations/preferences.svelte.ts create mode 100644 tools/ui/src/lib/stores/mcp/health.svelte.ts rename tools/ui/src/lib/stores/{mcp.svelte.ts => mcp/index.svelte.ts} (67%) rename tools/ui/src/lib/stores/{mcp-resources.svelte.ts => mcp/resources.svelte.ts} (96%) delete mode 100644 tools/ui/src/lib/stores/models.svelte.ts create mode 100644 tools/ui/src/lib/stores/models/index.svelte.ts create mode 100644 tools/ui/src/lib/stores/models/props.svelte.ts create mode 100644 tools/ui/src/lib/stores/models/status.svelte.ts rename tools/ui/src/lib/stores/{settings.svelte.ts => settings/index.svelte.ts} (89%) rename tools/ui/src/lib/stores/{settings-referrer.svelte.ts => settings/referrer.svelte.ts} (50%) create mode 100644 tools/ui/tests/unit/chat-activity.test.ts diff --git a/tools/ui/README.md b/tools/ui/README.md index 53b5925e2c..99abfaa41f 100644 --- a/tools/ui/README.md +++ b/tools/ui/README.md @@ -239,31 +239,44 @@ Routes → Components → Hooks → Stores → Services → Storage/API ### High-Level Architecture -See: [`docs/architecture/high-level-architecture-simplified.md`](docs/architecture/high-level-architecture-simplified.md) - ```mermaid flowchart TB subgraph Routes["📍 Routes"] R1["/ (Welcome)"] R2["/chat/[id]"] + R3["/mcp-servers"] + R4["/search"] + R5["/settings"] RL["+layout.svelte"] end subgraph Components["🧩 Components"] - C_Sidebar["ChatSidebar"] C_Screen["ChatScreen"] C_Form["ChatForm"] C_Messages["ChatMessages"] - C_ModelsSelector["ModelsSelector"] + C_Sidebar["ChatSidebar"] + C_Models["ModelsSelector"] C_Settings["ChatSettings"] + C_Mcp["McpServers"] + end + + subgraph Hooks["🔌 Hooks"] + H1["use-chat-screen-active-model"] + H2["use-processing-state"] + H3["use-context-gauge"] + H4["use-models-selector"] + H5["use-tools-panel"] end subgraph Stores["🗄️ Stores"] S1["chatStore"] S2["conversationsStore"] S3["modelsStore"] - S4["serverStore"] - S5["settingsStore"] + S4["mcpStore"] + S5["agenticStore"] + S6["serverStore"] + S7["settingsStore"] + S8["toolsStore"] end subgraph Services["⚙️ Services"] @@ -271,6 +284,9 @@ flowchart TB SV2["ModelsService"] SV3["PropsService"] SV4["DatabaseService"] + SV5["MCPService"] + SV6["ToolsService"] + SV7["SandboxService"] end subgraph Storage["💾 Storage"] @@ -282,19 +298,28 @@ flowchart TB API1["/v1/chat/completions"] API2["/props"] API3["/models/*"] + API4["/tools"] end R1 & R2 --> C_Screen RL --> C_Sidebar C_Screen --> C_Form & C_Messages & C_Settings - C_Screen --> S1 & S2 - C_ModelsSelector --> S3 & S4 + C_Screen --> H1 & H2 & H3 + C_Models --> H4 + C_Mcp --> S4 + C_Screen --> S1 & S2 & S3 + C_Models --> S3 + H1 --> S3 S1 --> SV1 & SV4 + S2 --> SV4 S3 --> SV2 & SV3 + S4 --> SV5 + S5 --> SV1 & SV5 & SV6 & SV7 SV4 --> ST1 SV1 --> API1 SV2 --> API3 SV3 --> API2 + SV6 --> API4 ``` ### Layer Breakdown @@ -303,6 +328,9 @@ flowchart TB - **`/`** - Welcome screen, creates new conversation - **`/chat/[id]`** - Active chat interface +- **`/mcp-servers`** - MCP server management +- **`/search`** - Conversation search +- **`/settings`** - Settings (optional `[[section]]`) - **`+layout.svelte`** - Sidebar, navigation, global initialization #### Components (`src/lib/components/`) @@ -348,28 +376,68 @@ Components are organized in `app/` (application-specific) and `ui/` (shadcn-svel #### Hooks (`src/lib/hooks/`) -- **`useModelChangeValidation`** - Validates model switch against conversation modalities -- **`useProcessingState`** - Tracks streaming progress and token generation +Hooks are the thin view-layer between components and stores: they own UI concerns (scroll, drag-and-drop, keyboard shortcuts, pickers, selection) and translate store state into view state. + +| Hook | Responsibility | +| ------------------------------- | -------------------------------------------------------------- | +| `use-chat-screen-active-model` | Active model resolution + modality capability detection | +| `use-processing-state` | View over `chatStore.processing` for streaming progress/tokens | +| `use-context-gauge` | View over `contextStatsStore` for the context usage gauge | +| `use-models-selector` | Model selector dropdown state (loaded/available groups) | +| `use-tools-panel` | Tools panel state | +| `use-reasoning-menu` | Reasoning-effort menu state | +| `use-attachment-menu` | Attachment menu + modality flags | +| `use-draft-messages` | Per-chat draft message/files persistence | +| `use-chat-form-pickers` | Chat form pickers (commands, mentions) | +| `use-debounced-search` | Shared debounced async search for pickers | +| `use-picker-navigation` | Picker keyboard navigation | +| `use-chat-message-edit-context` | Message edit context (content + extras) | +| `use-chat-screen-drag-and-drop` | Drag-and-drop state machine | +| `use-chat-screen-file-upload` | File upload queue + capability validation | +| `use-chat-screen-scroll` | Scroll container binding + navigation guard | +| `use-auto-scroll` | Auto-scroll controller for streaming | +| `use-marquee-selection` | Shift+click / marquee range selection | +| `use-keyboard-shortcuts` | Global keyboard shortcuts | +| `use-settings-navigation` | Settings section navigation | +| `use-pwa` | PWA install/update + version mismatch detection | #### Stores (`src/lib/stores/`) -| Store | Responsibility | -| -------------------- | --------------------------------------------------------- | -| `chatStore` | Message sending, streaming, abort control, error handling | -| `conversationsStore` | CRUD for conversations, message branching, navigation | -| `modelsStore` | Model list, selection, loading/unloading (ROUTER) | -| `serverStore` | Server properties, role detection, modalities | -| `settingsStore` | User preferences, parameter sync with server defaults | +Stores own reactive application state as Svelte 5 runes. Larger stores are split into directories and compose focused sub-stores behind a narrow host interface (see Architectural Patterns). + +| Store | Responsibility | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | +| `chatStore` | Chat lifecycle, streaming, abort control, error handling; composes `processing`, `activity`, `streams`, `flows` | +| `conversationsStore` | Conversation CRUD, message branching, navigation, import/export; composes `preferences` | +| `modelsStore` | Model list, selection, loading/unloading (ROUTER); composes `props`, `status` | +| `mcpStore` | MCP host role: multi-server lifecycle, tool routing; composes `health`, `resources` | +| `agenticStore` | Multi-turn agentic loop orchestration, tool execution; composes `gates` | +| `serverStore` | Server connection state, `/props`, role detection, modalities | +| `settingsStore` | User preferences, theme, parameter sync with server defaults | +| `toolsStore` | Tool registry: server + MCP tools, enabled set for the LLM | +| `permissionsStore` | Persisted tool permission grants | +| `contextStatsStore` | Context window usage for the active conversation | +| `draftMessagesStore` | Per-chat draft message/files | +| `deviceStore` | Browser environment signals (mobile, OS, theme) | +| `versionStore` | Build version information | #### Services (`src/lib/services/`) -| Service | Responsibility | -| ---------------------- | ----------------------------------------------- | -| `ChatService` | API calls to`/v1/chat/completions`, SSE parsing | -| `ModelsService` | `/models`, `/models/load`, `/models/unload` | -| `PropsService` | `/props`, `/props?model=` | -| `DatabaseService` | IndexedDB operations via Dexie | -| `ParameterSyncService` | Syncs settings with server defaults | +Services are a stateless protocol layer: static methods, pure I/O, no reactive state. Stores consume them for all API and storage access. + +| Service | Responsibility | +| ----------------------------- | ------------------------------------------------------------------------- | +| `ChatService` | `/v1/chat/completions` streaming + SSE parsing, message format conversion | +| `ModelsService` | `/models`, `/models/load`, `/models/unload` | +| `PropsService` | `/props`, `/props?model=` | +| `DatabaseService` | IndexedDB operations via Dexie | +| `MCPService` | MCP protocol: transports, connect, list/execute tools, prompts, resources | +| `ToolsService` | Server tool list/execute/stream (`/tools`) | +| `SandboxService` | Browser JS execution in a sandboxed worker | +| `ParameterSyncService` | Syncs settings with server defaults | +| `ConversationTransferService` | Conversation import/export JSONL + ZIP format | +| `MigrationService` | Non-destructive localStorage/IndexedDB migrations | +| `RouterService` | Dynamic route URL construction | --- @@ -377,8 +445,6 @@ Components are organized in `app/` (application-specific) and `ui/` (shadcn-svel ### MODEL Mode (Single Model) -See: [`docs/flows/data-flow-simplified-model-mode.md`](docs/flows/data-flow-simplified-model-mode.md) - ```mermaid sequenceDiagram participant User @@ -388,8 +454,9 @@ sequenceDiagram participant API as llama-server Note over User,API: Initialization - UI->>Stores: initialize() - Stores->>DB: load conversations + UI->>Stores: initStores() (awaited by route loads) + Stores->>Stores: run migrations + Stores->>DB: load conversations (background) Stores->>API: GET /props API-->>Stores: server config Stores->>API: GET /v1/models @@ -408,8 +475,6 @@ sequenceDiagram ### ROUTER Mode (Multi-Model) -See: [`docs/flows/data-flow-simplified-router-mode.md`](docs/flows/data-flow-simplified-router-mode.md) - ```mermaid sequenceDiagram participant User @@ -441,17 +506,6 @@ sequenceDiagram end ``` -### Detailed Flow Diagrams - -| Flow | Description | File | -| ------------- | ------------------------------------------ | ----------------------------------------------------------- | -| Chat | Message lifecycle, streaming, regeneration | [`chat-flow.md`](docs/flows/chat-flow.md) | -| Models | Loading, unloading, modality caching | [`models-flow.md`](docs/flows/models-flow.md) | -| Server | Props fetching, role detection | [`server-flow.md`](docs/flows/server-flow.md) | -| Conversations | CRUD, branching, import/export | [`conversations-flow.md`](docs/flows/conversations-flow.md) | -| Database | IndexedDB schema, operations | [`database-flow.md`](docs/flows/database-flow.md) | -| Settings | Parameter sync, user overrides | [`settings-flow.md`](docs/flows/settings-flow.md) | - --- ## Architectural Patterns @@ -505,13 +559,14 @@ Components dispatch actions to stores, stores coordinate with services for I/O, ### 3. Per-Conversation State -Enables concurrent streaming across multiple conversations: +Enables concurrent streaming across multiple conversations. Loading is tracked +per conversation by the activity ledger (`chatStore.activity`), while streaming +state and abort controllers live in per-conversation maps: ```typescript class ChatStore { - chatLoadingStates = new Map(); - chatStreamingStates = new Map(); - abortControllers = new Map(); + chatStreamingStates = new SvelteMap(); + abortControllers = new SvelteMap(); } ``` @@ -567,20 +622,14 @@ get isRouterMode() { ### 7. Modality Validation -Prevents sending attachments to incompatible models: +Prevents sending attachments to incompatible models. The +`use-chat-screen-active-model` hook derives the active model's capabilities +from `modelsStore.props`: ```typescript -// useModelChangeValidation hook -const validate = (modelId: string) => { - const modelModalities = modelsStore.getModelModalities(modelId); - const conversationModalities = conversationsStore.usedModalities; - - // Check if model supports all used modalities - if (conversationModalities.hasImages && !modelModalities.vision) { - return { valid: false, reason: 'Model does not support images' }; - } - // ... -}; +// use-chat-screen-active-model hook +const hasVisionModality = $derived.by(() => modelsStore.props.modelSupportsVision(activeModelId)); +const hasAudioModality = $derived.by(() => modelsStore.props.modelSupportsAudio(activeModelId)); ``` ### 8. Persistent Storage Strategy @@ -673,9 +722,6 @@ tools/ui/ │ └── styles/ # Global styles ├── static/ # Static assets ├── tests/ # Test files -├── docs/ # Architecture diagrams -│ ├── architecture/ # High-level architecture -│ └── flows/ # Feature-specific flows └── .storybook/ # Storybook configuration ``` diff --git a/tools/ui/docs/architecture/high-level-architecture-simplified.md b/tools/ui/docs/architecture/high-level-architecture-simplified.md deleted file mode 100644 index 500f477c9a..0000000000 --- a/tools/ui/docs/architecture/high-level-architecture-simplified.md +++ /dev/null @@ -1,145 +0,0 @@ -```mermaid -flowchart TB - subgraph Routes["📍 Routes"] - R1["/ (Welcome)"] - R2["/chat/[id]"] - RL["+layout.svelte"] - end - - subgraph Components["🧩 Components"] - C_Sidebar["ChatSidebar"] - C_Screen["ChatScreen"] - C_Form["ChatForm"] - C_Messages["ChatMessages"] - C_Message["ChatMessage"] - C_ChatMessageAgenticContent["ChatMessageAgenticContent"] - C_MessageEditForm["ChatMessageEditForm"] - C_ModelsSelector["ModelsSelector"] - C_Settings["ChatSettings"] - C_McpSettings["McpServersSettings"] - C_McpResourceBrowser["McpResourceBrowser"] - C_McpServersSelector["McpServersSelector"] - end - - subgraph Hooks["🪝 Hooks"] - H1["useModelChangeValidation"] - H2["useProcessingState"] - end - - subgraph Stores["🗄️ Stores"] - S1["chatStore
Chat interactions & streaming"] - SA["agenticStore
Multi-turn agentic loop orchestration"] - S2["conversationsStore
Conversation data, messages & MCP overrides"] - S3["modelsStore
Model selection & loading"] - S4["serverStore
Server props & role detection"] - S5["settingsStore
User configuration incl. MCP"] - S6["mcpStore
MCP servers, tools, prompts"] - S7["mcpResourceStore
MCP resources & attachments"] - end - - subgraph Services["⚙️ Services"] - SV1["ChatService"] - SV2["ModelsService"] - SV3["PropsService"] - SV4["DatabaseService"] - SV5["ParameterSyncService"] - SV6["MCPService
protocol operations"] - end - - subgraph Storage["💾 Storage"] - ST1["IndexedDB
conversations, messages"] - ST2["LocalStorage
config, userOverrides, mcpServers"] - end - - subgraph APIs["🌐 llama-server API"] - API1["/v1/chat/completions"] - API2["/props"] - API3["/models/*"] - API4["/v1/models"] - end - - subgraph ExternalMCP["🔌 External MCP Servers"] - EXT1["MCP Server 1
WebSocket/HTTP/SSE"] - EXT2["MCP Server N"] - end - - %% Routes → Components - R1 & R2 --> C_Screen - RL --> C_Sidebar - - %% Layout runs MCP health checks - RL --> S6 - - %% Component hierarchy - C_Screen --> C_Form & C_Messages & C_Settings - C_Messages --> C_Message - C_Message --> C_ChatMessageAgenticContent - C_Message --> C_MessageEditForm - C_Form & C_MessageEditForm --> C_ModelsSelector - C_Form --> C_McpServersSelector - C_Settings --> C_McpSettings - C_McpSettings --> C_McpResourceBrowser - - %% Components → Hooks → Stores - C_Form & C_Messages --> H1 & H2 - H1 --> S3 & S4 - H2 --> S1 & S5 - - %% Components → Stores - C_Screen --> S1 & S2 - C_Sidebar --> S2 - C_ModelsSelector --> S3 & S4 - C_Settings --> S5 - C_McpSettings --> S6 - C_McpResourceBrowser --> S6 & S7 - C_McpServersSelector --> S6 - C_Form --> S6 - - %% chatStore → agenticStore → mcpStore (agentic loop) - S1 --> SA - SA --> SV1 - SA --> S6 - - %% Stores → Services - S1 --> SV1 & SV4 - S2 --> SV4 - S3 --> SV2 & SV3 - S4 --> SV3 - S5 --> SV5 - S6 --> SV6 - S7 --> SV6 - - %% Services → Storage - SV4 --> ST1 - SV5 --> ST2 - - %% Services → APIs - SV1 --> API1 - SV2 --> API3 & API4 - SV3 --> API2 - - %% MCP → External Servers - SV6 --> EXT1 & EXT2 - - %% Styling - classDef routeStyle fill:#e1f5fe,stroke:#01579b,stroke-width:2px - classDef componentStyle fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px - classDef hookStyle fill:#fff8e1,stroke:#ff8f00,stroke-width:2px - classDef storeStyle fill:#fff3e0,stroke:#e65100,stroke-width:2px - classDef serviceStyle fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px - classDef storageStyle fill:#fce4ec,stroke:#c2185b,stroke-width:2px - classDef apiStyle fill:#e3f2fd,stroke:#1565c0,stroke-width:2px - classDef mcpStyle fill:#e0f2f1,stroke:#00695c,stroke-width:2px - classDef agenticStyle fill:#e8eaf6,stroke:#283593,stroke-width:2px - classDef externalStyle fill:#f3e5f5,stroke:#6a1b9a,stroke-width:2px,stroke-dasharray: 5 5 - - class R1,R2,RL routeStyle - class C_Sidebar,C_Screen,C_Form,C_Messages,C_Message,C_ChatMessageAgenticContent,C_MessageEditForm,C_ModelsSelector,C_Settings componentStyle - class C_McpSettings,C_McpResourceBrowser,C_McpServersSelector componentStyle - class H1,H2 hookStyle - class S1,S2,S3,S4,S5,SA,S6,S7 storeStyle - class SV1,SV2,SV3,SV4,SV5,SV6 serviceStyle - class ST1,ST2 storageStyle - class API1,API2,API3,API4 apiStyle - class EXT1,EXT2 externalStyle -``` diff --git a/tools/ui/docs/architecture/high-level-architecture.md b/tools/ui/docs/architecture/high-level-architecture.md deleted file mode 100644 index 42ddb3f4f5..0000000000 --- a/tools/ui/docs/architecture/high-level-architecture.md +++ /dev/null @@ -1,373 +0,0 @@ -```mermaid -flowchart TB -subgraph Routes["📍 Routes"] -R1["/ (+page.svelte)"] -R2["/chat/[id]"] -RL["+layout.svelte"] -end - - subgraph Components["🧩 Components"] - direction TB - subgraph LayoutComponents["Layout"] - C_Sidebar["ChatSidebar"] - C_Screen["ChatScreen"] - end - subgraph ChatUIComponents["Chat UI"] - C_Form["ChatForm"] - C_Messages["ChatMessages"] - C_Message["ChatMessage"] - C_MessageUser["ChatMessageUser"] - C_MessageEditForm["ChatMessageEditForm"] - C_Attach["ChatAttachments"] - C_ModelsSelector["ModelsSelector"] - C_Settings["ChatSettings"] - end - subgraph MCPComponents["MCP UI"] - C_McpSettings["McpServersSettings"] - C_McpServerCard["McpServerCard"] - C_McpResourceBrowser["McpResourceBrowser"] - C_McpResourcePreview["McpResourcePreview"] - C_McpServersSelector["McpServersSelector"] - end - end - - subgraph Hooks["🪝 Hooks"] - H1["useModelChangeValidation"] - H2["useProcessingState"] - H3["isMobile"] - end - - subgraph Stores["🗄️ Stores"] - direction TB - subgraph S1["chatStore"] - S1State["State:
isLoading, currentResponse
errorDialogState
activeProcessingState
chatLoadingStates
chatStreamingStates
abortControllers
processingStates
activeConversationId
isStreamingActive"] - S1LoadState["Loading State:
setChatLoading()
isChatLoading()
syncLoadingStateForChat()
clearUIState()
isChatLoadingPublic()
getAllLoadingChats()
getAllStreamingChats()"] - S1ProcState["Processing State:
setActiveProcessingConversation()
getProcessingState()
clearProcessingState()
getActiveProcessingState()
updateProcessingStateFromTimings()
getCurrentProcessingStateSync()
restoreProcessingStateFromMessages()"] - S1Stream["Streaming:
streamChatCompletion()
startStreaming()
stopStreaming()
stopGeneration()
isStreaming()"] - S1Error["Error Handling:
showErrorDialog()
dismissErrorDialog()
isAbortError()"] - S1Msg["Message Operations:
addMessage()
sendMessage()
updateMessage()
deleteMessage()
getDeletionInfo()"] - S1Regen["Regeneration:
regenerateMessage()
regenerateMessageWithBranching()
continueAssistantMessage()"] - S1Edit["Editing:
editAssistantMessage()
editUserMessagePreserveResponses()
editMessageWithBranching()
clearEditMode()
isEditModeActive()
getAddFilesHandler()
setEditModeActive()"] - S1Utils["Utilities:
getApiOptions()
parseTimingData()
getOrCreateAbortController()
getConversationModel()"] - end - subgraph SA["agenticStore"] - SAState["State:
sessions (Map)
isAnyRunning"] - SASession["Session Management:
getSession()
updateSession()
clearSession()
getActiveSessions()
isRunning()
currentTurn()
totalToolCalls()
lastError()
streamingToolCall()"] - SAConfig["Configuration:
getConfig()
maxTurns, maxToolPreviewLines"] - SAFlow["Agentic Loop:
runAgenticFlow()
executeAgenticLoop()
normalizeToolCalls()
emitToolCallResult()
extractBase64Attachments()"] - end - subgraph S2["conversationsStore"] - S2State["State:
conversations
activeConversation
activeMessages
isInitialized
pendingMcpServerOverrides
titleUpdateConfirmationCallback"] - S2Lifecycle["Lifecycle:
initialize()
loadConversations()
clearActiveConversation()"] - S2ConvCRUD["Conversation CRUD:
createConversation()
loadConversation()
deleteConversation()
deleteAll()
updateConversationName()
updateConversationTitleWithConfirmation()"] - S2MsgMgmt["Message Management:
refreshActiveMessages()
addMessageToActive()
updateMessageAtIndex()
findMessageIndex()
sliceActiveMessages()
removeMessageAtIndex()
getConversationMessages()"] - S2Nav["Navigation:
navigateToSibling()
updateCurrentNode()
updateConversationTimestamp()"] - S2McpOverrides["MCP Per-Chat Overrides:
getMcpServerOverride()
getAllMcpServerOverrides()
setMcpServerOverride()
toggleMcpServerForChat()
removeMcpServerOverride()
isMcpServerEnabledForChat()
clearPendingMcpServerOverrides()"] - S2Export["Import/Export:
downloadConversation()
exportAllConversations()
importConversations()
importConversationsData()
triggerDownload()"] - S2Utils["Utilities:
setTitleUpdateConfirmationCallback()"] - end - subgraph S3["modelsStore"] - S3State["State:
models, routerModels
selectedModelId
selectedModelName
loading, updating, error
modelLoadingStates
modelPropsCache
modelPropsFetching
propsCacheVersion"] - S3Getters["Computed Getters:
selectedModel
loadedModelIds
loadingModelIds
singleModelName"] - S3Modal["Modalities:
getModelModalities()
modelSupportsVision()
modelSupportsAudio()
getModelModalitiesArray()
getModelProps()
updateModelModalities()"] - S3Status["Status Queries:
isModelLoaded()
isModelOperationInProgress()
getModelStatus()
isModelPropsFetching()"] - S3Fetch["Data Fetching:
fetch()
fetchRouterModels()
fetchModelProps()
fetchModalitiesForLoadedModels()"] - S3Select["Model Selection:
selectModelById()
selectModelByName()
clearSelection()
findModelByName()
findModelById()
hasModel()"] - S3LoadUnload["Loading/Unloading Models:
loadModel()
unloadModel()
ensureModelLoaded()
waitForModelStatus()
pollForModelStatus()"] - S3Utils["Utilities:
toDisplayName()
clear()"] - end - subgraph S4["serverStore"] - S4State["State:
props
loading, error
role
fetchPromise"] - S4Getters["Getters:
defaultParams
contextSize
isRouterMode
isModelMode"] - S4Data["Data Handling:
fetch()
getErrorMessage()
clear()"] - S4Utils["Utilities:
detectRole()"] - end - subgraph S5["settingsStore"] - S5State["State:
config
theme
isInitialized
userOverrides"] - S5Lifecycle["Lifecycle:
initialize()
loadConfig()
saveConfig()
loadTheme()
saveTheme()"] - S5Update["Config Updates:
updateConfig()
updateMultipleConfig()
updateTheme()"] - S5Reset["Reset:
resetConfig()
resetTheme()
resetAll()
resetParameterToServerDefault()"] - S5Sync["Server Sync:
syncWithServerDefaults()
forceSyncWithServerDefaults()"] - S5Utils["Utilities:
getConfig()
getAllConfig()
getParameterInfo()
getParameterDiff()
getServerDefaults()
clearAllUserOverrides()"] - end - subgraph S6["mcpStore"] - S6State["State:
isInitializing, error
toolCount, connectedServers
healthChecks (Map)
connections (Map)
toolsIndex (Map)"] - S6Lifecycle["Lifecycle:
ensureInitialized()
initialize()
shutdown()
acquireConnection()
releaseConnection()"] - S6Health["Health Checks:
runHealthCheck()
runHealthChecksForServers()
updateHealthCheck()
getHealthCheckState()
clearHealthCheck()"] - S6Servers["Server Management:
getServers()
addServer()
updateServer()
removeServer()
getServerById()
getServerDisplayName()"] - S6Tools["Tool Operations:
getToolDefinitionsForLLM()
getToolNames()
hasTool()
getToolServer()
executeTool()
executeToolByName()"] - S6Prompts["Prompt Operations:
getAllPrompts()
getPrompt()
hasPromptsCapability()
getPromptCompletions()"] - end - subgraph S7["mcpResourceStore"] - S7State["State:
serverResources (Map)
cachedResources (Map)
subscriptions (Map)
attachments[]
isLoading"] - S7Resources["Resource Discovery:
setServerResources()
getServerResources()
getAllResourceInfos()
getAllTemplateInfos()
clearServerResources()"] - S7Cache["Caching:
cacheResourceContent()
getCachedContent()
invalidateCache()
clearCache()"] - S7Subs["Subscriptions:
addSubscription()
removeSubscription()
isSubscribed()
handleResourceUpdate()"] - S7Attach["Attachments:
addAttachment()
updateAttachmentContent()
removeAttachment()
clearAttachments()
toMessageExtras()"] - end - - subgraph ReactiveExports["⚡ Reactive Exports"] - direction LR - subgraph ChatExports["chatStore"] - RE1["isLoading()"] - RE2["currentResponse()"] - RE3["errorDialog()"] - RE4["activeProcessingState()"] - RE5["isChatStreaming()"] - RE6["isChatLoading()"] - RE7["getChatStreaming()"] - RE8["getAllLoadingChats()"] - RE9["getAllStreamingChats()"] - RE9a["isEditModeActive()"] - RE9b["getAddFilesHandler()"] - RE9c["setEditModeActive()"] - RE9d["clearEditMode()"] - end - subgraph AgenticExports["agenticStore"] - REA1["agenticIsRunning()"] - REA2["agenticCurrentTurn()"] - REA3["agenticTotalToolCalls()"] - REA4["agenticLastError()"] - REA5["agenticStreamingToolCall()"] - REA6["agenticIsAnyRunning()"] - end - subgraph ConvExports["conversationsStore"] - RE10["conversations()"] - RE11["activeConversation()"] - RE12["activeMessages()"] - RE13["isConversationsInitialized()"] - end - subgraph ModelsExports["modelsStore"] - RE15["modelOptions()"] - RE16["routerModels()"] - RE17["modelsLoading()"] - RE18["modelsUpdating()"] - RE19["modelsError()"] - RE20["selectedModelId()"] - RE21["selectedModelName()"] - RE22["selectedModelOption()"] - RE23["loadedModelIds()"] - RE24["loadingModelIds()"] - RE25["propsCacheVersion()"] - RE26["singleModelName()"] - end - subgraph ServerExports["serverStore"] - RE27["serverProps()"] - RE28["serverLoading()"] - RE29["serverError()"] - RE30["serverRole()"] - RE31["defaultParams()"] - RE32["contextSize()"] - RE33["isRouterMode()"] - RE34["isModelMode()"] - end - subgraph SettingsExports["settingsStore"] - RE35["config()"] - RE36["theme()"] - RE37["isInitialized()"] - end - subgraph MCPExports["mcpStore / mcpResourceStore"] - RE38["mcpResources()"] - RE39["mcpResourceAttachments()"] - RE40["mcpHasResourceAttachments()"] - RE41["mcpTotalResourceCount()"] - RE42["mcpResourcesLoading()"] - end - end - end - - subgraph Services["⚙️ Services"] - direction TB - subgraph SV1["ChatService"] - SV1Msg["Messaging:
sendMessage()"] - SV1Stream["Streaming:
handleStreamResponse()
handleNonStreamResponse()"] - SV1Convert["Conversion:
convertDbMessageToApiChatMessageData()
mergeToolCallDeltas()"] - SV1Utils["Utilities:
stripReasoningContent()
extractModelName()
parseErrorResponse()"] - end - subgraph SV2["ModelsService"] - SV2List["Listing:
list()
listRouter()"] - SV2LoadUnload["Load/Unload:
load()
unload()"] - SV2Status["Status:
isModelLoaded()
isModelLoading()"] - end - subgraph SV3["PropsService"] - SV3Fetch["Fetching:
fetch()
fetchForModel()"] - end - subgraph SV4["DatabaseService"] - SV4Conv["Conversations:
createConversation()
getConversation()
getAllConversations()
updateConversation()
deleteConversation()"] - SV4Msg["Messages:
createMessageBranch()
createRootMessage()
createSystemMessage()
getConversationMessages()
updateMessage()
deleteMessage()
deleteMessageCascading()"] - SV4Node["Navigation:
updateCurrentNode()"] - SV4Import["Import:
importConversations()"] - end - subgraph SV5["ParameterSyncService"] - SV5Extract["Extraction:
extractServerDefaults()"] - SV5Merge["Merging:
mergeWithServerDefaults()"] - SV5Info["Info:
getParameterInfo()
canSyncParameter()
getSyncableParameterKeys()
validateServerParameter()"] - SV5Diff["Diff:
createParameterDiff()"] - end - subgraph SV6["MCPService"] - SV6Transport["Transport:
createTransport()
WebSocket / StreamableHTTP / SSE"] - SV6Conn["Connection:
connect()
disconnect()"] - SV6Tools["Tools:
listTools()
callTool()"] - SV6Prompts["Prompts:
listPrompts()
getPrompt()"] - SV6Resources["Resources:
listResources()
listResourceTemplates()
readResource()
subscribeResource()
unsubscribeResource()"] - SV6Complete["Completions:
complete()"] - end - end - - subgraph ExternalMCP["🔌 External MCP Servers"] - EXT1["MCP Server 1
(WebSocket/StreamableHTTP/SSE)"] - EXT2["MCP Server N"] - end - - subgraph Storage["💾 Storage"] - ST1["IndexedDB"] - ST2["conversations"] - ST3["messages"] - ST5["LocalStorage"] - ST6["config"] - ST7["userOverrides"] - ST8["mcpServers"] - end - - subgraph APIs["🌐 llama-server API"] - API1["/v1/chat/completions"] - API2["/props
/props?model="] - API3["/models
/models/load
/models/unload"] - API4["/v1/models"] - end - - %% Routes render Components - R1 --> C_Screen - R2 --> C_Screen - RL --> C_Sidebar - - %% Layout runs MCP health checks on startup - RL --> S6 - - %% Component hierarchy - C_Screen --> C_Form & C_Messages & C_Settings - C_Messages --> C_Message - C_Message --> C_MessageUser - C_MessageUser --> C_MessageEditForm - C_MessageEditForm --> C_ModelsSelector - C_MessageEditForm --> C_Attach - C_Form --> C_ModelsSelector - C_Form --> C_Attach - C_Form --> C_McpServersSelector - C_Message --> C_Attach - - %% MCP Components hierarchy - C_Settings --> C_McpSettings - C_McpSettings --> C_McpServerCard - C_McpServerCard --> C_McpResourceBrowser - C_McpResourceBrowser --> C_McpResourcePreview - - %% Components use Hooks - C_Form --> H1 - C_Message --> H1 & H2 - C_MessageEditForm --> H1 - C_Screen --> H2 - - %% Hooks use Stores - H1 --> S3 & S4 - H2 --> S1 & S5 - - %% Components use Stores - C_Screen --> S1 & S2 - C_Messages --> S2 - C_Message --> S1 & S2 & S3 - C_Form --> S1 & S3 & S6 - C_Sidebar --> S2 - C_ModelsSelector --> S3 & S4 - C_Settings --> S5 - C_McpSettings --> S6 - C_McpServerCard --> S6 - C_McpResourceBrowser --> S6 & S7 - C_McpServersSelector --> S6 - - %% Stores export Reactive State - S1 -. exports .-> ChatExports - SA -. exports .-> AgenticExports - S2 -. exports .-> ConvExports - S3 -. exports .-> ModelsExports - S4 -. exports .-> ServerExports - S5 -. exports .-> SettingsExports - S6 -. exports .-> MCPExports - S7 -. exports .-> MCPExports - - %% chatStore → agenticStore (agentic loop orchestration) - S1 --> SA - SA --> SV1 - SA --> S6 - - %% Stores use Services - S1 --> SV1 & SV4 - S2 --> SV4 - S3 --> SV2 & SV3 - S4 --> SV3 - S5 --> SV5 - S6 --> SV6 - S7 --> SV6 - - %% Services to Storage - SV4 --> ST1 - ST1 --> ST2 & ST3 - SV5 --> ST5 - ST5 --> ST6 & ST7 & ST8 - - %% Services to APIs - SV1 --> API1 - SV2 --> API3 & API4 - SV3 --> API2 - - %% MCP → External Servers - SV6 --> EXT1 & EXT2 - - %% Styling - classDef routeStyle fill:#e1f5fe,stroke:#01579b,stroke-width:2px - classDef componentStyle fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px - classDef componentGroupStyle fill:#e1bee7,stroke:#7b1fa2,stroke-width:1px - classDef hookStyle fill:#fff8e1,stroke:#ff8f00,stroke-width:2px - classDef storeStyle fill:#fff3e0,stroke:#e65100,stroke-width:2px - classDef stateStyle fill:#ffe0b2,stroke:#e65100,stroke-width:1px - classDef methodStyle fill:#ffecb3,stroke:#e65100,stroke-width:1px - classDef reactiveStyle fill:#fffde7,stroke:#f9a825,stroke-width:1px - classDef serviceStyle fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px - classDef serviceMStyle fill:#c8e6c9,stroke:#2e7d32,stroke-width:1px - classDef externalStyle fill:#f3e5f5,stroke:#6a1b9a,stroke-width:2px,stroke-dasharray: 5 5 - classDef storageStyle fill:#fce4ec,stroke:#c2185b,stroke-width:2px - classDef apiStyle fill:#e3f2fd,stroke:#1565c0,stroke-width:2px - - class R1,R2,RL routeStyle - class C_Sidebar,C_Screen,C_Form,C_Messages,C_Message,C_MessageUser,C_MessageEditForm componentStyle - class C_ModelsSelector,C_Settings componentStyle - class C_Attach componentStyle - class C_McpSettings,C_McpServerCard,C_McpResourceBrowser,C_McpResourcePreview,C_McpServersSelector componentStyle - class H1,H2,H3 hookStyle - class LayoutComponents,ChatUIComponents,MCPComponents componentGroupStyle - class Hooks hookStyle - classDef agenticStyle fill:#e8eaf6,stroke:#283593,stroke-width:2px - classDef agenticMethodStyle fill:#c5cae9,stroke:#283593,stroke-width:1px - - class S1,S2,S3,S4,S5,SA,S6,S7 storeStyle - class S1State,S2State,S3State,S4State,S5State,SAState,S6State,S7State stateStyle - class S1Msg,S1Regen,S1Edit,S1Stream,S1LoadState,S1ProcState,S1Error,S1Utils methodStyle - class SASession,SAConfig,SAFlow methodStyle - class S2Lifecycle,S2ConvCRUD,S2MsgMgmt,S2Nav,S2McpOverrides,S2Export,S2Utils methodStyle - class S3Getters,S3Modal,S3Status,S3Fetch,S3Select,S3LoadUnload,S3Utils methodStyle - class S4Getters,S4Data,S4Utils methodStyle - class S5Lifecycle,S5Update,S5Reset,S5Sync,S5Utils methodStyle - class S6Lifecycle,S6Health,S6Servers,S6Tools,S6Prompts methodStyle - class S7Resources,S7Cache,S7Subs,S7Attach methodStyle - class ChatExports,AgenticExports,ConvExports,ModelsExports,ServerExports,SettingsExports,MCPExports reactiveStyle - class SV1,SV2,SV3,SV4,SV5,SV6 serviceStyle - class SV6Transport,SV6Conn,SV6Tools,SV6Prompts,SV6Resources,SV6Complete serviceMStyle - class EXT1,EXT2 externalStyle - class SV1Msg,SV1Stream,SV1Convert,SV1Utils serviceMStyle - class SV2List,SV2LoadUnload,SV2Status serviceMStyle - class SV3Fetch serviceMStyle - class SV4Conv,SV4Msg,SV4Node,SV4Import serviceMStyle - class SV5Extract,SV5Merge,SV5Info,SV5Diff serviceMStyle - class ST1,ST2,ST3,ST5,ST6,ST7,ST8 storageStyle - class API1,API2,API3,API4 apiStyle -``` diff --git a/tools/ui/docs/flows/chat-flow.md b/tools/ui/docs/flows/chat-flow.md deleted file mode 100644 index 296693c6a5..0000000000 --- a/tools/ui/docs/flows/chat-flow.md +++ /dev/null @@ -1,228 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 ChatForm / ChatMessage - participant chatStore as 🗄️ chatStore - participant agenticStore as 🗄️ agenticStore - participant convStore as 🗄️ conversationsStore - participant settingsStore as 🗄️ settingsStore - participant mcpStore as 🗄️ mcpStore - participant ChatSvc as ⚙️ ChatService - participant DbSvc as ⚙️ DatabaseService - participant API as 🌐 /v1/chat/completions - - Note over chatStore: State:
isLoading, currentResponse
errorDialogState, activeProcessingState
chatLoadingStates (Map)
chatStreamingStates (Map)
abortControllers (Map)
processingStates (Map) - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 💬 SEND MESSAGE - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: sendMessage(content, extras) - activate chatStore - - chatStore->>chatStore: setChatLoading(convId, true) - chatStore->>chatStore: clearChatStreaming(convId) - - alt no active conversation - chatStore->>convStore: createConversation() - Note over convStore: → see conversations-flow.mmd - end - - chatStore->>mcpStore: consumeResourceAttachmentsAsExtras() - Note right of mcpStore: Converts pending MCP resource
attachments into message extras - - chatStore->>chatStore: addMessage("user", content, extras) - chatStore->>DbSvc: createMessageBranch(userMsg, parentId) - chatStore->>convStore: addMessageToActive(userMsg) - chatStore->>convStore: updateCurrentNode(userMsg.id) - - chatStore->>chatStore: createAssistantMessage(userMsg.id) - chatStore->>DbSvc: createMessageBranch(assistantMsg, userMsg.id) - chatStore->>convStore: addMessageToActive(assistantMsg) - - chatStore->>chatStore: streamChatCompletion(messages, assistantMsg) - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🌊 STREAMING (with agentic flow detection) - %% ═══════════════════════════════════════════════════════════════════════════ - - activate chatStore - chatStore->>chatStore: startStreaming() - Note right of chatStore: isStreamingActive = true - - chatStore->>chatStore: setActiveProcessingConversation(convId) - chatStore->>chatStore: getOrCreateAbortController(convId) - Note right of chatStore: abortControllers.set(convId, new AbortController()) - - chatStore->>chatStore: getApiOptions() - Note right of chatStore: Merge from settingsStore.config:
temperature, max_tokens, top_p, etc. - - alt agenticConfig.enabled && mcpStore has connected servers - chatStore->>agenticStore: runAgenticFlow(convId, messages, assistantMsg, options, signal) - Note over agenticStore: Multi-turn agentic loop:
1. Call ChatService.sendMessage()
2. If response has tool_calls → execute via mcpStore
3. Append tool results as messages
4. Loop until no more tool_calls or maxTurns
→ see agentic flow details below - agenticStore-->>chatStore: final response with timings - else standard (non-agentic) flow - chatStore->>ChatSvc: sendMessage(messages, options, signal) - end - - activate ChatSvc - - ChatSvc->>ChatSvc: convertDbMessageToApiChatMessageData(messages) - Note right of ChatSvc: DatabaseMessage[] → ApiChatMessageData[]
Process attachments (images, PDFs, audio) - - ChatSvc->>API: POST /v1/chat/completions - Note right of API: {messages, model?, stream: true, ...params} - - loop SSE chunks - API-->>ChatSvc: data: {"choices":[{"delta":{...}}]} - ChatSvc->>ChatSvc: handleStreamResponse(response) - - alt content chunk - ChatSvc-->>chatStore: onChunk(content) - chatStore->>chatStore: setChatStreaming(convId, response, msgId) - Note right of chatStore: currentResponse = $state(accumulated) - chatStore->>convStore: updateMessageAtIndex(idx, {content}) - end - - alt reasoning chunk - ChatSvc-->>chatStore: onReasoningChunk(reasoning) - chatStore->>convStore: updateMessageAtIndex(idx, {thinking}) - end - - alt tool_calls chunk - ChatSvc-->>chatStore: onToolCallChunk(toolCalls) - chatStore->>convStore: updateMessageAtIndex(idx, {toolCalls}) - end - - alt model info - ChatSvc-->>chatStore: onModel(modelName) - chatStore->>chatStore: recordModel(modelName) - chatStore->>DbSvc: updateMessage(msgId, {model}) - end - - alt timings (during stream) - ChatSvc-->>chatStore: onTimings(timings, promptProgress) - chatStore->>chatStore: updateProcessingStateFromTimings() - end - - chatStore-->>UI: reactive $state update - end - - API-->>ChatSvc: data: [DONE] - ChatSvc-->>chatStore: onComplete(content, reasoning, timings, toolCalls) - deactivate ChatSvc - - chatStore->>chatStore: stopStreaming() - chatStore->>DbSvc: updateMessage(msgId, {content, timings, model}) - chatStore->>convStore: updateCurrentNode(msgId) - chatStore->>chatStore: setChatLoading(convId, false) - chatStore->>chatStore: clearChatStreaming(convId) - chatStore->>chatStore: clearProcessingState(convId) - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ⏹️ STOP GENERATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: stopGeneration() - activate chatStore - chatStore->>chatStore: savePartialResponseIfNeeded(convId) - Note right of chatStore: Save currentResponse to DB if non-empty - chatStore->>chatStore: abortControllers.get(convId).abort() - Note right of chatStore: fetch throws AbortError → caught by isAbortError() - chatStore->>chatStore: stopStreaming() - chatStore->>chatStore: setChatLoading(convId, false) - chatStore->>chatStore: clearChatStreaming(convId) - chatStore->>chatStore: clearProcessingState(convId) - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🔁 REGENERATE - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: regenerateMessageWithBranching(msgId, model?) - activate chatStore - chatStore->>convStore: findMessageIndex(msgId) - chatStore->>chatStore: Get parent of target message - chatStore->>chatStore: createAssistantMessage(parentId) - chatStore->>DbSvc: createMessageBranch(newAssistantMsg, parentId) - chatStore->>convStore: refreshActiveMessages() - Note right of chatStore: Same streaming flow - chatStore->>chatStore: streamChatCompletion(...) - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ➡️ CONTINUE - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: continueAssistantMessage(msgId) - activate chatStore - chatStore->>chatStore: Get existing content from message - chatStore->>chatStore: streamChatCompletion(..., existingContent) - Note right of chatStore: Appends to existing message content - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ✏️ EDIT USER MESSAGE - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: editMessageWithBranching(msgId, newContent, extras) - activate chatStore - chatStore->>chatStore: Get parent of target message - chatStore->>DbSvc: createMessageBranch(editedMsg, parentId) - chatStore->>convStore: refreshActiveMessages() - Note right of chatStore: Creates new branch, original preserved - chatStore->>chatStore: createAssistantMessage(editedMsg.id) - chatStore->>chatStore: streamChatCompletion(...) - Note right of chatStore: Automatically regenerates response - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ❌ ERROR HANDLING - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over chatStore: On stream error (non-abort): - chatStore->>chatStore: showErrorDialog(type, message) - Note right of chatStore: errorDialogState = {type: 'timeout'|'server', message} - chatStore->>convStore: removeMessageAtIndex(failedMsgIdx) - chatStore->>DbSvc: deleteMessage(failedMsgId) - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🤖 AGENTIC LOOP (when agenticConfig.enabled) - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over agenticStore: agenticStore.runAgenticFlow(convId, messages, assistantMsg, options, signal) - activate agenticStore - agenticStore->>agenticStore: getSession(convId) or create new - agenticStore->>agenticStore: updateSession(turn: 0, running: true) - - loop executeAgenticLoop (until no tool_calls or maxTurns) - agenticStore->>agenticStore: turn++ - agenticStore->>ChatSvc: sendMessage(messages, options, signal) - ChatSvc->>API: POST /v1/chat/completions - API-->>ChatSvc: response with potential tool_calls - ChatSvc-->>agenticStore: onComplete(content, reasoning, timings, toolCalls) - - alt response has tool_calls - agenticStore->>agenticStore: normalizeToolCalls(toolCalls) - loop for each tool_call - agenticStore->>agenticStore: updateSession(streamingToolCall) - agenticStore->>mcpStore: executeTool(mcpCall, signal) - mcpStore-->>agenticStore: tool result - agenticStore->>agenticStore: extractBase64Attachments(result) - agenticStore->>agenticStore: emitToolCallResult(convId, ...) - agenticStore->>convStore: addMessageToActive(toolResultMsg) - agenticStore->>DbSvc: createMessageBranch(toolResultMsg) - end - agenticStore->>agenticStore: Create new assistantMsg for next turn - Note right of agenticStore: Continue loop with updated messages - else no tool_calls (final response) - agenticStore->>agenticStore: buildFinalTimings(allTurns) - Note right of agenticStore: Break loop, return final response - end - end - - agenticStore->>agenticStore: updateSession(running: false) - agenticStore-->>chatStore: final content, timings, model - deactivate agenticStore -``` diff --git a/tools/ui/docs/flows/conversations-flow.md b/tools/ui/docs/flows/conversations-flow.md deleted file mode 100644 index bd2309bc03..0000000000 --- a/tools/ui/docs/flows/conversations-flow.md +++ /dev/null @@ -1,183 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 ChatSidebar / ChatScreen - participant convStore as 🗄️ conversationsStore - participant chatStore as 🗄️ chatStore - participant DbSvc as ⚙️ DatabaseService - participant IDB as 💾 IndexedDB - - Note over convStore: State:
conversations: DatabaseConversation[]
activeConversation: DatabaseConversation | null
activeMessages: DatabaseMessage[]
isInitialized: boolean
pendingMcpServerOverrides: Map<string, McpServerOverride> - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 🚀 INITIALIZATION - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over convStore: Auto-initialized in constructor (browser only) - convStore->>convStore: initialize() - activate convStore - convStore->>convStore: loadConversations() - convStore->>DbSvc: getAllConversations() - DbSvc->>IDB: SELECT * FROM conversations ORDER BY lastModified DESC - IDB-->>DbSvc: Conversation[] - DbSvc-->>convStore: conversations - convStore->>convStore: conversations = $state(data) - convStore->>convStore: isInitialized = true - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: ➕ CREATE CONVERSATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: createConversation(name?) - activate convStore - convStore->>DbSvc: createConversation(name || "New Chat") - DbSvc->>IDB: INSERT INTO conversations - IDB-->>DbSvc: conversation {id, name, lastModified, currNode: ""} - DbSvc-->>convStore: conversation - convStore->>convStore: conversations.unshift(conversation) - convStore->>convStore: activeConversation = $state(conversation) - convStore->>convStore: activeMessages = $state([]) - - alt pendingMcpServerOverrides has entries - loop each pending override - convStore->>DbSvc: Store MCP server override for new conversation - end - convStore->>convStore: clearPendingMcpServerOverrides() - end - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 📂 LOAD CONVERSATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: loadConversation(convId) - activate convStore - convStore->>DbSvc: getConversation(convId) - DbSvc->>IDB: SELECT * FROM conversations WHERE id = ? - IDB-->>DbSvc: conversation - convStore->>convStore: activeConversation = $state(conversation) - - convStore->>convStore: refreshActiveMessages() - convStore->>DbSvc: getConversationMessages(convId) - DbSvc->>IDB: SELECT * FROM messages WHERE convId = ? - IDB-->>DbSvc: allMessages[] - convStore->>convStore: filterByLeafNodeId(allMessages, currNode) - Note right of convStore: Filter to show only current branch path - convStore->>convStore: activeMessages = $state(filtered) - - Note right of convStore: Route (+page.svelte) then calls:
chatStore.syncLoadingStateForChat(convId) - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 🌳 MESSAGE BRANCHING MODEL - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over IDB: Message Tree Structure:
- Each message has parent (null for root)
- Each message has children[] array
- Conversation.currNode points to active leaf
- filterByLeafNodeId() traverses from root to currNode - - rect rgb(240, 240, 255) - Note over convStore: Example Branch Structure: - Note over convStore: root → user1 → assistant1 → user2 → assistant2a (currNode)
↘ assistant2b (alt branch) - end - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: ↔️ BRANCH NAVIGATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: navigateToSibling(msgId, direction) - activate convStore - convStore->>convStore: Find message in activeMessages - convStore->>convStore: Get parent message - convStore->>convStore: Find sibling in parent.children[] - convStore->>convStore: findLeafNode(siblingId, allMessages) - Note right of convStore: Navigate to leaf of sibling branch - convStore->>convStore: updateCurrentNode(leafId) - convStore->>DbSvc: updateCurrentNode(convId, leafId) - DbSvc->>IDB: UPDATE conversations SET currNode = ? - convStore->>convStore: refreshActiveMessages() - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 📝 UPDATE CONVERSATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: updateConversationName(convId, newName) - activate convStore - convStore->>DbSvc: updateConversation(convId, {name: newName}) - DbSvc->>IDB: UPDATE conversations SET name = ? - convStore->>convStore: Update in conversations array - deactivate convStore - - Note over convStore: Auto-title update (after first response): - convStore->>convStore: updateConversationTitleWithConfirmation() - convStore->>convStore: titleUpdateConfirmationCallback?() - Note right of convStore: Shows dialog if title would change - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 🗑️ DELETE CONVERSATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: deleteConversation(convId) - activate convStore - convStore->>DbSvc: deleteConversation(convId) - DbSvc->>IDB: DELETE FROM conversations WHERE id = ? - DbSvc->>IDB: DELETE FROM messages WHERE convId = ? - convStore->>convStore: conversations.filter(c => c.id !== convId) - alt deleted active conversation - convStore->>convStore: clearActiveConversation() - end - deactivate convStore - - UI->>convStore: deleteAll() - activate convStore - convStore->>DbSvc: Delete all conversations and messages - convStore->>convStore: conversations = [] - convStore->>convStore: clearActiveConversation() - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: � MCP SERVER PER-CHAT OVERRIDES - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over convStore: Conversations can override which MCP servers are enabled. - Note over convStore: Uses pendingMcpServerOverrides before conversation
is created, then persists to conversation metadata. - - UI->>convStore: setMcpServerOverride(convId, serverName, override) - Note right of convStore: override = {enabled: boolean} - - UI->>convStore: toggleMcpServerForChat(convId, serverName, enabled) - activate convStore - convStore->>convStore: setMcpServerOverride(convId, serverName, {enabled}) - deactivate convStore - - UI->>convStore: isMcpServerEnabledForChat(convId, serverName) - Note right of convStore: Check override → fall back to global MCP config - - UI->>convStore: getAllMcpServerOverrides(convId) - Note right of convStore: Returns all overrides for a conversation - - UI->>convStore: removeMcpServerOverride(convId, serverName) - UI->>convStore: getMcpServerOverride(convId, serverName) - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 📤 EXPORT / 📥 IMPORT - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: exportAllConversations() - activate convStore - convStore->>DbSvc: getAllConversations() - loop each conversation - convStore->>DbSvc: getConversationMessages(convId) - end - convStore->>convStore: triggerDownload(JSON blob) - deactivate convStore - - UI->>convStore: importConversations(file) - activate convStore - convStore->>convStore: Parse JSON file - convStore->>convStore: importConversationsData(parsed) - convStore->>DbSvc: importConversations(parsed) - Note right of DbSvc: Skips duplicate conversations
(checks existing by ID) - DbSvc->>IDB: INSERT conversations + messages (skip existing) - convStore->>convStore: loadConversations() - deactivate convStore -``` diff --git a/tools/ui/docs/flows/data-flow-simplified-model-mode.md b/tools/ui/docs/flows/data-flow-simplified-model-mode.md deleted file mode 100644 index 07b362147f..0000000000 --- a/tools/ui/docs/flows/data-flow-simplified-model-mode.md +++ /dev/null @@ -1,45 +0,0 @@ -```mermaid -%% MODEL Mode Data Flow (single model) -%% Detailed flows: ./flows/server-flow.mmd, ./flows/models-flow.mmd, ./flows/chat-flow.mmd - -sequenceDiagram - participant User as 👤 User - participant UI as 🧩 UI - participant Stores as 🗄️ Stores - participant DB as 💾 IndexedDB - participant API as 🌐 llama-server - - Note over User,API: 🚀 Initialization (see: server-flow.mmd, models-flow.mmd) - - UI->>Stores: initialize() - Stores->>DB: load conversations - Stores->>API: GET /props - API-->>Stores: server config + modalities - Stores->>API: GET /v1/models - API-->>Stores: single model (auto-selected) - - Note over User,API: 💬 Chat Flow (see: chat-flow.mmd) - - User->>UI: send message - UI->>Stores: sendMessage() - Stores->>DB: save user message - Stores->>API: POST /v1/chat/completions (stream) - loop streaming - API-->>Stores: SSE chunks - Stores-->>UI: reactive update - end - API-->>Stores: done + timings - Stores->>DB: save assistant message - - Note over User,API: 🔁 Regenerate - - User->>UI: regenerate - Stores->>DB: create message branch - Note right of Stores: same streaming flow - - Note over User,API: ⏹️ Stop - - User->>UI: stop - Stores->>Stores: abort stream - Stores->>DB: save partial response -``` diff --git a/tools/ui/docs/flows/data-flow-simplified-router-mode.md b/tools/ui/docs/flows/data-flow-simplified-router-mode.md deleted file mode 100644 index bccacf5684..0000000000 --- a/tools/ui/docs/flows/data-flow-simplified-router-mode.md +++ /dev/null @@ -1,77 +0,0 @@ -```mermaid -%% ROUTER Mode Data Flow (multi-model) -%% Detailed flows: ./flows/server-flow.mmd, ./flows/models-flow.mmd, ./flows/chat-flow.mmd - -sequenceDiagram - participant User as 👤 User - participant UI as 🧩 UI - participant Stores as 🗄️ Stores - participant DB as 💾 IndexedDB - participant API as 🌐 llama-server - - Note over User,API: 🚀 Initialization (see: server-flow.mmd, models-flow.mmd) - - UI->>Stores: initialize() - Stores->>DB: load conversations - Stores->>API: GET /props - API-->>Stores: {role: "router"} - Stores->>API: GET /v1/models - API-->>Stores: models[] with status (loaded/available) - loop each loaded model - Stores->>API: GET /props?model=X - API-->>Stores: modalities (vision/audio) - end - - Note over User,API: 🔄 Model Selection (see: models-flow.mmd) - - User->>UI: select model - alt model not loaded - Stores->>API: POST /models/load - loop poll status - Stores->>API: GET /v1/models - API-->>Stores: check if loaded - end - Stores->>API: GET /props?model=X - API-->>Stores: cache modalities - end - Stores->>Stores: validate modalities vs conversation - alt valid - Stores->>Stores: select model - else invalid - Stores->>API: POST /models/unload - UI->>User: show error toast - end - - Note over User,API: 💬 Chat Flow (see: chat-flow.mmd) - - User->>UI: send message - UI->>Stores: sendMessage() - Stores->>DB: save user message - Stores->>API: POST /v1/chat/completions {model: X} - Note right of API: router forwards to model - loop streaming - API-->>Stores: SSE chunks + model info - Stores-->>UI: reactive update - end - API-->>Stores: done + timings - Stores->>DB: save assistant message + model used - - Note over User,API: 🔁 Regenerate (optional: different model) - - User->>UI: regenerate - Stores->>Stores: validate modalities up to this message - Stores->>DB: create message branch - Note right of Stores: same streaming flow - - Note over User,API: ⏹️ Stop - - User->>UI: stop - Stores->>Stores: abort stream - Stores->>DB: save partial response - - Note over User,API: 🗑️ LRU Unloading - - Note right of API: Server auto-unloads LRU models
when cache full - User->>UI: select unloaded model - Note right of Stores: triggers load flow again -``` diff --git a/tools/ui/docs/flows/database-flow.md b/tools/ui/docs/flows/database-flow.md deleted file mode 100644 index 38cd6941cf..0000000000 --- a/tools/ui/docs/flows/database-flow.md +++ /dev/null @@ -1,174 +0,0 @@ -```mermaid -sequenceDiagram - participant Store as 🗄️ Stores - participant DbSvc as ⚙️ DatabaseService - participant Dexie as 📦 Dexie ORM - participant IDB as 💾 IndexedDB - - Note over DbSvc: Stateless service - all methods static
Database: "LlamacppWebui" - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 📊 SCHEMA - %% ═══════════════════════════════════════════════════════════════════════════ - - rect rgb(240, 248, 255) - Note over IDB: conversations table:
id (PK), lastModified, currNode, name - end - - rect rgb(255, 248, 240) - Note over IDB: messages table:
id (PK), convId (FK), type, role, timestamp,
parent, children[], content, thinking,
toolCalls, extra[], model, timings - end - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 💬 CONVERSATIONS CRUD - %% ═══════════════════════════════════════════════════════════════════════════ - - Store->>DbSvc: createConversation(name) - activate DbSvc - DbSvc->>DbSvc: Generate UUID - DbSvc->>Dexie: db.conversations.add({id, name, lastModified, currNode: ""}) - Dexie->>IDB: INSERT - IDB-->>Dexie: success - DbSvc-->>Store: DatabaseConversation - deactivate DbSvc - - Store->>DbSvc: getConversation(convId) - DbSvc->>Dexie: db.conversations.get(convId) - Dexie->>IDB: SELECT WHERE id = ? - IDB-->>DbSvc: DatabaseConversation - - Store->>DbSvc: getAllConversations() - DbSvc->>Dexie: db.conversations.orderBy('lastModified').reverse().toArray() - Dexie->>IDB: SELECT ORDER BY lastModified DESC - IDB-->>DbSvc: DatabaseConversation[] - - Store->>DbSvc: updateConversation(convId, updates) - DbSvc->>Dexie: db.conversations.update(convId, {...updates, lastModified}) - Dexie->>IDB: UPDATE - - Store->>DbSvc: deleteConversation(convId) - activate DbSvc - DbSvc->>Dexie: db.conversations.delete(convId) - Dexie->>IDB: DELETE FROM conversations - DbSvc->>Dexie: db.messages.where('convId').equals(convId).delete() - Dexie->>IDB: DELETE FROM messages WHERE convId = ? - deactivate DbSvc - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 📝 MESSAGES CRUD - %% ═══════════════════════════════════════════════════════════════════════════ - - Store->>DbSvc: createRootMessage(convId) - activate DbSvc - DbSvc->>DbSvc: Create root message {type: "root", parent: null} - DbSvc->>Dexie: db.messages.add(rootMsg) - Dexie->>IDB: INSERT - DbSvc-->>Store: rootMessageId - deactivate DbSvc - - Store->>DbSvc: createSystemMessage(convId, content, parentId) - activate DbSvc - DbSvc->>DbSvc: Create message {role: "system", parent: parentId} - DbSvc->>Dexie: db.messages.add(systemMsg) - Dexie->>IDB: INSERT - DbSvc-->>Store: DatabaseMessage - deactivate DbSvc - - Store->>DbSvc: createMessageBranch(message, parentId) - activate DbSvc - DbSvc->>DbSvc: Generate UUID for new message - DbSvc->>Dexie: db.messages.add({...message, id, parent: parentId}) - Dexie->>IDB: INSERT message - - alt parentId exists - DbSvc->>Dexie: db.messages.get(parentId) - Dexie->>IDB: SELECT parent - DbSvc->>DbSvc: parent.children.push(newId) - DbSvc->>Dexie: db.messages.update(parentId, {children}) - Dexie->>IDB: UPDATE parent.children - end - - DbSvc->>Dexie: db.conversations.update(convId, {currNode: newId}) - Dexie->>IDB: UPDATE conversation.currNode - DbSvc-->>Store: DatabaseMessage - deactivate DbSvc - - Store->>DbSvc: getConversationMessages(convId) - DbSvc->>Dexie: db.messages.where('convId').equals(convId).toArray() - Dexie->>IDB: SELECT WHERE convId = ? - IDB-->>DbSvc: DatabaseMessage[] - - Store->>DbSvc: updateMessage(msgId, updates) - DbSvc->>Dexie: db.messages.update(msgId, updates) - Dexie->>IDB: UPDATE - - Store->>DbSvc: deleteMessage(msgId) - DbSvc->>Dexie: db.messages.delete(msgId) - Dexie->>IDB: DELETE - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 🌳 BRANCHING OPERATIONS - %% ═══════════════════════════════════════════════════════════════════════════ - - Store->>DbSvc: updateCurrentNode(convId, nodeId) - DbSvc->>Dexie: db.conversations.update(convId, {currNode: nodeId, lastModified}) - Dexie->>IDB: UPDATE - - Store->>DbSvc: deleteMessageCascading(msgId) - activate DbSvc - DbSvc->>DbSvc: findDescendantMessages(msgId, allMessages) - Note right of DbSvc: Recursively find all children - loop each descendant - DbSvc->>Dexie: db.messages.delete(descendantId) - Dexie->>IDB: DELETE - end - DbSvc->>Dexie: db.messages.delete(msgId) - Dexie->>IDB: DELETE target message - - alt target message has a parent - DbSvc->>Dexie: db.messages.get(parentId) - DbSvc->>DbSvc: parent.children.filter(id !== msgId) - DbSvc->>Dexie: db.messages.update(parentId, {children}) - Note right of DbSvc: Remove deleted message from parent's children[] - end - deactivate DbSvc - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 📥 IMPORT - %% ═══════════════════════════════════════════════════════════════════════════ - - Store->>DbSvc: importConversations(data) - activate DbSvc - loop each conversation in data - DbSvc->>Dexie: db.conversations.get(conv.id) - alt conversation already exists - Note right of DbSvc: Skip duplicate (keep existing) - else conversation is new - DbSvc->>Dexie: db.conversations.add(conversation) - Dexie->>IDB: INSERT conversation - loop each message - DbSvc->>Dexie: db.messages.add(message) - Dexie->>IDB: INSERT message - end - end - end - deactivate DbSvc - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 🔗 MESSAGE TREE UTILITIES - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over DbSvc: Used by stores (imported from utils): - - rect rgb(240, 255, 240) - Note over DbSvc: filterByLeafNodeId(messages, leafId)
→ Returns path from root to leaf
→ Used to display current branch - end - - rect rgb(240, 255, 240) - Note over DbSvc: findLeafNode(startId, messages)
→ Traverse to deepest child
→ Used for branch navigation - end - - rect rgb(240, 255, 240) - Note over DbSvc: findDescendantMessages(msgId, messages)
→ Find all children recursively
→ Used for cascading deletes - end -``` diff --git a/tools/ui/docs/flows/mcp-flow.md b/tools/ui/docs/flows/mcp-flow.md deleted file mode 100644 index c8aa666599..0000000000 --- a/tools/ui/docs/flows/mcp-flow.md +++ /dev/null @@ -1,226 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 McpServersSettings / ChatForm - participant chatStore as 🗄️ chatStore - participant mcpStore as 🗄️ mcpStore - participant mcpResStore as 🗄️ mcpResourceStore - participant convStore as 🗄️ conversationsStore - participant MCPSvc as ⚙️ MCPService - participant LS as 💾 LocalStorage - participant ExtMCP as 🔌 External MCP Server - - Note over mcpStore: State:
isInitializing, error
toolCount, connectedServers
healthChecks (Map)
connections (Map)
toolsIndex (Map)
serverConfigs (Map) - - Note over mcpResStore: State:
serverResources (Map)
cachedResources (Map)
subscriptions (Map)
attachments[] - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 🚀 INITIALIZATION (App Startup) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpStore: ensureInitialized() - activate mcpStore - - mcpStore->>LS: get(MCP_SERVERS_LOCALSTORAGE_KEY) - LS-->>mcpStore: MCPServerSettingsEntry[] - - mcpStore->>mcpStore: parseServerSettings(servers) - Note right of mcpStore: Filter enabled servers
Build MCPServerConfig objects
Per-chat overrides checked via convStore - - loop For each enabled server - mcpStore->>mcpStore: runHealthCheck(serverId) - mcpStore->>mcpStore: updateHealthCheck(id, CONNECTING) - - mcpStore->>MCPSvc: connect(serverName, config, clientInfo, capabilities, onPhase) - activate MCPSvc - - MCPSvc->>MCPSvc: createTransport(config) - Note right of MCPSvc: WebSocket / StreamableHTTP / SSE
with optional CORS proxy - - MCPSvc->>ExtMCP: Transport handshake - ExtMCP-->>MCPSvc: Connection established - - MCPSvc->>ExtMCP: Initialize request - Note right of ExtMCP: Exchange capabilities
Server info, protocol version - - ExtMCP-->>MCPSvc: InitializeResult (serverInfo, capabilities) - - MCPSvc->>ExtMCP: listTools() - ExtMCP-->>MCPSvc: Tool[] - - MCPSvc-->>mcpStore: MCPConnection - deactivate MCPSvc - - mcpStore->>mcpStore: connections.set(serverName, connection) - mcpStore->>mcpStore: indexTools(connection.tools, serverName) - Note right of mcpStore: toolsIndex.set(toolName, serverName)
Handle name conflicts with prefixes - - mcpStore->>mcpStore: updateHealthCheck(id, SUCCESS) - mcpStore->>mcpStore: _connectedServers.push(serverName) - - alt Server supports resources - mcpStore->>MCPSvc: listAllResources(connection) - MCPSvc->>ExtMCP: listResources() - ExtMCP-->>MCPSvc: MCPResource[] - MCPSvc-->>mcpStore: resources - - mcpStore->>MCPSvc: listAllResourceTemplates(connection) - MCPSvc->>ExtMCP: listResourceTemplates() - ExtMCP-->>MCPSvc: MCPResourceTemplate[] - MCPSvc-->>mcpStore: templates - - mcpStore->>mcpResStore: setServerResources(serverName, resources, templates) - end - end - - mcpStore->>mcpStore: _isInitializing = false - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 🔧 TOOL EXECUTION (Chat with Tools) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpStore: executeTool(mcpCall: MCPToolCall, signal?) - activate mcpStore - - mcpStore->>mcpStore: toolsIndex.get(mcpCall.function.name) - Note right of mcpStore: Resolve serverName from toolsIndex
MCPToolCall = {id, type, function: {name, arguments}} - - mcpStore->>mcpStore: acquireConnection() - Note right of mcpStore: activeFlowCount++
Prevent shutdown during execution - - mcpStore->>mcpStore: connection = connections.get(serverName) - - mcpStore->>MCPSvc: callTool(connection, {name, arguments}, signal) - activate MCPSvc - - MCPSvc->>MCPSvc: throwIfAborted(signal) - MCPSvc->>ExtMCP: callTool(name, arguments) - - alt Tool execution success - ExtMCP-->>MCPSvc: ToolCallResult (content, isError) - MCPSvc->>MCPSvc: formatToolResult(result) - Note right of MCPSvc: Handle text, image (base64),
embedded resource content - MCPSvc-->>mcpStore: ToolExecutionResult - else Tool execution error - ExtMCP-->>MCPSvc: Error - MCPSvc-->>mcpStore: throw Error - else Aborted - MCPSvc-->>mcpStore: throw AbortError - end - - deactivate MCPSvc - - mcpStore->>mcpStore: releaseConnection() - Note right of mcpStore: activeFlowCount-- - - mcpStore-->>UI: ToolExecutionResult - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: � RESOURCE ATTACHMENT CONSUMPTION - %% ═══════════════════════════════════════════════════════════════════════════ - - chatStore->>mcpStore: consumeResourceAttachmentsAsExtras() - activate mcpStore - mcpStore->>mcpResStore: getAttachments() - mcpResStore-->>mcpStore: MCPResourceAttachment[] - mcpStore->>mcpStore: Convert attachments to message extras - mcpStore->>mcpResStore: clearAttachments() - mcpStore-->>chatStore: MessageExtra[] (for user message) - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: �📝 PROMPT OPERATIONS - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpStore: getAllPrompts() - activate mcpStore - - loop For each connected server with prompts capability - mcpStore->>MCPSvc: listPrompts(connection) - MCPSvc->>ExtMCP: listPrompts() - ExtMCP-->>MCPSvc: Prompt[] - MCPSvc-->>mcpStore: prompts - end - - mcpStore-->>UI: MCPPromptInfo[] (with serverName) - deactivate mcpStore - - UI->>mcpStore: getPrompt(serverName, promptName, args?) - activate mcpStore - - mcpStore->>MCPSvc: getPrompt(connection, name, args) - MCPSvc->>ExtMCP: getPrompt({name, arguments}) - ExtMCP-->>MCPSvc: GetPromptResult (messages) - MCPSvc-->>mcpStore: GetPromptResult - - mcpStore-->>UI: GetPromptResult - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 📁 RESOURCE OPERATIONS - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpResStore: addAttachment(resourceInfo) - activate mcpResStore - mcpResStore->>mcpResStore: Create MCPResourceAttachment (loading: true) - mcpResStore-->>UI: attachment - - UI->>mcpStore: readResource(serverName, uri) - activate mcpStore - - mcpStore->>MCPSvc: readResource(connection, uri) - MCPSvc->>ExtMCP: readResource({uri}) - ExtMCP-->>MCPSvc: MCPReadResourceResult (contents) - MCPSvc-->>mcpStore: contents - - mcpStore-->>UI: MCPResourceContent[] - deactivate mcpStore - - UI->>mcpResStore: updateAttachmentContent(attachmentId, content) - mcpResStore->>mcpResStore: cacheResourceContent(resource, content) - deactivate mcpResStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 🔄 AUTO-RECONNECTION - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over mcpStore: On WebSocket close or connection error: - mcpStore->>mcpStore: autoReconnect(serverName, attempt) - activate mcpStore - - mcpStore->>mcpStore: Calculate backoff delay - Note right of mcpStore: delay = min(30s, 1s * 2^attempt) - - mcpStore->>mcpStore: Wait for delay - mcpStore->>mcpStore: reconnectServer(serverName) - - alt Reconnection success - mcpStore->>mcpStore: updateHealthCheck(id, SUCCESS) - else Max attempts reached - mcpStore->>mcpStore: updateHealthCheck(id, ERROR) - end - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 🛑 SHUTDOWN - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpStore: shutdown() - activate mcpStore - - mcpStore->>mcpStore: Wait for activeFlowCount == 0 - - loop For each connection - mcpStore->>MCPSvc: disconnect(connection) - MCPSvc->>MCPSvc: transport.onclose = undefined - MCPSvc->>ExtMCP: close() - end - - mcpStore->>mcpStore: connections.clear() - mcpStore->>mcpStore: toolsIndex.clear() - mcpStore->>mcpStore: _connectedServers = [] - - mcpStore->>mcpResStore: clear() - deactivate mcpStore -``` diff --git a/tools/ui/docs/flows/models-flow.md b/tools/ui/docs/flows/models-flow.md deleted file mode 100644 index c3031b7292..0000000000 --- a/tools/ui/docs/flows/models-flow.md +++ /dev/null @@ -1,181 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 ModelsSelector - participant Hooks as 🪝 useModelChangeValidation - participant modelsStore as 🗄️ modelsStore - participant serverStore as 🗄️ serverStore - participant convStore as 🗄️ conversationsStore - participant ModelsSvc as ⚙️ ModelsService - participant PropsSvc as ⚙️ PropsService - participant API as 🌐 llama-server - - Note over modelsStore: State:
models: ModelOption[]
routerModels: ApiModelDataEntry[]
selectedModelId, selectedModelName
loading, updating, error
modelLoadingStates (Map)
modelPropsCache (Map)
propsCacheVersion - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🚀 INITIALIZATION (MODEL mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>modelsStore: fetch() - activate modelsStore - modelsStore->>modelsStore: loading = true - - alt serverStore.props not loaded - modelsStore->>serverStore: fetch() - Note over serverStore: → see server-flow.mmd - end - - modelsStore->>ModelsSvc: list() - ModelsSvc->>API: GET /v1/models - API-->>ModelsSvc: ApiModelListResponse {data: [model]} - - modelsStore->>modelsStore: models = $state(mapped) - Note right of modelsStore: Map to ModelOption[]:
{id, name, model, description, capabilities} - - Note over modelsStore: MODEL mode: Get modalities from serverStore.props - modelsStore->>modelsStore: modelPropsCache.set(model.id, serverStore.props) - modelsStore->>modelsStore: models[0].modalities = props.modalities - - modelsStore->>modelsStore: Auto-select single model - Note right of modelsStore: selectedModelId = models[0].id - modelsStore->>modelsStore: loading = false - deactivate modelsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🚀 INITIALIZATION (ROUTER mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>modelsStore: fetch() - activate modelsStore - modelsStore->>ModelsSvc: list() - ModelsSvc->>API: GET /v1/models - API-->>ModelsSvc: ApiModelListResponse - modelsStore->>modelsStore: models = $state(mapped) - deactivate modelsStore - - Note over UI: After models loaded, layout triggers: - UI->>modelsStore: fetchRouterModels() - activate modelsStore - modelsStore->>ModelsSvc: listRouter() - ModelsSvc->>API: GET /v1/models - API-->>ModelsSvc: ApiRouterModelsListResponse - Note right of API: {data: [{id, status, path, in_cache}]} - modelsStore->>modelsStore: routerModels = $state(data) - - modelsStore->>modelsStore: fetchModalitiesForLoadedModels() - loop each model where status === "loaded" - modelsStore->>PropsSvc: fetchForModel(modelId) - PropsSvc->>API: GET /props?model={modelId} - API-->>PropsSvc: ApiLlamaCppServerProps - modelsStore->>modelsStore: modelPropsCache.set(modelId, props) - end - modelsStore->>modelsStore: propsCacheVersion++ - deactivate modelsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🔄 MODEL SELECTION (ROUTER mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>Hooks: useModelChangeValidation({getRequiredModalities, onSuccess?, onValidationFailure?}) - Note over Hooks: Hook configured per-component:
ChatForm: getRequiredModalities = usedModalities
ChatMessage: getRequiredModalities = getModalitiesUpToMessage(msgId) - - UI->>Hooks: handleModelChange(modelId, modelName) - activate Hooks - Hooks->>Hooks: previousSelectedModelId = modelsStore.selectedModelId - Hooks->>modelsStore: isModelLoaded(modelName)? - - alt model NOT loaded - Hooks->>modelsStore: loadModel(modelName) - Note over modelsStore: → see LOAD MODEL section below - end - - Note over Hooks: Always fetch props (from cache or API) - Hooks->>modelsStore: fetchModelProps(modelName) - modelsStore-->>Hooks: props - - Hooks->>convStore: getRequiredModalities() - convStore-->>Hooks: {vision, audio} - - Hooks->>Hooks: Validate: model.modalities ⊇ required? - - alt validation PASSED - Hooks->>modelsStore: selectModelById(modelId) - Hooks-->>UI: return true - else validation FAILED - Hooks->>UI: toast.error("Model doesn't support required modalities") - alt model was just loaded - Hooks->>modelsStore: unloadModel(modelName) - end - alt onValidationFailure provided - Hooks->>modelsStore: selectModelById(previousSelectedModelId) - end - Hooks-->>UI: return false - end - deactivate Hooks - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ⬆️ LOAD MODEL (ROUTER mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - modelsStore->>modelsStore: loadModel(modelId) - activate modelsStore - - alt already loaded - modelsStore-->>modelsStore: return (no-op) - end - - modelsStore->>modelsStore: modelLoadingStates.set(modelId, true) - modelsStore->>ModelsSvc: load(modelId) - ModelsSvc->>API: POST /models/load {model: modelId} - API-->>ModelsSvc: {status: "loading"} - - modelsStore->>modelsStore: pollForModelStatus(modelId, LOADED) - loop poll every 500ms (max 60 attempts) - modelsStore->>modelsStore: fetchRouterModels() - modelsStore->>ModelsSvc: listRouter() - ModelsSvc->>API: GET /v1/models - API-->>ModelsSvc: models[] - modelsStore->>modelsStore: getModelStatus(modelId) - alt status === LOADED - Note right of modelsStore: break loop - else status === LOADING - Note right of modelsStore: wait 500ms, continue - end - end - - modelsStore->>modelsStore: updateModelModalities(modelId) - modelsStore->>PropsSvc: fetchForModel(modelId) - PropsSvc->>API: GET /props?model={modelId} - API-->>PropsSvc: props with modalities - modelsStore->>modelsStore: modelPropsCache.set(modelId, props) - modelsStore->>modelsStore: propsCacheVersion++ - - modelsStore->>modelsStore: modelLoadingStates.set(modelId, false) - deactivate modelsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ⬇️ UNLOAD MODEL (ROUTER mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - modelsStore->>modelsStore: unloadModel(modelId) - activate modelsStore - modelsStore->>modelsStore: modelLoadingStates.set(modelId, true) - modelsStore->>ModelsSvc: unload(modelId) - ModelsSvc->>API: POST /models/unload {model: modelId} - - modelsStore->>modelsStore: pollForModelStatus(modelId, UNLOADED) - loop poll until unloaded - modelsStore->>ModelsSvc: listRouter() - ModelsSvc->>API: GET /v1/models - end - - modelsStore->>modelsStore: modelLoadingStates.set(modelId, false) - deactivate modelsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 📊 COMPUTED GETTERS - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over modelsStore: Getters:
- selectedModel: ModelOption | null
- loadedModelIds: string[] (from routerModels)
- loadingModelIds: string[] (from modelLoadingStates)
- singleModelName: string | null (MODEL mode only) - - Note over modelsStore: Modality helpers:
- getModelModalities(modelId): {vision, audio}
- modelSupportsVision(modelId): boolean
- modelSupportsAudio(modelId): boolean -``` diff --git a/tools/ui/docs/flows/server-flow.md b/tools/ui/docs/flows/server-flow.md deleted file mode 100644 index d6a1611f6f..0000000000 --- a/tools/ui/docs/flows/server-flow.md +++ /dev/null @@ -1,76 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 +layout.svelte - participant serverStore as 🗄️ serverStore - participant PropsSvc as ⚙️ PropsService - participant API as 🌐 llama-server - - Note over serverStore: State:
props: ApiLlamaCppServerProps | null
loading, error
role: ServerRole | null (MODEL | ROUTER)
fetchPromise (deduplication) - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🚀 INITIALIZATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>serverStore: fetch() - activate serverStore - - alt fetchPromise exists (already fetching) - serverStore-->>UI: return fetchPromise - Note right of serverStore: Deduplicate concurrent calls - end - - serverStore->>serverStore: loading = true - serverStore->>serverStore: fetchPromise = new Promise() - - serverStore->>PropsSvc: fetch() - PropsSvc->>API: GET /props - API-->>PropsSvc: ApiLlamaCppServerProps - Note right of API: {role, model_path, model_alias,
modalities, default_generation_settings, ...} - - PropsSvc-->>serverStore: props - serverStore->>serverStore: props = $state(data) - - serverStore->>serverStore: detectRole(props) - Note right of serverStore: role = props.role === "router"
? ServerRole.ROUTER
: ServerRole.MODEL - - serverStore->>serverStore: loading = false - serverStore->>serverStore: fetchPromise = null - deactivate serverStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 📊 COMPUTED GETTERS - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over serverStore: Getters from props: - - rect rgb(240, 255, 240) - Note over serverStore: defaultParams
→ props.default_generation_settings.params
(temperature, top_p, top_k, etc.) - end - - rect rgb(240, 255, 240) - Note over serverStore: contextSize
→ props.default_generation_settings.n_ctx - end - - rect rgb(255, 240, 240) - Note over serverStore: isRouterMode
→ role === ServerRole.ROUTER - end - - rect rgb(255, 240, 240) - Note over serverStore: isModelMode
→ role === ServerRole.MODEL - end - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🔗 RELATIONSHIPS - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over serverStore: Used by: - Note right of serverStore: - modelsStore: role detection, MODEL mode modalities
- settingsStore: syncWithServerDefaults (defaultParams)
- chatStore: contextSize for processing state
- UI components: isRouterMode for conditional rendering - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ❌ ERROR HANDLING - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over serverStore: getErrorMessage(): string | null
Returns formatted error for UI display - - Note over serverStore: clear(): void
Resets all state (props, error, loading, role) -``` diff --git a/tools/ui/docs/flows/settings-flow.md b/tools/ui/docs/flows/settings-flow.md deleted file mode 100644 index 260713a17b..0000000000 --- a/tools/ui/docs/flows/settings-flow.md +++ /dev/null @@ -1,156 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 ChatSettings - participant settingsStore as 🗄️ settingsStore - participant serverStore as 🗄️ serverStore - participant ParamSvc as ⚙️ ParameterSyncService - participant LS as 💾 LocalStorage - - Note over settingsStore: State:
config: SettingsConfigType
theme: string ("auto" | "light" | "dark")
isInitialized: boolean
userOverrides: Set<string> - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 🚀 INITIALIZATION - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over settingsStore: Auto-initialized in constructor (browser only) - settingsStore->>settingsStore: initialize() - activate settingsStore - - settingsStore->>settingsStore: loadConfig() - settingsStore->>LS: get("llama-config") - LS-->>settingsStore: StoredConfig | null - - alt config exists - settingsStore->>settingsStore: Merge with SETTING_CONFIG_DEFAULT - Note right of settingsStore: Fill missing keys with defaults - else no config - settingsStore->>settingsStore: config = SETTING_CONFIG_DEFAULT - end - - settingsStore->>LS: get("llama-userOverrides") - LS-->>settingsStore: string[] | null - settingsStore->>settingsStore: userOverrides = new Set(data) - - settingsStore->>settingsStore: loadTheme() - settingsStore->>LS: get("llama-theme") - LS-->>settingsStore: theme | "auto" - - settingsStore->>settingsStore: isInitialized = true - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 🔄 SYNC WITH SERVER DEFAULTS - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over UI: Triggered from +layout.svelte when serverStore.props loaded - UI->>settingsStore: syncWithServerDefaults() - activate settingsStore - - settingsStore->>serverStore: defaultParams - serverStore-->>settingsStore: {temperature, top_p, top_k, ...} - - loop each SYNCABLE_PARAMETER - alt key NOT in userOverrides - settingsStore->>settingsStore: config[key] = serverDefault[key] - Note right of settingsStore: Non-overridden params adopt server default - else key in userOverrides - Note right of settingsStore: Keep user value, skip server default - end - end - - alt serverStore.props has uiSettings - settingsStore->>settingsStore: Apply uiSettings from server - Note right of settingsStore: Server-provided UI settings
(e.g. showRawOutputSwitch) - end - - settingsStore->>settingsStore: saveConfig() - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: ⚙️ UPDATE CONFIG - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>settingsStore: updateConfig(key, value) - activate settingsStore - settingsStore->>settingsStore: config[key] = value - - alt value matches server default for key - settingsStore->>settingsStore: userOverrides.delete(key) - Note right of settingsStore: Matches server default, remove override - else value differs from server default - settingsStore->>settingsStore: userOverrides.add(key) - Note right of settingsStore: Mark as user-modified (won't be overwritten) - end - - settingsStore->>settingsStore: saveConfig() - settingsStore->>LS: set(CONFIG_LOCALSTORAGE_KEY, config) - settingsStore->>LS: set(USER_OVERRIDES_LOCALSTORAGE_KEY, [...userOverrides]) - deactivate settingsStore - - UI->>settingsStore: updateMultipleConfig({key1: val1, key2: val2}) - activate settingsStore - Note right of settingsStore: Batch update, single save - settingsStore->>settingsStore: For each key: config[key] = value - settingsStore->>settingsStore: For each key: userOverrides.add(key) - settingsStore->>settingsStore: saveConfig() - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 🔄 RESET - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>settingsStore: resetConfig() - activate settingsStore - settingsStore->>settingsStore: config = {...SETTING_CONFIG_DEFAULT} - settingsStore->>settingsStore: userOverrides.clear() - Note right of settingsStore: All params reset to defaults
Next syncWithServerDefaults will adopt server values - settingsStore->>settingsStore: saveConfig() - deactivate settingsStore - - UI->>settingsStore: resetParameterToServerDefault(key) - activate settingsStore - settingsStore->>settingsStore: userOverrides.delete(key) - settingsStore->>serverStore: defaultParams[key] - settingsStore->>settingsStore: config[key] = serverDefault - settingsStore->>settingsStore: saveConfig() - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 🎨 THEME - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>settingsStore: updateTheme(newTheme) - activate settingsStore - settingsStore->>settingsStore: theme = newTheme - settingsStore->>settingsStore: saveTheme() - settingsStore->>LS: set("llama-theme", theme) - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 📊 PARAMETER INFO - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>settingsStore: getParameterInfo(key) - settingsStore->>ParamSvc: getParameterInfo(key, config, serverDefaults, userOverrides) - ParamSvc-->>settingsStore: ParameterInfo - Note right of ParamSvc: {
currentValue,
serverDefault,
isUserOverride: boolean,
canSync: boolean,
isDifferentFromServer: boolean
} - - UI->>settingsStore: getParameterDiff() - settingsStore->>ParamSvc: createParameterDiff(config, serverDefaults, userOverrides) - ParamSvc-->>settingsStore: ParameterDiff[] - Note right of ParamSvc: Array of parameters where user != server - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 📋 CONFIG CATEGORIES - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over settingsStore: Syncable with server (from /props): - rect rgb(240, 255, 240) - Note over settingsStore: temperature, top_p, top_k, min_p
repeat_penalty, presence_penalty, frequency_penalty
dynatemp_range, dynatemp_exponent
typ_p, xtc_probability, xtc_threshold
dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n - end - - Note over settingsStore: UI-only (not synced): - rect rgb(255, 240, 240) - Note over settingsStore: systemMessage, custom (JSON)
showStatistics, enableContinueGeneration
autoMicOnEmpty, disableAutoScroll
apiKey, pdfAsImage, disableReasoningParsing, showRawOutputSwitch - end -``` diff --git a/tools/ui/eslint.config.js b/tools/ui/eslint.config.js index b8bdb216e2..6ad065f5a0 100644 --- a/tools/ui/eslint.config.js +++ b/tools/ui/eslint.config.js @@ -12,6 +12,49 @@ import { fileURLToPath } from 'node:url'; import ts from 'typescript-eslint'; const gitignorePath = fileURLToPath(new URL('./.gitignore', import.meta.url)); +// Require a blank line between consecutive class accessors (get/set). The core +// `padding-line-between-statements` rule only handles statements, not class +// members, so this is enforced with a small custom rule. +const blankLineBetweenAccessors = { + create(context) { + return { + MethodDefinition(node) { + if (node.kind !== 'get' && node.kind !== 'set') return; + + const body = node.parent; + + if (!body || body.type !== 'ClassBody') return; + + const index = body.body.indexOf(node); + + if (index <= 0) return; + + const prev = body.body[index - 1]; + + if (prev.type !== 'MethodDefinition' || (prev.kind !== 'get' && prev.kind !== 'set')) + return; + + if (node.loc.start.line - prev.loc.end.line <= 1) { + context.report({ + fix(fixer) { + // Insert after the previous accessor's closing brace so the blank + // line keeps the current accessor's indentation. + return fixer.insertTextAfter(prev, '\n'); + }, + message: 'Expected a blank line between class accessors (get/set).', + node + }); + } + } + }; + }, + meta: { + docs: { description: 'Require a blank line between consecutive class accessors (get/set).' }, + fixable: 'whitespace', + schema: [], + type: 'layout' + } +}; export default ts.config( includeIgnoreFile(gitignorePath), @@ -22,7 +65,11 @@ export default ts.config( ...svelte.configs.prettier, { languageOptions: { globals: { ...globals.browser, ...globals.node } }, - plugins: { perfectionist, 'simple-import-sort': simpleImportSort }, + plugins: { + local: { rules: { 'blank-line-between-accessors': blankLineBetweenAccessors } }, + perfectionist, + 'simple-import-sort': simpleImportSort + }, rules: { // Snippet bodies often ignore one or more of the parent's params // (e.g. `{#snippet children(_meta, ctx)}` when only ctx is read). @@ -30,8 +77,11 @@ export default ts.config( 'error', { argsIgnorePattern: '^_', varsIgnorePattern: '^_' } ], + // Enforce empty line at end of file 'eol-last': 'error', + // Enforce a blank line between consecutive get/set accessors + 'local/blank-line-between-accessors': 'error', // typescript-eslint strongly recommend that you do not use the no-undef lint rule on TypeScript projects. // see: https://typescript-eslint.io/troubleshooting/faqs/eslint/#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors 'no-undef': 'off', @@ -61,6 +111,38 @@ export default ts.config( { blankLine: 'always', next: ['return', 'throw', 'break', 'continue'], prev: '*' } ], + // Class member order: public fields -> private fields -> constructor -> getters + // -> setters -> public methods -> private methods, alphabetical within each. + // Svelte $derived fields must stay in dependency order (forward references are + // rejected), so the two stores that rely on that are exempted below. + 'perfectionist/sort-classes': [ + 'error', + { + customGroups: [ + { groupName: 'public-field', modifiers: ['public'], selector: 'property' }, + { groupName: 'private-field', modifiers: ['private'], selector: 'property' }, + { groupName: 'get-method', selector: 'get-method' }, + { groupName: 'set-method', selector: 'set-method' }, + { groupName: 'public-method', modifiers: ['public'], selector: 'method' }, + { groupName: 'private-method', modifiers: ['private'], selector: 'method' } + ], + groups: [ + 'public-field', + 'private-field', + 'constructor', + 'get-method', + 'set-method', + 'public-method', + 'private-method', + 'unknown' + ], + type: 'natural', + // Keep members in dependency order (Svelte rejects forward references in + // $derived fields), while still sorting the rest alphabetically. + useExperimentalDependencyDetection: true + } + ], + // Alphabetical order for enum members 'perfectionist/sort-enums': ['error', { type: 'natural' }], diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte index 8e89491723..304e5a6009 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte @@ -139,7 +139,7 @@ let fileSize = $derived(currentItem?.size ? formatFileSize(currentItem.size) : ''); let hasVisionModality = $derived( - currentItem && activeModelId ? modelsStore.modelSupportsVision(activeModelId) : false + currentItem && activeModelId ? modelsStore.props.modelSupportsVision(activeModelId) : false ); let audioSrc = $derived( diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte index e937d27306..18061728a2 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -28,7 +28,6 @@ import { chatStore, conversationsStore, - mcpResourceStore, mcpStore, modelsStore, serverStore, @@ -140,7 +139,9 @@ // float above the box. let mentionAnchor: HTMLDivElement | null = $state(null); - let cwd = $derived(conversationsStore.activeConversation?.cwd ?? conversationsStore.pendingCwd); + let cwd = $derived( + conversationsStore.activeConversation?.cwd ?? conversationsStore.preferences.pendingCwd + ); const pickers = useChatFormPickers({ focusInput: refocusInput, @@ -151,7 +152,8 @@ getShowModelSelector: () => showModelSelector, getValue: () => value, hasCwdTools: () => toolsStore.hasEnabledCwdTools, - hasPrompts: () => mcpStore.hasPromptsCapability(conversationsStore.getAllMcpServerOverrides()), + hasPrompts: () => + mcpStore.hasPromptsCapability(conversationsStore.preferences.getAllMcpServerOverrides()), openModelSelector: () => chatFormActionsRef?.openModelSelector(), setCaretOffset: (offset) => inputRef?.setCaretOffset(offset), setValue: (v) => { @@ -170,7 +172,7 @@ onValueChange?.(''); } - await conversationsStore.setCwd(newDir); + await conversationsStore.preferences.setCwd(newDir); if (conversationsStore.activeConversation) { await chatStore.recordCwdChange(newDir?.trim() || null); @@ -595,7 +597,7 @@ {useRichInput} /> - {#if mcpResourceStore.hasAttachments} + {#if mcpStore.resources.hasAttachments} { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte index 3d04d14cb1..92f5b93490 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte @@ -38,11 +38,11 @@ } function isServerEnabledForChat(serverId: string): boolean { - return conversationsStore.isMcpServerEnabledForChat(serverId); + return conversationsStore.preferences.isMcpServerEnabledForChat(serverId); } async function toggleServerForChat(serverId: string) { - await conversationsStore.toggleMcpServerForChat(serverId); + await conversationsStore.preferences.toggleMcpServerForChat(serverId); } function handleMcpSubMenuOpen(open: boolean) { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte index 63a8c267d8..2e61bb07dd 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte @@ -218,12 +218,15 @@ {@const hasError = healthState.status === HealthCheckStatus.ERROR} {@const displayName = mcpStore.getServerLabel(server)} {@const faviconUrl = mcpStore.getServerFavicon(server.id)} - {@const isEnabled = conversationsStore.isMcpServerEnabledForChat(server.id)} + {@const isEnabled = conversationsStore.preferences.isMcpServerEnabledForChat( + server.id + )} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte index 518dee5d95..f76333a1cb 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte @@ -81,10 +81,10 @@ $effect(() => { if (activeModelId) { - const cached = modelsStore.getModelProps(activeModelId); + const cached = modelsStore.props.getModelProps(activeModelId); if (!cached) { - modelsStore.fetchModelProps(activeModelId).then(() => { + modelsStore.props.fetchModelProps(activeModelId).then(() => { modelPropsVersion++; }); } @@ -94,19 +94,21 @@ $effect(() => { void modelPropsVersion; - hasAudioModality = activeModelId ? modelsStore.modelSupportsAudio(activeModelId) : false; + hasAudioModality = activeModelId ? modelsStore.props.modelSupportsAudio(activeModelId) : false; }); $effect(() => { void modelPropsVersion; - hasVideoModality = activeModelId ? modelsStore.modelSupportsVideo(activeModelId) : false; + hasVideoModality = activeModelId ? modelsStore.props.modelSupportsVideo(activeModelId) : false; }); $effect(() => { void modelPropsVersion; - hasVisionModality = activeModelId ? modelsStore.modelSupportsVision(activeModelId) : false; + hasVisionModality = activeModelId + ? modelsStore.props.modelSupportsVision(activeModelId) + : false; }); $effect(() => { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte index d8fad772dd..118e54a0a2 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte @@ -58,13 +58,13 @@ let currentConfig = $derived(settingsStore.config); let hasMcpPromptsSupport = $derived.by(() => { - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); return mcpStore.hasPromptsCapability(perChatOverrides); }); let hasMcpResourcesSupport = $derived.by(() => { - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); return mcpStore.hasResourcesCapability(perChatOverrides); }); @@ -121,7 +121,7 @@ if (!chatStore.isLoading && !chatStore.isStreaming()) return false; - const processingState = chatStore.activeProcessingState; + const processingState = chatStore.processing.activeState; if (!processingState) return false; diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte index 7b071d99e0..d6bad98dc9 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte @@ -16,7 +16,7 @@ $effect(() => { const conv = conversationsStore.activeConversation; - untrack(() => chatStore.setActiveProcessingConversation(conv?.id ?? null)); + untrack(() => chatStore.processing.setActiveConversation(conv?.id ?? null)); }); $effect(() => { @@ -28,12 +28,12 @@ if (chatStore.isLoading || chatStore.isStreaming()) return; if (messages.length === 0) { - untrack(() => chatStore.clearProcessingState(conv.id)); + untrack(() => chatStore.processing.setState(conv.id, null)); return; } - untrack(() => chatStore.restoreProcessingStateFromMessages(messages, conv.id)); + untrack(() => chatStore.processing.restoreFromMessages(messages, conv.id)); }); $effect(() => { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte index 3f178da188..452fd7a68a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte @@ -3,7 +3,7 @@ ChatAttachmentsListItemMcpResource, HorizontalScrollCarousel } from '$lib/components/app'; - import { mcpResourceStore, mcpStore } from '$lib/stores'; + import { mcpStore } from '$lib/stores'; interface Props { class?: string; @@ -12,8 +12,8 @@ let { class: className, onResourceClick }: Props = $props(); - const attachments = $derived(mcpResourceStore.attachments); - const hasAttachments = $derived(mcpResourceStore.hasAttachments); + const attachments = $derived(mcpStore.resources.attachments); + const hasAttachments = $derived(mcpStore.resources.hasAttachments); function handleRemove(attachmentId: string) { mcpStore.removeResourceAttachment(attachmentId); diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte index 9b5a57b9b9..9a5c3e7470 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte @@ -87,7 +87,7 @@ isLoading = true; try { - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); const initialized = await mcpStore.ensureInitialized(perChatOverrides); if (!initialized) { diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte index b92be9fbd6..c92af719a5 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte @@ -59,7 +59,7 @@ message.model ?? chatStore.getResumeModel(message.convId) ?? modelsStore.selectedModelName ); let modelLoadProgress = $derived( - isRouter && loadTargetModel ? modelsStore.getLoadProgress(loadTargetModel) : null + isRouter && loadTargetModel ? modelsStore.status.getLoadProgress(loadTargetModel) : null ); let modelLoadingText = $derived(modelLoadProgressText(modelLoadProgress)); diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte index d3fb33a008..c5b80f1569 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte @@ -31,7 +31,7 @@ pendingModel = modelId; try { - await modelsStore.loadModel(modelId); + await modelsStore.status.load(modelId); } finally { pendingModel = null; } diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte index 5849799794..7a21c67666 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte @@ -43,14 +43,14 @@ ); const hasReasoningError = $derived( - isLastAssistantMessage ? !!agenticStore.lastError(message.convId) : false + isLastAssistantMessage ? !!agenticStore.getLastError(message.convId) : false ); let permissionDismissed = $state(false); const pendingPermission = $derived( isStreaming && isLastAssistantMessage - ? agenticStore.pendingPermissionRequest(message.convId) + ? agenticStore.getPendingPermissionRequest(message.convId) : null ); @@ -74,7 +74,7 @@ const pendingContinue = $derived( isStreaming && isLastAssistantMessage - ? agenticStore.pendingContinueRequest(message.convId) + ? agenticStore.getPendingContinueRequest(message.convId) : false ); @@ -97,7 +97,7 @@ const sections = $derived(deriveAgenticSections(message, toolMessages, [], isStreaming)); const currentlyExecutingToolCallId = $derived( - isStreaming ? agenticStore.executingToolCallId(message.convId) : null + isStreaming ? agenticStore.getExecutingToolCallId(message.convId) : null ); type TurnGroup = { diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte index 2a8f45ba53..c32c66d911 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte @@ -238,30 +238,30 @@ /> {/each} - {#if conversationsStore.activeConversation && agenticStore.pendingSteeringMessageContent(conversationsStore.activeConversation!.id)} + {#if conversationsStore.activeConversation && agenticStore.getPendingSteeringMessageContent(conversationsStore.activeConversation!.id)} {@const convId = conversationsStore.activeConversation!.id} - {@const pendingContent = agenticStore.pendingSteeringMessageContent(convId)} + {@const pendingContent = agenticStore.getPendingSteeringMessageContent(convId)} {#if pendingContent} chatStore.abortCurrentFlow(convId)} onEdit={(newContent, extras) => agenticStore.injectSteeringMessage(convId, newContent, extras)} onDelete={() => agenticStore.clearSteeringMessage(convId)} /> {/if} - {:else if conversationsStore.activeConversation && chatStore.pendingMessageContent(conversationsStore.activeConversation!.id)} + {:else if conversationsStore.activeConversation && chatStore.getPendingMessageContent(conversationsStore.activeConversation!.id)} {@const convId = conversationsStore.activeConversation!.id} - {@const pendingContent = chatStore.pendingMessageContent(convId)} + {@const pendingContent = chatStore.getPendingMessageContent(convId)} {#if pendingContent} chatStore.abortCurrentFlow(convId)} onEdit={(newContent, extras) => chatStore.injectPendingMessage(convId, newContent, extras)} onDelete={() => chatStore.clearPendingMessage(convId)} diff --git a/tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte b/tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte index 1ddad694b6..c48dcb38c7 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte @@ -8,7 +8,7 @@ import { Button } from '$lib/components/ui/button'; import * as Dialog from '$lib/components/ui/dialog'; import { ICON_CLASS_DEFAULT } from '$lib/constants'; - import { conversationsStore, mcpResourceStore, mcpStore } from '$lib/stores'; + import { conversationsStore, mcpStore } from '$lib/stores'; import type { MCPResourceContent, MCPResourceInfo, MCPResourceTemplateInfo } from '$lib/types'; import { getResourceDisplayName } from '$lib/utils'; import { SvelteSet } from 'svelte/reactivity'; @@ -33,7 +33,7 @@ let templatePreviewLoading = $state(false); let templatePreviewError = $state(null); - const totalCount = $derived(mcpResourceStore.totalResourceCount); + const totalCount = $derived(mcpStore.resources.totalResourceCount); $effect(() => { if (open) { @@ -48,7 +48,7 @@ }); async function loadResources() { - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); const initialized = await mcpStore.ensureInitialized(perChatOverrides); if (initialized) { @@ -126,16 +126,16 @@ isAttaching = true; try { - const knownResource = mcpResourceStore.findResourceByUri(templatePreviewUri); + const knownResource = mcpStore.resources.findResourceByUri(templatePreviewUri); if (knownResource) { - if (!mcpResourceStore.isAttached(knownResource.uri)) { + if (!mcpStore.resources.isAttached(knownResource.uri)) { await mcpStore.attachResource(knownResource.uri); } toast.success(`Resource attached: ${knownResource.title || knownResource.name}`); } else { - if (mcpResourceStore.isAttached(templatePreviewUri)) { + if (mcpStore.resources.isAttached(templatePreviewUri)) { toast.info('Resource already attached'); handleOpenChange(false); @@ -147,9 +147,9 @@ serverName: selectedTemplate.serverName, uri: templatePreviewUri }; - const attachment = mcpResourceStore.addAttachment(resourceInfo); + const attachment = mcpStore.resources.addAttachment(resourceInfo); - mcpResourceStore.updateAttachmentContent(attachment.id, templatePreviewContent); + mcpStore.resources.updateAttachmentContent(attachment.id, templatePreviewContent); toast.success(`Resource attached: ${resourceInfo.name}`); } @@ -199,7 +199,7 @@ function getAllResourcesFlatInTreeOrder(): MCPResourceInfo[] { const allResources: MCPResourceInfo[] = []; - const resourcesMap = mcpResourceStore.serverResources; + const resourcesMap = mcpStore.resources.serverResources; for (const [serverName, serverRes] of resourcesMap.entries()) { for (const resource of serverRes.resources) { diff --git a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte index 9123dcef99..a339c6a429 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte @@ -234,7 +234,7 @@ useProxy: newServerUseProxy }); - conversationsStore.setMcpServerOverride(newServerId, true); + conversationsStore.preferences.setMcpServerOverride(newServerId, true); handleOpenChange(false); } diff --git a/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte b/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte index 61155fceba..fb74988702 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte @@ -42,7 +42,7 @@ let modalities = $derived.by(() => { if (!firstModel?.id) return []; - return modelsStore.getModelModalitiesArray(firstModel.id); + return modelsStore.props.getModelModalitiesArray(firstModel.id); }); // Ensure models are fetched when dialog opens @@ -56,7 +56,7 @@ $effect(() => { if (open && isRouter && modelId) { isLoadingRouterProps = true; - modelsStore + modelsStore.props .fetchModelProps(modelId) .then((props) => { routerModelProps = props; diff --git a/tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte b/tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte index 301c396991..a8772aa1c9 100644 --- a/tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte @@ -14,7 +14,9 @@ let mcpServers = $derived(mcpStore.getServers().filter((s) => s.enabled)); let enabledMcpServersForChat = $derived( - mcpServers.filter((s) => conversationsStore.isMcpServerEnabledForChat(s.id) && s.url.trim()) + mcpServers.filter( + (s) => conversationsStore.preferences.isMcpServerEnabledForChat(s.id) && s.url.trim() + ) ); let healthyEnabledMcpServers = $derived( enabledMcpServersForChat.filter((s) => { diff --git a/tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte b/tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte index 18e9746532..c8cf8bbbc2 100644 --- a/tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte @@ -2,7 +2,7 @@ import McpResourcesBrowserEmptyState from './McpResourcesBrowserEmptyState.svelte'; import McpResourcesBrowserHeader from './McpResourcesBrowserHeader.svelte'; import McpResourcesBrowserServerItem from './McpResourcesBrowserServerItem.svelte'; - import { mcpResourceStore, mcpStore } from '$lib/stores'; + import { mcpStore } from '$lib/stores'; import type { MCPResourceInfo, MCPResourceTemplateInfo, MCPServerResources } from '$lib/types'; import { parseResourcePath } from '$lib/utils'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; @@ -31,8 +31,8 @@ let expandedFolders = new SvelteSet(); let searchQuery = $state(''); - const resources = $derived(mcpResourceStore.serverResources); - const isLoading = $derived(mcpResourceStore.isLoading); + const resources = $derived(mcpStore.resources.serverResources); + const isLoading = $derived(mcpStore.resources.isLoading); const filteredResources = $derived.by(() => { if (!searchQuery.trim()) { diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte index c23bca1aeb..1cd56c1246 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte @@ -116,7 +116,7 @@ if (status === ServerModelStatus.LOADING) return; - await modelsStore.unloadModel(modelId); + await modelsStore.status.unload(modelId); } export function open() { @@ -174,9 +174,9 @@ {@const triggerLoading = !!triggerModel && (triggerStatus === ServerModelStatus.LOADING || - modelsStore.isModelOperationInProgress(triggerModel))} + modelsStore.status.isOperationInProgress(triggerModel))} {@const triggerLoadPercent = triggerLoading - ? Math.round(modelLoadFraction(modelsStore.getLoadProgress(triggerModel)) * 100) + ? Math.round(modelLoadFraction(modelsStore.status.getLoadProgress(triggerModel)) * 100) : 0} {#if ms.isRouter} diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte index 18c885a62c..acdb36bcad 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte @@ -47,7 +47,7 @@ return (model?.status?.value as ServerModelStatus) ?? null; }); - let isOperationInProgress = $derived(modelsStore.isModelOperationInProgress(option.model)); + let isOperationInProgress = $derived(modelsStore.status.isOperationInProgress(option.model)); let isFailed = $derived(serverStatus === ServerModelStatus.FAILED); let isSleeping = $derived(serverStatus === ServerModelStatus.SLEEPING); let isLoaded = $derived( @@ -55,7 +55,7 @@ ); let isLoading = $derived(serverStatus === ServerModelStatus.LOADING || isOperationInProgress); - let loadProgress = $derived(isLoading ? modelsStore.getLoadProgress(option.model) : null); + let loadProgress = $derived(isLoading ? modelsStore.status.getLoadProgress(option.model) : null); let loadPercent = $derived(Math.round(modelLoadFraction(loadProgress) * 100)); let loadTitle = $derived(modelLoadProgressText(loadProgress)); @@ -138,7 +138,7 @@ icon={RotateCw} tooltip="Retry loading model" class="h-3 w-3 text-red-500 hover:text-foreground" - onclick={() => modelsStore.loadModel(option.model)} + onclick={() => modelsStore.status.load(option.model)} stopPropagationOnClick /> @@ -157,7 +157,7 @@ class="h-3 w-3 text-red-500 hover:text-red-600 [@media(pointer:coarse)]:text-amber-500 [@media(pointer:coarse)]:hover:text-amber-600" onclick={(e) => { e?.stopPropagation(); - modelsStore.unloadModel(option.model); + modelsStore.status.unload(option.model); }} /> @@ -174,7 +174,7 @@ icon={PowerOff} tooltip="Unload model" class="h-3 w-3 text-red-500 hover:text-red-600 [@media(pointer:coarse)]:text-green-500 [@media(pointer:coarse)]:hover:text-green-600" - onclick={() => modelsStore.unloadModel(option.model)} + onclick={() => modelsStore.status.unload(option.model)} stopPropagationOnClick /> @@ -191,7 +191,7 @@ icon={Power} tooltip="Load model" class="h-3 w-3 [@media(pointer:coarse)]:text-muted-foreground" - onclick={() => modelsStore.loadModel(option.model)} + onclick={() => modelsStore.status.load(option.model)} stopPropagationOnClick /> diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte index 7228a2e74a..0d10dd106c 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte @@ -72,9 +72,9 @@ {@const triggerLoading = !!triggerModel && (triggerStatus === ServerModelStatus.LOADING || - modelsStore.isModelOperationInProgress(triggerModel))} + modelsStore.status.isOperationInProgress(triggerModel))} {@const triggerLoadPercent = triggerLoading - ? Math.round(modelLoadFraction(modelsStore.getLoadProgress(triggerModel)) * 100) + ? Math.round(modelLoadFraction(modelsStore.status.getLoadProgress(triggerModel)) * 100) : 0} {#if ms.isRouter} diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte index c8b2c814cd..4233039eff 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte @@ -52,7 +52,7 @@ void modelsStore .fetch() .then(() => modelsStore.fetchRouterModels()) - .then(() => modelsStore.fetchModalitiesForLoadedModels()) + .then(() => modelsStore.props.fetchModalitiesForLoadedModels()) .then(() => modelsStore.ensureFirstModelSelected()); } }); diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte index 30f2b9b2a7..d5d93e11d7 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte @@ -23,13 +23,13 @@ let { fields, localConfig, onConfigChange, onThemeChange }: Props = $props(); let currentModelParams = $derived.by(() => { - void modelsStore.propsCacheVersion; + void modelsStore.props.cacheVersion; if (serverStore.isRouterMode) { const currentModelName = modelsStore.selectedModelName; if (currentModelName) { - const currentModelProps = modelsStore.getModelProps(currentModelName); + const currentModelProps = modelsStore.props.getModelProps(currentModelName); return (currentModelProps?.default_generation_settings?.params ?? {}) as Record< string, diff --git a/tools/ui/src/lib/components/app/settings/SettingsMcpServers.svelte b/tools/ui/src/lib/components/app/settings/SettingsMcpServers.svelte index 4ea4285322..23736ef1c1 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsMcpServers.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsMcpServers.svelte @@ -121,11 +121,13 @@ {:else} { - const wasEnabled = conversationsStore.isMcpServerEnabledForChat(server.id); + const wasEnabled = conversationsStore.preferences.isMcpServerEnabledForChat( + server.id + ); - await conversationsStore.toggleMcpServerForChat(server.id); + await conversationsStore.preferences.toggleMcpServerForChat(server.id); if (!wasEnabled) { // Promote the connection so tools/prompts/resources become diff --git a/tools/ui/src/lib/constants/attachment-menu.constants.ts b/tools/ui/src/lib/constants/attachment-menu.constants.ts index 62e03bea6e..07ca17fad1 100644 --- a/tools/ui/src/lib/constants/attachment-menu.constants.ts +++ b/tools/ui/src/lib/constants/attachment-menu.constants.ts @@ -74,7 +74,7 @@ export const ATTACHMENT_PROMPT_ITEMS: AttachmentMenuItem[] = [ enabledWhen: AttachmentItemEnabledWhen.ALWAYS, icon: Zap, id: AttachmentMenuItemId.MCP_PROMPT, - label: 'MCP Prompt', + label: 'MCP Prompts', visibleWhen: AttachmentItemVisibleWhen.HAS_MCP_PROMPTS_SUPPORT } ]; diff --git a/tools/ui/src/lib/constants/cache.constants.ts b/tools/ui/src/lib/constants/cache.constants.ts index b60792d995..9c6bfadf8a 100644 --- a/tools/ui/src/lib/constants/cache.constants.ts +++ b/tools/ui/src/lib/constants/cache.constants.ts @@ -32,13 +32,3 @@ export const MCP_RESOURCE_CACHE = { /** TTL for MCP resource cache entries in milliseconds (5 minutes) */ TTL_MS: 5 * 60 * 1000 } as const; - -/** - * Limits for pruning inactive conversation states held in memory. - */ -export const INACTIVE_CONVERSATION = { - /** Maximum age (in ms) for inactive conversation states before cleanup (30 minutes) */ - MAX_AGE_MS: 30 * 60 * 1000, - /** Maximum number of inactive conversation states to keep in memory */ - MAX_STATES: 10 -} as const; diff --git a/tools/ui/src/lib/constants/url.constants.ts b/tools/ui/src/lib/constants/url.constants.ts index 214c8afbac..8df4429346 100644 --- a/tools/ui/src/lib/constants/url.constants.ts +++ b/tools/ui/src/lib/constants/url.constants.ts @@ -1,3 +1,5 @@ +import { UrlProtocol } from '$lib/enums'; + const STD = ['com', 'net', 'org', 'gov', 'edu'] as const; const STD_MIL = [...STD, 'mil'] as const; const ccTLD_PREFIXES: Record = { @@ -184,3 +186,7 @@ export const WILDCARD_PUBLIC_SUFFIXES = buildSuffixSet(WILDCARD_BASES); // Matches one or more trailing "/" characters at the end of a URL/path. export const TRAILING_SLASHES_REGEX = /\/+$/; + +// Protocols that apiFetch treats as absolute and passes through untouched. +// Add a protocol here when a caller needs to fetch an absolute URL with it. +export const API_ABSOLUTE_URL_PROTOCOLS = [UrlProtocol.HTTP, UrlProtocol.HTTPS] as const; diff --git a/tools/ui/src/lib/hooks/use-auto-scroll.svelte.ts b/tools/ui/src/lib/hooks/use-auto-scroll.svelte.ts index 6ebce15dad..d55574efef 100644 --- a/tools/ui/src/lib/hooks/use-auto-scroll.svelte.ts +++ b/tools/ui/src/lib/hooks/use-auto-scroll.svelte.ts @@ -14,18 +14,14 @@ export interface AutoScrollOptions { */ export class AutoScrollController { private _autoScrollEnabled = $state(true); - private _userScrolledUp = $state(false); - private _lastScrollTop = $state(0); - private _scrollInterval: ReturnType | undefined; private _container: HTMLElement | undefined; private _disabled: boolean; + private _lastScrollTop = $state(0); private _mutationObserver: MutationObserver | null = null; - private _rafPending = false; private _observerEnabled = false; - constructor(options: AutoScrollOptions = {}) { - this._disabled = options.disabled ?? false; - } - + private _rafPending = false; + private _scrollInterval: ReturnType | undefined; + private _userScrolledUp = $state(false); get autoScrollEnabled(): boolean { return this._autoScrollEnabled; } @@ -34,6 +30,71 @@ export class AutoScrollController { return this._userScrolledUp; } + constructor(options: AutoScrollOptions = {}) { + this._disabled = options.disabled ?? false; + } + + /** + * Cleans up resources. Call this in onDestroy or when the component unmounts. + */ + destroy(): void { + this.stopInterval(); + this._doStopObserving(); + } + + /** + * Enables auto-scroll (e.g., when user sends a message). + */ + enable(): void { + if (this._disabled) return; + + this._userScrolledUp = false; + this._autoScrollEnabled = true; + } + + /** + * Handles scroll events to detect user scroll direction and toggle auto-scroll. + */ + handleScroll(): void { + if (this._disabled || !this._container) return; + + const { clientHeight, scrollHeight, scrollTop } = this._container; + const distanceFromBottom = scrollHeight - clientHeight - scrollTop; + const isScrollingUp = scrollTop < this._lastScrollTop; + const isAtBottom = distanceFromBottom < AUTO_SCROLL_AT_BOTTOM_THRESHOLD; + + if (isScrollingUp && !isAtBottom) { + this._userScrolledUp = true; + this._autoScrollEnabled = false; + } else if (isAtBottom && this._userScrolledUp) { + this._userScrolledUp = false; + this._autoScrollEnabled = true; + } + + this._lastScrollTop = scrollTop; + } + + /** + * Resets scroll state when switching conversations. + */ + resetScrollState(): void { + this._userScrolledUp = false; + this._autoScrollEnabled = !this._disabled; + + if (this._container) { + this._lastScrollTop = this._container.scrollTop; + } + } + + /** + * Scrolls the container to the bottom instantly. + */ + scrollToBottom(): void { + if (this._disabled || !this._container) return; + + this._container.scrollTop = this._container.scrollHeight; + } + /** * Binds the controller to a scrollable container element. */ @@ -63,59 +124,6 @@ export class AutoScrollController { } } - /** - * Handles scroll events to detect user scroll direction and toggle auto-scroll. - */ - handleScroll(): void { - if (this._disabled || !this._container) return; - - const { clientHeight, scrollHeight, scrollTop } = this._container; - const distanceFromBottom = scrollHeight - clientHeight - scrollTop; - const isScrollingUp = scrollTop < this._lastScrollTop; - const isAtBottom = distanceFromBottom < AUTO_SCROLL_AT_BOTTOM_THRESHOLD; - - if (isScrollingUp && !isAtBottom) { - this._userScrolledUp = true; - this._autoScrollEnabled = false; - } else if (isAtBottom && this._userScrolledUp) { - this._userScrolledUp = false; - this._autoScrollEnabled = true; - } - - this._lastScrollTop = scrollTop; - } - - /** - * Scrolls the container to the bottom instantly. - */ - scrollToBottom(): void { - if (this._disabled || !this._container) return; - - this._container.scrollTop = this._container.scrollHeight; - } - - /** - * Enables auto-scroll (e.g., when user sends a message). - */ - enable(): void { - if (this._disabled) return; - - this._userScrolledUp = false; - this._autoScrollEnabled = true; - } - - /** - * Resets scroll state when switching conversations. - */ - resetScrollState(): void { - this._userScrolledUp = false; - this._autoScrollEnabled = !this._disabled; - - if (this._container) { - this._lastScrollTop = this._container.scrollTop; - } - } - /** * Starts the auto-scroll interval for continuous scrolling during streaming. */ @@ -127,6 +135,18 @@ export class AutoScrollController { }, AUTO_SCROLL_INTERVAL); } + /** + * Starts a MutationObserver on the container that auto-scrolls to bottom + * on content changes. More responsive than interval-based polling. + */ + startObserving(): void { + this._observerEnabled = true; + + if (this._container && !this._disabled && !this._mutationObserver) { + this._doStartObserving(); + } + } + /** * Stops the auto-scroll interval. */ @@ -137,6 +157,14 @@ export class AutoScrollController { } } + /** + * Stops the MutationObserver. + */ + stopObserving(): void { + this._observerEnabled = false; + this._doStopObserving(); + } + /** * Updates the auto-scroll interval based on streaming state. * Call this in a $effect to automatically manage the interval. @@ -157,34 +185,6 @@ export class AutoScrollController { } } - /** - * Cleans up resources. Call this in onDestroy or when the component unmounts. - */ - destroy(): void { - this.stopInterval(); - this._doStopObserving(); - } - - /** - * Starts a MutationObserver on the container that auto-scrolls to bottom - * on content changes. More responsive than interval-based polling. - */ - startObserving(): void { - this._observerEnabled = true; - - if (this._container && !this._disabled && !this._mutationObserver) { - this._doStartObserving(); - } - } - - /** - * Stops the MutationObserver. - */ - stopObserving(): void { - this._observerEnabled = false; - this._doStopObserving(); - } - private _doStartObserving(): void { if (!this._container || this._mutationObserver) return; diff --git a/tools/ui/src/lib/hooks/use-chat-screen-active-model.svelte.ts b/tools/ui/src/lib/hooks/use-chat-screen-active-model.svelte.ts index ceffdd8a3a..b5a5d85ce9 100644 --- a/tools/ui/src/lib/hooks/use-chat-screen-active-model.svelte.ts +++ b/tools/ui/src/lib/hooks/use-chat-screen-active-model.svelte.ts @@ -22,10 +22,10 @@ export function useChatScreenActiveModel() { $effect(() => { if (activeModelId) { - const cached = modelsStore.getModelProps(activeModelId); + const cached = modelsStore.props.getModelProps(activeModelId); if (!cached) { - modelsStore.fetchModelProps(activeModelId).then(() => { + modelsStore.props.fetchModelProps(activeModelId).then(() => { modelPropsVersion++; }); } @@ -36,7 +36,7 @@ export function useChatScreenActiveModel() { if (activeModelId) { void modelPropsVersion; - return modelsStore.modelSupportsAudio(activeModelId); + return modelsStore.props.modelSupportsAudio(activeModelId); } return false; @@ -45,7 +45,7 @@ export function useChatScreenActiveModel() { if (activeModelId) { void modelPropsVersion; - return modelsStore.modelSupportsVideo(activeModelId); + return modelsStore.props.modelSupportsVideo(activeModelId); } return false; @@ -54,7 +54,7 @@ export function useChatScreenActiveModel() { if (activeModelId) { void modelPropsVersion; - return modelsStore.modelSupportsVision(activeModelId); + return modelsStore.props.modelSupportsVision(activeModelId); } return false; diff --git a/tools/ui/src/lib/hooks/use-context-gauge.svelte.ts b/tools/ui/src/lib/hooks/use-context-gauge.svelte.ts index 07d380224d..c6d55e3935 100644 --- a/tools/ui/src/lib/hooks/use-context-gauge.svelte.ts +++ b/tools/ui/src/lib/hooks/use-context-gauge.svelte.ts @@ -54,10 +54,10 @@ export function useContextGauge(): UseContextGaugeReturn { const modelId = contextStatsStore.activeModelId; if (modelId && contextStatsStore.isActiveModelLoaded) { - const cached = modelsStore.getModelProps(modelId); + const cached = modelsStore.props.getModelProps(modelId); if (!cached) { - void modelsStore.fetchModelProps(modelId); + void modelsStore.props.fetchModelProps(modelId); } } }); @@ -80,9 +80,9 @@ export function useContextGauge(): UseContextGaugeReturn { if (!modelId || contextStatsStore.isActiveModelLoading) return; try { - await modelsStore.loadModel(modelId); + await modelsStore.status.load(modelId); } catch { - // toast already surfaced by modelsStore.loadModel + // toast already surfaced by modelsStore.status.load } } diff --git a/tools/ui/src/lib/hooks/use-models-selector.svelte.ts b/tools/ui/src/lib/hooks/use-models-selector.svelte.ts index d56eeefcd3..7d2770a261 100644 --- a/tools/ui/src/lib/hooks/use-models-selector.svelte.ts +++ b/tools/ui/src/lib/hooks/use-models-selector.svelte.ts @@ -47,7 +47,7 @@ export interface UseModelsSelectorReturn { export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSelectorReturn { const options = $derived( modelsStore.models.filter((option) => { - const modelProps = modelsStore.getModelProps(option.model); + const modelProps = modelsStore.props.getModelProps(option.model); return modelProps?.ui !== false; }) @@ -103,7 +103,7 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele if (open) { modelsStore.fetchRouterModels().then(() => { - modelsStore.fetchModalitiesForLoadedModels(); + modelsStore.props.fetchModalitiesForLoadedModels(); }); } @@ -143,8 +143,8 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele if (!onModelChange && isRouter && !modelsStore.isModelLoaded(option.model)) { isLoadingModel = true; - modelsStore - .loadModel(option.model) + modelsStore.status + .load(option.model) .catch((error) => console.error('Failed to load model:', error)) .finally(() => (isLoadingModel = false)); } diff --git a/tools/ui/src/lib/hooks/use-processing-state.svelte.ts b/tools/ui/src/lib/hooks/use-processing-state.svelte.ts index 37e0748bcb..8a6f332f35 100644 --- a/tools/ui/src/lib/hooks/use-processing-state.svelte.ts +++ b/tools/ui/src/lib/hooks/use-processing-state.svelte.ts @@ -43,7 +43,7 @@ export function useProcessingState(): UseProcessingStateReturn { } // Read directly from the reactive state - return chatStore.activeProcessingState; + return chatStore.processing.activeState; }); $effect(() => { diff --git a/tools/ui/src/lib/hooks/use-reasoning-menu.svelte.ts b/tools/ui/src/lib/hooks/use-reasoning-menu.svelte.ts index 2ff67c9392..2cb9c90609 100644 --- a/tools/ui/src/lib/hooks/use-reasoning-menu.svelte.ts +++ b/tools/ui/src/lib/hooks/use-reasoning-menu.svelte.ts @@ -42,19 +42,20 @@ export function useReasoningMenu(): UseReasoningMenuReturn { }); const modelSupportsThinking = $derived.by(() => { void modelsStore.loadedModelIds; - void modelsStore.propsCacheVersion; + void modelsStore.props.cacheVersion; if (serverStore.isRouterMode) { const modelId = modelsStore.selectedModelName || conversationModel; return ( - modelsStore.checkModelSupportsThinking(modelId ?? '') || modelSupportsThinkingFromMessages + modelsStore.props.checkModelSupportsThinking(modelId ?? '') || + modelSupportsThinkingFromMessages ); } - return modelsStore.supportsThinking || modelSupportsThinkingFromMessages; + return modelsStore.props.supportsThinking || modelSupportsThinkingFromMessages; }); - const currentEffort = $derived(conversationsStore.getReasoningEffort()); + const currentEffort = $derived(conversationsStore.preferences.getReasoningEffort()); const thinkingEnabled = $derived( currentEffort !== ReasoningEffort.OFF && currentEffort !== ReasoningEffort.DEFAULT ); @@ -76,7 +77,7 @@ export function useReasoningMenu(): UseReasoningMenuReturn { return modelSupportsThinking; }, select(level: ReasoningEffortLevel): void { - conversationsStore.setReasoningEffort(level.value as ReasoningEffort); + conversationsStore.preferences.setReasoningEffort(level.value as ReasoningEffort); }, get thinkingEnabled() { return thinkingEnabled; diff --git a/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts b/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts index 80b3b85a99..e9dc0dcab6 100644 --- a/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts +++ b/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts @@ -35,7 +35,7 @@ export function useToolsPanel(): UseToolsPanelReturn { (g) => g.source !== ToolSource.MCP || !g.serverId || - conversationsStore.isMcpServerEnabledForChat(g.serverId) + conversationsStore.preferences.isMcpServerEnabledForChat(g.serverId) ) ); const totalToolCount = $derived(activeGroups.reduce((n, g) => n + g.tools.length, 0)); @@ -73,7 +73,7 @@ export function useToolsPanel(): UseToolsPanelReturn { return ( group.source === ToolSource.MCP && !!group.serverId && - !conversationsStore.isMcpServerEnabledForChat(group.serverId) + !conversationsStore.preferences.isMcpServerEnabledForChat(group.serverId) ); } diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index f609b4f4ec..b008b16db8 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -1,4 +1,11 @@ -import { settingsStore } from '../stores/settings.svelte'; +/** + * ChatService - Stateless chat completion and streaming API layer + * + * Wraps the /chat/completions and /stream endpoints: request building, SSE + * parsing, streaming callbacks, resume/probe logic and pre-encode KV-cache + * warming. No reactive state; consumed by chatStore and its managers. + */ + import { getAudioInputFormat } from '../utils/audio-format'; import { capImageDataURLSize } from '../utils/cap-img-size'; import { @@ -25,7 +32,8 @@ import { ReasoningFormat, StreamConnectionState } from '$lib/enums'; -import { modelsStore } from '$lib/stores/models.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { DatabaseMessageExtraMcpPrompt, DatabaseMessageExtraMcpResource } from '$lib/types'; import type { ApiChatCompletionToolCall, @@ -53,13 +61,310 @@ function streamStorageKey(conversationId: string): string { } export class ChatService { + // Per-chunk localStorage writes are throttled to at most one per + // conversation per interval (saveStreamStateThrottled). The resume offset + // only needs to be roughly current: on resume the server retransmits from + // a line boundary and the client discards its partial line. Guaranteed + // immediate writes happen at stream start, at resume boundaries and when + // the page goes hidden or away (pagehide/visibilitychange), so a reload + // always finds a usable offset. + private static readonly STREAM_STATE_SAVE_INTERVAL_MS = 500; + + private static streamStateSaveTrackers = new Map< + string, + { lastSavedAt: number; model: string | null; pendingBytes: number | null } + >(); + /** + * Checks whether all server slots are currently idle (not processing any requests). + * Queries the /slots endpoint (requires --slots flag on the server). + * Returns true if all slots are idle, false if any is processing. + * If the endpoint is unavailable or errors out, returns true (best-effort fallback). * - * - * Title Generation - * - * + * @param signal - Optional AbortSignal to cancel the request if needed + * @param model - Optional model name to check slots for (required in ROUTER mode) + * @returns {Promise} Promise that resolves to true if all slots are idle, false if any is processing */ + static async areAllSlotsIdle(model?: string | null, signal?: AbortSignal): Promise { + try { + const url = model ? `${API_SLOTS.LIST}?model=${encodeURIComponent(model)}` : API_SLOTS.LIST; + const res = await fetch(url, { signal }); + + if (!res.ok) return true; + + const slots: { is_processing: boolean }[] = await res.json(); + + return slots.every((s) => !s.is_processing); + } catch { + return true; + } + } + + /** + * Cancels the server-side replay buffer for a conversation, freeing its slot. + */ + static async cancelServerStream(conversationId: string, model?: string | null): Promise { + if (!conversationId) return; + + try { + const id = streamIdentity(conversationId, model); + + await fetch(ChatService.buildStreamUrl(id), { + headers: getAuthHeaders(), + method: 'DELETE' + }); + } catch (e) { + console.warn('cancelServerStream failed:', e); + } + } + + static clearStreamState(conversationId: string): void { + if (!conversationId) return; + + ChatService.streamStateSaveTrackers.delete(conversationId); + + try { + localStorage.removeItem(streamStorageKey(conversationId)); + } catch { + // nothing to do + } + } + + /** + * Converts a database message with attachments to API chat message format. + * Processes various attachment types (images, text files, PDFs) and formats them + * as content parts suitable for the chat completion API. + */ + static async convertDbMessageToApiChatMessageData( + message: DatabaseMessage & { extra?: DatabaseMessageExtra[] } + ): Promise { + // Handle tool result messages (role: 'tool') + if (message.role === MessageRole.TOOL && message.toolCallId) { + return { + content: message.content, + role: MessageRole.TOOL, + tool_call_id: message.toolCallId + }; + } + + // Parse tool calls for assistant messages + let toolCalls: ApiChatCompletionToolCall[] | undefined; + + if (message.toolCalls) { + try { + toolCalls = JSON.parse(message.toolCalls); + } catch { + // Ignore parse errors for malformed tool calls + } + } + + if (!message.extra || message.extra.length === 0) { + const result: ApiChatMessageData = { + content: message.content, + role: message.role as MessageRole + }; + + if (message.reasoningContent) { + result.reasoning_content = message.reasoningContent; + } + + if (toolCalls && toolCalls.length > 0) { + result.tool_calls = toolCalls; + } + + return result; + } + + const contentParts: ApiChatMessageContentPart[] = []; + const textFiles = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraTextFile => + extra.type === AttachmentType.TEXT + ); + + for (const textFile of textFiles) { + contentParts.push({ + text: formatAttachmentText(AttachmentLabel.FILE, textFile.name, textFile.content), + type: ContentPartType.TEXT + }); + } + + // Handle legacy 'context' type from the old UI (pasted content) + const legacyContextFiles = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraLegacyContext => + extra.type === AttachmentType.LEGACY_CONTEXT + ); + + for (const legacyContextFile of legacyContextFiles) { + contentParts.push({ + text: formatAttachmentText( + AttachmentLabel.FILE, + legacyContextFile.name, + legacyContextFile.content + ), + type: ContentPartType.TEXT + }); + } + + const imageFiles = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraImageFile => + extra.type === AttachmentType.IMAGE + ); + + for (const image of imageFiles) { + const maxImageResolution = settingsStore.getConfig(SETTINGS_KEYS.MAX_IMAGE_RESOLUTION); + // Caps the resolution and bakes the jpeg exif orientation in one pass, + // untouched images pass through as is + const base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution); + + contentParts.push({ + image_url: { url: base64Url }, + type: ContentPartType.IMAGE_URL + }); + } + + const audioFiles = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraAudioFile => + extra.type === AttachmentType.AUDIO + ); + + for (const audio of audioFiles) { + contentParts.push({ + input_audio: { + data: audio.base64Data, + format: getAudioInputFormat(audio.mimeType) + }, + type: ContentPartType.INPUT_AUDIO + }); + } + + if (message.content) { + contentParts.push({ + text: message.content, + type: ContentPartType.TEXT + }); + } + + const videoFiles = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraVideoFile => + extra.type === AttachmentType.VIDEO + ); + + for (const video of videoFiles) { + contentParts.push({ + input_video: { + data: video.base64Data, + format: video.mimeType.includes('mp4') + ? 'mp4' + : video.mimeType.includes('ogg') + ? 'ogg' + : 'auto' + }, + type: ContentPartType.INPUT_VIDEO + }); + } + + const pdfFiles = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraPdfFile => + extra.type === AttachmentType.PDF + ); + + for (const pdfFile of pdfFiles) { + if (pdfFile.processedAsImages && pdfFile.images) { + for (let i = 0; i < pdfFile.images.length; i++) { + contentParts.push({ + image_url: { url: pdfFile.images[i] }, + type: ContentPartType.IMAGE_URL + }); + } + } else { + contentParts.push({ + text: formatAttachmentText(AttachmentLabel.PDF_FILE, pdfFile.name, pdfFile.content), + type: ContentPartType.TEXT + }); + } + } + + const mcpPrompts = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraMcpPrompt => + extra.type === AttachmentType.MCP_PROMPT + ); + + for (const mcpPrompt of mcpPrompts) { + contentParts.push({ + text: formatAttachmentText( + AttachmentLabel.MCP_PROMPT, + mcpPrompt.name, + mcpPrompt.content, + mcpPrompt.serverName + ), + type: ContentPartType.TEXT + }); + } + + const mcpResources = message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraMcpResource => + extra.type === AttachmentType.MCP_RESOURCE + ); + + for (const mcpResource of mcpResources) { + contentParts.push({ + text: formatAttachmentText( + AttachmentLabel.MCP_RESOURCE, + mcpResource.name, + mcpResource.content, + mcpResource.serverName + ), + type: ContentPartType.TEXT + }); + } + + const result: ApiChatMessageData = { + content: contentParts, + role: message.role as MessageRole + }; + + if (message.reasoningContent) { + result.reasoning_content = message.reasoningContent; + } + + if (toolCalls && toolCalls.length > 0) { + result.tool_calls = toolCalls; + } + + return result; + } + + /** + * Fetch the full replay of a server-side stream from byte 0. Returns the raw Response so the + * caller can pipe it through the SSE parser like a fresh stream. + */ + static async fetchStreamReplay(streamId: string): Promise { + const resp = await fetch(ChatService.buildStreamUrl(streamId, 0), { + headers: getAuthHeaders() + }); + + if (!resp.ok) { + throw new ApiError(`Stream replay failed with HTTP ${resp.status}`, resp.status); + } + + return resp; + } + + // write a throttled-but-not-yet-persisted offset immediately; used at + // resume boundaries and on pagehide/visibilitychange so the persisted + // offset is the freshest one when it matters + static flushStreamState(conversationId: string): void { + const tracker = ChatService.streamStateSaveTrackers.get(conversationId); + + if (!tracker || tracker.pendingBytes === null) return; + + const { model, pendingBytes } = tracker; + + tracker.lastSavedAt = Date.now(); + tracker.pendingBytes = null; + + ChatService.writeStreamState(conversationId, pendingBytes, model); + } /** * Sends a streaming chat completion request for generating a chat title. @@ -99,13 +404,610 @@ export class ChatService { return titleResponse; } + static getStreamState(conversationId: string): ResumableStreamState | null { + if (!conversationId) return null; + + try { + const raw = localStorage.getItem(streamStorageKey(conversationId)); + + if (!raw) return null; + + const parsed = JSON.parse(raw) as ResumableStreamState; + + if (!parsed || typeof parsed.bytesReceived !== 'number') return null; + + return parsed; + } catch { + return null; + } + } + /** - * - * - * Messaging - * - * + * Handles streaming response from the chat completion API. */ + static async handleStreamResponse( + response: Response, + onChunk?: (chunk: string) => void, + onComplete?: ( + response: string, + reasoningContent?: string, + timings?: ChatMessageTimings, + toolCalls?: string + ) => void, + onError?: (error: Error) => void, + onReasoningChunk?: (chunk: string) => void, + onToolCallChunk?: (chunk: string) => void, + onModel?: (model: string) => void, + onCompletionId?: (id: string) => void, + onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void, + conversationId?: string, + abortSignal?: AbortSignal, + onConnectionState?: (state: StreamConnectionState) => void, + streamModel?: string | null + ): Promise { + let reader = response.body?.getReader(); + + if (!reader) { + throw new Error('No response body'); + } + + // bytesParsed is the absolute server side buffer offset of the next byte to parse + // segmentStartOffset is the absolute offset where the current reader started, reset on resume + // segmentBytesRead is wire bytes read by the current reader + let bytesParsed = 0; + let segmentStartOffset = 0; + let segmentBytesRead = 0; + let lastByteAt = Date.now(); + // each resume must produce at least one byte to be retried again + // if a resume returns 200 but yields nothing, we abandon + // since the session has a bounded size, the total number of retries is bounded by construction + let madeProgress = true; + + const encoder = new TextEncoder(); + + if (conversationId) { + ChatService.saveStreamState(conversationId, 0, streamModel); + } + + onConnectionState?.(StreamConnectionState.STREAMING); + + let decoder = new TextDecoder(); + let aggregatedContent = ''; + let fullReasoningContent = ''; + let aggregatedToolCalls: ApiChatCompletionToolCall[] = []; + let lastTimings: ChatMessageTimings | undefined; + let streamFinished = false; + let modelEmitted = false; + let idEmitted = false; + let toolCallIndexOffset = 0; + let hasOpenToolCallBatch = false; + + const finalizeOpenToolCallBatch = () => { + if (!hasOpenToolCallBatch) { + return; + } + + toolCallIndexOffset = aggregatedToolCalls.length; + hasOpenToolCallBatch = false; + }; + const processToolCallDelta = (toolCalls?: ApiChatCompletionToolCallDelta[]) => { + if (!toolCalls || toolCalls.length === 0) { + return; + } + + aggregatedToolCalls = ChatService.mergeToolCallDeltas( + aggregatedToolCalls, + toolCalls, + toolCallIndexOffset + ); + + if (aggregatedToolCalls.length === 0) { + return; + } + + hasOpenToolCallBatch = true; + + const serializedToolCalls = JSON.stringify(aggregatedToolCalls); + + if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log('[ChatService] Aggregated tool calls:', serializedToolCalls); + } + + if (!serializedToolCalls) { + return; + } + + if (!abortSignal?.aborted) { + onToolCallChunk?.(serializedToolCalls); + } + }; + const onVisibilityChange = () => { + if (typeof document === 'undefined') return; + + if (document.visibilityState === 'hidden') { + // the tab is going to the background and the OS may throttle or + // drop the socket shortly; persist the freshest resume offset now + if (conversationId) ChatService.flushStreamState(conversationId); + + return; + } + + if (streamFinished) return; + + if (!conversationId) return; + + // the bytes have been quiet for too long, the OS likely killed the socket + // kicking the reader unblocks reader.read with done=true so the outer loop can resume + if (Date.now() - lastByteAt > STREAM_VISIBILITY_KICK_MS) { + reader!.cancel().catch(() => {}); + } + }; + const onPageHide = () => { + // a reload or navigation is about to happen; make sure the resume + // offset that getStreamState() will read is not a stale throttled one + if (conversationId) ChatService.flushStreamState(conversationId); + }; + + if (typeof document !== 'undefined') { + document.addEventListener('visibilitychange', onVisibilityChange); + window.addEventListener('pagehide', onPageHide); + } + + try { + let chunk = ''; + + // outer loop drives the resume cycle, swaps reader on premature end of stream + while (true) { + while (true) { + if (abortSignal?.aborted) break; + + let done: boolean; + let value: Uint8Array | undefined; + + try { + const r = await reader.read(); + + done = r.done; + value = r.value; + } catch (readErr) { + // reader.read() rejects with TypeError when the underlying connection drops + // instead of just resolving with done=true. treat it like done so the outer + // loop swaps reader via the resume path + if (isAbortError(readErr)) { + throw readErr; + } + + console.warn('reader.read() rejected, treating as premature end:', readErr); + done = true; + value = undefined; + } + + if (done) break; + + if (abortSignal?.aborted) break; + + if (value && value.byteLength > 0) { + segmentBytesRead += value.byteLength; + lastByteAt = Date.now(); + + if (!madeProgress) { + madeProgress = true; + onConnectionState?.(StreamConnectionState.STREAMING); + } + } + + chunk += decoder.decode(value, { stream: true }); + const lines = chunk.split(SSE_LINE_SEPARATOR); + + chunk = lines.pop() || ''; + + // the persisted offset must point right after the last fully parsed line, + // the trailing `chunk` is partial bytes still waiting for a newline + if (conversationId) { + const tailBytes = encoder.encode(chunk).byteLength; + + bytesParsed = segmentStartOffset + segmentBytesRead - tailBytes; + ChatService.saveStreamStateThrottled(conversationId, bytesParsed, streamModel); + } + + for (const line of lines) { + if (abortSignal?.aborted) break; + + if (line.startsWith(SSE_DATA_PREFIX)) { + const data = line.slice(SSE_DATA_PREFIX.length).trim(); + + if (data === SSE_DONE_MARKER) { + streamFinished = true; + + continue; + } + + try { + const parsed: ApiChatCompletionStreamChunk = JSON.parse(data); + const choice = parsed.choices?.[0]; + const content = choice?.delta?.content; + const reasoningContent = choice?.delta?.reasoning_content; + const toolCalls = choice?.delta?.tool_calls; + const timings = parsed.timings; + const promptProgress = parsed.prompt_progress; + const chunkModel = ChatService.extractModelName(parsed); + + if (chunkModel && !modelEmitted) { + modelEmitted = true; + onModel?.(chunkModel); + } + + if (parsed.id && !idEmitted) { + idEmitted = true; + onCompletionId?.(parsed.id); + } + + if (promptProgress) { + ChatService.notifyTimings(undefined, promptProgress, onTimings); + } + + if (timings) { + ChatService.notifyTimings(timings, promptProgress, onTimings); + lastTimings = timings; + } + + if (content) { + finalizeOpenToolCallBatch(); + aggregatedContent += content; + + if (!abortSignal?.aborted) { + onChunk?.(content); + } + } + + if (reasoningContent) { + finalizeOpenToolCallBatch(); + fullReasoningContent += reasoningContent; + + if (!abortSignal?.aborted) { + onReasoningChunk?.(reasoningContent); + } + } + + processToolCallDelta(toolCalls); + } catch (e) { + console.error('Error parsing JSON chunk:', e); + } + } + } + + if (abortSignal?.aborted) break; + + if (streamFinished) break; + } + + // inner reader done, decide whether to try a resume + if (abortSignal?.aborted) break; + + if (streamFinished) break; + + if (!conversationId) break; + + if (!madeProgress) { + onConnectionState?.(StreamConnectionState.LOST); + onError?.(new Error('Stream resume produced no new bytes, giving up')); + + break; + } + + onConnectionState?.(StreamConnectionState.RESUMING); + madeProgress = false; + + // the server resends starting at bytesParsed, discard any partial line we held, it + // will be retransmitted from a clean line boundary. reuse the frozen model, not the + // live dropdown + // resumeStream reads the offset from localStorage, so persist the + // freshest bytesParsed before asking the server to replay from it + ChatService.flushStreamState(conversationId); + const resumeResp = await ChatService.resumeStream( + conversationId, + abortSignal, + streamModel + ).catch(() => null); + + // an abort landing during the resume request is intentional, not a lost connection + if (abortSignal?.aborted) break; + + if (!resumeResp || resumeResp.status !== 200) { + onConnectionState?.(StreamConnectionState.LOST); + onError?.(new Error('Stream connection lost and could not be resumed')); + + break; + } + + const newReader = resumeResp.body?.getReader(); + + if (!newReader) break; + + try { + reader.releaseLock(); + } catch { + /* ignore */ + } + reader = newReader; + decoder = new TextDecoder(); + chunk = ''; + segmentStartOffset = bytesParsed; + segmentBytesRead = 0; + lastByteAt = Date.now(); + } + + if (abortSignal?.aborted) return; + + if (streamFinished) { + finalizeOpenToolCallBatch(); + + if (conversationId) { + ChatService.clearStreamState(conversationId); + } + + const finalToolCalls = + aggregatedToolCalls.length > 0 ? JSON.stringify(aggregatedToolCalls) : undefined; + + onComplete?.( + aggregatedContent, + fullReasoningContent || undefined, + lastTimings, + finalToolCalls + ); + } + } catch (error) { + const err = error instanceof Error ? error : new Error('Stream error'); + + onError?.(err); + + throw err; + } finally { + if (typeof document !== 'undefined') { + document.removeEventListener('visibilitychange', onVisibilityChange); + window.removeEventListener('pagehide', onPageHide); + } + + try { + reader.releaseLock(); + } catch { + /* ignore */ + } + } + } + + /** + * Look up server-side stream sessions for the given conversation ids. Ids carry the frozen + * conv::model identity when a model was bound at POST time. + */ + static async lookupStreamSessions(conversationIds: string[]): Promise { + const resp = await fetch(API_STREAM.LOOKUP, { + body: JSON.stringify({ conversation_ids: conversationIds }), + headers: getJsonHeaders(), + method: 'POST' + }); + + if (!resp.ok) { + throw new ApiError(`Stream lookup failed with HTTP ${resp.status}`, resp.status); + } + + const body = (await resp.json()) as unknown; + + if (!Array.isArray(body)) { + throw new Error('Stream lookup returned a non-array response'); + } + + return body as ApiStreamSession[]; + } + + /** + * Normalizes an array of messages (database or already-API-shaped) into + * API chat message data, converting DB messages and dropping empty system + * messages. Shared by sendMessage, preEncode and the agentic flow. + */ + static async normalizeMessagesForApi( + messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[] + ): Promise { + return ( + await Promise.all( + messages.map((msg) => { + if ('id' in msg && 'convId' in msg && 'timestamp' in msg) { + return ChatService.convertDbMessageToApiChatMessageData( + msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] } + ); + } + + return msg as ApiChatMessageData; + }) + ) + ).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => { + // Filter out empty system messages + if (msg.role === MessageRole.SYSTEM) { + const content = typeof msg.content === 'string' ? msg.content : ''; + + return content.trim().length > 0; + } + + return true; + }); + } + + /** + * Fire-and-forget request to pre-encode the conversation in the server's KV cache. + * Re-submits the full conversation with n_predict=0 so the server processes the prompt + * without generating tokens, warming the cache for the next turn. + */ + static async preEncode( + messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[], + model?: string | null, + excludeReasoning?: boolean, + signal?: AbortSignal + ): Promise { + const normalizedMessages: ApiChatMessageData[] = + await ChatService.normalizeMessagesForApi(messages); + const requestBody: Record = { + messages: normalizedMessages.map((msg: ApiChatMessageData) => { + const mapped: Record = { + content: excludeReasoning ? ChatService.stripReasoningContent(msg.content) : msg.content, + role: msg.role, + tool_call_id: msg.tool_call_id, + tool_calls: msg.tool_calls + }; + + if (!excludeReasoning && msg.reasoning_content) { + mapped.reasoning_content = msg.reasoning_content; + } + + return mapped; + }), + n_predict: 0, + stream: false + }; + + if (model) { + requestBody.model = model; + } + + try { + await fetch(API_CHAT.COMPLETIONS, { + body: JSON.stringify(requestBody), + headers: getJsonHeaders(), + method: 'POST', + signal + }); + } catch (error) { + if (!isAbortError(error)) { + console.warn('[ChatService] Pre-encode request failed:', error); + } + } + } + + // probe the resume route status without consuming the stream: the SSE route has no HEAD, + // so issue the GET and abort it right after the status line. 0 on network error + static async probeResumeStatus(streamId: string): Promise { + if (!streamId) return 0; + + const ac = new AbortController(); + + try { + const resp = await fetch(ChatService.buildStreamUrl(streamId, 0), { + headers: getAuthHeaders(), + signal: ac.signal + }); + + ac.abort(); + + return resp.status; + } catch { + return 0; + } + } + + static async resumeStream( + conversationId: string, + signal?: AbortSignal, + model?: string | null + ): Promise { + if (!conversationId) return null; + + const state = ChatService.getStreamState(conversationId); + const from = state?.bytesReceived ?? 0; + const id = streamIdentity(conversationId, model); + const url = ChatService.buildStreamUrl(id, from); + + return await fetch(url, { headers: getAuthHeaders(), method: 'GET', signal }); + } + + /** + * Rebuild the stream identity for a resume. The model persisted at POST time wins, including a + * stored null which means the POST carried no explicit model so the identity stays the bare conv + * id. Only fall back to the caller supplied current model when nothing was persisted. + */ + static resumeStreamIdentity( + conversationId: string, + state: ResumableStreamState | null, + fallbackModel: string | null + ): string { + const model = state && state.model !== undefined ? state.model : fallbackModel; + + return streamIdentity(conversationId, model); + } + + // persist the running byte count and the frozen model for a conversation, a later visit + // resumes the SSE replay at the right offset under the same conv::model + // identity. Writes immediately; the per-chunk read loop uses the throttled + // variant instead. + static saveStreamState( + conversationId: string, + bytesReceived: number, + model?: string | null + ): void { + if (!conversationId) return; + + ChatService.writeStreamState(conversationId, bytesReceived, model); + // record the write so a throttled save landing inside the interval + // holds its value pending instead of re-writing + ChatService.streamStateSaveTrackers.set(conversationId, { + lastSavedAt: Date.now(), + model: model ?? null, + pendingBytes: null + }); + } + + // throttled variant for the per-chunk read loop: writes at most once per + // conversation per STREAM_STATE_SAVE_INTERVAL_MS, holding the latest value + // pending until the interval elapses or flushStreamState() forces it out + static saveStreamStateThrottled( + conversationId: string, + bytesReceived: number, + model?: string | null + ): void { + if (!conversationId) return; + + const tracker = ChatService.streamStateSaveTrackers.get(conversationId) ?? { + lastSavedAt: 0, + model: null, + pendingBytes: null + }; + + tracker.model = model ?? null; + + if (Date.now() - tracker.lastSavedAt >= ChatService.STREAM_STATE_SAVE_INTERVAL_MS) { + tracker.lastSavedAt = Date.now(); + tracker.pendingBytes = null; + ChatService.writeStreamState(conversationId, bytesReceived, model); + } else { + tracker.pendingBytes = bytesReceived; + } + + ChatService.streamStateSaveTrackers.set(conversationId, tracker); + } + + /** + * Pick the running session to splice into when discoverActiveStream lists candidates for a + * conversation. Finalized sessions are not candidates: their final content was already written + * to the DB by the original onComplete handler, so attaching to them would replay a buffer that + * may not match what the DB holds. A continue session's buffer holds only the appended deltas, + * not the pre continue prefix, so replaying it as a fresh generation would erase the original. + * + * Among running sessions we tie break on the most recent started_at, which covers the case of + * multiple inferences left running on the same conversation. + */ + static selectActiveStream( + sessions: ApiStreamSession[] | null | undefined + ): ApiStreamSession | null { + if (!Array.isArray(sessions) || sessions.length === 0) { + return null; + } + + const running = sessions.filter((s) => !s.is_done); + + if (running.length === 0) { + return null; + } + + return running.reduce((best, cur) => (cur.started_at > best.started_at ? cur : best)); + } /** * Sends a chat completion request to the llama-server. @@ -169,31 +1071,11 @@ export class ChatService { xtc_probability, xtc_threshold } = options; - const normalizedMessages: ApiChatMessageData[] = ( - await Promise.all( - messages.map((msg) => { - if ('id' in msg && 'convId' in msg && 'timestamp' in msg) { - const dbMsg = msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }; - - return ChatService.convertDbMessageToApiChatMessageData(dbMsg); - } else { - return msg as ApiChatMessageData; - } - }) - ) - ).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => { - // Filter out empty system messages - if (msg.role === MessageRole.SYSTEM) { - const content = typeof msg.content === 'string' ? msg.content : ''; - - return content.trim().length > 0; - } - - return true; - }); + const normalizedMessages: ApiChatMessageData[] = + await ChatService.normalizeMessagesForApi(messages); // Filter out image attachments if the model doesn't support vision - if (options.model && !modelsStore.modelSupportsVision(options.model)) { + if (options.model && !modelsStore.props.modelSupportsVision(options.model)) { normalizedMessages.forEach((msg) => { if (Array.isArray(msg.content)) { msg.content = msg.content.filter((part: ApiChatMessageContentPart) => { @@ -436,31 +1318,6 @@ export class ChatService { } } - /** - * Checks whether all server slots are currently idle (not processing any requests). - * Queries the /slots endpoint (requires --slots flag on the server). - * Returns true if all slots are idle, false if any is processing. - * If the endpoint is unavailable or errors out, returns true (best-effort fallback). - * - * @param signal - Optional AbortSignal to cancel the request if needed - * @param model - Optional model name to check slots for (required in ROUTER mode) - * @returns {Promise} Promise that resolves to true if all slots are idle, false if any is processing - */ - static async areAllSlotsIdle(model?: string | null, signal?: AbortSignal): Promise { - try { - const url = model ? `${API_SLOTS.LIST}?model=${encodeURIComponent(model)}` : API_SLOTS.LIST; - const res = await fetch(url, { signal }); - - if (!res.ok) return true; - - const slots: { is_processing: boolean }[] = await res.json(); - - return slots.every((s) => !s.is_processing); - } catch { - return true; - } - } - /** * Ends the current reasoning block of a running completion, targeted by its * chat completion id (streamed back as `id`). Matching the completion rather @@ -510,167 +1367,6 @@ export class ChatService { } } - /** - * Sends a fire-and-forget request to pre-encode the conversation in the server's KV cache. - * After a response completes, this re-submits the full conversation - * using n_predict=0 and stream=false so the server processes the prompt without generating tokens. - * This warms the cache for the next turn, making it faster. - * - * When excludeReasoningFromContext is true, reasoning content is stripped from the messages - * to match what sendMessage would send on the next turn (avoiding cache misses). - * When false, reasoning_content is preserved so the cached prompt matches the next request. - * - * @param messages - The full conversation including the latest assistant response - * @param model - Optional model name (required in ROUTER mode) - * @param excludeReasoning - Whether to strip reasoning content (should match excludeReasoningFromContext setting) - * @param signal - Optional AbortSignal to cancel the pre-encode request - */ - static async cancelServerStream(conversationId: string, model?: string | null): Promise { - if (!conversationId) return; - - try { - const id = streamIdentity(conversationId, model); - - await fetch(ChatService.buildStreamUrl(id), { - headers: getAuthHeaders(), - method: 'DELETE' - }); - } catch (e) { - console.warn('cancelServerStream failed:', e); - } - } - - /** - * Look up server-side stream sessions for the given conversation ids. Ids carry the frozen - * conv::model identity when a model was bound at POST time. - */ - static async lookupStreamSessions(conversationIds: string[]): Promise { - const resp = await fetch(API_STREAM.LOOKUP, { - body: JSON.stringify({ conversation_ids: conversationIds }), - headers: getJsonHeaders(), - method: 'POST' - }); - - if (!resp.ok) { - throw new ApiError(`Stream lookup failed with HTTP ${resp.status}`, resp.status); - } - - const body = (await resp.json()) as unknown; - - if (!Array.isArray(body)) { - throw new Error('Stream lookup returned a non-array response'); - } - - return body as ApiStreamSession[]; - } - - /** - * Fetch the full replay of a server-side stream from byte 0. Returns the raw Response so the - * caller can pipe it through the SSE parser like a fresh stream. - */ - static async fetchStreamReplay(streamId: string): Promise { - const resp = await fetch(ChatService.buildStreamUrl(streamId, 0), { - headers: getAuthHeaders() - }); - - if (!resp.ok) { - throw new ApiError(`Stream replay failed with HTTP ${resp.status}`, resp.status); - } - - return resp; - } - - /** - * Pick the running session to splice into when discoverActiveStream lists candidates for a - * conversation. Finalized sessions are not candidates: their final content was already written - * to the DB by the original onComplete handler, so attaching to them would replay a buffer that - * may not match what the DB holds. A continue session's buffer holds only the appended deltas, - * not the pre continue prefix, so replaying it as a fresh generation would erase the original. - * - * Among running sessions we tie break on the most recent started_at, which covers the case of - * multiple inferences left running on the same conversation. - */ - static selectActiveStream( - sessions: ApiStreamSession[] | null | undefined - ): ApiStreamSession | null { - if (!Array.isArray(sessions) || sessions.length === 0) { - return null; - } - - const running = sessions.filter((s) => !s.is_done); - - if (running.length === 0) { - return null; - } - - return running.reduce((best, cur) => (cur.started_at > best.started_at ? cur : best)); - } - - // persist the running byte count and the frozen model for a conversation, a later visit - // resumes the SSE replay at the right offset under the same conv::model identity - static saveStreamState( - conversationId: string, - bytesReceived: number, - model?: string | null - ): void { - if (!conversationId) return; - - try { - const state: ResumableStreamState = { - bytesReceived, - model: model ?? null, - updatedAt: Date.now() - }; - - localStorage.setItem(streamStorageKey(conversationId), JSON.stringify(state)); - } catch { - // localStorage may be full or disabled, silently ignore - } - } - - static getStreamState(conversationId: string): ResumableStreamState | null { - if (!conversationId) return null; - - try { - const raw = localStorage.getItem(streamStorageKey(conversationId)); - - if (!raw) return null; - - const parsed = JSON.parse(raw) as ResumableStreamState; - - if (!parsed || typeof parsed.bytesReceived !== 'number') return null; - - return parsed; - } catch { - return null; - } - } - - static clearStreamState(conversationId: string): void { - if (!conversationId) return; - - try { - localStorage.removeItem(streamStorageKey(conversationId)); - } catch { - // nothing to do - } - } - - /** - * Rebuild the stream identity for a resume. The model persisted at POST time wins, including a - * stored null which means the POST carried no explicit model so the identity stays the bare conv - * id. Only fall back to the caller supplied current model when nothing was persisted. - */ - static resumeStreamIdentity( - conversationId: string, - state: ResumableStreamState | null, - fallbackModel: string | null - ): string { - const model = state && state.model !== undefined ? state.model : fallbackModel; - - return streamIdentity(conversationId, model); - } - // build the replay route url for a stream identity, from is the resume byte offset, omitted // for the cancel route private static buildStreamUrl(streamId: string, from?: number): string { @@ -681,462 +1377,58 @@ export class ChatService { } /** - * Reconnect to an interrupted stream for this conversation. Returns the fetch Response so the - * existing SSE parser drains it like a fresh stream. The server returns 200 on success, 404 if - * no session exists for the conv_id, and 400 if the offset is below the dropped prefix. + * Extracts model name from Chat Completions API response data. + * Handles various response formats including streaming chunks and final responses. + * + * WORKAROUND: In single model mode, llama-server returns a default/incorrect model name + * in the response. We override it with the actual model name from serverStore. + * + * @param data - Raw response data from the Chat Completions API + * @returns Model name string if found, undefined otherwise + * @private */ - // probe the resume route status without consuming the stream: the SSE route has no HEAD, - // so issue the GET and abort it right after the status line. 0 on network error - static async probeResumeStatus(streamId: string): Promise { - if (!streamId) return 0; - - const ac = new AbortController(); - - try { - const resp = await fetch(ChatService.buildStreamUrl(streamId, 0), { - headers: getAuthHeaders(), - signal: ac.signal - }); - - ac.abort(); - - return resp.status; - } catch { - return 0; - } - } - - static async resumeStream( - conversationId: string, - signal?: AbortSignal, - model?: string | null - ): Promise { - if (!conversationId) return null; - - const state = ChatService.getStreamState(conversationId); - const from = state?.bytesReceived ?? 0; - const id = streamIdentity(conversationId, model); - const url = ChatService.buildStreamUrl(id, from); - - return await fetch(url, { headers: getAuthHeaders(), method: 'GET', signal }); - } - - static async preEncode( - messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[], - model?: string | null, - excludeReasoning?: boolean, - signal?: AbortSignal - ): Promise { - const normalizedMessages: ApiChatMessageData[] = ( - await Promise.all( - messages.map((msg) => { - if ('id' in msg && 'convId' in msg && 'timestamp' in msg) { - return ChatService.convertDbMessageToApiChatMessageData( - msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] } - ); - } - - return msg as ApiChatMessageData; - }) - ) - ).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => { - if (msg.role === MessageRole.SYSTEM) { - const content = typeof msg.content === 'string' ? msg.content : ''; - - return content.trim().length > 0; - } - - return true; - }); - const requestBody: Record = { - messages: normalizedMessages.map((msg: ApiChatMessageData) => { - const mapped: Record = { - content: excludeReasoning ? ChatService.stripReasoningContent(msg.content) : msg.content, - role: msg.role, - tool_call_id: msg.tool_call_id, - tool_calls: msg.tool_calls - }; - - if (!excludeReasoning && msg.reasoning_content) { - mapped.reasoning_content = msg.reasoning_content; - } - - return mapped; - }), - n_predict: 0, - stream: false + private static extractModelName(data: unknown): string | undefined { + const asRecord = (value: unknown): Record | undefined => { + return typeof value === 'object' && value !== null + ? (value as Record) + : undefined; }; - - if (model) { - requestBody.model = model; - } - - try { - await fetch(API_CHAT.COMPLETIONS, { - body: JSON.stringify(requestBody), - headers: getJsonHeaders(), - method: 'POST', - signal - }); - } catch (error) { - if (!isAbortError(error)) { - console.warn('[ChatService] Pre-encode request failed:', error); - } - } - } - - /** - * - * - * Streaming - * - * - */ - - /** - * Handles streaming response from the chat completion API - * @param response - The Response object from the fetch request - * @param onChunk - Optional callback invoked for each content chunk received - * @param onComplete - Optional callback invoked when the stream is complete with full response - * @param onError - Optional callback invoked if an error occurs during streaming - * @param onReasoningChunk - Optional callback invoked for each reasoning content chunk - * @param conversationId - Optional conversation ID for per-conversation state tracking - * @returns {Promise} Promise that resolves when streaming is complete - * @throws {Error} if the stream cannot be read or parsed - */ - static async handleStreamResponse( - response: Response, - onChunk?: (chunk: string) => void, - onComplete?: ( - response: string, - reasoningContent?: string, - timings?: ChatMessageTimings, - toolCalls?: string - ) => void, - onError?: (error: Error) => void, - onReasoningChunk?: (chunk: string) => void, - onToolCallChunk?: (chunk: string) => void, - onModel?: (model: string) => void, - onCompletionId?: (id: string) => void, - onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void, - conversationId?: string, - abortSignal?: AbortSignal, - onConnectionState?: (state: StreamConnectionState) => void, - streamModel?: string | null - ): Promise { - let reader = response.body?.getReader(); - - if (!reader) { - throw new Error('No response body'); - } - - // bytesParsed is the absolute server side buffer offset of the next byte to parse - // segmentStartOffset is the absolute offset where the current reader started, reset on resume - // segmentBytesRead is wire bytes read by the current reader - let bytesParsed = 0; - let segmentStartOffset = 0; - let segmentBytesRead = 0; - let lastByteAt = Date.now(); - // each resume must produce at least one byte to be retried again - // if a resume returns 200 but yields nothing, we abandon - // since the session has a bounded size, the total number of retries is bounded by construction - let madeProgress = true; - - const encoder = new TextEncoder(); - - if (conversationId) { - ChatService.saveStreamState(conversationId, 0, streamModel); - } - - onConnectionState?.(StreamConnectionState.STREAMING); - - let decoder = new TextDecoder(); - let aggregatedContent = ''; - let fullReasoningContent = ''; - let aggregatedToolCalls: ApiChatCompletionToolCall[] = []; - let lastTimings: ChatMessageTimings | undefined; - let streamFinished = false; - let modelEmitted = false; - let idEmitted = false; - let toolCallIndexOffset = 0; - let hasOpenToolCallBatch = false; - - const finalizeOpenToolCallBatch = () => { - if (!hasOpenToolCallBatch) { - return; - } - - toolCallIndexOffset = aggregatedToolCalls.length; - hasOpenToolCallBatch = false; + const getTrimmedString = (value: unknown): string | undefined => { + return typeof value === 'string' && value.trim() ? value.trim() : undefined; }; - const processToolCallDelta = (toolCalls?: ApiChatCompletionToolCallDelta[]) => { - if (!toolCalls || toolCalls.length === 0) { - return; - } + const root = asRecord(data); - aggregatedToolCalls = ChatService.mergeToolCallDeltas( - aggregatedToolCalls, - toolCalls, - toolCallIndexOffset - ); + if (!root) return undefined; - if (aggregatedToolCalls.length === 0) { - return; - } + // 1) root (some implementations provide `model` at the top level) + const rootModel = getTrimmedString(root.model); - hasOpenToolCallBatch = true; - - const serializedToolCalls = JSON.stringify(aggregatedToolCalls); - - if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { - console.log('[ChatService] Aggregated tool calls:', serializedToolCalls); - } - - if (!serializedToolCalls) { - return; - } - - if (!abortSignal?.aborted) { - onToolCallChunk?.(serializedToolCalls); - } - }; - const onVisibilityChange = () => { - if (typeof document === 'undefined') return; - - if (document.visibilityState !== 'visible') return; - - if (streamFinished) return; - - if (!conversationId) return; - - // the bytes have been quiet for too long, the OS likely killed the socket - // kicking the reader unblocks reader.read with done=true so the outer loop can resume - if (Date.now() - lastByteAt > STREAM_VISIBILITY_KICK_MS) { - reader!.cancel().catch(() => {}); - } - }; - - if (typeof document !== 'undefined') { - document.addEventListener('visibilitychange', onVisibilityChange); + if (rootModel) { + return rootModel; } - try { - let chunk = ''; + // 2) streaming choice (delta) or final response (message) + const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined; - // outer loop drives the resume cycle, swaps reader on premature end of stream - while (true) { - while (true) { - if (abortSignal?.aborted) break; - - let done: boolean; - let value: Uint8Array | undefined; - - try { - const r = await reader.read(); - - done = r.done; - value = r.value; - } catch (readErr) { - // reader.read() rejects with TypeError when the underlying connection drops - // instead of just resolving with done=true. treat it like done so the outer - // loop swaps reader via the resume path - if (isAbortError(readErr)) { - throw readErr; - } - - console.warn('reader.read() rejected, treating as premature end:', readErr); - done = true; - value = undefined; - } - - if (done) break; - - if (abortSignal?.aborted) break; - - if (value && value.byteLength > 0) { - segmentBytesRead += value.byteLength; - lastByteAt = Date.now(); - - if (!madeProgress) { - madeProgress = true; - onConnectionState?.(StreamConnectionState.STREAMING); - } - } - - chunk += decoder.decode(value, { stream: true }); - const lines = chunk.split(SSE_LINE_SEPARATOR); - - chunk = lines.pop() || ''; - - // the persisted offset must point right after the last fully parsed line, - // the trailing `chunk` is partial bytes still waiting for a newline - if (conversationId) { - const tailBytes = encoder.encode(chunk).byteLength; - - bytesParsed = segmentStartOffset + segmentBytesRead - tailBytes; - ChatService.saveStreamState(conversationId, bytesParsed, streamModel); - } - - for (const line of lines) { - if (abortSignal?.aborted) break; - - if (line.startsWith(SSE_DATA_PREFIX)) { - const data = line.slice(SSE_DATA_PREFIX.length).trim(); - - if (data === SSE_DONE_MARKER) { - streamFinished = true; - - continue; - } - - try { - const parsed: ApiChatCompletionStreamChunk = JSON.parse(data); - const choice = parsed.choices?.[0]; - const content = choice?.delta?.content; - const reasoningContent = choice?.delta?.reasoning_content; - const toolCalls = choice?.delta?.tool_calls; - const timings = parsed.timings; - const promptProgress = parsed.prompt_progress; - const chunkModel = ChatService.extractModelName(parsed); - - if (chunkModel && !modelEmitted) { - modelEmitted = true; - onModel?.(chunkModel); - } - - if (parsed.id && !idEmitted) { - idEmitted = true; - onCompletionId?.(parsed.id); - } - - if (promptProgress) { - ChatService.notifyTimings(undefined, promptProgress, onTimings); - } - - if (timings) { - ChatService.notifyTimings(timings, promptProgress, onTimings); - lastTimings = timings; - } - - if (content) { - finalizeOpenToolCallBatch(); - aggregatedContent += content; - - if (!abortSignal?.aborted) { - onChunk?.(content); - } - } - - if (reasoningContent) { - finalizeOpenToolCallBatch(); - fullReasoningContent += reasoningContent; - - if (!abortSignal?.aborted) { - onReasoningChunk?.(reasoningContent); - } - } - - processToolCallDelta(toolCalls); - } catch (e) { - console.error('Error parsing JSON chunk:', e); - } - } - } - - if (abortSignal?.aborted) break; - - if (streamFinished) break; - } - - // inner reader done, decide whether to try a resume - if (abortSignal?.aborted) break; - - if (streamFinished) break; - - if (!conversationId) break; - - if (!madeProgress) { - onConnectionState?.(StreamConnectionState.LOST); - onError?.(new Error('Stream resume produced no new bytes, giving up')); - - break; - } - - onConnectionState?.(StreamConnectionState.RESUMING); - madeProgress = false; - - // the server resends starting at bytesParsed, discard any partial line we held, it - // will be retransmitted from a clean line boundary. reuse the frozen model, not the - // live dropdown - const resumeResp = await ChatService.resumeStream( - conversationId, - abortSignal, - streamModel - ).catch(() => null); - - // an abort landing during the resume request is intentional, not a lost connection - if (abortSignal?.aborted) break; - - if (!resumeResp || resumeResp.status !== 200) { - onConnectionState?.(StreamConnectionState.LOST); - onError?.(new Error('Stream connection lost and could not be resumed')); - - break; - } - - const newReader = resumeResp.body?.getReader(); - - if (!newReader) break; - - try { - reader.releaseLock(); - } catch { - /* ignore */ - } - reader = newReader; - decoder = new TextDecoder(); - chunk = ''; - segmentStartOffset = bytesParsed; - segmentBytesRead = 0; - lastByteAt = Date.now(); - } - - if (abortSignal?.aborted) return; - - if (streamFinished) { - finalizeOpenToolCallBatch(); - - if (conversationId) { - ChatService.clearStreamState(conversationId); - } - - const finalToolCalls = - aggregatedToolCalls.length > 0 ? JSON.stringify(aggregatedToolCalls) : undefined; - - onComplete?.( - aggregatedContent, - fullReasoningContent || undefined, - lastTimings, - finalToolCalls - ); - } - } catch (error) { - const err = error instanceof Error ? error : new Error('Stream error'); - - onError?.(err); - - throw err; - } finally { - if (typeof document !== 'undefined') { - document.removeEventListener('visibilitychange', onVisibilityChange); - } - - try { - reader.releaseLock(); - } catch { - /* ignore */ - } + if (!firstChoice) { + return undefined; } + + // priority: delta.model (first chunk) else message.model (final response) + const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model); + + if (deltaModel) { + return deltaModel; + } + + const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model); + + if (messageModel) { + return messageModel; + } + + // avoid guessing from non-standard locations (metadata, etc.) + return undefined; } /** @@ -1271,253 +1563,23 @@ export class ChatService { } /** + * Calls the onTimings callback with timing data from streaming response. * - * - * Conversion - * - * + * @param timings - Timing information from the Chat Completions API response + * @param promptProgress - Prompt processing progress data + * @param onTimingsCallback - Callback function to invoke with timing data + * @private */ + private static notifyTimings( + timings: ChatMessageTimings | undefined, + promptProgress: ChatMessagePromptProgress | undefined, + onTimingsCallback: + | ((timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void) + | undefined + ): void { + if (!onTimingsCallback || (!timings && !promptProgress)) return; - /** - * Converts a database message with attachments to API chat message format. - * Processes various attachment types (images, text files, PDFs) and formats them - * as content parts suitable for the chat completion API. - * - * @param message - Database message object with optional extra attachments - * @param message.content - The text content of the message - * @param message.role - The role of the message sender (user, assistant, system) - * @param message.extra - Optional array of message attachments (images, files, etc.) - * @returns {ApiChatMessageData} object formatted for the chat completion API - * @static - */ - static async convertDbMessageToApiChatMessageData( - message: DatabaseMessage & { extra?: DatabaseMessageExtra[] } - ): Promise { - // Handle tool result messages (role: 'tool') - if (message.role === MessageRole.TOOL && message.toolCallId) { - return { - content: message.content, - role: MessageRole.TOOL, - tool_call_id: message.toolCallId - }; - } - - // Parse tool calls for assistant messages - let toolCalls: ApiChatCompletionToolCall[] | undefined; - - if (message.toolCalls) { - try { - toolCalls = JSON.parse(message.toolCalls); - } catch { - // Ignore parse errors for malformed tool calls - } - } - - if (!message.extra || message.extra.length === 0) { - const result: ApiChatMessageData = { - content: message.content, - role: message.role as MessageRole - }; - - if (message.reasoningContent) { - result.reasoning_content = message.reasoningContent; - } - - if (toolCalls && toolCalls.length > 0) { - result.tool_calls = toolCalls; - } - - return result; - } - - const contentParts: ApiChatMessageContentPart[] = []; - const textFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraTextFile => - extra.type === AttachmentType.TEXT - ); - - for (const textFile of textFiles) { - contentParts.push({ - text: formatAttachmentText(AttachmentLabel.FILE, textFile.name, textFile.content), - type: ContentPartType.TEXT - }); - } - - // Handle legacy 'context' type from the old UI (pasted content) - const legacyContextFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraLegacyContext => - extra.type === AttachmentType.LEGACY_CONTEXT - ); - - for (const legacyContextFile of legacyContextFiles) { - contentParts.push({ - text: formatAttachmentText( - AttachmentLabel.FILE, - legacyContextFile.name, - legacyContextFile.content - ), - type: ContentPartType.TEXT - }); - } - - const imageFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraImageFile => - extra.type === AttachmentType.IMAGE - ); - - for (const image of imageFiles) { - const maxImageResolution = settingsStore.getConfig(SETTINGS_KEYS.MAX_IMAGE_RESOLUTION); - // Caps the resolution and bakes the jpeg exif orientation in one pass, - // untouched images pass through as is - const base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution); - - contentParts.push({ - image_url: { url: base64Url }, - type: ContentPartType.IMAGE_URL - }); - } - - const audioFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraAudioFile => - extra.type === AttachmentType.AUDIO - ); - - for (const audio of audioFiles) { - contentParts.push({ - input_audio: { - data: audio.base64Data, - format: getAudioInputFormat(audio.mimeType) - }, - type: ContentPartType.INPUT_AUDIO - }); - } - - if (message.content) { - contentParts.push({ - text: message.content, - type: ContentPartType.TEXT - }); - } - - const videoFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraVideoFile => - extra.type === AttachmentType.VIDEO - ); - - for (const video of videoFiles) { - contentParts.push({ - input_video: { - data: video.base64Data, - format: video.mimeType.includes('mp4') - ? 'mp4' - : video.mimeType.includes('ogg') - ? 'ogg' - : 'auto' - }, - type: ContentPartType.INPUT_VIDEO - }); - } - - const pdfFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraPdfFile => - extra.type === AttachmentType.PDF - ); - - for (const pdfFile of pdfFiles) { - if (pdfFile.processedAsImages && pdfFile.images) { - for (let i = 0; i < pdfFile.images.length; i++) { - contentParts.push({ - image_url: { url: pdfFile.images[i] }, - type: ContentPartType.IMAGE_URL - }); - } - } else { - contentParts.push({ - text: formatAttachmentText(AttachmentLabel.PDF_FILE, pdfFile.name, pdfFile.content), - type: ContentPartType.TEXT - }); - } - } - - const mcpPrompts = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraMcpPrompt => - extra.type === AttachmentType.MCP_PROMPT - ); - - for (const mcpPrompt of mcpPrompts) { - contentParts.push({ - text: formatAttachmentText( - AttachmentLabel.MCP_PROMPT, - mcpPrompt.name, - mcpPrompt.content, - mcpPrompt.serverName - ), - type: ContentPartType.TEXT - }); - } - - const mcpResources = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraMcpResource => - extra.type === AttachmentType.MCP_RESOURCE - ); - - for (const mcpResource of mcpResources) { - contentParts.push({ - text: formatAttachmentText( - AttachmentLabel.MCP_RESOURCE, - mcpResource.name, - mcpResource.content, - mcpResource.serverName - ), - type: ContentPartType.TEXT - }); - } - - const result: ApiChatMessageData = { - content: contentParts, - role: message.role as MessageRole - }; - - if (message.reasoningContent) { - result.reasoning_content = message.reasoningContent; - } - - if (toolCalls && toolCalls.length > 0) { - result.tool_calls = toolCalls; - } - - return result; - } - - /** - * - * - * Utilities - * - * - */ - - /** - * Strips legacy inline reasoning content tags from message content. - * Handles both plain string content and multipart content arrays. - */ - private static stripReasoningContent( - content: string | ApiChatMessageContentPart[] - ): string | ApiChatMessageContentPart[] { - const stripFromString = (text: string): string => - text.replace(LEGACY_AGENTIC_REGEX.REASONING_BLOCK, '').trim(); - - if (typeof content === 'string') { - return stripFromString(content); - } - - return content.map((part) => { - if (part.type === ContentPartType.TEXT && part.text) { - return { ...part, text: stripFromString(part.text) }; - } - - return part; - }); + onTimingsCallback(timings, promptProgress); } /** @@ -1560,77 +1622,44 @@ export class ChatService { } /** - * Extracts model name from Chat Completions API response data. - * Handles various response formats including streaming chunks and final responses. - * - * WORKAROUND: In single model mode, llama-server returns a default/incorrect model name - * in the response. We override it with the actual model name from serverStore. - * - * @param data - Raw response data from the Chat Completions API - * @returns Model name string if found, undefined otherwise - * @private + * Strips legacy inline reasoning content tags from message content. + * Handles both plain string content and multipart content arrays. */ - private static extractModelName(data: unknown): string | undefined { - const asRecord = (value: unknown): Record | undefined => { - return typeof value === 'object' && value !== null - ? (value as Record) - : undefined; - }; - const getTrimmedString = (value: unknown): string | undefined => { - return typeof value === 'string' && value.trim() ? value.trim() : undefined; - }; - const root = asRecord(data); + private static stripReasoningContent( + content: string | ApiChatMessageContentPart[] + ): string | ApiChatMessageContentPart[] { + const stripFromString = (text: string): string => + text.replace(LEGACY_AGENTIC_REGEX.REASONING_BLOCK, '').trim(); - if (!root) return undefined; - - // 1) root (some implementations provide `model` at the top level) - const rootModel = getTrimmedString(root.model); - - if (rootModel) { - return rootModel; + if (typeof content === 'string') { + return stripFromString(content); } - // 2) streaming choice (delta) or final response (message) - const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined; + return content.map((part) => { + if (part.type === ContentPartType.TEXT && part.text) { + return { ...part, text: stripFromString(part.text) }; + } - if (!firstChoice) { - return undefined; - } - - // priority: delta.model (first chunk) else message.model (final response) - const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model); - - if (deltaModel) { - return deltaModel; - } - - const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model); - - if (messageModel) { - return messageModel; - } - - // avoid guessing from non-standard locations (metadata, etc.) - return undefined; + return part; + }); } - /** - * Calls the onTimings callback with timing data from streaming response. - * - * @param timings - Timing information from the Chat Completions API response - * @param promptProgress - Prompt processing progress data - * @param onTimingsCallback - Callback function to invoke with timing data - * @private - */ - private static notifyTimings( - timings: ChatMessageTimings | undefined, - promptProgress: ChatMessagePromptProgress | undefined, - onTimingsCallback: - | ((timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void) - | undefined + // write the resume state straight to localStorage, bypassing the throttle + private static writeStreamState( + conversationId: string, + bytesReceived: number, + model?: string | null ): void { - if (!onTimingsCallback || (!timings && !promptProgress)) return; + try { + const state: ResumableStreamState = { + bytesReceived, + model: model ?? null, + updatedAt: Date.now() + }; - onTimingsCallback(timings, promptProgress); + localStorage.setItem(streamStorageKey(conversationId), JSON.stringify(state)); + } catch { + // localStorage may be full or disabled, silently ignore + } } } diff --git a/tools/ui/src/lib/services/conversation-transfer.service.ts b/tools/ui/src/lib/services/conversation-transfer.service.ts index acef580053..40a09477a3 100644 --- a/tools/ui/src/lib/services/conversation-transfer.service.ts +++ b/tools/ui/src/lib/services/conversation-transfer.service.ts @@ -16,187 +16,6 @@ import { import { strFromU8, strToU8, unzipSync, zipSync } from 'fflate'; export class ConversationTransferService { - /** - * - * - * JSONL Session Format - * - * - */ - - /** - * Serializes a session (a conversation with its messages) as JSONL. - * The first line is the session header (a `SessionRecordType.SESSION` record - * carrying the conversation properties); each subsequent line is a single message. - * @param data - The exported conversation payload - * @returns The JSONL string (one record per line) - */ - static serializeSessionToJsonl(data: ExportedConversation): string { - const { conv, messages } = data; - const sessionLine = JSON.stringify({ - harness: EXPORT_CONV.HARNESS, - type: SessionRecordType.SESSION, - ...conv - }); - const messageLines = messages.map((message: DatabaseMessage) => { - // `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it. - const { toolCalls, ...rest } = message; - const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest; - - return JSON.stringify({ message: normalized, type: SessionRecordType.MESSAGE }); - }); - - return [sessionLine, ...messageLines].join(NEWLINE); - } - - /** - * Parses the JSONL session format produced by {@link serializeSessionToJsonl}. - * A `SessionRecordType.SESSION` line starts a new session; following - * `SessionRecordType.MESSAGE` lines are appended to it. Supports multiple - * sessions in a single file. - * @param text - The JSONL file contents - * @returns The parsed conversations with their messages - */ - static parseSessionsJsonl(text: string): ExportedConversation[] { - const sessions: ExportedConversation[] = []; - - let current: ExportedConversation | null = null; - - for (const line of text.split(NEWLINE)) { - const trimmed = line.trim(); - - if (!trimmed) continue; - - const record = JSON.parse(trimmed); - - if (record.type === SessionRecordType.SESSION) { - // Drop the discriminator and harness marker; the rest is the conversation. - const conv = { ...record }; - - delete conv.type; - delete conv.harness; - current = { conv: conv as DatabaseConversation, messages: [] }; - sessions.push(current); - } else if (record.type === SessionRecordType.MESSAGE) { - if (!current) { - throw new Error('Invalid JSONL: message record before any session record'); - } - - const message = record.message as DatabaseMessage; - - // `toolCalls` is parsed to an array on export; the DB stores it as a string. - if (message.toolCalls !== undefined && typeof message.toolCalls !== 'string') { - message.toolCalls = JSON.stringify(message.toolCalls); - } - - current.messages.push(message); - } - // Ignore unknown record types for forward compatibility. - } - - return sessions; - } - - /** - * Reports whether the text is the JSONL session format, whose first non-empty - * line is a `SessionRecordType.SESSION` record. A legacy JSON export starts - * with an array or an object that has no such discriminator. - * @param text - The file contents - */ - private static isSessionsJsonl(text: string): boolean { - const trimmed = text.trimStart(); - const lineEnd = trimmed.indexOf(NEWLINE); - const firstLine = lineEnd === -1 ? trimmed : trimmed.slice(0, lineEnd); - - try { - return JSON.parse(firstLine).type === SessionRecordType.SESSION; - } catch { - // Not a standalone JSON record, so not the JSONL format. - return false; - } - } - - /** - * Parses an import file into conversations, accepting the current JSONL and - * ZIP formats as well as the legacy JSON format. The format comes from the - * contents, so an import works whatever the file is named. - * @param file - The user-selected file - * @returns The parsed conversations with their messages - */ - static async parseImportFile(file: File): Promise { - const bytes = new Uint8Array(await file.arrayBuffer()); - - if (ZIP_MAGIC.every((byte, index) => bytes[index] === byte)) { - const entries = unzipSync(bytes); - const sessions: ExportedConversation[] = []; - - for (const [entryName, entryBytes] of Object.entries(entries)) { - if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue; - - sessions.push(...ConversationTransferService.parseSessionsJsonl(strFromU8(entryBytes))); - } - - return sessions; - } - - const text = strFromU8(bytes); - - if (ConversationTransferService.isSessionsJsonl(text)) { - return ConversationTransferService.parseSessionsJsonl(text); - } - - // Legacy JSON format: an array of conversations or a single conversation object. - const parsed = JSON.parse(text); - - if (Array.isArray(parsed)) { - return parsed; - } - - if (parsed && typeof parsed === 'object' && 'conv' in parsed && 'messages' in parsed) { - return [parsed]; - } - - throw new Error( - 'Invalid file format: expected array of conversations or single conversation object' - ); - } - - /** - * - * - * Downloads - * - * - */ - - /** - * Generates a sanitized filename for a conversation export - * @param conversation - The conversation metadata - * @param msgs - Optional array of messages belonging to the conversation - * @returns The generated filename string - */ - static generateConversationFilename( - conversation: { id?: string; name?: string }, - msgs?: DatabaseMessage[] - ): string { - const conversationName = (conversation.name ?? '').trim().toLowerCase(); - const sanitizedName = conversationName - .replace(EXPORT_CONV.NON_ALPHANUMERIC_REGEX, EXPORT_CONV.NONALNUM_REPLACEMENT) - .replace(EXPORT_CONV.MULTIPLE_UNDERSCORE_REGEX, '_') - .substring(0, EXPORT_CONV.NAME_SUFFIX_MAX_LENGTH); - // If we have messages, use the timestamp of the newest message - const referenceDate = msgs?.length - ? new Date(Math.max(...msgs.map((m) => m.timestamp))) - : new Date(); - const iso = referenceDate.toISOString().slice(0, EXPORT_CONV.ISO_TIMESTAMP_SLICE); - const formattedDate = iso - .replace(EXPORT_CONV.ISO_DATE_TIME_SEPARATOR, EXPORT_CONV.ISO_DATE_TIME_SEPARATOR_REPLACEMENT) - .replaceAll(EXPORT_CONV.ISO_TIME_SEPARATOR, EXPORT_CONV.ISO_TIME_SEPARATOR_REPLACEMENT); - const trimmedConvId = conversation.id?.slice(0, EXPORT_CONV.ID_TRIM_LENGTH) ?? ''; - - return `${formattedDate}_conv_${trimmedConvId}_${sanitizedName}${FileExtensionText.JSONL}`; - } - /** * Triggers a browser download of the provided exported conversation data * @param data - The exported conversation payload (a single conversation with its messages) @@ -262,6 +81,171 @@ export class ConversationTransferService { ConversationTransferService.triggerDownload(blob, archiveName); } + /** + * Generates a sanitized filename for a conversation export + * @param conversation - The conversation metadata + * @param msgs - Optional array of messages belonging to the conversation + * @returns The generated filename string + */ + static generateConversationFilename( + conversation: { id?: string; name?: string }, + msgs?: DatabaseMessage[] + ): string { + const conversationName = (conversation.name ?? '').trim().toLowerCase(); + const sanitizedName = conversationName + .replace(EXPORT_CONV.NON_ALPHANUMERIC_REGEX, EXPORT_CONV.NONALNUM_REPLACEMENT) + .replace(EXPORT_CONV.MULTIPLE_UNDERSCORE_REGEX, '_') + .substring(0, EXPORT_CONV.NAME_SUFFIX_MAX_LENGTH); + // If we have messages, use the timestamp of the newest message + const referenceDate = msgs?.length + ? new Date(Math.max(...msgs.map((m) => m.timestamp))) + : new Date(); + const iso = referenceDate.toISOString().slice(0, EXPORT_CONV.ISO_TIMESTAMP_SLICE); + const formattedDate = iso + .replace(EXPORT_CONV.ISO_DATE_TIME_SEPARATOR, EXPORT_CONV.ISO_DATE_TIME_SEPARATOR_REPLACEMENT) + .replaceAll(EXPORT_CONV.ISO_TIME_SEPARATOR, EXPORT_CONV.ISO_TIME_SEPARATOR_REPLACEMENT); + const trimmedConvId = conversation.id?.slice(0, EXPORT_CONV.ID_TRIM_LENGTH) ?? ''; + + return `${formattedDate}_conv_${trimmedConvId}_${sanitizedName}${FileExtensionText.JSONL}`; + } + + /** + * Parses an import file into conversations, accepting the current JSONL and + * ZIP formats as well as the legacy JSON format. The format comes from the + * contents, so an import works whatever the file is named. + * @param file - The user-selected file + * @returns The parsed conversations with their messages + */ + static async parseImportFile(file: File): Promise { + const bytes = new Uint8Array(await file.arrayBuffer()); + + if (ZIP_MAGIC.every((byte, index) => bytes[index] === byte)) { + const entries = unzipSync(bytes); + const sessions: ExportedConversation[] = []; + + for (const [entryName, entryBytes] of Object.entries(entries)) { + if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue; + + sessions.push(...ConversationTransferService.parseSessionsJsonl(strFromU8(entryBytes))); + } + + return sessions; + } + + const text = strFromU8(bytes); + + if (ConversationTransferService.isSessionsJsonl(text)) { + return ConversationTransferService.parseSessionsJsonl(text); + } + + // Legacy JSON format: an array of conversations or a single conversation object. + const parsed = JSON.parse(text); + + if (Array.isArray(parsed)) { + return parsed; + } + + if (parsed && typeof parsed === 'object' && 'conv' in parsed && 'messages' in parsed) { + return [parsed]; + } + + throw new Error( + 'Invalid file format: expected array of conversations or single conversation object' + ); + } + + /** + * Parses the JSONL session format produced by {@link serializeSessionToJsonl}. + * A `SessionRecordType.SESSION` line starts a new session; following + * `SessionRecordType.MESSAGE` lines are appended to it. Supports multiple + * sessions in a single file. + * @param text - The JSONL file contents + * @returns The parsed conversations with their messages + */ + static parseSessionsJsonl(text: string): ExportedConversation[] { + const sessions: ExportedConversation[] = []; + + let current: ExportedConversation | null = null; + + for (const line of text.split(NEWLINE)) { + const trimmed = line.trim(); + + if (!trimmed) continue; + + const record = JSON.parse(trimmed); + + if (record.type === SessionRecordType.SESSION) { + // Drop the discriminator and harness marker; the rest is the conversation. + const conv = { ...record }; + + delete conv.type; + delete conv.harness; + current = { conv: conv as DatabaseConversation, messages: [] }; + sessions.push(current); + } else if (record.type === SessionRecordType.MESSAGE) { + if (!current) { + throw new Error('Invalid JSONL: message record before any session record'); + } + + const message = record.message as DatabaseMessage; + + // `toolCalls` is parsed to an array on export; the DB stores it as a string. + if (message.toolCalls !== undefined && typeof message.toolCalls !== 'string') { + message.toolCalls = JSON.stringify(message.toolCalls); + } + + current.messages.push(message); + } + // Ignore unknown record types for forward compatibility. + } + + return sessions; + } + + /** + * Serializes a session (a conversation with its messages) as JSONL. + * The first line is the session header (a `SessionRecordType.SESSION` record + * carrying the conversation properties); each subsequent line is a single message. + * @param data - The exported conversation payload + * @returns The JSONL string (one record per line) + */ + static serializeSessionToJsonl(data: ExportedConversation): string { + const { conv, messages } = data; + const sessionLine = JSON.stringify({ + harness: EXPORT_CONV.HARNESS, + type: SessionRecordType.SESSION, + ...conv + }); + const messageLines = messages.map((message: DatabaseMessage) => { + // `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it. + const { toolCalls, ...rest } = message; + const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest; + + return JSON.stringify({ message: normalized, type: SessionRecordType.MESSAGE }); + }); + + return [sessionLine, ...messageLines].join(NEWLINE); + } + + /** + * Reports whether the text is the JSONL session format, whose first non-empty + * line is a `SessionRecordType.SESSION` record. A legacy JSON export starts + * with an array or an object that has no such discriminator. + * @param text - The file contents + */ + private static isSessionsJsonl(text: string): boolean { + const trimmed = text.trimStart(); + const lineEnd = trimmed.indexOf(NEWLINE); + const firstLine = lineEnd === -1 ? trimmed : trimmed.slice(0, lineEnd); + + try { + return JSON.parse(firstLine).type === SessionRecordType.SESSION; + } catch { + // Not a standalone JSON record, so not the JSONL format. + return false; + } + } + /** * Triggers a browser download of a blob under the given filename. */ diff --git a/tools/ui/src/lib/services/database.service.ts b/tools/ui/src/lib/services/database.service.ts index 89dc58b005..a466f84831 100644 --- a/tools/ui/src/lib/services/database.service.ts +++ b/tools/ui/src/lib/services/database.service.ts @@ -1,3 +1,11 @@ +/** + * DatabaseService - IndexedDB persistence for conversations and messages + * + * Thin Dexie layer over the conversations/messages tables: CRUD, tree + * navigation (descendants, reparenting) and cascading deletes. No reactive + * state; consumed by conversationsStore and the chat flows. + */ + import { IDXDB_STORES, IDXDB_TABLES, STORAGE_APP_NAME } from '$lib/constants'; import { MessageRole } from '$lib/enums'; import type { McpServerOverride } from '$lib/types/database'; @@ -20,12 +28,99 @@ const db = new LlamaUiDatabase(); export class DatabaseService { /** + * Deletes multiple conversations in a single transaction. Each deleted + * conversation has its direct children reparented to the nearest surviving + * ancestor (or promoted to top-level). Children also in `ids` are dropped + * entirely rather than reparented. * - * - * Conversations - * - * + * @param ids - Conversation IDs to delete */ + static async bulkDeleteConversations(ids: string[]): Promise { + const cleanIds = ids.filter((id): id is string => typeof id === 'string' && id.length > 0); + + if (cleanIds.length === 0) return; + + const idSet = new Set(cleanIds); + + await db.transaction( + 'rw', + [db[IDXDB_TABLES.conversations], db[IDXDB_TABLES.messages]], + async () => { + // Pre-load each to-delete conversation so the per-id reparent + // walk-up doesn't ping-pong the same ancestry chain. + const prefetched = new Map(); + + let frontier = [...cleanIds]; + + const requested = new Set(frontier); + + while (frontier.length > 0) { + const fetched = await db[IDXDB_TABLES.conversations].bulkGet(frontier); + + frontier = []; + for (let i = 0; i < fetched.length; i++) { + const conv = fetched[i]; + + if (!conv || !conv.id) continue; + + prefetched.set(conv.id, conv); + const ancestor = conv.forkedFromConversationId; + + if (ancestor && !prefetched.has(ancestor) && !requested.has(ancestor)) { + frontier.push(ancestor); + requested.add(ancestor); + } + } + } + + for (const id of cleanIds) { + await this.reparentDirectChildren(id, idSet, prefetched); + } + + await db[IDXDB_TABLES.conversations].bulkDelete(cleanIds); + await db[IDXDB_TABLES.messages].where('convId').anyOf(cleanIds).delete(); + } + ); + } + + /** + * Toggles the pinned status of each conversation in `ids` inside a single + * transaction. Treats `pinned === undefined` as `false`, matching the + * semantics of {@link toggleConversationPin} where `!undefined` evaluates + * to `true`. Returns the resulting pinned state for every id that was + * updated; missing ids are omitted from the map. + * + * @param ids - Conversation IDs to toggle + * @returns Map of id -> new pinned state + */ + static async bulkToggleConversationPins(ids: string[]): Promise> { + const cleanIds = ids.filter((id): id is string => typeof id === 'string' && id.length > 0); + const result = new Map(); + + if (cleanIds.length === 0) return result; + + await db.transaction('rw', db[IDXDB_TABLES.conversations], async () => { + const convs = await db[IDXDB_TABLES.conversations].bulkGet(cleanIds); + const updates: DatabaseConversation[] = []; + + for (let i = 0; i < cleanIds.length; i++) { + const conv = convs[i]; + + if (!conv) continue; + + const newPinned = !conv.pinned; + + updates.push({ ...conv, pinned: newPinned }); + result.set(cleanIds[i], newPinned); + } + + if (updates.length === 0) return; + + await db[IDXDB_TABLES.conversations].bulkPut(updates); + }); + + return result; + } /** * Creates a new conversation. @@ -51,14 +146,6 @@ export class DatabaseService { return conversation; } - /** - * - * - * Messages - * - * - */ - /** * Creates a new message branch by adding a message and updating parent/child relationships. * Also updates the conversation's currNode to point to the new message. @@ -96,13 +183,7 @@ export class DatabaseService { // Update parent's children array if parent exists if (parentId !== null) { - const parentMessage = await db[IDXDB_TABLES.messages].get(parentId); - - if (parentMessage) { - await db[IDXDB_TABLES.messages].update(parentId, { - children: [...parentMessage.children, newMessage.id] - }); - } + await this.addChildToParent(parentId, newMessage.id); } await this.updateConversation(message.convId, { @@ -178,9 +259,7 @@ export class DatabaseService { }; await db[IDXDB_TABLES.messages].add(systemMessage); - await db[IDXDB_TABLES.messages].update(parentId, { - children: [...parentMessage.children, systemMessage.id] - }); + await this.addChildToParent(parentId, systemMessage.id); return systemMessage; }); @@ -230,121 +309,6 @@ export class DatabaseService { ); } - /** - * Reparents direct children of `parentId` to the nearest surviving - * ancestor (or promotes them to top-level when the immediate parent was - * top-level). Walking skips any ancestor listed in `excludeIds`, since - * those will be deleted in the same batch — leaving a grandchild pointing - * at an `excludeIds` entry would orphan it. Children whose own id is in - * `excludeIds` are dropped from the updates (the bulk-delete pass will - * remove them). `prefetched` may carry a pre-fetched ancestor map to - * avoid repeat reads inside a bulk transaction. - */ - private static async reparentDirectChildren( - parentId: string, - excludeIds: ReadonlySet = new Set(), - prefetched?: ReadonlyMap - ): Promise { - const conv = prefetched?.get(parentId) ?? (await db[IDXDB_TABLES.conversations].get(parentId)); - - if (!conv) return; - - let newParent = conv.forkedFromConversationId; - - const visited = new Set([parentId]); - - while (newParent && excludeIds.has(newParent)) { - if (visited.has(newParent)) { - newParent = undefined; - - break; - } - - visited.add(newParent); - const next = - prefetched?.get(newParent) ?? (await db[IDXDB_TABLES.conversations].get(newParent)); - - if (!next) { - newParent = undefined; - - break; - } - - newParent = next.forkedFromConversationId; - } - - const directChildren = await db[IDXDB_TABLES.conversations] - .filter((c) => c.forkedFromConversationId === parentId) - .toArray(); - const updates: DatabaseConversation[] = []; - - for (const child of directChildren) { - if (excludeIds.has(child.id)) continue; - - updates.push({ ...child, forkedFromConversationId: newParent }); - } - - if (updates.length === 0) return; - - await db[IDXDB_TABLES.conversations].bulkPut(updates); - } - - /** - * Deletes multiple conversations in a single transaction. Each deleted - * conversation has its direct children reparented to the nearest surviving - * ancestor (or promoted to top-level). Children also in `ids` are dropped - * entirely rather than reparented. - * - * @param ids - Conversation IDs to delete - */ - static async bulkDeleteConversations(ids: string[]): Promise { - const cleanIds = ids.filter((id): id is string => typeof id === 'string' && id.length > 0); - - if (cleanIds.length === 0) return; - - const idSet = new Set(cleanIds); - - await db.transaction( - 'rw', - [db[IDXDB_TABLES.conversations], db[IDXDB_TABLES.messages]], - async () => { - // Pre-load each to-delete conversation so the per-id reparent - // walk-up doesn't ping-pong the same ancestry chain. - const prefetched = new Map(); - - let frontier = [...cleanIds]; - - const requested = new Set(frontier); - - while (frontier.length > 0) { - const fetched = await db[IDXDB_TABLES.conversations].bulkGet(frontier); - - frontier = []; - for (let i = 0; i < fetched.length; i++) { - const conv = fetched[i]; - - if (!conv || !conv.id) continue; - - prefetched.set(conv.id, conv); - const ancestor = conv.forkedFromConversationId; - - if (ancestor && !prefetched.has(ancestor) && !requested.has(ancestor)) { - frontier.push(ancestor); - requested.add(ancestor); - } - } - } - - for (const id of cleanIds) { - await this.reparentDirectChildren(id, idSet, prefetched); - } - - await db[IDXDB_TABLES.conversations].bulkDelete(cleanIds); - await db[IDXDB_TABLES.messages].where('convId').anyOf(cleanIds).delete(); - } - ); - } - /** * Deletes a message and removes it from its parent's children array. * @@ -356,17 +320,8 @@ export class DatabaseService { if (!message) return; - // Remove this message from its parent's children array - if (message.parent) { - const parent = await db[IDXDB_TABLES.messages].get(message.parent); + await this.removeChildFromParent(messageId); - if (parent) { - parent.children = parent.children.filter((childId: string) => childId !== messageId); - await db[IDXDB_TABLES.messages].put(parent); - } - } - - // Delete the message await db[IDXDB_TABLES.messages].delete(messageId); }); } @@ -389,20 +344,10 @@ export class DatabaseService { .where('convId') .equals(conversationId) .toArray(); - // Find all descendant messages const descendants = findDescendantMessages(allMessages, messageId); const allToDelete = [messageId, ...descendants]; - // Get the message to delete for parent cleanup - const message = await db[IDXDB_TABLES.messages].get(messageId); - if (message && message.parent) { - const parent = await db[IDXDB_TABLES.messages].get(message.parent); - - if (parent) { - parent.children = parent.children.filter((childId: string) => childId !== messageId); - await db[IDXDB_TABLES.messages].put(parent); - } - } + await this.removeChildFromParent(messageId); // Delete all messages in the branch await db[IDXDB_TABLES.messages].bulkDelete(allToDelete); @@ -411,243 +356,6 @@ export class DatabaseService { }); } - /** - * Gets all conversations, sorted by last modified time (newest first). - * - * @returns Array of conversations - */ - static async getAllConversations(): Promise { - return await db[IDXDB_TABLES.conversations].orderBy('lastModified').reverse().toArray(); - } - - /** - * Gets a conversation by ID. - * - * @param id - Conversation ID - * @returns The conversation if found, otherwise undefined - */ - static async getConversation(id: string): Promise { - return await db[IDXDB_TABLES.conversations].get(id); - } - - /** - * Gets all messages in a conversation, sorted by timestamp (oldest first). - * - * @param convId - Conversation ID - * @returns Array of messages in the conversation - */ - static async getConversationMessages(convId: string): Promise { - return await db[IDXDB_TABLES.messages].where('convId').equals(convId).sortBy('timestamp'); - } - - /** - * Loads multiple conversations with all of their messages in two bulk - * reads. Missing conversations are silently omitted from the result. - * - * @param convIds - Conversation IDs to load - * @returns Map of id -> { conv, messages }. Messages are sorted ascending by timestamp. - */ - static async getConversationsWithMessages( - convIds: string[] - ): Promise> { - const result = new Map(); - const cleanIds = convIds.filter((id): id is string => typeof id === 'string' && id.length > 0); - - if (cleanIds.length === 0) return result; - - const [convs, allMessages] = await Promise.all([ - db[IDXDB_TABLES.conversations].bulkGet(cleanIds), - db[IDXDB_TABLES.messages].where('convId').anyOf(cleanIds).toArray() - ]); - const messagesByConv = new Map(); - - for (const msg of allMessages) { - const bucket = messagesByConv.get(msg.convId); - - if (bucket) bucket.push(msg); - else messagesByConv.set(msg.convId, [msg]); - } - - for (let i = 0; i < cleanIds.length; i++) { - const conv = convs[i]; - - if (!conv) continue; - - const messages = (messagesByConv.get(conv.id) ?? []).sort( - (a, b) => a.timestamp - b.timestamp - ); - - result.set(conv.id, { conv, messages }); - } - - return result; - } - - /** - * Updates a conversation. `lastModified` is never stamped implicitly; - * pass it in `updates` to bump the conversation in recency ordering. - * - * @param id - Conversation ID - * @param updates - Partial updates to apply - * @returns Promise that resolves when the conversation is updated - */ - static async updateConversation( - id: string, - updates: Partial> - ): Promise { - await db[IDXDB_TABLES.conversations].update(id, updates); - } - - /** - * - * - * Navigation - * - * - */ - - /** - * Toggles the pinned status of a conversation. - * - * @param id - Conversation ID - * @returns The new pinned status - */ - static async toggleConversationPin(id: string): Promise { - const conversation = await db[IDXDB_TABLES.conversations].get(id); - - if (!conversation) { - throw new Error(`Conversation ${id} not found`); - } - - const newPinnedState = !conversation.pinned; - - await this.updateConversation(id, { pinned: newPinnedState }); - - return newPinnedState; - } - - /** - * Toggles the pinned status of each conversation in `ids` inside a single - * transaction. Treats `pinned === undefined` as `false`, matching the - * semantics of {@link toggleConversationPin} where `!undefined` evaluates - * to `true`. Returns the resulting pinned state for every id that was - * updated; missing ids are omitted from the map. - * - * @param ids - Conversation IDs to toggle - * @returns Map of id -> new pinned state - */ - static async bulkToggleConversationPins(ids: string[]): Promise> { - const cleanIds = ids.filter((id): id is string => typeof id === 'string' && id.length > 0); - const result = new Map(); - - if (cleanIds.length === 0) return result; - - await db.transaction('rw', db[IDXDB_TABLES.conversations], async () => { - const convs = await db[IDXDB_TABLES.conversations].bulkGet(cleanIds); - const updates: DatabaseConversation[] = []; - - for (let i = 0; i < cleanIds.length; i++) { - const conv = convs[i]; - - if (!conv) continue; - - const newPinned = !conv.pinned; - - updates.push({ ...conv, pinned: newPinned }); - result.set(cleanIds[i], newPinned); - } - - if (updates.length === 0) return; - - await db[IDXDB_TABLES.conversations].bulkPut(updates); - }); - - return result; - } - - /** - * Updates the conversation's current node (active branch). - * This determines which conversation path is currently being viewed. - * - * @param convId - Conversation ID - * @param nodeId - Message ID to set as current node - */ - static async updateCurrentNode(convId: string, nodeId: string): Promise { - await this.updateConversation(convId, { - currNode: nodeId - }); - } - - /** - * Updates a message. - * - * @param id - Message ID - * @param updates - Partial updates to apply - * @returns Promise that resolves when the message is updated - */ - static async updateMessage( - id: string, - updates: Partial> - ): Promise { - await db[IDXDB_TABLES.messages].update(id, updates); - } - - /** - * - * - * Import - * - * - */ - - /** - * Imports multiple conversations and their messages. - * Skips conversations that already exist. - * - * @param data - Array of { conv, messages } objects - * @returns The conversations written to the database and the ones skipped - */ - static async importConversations( - data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[] - ): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> { - const imported: DatabaseConversation[] = []; - const skipped: DatabaseConversation[] = []; - - return await db.transaction( - 'rw', - [db[IDXDB_TABLES.conversations], db[IDXDB_TABLES.messages]], - async () => { - for (const item of data) { - const { conv, messages } = item; - const existing = await db[IDXDB_TABLES.conversations].get(conv.id); - - if (existing) { - skipped.push(conv); - - continue; - } - - await db[IDXDB_TABLES.conversations].add(conv); - for (const msg of messages) { - await db[IDXDB_TABLES.messages].put(msg); - } - - imported.push(conv); - } - - return { imported, skipped }; - } - ); - } - - /** - * - * - * Forking - * - * - */ - /** * Forks a conversation at a specific message, creating a new conversation * containing all messages from the root up to (and including) the target message. @@ -726,13 +434,272 @@ export class DatabaseService { }; await db[IDXDB_TABLES.conversations].add(newConv); - - for (const msg of clonedMessages) { - await db[IDXDB_TABLES.messages].add(msg); - } + await db[IDXDB_TABLES.messages].bulkAdd(clonedMessages); return newConv; } ); } + + /** + * Gets all conversations, sorted by last modified time (newest first). + * + * @returns Array of conversations + */ + static async getAllConversations(): Promise { + return await db[IDXDB_TABLES.conversations].orderBy('lastModified').reverse().toArray(); + } + + /** + * Gets a conversation by ID. + * + * @param id - Conversation ID + * @returns The conversation if found, otherwise undefined + */ + static async getConversation(id: string): Promise { + return await db[IDXDB_TABLES.conversations].get(id); + } + + /** + * Gets all messages in a conversation, sorted by timestamp (oldest first). + * + * @param convId - Conversation ID + * @returns Array of messages in the conversation + */ + static async getConversationMessages(convId: string): Promise { + return await db[IDXDB_TABLES.messages].where('convId').equals(convId).sortBy('timestamp'); + } + + /** + * Loads multiple conversations with all of their messages in two bulk + * reads. Missing conversations are silently omitted from the result. + * + * @param convIds - Conversation IDs to load + * @returns Map of id -> { conv, messages }. Messages are sorted ascending by timestamp. + */ + static async getConversationsWithMessages( + convIds: string[] + ): Promise> { + const result = new Map(); + const cleanIds = convIds.filter((id): id is string => typeof id === 'string' && id.length > 0); + + if (cleanIds.length === 0) return result; + + const [convs, allMessages] = await Promise.all([ + db[IDXDB_TABLES.conversations].bulkGet(cleanIds), + db[IDXDB_TABLES.messages].where('convId').anyOf(cleanIds).toArray() + ]); + const messagesByConv = new Map(); + + for (const msg of allMessages) { + const bucket = messagesByConv.get(msg.convId); + + if (bucket) bucket.push(msg); + else messagesByConv.set(msg.convId, [msg]); + } + + for (let i = 0; i < cleanIds.length; i++) { + const conv = convs[i]; + + if (!conv) continue; + + const messages = (messagesByConv.get(conv.id) ?? []).sort( + (a, b) => a.timestamp - b.timestamp + ); + + result.set(conv.id, { conv, messages }); + } + + return result; + } + + /** + * Imports multiple conversations and their messages. + * Skips conversations that already exist. + * + * @param data - Array of { conv, messages } objects + * @returns The conversations written to the database and the ones skipped + */ + static async importConversations( + data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[] + ): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> { + const imported: DatabaseConversation[] = []; + const skipped: DatabaseConversation[] = []; + + return await db.transaction( + 'rw', + [db[IDXDB_TABLES.conversations], db[IDXDB_TABLES.messages]], + async () => { + for (const item of data) { + const { conv, messages } = item; + const existing = await db[IDXDB_TABLES.conversations].get(conv.id); + + if (existing) { + skipped.push(conv); + + continue; + } + + await db[IDXDB_TABLES.conversations].add(conv); + for (const msg of messages) { + await db[IDXDB_TABLES.messages].put(msg); + } + + imported.push(conv); + } + + return { imported, skipped }; + } + ); + } + + /** + * Toggles the pinned status of a conversation. + * + * @param id - Conversation ID + * @returns The new pinned status + */ + static async toggleConversationPin(id: string): Promise { + const conversation = await db[IDXDB_TABLES.conversations].get(id); + + if (!conversation) { + throw new Error(`Conversation ${id} not found`); + } + + const newPinnedState = !conversation.pinned; + + await this.updateConversation(id, { pinned: newPinnedState }); + + return newPinnedState; + } + + /** + * Updates a conversation. `lastModified` is never stamped implicitly; + * pass it in `updates` to bump the conversation in recency ordering. + * + * @param id - Conversation ID + * @param updates - Partial updates to apply + * @returns Promise that resolves when the conversation is updated + */ + static async updateConversation( + id: string, + updates: Partial> + ): Promise { + await db[IDXDB_TABLES.conversations].update(id, updates); + } + + /** + * Updates the conversation's current node (active branch). + * This determines which conversation path is currently being viewed. + * + * @param convId - Conversation ID + * @param nodeId - Message ID to set as current node + */ + static async updateCurrentNode(convId: string, nodeId: string): Promise { + await this.updateConversation(convId, { + currNode: nodeId + }); + } + + /** + * Updates a message. + * + * @param id - Message ID + * @param updates - Partial updates to apply + * @returns Promise that resolves when the message is updated + */ + static async updateMessage( + id: string, + updates: Partial> + ): Promise { + await db[IDXDB_TABLES.messages].update(id, updates); + } + + /** + * Appends a child id to a parent message's children array. + */ + private static async addChildToParent(parentId: string, childId: string): Promise { + const parent = await db[IDXDB_TABLES.messages].get(parentId); + + if (!parent) return; + + await db[IDXDB_TABLES.messages].update(parentId, { + children: [...parent.children, childId] + }); + } + + /** + * Removes a child id from its parent message's children array. + */ + private static async removeChildFromParent(messageId: string): Promise { + const message = await db[IDXDB_TABLES.messages].get(messageId); + + if (!message?.parent) return; + + const parent = await db[IDXDB_TABLES.messages].get(message.parent); + + if (!parent) return; + + parent.children = parent.children.filter((childId: string) => childId !== messageId); + await db[IDXDB_TABLES.messages].put(parent); + } + + /** + * Reparents direct children of `parentId` to the nearest surviving + * ancestor (or promotes them to top-level when the immediate parent was + * top-level). Walking skips any ancestor listed in `excludeIds`, since + * those will be deleted in the same batch — leaving a grandchild pointing + * at an `excludeIds` entry would orphan it. Children whose own id is in + * `excludeIds` are dropped from the updates (the bulk-delete pass will + * remove them). `prefetched` may carry a pre-fetched ancestor map to + * avoid repeat reads inside a bulk transaction. + */ + private static async reparentDirectChildren( + parentId: string, + excludeIds: ReadonlySet = new Set(), + prefetched?: ReadonlyMap + ): Promise { + const conv = prefetched?.get(parentId) ?? (await db[IDXDB_TABLES.conversations].get(parentId)); + + if (!conv) return; + + let newParent = conv.forkedFromConversationId; + + const visited = new Set([parentId]); + + while (newParent && excludeIds.has(newParent)) { + if (visited.has(newParent)) { + newParent = undefined; + + break; + } + + visited.add(newParent); + const next = + prefetched?.get(newParent) ?? (await db[IDXDB_TABLES.conversations].get(newParent)); + + if (!next) { + newParent = undefined; + + break; + } + + newParent = next.forkedFromConversationId; + } + + const directChildren = await db[IDXDB_TABLES.conversations] + .filter((c) => c.forkedFromConversationId === parentId) + .toArray(); + const updates: DatabaseConversation[] = []; + + for (const child of directChildren) { + if (excludeIds.has(child.id)) continue; + + updates.push({ ...child, forkedFromConversationId: newParent }); + } + + if (updates.length === 0) return; + + await db[IDXDB_TABLES.conversations].bulkPut(updates); + } } diff --git a/tools/ui/src/lib/services/index.ts b/tools/ui/src/lib/services/index.ts index fe739a3bc2..7ae9e23d48 100644 --- a/tools/ui/src/lib/services/index.ts +++ b/tools/ui/src/lib/services/index.ts @@ -53,9 +53,9 @@ * - Reasoning content stripping from prompt history to avoid KV cache pollution * - Error translation (network, timeout, server errors → user-friendly messages) * - * @see chatStore in stores/chat.svelte.ts — primary consumer for chat state management - * @see agenticStore in stores/agentic.svelte.ts — uses ChatService for agentic loop streaming - * @see conversationsStore in stores/conversations.svelte.ts — provides message context + * @see chatStore in stores/chat/index.svelte.ts — primary consumer for chat state management + * @see agenticStore in stores/agentic/index.svelte.ts — uses ChatService for agentic loop streaming + * @see conversationsStore in stores/conversations/index.svelte.ts — provides message context */ export { ChatService } from './chat.service'; @@ -98,8 +98,8 @@ export { ChatService } from './chat.service'; * enabling conversation branching and alternative response paths. The conversation's * `currNode` tracks the currently active branch endpoint. * - * @see conversationsStore in stores/conversations.svelte.ts — reactive layer on top of DatabaseService - * @see chatStore in stores/chat.svelte.ts — uses DatabaseService directly for message CRUD during streaming + * @see conversationsStore in stores/conversations/index.svelte.ts — reactive layer on top of DatabaseService + * @see chatStore in stores/chat/index.svelte.ts — uses DatabaseService directly for message CRUD during streaming */ export { DatabaseService } from './database.service'; @@ -143,7 +143,7 @@ export { ConversationTransferService } from './conversation-transfer.service'; * - `POST /models/load` — Load a model (ROUTER mode only) * - `POST /models/unload` — Unload a model (ROUTER mode only) * - * @see modelsStore in stores/models.svelte.ts — primary consumer for reactive model state + * @see modelsStore in stores/models/index.svelte.ts — primary consumer for reactive model state */ export { ModelsService } from './models.service'; @@ -174,8 +174,8 @@ export { ModelsService } from './models.service'; * - `&autoload=false` → Prevents model auto-loading when querying props * * @see serverStore in stores/server.svelte.ts — consumes global server props - * @see modelsStore in stores/models.svelte.ts — consumes per-model props for modalities - * @see settingsStore in stores/settings.svelte.ts — syncs default generation params from props + * @see modelsStore in stores/models/index.svelte.ts — consumes per-model props for modalities + * @see settingsStore in stores/settings/index.svelte.ts — syncs default generation params from props */ export { PropsService } from './props.service'; @@ -217,7 +217,7 @@ export { PropsService } from './props.service'; * - `ParameterSyncService` class — static methods for sync logic * - `SYNCABLE_PARAMETERS` — mapping of UI setting keys to server parameter keys * - * @see settingsStore in stores/settings.svelte.ts — primary consumer for settings sync + * @see settingsStore in stores/settings/index.svelte.ts — primary consumer for settings sync * @see SettingsChatParameterSourceIndicator — displays parameter source badges in UI */ export { ParameterSyncService } from './parameter-sync.service'; @@ -241,7 +241,7 @@ export { ParameterSyncService } from './parameter-sync.service'; * - Manages connection lifecycle, health checks, reconnection * - Handles tool name conflict resolution and server coordination * - * - **mcpResourceStore**: Reactive resource state + * - **mcpResourceStore** (composed as mcpStore.resources): Reactive resource state * - Receives resource data fetched via MCPService * - Manages resource caching, subscriptions, and attachments * @@ -263,9 +263,9 @@ export { ParameterSyncService } from './parameter-sync.service'; * 2. **StreamableHTTP** — modern HTTP-based, supports CORS proxy * 3. **SSE** — legacy fallback, supports CORS proxy * - * @see mcpStore in stores/mcp.svelte.ts — reactive business logic facade on top of MCPService - * @see mcpResourceStore in stores/mcp-resources.svelte.ts — reactive resource state management - * @see agenticStore in stores/agentic.svelte.ts — uses MCPService (via mcpStore) for tool execution + * @see mcpStore in stores/mcp/index.svelte.ts — reactive business logic facade on top of MCPService + * @see mcpStore.resources in stores/mcp/resources.svelte.ts — reactive resource state management + * @see agenticStore in stores/agentic/index.svelte.ts — uses MCPService (via mcpStore) for tool execution * @see MCP Protocol Specification: https://modelcontextprotocol.io/specification/2025-06-18 */ export { MCPService } from './mcp.service'; @@ -286,7 +286,7 @@ export { MCPService } from './mcp.service'; * - **agenticStore**: Dispatches ToolSource.BROWSER calls here * * @see buildSandboxToolDefinition in utils/sandbox-tool - tool schema sent to the LLM - * @see agenticStore in stores/agentic.svelte.ts - tool dispatch + * @see agenticStore in stores/agentic/index.svelte.ts - tool dispatch */ export { SandboxService } from './sandbox.service'; diff --git a/tools/ui/src/lib/services/mcp.service.ts b/tools/ui/src/lib/services/mcp.service.ts index 65e9e59d6e..7b857fd438 100644 --- a/tools/ui/src/lib/services/mcp.service.ts +++ b/tools/ui/src/lib/services/mcp.service.ts @@ -1,3 +1,11 @@ +/** + * MCPService - Stateless MCP protocol layer + * + * Implements the client side of the MCP spec over WebSocket, StreamableHTTP + * and SSE transports: connect, tool/prompt/resource operations and result + * formatting. No reactive state; consumed by mcpStore and its managers. + */ + import { Client } from '@modelcontextprotocol/sdk/client'; import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js'; import { @@ -88,493 +96,79 @@ interface DiagnosticRequestDetails { export class MCPService { /** - * Create a connection log entry for phase tracking. + * Execute a tool call on a connection. + * Supports abort signal for cancellable operations (e.g., when user stops generation). + * Formats the raw tool result into a string representation. * - * @param phase - The connection phase this log belongs to - * @param message - Human-readable log message - * @param level - Log severity level (default: INFO) - * @param details - Optional structured details for debugging - * @returns Formatted connection log entry + * @param connection - The MCP connection to execute against + * @param params - Tool name and arguments to execute + * @param signal - Optional AbortSignal for cancellation support + * @returns Formatted tool execution result with content string and error flag + * @throws {Error} If tool execution fails or is aborted */ - private static createLog( - phase: MCPConnectionPhase, - message: string, - level: MCPLogLevel = MCPLogLevel.INFO, - details?: unknown - ): MCPConnectionLog { - return { - details, - level, - message, - phase, - timestamp: new Date() - }; - } - - private static createDiagnosticRequestDetails( - input: RequestInfo | URL, - init: RequestInit | undefined, - baseInit: RequestInit, - requestHeaders: Headers, - extraRedactedHeaders?: Iterable - ): DiagnosticRequestDetails { - const body = getRequestBody(input, init); - const details: DiagnosticRequestDetails = { - body: summarizeRequestBody(body), - credentials: init?.credentials ?? baseInit.credentials, - headers: sanitizeHeaders(requestHeaders, extraRedactedHeaders, HEADERS.PARTIAL_REDACT), - method: getRequestMethod(input, init, baseInit).toUpperCase(), - mode: init?.mode ?? baseInit.mode, - url: getRequestUrl(input) - }; - const jsonRpcMethods = extractJsonRpcMethods(body); - - if (jsonRpcMethods) { - details.jsonRpcMethods = jsonRpcMethods; - } - - return details; - } - - private static addRequestHeaders( - requestHeaders: Headers, - headers: HeadersInit, - useProxy: boolean - ) { - for (const [key, value] of new Headers(headers).entries()) { - const proxiedKey = - useProxy && !key.toLowerCase().startsWith(CORS_PROXY.HEADER_PREFIX) - ? `${CORS_PROXY.HEADER_PREFIX}${key}` - : key; - - requestHeaders.set(proxiedKey, value); - } - } - - private static summarizeError(error: unknown): Record { - if (error instanceof Error) { - return { - cause: - error.cause instanceof Error - ? { message: error.cause.message, name: error.cause.name } - : error.cause, - message: error.message, - name: error.name, - stack: error.stack?.split('\n').slice(0, 6).join('\n') - }; - } - - return { value: String(error) }; - } - - private static getBrowserContext( - targetUrl: URL, - useProxy: boolean - ): Record | undefined { - if (typeof window === 'undefined') { - return undefined; - } - - return { - isSecureContext: window.isSecureContext, - location: window.location.href, - origin: window.location.origin, - protocol: window.location.protocol, - sameOrigin: window.location.origin === targetUrl.origin, - targetOrigin: targetUrl.origin, - targetProtocol: targetUrl.protocol, - useProxy - }; - } - - private static getConnectionHints( - targetUrl: URL, - config: MCPServerConfig, - error: unknown - ): string[] { - const hints: string[] = []; - const message = error instanceof Error ? error.message : String(error); - const headerNames = Object.keys(config.headers ?? {}); - - if (typeof window !== 'undefined') { - if ( - window.location.protocol === 'https:' && - targetUrl.protocol === 'http:' && - !config.useProxy - ) { - hints.push( - 'The page is running over HTTPS but the MCP server is HTTP. Browsers often block this as mixed content; enable the proxy or use HTTPS/WSS for the MCP server.' - ); - } - - if (window.location.origin !== targetUrl.origin && !config.useProxy) { - hints.push( - 'This is a cross-origin browser request. If the server is reachable from curl or Node but not from the browser, missing CORS headers are the most likely cause.' - ); - } - } - - if (headerNames.length > 0) { - hints.push( - `Custom request headers are configured (${headerNames.join(', ')}). That triggers a CORS preflight, so the server must allow OPTIONS and include the matching Access-Control-Allow-Headers response.` - ); - } - - if (config.credentials && config.credentials !== 'omit') { - hints.push( - 'Credentials are enabled for this connection. Cross-origin credentialed requests need Access-Control-Allow-Credentials: true and cannot use a wildcard Access-Control-Allow-Origin.' - ); - } - - if (message.includes('Failed to fetch')) { - hints.push( - '"Failed to fetch" is a browser-level network failure. Common causes are CORS rejection, mixed-content blocking, certificate/TLS errors, DNS failures, or nothing listening on the target port.' - ); - } - - return hints; - } - - private static createDiagnosticFetch( - serverName: string, - config: MCPServerConfig, - baseInit: RequestInit, - targetUrl: URL, - useProxy: boolean, - onLog?: (log: MCPConnectionLog) => void - ): { - fetch: typeof fetch; - disable: () => void; - } { - let enabled = true; - - const logIfEnabled = (log: MCPConnectionLog) => { - if (enabled) { - onLog?.(log); - } - }; - - return { - disable: () => { - enabled = false; - }, - fetch: async (input, init) => { - if (useProxy && typeof window !== 'undefined') { - let requestUrlStr = ''; - - if (typeof input === 'string') { - requestUrlStr = input; - } else if (input instanceof URL) { - requestUrlStr = input.href; - } - - if (requestUrlStr) { - const parsedRequestUrl = new URL(requestUrlStr, window.location.origin); - - if ( - parsedRequestUrl.origin === window.location.origin && - !parsedRequestUrl.pathname.includes(CORS_PROXY_ENDPOINT) - ) { - const originalConfigUrl = new URL(config.url); - const realTargetUrl = new URL( - parsedRequestUrl.pathname + parsedRequestUrl.search, - originalConfigUrl.origin - ); - const proxiedUrl = buildProxiedUrl(realTargetUrl.href); - - if (typeof input === 'string') { - input = proxiedUrl.href; - } else if (input instanceof URL) { - input = proxiedUrl; - } - } - } - } - - const startedAt = performance.now(); - const requestHeaders = new Headers(baseInit.headers); - - if (typeof Request !== 'undefined' && input instanceof Request) { - this.addRequestHeaders(requestHeaders, input.headers, useProxy); - } - - if (init?.headers) { - this.addRequestHeaders(requestHeaders, init.headers, useProxy); - } - - const request = this.createDiagnosticRequestDetails( - input, - init, - baseInit, - requestHeaders, - Object.keys(config.headers ?? {}) - ); - const { method, url } = request; - - logIfEnabled( - this.createLog( - MCPConnectionPhase.INITIALIZING, - `HTTP ${method} ${url}`, - MCPLogLevel.INFO, - { - request, - serverName - } - ) - ); - - if (method === 'DELETE' && url.includes(CORS_PROXY_ENDPOINT)) { - const response = new Response(null, { status: 200, statusText: 'OK' }); - - logIfEnabled( - this.createLog( - MCPConnectionPhase.INITIALIZING, - `HTTP 200 ${method} ${url} (fake response)`, - MCPLogLevel.INFO, - { - response: { - durationMs: 0, - isFake: true, - status: response.status, - statusText: response.statusText, - url - } - } - ) - ); - - // fake response, bypass real fetch() - return response; - } - - try { - const response = await fetch(input, { - ...baseInit, - ...init, - headers: requestHeaders - }); - const durationMs = Math.round(performance.now() - startedAt); - - logIfEnabled( - this.createLog( - MCPConnectionPhase.INITIALIZING, - `HTTP ${response.status} ${method} ${url} (${durationMs}ms)`, - response.ok ? MCPLogLevel.INFO : MCPLogLevel.WARN, - { - response: { - durationMs, - headers: sanitizeHeaders(response.headers, undefined, HEADERS.PARTIAL_REDACT), - status: response.status, - statusText: response.statusText, - url - } - } - ) - ); - - return response; - } catch (error) { - const durationMs = Math.round(performance.now() - startedAt); - - logIfEnabled( - this.createLog( - MCPConnectionPhase.ERROR, - `HTTP ${method} ${url} failed: ${formatDiagnosticErrorMessage(error)}`, - MCPLogLevel.ERROR, - { - browser: this.getBrowserContext(targetUrl, useProxy), - durationMs, - error: this.summarizeError(error), - hints: this.getConnectionHints(targetUrl, config, error), - request, - serverName - } - ) - ); - - throw error; - } - } - }; - } - - /** - * Detect if an error indicates an expired/invalidated MCP session. - * Per MCP spec 2025-11-25: HTTP 404 means session invalidated, client MUST - * discard its session ID and start a new session with a fresh initialize request. - * - * @param error - The caught error to inspect - * @returns true if the error is a StreamableHTTP 404 (session not found) - */ - static isSessionExpiredError(error: unknown): boolean { - return error instanceof StreamableHTTPError && error.code === 404; - } - - /** - * Create transport based on server configuration. - * Supports WebSocket, StreamableHTTP (modern), and SSE (legacy) transports. - * When `useProxy` is enabled, routes HTTP requests through llama-server's CORS proxy. - * - * **Fallback Order:** - * 1. WebSocket — if explicitly configured (no CORS proxy support) - * 2. StreamableHTTP — default for HTTP connections - * 3. SSE — automatic fallback if StreamableHTTP fails - * - * @param config - Server configuration with url, transport type, proxy, and auth settings - * @returns Object containing the created transport and the transport type used - * @throws {Error} If url is missing, WebSocket + proxy combination, or all transports fail - */ - static createTransport( - serverName: string, - config: MCPServerConfig, - onLog?: (log: MCPConnectionLog) => void - ): { - transport: Transport; - type: MCPTransportType; - stopPhaseLogging: () => void; - } { - if (!config.url) { - throw new Error('MCP server configuration is missing url'); - } - - const useProxy = config.useProxy ?? false; - const requestInit: RequestInit = {}; - - if (config.headers) { - requestInit.headers = config.useProxy ? buildProxiedHeaders(config.headers) : config.headers; - } - - if (useProxy) { - requestInit.headers = { - ...getAuthHeaders(), - ...(requestInit.headers as Record) - }; - } - - if (config.credentials) { - requestInit.credentials = config.credentials; - } - - if (config.transport === MCPTransportType.WEBSOCKET) { - if (useProxy) { - throw new Error( - 'WebSocket transport is not supported when using CORS proxy. Use HTTP transport instead.' - ); - } - - const url = new URL(config.url); - - if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { - console.log(`[MCPService] Creating WebSocket transport for ${url.href}`); - } - - return { - stopPhaseLogging: () => {}, - transport: new WebSocketClientTransport(url), - type: MCPTransportType.WEBSOCKET - }; - } - - if (config.transport === MCPTransportType.SSE) { - const url = useProxy ? buildProxiedUrl(config.url) : new URL(config.url); - const { disable: stopPhaseLogging, fetch: diagnosticFetch } = this.createDiagnosticFetch( - serverName, - config, - requestInit, - url, - useProxy, - onLog - ); - - if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { - console.log(`[MCPService] Creating SSE transport for ${url.href}`); - } - - return { - stopPhaseLogging, - transport: new SSEClientTransport(url, { - eventSourceInit: { fetch: diagnosticFetch }, - fetch: diagnosticFetch, - requestInit - }), - type: MCPTransportType.SSE - }; - } - - const url = useProxy ? buildProxiedUrl(config.url) : new URL(config.url); - const { disable: stopPhaseLogging, fetch: diagnosticFetch } = this.createDiagnosticFetch( - serverName, - config, - requestInit, - url, - useProxy, - onLog - ); - - if (useProxy && import.meta.env.DEV && import.meta.env.VITE_DEBUG) { - console.log(`[MCPService] Using CORS proxy for ${config.url} -> ${url.href}`); - } + static async callTool( + connection: MCPConnection, + params: ToolCallParams, + signal?: AbortSignal + ): Promise { + throwIfAborted(signal); try { - if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { - console.log(`[MCPService] Creating StreamableHTTP transport for ${url.href}`); - } + const result = await connection.client.callTool( + { arguments: params.arguments, name: params.name }, + undefined, + { signal, timeout: connection.requestTimeoutMs } + ); return { - stopPhaseLogging, - transport: new StreamableHTTPClientTransport(url, { - fetch: diagnosticFetch, - requestInit - }), - type: MCPTransportType.STREAMABLE_HTTP + content: this.formatToolResult(result as ToolCallResult), + isError: (result as ToolCallResult).isError ?? false }; - } catch (httpError) { - console.warn(`[MCPService] StreamableHTTP failed, trying SSE transport...`, httpError); - - try { - return { - stopPhaseLogging, - transport: new SSEClientTransport(url, { - eventSourceInit: { fetch: diagnosticFetch }, - fetch: diagnosticFetch, - requestInit - }), - type: MCPTransportType.SSE - }; - } catch (sseError) { - const httpMsg = httpError instanceof Error ? httpError.message : String(httpError); - const sseMsg = sseError instanceof Error ? sseError.message : String(sseError); - - throw new Error(`Failed to create transport. StreamableHTTP: ${httpMsg}; SSE: ${sseMsg}`); + } catch (error) { + if (isAbortError(error)) { + throw error; } + + // Let session-expired errors propagate unwrapped for reconnection handling + if (this.isSessionExpiredError(error)) { + throw error; + } + + const message = error instanceof Error ? error.message : String(error); + + throw new Error( + `Tool "${params.name}" execution failed on server "${connection.serverName}": ${message}`, + { cause: error instanceof Error ? error : undefined } + ); } } /** - * Extract server info from SDK Implementation type. - * Normalizes the SDK's server version response into our MCPServerInfo type. + * Request completion suggestions from a server. + * Used for autocompleting prompt arguments or resource URI templates. * - * @param impl - Raw Implementation object from MCP SDK - * @returns Normalized server info or undefined if input is empty + * @param connection - The MCP connection to use + * @param ref - Reference to the prompt or resource template + * @param argument - The argument being completed (name and current value) + * @returns Completion result with suggested values */ - private static extractServerInfo(impl: Implementation | undefined): MCPServerInfo | undefined { - if (!impl) { - return undefined; - } + static async complete( + connection: MCPConnection, + ref: { type: MCPRefType.PROMPT; name: string } | { type: MCPRefType.RESOURCE; uri: string }, + argument: { name: string; value: string } + ): Promise<{ values: string[]; total?: number; hasMore?: boolean } | null> { + try { + const result = await connection.client.complete({ + argument, + ref + }); - return { - description: impl.description, - icons: impl.icons?.map((icon: MCPResourceIcon) => ({ - mimeType: icon.mimeType, - sizes: icon.sizes, - src: icon.src, - theme: icon.theme - })), - name: impl.name, - title: impl.title, - version: impl.version, - websiteUrl: impl.websiteUrl - }; + return result.completion; + } catch (error) { + console.error(`[MCPService] Failed to get completions:`, error); + + return null; + } } /** @@ -847,6 +441,146 @@ export class MCPService { }; } + /** + * Create transport based on server configuration. + * Supports WebSocket, StreamableHTTP (modern), and SSE (legacy) transports. + * When `useProxy` is enabled, routes HTTP requests through llama-server's CORS proxy. + * + * **Fallback Order:** + * 1. WebSocket — if explicitly configured (no CORS proxy support) + * 2. StreamableHTTP — default for HTTP connections + * 3. SSE — automatic fallback if StreamableHTTP fails + * + * @param config - Server configuration with url, transport type, proxy, and auth settings + * @returns Object containing the created transport and the transport type used + * @throws {Error} If url is missing, WebSocket + proxy combination, or all transports fail + */ + static createTransport( + serverName: string, + config: MCPServerConfig, + onLog?: (log: MCPConnectionLog) => void + ): { + transport: Transport; + type: MCPTransportType; + stopPhaseLogging: () => void; + } { + if (!config.url) { + throw new Error('MCP server configuration is missing url'); + } + + const useProxy = config.useProxy ?? false; + const requestInit: RequestInit = {}; + + if (config.headers) { + requestInit.headers = config.useProxy ? buildProxiedHeaders(config.headers) : config.headers; + } + + if (useProxy) { + requestInit.headers = { + ...getAuthHeaders(), + ...(requestInit.headers as Record) + }; + } + + if (config.credentials) { + requestInit.credentials = config.credentials; + } + + if (config.transport === MCPTransportType.WEBSOCKET) { + if (useProxy) { + throw new Error( + 'WebSocket transport is not supported when using CORS proxy. Use HTTP transport instead.' + ); + } + + const url = new URL(config.url); + + if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log(`[MCPService] Creating WebSocket transport for ${url.href}`); + } + + return { + stopPhaseLogging: () => {}, + transport: new WebSocketClientTransport(url), + type: MCPTransportType.WEBSOCKET + }; + } + + if (config.transport === MCPTransportType.SSE) { + const url = useProxy ? buildProxiedUrl(config.url) : new URL(config.url); + const { disable: stopPhaseLogging, fetch: diagnosticFetch } = this.createDiagnosticFetch( + serverName, + config, + requestInit, + url, + useProxy, + onLog + ); + + if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log(`[MCPService] Creating SSE transport for ${url.href}`); + } + + return { + stopPhaseLogging, + transport: new SSEClientTransport(url, { + eventSourceInit: { fetch: diagnosticFetch }, + fetch: diagnosticFetch, + requestInit + }), + type: MCPTransportType.SSE + }; + } + + const url = useProxy ? buildProxiedUrl(config.url) : new URL(config.url); + const { disable: stopPhaseLogging, fetch: diagnosticFetch } = this.createDiagnosticFetch( + serverName, + config, + requestInit, + url, + useProxy, + onLog + ); + + if (useProxy && import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log(`[MCPService] Using CORS proxy for ${config.url} -> ${url.href}`); + } + + try { + if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log(`[MCPService] Creating StreamableHTTP transport for ${url.href}`); + } + + return { + stopPhaseLogging, + transport: new StreamableHTTPClientTransport(url, { + fetch: diagnosticFetch, + requestInit + }), + type: MCPTransportType.STREAMABLE_HTTP + }; + } catch (httpError) { + console.warn(`[MCPService] StreamableHTTP failed, trying SSE transport...`, httpError); + + try { + return { + stopPhaseLogging, + transport: new SSEClientTransport(url, { + eventSourceInit: { fetch: diagnosticFetch }, + fetch: diagnosticFetch, + requestInit + }), + type: MCPTransportType.SSE + }; + } catch (sseError) { + const httpMsg = httpError instanceof Error ? httpError.message : String(httpError); + const sseMsg = sseError instanceof Error ? sseError.message : String(sseError); + + throw new Error(`Failed to create transport. StreamableHTTP: ${httpMsg}; SSE: ${sseMsg}`); + } + } + } + /** * Disconnect from a server. * Clears the `onclose` handler to prevent reconnection attempts on voluntary disconnect. @@ -882,29 +616,68 @@ export class MCPService { } /** - * List tools from a connection. - * Silently returns empty array on failure (logged as warning). + * Get a specific prompt with arguments. + * Unlike list operations, this throws on failure since the caller explicitly + * requested a specific prompt and needs to handle the error. * - * @param connection - The MCP connection to query - * @returns Array of available tools, or empty array on error + * @param connection - The MCP connection to use + * @param name - The prompt name to retrieve + * @param args - Optional key-value arguments to pass to the prompt + * @returns The prompt result with messages and metadata + * @throws {Error} If the prompt retrieval fails */ - static async listTools(connection: MCPConnection): Promise { + static async getPrompt( + connection: MCPConnection, + name: string, + args?: Record + ): Promise { try { - const result = await connection.client.listTools(); - - return result.tools ?? []; + return await connection.client.getPrompt({ arguments: args, name }); } catch (error) { - // Let session-expired errors propagate for reconnection handling - if (this.isSessionExpiredError(error)) { - throw error; - } + console.error(`[MCPService][${connection.serverName}] Failed to get prompt:`, error); - console.warn(`[MCPService][${connection.serverName}] Failed to list tools:`, error); - - return []; + throw error; } } + /** + * Detect if an error indicates an expired/invalidated MCP session. + * Per MCP spec 2025-11-25: HTTP 404 means session invalidated, client MUST + * discard its session ID and start a new session with a fresh initialize request. + * + * @param error - The caught error to inspect + * @returns true if the error is a StreamableHTTP 404 (session not found) + */ + static isSessionExpiredError(error: unknown): boolean { + return error instanceof StreamableHTTPError && error.code === 404; + } + + /** + * List all resources from a connection (handles pagination automatically). + * @param connection - The MCP connection to use + * @returns Array of all available resources + */ + static async listAllResources(connection: MCPConnection): Promise { + return this.paginate( + connection, + (cursor) => this.listResources(connection, cursor), + (result) => result.resources + ); + } + + /** + * List all resource templates from a connection (handles pagination automatically). + * @param connection - The MCP connection to use + * @returns Array of all available resource templates + */ + static async listAllResourceTemplates(connection: MCPConnection): Promise { + return this.paginate( + connection, + (cursor) => this.listResourceTemplates(connection, cursor), + (result) => result.resourceTemplates + ); + } + /** * List prompts from a connection. * Silently returns empty array on failure (logged as warning). @@ -929,177 +702,6 @@ export class MCPService { } } - /** - * Get a specific prompt with arguments. - * Unlike list operations, this throws on failure since the caller explicitly - * requested a specific prompt and needs to handle the error. - * - * @param connection - The MCP connection to use - * @param name - The prompt name to retrieve - * @param args - Optional key-value arguments to pass to the prompt - * @returns The prompt result with messages and metadata - * @throws {Error} If the prompt retrieval fails - */ - static async getPrompt( - connection: MCPConnection, - name: string, - args?: Record - ): Promise { - try { - return await connection.client.getPrompt({ arguments: args, name }); - } catch (error) { - console.error(`[MCPService][${connection.serverName}] Failed to get prompt:`, error); - - throw error; - } - } - - /** - * Execute a tool call on a connection. - * Supports abort signal for cancellable operations (e.g., when user stops generation). - * Formats the raw tool result into a string representation. - * - * @param connection - The MCP connection to execute against - * @param params - Tool name and arguments to execute - * @param signal - Optional AbortSignal for cancellation support - * @returns Formatted tool execution result with content string and error flag - * @throws {Error} If tool execution fails or is aborted - */ - static async callTool( - connection: MCPConnection, - params: ToolCallParams, - signal?: AbortSignal - ): Promise { - throwIfAborted(signal); - - try { - const result = await connection.client.callTool( - { arguments: params.arguments, name: params.name }, - undefined, - { signal, timeout: connection.requestTimeoutMs } - ); - - return { - content: this.formatToolResult(result as ToolCallResult), - isError: (result as ToolCallResult).isError ?? false - }; - } catch (error) { - if (isAbortError(error)) { - throw error; - } - - // Let session-expired errors propagate unwrapped for reconnection handling - if (this.isSessionExpiredError(error)) { - throw error; - } - - const message = error instanceof Error ? error.message : String(error); - - throw new Error( - `Tool "${params.name}" execution failed on server "${connection.serverName}": ${message}`, - { cause: error instanceof Error ? error : undefined } - ); - } - } - - /** - * Format tool result content items to a single string. - * Handles text, image (base64 data URL), and embedded resource content types. - * - * @param result - Raw tool call result from MCP SDK - * @returns Concatenated string representation of all content items - */ - private static formatToolResult(result: ToolCallResult): string { - const content = result.content; - - if (!Array.isArray(content)) return ''; - - const formatted = content - .map((item) => this.formatSingleContent(item)) - .filter(Boolean) - .join(NEWLINE); - - if (formatted !== '') { - return formatted; - } - - if (result.structuredContent && typeof result.structuredContent === 'object') { - return JSON.stringify(result.structuredContent); - } - - return ''; - } - - private static formatSingleContent(content: ToolResultContentItem): string { - if (content.type === MCPContentType.TEXT && content.text) { - return content.text; - } - - if (content.type === MCPContentType.IMAGE && content.data) { - return createBase64DataUrl(content.mimeType ?? DEFAULT_IMAGE_MIME_TYPE, content.data); - } - - if (content.type === MCPContentType.RESOURCE && content.resource) { - const resource = content.resource; - - if (resource.text) return resource.text; - - if (resource.blob) return resource.blob; - - return JSON.stringify(resource); - } - - if (content.data && content.mimeType) { - return createBase64DataUrl(content.mimeType, content.data); - } - - return JSON.stringify(content); - } - - /** - * - * - * Completions Operations - * - * - */ - - /** - * Request completion suggestions from a server. - * Used for autocompleting prompt arguments or resource URI templates. - * - * @param connection - The MCP connection to use - * @param ref - Reference to the prompt or resource template - * @param argument - The argument being completed (name and current value) - * @returns Completion result with suggested values - */ - static async complete( - connection: MCPConnection, - ref: { type: MCPRefType.PROMPT; name: string } | { type: MCPRefType.RESOURCE; uri: string }, - argument: { name: string; value: string } - ): Promise<{ values: string[]; total?: number; hasMore?: boolean } | null> { - try { - const result = await connection.client.complete({ - argument, - ref - }); - - return result.completion; - } catch (error) { - console.error(`[MCPService] Failed to get completions:`, error); - - return null; - } - } - - /** - * - * - * Resources Operations - * - * - */ - /** * List resources from a connection. * @param connection - The MCP connection to use @@ -1128,26 +730,6 @@ export class MCPService { } } - /** - * List all resources from a connection (handles pagination automatically). - * @param connection - The MCP connection to use - * @returns Array of all available resources - */ - static async listAllResources(connection: MCPConnection): Promise { - const allResources: MCPResource[] = []; - - let cursor: string | undefined; - - do { - const result = await this.listResources(connection, cursor); - - allResources.push(...result.resources); - cursor = result.nextCursor; - } while (cursor); - - return allResources; - } - /** * List resource templates from a connection. * @param connection - The MCP connection to use @@ -1180,23 +762,27 @@ export class MCPService { } /** - * List all resource templates from a connection (handles pagination automatically). - * @param connection - The MCP connection to use - * @returns Array of all available resource templates + * List tools from a connection. + * Silently returns empty array on failure (logged as warning). + * + * @param connection - The MCP connection to query + * @returns Array of available tools, or empty array on error */ - static async listAllResourceTemplates(connection: MCPConnection): Promise { - const allTemplates: MCPResourceTemplate[] = []; + static async listTools(connection: MCPConnection): Promise { + try { + const result = await connection.client.listTools(); - let cursor: string | undefined; + return result.tools ?? []; + } catch (error) { + // Let session-expired errors propagate for reconnection handling + if (this.isSessionExpiredError(error)) { + throw error; + } - do { - const result = await this.listResourceTemplates(connection, cursor); + console.warn(`[MCPService][${connection.serverName}] Failed to list tools:`, error); - allTemplates.push(...result.resourceTemplates); - cursor = result.nextCursor; - } while (cursor); - - return allTemplates; + return []; + } } /** @@ -1244,28 +830,6 @@ export class MCPService { } } - /** - * Unsubscribe from updates for a resource. - * @param connection - The MCP connection to use - * @param uri - The URI of the resource to unsubscribe from - */ - static async unsubscribeResource(connection: MCPConnection, uri: string): Promise { - try { - await connection.client.unsubscribeResource({ uri }); - - if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { - console.log(`[MCPService][${connection.serverName}] Unsubscribed from resource: ${uri}`); - } - } catch (error) { - console.error( - `[MCPService][${connection.serverName}] Failed to unsubscribe from resource:`, - error - ); - - throw error; - } - } - /** * Check if a connection supports resources. * Per MCP spec: presence of the `resources` key (even as empty object `{}`) indicates support. @@ -1288,4 +852,440 @@ export class MCPService { static supportsResourceSubscriptions(connection: MCPConnection): boolean { return !!connection.serverCapabilities?.resources?.subscribe; } + + /** + * Unsubscribe from updates for a resource. + * @param connection - The MCP connection to use + * @param uri - The URI of the resource to unsubscribe from + */ + static async unsubscribeResource(connection: MCPConnection, uri: string): Promise { + try { + await connection.client.unsubscribeResource({ uri }); + + if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log(`[MCPService][${connection.serverName}] Unsubscribed from resource: ${uri}`); + } + } catch (error) { + console.error( + `[MCPService][${connection.serverName}] Failed to unsubscribe from resource:`, + error + ); + + throw error; + } + } + + private static addRequestHeaders( + requestHeaders: Headers, + headers: HeadersInit, + useProxy: boolean + ) { + for (const [key, value] of new Headers(headers).entries()) { + const proxiedKey = + useProxy && !key.toLowerCase().startsWith(CORS_PROXY.HEADER_PREFIX) + ? `${CORS_PROXY.HEADER_PREFIX}${key}` + : key; + + requestHeaders.set(proxiedKey, value); + } + } + + private static createDiagnosticFetch( + serverName: string, + config: MCPServerConfig, + baseInit: RequestInit, + targetUrl: URL, + useProxy: boolean, + onLog?: (log: MCPConnectionLog) => void + ): { + fetch: typeof fetch; + disable: () => void; + } { + let enabled = true; + + const logIfEnabled = (log: MCPConnectionLog) => { + if (enabled) { + onLog?.(log); + } + }; + + return { + disable: () => { + enabled = false; + }, + fetch: async (input, init) => { + if (useProxy && typeof window !== 'undefined') { + let requestUrlStr = ''; + + if (typeof input === 'string') { + requestUrlStr = input; + } else if (input instanceof URL) { + requestUrlStr = input.href; + } + + if (requestUrlStr) { + const parsedRequestUrl = new URL(requestUrlStr, window.location.origin); + + if ( + parsedRequestUrl.origin === window.location.origin && + !parsedRequestUrl.pathname.includes(CORS_PROXY_ENDPOINT) + ) { + const originalConfigUrl = new URL(config.url); + const realTargetUrl = new URL( + parsedRequestUrl.pathname + parsedRequestUrl.search, + originalConfigUrl.origin + ); + const proxiedUrl = buildProxiedUrl(realTargetUrl.href); + + if (typeof input === 'string') { + input = proxiedUrl.href; + } else if (input instanceof URL) { + input = proxiedUrl; + } + } + } + } + + const startedAt = performance.now(); + const requestHeaders = new Headers(baseInit.headers); + + if (typeof Request !== 'undefined' && input instanceof Request) { + this.addRequestHeaders(requestHeaders, input.headers, useProxy); + } + + if (init?.headers) { + this.addRequestHeaders(requestHeaders, init.headers, useProxy); + } + + const request = this.createDiagnosticRequestDetails( + input, + init, + baseInit, + requestHeaders, + Object.keys(config.headers ?? {}) + ); + const { method, url } = request; + + logIfEnabled( + this.createLog( + MCPConnectionPhase.INITIALIZING, + `HTTP ${method} ${url}`, + MCPLogLevel.INFO, + { + request, + serverName + } + ) + ); + + if (method === 'DELETE' && url.includes(CORS_PROXY_ENDPOINT)) { + const response = new Response(null, { status: 200, statusText: 'OK' }); + + logIfEnabled( + this.createLog( + MCPConnectionPhase.INITIALIZING, + `HTTP 200 ${method} ${url} (fake response)`, + MCPLogLevel.INFO, + { + response: { + durationMs: 0, + isFake: true, + status: response.status, + statusText: response.statusText, + url + } + } + ) + ); + + // fake response, bypass real fetch() + return response; + } + + try { + const response = await fetch(input, { + ...baseInit, + ...init, + headers: requestHeaders + }); + const durationMs = Math.round(performance.now() - startedAt); + + logIfEnabled( + this.createLog( + MCPConnectionPhase.INITIALIZING, + `HTTP ${response.status} ${method} ${url} (${durationMs}ms)`, + response.ok ? MCPLogLevel.INFO : MCPLogLevel.WARN, + { + response: { + durationMs, + headers: sanitizeHeaders(response.headers, undefined, HEADERS.PARTIAL_REDACT), + status: response.status, + statusText: response.statusText, + url + } + } + ) + ); + + return response; + } catch (error) { + const durationMs = Math.round(performance.now() - startedAt); + + logIfEnabled( + this.createLog( + MCPConnectionPhase.ERROR, + `HTTP ${method} ${url} failed: ${formatDiagnosticErrorMessage(error)}`, + MCPLogLevel.ERROR, + { + browser: this.getBrowserContext(targetUrl, useProxy), + durationMs, + error: this.summarizeError(error), + hints: this.getConnectionHints(targetUrl, config, error), + request, + serverName + } + ) + ); + + throw error; + } + } + }; + } + + private static createDiagnosticRequestDetails( + input: RequestInfo | URL, + init: RequestInit | undefined, + baseInit: RequestInit, + requestHeaders: Headers, + extraRedactedHeaders?: Iterable + ): DiagnosticRequestDetails { + const body = getRequestBody(input, init); + const details: DiagnosticRequestDetails = { + body: summarizeRequestBody(body), + credentials: init?.credentials ?? baseInit.credentials, + headers: sanitizeHeaders(requestHeaders, extraRedactedHeaders, HEADERS.PARTIAL_REDACT), + method: getRequestMethod(input, init, baseInit).toUpperCase(), + mode: init?.mode ?? baseInit.mode, + url: getRequestUrl(input) + }; + const jsonRpcMethods = extractJsonRpcMethods(body); + + if (jsonRpcMethods) { + details.jsonRpcMethods = jsonRpcMethods; + } + + return details; + } + + /** + * Create a connection log entry for phase tracking. + * + * @param phase - The connection phase this log belongs to + * @param message - Human-readable log message + * @param level - Log severity level (default: INFO) + * @param details - Optional structured details for debugging + * @returns Formatted connection log entry + */ + private static createLog( + phase: MCPConnectionPhase, + message: string, + level: MCPLogLevel = MCPLogLevel.INFO, + details?: unknown + ): MCPConnectionLog { + return { + details, + level, + message, + phase, + timestamp: new Date() + }; + } + + /** + * Extract server info from SDK Implementation type. + * Normalizes the SDK's server version response into our MCPServerInfo type. + * + * @param impl - Raw Implementation object from MCP SDK + * @returns Normalized server info or undefined if input is empty + */ + private static extractServerInfo(impl: Implementation | undefined): MCPServerInfo | undefined { + if (!impl) { + return undefined; + } + + return { + description: impl.description, + icons: impl.icons?.map((icon: MCPResourceIcon) => ({ + mimeType: icon.mimeType, + sizes: icon.sizes, + src: icon.src, + theme: icon.theme + })), + name: impl.name, + title: impl.title, + version: impl.version, + websiteUrl: impl.websiteUrl + }; + } + + private static formatSingleContent(content: ToolResultContentItem): string { + if (content.type === MCPContentType.TEXT && content.text) { + return content.text; + } + + if (content.type === MCPContentType.IMAGE && content.data) { + return createBase64DataUrl(content.mimeType ?? DEFAULT_IMAGE_MIME_TYPE, content.data); + } + + if (content.type === MCPContentType.RESOURCE && content.resource) { + const resource = content.resource; + + if (resource.text) return resource.text; + + if (resource.blob) return resource.blob; + + return JSON.stringify(resource); + } + + if (content.data && content.mimeType) { + return createBase64DataUrl(content.mimeType, content.data); + } + + return JSON.stringify(content); + } + + /** + * Format tool result content items to a single string. + * Handles text, image (base64 data URL), and embedded resource content types. + * + * @param result - Raw tool call result from MCP SDK + * @returns Concatenated string representation of all content items + */ + private static formatToolResult(result: ToolCallResult): string { + const content = result.content; + + if (!Array.isArray(content)) return ''; + + const formatted = content + .map((item) => this.formatSingleContent(item)) + .filter(Boolean) + .join(NEWLINE); + + if (formatted !== '') { + return formatted; + } + + if (result.structuredContent && typeof result.structuredContent === 'object') { + return JSON.stringify(result.structuredContent); + } + + return ''; + } + + private static getBrowserContext( + targetUrl: URL, + useProxy: boolean + ): Record | undefined { + if (typeof window === 'undefined') { + return undefined; + } + + return { + isSecureContext: window.isSecureContext, + location: window.location.href, + origin: window.location.origin, + protocol: window.location.protocol, + sameOrigin: window.location.origin === targetUrl.origin, + targetOrigin: targetUrl.origin, + targetProtocol: targetUrl.protocol, + useProxy + }; + } + + private static getConnectionHints( + targetUrl: URL, + config: MCPServerConfig, + error: unknown + ): string[] { + const hints: string[] = []; + const message = error instanceof Error ? error.message : String(error); + const headerNames = Object.keys(config.headers ?? {}); + + if (typeof window !== 'undefined') { + if ( + window.location.protocol === 'https:' && + targetUrl.protocol === 'http:' && + !config.useProxy + ) { + hints.push( + 'The page is running over HTTPS but the MCP server is HTTP. Browsers often block this as mixed content; enable the proxy or use HTTPS/WSS for the MCP server.' + ); + } + + if (window.location.origin !== targetUrl.origin && !config.useProxy) { + hints.push( + 'This is a cross-origin browser request. If the server is reachable from curl or Node but not from the browser, missing CORS headers are the most likely cause.' + ); + } + } + + if (headerNames.length > 0) { + hints.push( + `Custom request headers are configured (${headerNames.join(', ')}). That triggers a CORS preflight, so the server must allow OPTIONS and include the matching Access-Control-Allow-Headers response.` + ); + } + + if (config.credentials && config.credentials !== 'omit') { + hints.push( + 'Credentials are enabled for this connection. Cross-origin credentialed requests need Access-Control-Allow-Credentials: true and cannot use a wildcard Access-Control-Allow-Origin.' + ); + } + + if (message.includes('Failed to fetch')) { + hints.push( + '"Failed to fetch" is a browser-level network failure. Common causes are CORS rejection, mixed-content blocking, certificate/TLS errors, DNS failures, or nothing listening on the target port.' + ); + } + + return hints; + } + + /** + * Walk a cursor-paginated MCP list endpoint, collecting every page. + */ + private static async paginate( + connection: MCPConnection, + fetchPage: (cursor?: string) => Promise, + extract: (result: R) => T[] + ): Promise { + const all: T[] = []; + + let cursor: string | undefined; + + do { + const result = await fetchPage(cursor); + + all.push(...extract(result)); + cursor = result.nextCursor; + } while (cursor); + + return all; + } + + private static summarizeError(error: unknown): Record { + if (error instanceof Error) { + return { + cause: + error.cause instanceof Error + ? { message: error.cause.message, name: error.cause.name } + : error.cause, + message: error.message, + name: error.name, + stack: error.stack?.split('\n').slice(0, 6).join('\n') + }; + } + + return { value: String(error) }; + } } diff --git a/tools/ui/src/lib/services/migration.service.ts b/tools/ui/src/lib/services/migration.service.ts index 2626a42b3c..5d321b3ba2 100644 --- a/tools/ui/src/lib/services/migration.service.ts +++ b/tools/ui/src/lib/services/migration.service.ts @@ -1,20 +1,11 @@ /** - * Migration Service - Unified data migration hook + * MigrationService - Unified data migration hook * - * Centralizes all data migrations (localStorage, IndexedDB, legacy formats) into a single - * initialization point. Each migration copies data to new format WITHOUT deleting the old. - * - * **Architecture:** - * - Migrations are defined as objects with `id` and `run()` methods - * - Migration state is tracked in localStorage to avoid re-running - * - `runAllMigrations()` should be called once at app startup - * - All migrations are NON-DESTRUCTIVE - legacy data is preserved for downgrade compatibility - * - * **Current Migrations:** - * 1. localStorage prefix: Copy LlamaCppWebui.* → LlamaUi.* (both preserved) - * 2. IndexedDB database: Copy LlamacppWebui → LlamaUi (both preserved) - * 3. Legacy message format: Transform in-place (preserves structure, migrates markers) - * 4. Theme key: Copy standalone `theme` → config object (both preserved) + * Centralizes all data migrations (localStorage, IndexedDB, legacy formats) + * into a single initialization point. Each migration copies data to the new + * format WITHOUT deleting the old, and state is tracked in localStorage so + * `runAllMigrations()` (called once at startup) never re-runs a completed + * migration. All migrations are non-destructive for downgrade compatibility. */ import { diff --git a/tools/ui/src/lib/services/models.service.ts b/tools/ui/src/lib/services/models.service.ts index 84832e086f..bb1bbd356a 100644 --- a/tools/ui/src/lib/services/models.service.ts +++ b/tools/ui/src/lib/services/models.service.ts @@ -1,25 +1,55 @@ +/** + * ModelsService - Stateless model management API layer + * + * Wraps the /models endpoints (list, load, unload) and the /models/sse + * status feed in MODEL and ROUTER modes. No reactive state; consumed by + * modelsStore and its status manager. + */ + import { base } from '$app/paths'; -import { - API_MODELS, - MODEL_ID, - SSE_DATA_PREFIX, - SSE_LINE_SEPARATOR, - SSE_RECORD_SEPARATOR -} from '$lib/constants'; +import { API_MODELS, MODEL_ID } from '$lib/constants'; import { ServerModelStatus } from '$lib/enums'; import type { ParsedModelId } from '$lib/types/models'; -import { apiFetch, apiPost, normalizeModelName } from '$lib/utils'; +import { + apiFetch, + apiPost, + extractSseDataPayload, + normalizeModelName, + splitSseRecords +} from '$lib/utils'; import { getAuthHeaders } from '$lib/utils/api-headers'; export class ModelsService { + private static readonly SSE_RECONNECT_MS = 1000; + + /** + * Check if a model is loaded based on its metadata. + * + * @param model - Model data entry from the API response + * @returns True if the model status is LOADED + */ + static isModelLoaded(model: ApiModelDataEntry): boolean { + return model.status.value === ServerModelStatus.LOADED; + } + /** * * - * Listing + * Load/Unload * * */ + /** + * Check if a model is currently loading. + * + * @param model - Model data entry from the API response + * @returns True if the model status is LOADING + */ + static isModelLoading(model: ApiModelDataEntry): boolean { + return model.status.value === ServerModelStatus.LOADING; + } + /** * Fetch list of models from OpenAI-compatible endpoint. * Works in both MODEL and ROUTER modes. @@ -41,14 +71,6 @@ export class ModelsService { return apiFetch(API_MODELS.LIST); } - /** - * - * - * Load/Unload - * - * - */ - /** * Load a model (ROUTER mode only). * Sends POST request to `/models/load`. Note: the endpoint returns success @@ -68,137 +90,6 @@ export class ModelsService { return apiPost(API_MODELS.LOAD, payload); } - /** - * Unload a model (ROUTER mode only). - * Sends POST request to `/models/unload`. Note: the endpoint returns success - * before unloading completes — use polling to await actual unload status. - * - * @param modelId - Model identifier to unload - * @returns Unload response from the server - */ - static async unload(modelId: string): Promise { - return apiPost(API_MODELS.UNLOAD, { model: modelId }); - } - - /** - * - * - * Status - * - * - */ - - /** - * Check if a model is loaded based on its metadata. - * - * @param model - Model data entry from the API response - * @returns True if the model status is LOADED - */ - static isModelLoaded(model: ApiModelDataEntry): boolean { - return model.status.value === ServerModelStatus.LOADED; - } - - /** - * Check if a model is currently loading. - * - * @param model - Model data entry from the API response - * @returns True if the model status is LOADING - */ - static isModelLoading(model: ApiModelDataEntry): boolean { - return model.status.value === ServerModelStatus.LOADING; - } - - /** - * - * - * Status Feed - * - * - */ - - private static readonly SSE_RECONNECT_MS = 1000; - - /** - * Read the /models/sse feed and invoke onEvent for each parsed envelope. - * Reconnects on network drops until the signal aborts. Splits the byte - * stream into SSE records on the blank line boundary; the payload rides in - * the data lines as a JSON envelope with its own model, event and data fields. - */ - static async watchModelEvents( - signal: AbortSignal, - onEvent: (event: ApiModelsSseEvent) => void - ): Promise { - const decoder = new TextDecoder(); - - while (!signal.aborted) { - try { - const response = await fetch(`${base}${API_MODELS.SSE}`, { - headers: getAuthHeaders(), - signal - }); - - if (response.ok && response.body) { - const reader = response.body.getReader(); - - let buffer = ''; - - while (!signal.aborted) { - const { done, value } = await reader.read(); - - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - - let boundary = buffer.indexOf(SSE_RECORD_SEPARATOR); - - while (boundary !== -1) { - const event = ModelsService.parseStatusRecord(buffer.slice(0, boundary)); - - if (event) onEvent(event); - - buffer = buffer.slice(boundary + SSE_RECORD_SEPARATOR.length); - boundary = buffer.indexOf(SSE_RECORD_SEPARATOR); - } - } - } - } catch { - // network drop or abort falls through to the reconnect delay - } - - if (signal.aborted) return; - - await new Promise((resolve) => setTimeout(resolve, ModelsService.SSE_RECONNECT_MS)); - } - } - - /** - * Parse one SSE record into its JSON envelope, or null when the record - * carries no data payload or malformed JSON. - */ - private static parseStatusRecord(record: string): ApiModelsSseEvent | null { - const payload = record - .split(SSE_LINE_SEPARATOR) - .filter((line) => line.startsWith(SSE_DATA_PREFIX)) - .map((line) => line.slice(SSE_DATA_PREFIX.length).trim()) - .join(SSE_LINE_SEPARATOR); - - if (payload.length === 0) return null; - - try { - return JSON.parse(payload) as ApiModelsSseEvent; - } catch { - return null; - } - } - - /** - * - * - * Parsing - * - * - */ - /** * Parse a model ID string into its structured components. * @@ -311,4 +202,84 @@ export class ModelsService { return result; } + + /** + * Unload a model (ROUTER mode only). + * Sends POST request to `/models/unload`. Note: the endpoint returns success + * before unloading completes — use polling to await actual unload status. + * + * @param modelId - Model identifier to unload + * @returns Unload response from the server + */ + static async unload(modelId: string): Promise { + return apiPost(API_MODELS.UNLOAD, { model: modelId }); + } + + /** + * Read the /models/sse feed and invoke onEvent for each parsed envelope. + * Reconnects on network drops until the signal aborts. Splits the byte + * stream into SSE records on the blank line boundary; the payload rides in + * the data lines as a JSON envelope with its own model, event and data fields. + */ + static async watchModelEvents( + signal: AbortSignal, + onEvent: (event: ApiModelsSseEvent) => void + ): Promise { + const decoder = new TextDecoder(); + + while (!signal.aborted) { + try { + const response = await fetch(`${base}${API_MODELS.SSE}`, { + headers: getAuthHeaders(), + signal + }); + + if (response.ok && response.body) { + const reader = response.body.getReader(); + + let buffer = ''; + + while (!signal.aborted) { + const { done, value } = await reader.read(); + + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + const { records, rest } = splitSseRecords(buffer); + + buffer = rest; + + for (const record of records) { + const event = ModelsService.parseStatusRecord(record); + + if (event) onEvent(event); + } + } + } + } catch { + // network drop or abort falls through to the reconnect delay + } + + if (signal.aborted) return; + + await new Promise((resolve) => setTimeout(resolve, ModelsService.SSE_RECONNECT_MS)); + } + } + + /** + * Parse one SSE record into its JSON envelope, or null when the record + * carries no data payload or malformed JSON. + */ + private static parseStatusRecord(record: string): ApiModelsSseEvent | null { + const payload = extractSseDataPayload(record); + + if (payload.length === 0) return null; + + try { + return JSON.parse(payload) as ApiModelsSseEvent; + } catch { + return null; + } + } } diff --git a/tools/ui/src/lib/services/parameter-sync.service.ts b/tools/ui/src/lib/services/parameter-sync.service.ts index 0ed9ebd48c..467e7c2dbd 100644 --- a/tools/ui/src/lib/services/parameter-sync.service.ts +++ b/tools/ui/src/lib/services/parameter-sync.service.ts @@ -1,3 +1,11 @@ +/** + * ParameterSyncService - Syncs sampling parameters with the server + * + * Decides for each sampling parameter whether the user's setting is an + * override of the server default, and normalizes floating-point values. + * No reactive state; consumed by settingsStore. + */ + import { SETTINGS_KEYS, SYNCABLE_PARAMETERS } from '$lib/constants'; import { ParameterSource, SyncableParameterType } from '$lib/enums'; import type { ParameterInfo, ParameterRecord, ParameterValue } from '$lib/types'; @@ -5,22 +13,47 @@ import { normalizeFloatingPoint } from '$lib/utils'; export class ParameterSyncService { /** + * Check if a parameter can be synced from server. * - * - * Extraction - * - * + * @param key - The parameter key to check + * @returns True if the parameter is in the syncable parameters list */ + static canSyncParameter(key: string): boolean { + return SYNCABLE_PARAMETERS.some((param) => param.key === key && param.canSync); + } /** - * Round floating-point numbers to avoid JavaScript precision issues. - * E.g., 0.1 + 0.2 = 0.30000000000000004 → 0.3 + * Create a diff between current settings and server defaults. + * Shows which parameters differ from server values, useful for debugging + * and for the "Reset to defaults" functionality. * - * @param value - Parameter value to normalize - * @returns Precision-normalized value + * @param currentSettings - Current parameter values in the settings store + * @param serverDefaults - Default values extracted from server props + * @returns Record of parameter diffs with current value, server value, and whether they differ */ - private static roundFloatingPoint(value: ParameterValue): ParameterValue { - return normalizeFloatingPoint(value) as ParameterValue; + static createParameterDiff( + currentSettings: ParameterRecord, + serverDefaults: ParameterRecord + ): Record { + const diff: Record< + string, + { current: ParameterValue; server: ParameterValue; differs: boolean } + > = {}; + + for (const key of this.getSyncableParameterKeys()) { + const currentValue = currentSettings[key]; + const serverValue = serverDefaults[key]; + + if (serverValue !== undefined) { + diff[key] = { + current: currentValue, + differs: currentValue !== serverValue, + server: serverValue + }; + } + } + + return diff; } /** @@ -59,49 +92,6 @@ export class ParameterSyncService { return extracted; } - /** - * - * - * Merging - * - * - */ - - /** - * Merge server defaults with current user settings. - * User overrides always take priority — only parameters not in `userOverrides` - * set will be updated from server defaults. - * - * @param currentSettings - Current parameter values in the settings store - * @param serverDefaults - Default values extracted from server props - * @param userOverrides - Set of parameter keys explicitly overridden by the user - * @returns Merged parameter record with user overrides preserved - */ - static mergeWithServerDefaults( - currentSettings: ParameterRecord, - serverDefaults: ParameterRecord, - userOverrides: Set = new Set() - ): ParameterRecord { - const merged = { ...currentSettings }; - - for (const [key, serverValue] of Object.entries(serverDefaults)) { - // Only update if user hasn't explicitly overridden this parameter - if (!userOverrides.has(key)) { - merged[key] = this.roundFloatingPoint(serverValue); - } - } - - return merged; - } - - /** - * - * - * Info - * - * - */ - /** * Get parameter information including source and values. * Used by SettingsChatParameterSourceIndicator to display the correct badge @@ -132,16 +122,6 @@ export class ParameterSyncService { }; } - /** - * Check if a parameter can be synced from server. - * - * @param key - The parameter key to check - * @returns True if the parameter is in the syncable parameters list - */ - static canSyncParameter(key: string): boolean { - return SYNCABLE_PARAMETERS.some((param) => param.key === key && param.canSync); - } - /** * Get all syncable parameter keys. * @@ -151,6 +131,33 @@ export class ParameterSyncService { return SYNCABLE_PARAMETERS.filter((param) => param.canSync).map((param) => param.key); } + /** + * Merge server defaults with current user settings. + * User overrides always take priority — only parameters not in `userOverrides` + * set will be updated from server defaults. + * + * @param currentSettings - Current parameter values in the settings store + * @param serverDefaults - Default values extracted from server props + * @param userOverrides - Set of parameter keys explicitly overridden by the user + * @returns Merged parameter record with user overrides preserved + */ + static mergeWithServerDefaults( + currentSettings: ParameterRecord, + serverDefaults: ParameterRecord, + userOverrides: Set = new Set() + ): ParameterRecord { + const merged = { ...currentSettings }; + + for (const [key, serverValue] of Object.entries(serverDefaults)) { + // Only update if user hasn't explicitly overridden this parameter + if (!userOverrides.has(key)) { + merged[key] = this.roundFloatingPoint(serverValue); + } + } + + return merged; + } + /** * Validate a server parameter value against its expected type. * @@ -176,44 +183,13 @@ export class ParameterSyncService { } /** + * Round floating-point numbers to avoid JavaScript precision issues. + * E.g., 0.1 + 0.2 = 0.30000000000000004 → 0.3 * - * - * Diff - * - * + * @param value - Parameter value to normalize + * @returns Precision-normalized value */ - - /** - * Create a diff between current settings and server defaults. - * Shows which parameters differ from server values, useful for debugging - * and for the "Reset to defaults" functionality. - * - * @param currentSettings - Current parameter values in the settings store - * @param serverDefaults - Default values extracted from server props - * @returns Record of parameter diffs with current value, server value, and whether they differ - */ - static createParameterDiff( - currentSettings: ParameterRecord, - serverDefaults: ParameterRecord - ): Record { - const diff: Record< - string, - { current: ParameterValue; server: ParameterValue; differs: boolean } - > = {}; - - for (const key of this.getSyncableParameterKeys()) { - const currentValue = currentSettings[key]; - const serverValue = serverDefaults[key]; - - if (serverValue !== undefined) { - diff[key] = { - current: currentValue, - differs: currentValue !== serverValue, - server: serverValue - }; - } - } - - return diff; + private static roundFloatingPoint(value: ParameterValue): ParameterValue { + return normalizeFloatingPoint(value) as ParameterValue; } } diff --git a/tools/ui/src/lib/services/props.service.ts b/tools/ui/src/lib/services/props.service.ts index 46f4915fad..488a67b641 100644 --- a/tools/ui/src/lib/services/props.service.ts +++ b/tools/ui/src/lib/services/props.service.ts @@ -1,14 +1,14 @@ +/** + * PropsService - Fetches server properties from /props + * + * Returns global server settings and capabilities, including per-model + * modalities in MODEL mode. No reactive state; consumed by serverStore and + * the model props manager. + */ + import { apiFetchWithParams } from '$lib/utils'; export class PropsService { - /** - * - * - * Fetching - * - * - */ - /** * Fetches global server properties from the `/props` endpoint. * In MODEL mode, returns modalities for the single loaded model. diff --git a/tools/ui/src/lib/services/read-media.service.ts b/tools/ui/src/lib/services/read-media.service.ts index 8de9bbbea5..2858795e8b 100644 --- a/tools/ui/src/lib/services/read-media.service.ts +++ b/tools/ui/src/lib/services/read-media.service.ts @@ -1,3 +1,10 @@ +/** + * ReadMediaService - Reads local media files for the read_media tool + * + * Encodes image and audio files as base64 data URLs with the metadata the + * model needs. No reactive state; consumed by toolsStore. + */ + import { ToolsService } from './tools.service'; import { FILE_EXTENSION_SEPARATOR, @@ -40,7 +47,7 @@ function fileExtension(path: string): string { * actually use the result - the server has no idea which model is selected. * * @see buildReadMediaToolDefinition in constants/read-media.ts - tool schema sent to the LLM - * @see agenticStore in stores/agentic.svelte.ts - tool dispatch and attachment extraction + * @see agenticStore in stores/agentic/index.svelte.ts - tool dispatch and attachment extraction */ export class ReadMediaService { static async executeTool( diff --git a/tools/ui/src/lib/services/router.service.ts b/tools/ui/src/lib/services/router.service.ts index 59de4cb6fe..217de38f3a 100644 --- a/tools/ui/src/lib/services/router.service.ts +++ b/tools/ui/src/lib/services/router.service.ts @@ -1,3 +1,10 @@ +/** + * RouterService - Builds app route paths + * + * Returns chat and settings route strings from a single source of truth + * (ROUTES). No state. + */ + import { ROUTES } from '$lib/constants'; export class RouterService { diff --git a/tools/ui/src/lib/services/sandbox-harness.ts b/tools/ui/src/lib/services/sandbox-harness.ts index 189ff59a5e..29f9ad2a56 100644 --- a/tools/ui/src/lib/services/sandbox-harness.ts +++ b/tools/ui/src/lib/services/sandbox-harness.ts @@ -1,3 +1,10 @@ +/** + * Sandbox harness - builds the srcdoc document for the sandboxed iframe + * + * Produces the HTML/CSP/worker shim that runs untrusted model code in an + * opaque origin. Consumed by sandbox.service. + */ + import WORKER_SHIM from './sandbox-worker.js?raw'; import { NEWLINE } from '$lib/constants'; diff --git a/tools/ui/src/lib/services/sandbox.service.ts b/tools/ui/src/lib/services/sandbox.service.ts index 27da9d2634..bdc63e4edf 100644 --- a/tools/ui/src/lib/services/sandbox.service.ts +++ b/tools/ui/src/lib/services/sandbox.service.ts @@ -1,3 +1,11 @@ +/** + * SandboxService - Runs untrusted code in a sandboxed worker + * + * Executes model-generated code inside a CSP-restricted, opaque-origin + * iframe worker with output and timeout limits. No reactive state; consumed + * by toolsStore for code-execution tools. + */ + import { buildSandboxHarness } from './sandbox-harness'; import { NEWLINE, @@ -8,7 +16,7 @@ import { SANDBOX_TOOL_NAME, SANDBOX_TRUNCATION_NOTICE } from '$lib/constants'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { ToolExecutionResult } from '$lib/types'; /** Cached harnesses keyed by whether nerdamer is included. */ diff --git a/tools/ui/src/lib/services/tools.service.ts b/tools/ui/src/lib/services/tools.service.ts index 2b3a2c0dc7..78229756ce 100644 --- a/tools/ui/src/lib/services/tools.service.ts +++ b/tools/ui/src/lib/services/tools.service.ts @@ -1,3 +1,10 @@ +/** + * ToolsService - Stateless server tools API layer + * + * Fetches the server's /tools listing and streams tool execution results. + * No reactive state; consumed by toolsStore. + */ + import { base } from '$app/paths'; import { API_TOOLS, HEADERS } from '$lib/constants'; import { ToolResponseField } from '$lib/enums'; @@ -7,15 +14,6 @@ import { getJsonHeaders } from '$lib/utils/api-headers'; import { parseSseJsonStream, type SseJsonEvent } from '$lib/utils/sse'; export class ToolsService { - /** - * Fetch the list of server tools from the server. - * - * @returns Array of tool definitions in OpenAI-compatible format - */ - static async list(): Promise { - return apiFetch(API_TOOLS.LIST); - } - /** * Execute a server tool on the server. * @@ -76,6 +74,15 @@ export class ToolsService { }); } + /** + * Fetch the list of server tools from the server. + * + * @returns Array of tool definitions in OpenAI-compatible format + */ + static async list(): Promise { + return apiFetch(API_TOOLS.LIST); + } + /** * Stream a server tool's output chunks from the server. The server * `POST /tools` endpoint with `{stream: true}` emits `data: {"chunk": "..."}` diff --git a/tools/ui/src/lib/stores/agentic/gates.svelte.ts b/tools/ui/src/lib/stores/agentic/gates.svelte.ts new file mode 100644 index 0000000000..6b52fa3aff --- /dev/null +++ b/tools/ui/src/lib/stores/agentic/gates.svelte.ts @@ -0,0 +1,208 @@ +/** + * AgenticGates - User interaction gates for the agentic loop + * + * Owns the state the loop waits on between turns: tool permission requests, + * turn-limit continue prompts and queued steering messages. The loop awaits + * requestPermission/requestContinue; the UI resolves them through + * resolvePermission/resolveContinue. Owned by agenticStore, no host coupling. + */ + +import { ToolPermissionDecision } from '$lib/enums'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { permissionsStore } from '$lib/stores/permissions.svelte'; +import { toolsStore } from '$lib/stores/tools.svelte'; +import type { DatabaseMessageExtra, SteeringMessage } from '$lib/types'; +import { SvelteMap } from 'svelte/reactivity'; + +export class AgenticGates { + /** Resolve functions for pending continue Promises; nothing derives from this map */ + private continueResolvers = new SvelteMap void>(); + /** Dedicated reactive state for pending continue requests (turn limit reached) */ + private pendingContinueRequests = new SvelteMap(); + + /** Dedicated reactive state for pending permission requests (ensures immediate UI updates) */ + private pendingPermissions = new SvelteMap< + string, + { toolName: string; serverLabel: string } | null + >(); + /** Resolve functions for pending permission Promises; nothing derives from this map */ + private permissionResolvers = new SvelteMap void>(); + + /** Reactive: queued steering messages to inject between turns */ + private steeringMessages = new SvelteMap(); + + /** + * Drop all pending gate state for a conversation, e.g. when a flow exits. + */ + clear(conversationId: string): void { + this.pendingPermissions.set(conversationId, null); + this.permissionResolvers.delete(conversationId); + this.pendingContinueRequests.set(conversationId, false); + this.continueResolvers.delete(conversationId); + this.steeringMessages.delete(conversationId); + } + + /** + * Clear the pending steering message without consuming it. + */ + clearSteeringMessage(conversationId: string): void { + this.steeringMessages.delete(conversationId); + } + + /** + * Consume and return the pending steering message for re-sending. + * Called by chatStore after the agentic flow exits. + */ + consumePendingSteeringMessage(conversationId: string): SteeringMessage | null { + const msg = this.steeringMessages.get(conversationId); + + if (!msg) return null; + + this.steeringMessages.delete(conversationId); + + return msg; + } + + getPendingContinueRequest(conversationId: string): boolean { + return this.pendingContinueRequests.get(conversationId) ?? false; + } + + getPendingPermissionRequest( + conversationId: string + ): { toolName: string; serverLabel: string } | null { + return this.pendingPermissions.get(conversationId) ?? null; + } + + getPendingSteeringMessageContent(conversationId: string): string | null { + return this.steeringMessages.get(conversationId)?.content ?? null; + } + + getPendingSteeringMessageExtras(conversationId: string): DatabaseMessageExtra[] | undefined { + return this.steeringMessages.get(conversationId)?.extras; + } + + hasPendingSteeringMessage(conversationId: string): boolean { + return this.steeringMessages.has(conversationId); + } + + /** + * Queue a steering message. When the current agentic turn completes, + * the flow exits and the caller re-sends the message as a normal chat message. + */ + injectSteeringMessage( + conversationId: string, + content: string, + extras?: DatabaseMessageExtra[] + ): void { + this.steeringMessages.set(conversationId, { content, extras }); + } + + async requestContinue(conversationId: string, signal?: AbortSignal): Promise { + this.pendingContinueRequests.set(conversationId, true); + + return new Promise((resolve) => { + if (signal?.aborted) { + this.pendingContinueRequests.set(conversationId, false); + resolve(false); + + return; + } + + this.continueResolvers.set(conversationId, (shouldContinue) => { + this.pendingContinueRequests.set(conversationId, false); + resolve(shouldContinue); + }); + + signal?.addEventListener( + 'abort', + () => { + const resolver = this.continueResolvers.get(conversationId); + + if (resolver) { + this.continueResolvers.delete(conversationId); + this.pendingContinueRequests.set(conversationId, false); + resolve(false); + } + }, + { once: true } + ); + }); + } + + async requestPermission( + conversationId: string, + toolName: string, + serverLabel: string, + signal?: AbortSignal + ): Promise { + const permissionKey = toolsStore.getPermissionKey(toolName); + + if (permissionKey && permissionsStore.hasTool(permissionKey)) { + return ToolPermissionDecision.ONCE; + } + + this.pendingPermissions.set(conversationId, { serverLabel, toolName }); + + return new Promise((resolve) => { + if (signal?.aborted) { + this.pendingPermissions.set(conversationId, null); + resolve(ToolPermissionDecision.DENY); + + return; + } + + this.permissionResolvers.set(conversationId, (decision) => { + this.pendingPermissions.set(conversationId, null); + + if (decision === ToolPermissionDecision.ALWAYS && permissionKey) { + permissionsStore.allowTool(permissionKey); + } else if (decision === ToolPermissionDecision.ALWAYS_SERVER) { + const serverToolKeys = toolsStore.allTools + .filter((t) => + t.serverName + ? t.serverName === serverLabel + : toolsStore.getToolServerLabel(t.definition.function.name) === serverLabel + ) + .map((t) => toolsStore.getPermissionKey(t.definition.function.name)!) + .filter((k): k is string => k !== null); + + permissionsStore.allowTools(serverToolKeys); + } + + resolve(decision); + }); + + signal?.addEventListener( + 'abort', + () => { + const resolver = this.permissionResolvers.get(conversationId); + + if (resolver) { + this.permissionResolvers.delete(conversationId); + this.pendingPermissions.set(conversationId, null); + resolve(ToolPermissionDecision.DENY); + } + }, + { once: true } + ); + }); + } + + resolveContinue(conversationId: string, shouldContinue: boolean): void { + const resolver = this.continueResolvers.get(conversationId); + + if (resolver) { + this.continueResolvers.delete(conversationId); + resolver(shouldContinue); + } + } + + resolvePermission(conversationId: string, decision: ToolPermissionDecision): void { + const resolver = this.permissionResolvers.get(conversationId); + + if (resolver) { + this.permissionResolvers.delete(conversationId); + resolver(decision); + } + } +} diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic/index.svelte.ts similarity index 77% rename from tools/ui/src/lib/stores/agentic.svelte.ts rename to tools/ui/src/lib/stores/agentic/index.svelte.ts index d2a2ea8871..a91e0ba46f 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic/index.svelte.ts @@ -1,23 +1,13 @@ /** - * agenticStore - Reactive State Store for Agentic Loop Orchestration + * AgenticStore - Multi-turn agentic loop orchestration * - * Manages multi-turn agentic loop with MCP tools: - * - LLM streaming with tool call detection - * - Tool execution via mcpStore - * - Session state management - * - Turn limit enforcement + * Drives the agentic loop over MCP tools: streams each LLM turn, detects + * tool calls, executes them via mcpStore, and enforces the turn limit. Each + * turn produces one assistant message (with tool_calls) and one tool result + * message per executed call, persisted as separate DB rows. * - * Each agentic turn produces separate DB messages: - * - One assistant message per LLM turn (with tool_calls if any) - * - One tool result message per tool call execution - * - * **Architecture & Relationships:** - * - **ChatService**: Stateless API layer (sendMessage, streaming) - * - **mcpStore**: MCP connection management and tool execution - * - **agenticStore** (this): Reactive state + business logic - * - * @see ChatService in services/chat.service.ts for API operations - * @see mcpStore in stores/mcp.svelte.ts for MCP operations + * Uses ChatService for streaming and mcpStore for tool execution; waits on + * the permission/continue/steering gates owned by {@link AgenticGates}. */ import { DEFAULT_AGENTIC_CONFIG, NEWLINE } from '$lib/constants'; @@ -43,11 +33,11 @@ import { ReadMediaService } from '$lib/services/read-media.service'; import { SandboxService } from '$lib/services/sandbox.service'; import { ToolsService } from '$lib/services/tools.service'; // direct imports between stores, not via the barrel, to avoid circular deps -import { conversationsStore } from '$lib/stores/conversations.svelte'; -import { mcpStore } from '$lib/stores/mcp.svelte'; -import { modelsStore } from '$lib/stores/models.svelte'; -import { permissionsStore } from '$lib/stores/permissions.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { AgenticGates } from '$lib/stores/agentic/gates.svelte'; +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import { mcpStore } from '$lib/stores/mcp/index.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { toolsStore } from '$lib/stores/tools.svelte'; import type { AgenticConfig, @@ -152,160 +142,45 @@ function toAgenticMessages(messages: ApiChatMessageData[]): AgenticMessage[] { } class AgenticStore { - private _sessions = new SvelteMap(); - /** Dedicated reactive state for pending permission requests (ensures immediate UI updates) */ - private _pendingPermissions = new SvelteMap< - string, - { toolName: string; serverLabel: string } | null - >(); - /** Non-reactive: stores resolve functions for pending permission Promises */ - private _permissionResolvers = new Map void>(); + // permission, continue and steering gates the loop waits on between turns + private gates = new AgenticGates(); + private sessions = new SvelteMap(); - /** Dedicated reactive state for pending continue requests (turn limit reached) */ - private _pendingContinueRequests = new SvelteMap(); - /** Non-reactive: stores resolve functions for pending continue Promises */ - private _continueResolvers = new Map void>(); - - /** Reactive: queued steering messages to inject between turns */ - private _steeringMessages = new SvelteMap(); - - get isReady(): boolean { - return true; - } get isAnyRunning(): boolean { - for (const session of this._sessions.values()) { + for (const session of this.sessions.values()) { if (session.isRunning) return true; } return false; } - getSession(conversationId: string): AgenticSession { - let session = this._sessions.get(conversationId); - - if (!session) { - session = createDefaultSession(); - this._sessions.set(conversationId, session); - } - - return session; - } - - private updateSession(conversationId: string, update: Partial): void { - const session = this.getSession(conversationId); - - this._sessions.set(conversationId, { ...session, ...update }); - } - - clearSession(conversationId: string): void { - this._sessions.delete(conversationId); - } - - getActiveSessions(): Array<{ conversationId: string; session: AgenticSession }> { - const active: Array<{ conversationId: string; session: AgenticSession }> = []; - - for (const [conversationId, session] of this._sessions.entries()) { - if (session.isRunning) active.push({ conversationId, session }); - } - - return active; - } - - isRunning(conversationId: string): boolean { - return this._sessions.get(conversationId)?.isRunning ?? false; - } - - // read-only: safe to call from derivations, unlike getSession - getLiveLlmTotals(conversationId: string): AgenticSession['liveLlm'] { - return this._sessions.get(conversationId)?.liveLlm ?? null; - } - - // read-only: safe to call from derivations, unlike getSession - getFlowRootMessageId(conversationId: string): string | null { - return this._sessions.get(conversationId)?.flowRootMessageId ?? null; - } - - currentTurn(conversationId: string): number { - return this._sessions.get(conversationId)?.currentTurn ?? 0; - } - - totalToolCalls(conversationId: string): number { - return this._sessions.get(conversationId)?.totalToolCalls ?? 0; - } - - lastError(conversationId: string): Error | null { - return this._sessions.get(conversationId)?.lastError ?? null; - } - - streamingToolCall(conversationId: string): { name: string; arguments: string } | null { - return this._sessions.get(conversationId)?.streamingToolCall ?? null; - } - - executingToolCallId(conversationId: string): string | null { - return this._sessions.get(conversationId)?.executingToolCallId ?? null; - } - - pendingPermissionRequest( - conversationId: string - ): { toolName: string; serverLabel: string } | null { - return this._pendingPermissions.get(conversationId) ?? null; - } - - pendingContinueRequest(conversationId: string): boolean { - return this._pendingContinueRequests.get(conversationId) ?? false; - } - - resolveContinue(conversationId: string, shouldContinue: boolean): void { - const resolver = this._continueResolvers.get(conversationId); - - if (resolver) { - this._continueResolvers.delete(conversationId); - resolver(shouldContinue); - } - } - - resolvePermission(conversationId: string, decision: ToolPermissionDecision): void { - const resolver = this._permissionResolvers.get(conversationId); - - if (resolver) { - this._permissionResolvers.delete(conversationId); - resolver(decision); - } + get isReady(): boolean { + return true; } clearError(conversationId: string): void { this.updateSession(conversationId, { lastError: null }); } - hasPendingSteeringMessage(conversationId: string): boolean { - return this._steeringMessages.has(conversationId); - } - - pendingSteeringMessageContent(conversationId: string): string | null { - return this._steeringMessages.get(conversationId)?.content ?? null; - } - - pendingSteeringMessageExtras(conversationId: string): DatabaseMessageExtra[] | undefined { - return this._steeringMessages.get(conversationId)?.extras; - } - - /** - * Queue a steering message. When the current agentic turn completes, - * the flow exits and the caller re-sends the message as a normal chat message. - */ - injectSteeringMessage( - conversationId: string, - content: string, - extras?: DatabaseMessageExtra[] - ): void { - this._steeringMessages.set(conversationId, { content, extras }); + clearSession(conversationId: string): void { + this.sessions.delete(conversationId); } /** * Clear the pending steering message without consuming it. */ clearSteeringMessage(conversationId: string): void { - this._steeringMessages.delete(conversationId); + this.gates.clearSteeringMessage(conversationId); + } + + constructor() { + // drop per-conversation session state when the conversation is deleted, + // otherwise every conversation that ever ran a flow leaks a session here + conversationsStore.onConversationsDeleted((convIds) => { + for (const convId of convIds) { + this.sessions.delete(convId); + } + }); } /** @@ -313,13 +188,17 @@ class AgenticStore { * Called by chatStore after the agentic flow exits. */ consumePendingSteeringMessage(conversationId: string): SteeringMessage | null { - const msg = this._steeringMessages.get(conversationId); + return this.gates.consumePendingSteeringMessage(conversationId); + } - if (!msg) return null; + getActiveSessions(): Array<{ conversationId: string; session: AgenticSession }> { + const active: Array<{ conversationId: string; session: AgenticSession }> = []; - this._steeringMessages.delete(conversationId); + for (const [conversationId, session] of this.sessions.entries()) { + if (session.isRunning) active.push({ conversationId, session }); + } - return msg; + return active; } getConfig(settings: SettingsConfigType, perChatOverrides?: McpServerOverride[]): AgenticConfig { @@ -336,105 +215,91 @@ class AgenticStore { }; } - private parseToolArguments(args: string | Record): Record { - if (typeof args === 'object') return args; - - const trimmed = args.trim(); - - if (trimmed === '') return {}; - - return JSON.parse(trimmed) as Record; + getCurrentTurn(conversationId: string): number { + return this.sessions.get(conversationId)?.currentTurn ?? 0; } - private async requestPermission( - conversationId: string, - toolName: string, - serverLabel: string, - signal?: AbortSignal - ): Promise { - const permissionKey = toolsStore.getPermissionKey(toolName); + getExecutingToolCallId(conversationId: string): string | null { + return this.sessions.get(conversationId)?.executingToolCallId ?? null; + } - if (permissionKey && permissionsStore.hasTool(permissionKey)) { - return ToolPermissionDecision.ONCE; + // read-only: safe to call from derivations, unlike getSession + getFlowRootMessageId(conversationId: string): string | null { + return this.sessions.get(conversationId)?.flowRootMessageId ?? null; + } + + getLastError(conversationId: string): Error | null { + return this.sessions.get(conversationId)?.lastError ?? null; + } + + // read-only: safe to call from derivations, unlike getSession + getLiveLlmTotals(conversationId: string): AgenticSession['liveLlm'] { + return this.sessions.get(conversationId)?.liveLlm ?? null; + } + + getPendingContinueRequest(conversationId: string): boolean { + return this.gates.getPendingContinueRequest(conversationId); + } + + getPendingPermissionRequest( + conversationId: string + ): { toolName: string; serverLabel: string } | null { + return this.gates.getPendingPermissionRequest(conversationId); + } + + getPendingSteeringMessageContent(conversationId: string): string | null { + return this.gates.getPendingSteeringMessageContent(conversationId); + } + + getPendingSteeringMessageExtras(conversationId: string): DatabaseMessageExtra[] | undefined { + return this.gates.getPendingSteeringMessageExtras(conversationId); + } + + getSession(conversationId: string): AgenticSession { + let session = this.sessions.get(conversationId); + + if (!session) { + session = createDefaultSession(); + this.sessions.set(conversationId, session); } - this._pendingPermissions.set(conversationId, { serverLabel, toolName }); - - return new Promise((resolve) => { - if (signal?.aborted) { - this._pendingPermissions.set(conversationId, null); - resolve(ToolPermissionDecision.DENY); - - return; - } - - this._permissionResolvers.set(conversationId, (decision) => { - this._pendingPermissions.set(conversationId, null); - - if (decision === ToolPermissionDecision.ALWAYS && permissionKey) { - permissionsStore.allowTool(permissionKey); - } else if (decision === ToolPermissionDecision.ALWAYS_SERVER) { - const serverToolKeys = toolsStore.allTools - .filter((t) => - t.serverName - ? t.serverName === serverLabel - : toolsStore.getToolServerLabel(t.definition.function.name) === serverLabel - ) - .map((t) => toolsStore.getPermissionKey(t.definition.function.name)!) - .filter((k): k is string => k !== null); - - permissionsStore.allowTools(serverToolKeys); - } - - resolve(decision); - }); - - signal?.addEventListener( - 'abort', - () => { - const resolver = this._permissionResolvers.get(conversationId); - - if (resolver) { - this._permissionResolvers.delete(conversationId); - this._pendingPermissions.set(conversationId, null); - resolve(ToolPermissionDecision.DENY); - } - }, - { once: true } - ); - }); + return session; } - private async requestContinue(conversationId: string, signal?: AbortSignal): Promise { - this._pendingContinueRequests.set(conversationId, true); + getStreamingToolCall(conversationId: string): { name: string; arguments: string } | null { + return this.sessions.get(conversationId)?.streamingToolCall ?? null; + } - return new Promise((resolve) => { - if (signal?.aborted) { - this._pendingContinueRequests.set(conversationId, false); - resolve(false); + getTotalToolCalls(conversationId: string): number { + return this.sessions.get(conversationId)?.totalToolCalls ?? 0; + } - return; - } + hasPendingSteeringMessage(conversationId: string): boolean { + return this.gates.hasPendingSteeringMessage(conversationId); + } - this._continueResolvers.set(conversationId, (shouldContinue) => { - this._pendingContinueRequests.set(conversationId, false); - resolve(shouldContinue); - }); + /** + * Queue a steering message. When the current agentic turn completes, + * the flow exits and the caller re-sends the message as a normal chat message. + */ + injectSteeringMessage( + conversationId: string, + content: string, + extras?: DatabaseMessageExtra[] + ): void { + this.gates.injectSteeringMessage(conversationId, content, extras); + } - signal?.addEventListener( - 'abort', - () => { - const resolver = this._continueResolvers.get(conversationId); + isRunning(conversationId: string): boolean { + return this.sessions.get(conversationId)?.isRunning ?? false; + } - if (resolver) { - this._continueResolvers.delete(conversationId); - this._pendingContinueRequests.set(conversationId, false); - resolve(false); - } - }, - { once: true } - ); - }); + resolveContinue(conversationId: string, shouldContinue: boolean): void { + this.gates.resolveContinue(conversationId, shouldContinue); + } + + resolvePermission(conversationId: string, decision: ToolPermissionDecision): void { + this.gates.resolvePermission(conversationId, decision); } async runAgenticFlow(params: AgenticFlowParams): Promise { @@ -449,11 +314,7 @@ class AgenticStore { } = params; // Clear any pending permissions/continue requests for this conversation when starting a new flow - this._pendingPermissions.set(conversationId, null); - this._permissionResolvers.delete(conversationId); - this._pendingContinueRequests.set(conversationId, false); - this._continueResolvers.delete(conversationId); - this._steeringMessages.delete(conversationId); + this.gates.clear(conversationId); // Ensure server tools are fetched before checking if agentic is enabled if (toolsStore.serverTools.length === 0 && !toolsStore.loading) { @@ -482,26 +343,8 @@ class AgenticStore { console.log(`[AgenticStore] Starting agentic flow with ${tools.length} tools`); - const normalizedMessages: ApiChatMessageData[] = ( - await Promise.all( - messages.map((msg) => { - if ('id' in msg && 'convId' in msg && 'timestamp' in msg) - return ChatService.convertDbMessageToApiChatMessageData( - msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] } - ); - - return msg as ApiChatMessageData; - }) - ) - ).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => { - if (msg.role === MessageRole.SYSTEM) { - const content = typeof msg.content === 'string' ? msg.content : ''; - - return content.trim().length > 0; - } - - return true; - }); + const normalizedMessages: ApiChatMessageData[] = + await ChatService.normalizeMessagesForApi(messages); this.updateSession(conversationId, { currentTurn: 0, @@ -550,6 +393,30 @@ class AgenticStore { } } + private buildAttachmentName(mimeType: string, index: number): string { + const extension = mimeType.startsWith(MimeTypePrefix.AUDIO) + ? (AUDIO_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_AUDIO_EXTENSION) + : (IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION); + + return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; + } + + private buildFinalTimings( + capturedTimings: ChatMessageTimings | undefined, + agenticTimings: ChatMessageAgenticTimings + ): ChatMessageTimings | undefined { + if (agenticTimings.toolCallsCount === 0) return capturedTimings; + + return { + agentic: agenticTimings, + cache_n: capturedTimings?.cache_n, + predicted_ms: capturedTimings?.predicted_ms, + predicted_n: capturedTimings?.predicted_n, + prompt_ms: capturedTimings?.prompt_ms, + prompt_n: capturedTimings?.prompt_n + }; + } + private async executeAgenticLoop(params: { conversationId: string; messages: ApiChatMessageData[]; @@ -596,7 +463,7 @@ class AgenticStore { while (true) { if (turn >= maxTurns) { // Turn limit reached - ask user whether to continue - const shouldContinue = await this.requestContinue(conversationId, signal); + const shouldContinue = await this.gates.requestContinue(conversationId, signal); // Yield to allow Svelte to flush the UI update await new Promise((r) => setTimeout(r, 0)); @@ -769,7 +636,7 @@ class AgenticStore { // === Steering check: if a user message was queued during this turn, exit the flow. // The caller (chatStore) will consume the pending message and re-send it normally. - if (this._steeringMessages.has(conversationId)) { + if (this.gates.hasPendingSteeringMessage(conversationId)) { console.log('[AgenticStore] Steering message detected after turn, exiting agentic flow'); await onAssistantTurnComplete?.( turnContent, @@ -847,7 +714,7 @@ class AgenticStore { } // Check for pending steering message - skip remaining tool calls - if (this._steeringMessages.has(conversationId)) { + if (this.gates.hasPendingSteeringMessage(conversationId)) { console.log( `[AgenticStore] Steering message detected, skipping ${normalizedCalls.length - i} remaining tool call(s)` ); @@ -872,7 +739,7 @@ class AgenticStore { const toolName = toolCall.function.name; const serverLabel = toolsStore.getToolServerLabel(toolName); // Ask for permission before executing the tool - const permission = await this.requestPermission( + const permission = await this.gates.requestPermission( conversationId, toolName, serverLabel, @@ -959,8 +826,8 @@ class AgenticStore { executionResult = await ReadMediaService.executeTool( args, { - audio: modelsStore.modelSupportsAudio(effectiveModel), - vision: modelsStore.modelSupportsVision(effectiveModel) + audio: modelsStore.props.modelSupportsAudio(effectiveModel), + vision: modelsStore.props.modelSupportsVision(effectiveModel) }, signal, conversationsStore.activeConversation?.cwd @@ -1058,7 +925,7 @@ class AgenticStore { for (const attachment of attachments) { if (attachment.type === AttachmentType.AUDIO) { - if (modelsStore.modelSupportsAudio(effectiveModel)) { + if (modelsStore.props.modelSupportsAudio(effectiveModel)) { contentParts.push({ input_audio: { data: (attachment as DatabaseMessageExtraAudioFile).base64Data, @@ -1070,7 +937,7 @@ class AgenticStore { }); } } else if (attachment.type === AttachmentType.IMAGE) { - if (modelsStore.modelSupportsVision(effectiveModel)) { + if (modelsStore.props.modelSupportsVision(effectiveModel)) { contentParts.push({ image_url: { url: (attachment as DatabaseMessageExtraImageFile).base64Url @@ -1101,7 +968,7 @@ class AgenticStore { } // If tools were interrupted by a steering message, exit now instead of starting another LLM turn - if (this._steeringMessages.has(conversationId)) { + if (this.gates.hasPendingSteeringMessage(conversationId)) { console.log( '[AgenticStore] Steering message detected after tool execution, exiting agentic flow' ); @@ -1114,35 +981,6 @@ class AgenticStore { } } - private buildFinalTimings( - capturedTimings: ChatMessageTimings | undefined, - agenticTimings: ChatMessageAgenticTimings - ): ChatMessageTimings | undefined { - if (agenticTimings.toolCallsCount === 0) return capturedTimings; - - return { - agentic: agenticTimings, - cache_n: capturedTimings?.cache_n, - predicted_ms: capturedTimings?.predicted_ms, - predicted_n: capturedTimings?.predicted_n, - prompt_ms: capturedTimings?.prompt_ms, - prompt_n: capturedTimings?.prompt_n - }; - } - - private normalizeToolCalls(toolCalls: ApiChatCompletionToolCall[]): AgenticToolCallList { - if (!toolCalls) return []; - - return toolCalls.map((call, index) => ({ - function: { - arguments: call?.function?.arguments ?? '', - name: call?.function?.name ?? '' - }, - id: call?.id ?? `tool_${index}`, - type: (call?.type as ToolCallType.FUNCTION) ?? ToolCallType.FUNCTION - })); - } - private extractBase64Attachments(result: string): { cleanedResult: string; attachments: DatabaseMessageExtra[]; @@ -1198,12 +1036,33 @@ class AgenticStore { return { attachments, cleanedResult: cleanedLines.join(NEWLINE) }; } - private buildAttachmentName(mimeType: string, index: number): string { - const extension = mimeType.startsWith(MimeTypePrefix.AUDIO) - ? (AUDIO_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_AUDIO_EXTENSION) - : (IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION); + private normalizeToolCalls(toolCalls: ApiChatCompletionToolCall[]): AgenticToolCallList { + if (!toolCalls) return []; - return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; + return toolCalls.map((call, index) => ({ + function: { + arguments: call?.function?.arguments ?? '', + name: call?.function?.name ?? '' + }, + id: call?.id ?? `tool_${index}`, + type: (call?.type as ToolCallType.FUNCTION) ?? ToolCallType.FUNCTION + })); + } + + private parseToolArguments(args: string | Record): Record { + if (typeof args === 'object') return args; + + const trimmed = args.trim(); + + if (trimmed === '') return {}; + + return JSON.parse(trimmed) as Record; + } + + private updateSession(conversationId: string, update: Partial): void { + const session = this.getSession(conversationId); + + this.sessions.set(conversationId, { ...session, ...update }); } } diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts deleted file mode 100644 index b7add77779..0000000000 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ /dev/null @@ -1,2868 +0,0 @@ -/** - * chatStore - Reactive State Store for Chat Operations - * - * Manages chat lifecycle, streaming, message operations, and processing state. - * - * **Architecture & Relationships:** - * - **ChatService**: Stateless API layer (sendMessage, streaming) - * - **chatStore** (this): Reactive state + business logic - * - **conversationsStore**: Conversation persistence and navigation - * - * @see ChatService in services/chat.service.ts for API operations - */ - -import { - CONVERSATION_ID_SEPARATOR, - CWD_CLEARED_TEXT, - INACTIVE_CONVERSATION, - STREAM_RESUME_RETRY_MS, - SYSTEM_MESSAGE_PLACEHOLDER, - TITLE_GENERATION -} from '$lib/constants'; -import { - ContinueIntentKind, - ErrorDialogType, - MessageRole, - MessageType, - ReasoningEffort, - StreamConnectionState -} from '$lib/enums'; -import { ChatService } from '$lib/services/chat.service'; -import { DatabaseService } from '$lib/services/database.service'; -// direct imports between stores, not via the barrel, to avoid circular deps -import { agenticStore } from '$lib/stores/agentic.svelte'; -import { conversationsStore } from '$lib/stores/conversations.svelte'; -import { mcpStore } from '$lib/stores/mcp.svelte'; -import { modelsStore } from '$lib/stores/models.svelte'; -import { serverStore } from '$lib/stores/server.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; -import { toolsStore } from '$lib/stores/tools.svelte'; -import type { - ApiChatMessageData, - ApiProcessingState, - ApiStreamSession, - ChatMessagePromptProgress, - ChatMessageTimings, - ChatStreamCallbacks, - DatabaseMessage, - DatabaseMessageExtra, - ErrorDialogState -} from '$lib/types'; -import { - classifyContinueIntent, - filterByLeafNodeId, - findDescendantMessages, - findLeafNode, - findMessageById, - formatCwdMessage, - generateConversationTitle, - getConversationModel, - isAbortError, - normalizeModelName, - streamIdentity -} from '$lib/utils'; -import { SvelteMap, SvelteSet } from 'svelte/reactivity'; - -interface ConversationStateEntry { - lastAccessed: number; -} - -class ChatStore { - activeProcessingState = $state(null); - currentResponse = $state(''); - errorDialogState = $state(null); - isLoading = $state(false); - // true while the active conversation streams reasoning content but no visible content yet - isReasoning = $state(false); - // resumable stream connection state for the active conversation - // streaming -> bytes flowing normally, resuming -> waiting on /v1/stream reconnect, lost -> unrecoverable - streamConnectionState = $state(StreamConnectionState.STREAMING); - chatLoadingStates = new SvelteMap(); - chatReasoningStates = new SvelteMap(); - chatStreamingStates = new SvelteMap< - string, - { response: string; messageId: string; model?: string | null } - >(); - // convs that the backend reports as having a running session, populated by the global sync - // at app mount and on visibilitychange. it does not overlap with chatLoadingStates which - // tracks inferences driven by this browser, both are unioned to feed the sidebar spinners - private remoteRunningConvs = new SvelteSet(); - // per conv attach lifecycle, used to derive the global streaming flag without flipping it - // off when one conv finishes while another is still streaming. mirrors chatLoadingStates - // in scope but tracks the attach + tee replay path specifically - private attachingConvs = new SvelteSet(); - // pending resume retry timers while an owning model loads, one per conv - private resumeRetryTimers = new SvelteMap>(); - // convs whose resume waits on a model load: their loading state belongs to the retry loop, - // so discoverActiveStream must not treat it as a live send and bail - private resumePendingConvs = new SvelteSet(); - // in-flight discoverActiveStream guard, keyed by conv id - private discoveringConvs = new SvelteSet(); - private abortControllers = new SvelteMap(); - private preEncodeAbortController: AbortController | null = null; - private processingStates = new SvelteMap(); - private conversationStateTimestamps = new SvelteMap(); - private activeConversationId = $state(null); - private isStreamingActive = $state(false); - private isEditModeActive = $state(false); - private addFilesHandler: ((files: File[]) => void) | null = $state(null); - pendingEditMessageId = $state(null); - private _pendingDraftMessage = $state(''); - private _pendingDraftFiles = $state([]); - - /** Reactive: queued pending messages for non-agentic streaming */ - private _pendingMessages = new SvelteMap< - string, - { content: string; extras?: DatabaseMessageExtra[] } - >(); - - private setChatLoading(convId: string, loading: boolean): void { - this.touchConversationState(convId); - - if (loading) { - this.chatLoadingStates.set(convId, true); - - if (convId === conversationsStore.activeConversation?.id) this.isLoading = true; - } else { - this.chatLoadingStates.delete(convId); - - if (convId === conversationsStore.activeConversation?.id) this.isLoading = false; - - this.setChatReasoning(convId, false); - // the local pipe is the authoritative observer of session end: when it finishes (clean - // onComplete or explicit Stop), the backend session is finalized too, so we drop the - // sidebar hint for this conv right away instead of waiting for the next visibilitychange - // snapshot. without this the spinner ghosts until the user toggles the tab - this.remoteRunningConvs.delete(convId); - } - } - - private setChatReasoning(convId: string, reasoning: boolean): void { - if (reasoning) { - this.chatReasoningStates.set(convId, true); - - if (convId === conversationsStore.activeConversation?.id) this.isReasoning = true; - } else { - this.chatReasoningStates.delete(convId); - - if (convId === conversationsStore.activeConversation?.id) this.isReasoning = false; - } - } - private setChatStreaming( - convId: string, - response: string, - messageId: string, - model?: string | null - ): void { - this.touchConversationState(convId); - this.chatStreamingStates.set(convId, { - messageId, - model: model ?? this.chatStreamingStates.get(convId)?.model, - response - }); - - if (convId === conversationsStore.activeConversation?.id) this.currentResponse = response; - } - private clearChatStreaming(convId: string, messageId?: string): void { - // session aware: a stale generation must not wipe a newer one's streaming state on the - // same conversation, that would drop the frozen stop identity and stop the wrong session - if (messageId !== undefined) { - const cur = this.chatStreamingStates.get(convId); - - if (cur && cur.messageId !== messageId) return; - } - - this.chatStreamingStates.delete(convId); - - if (convId === conversationsStore.activeConversation?.id) this.currentResponse = ''; - } - private getChatStreamingState( - convId: string - ): { response: string; messageId: string } | undefined { - return this.chatStreamingStates.get(convId); - } - syncLoadingStateForChat(convId: string): void { - this.isLoading = this.chatLoadingStates.get(convId) || false; - this.isReasoning = this.chatReasoningStates.get(convId) || false; - const s = this.chatStreamingStates.get(convId); - - this.currentResponse = s?.response || ''; - this.isStreamingActive = s !== undefined; - this.setActiveProcessingConversation(convId); - - // Sync streaming content to activeMessages so UI displays current content - if (s?.response && s?.messageId) { - const idx = conversationsStore.findMessageIndex(s.messageId); - - if (idx !== -1) { - conversationsStore.updateMessageAtIndex(idx, { content: s.response }); - } - } - } - /** - * Server side stream discovery, split in three pieces: - * - * probeServerStream(convId) -> hits POST /v1/streams/lookup with the conv id, returns the session to attach - * to or null. Pure read, no side effect, no UI lock. Safe to fire in parallel with anything. - * - * attachServerStream(convId) -> flips the spinner immediately, fetches the replay stream - * from byte 0, finds the assistant slot to splice into (creates a placeholder if the conv has - * no assistant message yet, for cross device or fresh local DB cases), and pipes the SSE bytes - * into the message via handleStreamResponse. - * - * discoverActiveStream(convId) -> probe + attach in one call. Used by callers that do not need - * to overlap the probe with other async work. - * - * The mount of the chat page in +page.svelte calls probeServerStream in parallel with - * loadConversation, then attachServerStream once both have settled. This gives the earliest - * possible time to spinner and avoids racing against an empty activeMessages array. - */ - async probeServerStream(convId: string): Promise { - if (!convId) return null; - - let sessions: ApiStreamSession[]; - - try { - sessions = await ChatService.lookupStreamSessions([convId]); - } catch (e) { - console.warn(`probeServerStream failed for conv ${convId}:`, e); - - return null; - } - - return ChatService.selectActiveStream(sessions); - } - - async attachServerStream(convId: string, streamId?: string): Promise { - if (!convId) return; - - if (this.chatStreamingStates.has(convId)) return; - - // flip the spinner immediately, the user sees activity as soon as the conv becomes active. - // the global isStreamingActive flag is derived from attachingConvs.size, so adding here - // turns it on, and removing in unlock only turns it off when this is the last attach - this.setChatLoading(convId, true); - this.attachingConvs.add(convId); - this.setStreamingActive(true); - - // only set the active processing conv if we are looking at it, otherwise a background - // attach would steal the indicator from the conv the user is currently viewing - if (convId === conversationsStore.activeConversation?.id) { - this.setActiveProcessingConversation(convId); - } - - const unlock = () => { - this.attachingConvs.delete(convId); - - // flip the global flag off only when no other conv is still attaching - if (this.attachingConvs.size === 0) { - this.setStreamingActive(false); - } - - this.setChatLoading(convId, false); - this.clearChatStreaming(convId); - }; - // fetch the replay stream from byte 0, rebuild the assistant message from scratch. - // resolve the server side identity, fall back to streamIdentity when the caller does not - // pass a streamId. probeServerStream returns the full id (with ::model suffix when present) - const id = streamId || streamIdentity(convId, modelsStore.selectedModelName); - - let response: Response; - - try { - response = await ChatService.fetchStreamReplay(id); - } catch (e) { - console.error(`attachServerStream replay failed for conv ${convId}:`, e); - unlock(); - - return; - } - - // load the target conversation messages by id, not via the active store. when multiple - // attaches run in parallel the active store may reflect another conv and writing through - // its index mixes content across convs (CoT flicker, message bleed). by going through the - // DB we stay isolated, and only mirror into the active store when the attached conv is - // the one currently displayed - let messages: DatabaseMessage[]; - - try { - messages = await DatabaseService.getConversationMessages(convId); - } catch (e) { - console.error('attachServerStream load messages failed:', e); - unlock(); - - return; - } - - // locate the slot to splice into, create a placeholder assistant message if there is none. - // we use the conv-scoped findLastAssistantIdx helpers, they only depend on the array - let targetIdx = this.findLastAssistantIdx(messages); - - if (targetIdx === -1) { - const lastUserIdx = this.findLastUserIdx(messages); - - if (lastUserIdx === -1) { - console.warn( - `attachServerStream: conv ${convId} has no user or assistant message, cannot splice` - ); - unlock(); - - return; - } - - try { - const placeholder = await DatabaseService.createMessageBranch( - { - children: [], - content: '', - convId, - parent: messages[lastUserIdx].id, - role: MessageRole.ASSISTANT, - timestamp: Date.now(), - toolCalls: '', - type: MessageType.TEXT - } as Omit, - messages[lastUserIdx].id - ); - - messages = [...messages, placeholder]; - targetIdx = messages.length - 1; - - // only push into the active store when this conv is the one displayed right now - if (convId === conversationsStore.activeConversation?.id) { - conversationsStore.addMessageToActive(placeholder); - } - } catch (e) { - console.error('attachServerStream placeholder creation failed:', e); - unlock(); - - return; - } - } - - if (targetIdx === -1) { - unlock(); - - return; - } - - const targetMessage = messages[targetIdx]; - const targetMessageId = targetMessage.id; - // when the assistant slot already has content, the running session is a continue or - // another append flow and its buffer holds only the appended deltas. preserve the prefix - // and let the replay add to it. when the slot is empty the session buffer holds the whole - // message so we wipe and rebuild from byte 0 - const existingContent = targetMessage.content ?? ''; - const existingReasoning = targetMessage.reasoningContent ?? ''; - const isAppendMode = existingContent.length > 0; - // helper: write to the active store only when the attached conv is currently displayed. - // the lookup by message id is robust to reordering of activeMessages, two parallel attaches - // can no longer step on each other's indices - const writeActive = (updates: Partial) => { - if (convId !== conversationsStore.activeConversation?.id) { - return; - } - - const liveIdx = conversationsStore.findMessageIndex(targetMessageId); - - if (liveIdx === -1) return; - - conversationsStore.updateMessageAtIndex(liveIdx, updates); - }; - - if (!isAppendMode) { - writeActive({ content: '', reasoningContent: undefined }); - } - - // extract the model suffix, the resume calls in handleStreamResponse must reuse the model - // the session was tagged with, not the live dropdown - const sepIdx = id.indexOf(CONVERSATION_ID_SEPARATOR); - const attachedModel: string | null = sepIdx === -1 ? null : id.slice(sepIdx + 2); - - this.setChatStreaming(convId, existingContent, targetMessageId, attachedModel); - const abortController = this.getOrCreateAbortController(convId); - - let streamedContent = ''; - let streamedReasoningContent = ''; - - const cleanup = () => { - unlock(); - this.setProcessingState(convId, null); - }; - - try { - await ChatService.handleStreamResponse( - response, - (chunk: string) => { - streamedContent += chunk; - const displayed = isAppendMode ? existingContent + streamedContent : streamedContent; - - writeActive({ content: displayed }); - this.setChatStreaming(convId, displayed, targetMessageId); - }, - async ( - finalContent?: string, - reasoningContent?: string, - timings?: ChatMessageTimings, - toolCalls?: string - ) => { - const streamed = streamedContent || finalContent || ''; - const streamedR = streamedReasoningContent || reasoningContent || ''; - const content = isAppendMode ? existingContent + streamed : streamed; - const reasoning = isAppendMode ? existingReasoning + streamedR : streamedR; - - // the DB write is the source of truth, mirror to the active store only when - // the conv is currently displayed - await DatabaseService.updateMessage(targetMessageId, { - content, - reasoningContent: reasoning || undefined, - timings, - toolCalls: toolCalls || '' - }); - writeActive({ - content, - reasoningContent: reasoning || undefined, - timings - }); - cleanup(); - }, - (err: Error) => { - console.error('attachServerStream pipe error:', err); - cleanup(); - }, - (chunk: string) => { - streamedReasoningContent += chunk; - const displayed = isAppendMode - ? existingReasoning + streamedReasoningContent - : streamedReasoningContent; - - writeActive({ reasoningContent: displayed }); - }, - undefined, - undefined, - undefined, - undefined, - convId, - abortController.signal, - (connState: StreamConnectionState) => { - if (convId === conversationsStore.activeConversation?.id) { - this.streamConnectionState = connState; - } - }, - attachedModel - ); - } catch (e) { - console.error('attachServerStream pipe crashed:', e); - cleanup(); - } - } - - /** - * Model frozen at send time for a stream awaiting resume, from the persisted stream state. - * The load progress indicator targets it after a reload, when the message row has no model - * yet and the dropdown selection may not be restored. - */ - getResumeModel(convId: string): string | null { - return ChatService.getStreamState(convId)?.model ?? null; - } - - async discoverActiveStream(convId: string): Promise { - if (!convId) return; - - if (this.chatStreamingStates.has(convId)) return; - - if (this.chatLoadingStates.get(convId) && !this.resumePendingConvs.has(convId)) return; - - // concurrency guard: another discover may already be running for this conv (typical race - // between mount and visibilitychange on tab switch). a second concurrent fetch on the same - // /v1/stream would duplicate every byte into the DB message, this guard bounces it - if (this.discoveringConvs.has(convId)) return; - - this.discoveringConvs.add(convId); - - try { - // the model is frozen at POST time, rebuild the exact conv::model identity from the - // persisted state so the lookup key matches what the server stored. null means a single - // model conv with no ::suffix, only guess from the dropdown with no persisted state - const localState = ChatService.getStreamState(convId); - const streamId = ChatService.resumeStreamIdentity( - convId, - localState, - modelsStore.selectedModelName - ); - // primary path: ask the server which sessions exist for this identity - const serverTarget = await this.probeServerStream(streamId); - - if (serverTarget) { - // pass the full server side identity (may carry a ::model suffix) so the GET routes - // straight to the owning session, no probe or fan out - await this.attachServerStream(convId, serverTarget.conversation_id); - - return; - } - - // fallback: local state remembers an interrupted byte offset for this conv, the server may - // still have a live session matching that identity (we just lost the bytes mid stream). retry - // with the frozen identity, the server probe inside attachServerStream tells us if it exists - if (!localState) { - return; - } - - // quiet status probe first: a full attach flips the loading UI on every try, probing - // keeps the retry loop invisible while the owning model is still loading (503) - const status = await ChatService.probeResumeStatus(streamId); - - if (status === 503) { - // make the wait visible: the empty assistant row persisted at send time renders - // the processing info, whose model load percentage flows from the models feed - this.resumePendingConvs.add(convId); - this.setChatLoading(convId, true); - - if (!this.resumeRetryTimers.has(convId)) { - this.resumeRetryTimers.set( - convId, - setTimeout(() => { - this.resumeRetryTimers.delete(convId); - void this.discoverActiveStream(convId); - }, STREAM_RESUME_RETRY_MS) - ); - } - - return; - } - - if (this.resumePendingConvs.delete(convId) && status !== 200) { - // the wait is over without a session to attach, drop the visible loading state - this.setChatLoading(convId, false); - } - - if (status === 0) { - // transient network failure, the next mount or visibility change retries - return; - } - - if (status !== 200) { - // the session is gone (stopped, TTL expired), nothing to resume anymore - ChatService.clearStreamState(convId); - - return; - } - - await this.attachServerStream(convId, streamId); - - // if attachServerStream failed (session gone, TTL expired), clear the local state to avoid retrying forever - if (!this.chatStreamingStates.has(convId) && !this.chatLoadingStates.get(convId)) { - ChatService.clearStreamState(convId); - } - } finally { - this.discoveringConvs.delete(convId); - } - } - - private findLastAssistantIdx(messages: DatabaseMessage[]): number { - for (let i = messages.length - 1; i >= 0; i--) { - if (messages[i].role === MessageRole.ASSISTANT) return i; - } - - return -1; - } - - private findLastUserIdx(messages: DatabaseMessage[]): number { - for (let i = messages.length - 1; i >= 0; i--) { - if (messages[i].role === MessageRole.USER) return i; - } - - return -1; - } - - clearUIState(): void { - this.isLoading = false; - this.currentResponse = ''; - this.isStreamingActive = false; - } - - setActiveProcessingConversation(conversationId: string | null): void { - this.activeConversationId = conversationId; - this.activeProcessingState = conversationId - ? this.processingStates.get(conversationId) || null - : null; - } - - getProcessingState(conversationId: string): ApiProcessingState | null { - return this.processingStates.get(conversationId) || null; - } - - private setProcessingState(conversationId: string, state: ApiProcessingState | null): void { - if (state === null) this.processingStates.delete(conversationId); - else this.processingStates.set(conversationId, state); - - if (conversationId === this.activeConversationId) this.activeProcessingState = state; - } - - clearProcessingState(conversationId: string): void { - this.processingStates.delete(conversationId); - - if (conversationId === this.activeConversationId) this.activeProcessingState = null; - } - - getActiveProcessingState(): ApiProcessingState | null { - return this.activeProcessingState; - } - - getCurrentProcessingStateSync(): ApiProcessingState | null { - return this.activeProcessingState; - } - - private setStreamingActive(active: boolean): void { - this.isStreamingActive = active; - } - - isStreaming(): boolean { - return this.isStreamingActive; - } - - private getOrCreateAbortController(convId: string): AbortController { - let c = this.abortControllers.get(convId); - - if (!c || c.signal.aborted) { - c = new AbortController(); - this.abortControllers.set(convId, c); - } - - return c; - } - - private abortRequest(convId?: string): void { - if (convId) { - const c = this.abortControllers.get(convId); - - if (c) { - c.abort(); - this.abortControllers.delete(convId); - } - } else { - for (const c of this.abortControllers.values()) c.abort(); - this.abortControllers.clear(); - } - } - - /** - * Abort the current agentic flow signal without clearing loading state. - * Used by "Send immediately" to force the agentic loop to exit so that - * the pending steering message can be re-sent. - * - * Any tool calls captured mid-stream are dropped before the abort so the - * pending message (or a manual follow-up) does not re-send a half-received - * tool call with invalid JSON arguments to the server. Mirrors what the - * Stop button already does through stopGenerationForChat. - */ - async abortCurrentFlow(convId: string): Promise { - await this.savePartialResponseIfNeeded(convId); - const c = this.abortControllers.get(convId); - - if (c) { - c.abort(); - this.abortControllers.delete(convId); - } - } - - private showErrorDialog(state: ErrorDialogState | null): void { - this.errorDialogState = state; - } - - dismissErrorDialog(): void { - this.errorDialogState = null; - } - - clearEditMode(): void { - this.isEditModeActive = false; - this.addFilesHandler = null; - } - - isEditing(): boolean { - return this.isEditModeActive; - } - - setEditModeActive(handler: (files: File[]) => void): void { - this.isEditModeActive = true; - this.addFilesHandler = handler; - } - - getAddFilesHandler(): ((files: File[]) => void) | null { - return this.addFilesHandler; - } - - clearPendingEditMessageId(): void { - this.pendingEditMessageId = null; - } - - savePendingDraft(message: string, files: ChatUploadedFile[]): void { - this._pendingDraftMessage = message; - this._pendingDraftFiles = [...files]; - } - - consumePendingDraft(): { message: string; files: ChatUploadedFile[] } | null { - if (!this._pendingDraftMessage && this._pendingDraftFiles.length === 0) return null; - - const d = { files: [...this._pendingDraftFiles], message: this._pendingDraftMessage }; - - this._pendingDraftMessage = ''; - this._pendingDraftFiles = []; - - return d; - } - - hasPendingDraft(): boolean { - return Boolean(this._pendingDraftMessage) || this._pendingDraftFiles.length > 0; - } - - getAllLoadingChats(): string[] { - // union of local (this browser is piping) and remote (backend reports a running session - // for this conv but no local pipe yet) sources. the sidebar shows one spinner per entry - const out = new SvelteSet(this.chatLoadingStates.keys()); - - for (const id of this.remoteRunningConvs) { - out.add(id); - } - - return Array.from(out); - } - - getAllStreamingChats(): string[] { - return Array.from(this.chatStreamingStates.keys()); - } - - /** - * Resync the remote running convs set from the backend. Called by the layout at mount and on - * visibilitychange, no polling. A snapshot semantic: the set is replaced wholesale, stale entries - * for sessions that finalized while the browser was elsewhere are dropped naturally. - */ - async syncRemoteRunningStreams(): Promise { - // the conversations store loads from IndexedDB asynchronously, the +layout onMount caller - // fires before that finishes. read ids straight from the DB so the result does not depend - // on the store init race, and the sidebar spinners light up at first paint for every conv - // the user owns even if it has not been hydrated into the store yet - let ids: string[]; - - try { - const all = await DatabaseService.getAllConversations(); - - ids = all.map((c) => c.id).filter((id) => !!id); - } catch (e) { - console.warn('syncRemoteRunningStreams DB read failed:', e); - - return; - } - - // only ask about conv ids the user already owns - if (ids.length === 0) { - for (const id of Array.from(this.remoteRunningConvs)) { - this.remoteRunningConvs.delete(id); - } - - return; - } - - // rebuild the frozen conv::model identity per conv so a session started with a model still - // matches. the server response is mapped back to the bare id below for the sidebar set - const lookupIds = ids.map((id) => - ChatService.resumeStreamIdentity(id, ChatService.getStreamState(id), null) - ); - - let sessions: ApiStreamSession[]; - - try { - sessions = await ChatService.lookupStreamSessions(lookupIds); - } catch (e) { - console.warn('syncRemoteRunningStreams lookup failed:', e); - - return; - } - const running = new SvelteSet(); - - for (const s of sessions) { - if (s && !s.is_done && typeof s.conversation_id === 'string' && s.conversation_id) { - // strip the optional ::model suffix, the sidebar set is keyed by the bare conv id - const sepIdx = s.conversation_id.indexOf(CONVERSATION_ID_SEPARATOR); - const bareId = sepIdx === -1 ? s.conversation_id : s.conversation_id.slice(0, sepIdx); - - running.add(bareId); - } - } - for (const id of Array.from(this.remoteRunningConvs)) { - if (!running.has(id)) { - this.remoteRunningConvs.delete(id); - } - } - for (const id of running) { - this.remoteRunningConvs.add(id); - } - } - - getChatStreaming(convId: string): { response: string; messageId: string } | undefined { - return this.getChatStreamingState(convId); - } - - isChatLoading(convId: string): boolean { - return this.chatLoadingStates.get(convId) || false; - } - - private isChatLoadingInternal(convId: string): boolean { - return this.chatLoadingStates.has(convId) || this.chatStreamingStates.has(convId); - } - - hasPendingMessage(convId: string): boolean { - return this._pendingMessages.has(convId); - } - - pendingMessageContent(convId: string): string | null { - return this._pendingMessages.get(convId)?.content ?? null; - } - - pendingMessageExtras(convId: string): DatabaseMessageExtra[] | undefined { - return this._pendingMessages.get(convId)?.extras; - } - - injectPendingMessage(convId: string, content: string, extras?: DatabaseMessageExtra[]): void { - this._pendingMessages.set(convId, { content, extras }); - } - - clearPendingMessage(convId: string): void { - this._pendingMessages.delete(convId); - } - - consumePendingMessage( - convId: string - ): { content: string; extras?: DatabaseMessageExtra[] } | null { - const msg = this._pendingMessages.get(convId); - - if (!msg) return null; - - this._pendingMessages.delete(convId); - - return msg; - } - - private touchConversationState(convId: string): void { - this.conversationStateTimestamps.set(convId, { lastAccessed: Date.now() }); - } - - cleanupOldConversationStates(activeConversationIds?: string[]): number { - const now = Date.now(); - const activeIdsList = activeConversationIds ?? []; - const preserveIds = this.activeConversationId - ? [...activeIdsList, this.activeConversationId] - : activeIdsList; - const allConvIds = [ - ...new Set([ - ...this.chatLoadingStates.keys(), - ...this.chatStreamingStates.keys(), - ...this.abortControllers.keys(), - ...this.processingStates.keys(), - ...this.conversationStateTimestamps.keys() - ]) - ]; - const cleanupCandidates: Array<{ convId: string; lastAccessed: number }> = []; - - for (const convId of allConvIds) { - if (preserveIds.includes(convId)) continue; - - if (this.chatLoadingStates.get(convId)) continue; - - if (this.chatStreamingStates.has(convId)) continue; - - const ts = this.conversationStateTimestamps.get(convId); - - cleanupCandidates.push({ convId, lastAccessed: ts?.lastAccessed ?? 0 }); - } - cleanupCandidates.sort((a, b) => a.lastAccessed - b.lastAccessed); - let cleanedUp = 0; - - for (const { convId, lastAccessed } of cleanupCandidates) { - if ( - cleanupCandidates.length - cleanedUp > INACTIVE_CONVERSATION.MAX_STATES || - now - lastAccessed > INACTIVE_CONVERSATION.MAX_AGE_MS - ) { - this.cleanupConversationState(convId); - cleanedUp++; - } - } - - return cleanedUp; - } - private cleanupConversationState(convId: string): void { - const c = this.abortControllers.get(convId); - - if (c && !c.signal.aborted) c.abort(); - - this.chatLoadingStates.delete(convId); - this.chatStreamingStates.delete(convId); - this.abortControllers.delete(convId); - this.processingStates.delete(convId); - this.conversationStateTimestamps.delete(convId); - } - getTrackedConversationCount(): number { - return new Set([ - ...this.chatLoadingStates.keys(), - ...this.chatStreamingStates.keys(), - ...this.abortControllers.keys(), - ...this.processingStates.keys() - ]).size; - } - - private getMessageByIdWithRole( - messageId: string, - expectedRole?: MessageRole - ): { message: DatabaseMessage; index: number } | null { - const index = conversationsStore.findMessageIndex(messageId); - - if (index === -1) return null; - - const message = conversationsStore.activeMessages[index]; - - if (expectedRole && message.role !== expectedRole) return null; - - return { index, message }; - } - - async addMessage( - role: MessageRole, - content: string, - type: MessageType = MessageType.TEXT, - parent: string = '-1', - extras?: DatabaseMessageExtra[], - isSynthetic?: boolean - ): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) throw new Error('No active conversation'); - - let parentId: string | null = null; - - if (parent === '-1') { - const am = conversationsStore.activeMessages; - - if (am.length > 0) parentId = am[am.length - 1].id; - else { - const all = await conversationsStore.getConversationMessages(activeConv.id); - const r = all.find((m) => m.parent === null && m.type === 'root'); - - parentId = r ? r.id : await DatabaseService.createRootMessage(activeConv.id); - } - } else parentId = parent; - - const message = await DatabaseService.createMessageBranch( - { - children: [], - content, - convId: activeConv.id, - extra: extras, - isSynthetic, - role, - timestamp: Date.now(), - toolCalls: '', - type - }, - parentId - ); - - conversationsStore.addMessageToActive(message); - await conversationsStore.updateCurrentNode(message.id); - conversationsStore.updateConversationTimestamp(); - - return message; - } - - /** - * Record a working-directory change into chat history as a synthetic - * user message, so the model sees it on its next turn (the client - * sends the cwd itself via the x-tool-cwd header on tool calls). - * A plain user message is used because some chat templates reject - * tool messages without a preceding tool call. - */ - async recordCwdChange(cwd: string | null): Promise { - const content = cwd - ? formatCwdMessage(cwd, await toolsStore.resolveServerHome()) - : CWD_CLEARED_TEXT; - // Reuse the trailing cwd row when it is already the last message, so - // repeated picks update it in place instead of stacking another row. - const last = conversationsStore.activeMessages[conversationsStore.activeMessages.length - 1]; - - if (last && last.role === MessageRole.USER && last.isSynthetic === true) { - await DatabaseService.updateMessage(last.id, { content, isSynthetic: true }); - conversationsStore.updateMessageAtIndex(conversationsStore.activeMessages.length - 1, { - content, - isSynthetic: true - }); - - return; - } - - await this.addMessage(MessageRole.USER, content, MessageType.TEXT, '-1', undefined, true); - } - - async addSystemPrompt(): Promise { - let activeConv = conversationsStore.activeConversation; - - if (!activeConv) { - await conversationsStore.createConversation(); - activeConv = conversationsStore.activeConversation; - } - - if (!activeConv) return; - - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); - const rootId = rootMessage - ? rootMessage.id - : await DatabaseService.createRootMessage(activeConv.id); - const existingSystemMessage = allMessages.find( - (m) => m.role === MessageRole.SYSTEM && m.parent === rootId - ); - - if (existingSystemMessage) { - this.pendingEditMessageId = existingSystemMessage.id; - - if (!conversationsStore.activeMessages.some((m) => m.id === existingSystemMessage.id)) - conversationsStore.activeMessages.unshift(existingSystemMessage); - - return; - } - - const am = conversationsStore.activeMessages; - const firstActiveMessage = am.find((m) => m.parent === rootId); - const systemMessage = await DatabaseService.createSystemMessage( - activeConv.id, - SYSTEM_MESSAGE_PLACEHOLDER, - rootId - ); - - if (firstActiveMessage) { - await DatabaseService.updateMessage(firstActiveMessage.id, { - parent: systemMessage.id - }); - await DatabaseService.updateMessage(systemMessage.id, { - children: [firstActiveMessage.id] - }); - const updatedRootChildren = rootMessage - ? rootMessage.children.filter((id: string) => id !== firstActiveMessage.id) - : []; - - await DatabaseService.updateMessage(rootId, { - children: [ - ...updatedRootChildren.filter((id: string) => id !== systemMessage.id), - systemMessage.id - ] - }); - const firstMsgIndex = conversationsStore.findMessageIndex(firstActiveMessage.id); - - if (firstMsgIndex !== -1) - conversationsStore.updateMessageAtIndex(firstMsgIndex, { - parent: systemMessage.id - }); - } - - conversationsStore.activeMessages.unshift(systemMessage); - this.pendingEditMessageId = systemMessage.id; - conversationsStore.updateConversationTimestamp(); - } catch (error) { - console.error('Failed to add system prompt:', error); - } - } - - async removeSystemPromptPlaceholder(messageId: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return false; - - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const systemMessage = findMessageById(allMessages, messageId); - - if (!systemMessage || systemMessage.role !== MessageRole.SYSTEM) return false; - - const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); - - if (!rootMessage) return false; - - if (allMessages.length === 2 && systemMessage.children.length === 0) { - await conversationsStore.deleteConversation(activeConv.id); - - return true; - } - - for (const childId of systemMessage.children) { - await DatabaseService.updateMessage(childId, { parent: rootMessage.id }); - const childIndex = conversationsStore.findMessageIndex(childId); - - if (childIndex !== -1) - conversationsStore.updateMessageAtIndex(childIndex, { parent: rootMessage.id }); - } - await DatabaseService.updateMessage(rootMessage.id, { - children: [ - ...rootMessage.children.filter((id: string) => id !== messageId), - ...systemMessage.children - ] - }); - await DatabaseService.deleteMessage(messageId); - const systemIndex = conversationsStore.findMessageIndex(messageId); - - if (systemIndex !== -1) conversationsStore.activeMessages.splice(systemIndex, 1); - - conversationsStore.updateConversationTimestamp(); - - return false; - } catch (error) { - console.error('Failed to remove system prompt placeholder:', error); - - return false; - } - } - - private async createAssistantMessage(parentId?: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) throw new Error('No active conversation'); - - return await DatabaseService.createMessageBranch( - { - children: [], - content: '', - convId: activeConv.id, - model: null, - role: MessageRole.ASSISTANT, - timestamp: Date.now(), - toolCalls: '', - type: MessageType.TEXT - }, - parentId || null - ); - } - - async sendMessage(content: string, extras?: DatabaseMessageExtra[]): Promise { - if (!content.trim() && (!extras || extras.length === 0)) return; - - const activeConv = conversationsStore.activeConversation; - - // If agentic loop is running, inject as a steering message instead of starting a new flow - if (activeConv && agenticStore.isRunning(activeConv.id)) { - agenticStore.injectSteeringMessage(activeConv.id, content, extras); - - return; - } - - // If non-agentic streaming is active, queue as a pending message to send after completion - if (activeConv && this.isChatLoadingInternal(activeConv.id)) { - this.injectPendingMessage(activeConv.id, content, extras); - - return; - } - - // Cancel any in-flight pre-encode request - this.cancelPreEncode(); - - // Consume MCP resource attachments - converts them to extras and clears the live store - const resourceExtras = mcpStore.consumeResourceAttachmentsAsExtras(); - const allExtras = resourceExtras.length > 0 ? [...(extras || []), ...resourceExtras] : extras; - - let isNewConversation = false; - - if (!activeConv) { - await conversationsStore.createConversation(); - isNewConversation = true; - } - - const currentConv = conversationsStore.activeConversation; - - if (!currentConv) return; - - this.showErrorDialog(null); - this.setChatLoading(currentConv.id, true); - this.clearChatStreaming(currentConv.id); - try { - let parentIdForUserMessage: string | undefined; - - if (isNewConversation) { - const rootId = await DatabaseService.createRootMessage(currentConv.id); - const currentConfig = settingsStore.config; - const systemPrompt = currentConfig.systemMessage?.toString().trim(); - - let sysOrRootId = rootId; - - if (systemPrompt) { - const systemMessage = await DatabaseService.createSystemMessage( - currentConv.id, - systemPrompt, - rootId - ); - - conversationsStore.addMessageToActive(systemMessage); - sysOrRootId = systemMessage.id; - } - - // Reflect a working directory picked on the new-chat screen into - // chat history before the first user message, so the model sees - // it on its first turn. createConversation() has already threaded - // the pending pick onto the conversation. - if (currentConv.cwd) { - const cwdMessage = await this.addMessage( - MessageRole.USER, - formatCwdMessage(currentConv.cwd, await toolsStore.resolveServerHome()), - MessageType.TEXT, - sysOrRootId, - undefined, - true - ); - - parentIdForUserMessage = cwdMessage.id; - } else { - parentIdForUserMessage = sysOrRootId; - } - } - - const userMessage = await this.addMessage( - MessageRole.USER, - content, - MessageType.TEXT, - parentIdForUserMessage ?? '-1', - allExtras - ); - - if (isNewConversation && content) - await conversationsStore.updateConversationName( - currentConv.id, - generateConversationTitle( - content, - Boolean(settingsStore.config.titleGenerationUseFirstLine) - ) - ); - - const assistantMessage = await this.createAssistantMessage(userMessage.id); - - conversationsStore.addMessageToActive(assistantMessage); - await this.streamChatCompletion( - conversationsStore.activeMessages.slice(0, -1), - assistantMessage, - undefined, - undefined, - undefined, - settingsStore.config.titleGenerationUseLLM && isNewConversation ? content : undefined - ); - } catch (error) { - if (isAbortError(error)) { - this.setChatLoading(currentConv.id, false); - - return; - } - - console.error('Failed to send message:', error); - this.setChatLoading(currentConv.id, false); - const dialogType = - error instanceof Error && error.name === 'TimeoutError' - ? ErrorDialogType.TIMEOUT - : ErrorDialogType.SERVER; - const contextInfo = ( - error as Error & { contextInfo?: { n_prompt_tokens: number; n_ctx: number } } - ).contextInfo; - - this.showErrorDialog({ - contextInfo, - message: error instanceof Error ? error.message : 'Unknown error', - type: dialogType - }); - } - } - - private async streamChatCompletion( - allMessages: DatabaseMessage[], - assistantMessage: DatabaseMessage, - onComplete?: (content: string) => Promise, - onError?: (error: Error) => void, - modelOverride?: string | null, - firstUserMessageContent?: string - ): Promise { - // the ::model suffix in the stream identity is only for router mode, where it routes to the - // owning child. in single-model mode the identity stays the bare conv id so that attach, stop - // and reattach all agree, regardless of fresh send vs regenerate passing a resolved model - let effectiveModel: string | null | undefined = undefined; - - if (serverStore.isRouterMode) { - const conversationModel = getConversationModel(allMessages); - - effectiveModel = modelOverride || modelsStore.selectedModelName || conversationModel; - } - - if (serverStore.isRouterMode && effectiveModel) { - if (!modelsStore.getModelProps(effectiveModel)) - await modelsStore.fetchModelProps(effectiveModel); - } - - // Mutable state for the current message being streamed - let currentMessageId = assistantMessage.id; - let streamedContent = ''; - let streamedReasoningContent = ''; - let resolvedModel: string | null = null; - let modelPersisted = false; - - const convId = assistantMessage.convId; - - // Tracks the last message created in this flow. Used as the parent for the next - // turn's assistant message so createAssistantMessage does not have to read - // conversationsStore.activeMessages, which may belong to a different conversation - // after the user navigates while the loop is still running. - let lastCreatedInFlow = currentMessageId; - - // freeze the POST identity from t0 so a stop cancels with the exact session key, - // never a stale or empty model resolved later - this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); - - const recordModel = (modelName: string | null | undefined, persistImmediately = true): void => { - if (!modelName) return; - - const n = normalizeModelName(modelName); - - if (!n || n === resolvedModel) return; - - resolvedModel = n; - const idx = conversationsStore.findMessageIndex(currentMessageId); - - conversationsStore.updateMessageAtIndex(idx, { model: n }); - - if (persistImmediately && !modelPersisted) { - modelPersisted = true; - DatabaseService.updateMessage(currentMessageId, { model: n }).catch(() => { - modelPersisted = false; - resolvedModel = null; - }); - } - }; - - let completionIdRecorded = false; - - const recordCompletionId = (id: string): void => { - if (!id || completionIdRecorded) return; - - completionIdRecorded = true; - const idx = conversationsStore.findMessageIndex(currentMessageId); - - conversationsStore.updateMessageAtIndex(idx, { completionId: id }); - DatabaseService.updateMessage(currentMessageId, { completionId: id }).catch(() => { - completionIdRecorded = false; - }); - }; - const updateStreamingUI = () => { - this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); - const idx = conversationsStore.findMessageIndex(currentMessageId); - - conversationsStore.updateMessageAtIndex(idx, { content: streamedContent }); - }; - const cleanupStreamingState = () => { - this.setStreamingActive(false); - this.setChatLoading(convId, false); - this.clearChatStreaming(convId, currentMessageId); - this.setProcessingState(convId, null); - }; - - this.setStreamingActive(true); - this.setActiveProcessingConversation(convId); - const abortController = this.getOrCreateAbortController(convId); - const streamCallbacks: ChatStreamCallbacks = { - createAssistantMessage: async () => { - // Reset streaming state for new message - streamedContent = ''; - streamedReasoningContent = ''; - - const msg = await DatabaseService.createMessageBranch( - { - children: [], - content: '', - convId, - model: resolvedModel, - role: MessageRole.ASSISTANT, - timestamp: Date.now(), - toolCalls: '', - type: MessageType.TEXT - }, - lastCreatedInFlow - ); - - if (conversationsStore.activeConversation?.id === convId) { - conversationsStore.addMessageToActive(msg); - } - - currentMessageId = msg.id; - lastCreatedInFlow = msg.id; - - return msg; - }, - createToolResultMessage: async ( - toolCallId: string, - content: string, - extras?: DatabaseMessageExtra[], - toolCwd?: string - ) => { - const msg = await DatabaseService.createMessageBranch( - { - children: [], - content, - convId, - extra: extras, - role: MessageRole.TOOL, - timestamp: Date.now(), - toolCallId, - toolCalls: '', - toolCwd, - type: MessageType.TEXT - }, - currentMessageId - ); - - // mirror into the active store and move the node pointer only when this - // conversation is displayed; otherwise persist the node move straight to - // the db for the owning conv so a foreign conv's currNode stays untouched - if (conversationsStore.activeConversation?.id === convId) { - conversationsStore.addMessageToActive(msg); - await conversationsStore.updateCurrentNode(msg.id); - } else { - await DatabaseService.updateCurrentNode(convId, msg.id); - } - - lastCreatedInFlow = msg.id; - - return msg; - }, - onAssistantTurnComplete: async ( - content: string, - reasoningContent: string | undefined, - timings: ChatMessageTimings | undefined, - toolCalls: import('$lib/types/api').ApiChatCompletionToolCall[] | undefined - ) => { - const updateData: Record = { - content, - reasoningContent: reasoningContent || undefined, - timings, - toolCalls: toolCalls ? JSON.stringify(toolCalls) : '' - }; - - if (resolvedModel && !modelPersisted) updateData.model = resolvedModel; - - await DatabaseService.updateMessage(currentMessageId, updateData); - const idx = conversationsStore.findMessageIndex(currentMessageId); - const uiUpdate: Partial = { - content, - reasoningContent: reasoningContent || undefined, - toolCalls: toolCalls ? JSON.stringify(toolCalls) : '' - }; - - if (timings) uiUpdate.timings = timings; - - if (resolvedModel) uiUpdate.model = resolvedModel; - - // touch the active ui array and node pointer only when this conversation - // is displayed; otherwise persist the node move straight to the db so a - // foreign conv's currNode stays untouched - if (conversationsStore.activeConversation?.id === convId) { - conversationsStore.updateMessageAtIndex(idx, uiUpdate); - await conversationsStore.updateCurrentNode(currentMessageId); - } else { - await DatabaseService.updateCurrentNode(convId, currentMessageId); - } - }, - onAttachments: (messageId: string, extras: DatabaseMessageExtra[]) => { - if (!extras.length) return; - - const idx = conversationsStore.findMessageIndex(messageId); - - if (idx === -1) return; - - const msg = conversationsStore.activeMessages[idx]; - const updatedExtras = [...(msg.extra || []), ...extras]; - - conversationsStore.updateMessageAtIndex(idx, { extra: updatedExtras }); - DatabaseService.updateMessage(messageId, { extra: updatedExtras }).catch(console.error); - }, - onChunk: (chunk: string) => { - streamedContent += chunk; - updateStreamingUI(); - this.setChatReasoning(convId, false); - }, - onCompletionId: (id: string) => recordCompletionId(id), - onError: async (error: Error) => { - this.setStreamingActive(false); - - if (isAbortError(error)) { - cleanupStreamingState(); - // If aborted with a pending message (e.g. "Send immediately"), re-send it - const pending = this.consumePendingMessage(convId); - - if (pending) { - this.sendMessage(pending.content, pending.extras); - } - - return; - } - - console.error('Streaming error:', error); - // keep whatever was streamed so far, the message stays in memory and in DB - await this.savePartialResponseIfNeeded(convId); - cleanupStreamingState(); - this.clearPendingMessage(convId); - - const contextInfo = ( - error as Error & { contextInfo?: { n_prompt_tokens: number; n_ctx: number } } - ).contextInfo; - - this.showErrorDialog({ - contextInfo, - message: error.message, - type: error.name === 'TimeoutError' ? ErrorDialogType.TIMEOUT : ErrorDialogType.SERVER - }); - - if (onError) onError(error); - }, - onFlowComplete: (finalTimings?: ChatMessageTimings) => { - if (finalTimings) { - const idx = conversationsStore.findMessageIndex(assistantMessage.id); - - conversationsStore.updateMessageAtIndex(idx, { timings: finalTimings }); - DatabaseService.updateMessage(assistantMessage.id, { - timings: finalTimings - }).catch(console.error); - } - - cleanupStreamingState(); - - if (onComplete) onComplete(streamedContent); - - if (serverStore.isRouterMode) modelsStore.fetchRouterModels().catch(console.error); - - // Pre-encode conversation in KV cache for faster next turn - if (settingsStore.config.preEncodeConversation) { - this.triggerPreEncode( - allMessages, - assistantMessage, - streamedContent, - effectiveModel, - !!settingsStore.config.excludeReasoningFromContext - ); - } - }, - onModel: (modelName: string) => recordModel(modelName), - onReasoningChunk: (chunk: string) => { - streamedReasoningContent += chunk; - // mark streaming state so a stop mid-thinking can persist the partial reasoning - this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); - const idx = conversationsStore.findMessageIndex(currentMessageId); - - conversationsStore.updateMessageAtIndex(idx, { - reasoningContent: streamedReasoningContent - }); - this.setChatReasoning(convId, true); - }, - onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { - const tokensPerSecond = - timings?.predicted_ms && timings?.predicted_n - ? (timings.predicted_n / timings.predicted_ms) * 1000 - : 0; - - this.updateProcessingStateFromTimings( - { - cache_n: timings?.cache_n || 0, - predicted_n: timings?.predicted_n || 0, - predicted_per_second: tokensPerSecond, - prompt_ms: timings?.prompt_ms, - prompt_n: timings?.prompt_n || 0, - prompt_progress: promptProgress - }, - convId - ); - }, - onToolCallsStreaming: (toolCalls) => { - const idx = conversationsStore.findMessageIndex(currentMessageId); - - conversationsStore.updateMessageAtIndex(idx, { - toolCalls: JSON.stringify(toolCalls) - }); - }, - onTurnComplete: (intermediateTimings: ChatMessageTimings) => { - // Update the first assistant message with cumulative agentic timings - const idx = conversationsStore.findMessageIndex(assistantMessage.id); - - conversationsStore.updateMessageAtIndex(idx, { timings: intermediateTimings }); - }, - updateToolResultMessage: async ( - messageId: string, - content: string, - extras?: DatabaseMessageExtra[] - ) => { - // Persist latest content + merged extras; mirror into the active - // store so the chat view sees live updates for streaming tools - // (e.g. exec_shell_command). The existing tool message node - // pointer stays put - the renderer is already scoped to it. - const updates: Partial = { content }; - - if (extras) { - const idx = conversationsStore.findMessageIndex(messageId); - const existing = idx >= 0 ? (conversationsStore.activeMessages[idx]?.extra ?? []) : []; - const merged = [...existing, ...extras]; - - updates.extra = merged; - } - - if (conversationsStore.activeConversation?.id === convId) { - const idx = conversationsStore.findMessageIndex(messageId); - - if (idx >= 0) conversationsStore.updateMessageAtIndex(idx, updates); - } - - await DatabaseService.updateMessage(messageId, updates); - } - }; - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); - - { - const agenticResult = await agenticStore.runAgenticFlow({ - callbacks: streamCallbacks, - conversationId: convId, - flowRootMessageId: assistantMessage.id, - messages: allMessages, - options: { - ...this.getApiOptions(), - ...(effectiveModel ? { model: effectiveModel } : {}) - }, - perChatOverrides, - signal: abortController.signal - }); - - if (agenticResult.handled) { - // Generate LLM based title for new conversations after agentic flow completes - if (firstUserMessageContent) { - await this.generateTitleWithLLM(firstUserMessageContent, streamedContent, convId); - } - - // Check if there's a pending steering message to re-send - const pending = agenticStore.consumePendingSteeringMessage(convId); - - if (pending) { - await this.sendMessage(pending.content, pending.extras); - } - - return; - } - } - - await ChatService.sendMessage( - allMessages, - { - ...this.getApiOptions(), - ...(effectiveModel ? { model: effectiveModel } : {}), - onChunk: streamCallbacks.onChunk, - onComplete: async ( - finalContent?: string, - reasoningContent?: string, - timings?: ChatMessageTimings, - toolCalls?: string - ) => { - const content = streamedContent || finalContent || ''; - const reasoning = streamedReasoningContent || reasoningContent; - const updateData: Record = { - content, - reasoningContent: reasoning || undefined, - timings, - toolCalls: toolCalls || '' - }; - - if (resolvedModel && !modelPersisted) updateData.model = resolvedModel; - - await DatabaseService.updateMessage(currentMessageId, updateData); - const idx = conversationsStore.findMessageIndex(currentMessageId); - const uiUpdate: Partial = { - content, - reasoningContent: reasoning || undefined, - toolCalls: toolCalls || '' - }; - - if (timings) uiUpdate.timings = timings; - - if (resolvedModel) uiUpdate.model = resolvedModel; - - conversationsStore.updateMessageAtIndex(idx, uiUpdate); - await conversationsStore.updateCurrentNode(currentMessageId); - cleanupStreamingState(); - - if (onComplete) await onComplete(content); - - if (serverStore.isRouterMode) modelsStore.fetchRouterModels().catch(console.error); - - // Generate LLM based title for new conversations (avoids stale reference - // issue when user switches conversations while streaming) - if (firstUserMessageContent) { - await this.generateTitleWithLLM(firstUserMessageContent, streamedContent, convId); - } - - // Check if there's a pending message queued during streaming - const pending = this.consumePendingMessage(convId); - - if (pending) { - await this.sendMessage(pending.content, pending.extras); - } - }, - onCompletionId: streamCallbacks.onCompletionId, - onConnectionState: (state: StreamConnectionState) => { - if (convId === conversationsStore.activeConversation?.id) { - this.streamConnectionState = state; - } - }, - onError: streamCallbacks.onError, - onModel: streamCallbacks.onModel, - onReasoningChunk: streamCallbacks.onReasoningChunk, - onTimings: streamCallbacks.onTimings, - stream: true - }, - convId, - abortController.signal - ); - } - - async stopGeneration(): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return; - - await this.stopGenerationForChat(activeConv.id); - } - async stopGenerationForChat(convId: string): Promise { - await this.savePartialResponseIfNeeded(convId); - this.setStreamingActive(false); - // tell the server to stop the generation, not just drop the HTTP socket. without this the - // detached drain keeps producing tokens until eos or max_tokens. use the frozen identity - // captured when the session started, not the live dropdown - const streamStateForStop = this.chatStreamingStates.get(convId); - const modelForStop = streamStateForStop?.model ?? ChatService.getStreamState(convId)?.model; - - void ChatService.cancelServerStream(convId, modelForStop); - // an explicit stop leaves nothing to resume and kills a pending resume retry - ChatService.clearStreamState(convId); - const retryTimer = this.resumeRetryTimers.get(convId); - - if (retryTimer !== undefined) { - clearTimeout(retryTimer); - this.resumeRetryTimers.delete(convId); - } - - this.resumePendingConvs.delete(convId); - this.abortRequest(convId); - this.setChatLoading(convId, false); - this.clearChatStreaming(convId); - this.setProcessingState(convId, null); - this.clearPendingMessage(convId); - } - - private async generateTitleWithLLM( - userContent: string, - assistantContent: string, - convId: string - ): Promise { - const effectiveModel = - serverStore.isRouterMode && modelsStore.selectedModelName - ? modelsStore.selectedModelName - : undefined; - const configValue = settingsStore.config; - const titlePromptTemplate = - typeof configValue.titleGenerationPrompt === 'string' && - configValue.titleGenerationPrompt.trim() - ? configValue.titleGenerationPrompt - : TITLE_GENERATION.DEFAULT_PROMPT; - const titlePrompt = titlePromptTemplate - .replace('{{USER}}', String(userContent || '')) - .replace('{{ASSISTANT}}', String(assistantContent || '')); - const titleMessage: ApiChatMessageData = { - content: titlePrompt, - role: MessageRole.USER - }; - const titleResponse = await ChatService.generateTitle(titleMessage, effectiveModel); - - if (!titleResponse) { - return; - } - - let cleanTitle = titleResponse.trim(); - - cleanTitle = cleanTitle - .replace(TITLE_GENERATION.PREFIX_PATTERN, '') - .replace(TITLE_GENERATION.QUOTE_PATTERN, '') - .trim(); - - if (!cleanTitle || cleanTitle.length < TITLE_GENERATION.MIN_LENGTH) { - const firstLine = userContent.split('\n').find((l) => l.trim().length > 0); - - cleanTitle = firstLine ? firstLine.trim() : TITLE_GENERATION.FALLBACK; - } - - if (cleanTitle && cleanTitle.length >= TITLE_GENERATION.MIN_LENGTH) { - await conversationsStore.updateConversationName(convId, cleanTitle); - } - } - - private async savePartialResponseIfNeeded(convId?: string): Promise { - const conversationId = convId || conversationsStore.activeConversation?.id; - - if (!conversationId) return; - - const streamingState = this.getChatStreamingState(conversationId); - - if (!streamingState) return; - - const messages = - conversationId === conversationsStore.activeConversation?.id - ? conversationsStore.activeMessages - : await conversationsStore.getConversationMessages(conversationId); - - if (!messages.length) return; - - const lastMessage = messages[messages.length - 1]; - - if (lastMessage?.role !== MessageRole.ASSISTANT) return; - - const partialContent = streamingState.response; - const partialReasoning = lastMessage.reasoningContent || ''; - // snapshot the streamed tool calls before clearing so we still know whether - // anything was captured when deciding to skip the DB write below - const hadPartialToolCalls = !!lastMessage.toolCalls?.trim(); - - // nothing to persist when content, reasoning, and streamed tool calls are all empty - // (e.g. stop before any token). otherwise drop the partial tool call and write whatever - // was streamed: incomplete arguments (truncated JSON, missing closing quote) would - // otherwise be re-sent to the server on the next turn and rejected. - if (!partialContent.trim() && !partialReasoning.trim() && !hadPartialToolCalls) return; - - try { - const updateData: { - content?: string; - reasoningContent?: string; - toolCalls?: string; - timings?: ChatMessageTimings; - } = { - toolCalls: '' - }; - - if (partialContent.trim()) updateData.content = partialContent; - - if (partialReasoning.trim()) updateData.reasoningContent = partialReasoning; - - const lastKnownState = this.getProcessingState(conversationId); - - if (lastKnownState) { - updateData.timings = { - cache_n: lastKnownState.cacheTokens || 0, - predicted_ms: - lastKnownState.tokensPerSecond && lastKnownState.tokensDecoded - ? (lastKnownState.tokensDecoded / lastKnownState.tokensPerSecond) * 1000 - : undefined, - predicted_n: lastKnownState.tokensDecoded || 0, - prompt_ms: lastKnownState.promptMs, - prompt_n: lastKnownState.promptTokens || 0 - }; - } - - await DatabaseService.updateMessage(lastMessage.id, updateData); - lastMessage.content = partialContent; - // mirror the drop into the in-memory message so the next request sent via - // sendMessage (queued pending, Send immediately, or manual follow-up) reads - // the cleared value, not whatever the streaming widget had been showing - lastMessage.toolCalls = ''; - - if (updateData.timings) lastMessage.timings = updateData.timings; - } catch (error) { - lastMessage.content = partialContent; - lastMessage.toolCalls = ''; - console.error('Failed to save partial response:', error); - } - } - - async updateMessage(messageId: string, newContent: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return; - - if (this.isChatLoadingInternal(activeConv.id)) await this.stopGeneration(); - - const result = this.getMessageByIdWithRole(messageId, MessageRole.USER); - - if (!result) return; - - const { index: messageIndex, message: messageToUpdate } = result; - const originalContent = messageToUpdate.content; - - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); - const isFirstUserMessage = rootMessage && messageToUpdate.parent === rootMessage.id; - - conversationsStore.updateMessageAtIndex(messageIndex, { content: newContent }); - await DatabaseService.updateMessage(messageId, { content: newContent }); - - if (isFirstUserMessage && newContent.trim()) - await conversationsStore.updateConversationName( - activeConv.id, - generateConversationTitle( - newContent, - Boolean(settingsStore.config.titleGenerationUseFirstLine) - ) - ); - - const messagesToRemove = conversationsStore.activeMessages.slice(messageIndex + 1); - - if (messagesToRemove.length > 0) - await DatabaseService.deleteMessageCascading(activeConv.id, messagesToRemove[0].id); - - conversationsStore.sliceActiveMessages(messageIndex + 1); - conversationsStore.updateConversationTimestamp(); - this.setChatLoading(activeConv.id, true); - this.clearChatStreaming(activeConv.id); - const assistantMessage = await this.createAssistantMessage(); - - conversationsStore.addMessageToActive(assistantMessage); - await conversationsStore.updateCurrentNode(assistantMessage.id); - await this.streamChatCompletion( - conversationsStore.activeMessages.slice(0, -1), - assistantMessage, - undefined, - () => { - conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(messageId), { - content: originalContent - }); - } - ); - } catch (error) { - if (!isAbortError(error)) console.error('Failed to update message:', error); - } - } - - async regenerateMessage(messageId: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv || this.isChatLoadingInternal(activeConv.id)) return; - - this.cancelPreEncode(); - const result = this.getMessageByIdWithRole(messageId, MessageRole.ASSISTANT); - - if (!result) return; - - const { index: messageIndex } = result; - - try { - const messagesToRemove = conversationsStore.activeMessages.slice(messageIndex); - - await DatabaseService.deleteMessageCascading(activeConv.id, messagesToRemove[0].id); - conversationsStore.sliceActiveMessages(messageIndex); - conversationsStore.updateConversationTimestamp(); - this.setChatLoading(activeConv.id, true); - this.clearChatStreaming(activeConv.id); - const parentMessageId = - conversationsStore.activeMessages.length > 0 - ? conversationsStore.activeMessages[conversationsStore.activeMessages.length - 1].id - : undefined; - const assistantMessage = await this.createAssistantMessage(parentMessageId); - - conversationsStore.addMessageToActive(assistantMessage); - await this.streamChatCompletion( - conversationsStore.activeMessages.slice(0, -1), - assistantMessage - ); - } catch (error) { - if (!isAbortError(error)) console.error('Failed to regenerate message:', error); - - this.setChatLoading(activeConv?.id || '', false); - } - } - - async regenerateMessageWithBranching(messageId: string, modelOverride?: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv || this.isChatLoadingInternal(activeConv.id)) return; - - this.cancelPreEncode(); - try { - const idx = conversationsStore.findMessageIndex(messageId); - - if (idx === -1) return; - - const msg = conversationsStore.activeMessages[idx]; - - if (msg.role !== MessageRole.ASSISTANT) return; - - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const parentMessage = findMessageById(allMessages, msg.parent); - - if (!parentMessage) return; - - this.setChatLoading(activeConv.id, true); - this.clearChatStreaming(activeConv.id); - const newAssistantMessage = await DatabaseService.createMessageBranch( - { - children: [], - content: '', - convId: msg.convId, - model: null, - role: msg.role, - timestamp: Date.now(), - toolCalls: '', - type: msg.type - }, - parentMessage.id - ); - - await conversationsStore.updateCurrentNode(newAssistantMessage.id); - conversationsStore.updateConversationTimestamp(); - await conversationsStore.refreshActiveMessages(); - const conversationPath = filterByLeafNodeId( - allMessages, - parentMessage.id, - false - ) as DatabaseMessage[]; - const modelToUse = modelOverride || msg.model || undefined; - - await this.streamChatCompletion( - conversationPath, - newAssistantMessage, - undefined, - undefined, - modelToUse - ); - } catch (error) { - if (!isAbortError(error)) - console.error('Failed to regenerate message with branching:', error); - - this.setChatLoading(activeConv?.id || '', false); - } - } - - async getDeletionInfo(messageId: string): Promise<{ - totalCount: number; - userMessages: number; - assistantMessages: number; - messageTypes: string[]; - }> { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) - return { assistantMessages: 0, messageTypes: [], totalCount: 0, userMessages: 0 }; - - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const messageToDelete = findMessageById(allMessages, messageId); - - // For system messages, don't count descendants as they will be preserved (reparented to root) - if (messageToDelete?.role === MessageRole.SYSTEM) { - const messagesToDelete = allMessages.filter((m) => m.id === messageId); - - let assistantMessages = 0, - userMessages = 0; - - const messageTypes: string[] = []; - - for (const msg of messagesToDelete) { - if (msg.role === MessageRole.USER) { - userMessages++; - - if (!messageTypes.includes('user message')) messageTypes.push('user message'); - } else if (msg.role === MessageRole.ASSISTANT) { - assistantMessages++; - - if (!messageTypes.includes('assistant response')) messageTypes.push('assistant response'); - } - } - - return { assistantMessages, messageTypes, totalCount: 1, userMessages }; - } - - const descendants = findDescendantMessages(allMessages, messageId); - const allToDelete = [messageId, ...descendants]; - const messagesToDelete = allMessages.filter((m) => allToDelete.includes(m.id)); - - let assistantMessages = 0, - userMessages = 0; - - const messageTypes: string[] = []; - - for (const msg of messagesToDelete) { - if (msg.role === MessageRole.USER) { - userMessages++; - - if (!messageTypes.includes('user message')) messageTypes.push('user message'); - } else if (msg.role === MessageRole.ASSISTANT) { - assistantMessages++; - - if (!messageTypes.includes('assistant response')) messageTypes.push('assistant response'); - } - } - - return { assistantMessages, messageTypes, totalCount: allToDelete.length, userMessages }; - } - - async deleteMessage(messageId: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return; - - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const messageToDelete = findMessageById(allMessages, messageId); - - if (!messageToDelete) return; - - const currentPath = filterByLeafNodeId(allMessages, activeConv.currNode || '', false); - const isInCurrentPath = currentPath.some((m) => m.id === messageId); - - if (isInCurrentPath && messageToDelete.parent) { - const siblings = allMessages.filter( - (m) => m.parent === messageToDelete.parent && m.id !== messageId - ); - - if (siblings.length > 0) { - const latestSibling = siblings.reduce((latest, sibling) => - sibling.timestamp > latest.timestamp ? sibling : latest - ); - - await conversationsStore.updateCurrentNode(findLeafNode(allMessages, latestSibling.id)); - } else if (messageToDelete.parent) { - await conversationsStore.updateCurrentNode( - findLeafNode(allMessages, messageToDelete.parent) - ); - } - } - - await DatabaseService.deleteMessageCascading(activeConv.id, messageId); - await conversationsStore.refreshActiveMessages(); - - conversationsStore.updateConversationTimestamp(); - } catch (error) { - console.error('Failed to delete message:', error); - } - } - - /** - * Open a fresh assistant turn anchored at the last tool result of a resolved - * agentic round and let streamChatCompletion route through runAgenticFlow. - * Used by continueAssistantMessage when classifyContinueIntent returns - * next_turn, meaning the target assistant already has its tool_calls paired - * with trailing tool results and the next thing to generate is a brand new - * turn rather than a token level continuation. - */ - private async continueAsNextAgenticTurn(anchorIndex: number): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return; - - const anchor = conversationsStore.activeMessages[anchorIndex]; - - if (!anchor) return; - - this.cancelPreEncode(); - this.setChatLoading(activeConv.id, true); - this.clearChatStreaming(activeConv.id); - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const anchorMessage = findMessageById(allMessages, anchor.id); - - if (!anchorMessage) { - this.setChatLoading(activeConv.id, false); - - return; - } - - const newAssistantMessage = await DatabaseService.createMessageBranch( - { - children: [], - content: '', - convId: activeConv.id, - model: null, - role: MessageRole.ASSISTANT, - timestamp: Date.now(), - toolCalls: '', - type: MessageType.TEXT - }, - anchorMessage.id - ); - - await conversationsStore.updateCurrentNode(newAssistantMessage.id); - conversationsStore.updateConversationTimestamp(); - await conversationsStore.refreshActiveMessages(); - const conversationPath = filterByLeafNodeId( - allMessages, - anchorMessage.id, - false - ) as DatabaseMessage[]; - - await this.streamChatCompletion(conversationPath, newAssistantMessage); - } catch (error) { - if (!isAbortError(error)) console.error('Failed to continue agentic turn:', error); - - this.setChatLoading(activeConv.id, false); - } - } - - async continueAssistantMessage(messageId: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv || this.isChatLoadingInternal(activeConv.id)) return; - - const result = this.getMessageByIdWithRole(messageId, MessageRole.ASSISTANT); - - if (!result) return; - - const { index: idx, message: msg } = result; - // Decide which resume path applies. tool_calls without tool results can - // not be resumed mid sequence by continue_final_message, branch instead. - // tool_calls already paired with tool results need a fresh next turn, - // not a token level continuation of the target assistant. - const intent = classifyContinueIntent(conversationsStore.activeMessages, idx); - - if (intent.kind === ContinueIntentKind.RERUN_TURN) { - return this.regenerateMessageWithBranching(messageId); - } - - if (intent.kind === ContinueIntentKind.NEXT_TURN) { - return this.continueAsNextAgenticTurn(intent.truncateAfter); - } - - try { - this.showErrorDialog(null); - this.setChatLoading(activeConv.id, true); - this.clearChatStreaming(activeConv.id); - - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const dbMessage = findMessageById(allMessages, messageId); - - if (!dbMessage) { - this.setChatLoading(activeConv.id, false); - - return; - } - - const originalContent = dbMessage.content; - const originalReasoning = dbMessage.reasoningContent || ''; - // Hand the persisted DatabaseMessage straight to sendMessage so its - // internal converter preserves tool_calls and extras when present. - // Reconstructing a bare {role, content} here would drop those fields - // and break continue_final_message for messages with tool calls. - const contextWithContinue = conversationsStore.activeMessages.slice(0, idx + 1); - - let appendedContent = ''; - let appendedReasoning = ''; - let hasReceivedContent = false; - - const updateStreamingContent = (fullContent: string) => { - this.setChatStreaming(msg.convId, fullContent, msg.id); - // resolve the row by id on every write, switching to another conv mid continue makes - // this a no op instead of writing positionally into the now displayed conversation - conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { - content: fullContent - }); - }; - const abortController = this.getOrCreateAbortController(msg.convId); - - await ChatService.sendMessage( - contextWithContinue, - { - ...this.getApiOptions(), - continueFinalMessage: true, - onChunk: (chunk: string) => { - appendedContent += chunk; - hasReceivedContent = true; - updateStreamingContent(originalContent + appendedContent); - this.setChatReasoning(msg.convId, false); - }, - onComplete: async ( - finalContent?: string, - reasoningContent?: string, - timings?: ChatMessageTimings - ) => { - const finalAppendedContent = hasReceivedContent ? appendedContent : finalContent || ''; - const finalAppendedReasoning = hasReceivedContent - ? appendedReasoning - : reasoningContent || ''; - const fullContent = originalContent + finalAppendedContent; - const fullReasoning = originalReasoning + finalAppendedReasoning || undefined; - - await DatabaseService.updateMessage(msg.id, { - content: fullContent, - reasoningContent: fullReasoning, - timestamp: Date.now(), - timings - }); - - conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { - content: fullContent, - reasoningContent: fullReasoning, - timestamp: Date.now(), - timings - }); - - conversationsStore.updateConversationTimestamp(msg.convId); - - this.setChatLoading(msg.convId, false); - this.clearChatStreaming(msg.convId); - this.setProcessingState(msg.convId, null); - }, - onCompletionId: (id: string) => { - if (!id) return; - - // refresh the message id so a later skip targets the live slot after a continue - conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { - completionId: id - }); - DatabaseService.updateMessage(msg.id, { completionId: id }).catch(() => {}); - }, - onConnectionState: (state: StreamConnectionState) => { - if (msg.convId === conversationsStore.activeConversation?.id) { - this.streamConnectionState = state; - } - }, - onError: async (error: Error) => { - if (isAbortError(error)) { - if (hasReceivedContent && appendedContent) { - await DatabaseService.updateMessage(msg.id, { - content: originalContent + appendedContent, - reasoningContent: originalReasoning + appendedReasoning || undefined, - timestamp: Date.now() - }); - - conversationsStore.updateMessageAtIndex( - conversationsStore.findMessageIndex(msg.id), - { - content: originalContent + appendedContent, - reasoningContent: originalReasoning + appendedReasoning || undefined, - timestamp: Date.now() - } - ); - } - - this.setChatLoading(msg.convId, false); - this.clearChatStreaming(msg.convId); - this.setProcessingState(msg.convId, null); - - return; - } - - console.error('Continue generation error:', error); - // keep whatever was appended so far, the message stays in memory and in DB - await DatabaseService.updateMessage(msg.id, { - content: originalContent + appendedContent, - reasoningContent: originalReasoning + appendedReasoning || undefined, - timestamp: Date.now() - }); - conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { - content: originalContent + appendedContent, - reasoningContent: originalReasoning + appendedReasoning || undefined, - timestamp: Date.now() - }); - - this.setChatLoading(msg.convId, false); - this.clearChatStreaming(msg.convId); - this.setProcessingState(msg.convId, null); - this.showErrorDialog({ - message: error.message, - type: error.name === 'TimeoutError' ? ErrorDialogType.TIMEOUT : ErrorDialogType.SERVER - }); - }, - onReasoningChunk: (chunk: string) => { - appendedReasoning += chunk; - hasReceivedContent = true; - // mark streaming state so a stop mid-thinking can persist the partial reasoning - this.setChatStreaming(msg.convId, originalContent + appendedContent, msg.id); - conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { - reasoningContent: originalReasoning + appendedReasoning - }); - this.setChatReasoning(msg.convId, true); - }, - onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { - const tokensPerSecond = - timings?.predicted_ms && timings?.predicted_n - ? (timings.predicted_n / timings.predicted_ms) * 1000 - : 0; - - this.updateProcessingStateFromTimings( - { - cache_n: timings?.cache_n || 0, - predicted_n: timings?.predicted_n || 0, - predicted_per_second: tokensPerSecond, - prompt_ms: timings?.prompt_ms, - prompt_n: timings?.prompt_n || 0, - prompt_progress: promptProgress - }, - msg.convId - ); - } - }, - - msg.convId, - abortController.signal - ); - } catch (error) { - if (!isAbortError(error)) console.error('Failed to continue message:', error); - - if (activeConv) this.setChatLoading(activeConv.id, false); - } - } - - async editAssistantMessage( - messageId: string, - newContent: string, - shouldBranch: boolean - ): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv || this.isChatLoadingInternal(activeConv.id)) return; - - const result = this.getMessageByIdWithRole(messageId, MessageRole.ASSISTANT); - - if (!result) return; - - const { index: idx, message: msg } = result; - - try { - if (shouldBranch) { - const newMessage = await DatabaseService.createMessageBranch( - { - children: [], - content: newContent, - convId: msg.convId, - model: msg.model, - role: msg.role, - timestamp: Date.now(), - toolCalls: msg.toolCalls || '', - type: msg.type - }, - msg.parent! - ); - - await conversationsStore.updateCurrentNode(newMessage.id); - } else { - await DatabaseService.updateMessage(msg.id, { content: newContent }); - conversationsStore.updateMessageAtIndex(idx, { content: newContent }); - } - - conversationsStore.updateConversationTimestamp(); - - await conversationsStore.refreshActiveMessages(); - } catch (error) { - console.error('Failed to edit assistant message:', error); - } - } - - async editUserMessagePreserveResponses( - messageId: string, - newContent: string, - newExtras?: DatabaseMessageExtra[] - ): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return; - - const result = this.getMessageByIdWithRole(messageId, MessageRole.USER); - - if (!result) return; - - const { index: idx, message: msg } = result; - - try { - const updateData: Partial = { content: newContent }; - - if (newExtras !== undefined) updateData.extra = JSON.parse(JSON.stringify(newExtras)); - - await DatabaseService.updateMessage(messageId, updateData); - - conversationsStore.updateMessageAtIndex(idx, updateData); - - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); - - if (rootMessage && msg.parent === rootMessage.id && newContent.trim()) { - await conversationsStore.updateConversationName( - activeConv.id, - generateConversationTitle( - newContent, - Boolean(settingsStore.config.titleGenerationUseFirstLine) - ) - ); - } - - conversationsStore.updateConversationTimestamp(); - } catch (error) { - console.error('Failed to edit user message:', error); - } - } - - async editMessageWithBranching( - messageId: string, - newContent: string, - newExtras?: DatabaseMessageExtra[] - ): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv || this.isChatLoadingInternal(activeConv.id)) return; - - let result = this.getMessageByIdWithRole(messageId, MessageRole.USER); - - if (!result) result = this.getMessageByIdWithRole(messageId, MessageRole.SYSTEM); - - if (!result) return; - - const { index: idx, message: msg } = result; - - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); - const isFirstUserMessage = - msg.role === MessageRole.USER && rootMessage && msg.parent === rootMessage.id; - const extrasToUse = - newExtras !== undefined - ? JSON.parse(JSON.stringify(newExtras)) - : msg.extra - ? JSON.parse(JSON.stringify(msg.extra)) - : undefined; - - let messageIdForResponse: string; - - const dbMsg = findMessageById(allMessages, msg.id); - const hasChildren = dbMsg ? dbMsg.children.length > 0 : msg.children.length > 0; - - if (!hasChildren) { - // No responses after this message — update in place instead of branching - const updates: Partial = { - content: newContent, - extra: extrasToUse, - timestamp: Date.now() - }; - - await DatabaseService.updateMessage(msg.id, updates); - conversationsStore.updateMessageAtIndex(idx, updates); - messageIdForResponse = msg.id; - } else { - // Has children — create a new branch as sibling - const parentId = msg.parent || rootMessage?.id; - - if (!parentId) return; - - const newMessage = await DatabaseService.createMessageBranch( - { - children: [], - content: newContent, - convId: msg.convId, - extra: extrasToUse, - model: msg.model, - role: msg.role, - timestamp: Date.now(), - toolCalls: msg.toolCalls || '', - type: msg.type - }, - parentId - ); - - await conversationsStore.updateCurrentNode(newMessage.id); - messageIdForResponse = newMessage.id; - } - - conversationsStore.updateConversationTimestamp(); - - if (isFirstUserMessage && newContent.trim()) - await conversationsStore.updateConversationName( - activeConv.id, - generateConversationTitle( - newContent, - Boolean(settingsStore.config.titleGenerationUseFirstLine) - ) - ); - - await conversationsStore.refreshActiveMessages(); - - if (msg.role === MessageRole.USER) - await this.generateResponseForMessage(messageIdForResponse); - } catch (error) { - console.error('Failed to edit message with branching:', error); - } - } - - private async generateResponseForMessage(userMessageId: string): Promise { - const activeConv = conversationsStore.activeConversation; - - if (!activeConv) return; - - this.showErrorDialog(null); - this.setChatLoading(activeConv.id, true); - this.clearChatStreaming(activeConv.id); - - try { - const allMessages = await conversationsStore.getConversationMessages(activeConv.id); - const conversationPath = filterByLeafNodeId( - allMessages, - userMessageId, - false - ) as DatabaseMessage[]; - const assistantMessage = await DatabaseService.createMessageBranch( - { - children: [], - content: '', - convId: activeConv.id, - model: null, - role: MessageRole.ASSISTANT, - timestamp: Date.now(), - toolCalls: '', - type: MessageType.TEXT - }, - userMessageId - ); - - conversationsStore.addMessageToActive(assistantMessage); - - await this.streamChatCompletion(conversationPath, assistantMessage); - } catch (error) { - console.error('Failed to generate response:', error); - this.setChatLoading(activeConv.id, false); - } - } - - private getContextTotal(): number | null { - const activeConvId = this.activeConversationId; - const activeState = activeConvId ? this.getProcessingState(activeConvId) : null; - - if (activeState && typeof activeState.contextTotal === 'number' && activeState.contextTotal > 0) - return activeState.contextTotal; - - if (serverStore.isRouterMode) { - const modelContextSize = modelsStore.selectedModelContextSize; - - if (typeof modelContextSize === 'number' && modelContextSize > 0) { - return modelContextSize; - } - } else { - const propsContextSize = serverStore.contextSize; - - if (typeof propsContextSize === 'number' && propsContextSize > 0) { - return propsContextSize; - } - } - - return null; - } - - updateProcessingStateFromTimings( - timingData: { - prompt_n: number; - prompt_ms?: number; - predicted_n: number; - predicted_per_second: number; - cache_n: number; - prompt_progress?: ChatMessagePromptProgress; - }, - conversationId?: string - ): void { - const processingState = this.parseTimingData(timingData); - - if (processingState === null) { - console.warn('Failed to parse timing data - skipping update'); - - return; - } - - const targetId = conversationId || this.activeConversationId; - - if (targetId) { - this.setProcessingState(targetId, processingState); - } - } - - private parseTimingData(timingData: Record): ApiProcessingState | null { - const cacheTokens = (timingData.cache_n as number) || 0, - predictedTokens = (timingData.predicted_n as number) || 0, - promptMs = (timingData.prompt_ms as number) || undefined, - promptTokens = (timingData.prompt_n as number) || 0, - tokensPerSecond = (timingData.predicted_per_second as number) || 0; - const promptProgress = timingData.prompt_progress as - | { total: number; cache: number; processed: number; time_ms: number } - | undefined; - const contextTotal = this.getContextTotal(); - const currentConfig = settingsStore.config; - const outputTokensMax = currentConfig.max_tokens || -1; - const contextUsed = promptTokens + cacheTokens + predictedTokens, - outputTokensUsed = predictedTokens; - const progressCache = promptProgress?.cache || 0, - progressActualDone = (promptProgress?.processed ?? 0) - progressCache, - progressActualTotal = (promptProgress?.total ?? 0) - progressCache; - const progressPercent = promptProgress - ? Math.round((progressActualDone / progressActualTotal) * 100) - : undefined; - - return { - cacheTokens, - contextTotal, - contextUsed, - hasNextToken: predictedTokens > 0, - outputTokensMax, - outputTokensUsed, - progressPercent, - promptMs, - promptProgress, - promptTokens, - speculative: false, - status: predictedTokens > 0 ? 'generating' : promptProgress ? 'preparing' : 'idle', - temperature: currentConfig.temperature ?? 0.8, - tokensDecoded: predictedTokens, - tokensPerSecond, - tokensRemaining: outputTokensMax - predictedTokens, - topP: currentConfig.top_p ?? 0.95 - }; - } - - restoreProcessingStateFromMessages(messages: DatabaseMessage[], conversationId: string): void { - for (let i = messages.length - 1; i >= 0; i--) { - const message = messages[i]; - - if (message.role === MessageRole.ASSISTANT && message.timings) { - const restoredState = this.parseTimingData({ - cache_n: message.timings.cache_n || 0, - predicted_n: message.timings.predicted_n || 0, - predicted_per_second: - message.timings.predicted_n && message.timings.predicted_ms - ? (message.timings.predicted_n / message.timings.predicted_ms) * 1000 - : 0, - prompt_ms: message.timings.prompt_ms, - prompt_n: message.timings.prompt_n || 0 - }); - - if (restoredState) { - this.setProcessingState(conversationId, restoredState); - - return; - } - } - } - } - - private getApiOptions(): Record { - const currentConfig = settingsStore.config; - const hasValue = (value: unknown): boolean => - value !== undefined && value !== null && value !== ''; - const apiOptions: Record = { stream: true, timings_per_token: true }; - - if (serverStore.isRouterMode) { - const modelName = modelsStore.selectedModelName; - - if (modelName) apiOptions.model = modelName; - } - - if (currentConfig.systemMessage) apiOptions.systemMessage = currentConfig.systemMessage; - - if (currentConfig.disableReasoningParsing) apiOptions.disableReasoningParsing = true; - - if (currentConfig.excludeReasoningFromContext) apiOptions.excludeReasoningFromContext = true; - - // an explicit reasoning choice overrides the server default, DEFAULT sends nothing - const effort = conversationsStore.getReasoningEffort(); - - if (effort !== ReasoningEffort.DEFAULT) { - apiOptions.enableThinking = effort !== ReasoningEffort.OFF; - - if (effort !== ReasoningEffort.OFF) apiOptions.reasoningEffort = effort; - } - - if (hasValue(currentConfig.temperature)) - apiOptions.temperature = Number(currentConfig.temperature); - - if (hasValue(currentConfig.max_tokens)) - apiOptions.max_tokens = Number(currentConfig.max_tokens); - - if (hasValue(currentConfig.dynatemp_range)) - apiOptions.dynatemp_range = Number(currentConfig.dynatemp_range); - - if (hasValue(currentConfig.dynatemp_exponent)) - apiOptions.dynatemp_exponent = Number(currentConfig.dynatemp_exponent); - - if (hasValue(currentConfig.top_k)) apiOptions.top_k = Number(currentConfig.top_k); - - if (hasValue(currentConfig.top_p)) apiOptions.top_p = Number(currentConfig.top_p); - - if (hasValue(currentConfig.min_p)) apiOptions.min_p = Number(currentConfig.min_p); - - if (hasValue(currentConfig.xtc_probability)) - apiOptions.xtc_probability = Number(currentConfig.xtc_probability); - - if (hasValue(currentConfig.xtc_threshold)) - apiOptions.xtc_threshold = Number(currentConfig.xtc_threshold); - - if (hasValue(currentConfig.typ_p)) apiOptions.typ_p = Number(currentConfig.typ_p); - - if (hasValue(currentConfig.repeat_last_n)) - apiOptions.repeat_last_n = Number(currentConfig.repeat_last_n); - - if (hasValue(currentConfig.repeat_penalty)) - apiOptions.repeat_penalty = Number(currentConfig.repeat_penalty); - - if (hasValue(currentConfig.presence_penalty)) - apiOptions.presence_penalty = Number(currentConfig.presence_penalty); - - if (hasValue(currentConfig.frequency_penalty)) - apiOptions.frequency_penalty = Number(currentConfig.frequency_penalty); - - if (hasValue(currentConfig.dry_multiplier)) - apiOptions.dry_multiplier = Number(currentConfig.dry_multiplier); - - if (hasValue(currentConfig.dry_base)) apiOptions.dry_base = Number(currentConfig.dry_base); - - if (hasValue(currentConfig.dry_allowed_length)) - apiOptions.dry_allowed_length = Number(currentConfig.dry_allowed_length); - - if (hasValue(currentConfig.dry_penalty_last_n)) - apiOptions.dry_penalty_last_n = Number(currentConfig.dry_penalty_last_n); - - if (currentConfig.samplers) apiOptions.samplers = currentConfig.samplers; - - if (hasValue(currentConfig.backend_sampling)) - apiOptions.backend_sampling = currentConfig.backend_sampling; - - if (currentConfig.customJson) apiOptions.custom = currentConfig.customJson; - - return apiOptions; - } - - private cancelPreEncode(): void { - if (this.preEncodeAbortController) { - this.preEncodeAbortController.abort(); - this.preEncodeAbortController = null; - } - } - - private async triggerPreEncode( - allMessages: DatabaseMessage[], - assistantMessage: DatabaseMessage, - assistantContent: string, - model?: string | null, - excludeReasoning?: boolean - ): Promise { - this.cancelPreEncode(); - this.preEncodeAbortController = new AbortController(); - - const signal = this.preEncodeAbortController.signal; - - try { - const allIdle = await ChatService.areAllSlotsIdle(model, signal); - - if (!allIdle || signal.aborted) return; - - const messagesWithAssistant: DatabaseMessage[] = [ - ...allMessages, - { ...assistantMessage, content: assistantContent } - ]; - - await ChatService.preEncode(messagesWithAssistant, model, excludeReasoning, signal); - } catch (err) { - if (!isAbortError(err)) { - console.warn('[ChatStore] Pre-encode failed:', err); - } - } - } -} - -export const chatStore = new ChatStore(); diff --git a/tools/ui/src/lib/stores/chat/activity.svelte.ts b/tools/ui/src/lib/stores/chat/activity.svelte.ts new file mode 100644 index 0000000000..cd4e0497bf --- /dev/null +++ b/tools/ui/src/lib/stores/chat/activity.svelte.ts @@ -0,0 +1,74 @@ +/** + * ChatActivityStore - Conversation activity ledger + * + * Single owner of the "is this conversation doing something" state: + * - `local` - this browser is piping a stream (send, server-stream attach, + * or resume-wait while the owning model loads) + * - `remote` - the backend reports a running session, no local pipe yet + * (global snapshot on mount / visibilitychange) + * + * The union of both drives the sidebar spinners (`loadingConvs`); `local` + * drives the per-conversation loading flags. When a local pipe ends it is + * the authoritative observer of session end, so it also drops the stale + * remote hint in the same call - no cross-owner cleanup, no ghosted + * spinners waiting for the next visibilitychange snapshot. + * + * Composed under chatStore.activity; not exported from the stores barrel. + */ + +import { SvelteSet } from 'svelte/reactivity'; + +export class ChatActivityStore { + /** Convs this browser is piping a stream for (send, attach, resume-wait). */ + private local = new SvelteSet(); + /** Convs the backend reports as having a running session (snapshot sync). */ + private remote = new SvelteSet(); + + /** Convs with any activity, the union the sidebar spinners render. */ + loadingConvs = $derived.by(() => { + const out = new SvelteSet(this.local); + + for (const id of this.remote) out.add(id); + + return Array.from(out); + }); + + /** + * Apply a backend snapshot of running sessions (mount / visibilitychange). + * Diffed so unchanged entries do not re-trigger reactivity. + */ + applyRemoteSnapshot(running: Iterable): void { + const next = new SvelteSet(running); + + for (const id of Array.from(this.remote)) { + if (!next.has(id)) this.remote.delete(id); + } + + for (const id of next) this.remote.add(id); + } + + isLocal(convId: string): boolean { + return this.local.has(convId); + } + + isRemote(convId: string): boolean { + return this.remote.has(convId); + } + + /** + * A local pipe ended for the conv. Also drops the remote hint: the local + * pipe is the authoritative observer of session end, so the sidebar hint + * goes away right away instead of ghosting until the next snapshot. + */ + localEnded(convId: string): void { + this.local.delete(convId); + this.remote.delete(convId); + } + + /** A local pipe (send, attach or resume-wait) started for the conv. */ + markLocal(convId: string): void { + this.local.add(convId); + } +} + +export const chatActivityStore = new ChatActivityStore(); diff --git a/tools/ui/src/lib/stores/context-stats.svelte.ts b/tools/ui/src/lib/stores/chat/context-stats.svelte.ts similarity index 56% rename from tools/ui/src/lib/stores/context-stats.svelte.ts rename to tools/ui/src/lib/stores/chat/context-stats.svelte.ts index 1491845630..b5d22cfbda 100644 --- a/tools/ui/src/lib/stores/context-stats.svelte.ts +++ b/tools/ui/src/lib/stores/chat/context-stats.svelte.ts @@ -1,5 +1,5 @@ /** - * contextStatsStore - Context window usage stats for the active conversation + * ContextStatsStore - Context window usage stats for the active conversation * * Combines token usage persisted in message timings metadata with * server-originating data: model context size from /props (modelsStore) @@ -8,12 +8,17 @@ import { MessageRole } from '$lib/enums'; // direct imports between stores, not via the barrel, to avoid circular deps -import { agenticStore } from '$lib/stores/agentic.svelte'; -import { chatStore } from '$lib/stores/chat.svelte'; -import { conversationsStore } from '$lib/stores/conversations.svelte'; -import { modelsStore } from '$lib/stores/models.svelte'; +import { agenticStore } from '$lib/stores/agentic/index.svelte'; +import { chatStore } from '$lib/stores/chat/index.svelte'; +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; import { serverStore } from '$lib/stores/server.svelte'; -import type { ApiProcessingState, ChatMessageTimings, DatabaseMessage } from '$lib/types'; +import type { + ApiProcessingState, + ChatMessageAgenticTimings, + ChatMessageTimings, + DatabaseMessage +} from '$lib/types'; interface LiveStats { freshTokens: number; @@ -22,14 +27,46 @@ interface LiveStats { outputTokens: number; } -function lastAssistantTimings(messages: DatabaseMessage[]): ChatMessageTimings | undefined { - for (let i = messages.length - 1; i >= 0; i--) { - const m = messages[i]; +interface AssistantTimingsSummary { + lastAgenticLlm: ChatMessageAgenticTimings['llm'] | undefined; + lastTimings: ChatMessageTimings | undefined; + cacheTotal: number; + output: number; + outputMs: number; + read: number; +} - if (m.role === MessageRole.ASSISTANT && m.timings) return m.timings; +/** + * One forward pass over the messages computing everything the deriveds + * below need: the last assistant timings (per-turn gauges), the last + * agentic llm totals (cumulative gauge) and the cumulative sums. During + * streaming activeMessages churns every chunk, and each of these used to be + * its own O(n) scan re-run per chunk. + */ +function summarizeAssistantTimings(messages: DatabaseMessage[]): AssistantTimingsSummary { + let lastAgenticLlm: ChatMessageAgenticTimings['llm'] | undefined; + let lastTimings: ChatMessageTimings | undefined; + let read = 0; + let cacheTotal = 0; + let output = 0; + let outputMs = 0; + + for (const m of messages) { + if (m.role !== MessageRole.ASSISTANT || !m.timings) continue; + + lastTimings = m.timings; + + if (m.timings.agentic?.llm?.predicted_n != null) { + lastAgenticLlm = m.timings.agentic.llm; + } + + read += m.timings.prompt_n ?? 0; + cacheTotal += m.timings.cache_n ?? 0; + output += m.timings.predicted_n ?? 0; + outputMs += m.timings.predicted_ms ?? 0; } - return undefined; + return { cacheTotal, lastAgenticLlm, lastTimings, output, outputMs, read }; } function deriveLiveStats(state: ApiProcessingState | null): LiveStats | null { @@ -52,83 +89,14 @@ class ContextStatsStore { // The canonical resolution lives in modelsStore.activeModelId. activeModelId = $derived(modelsStore.activeModelId); - isActiveModelLoaded = $derived( - this.activeModelId !== null && - (!serverStore.isRouterMode || modelsStore.isModelLoaded(this.activeModelId)) + // shared by currentRead/Fresh/Cache/Output and cumulative so a per-chunk + // churn of activeMessages triggers exactly one scan instead of one per + // derived + private assistantTimings = $derived.by(() => + summarizeAssistantTimings(conversationsStore.activeMessages as DatabaseMessage[]) ); - isActiveModelLoading = $derived( - this.activeModelId !== null && modelsStore.isModelOperationInProgress(this.activeModelId) - ); - - contextTotal = $derived.by(() => { - void modelsStore.propsCacheVersion; - - return this.activeModelId ? modelsStore.getModelContextSize(this.activeModelId) : null; - }); - - private liveStats = $derived(deriveLiveStats(chatStore.activeProcessingState)); - - currentRead = $derived.by(() => { - const timings = lastAssistantTimings(conversationsStore.activeMessages as DatabaseMessage[]); - - let read = 0; - - if (timings) { - read = (timings.prompt_n ?? 0) + (timings.cache_n ?? 0); - } - - // live.promptTokens is already the combined reading (prompt + cache), - // so do not also add live.cacheTokens. - if (this.liveStats && this.liveStats.promptTokens > 0) { - read = Math.max(read, this.liveStats.promptTokens); - } - - return read; - }); - - currentFresh = $derived.by(() => { - const timings = lastAssistantTimings(conversationsStore.activeMessages as DatabaseMessage[]); - const fresh = timings?.prompt_n ?? 0; - - return Math.max(fresh, this.liveStats?.freshTokens ?? 0); - }); - - currentCache = $derived.by(() => { - const timings = lastAssistantTimings(conversationsStore.activeMessages as DatabaseMessage[]); - const cached = timings?.cache_n ?? 0; - - if (this.liveStats && this.liveStats.promptTokens > 0) { - return Math.max(cached, this.liveStats.cacheTokens); - } - - return cached; - }); - - currentOutput = $derived.by(() => { - if (this.liveStats && this.liveStats.outputTokens > 0) return this.liveStats.outputTokens; - - const timings = lastAssistantTimings(conversationsStore.activeMessages as DatabaseMessage[]); - - return timings?.predicted_n ?? 0; - }); - - kvTotal = $derived(this.currentRead + this.currentOutput); - - contextUsed = $derived(this.currentRead + this.currentOutput); - - contextAvailable = $derived( - this.contextTotal !== null ? this.contextTotal - this.contextUsed : null - ); - - contextPercent = $derived.by(() => { - if (this.contextTotal === null || this.contextTotal <= 0) return null; - - return Math.round((this.contextUsed / this.contextTotal) * 100); - }); - private cumulative = $derived.by(() => { - const messages = conversationsStore.activeMessages as DatabaseMessage[]; const convId = conversationsStore.activeConversation?.id; // A running agentic flow stamps llm totals on messages only when it // exits, so read its live session totals instead. @@ -147,51 +115,107 @@ class ContextStatsStore { }; } + const { cacheTotal, lastAgenticLlm, output, outputMs, read } = this.assistantTimings; + // Agentic sessions stamp the same agentic.llm totals onto every // assistant message; cache_n is never per-turn so cache_total stays 0. - const agenticMessages = messages.filter( - (m) => m.role === MessageRole.ASSISTANT && m.timings?.agentic?.llm?.predicted_n != null - ); - - if (agenticMessages.length > 0) { - const llm = agenticMessages[agenticMessages.length - 1].timings!.agentic!.llm; - const output = llm.predicted_n ?? 0; - const outputMs = llm.predicted_ms ?? 0; - const averageTokensPerSecond = outputMs > 0 && output > 0 ? (output / outputMs) * 1000 : null; + if (lastAgenticLlm) { + const averageTokensPerSecond = + lastAgenticLlm.predicted_ms > 0 && lastAgenticLlm.predicted_n > 0 + ? (lastAgenticLlm.predicted_n / lastAgenticLlm.predicted_ms) * 1000 + : null; return { averageTokensPerSecond, cacheTotal: 0, - output, - read: llm.prompt_n ?? 0 + output: lastAgenticLlm.predicted_n ?? 0, + read: lastAgenticLlm.prompt_n ?? 0 }; } - let read = 0; - let output = 0; - let outputMs = 0; - let cacheTotal = 0; - - for (const m of messages) { - if (m.role !== MessageRole.ASSISTANT || !m.timings) continue; - - read += m.timings.prompt_n ?? 0; - cacheTotal += m.timings.cache_n ?? 0; - output += m.timings.predicted_n ?? 0; - outputMs += m.timings.predicted_ms ?? 0; - } const averageTokensPerSecond = outputMs > 0 && output > 0 ? (output / outputMs) * 1000 : null; return { averageTokensPerSecond, cacheTotal, output, read }; }); - cumulativeRead = $derived(this.cumulative.read); + averageTokensPerSecond = $derived(this.cumulative.averageTokensPerSecond); - cumulativeOutput = $derived(this.cumulative.output); + contextTotal = $derived.by(() => { + void modelsStore.props.cacheVersion; + + return this.activeModelId ? modelsStore.props.getModelContextSize(this.activeModelId) : null; + }); + + private liveStats = $derived(deriveLiveStats(chatStore.processing.activeState)); + + currentOutput = $derived.by(() => { + if (this.liveStats && this.liveStats.outputTokens > 0) return this.liveStats.outputTokens; + + return this.assistantTimings.lastTimings?.predicted_n ?? 0; + }); + + currentRead = $derived.by(() => { + const timings = this.assistantTimings.lastTimings; + + let read = 0; + + if (timings) { + read = (timings.prompt_n ?? 0) + (timings.cache_n ?? 0); + } + + // live.promptTokens is already the combined reading (prompt + cache), + // so do not also add live.cacheTokens. + if (this.liveStats && this.liveStats.promptTokens > 0) { + read = Math.max(read, this.liveStats.promptTokens); + } + + return read; + }); + + contextUsed = $derived(this.currentRead + this.currentOutput); + + contextAvailable = $derived( + this.contextTotal !== null ? this.contextTotal - this.contextUsed : null + ); + + contextPercent = $derived.by(() => { + if (this.contextTotal === null || this.contextTotal <= 0) return null; + + return Math.round((this.contextUsed / this.contextTotal) * 100); + }); cumulativeCacheTotal = $derived(this.cumulative.cacheTotal); - averageTokensPerSecond = $derived(this.cumulative.averageTokensPerSecond); + cumulativeOutput = $derived(this.cumulative.output); + + cumulativeRead = $derived(this.cumulative.read); + + currentCache = $derived.by(() => { + const cached = this.assistantTimings.lastTimings?.cache_n ?? 0; + + if (this.liveStats && this.liveStats.promptTokens > 0) { + return Math.max(cached, this.liveStats.cacheTokens); + } + + return cached; + }); + + currentFresh = $derived.by(() => { + const fresh = this.assistantTimings.lastTimings?.prompt_n ?? 0; + + return Math.max(fresh, this.liveStats?.freshTokens ?? 0); + }); + + isActiveModelLoaded = $derived( + this.activeModelId !== null && + (!serverStore.isRouterMode || modelsStore.isModelLoaded(this.activeModelId)) + ); + + isActiveModelLoading = $derived( + this.activeModelId !== null && modelsStore.status.isOperationInProgress(this.activeModelId) + ); + + kvTotal = $derived(this.currentRead + this.currentOutput); } export const contextStatsStore = new ContextStatsStore(); diff --git a/tools/ui/src/lib/stores/draft-messages.svelte.ts b/tools/ui/src/lib/stores/chat/drafts.svelte.ts similarity index 76% rename from tools/ui/src/lib/stores/draft-messages.svelte.ts rename to tools/ui/src/lib/stores/chat/drafts.svelte.ts index 235a59122e..f480e1efd4 100644 --- a/tools/ui/src/lib/stores/draft-messages.svelte.ts +++ b/tools/ui/src/lib/stores/chat/drafts.svelte.ts @@ -1,3 +1,11 @@ +/** + * DraftMessagesStore - Per-conversation input drafts + * + * Keeps in-memory drafts (message text + files) keyed by conversation id, + * plus a dedicated key for the new-chat screen, so the input box restores + * its content when switching conversations. + */ + import { NEW_CHAT_DRAFT_KEY } from '$lib/constants'; interface DraftMessage { @@ -8,6 +16,12 @@ interface DraftMessage { class DraftMessagesStore { private drafts = new Map(); + clearDraftMessage(chatId: string | undefined): void { + const key = chatId ?? NEW_CHAT_DRAFT_KEY; + + this.drafts.delete(key); + } + getDraftMessage(chatId: string | undefined): DraftMessage { const key = chatId ?? NEW_CHAT_DRAFT_KEY; @@ -23,12 +37,6 @@ class DraftMessagesStore { this.drafts.delete(key); } } - - clearDraftMessage(chatId: string | undefined): void { - const key = chatId ?? NEW_CHAT_DRAFT_KEY; - - this.drafts.delete(key); - } } export const draftMessagesStore = new DraftMessagesStore(); diff --git a/tools/ui/src/lib/stores/chat/flows.svelte.ts b/tools/ui/src/lib/stores/chat/flows.svelte.ts new file mode 100644 index 0000000000..16c377bb61 --- /dev/null +++ b/tools/ui/src/lib/stores/chat/flows.svelte.ts @@ -0,0 +1,794 @@ +/** + * ChatMessageFlows - Message-level flows for the active conversation + * + * Owns the operations that mutate chat history and (re)stream a response: + * editing, regeneration, continuation and deletion of messages. Created and + * owned by chatStore; the host exposes the streaming core and the + * per-conversation state setters these flows drive. + */ + +import { + ContinueIntentKind, + ErrorDialogType, + MessageRole, + MessageType, + StreamConnectionState +} from '$lib/enums'; +import { ChatService } from '$lib/services/chat.service'; +import { DatabaseService } from '$lib/services/database.service'; +import type { ChatProcessingStore } from '$lib/stores/chat/processing.svelte'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import type { + ChatMessagePromptProgress, + ChatMessageTimings, + DatabaseMessage, + DatabaseMessageExtra, + ErrorDialogState +} from '$lib/types'; +import { + classifyContinueIntent, + filterByLeafNodeId, + findDescendantMessages, + findLeafNode, + findMessageById, + isAbortError +} from '$lib/utils'; + +/** + * The slice of chatStore the flows drive. Kept narrow on purpose so the flows + * cannot reach around the host's full surface; chatStore implements this + * structurally. + */ +export interface ChatFlowsHost { + processing: ChatProcessingStore; + streamConnectionState: StreamConnectionState; + cancelPreEncode(): void; + clearChatStreaming(convId: string, messageId?: string): void; + cleanupStreaming(convId: string): void; + createAssistantMessage(parentId?: string): Promise; + getApiOptions(): Record; + getOrCreateAbortController(convId: string): AbortController; + isChatLoadingInternal(convId: string): boolean; + setChatLoading(convId: string, loading: boolean): void; + setChatReasoning(convId: string, reasoning: boolean): void; + setChatStreaming( + convId: string, + response: string, + messageId: string, + model?: string | null + ): void; + showErrorDialog(state: ErrorDialogState | null): void; + stopGeneration(): Promise; + streamChatCompletion( + allMessages: DatabaseMessage[], + assistantMessage: DatabaseMessage, + onComplete?: (content: string) => Promise, + onError?: (error: Error) => void, + modelOverride?: string | null, + firstUserMessageContent?: string + ): Promise; +} + +export class ChatMessageFlows { + constructor(private host: ChatFlowsHost) {} + + async continueAssistantMessage(messageId: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv || this.host.isChatLoadingInternal(activeConv.id)) return; + + const result = this.getMessageByIdWithRole(messageId, MessageRole.ASSISTANT); + + if (!result) return; + + const { index: idx, message: msg } = result; + // Decide which resume path applies. tool_calls without tool results can + // not be resumed mid sequence by continue_final_message, branch instead. + // tool_calls already paired with tool results need a fresh next turn, + // not a token level continuation of the target assistant. + const intent = classifyContinueIntent(conversationsStore.activeMessages, idx); + + if (intent.kind === ContinueIntentKind.RERUN_TURN) { + return this.regenerateMessageWithBranching(messageId); + } + + if (intent.kind === ContinueIntentKind.NEXT_TURN) { + return this.continueAsNextAgenticTurn(intent.truncateAfter); + } + + try { + this.host.showErrorDialog(null); + this.host.setChatLoading(activeConv.id, true); + this.host.clearChatStreaming(activeConv.id); + + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const dbMessage = findMessageById(allMessages, messageId); + + if (!dbMessage) { + this.host.setChatLoading(activeConv.id, false); + + return; + } + + const originalContent = dbMessage.content; + const originalReasoning = dbMessage.reasoningContent || ''; + // Hand the persisted DatabaseMessage straight to sendMessage so its + // internal converter preserves tool_calls and extras when present. + // Reconstructing a bare {role, content} here would drop those fields + // and break continue_final_message for messages with tool calls. + const contextWithContinue = conversationsStore.activeMessages.slice(0, idx + 1); + + let appendedContent = ''; + let appendedReasoning = ''; + let hasReceivedContent = false; + + const updateStreamingContent = (fullContent: string) => { + this.host.setChatStreaming(msg.convId, fullContent, msg.id); + // resolve the row by id on every write, switching to another conv mid continue makes + // this a no op instead of writing positionally into the now displayed conversation + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { + content: fullContent + }); + }; + const abortController = this.host.getOrCreateAbortController(msg.convId); + + await ChatService.sendMessage( + contextWithContinue, + { + ...this.host.getApiOptions(), + continueFinalMessage: true, + onChunk: (chunk: string) => { + appendedContent += chunk; + hasReceivedContent = true; + updateStreamingContent(originalContent + appendedContent); + this.host.setChatReasoning(msg.convId, false); + }, + onComplete: async ( + finalContent?: string, + reasoningContent?: string, + timings?: ChatMessageTimings + ) => { + const finalAppendedContent = hasReceivedContent ? appendedContent : finalContent || ''; + const finalAppendedReasoning = hasReceivedContent + ? appendedReasoning + : reasoningContent || ''; + const fullContent = originalContent + finalAppendedContent; + const fullReasoning = originalReasoning + finalAppendedReasoning || undefined; + + await DatabaseService.updateMessage(msg.id, { + content: fullContent, + reasoningContent: fullReasoning, + timestamp: Date.now(), + timings + }); + + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { + content: fullContent, + reasoningContent: fullReasoning, + timestamp: Date.now(), + timings + }); + + conversationsStore.updateConversationTimestamp(msg.convId); + + this.host.cleanupStreaming(msg.convId); + }, + onCompletionId: (id: string) => { + if (!id) return; + + // refresh the message id so a later skip targets the live slot after a continue + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { + completionId: id + }); + DatabaseService.updateMessage(msg.id, { completionId: id }).catch(() => {}); + }, + onConnectionState: (state: StreamConnectionState) => { + if (msg.convId === conversationsStore.activeConversation?.id) { + this.host.streamConnectionState = state; + } + }, + onError: async (error: Error) => { + if (isAbortError(error)) { + if (hasReceivedContent && appendedContent) { + await DatabaseService.updateMessage(msg.id, { + content: originalContent + appendedContent, + reasoningContent: originalReasoning + appendedReasoning || undefined, + timestamp: Date.now() + }); + + conversationsStore.updateMessageAtIndex( + conversationsStore.findMessageIndex(msg.id), + { + content: originalContent + appendedContent, + reasoningContent: originalReasoning + appendedReasoning || undefined, + timestamp: Date.now() + } + ); + } + + this.host.cleanupStreaming(msg.convId); + + return; + } + + console.error('Continue generation error:', error); + // keep whatever was appended so far, the message stays in memory and in DB + await DatabaseService.updateMessage(msg.id, { + content: originalContent + appendedContent, + reasoningContent: originalReasoning + appendedReasoning || undefined, + timestamp: Date.now() + }); + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { + content: originalContent + appendedContent, + reasoningContent: originalReasoning + appendedReasoning || undefined, + timestamp: Date.now() + }); + + this.host.cleanupStreaming(msg.convId); + this.host.showErrorDialog({ + message: error.message, + type: error.name === 'TimeoutError' ? ErrorDialogType.TIMEOUT : ErrorDialogType.SERVER + }); + }, + onReasoningChunk: (chunk: string) => { + appendedReasoning += chunk; + hasReceivedContent = true; + // mark streaming state so a stop mid-thinking can persist the partial reasoning + this.host.setChatStreaming(msg.convId, originalContent + appendedContent, msg.id); + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { + reasoningContent: originalReasoning + appendedReasoning + }); + this.host.setChatReasoning(msg.convId, true); + }, + onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { + this.host.processing.applyStreamTimings(timings, promptProgress, msg.convId); + } + }, + + msg.convId, + abortController.signal + ); + } catch (error) { + if (!isAbortError(error)) console.error('Failed to continue message:', error); + + if (activeConv) this.host.setChatLoading(activeConv.id, false); + } + } + + async deleteMessage(messageId: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return; + + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const messageToDelete = findMessageById(allMessages, messageId); + + if (!messageToDelete) return; + + const currentPath = filterByLeafNodeId(allMessages, activeConv.currNode || '', false); + const isInCurrentPath = currentPath.some((m) => m.id === messageId); + + if (isInCurrentPath && messageToDelete.parent) { + const siblings = allMessages.filter( + (m) => m.parent === messageToDelete.parent && m.id !== messageId + ); + + if (siblings.length > 0) { + const latestSibling = siblings.reduce((latest, sibling) => + sibling.timestamp > latest.timestamp ? sibling : latest + ); + + await conversationsStore.updateCurrentNode(findLeafNode(allMessages, latestSibling.id)); + } else if (messageToDelete.parent) { + await conversationsStore.updateCurrentNode( + findLeafNode(allMessages, messageToDelete.parent) + ); + } + } + + await DatabaseService.deleteMessageCascading(activeConv.id, messageId); + await conversationsStore.refreshActiveMessages(); + + conversationsStore.updateConversationTimestamp(); + } catch (error) { + console.error('Failed to delete message:', error); + } + } + + async editAssistantMessage( + messageId: string, + newContent: string, + shouldBranch: boolean + ): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv || this.host.isChatLoadingInternal(activeConv.id)) return; + + const result = this.getMessageByIdWithRole(messageId, MessageRole.ASSISTANT); + + if (!result) return; + + const { index: idx, message: msg } = result; + + try { + if (shouldBranch) { + const newMessage = await DatabaseService.createMessageBranch( + { + children: [], + content: newContent, + convId: msg.convId, + model: msg.model, + role: msg.role, + timestamp: Date.now(), + toolCalls: msg.toolCalls || '', + type: msg.type + }, + msg.parent! + ); + + await conversationsStore.updateCurrentNode(newMessage.id); + } else { + await DatabaseService.updateMessage(msg.id, { content: newContent }); + conversationsStore.updateMessageAtIndex(idx, { content: newContent }); + } + + conversationsStore.updateConversationTimestamp(); + + await conversationsStore.refreshActiveMessages(); + } catch (error) { + console.error('Failed to edit assistant message:', error); + } + } + + async editMessageWithBranching( + messageId: string, + newContent: string, + newExtras?: DatabaseMessageExtra[] + ): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv || this.host.isChatLoadingInternal(activeConv.id)) return; + + let result = this.getMessageByIdWithRole(messageId, MessageRole.USER); + + if (!result) result = this.getMessageByIdWithRole(messageId, MessageRole.SYSTEM); + + if (!result) return; + + const { index: idx, message: msg } = result; + + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); + const isFirstUserMessage = + msg.role === MessageRole.USER && rootMessage && msg.parent === rootMessage.id; + const extrasToUse = + newExtras !== undefined + ? JSON.parse(JSON.stringify(newExtras)) + : msg.extra + ? JSON.parse(JSON.stringify(msg.extra)) + : undefined; + + let messageIdForResponse: string; + + const dbMsg = findMessageById(allMessages, msg.id); + const hasChildren = dbMsg ? dbMsg.children.length > 0 : msg.children.length > 0; + + if (!hasChildren) { + // No responses after this message - update in place instead of branching + const updates: Partial = { + content: newContent, + extra: extrasToUse, + timestamp: Date.now() + }; + + await DatabaseService.updateMessage(msg.id, updates); + conversationsStore.updateMessageAtIndex(idx, updates); + messageIdForResponse = msg.id; + } else { + // Has children - create a new branch as sibling + const parentId = msg.parent || rootMessage?.id; + + if (!parentId) return; + + const newMessage = await DatabaseService.createMessageBranch( + { + children: [], + content: newContent, + convId: msg.convId, + extra: extrasToUse, + model: msg.model, + role: msg.role, + timestamp: Date.now(), + toolCalls: msg.toolCalls || '', + type: msg.type + }, + parentId + ); + + await conversationsStore.updateCurrentNode(newMessage.id); + messageIdForResponse = newMessage.id; + } + + conversationsStore.updateConversationTimestamp(); + + if (isFirstUserMessage && newContent.trim()) + await conversationsStore.applyTitleFromContent(activeConv.id, newContent); + + await conversationsStore.refreshActiveMessages(); + + if (msg.role === MessageRole.USER) + await this.generateResponseForMessage(messageIdForResponse); + } catch (error) { + console.error('Failed to edit message with branching:', error); + } + } + + async editUserMessagePreserveResponses( + messageId: string, + newContent: string, + newExtras?: DatabaseMessageExtra[] + ): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return; + + const result = this.getMessageByIdWithRole(messageId, MessageRole.USER); + + if (!result) return; + + const { index: idx, message: msg } = result; + + try { + const updateData: Partial = { content: newContent }; + + if (newExtras !== undefined) updateData.extra = JSON.parse(JSON.stringify(newExtras)); + + await DatabaseService.updateMessage(messageId, updateData); + + conversationsStore.updateMessageAtIndex(idx, updateData); + + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); + + if (rootMessage && msg.parent === rootMessage.id && newContent.trim()) { + await conversationsStore.applyTitleFromContent(activeConv.id, newContent); + } + + conversationsStore.updateConversationTimestamp(); + } catch (error) { + console.error('Failed to edit user message:', error); + } + } + + async getDeletionInfo(messageId: string): Promise<{ + totalCount: number; + userMessages: number; + assistantMessages: number; + messageTypes: string[]; + }> { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) + return { assistantMessages: 0, messageTypes: [], totalCount: 0, userMessages: 0 }; + + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const messageToDelete = findMessageById(allMessages, messageId); + + // For system messages, don't count descendants as they will be preserved (reparented to root) + if (messageToDelete?.role === MessageRole.SYSTEM) { + const messagesToDelete = allMessages.filter((m) => m.id === messageId); + + let assistantMessages = 0, + userMessages = 0; + + const messageTypes: string[] = []; + + for (const msg of messagesToDelete) { + if (msg.role === MessageRole.USER) { + userMessages++; + + if (!messageTypes.includes('user message')) messageTypes.push('user message'); + } else if (msg.role === MessageRole.ASSISTANT) { + assistantMessages++; + + if (!messageTypes.includes('assistant response')) messageTypes.push('assistant response'); + } + } + + return { assistantMessages, messageTypes, totalCount: 1, userMessages }; + } + + const descendants = findDescendantMessages(allMessages, messageId); + const allToDelete = [messageId, ...descendants]; + const messagesToDelete = allMessages.filter((m) => allToDelete.includes(m.id)); + + let assistantMessages = 0, + userMessages = 0; + + const messageTypes: string[] = []; + + for (const msg of messagesToDelete) { + if (msg.role === MessageRole.USER) { + userMessages++; + + if (!messageTypes.includes('user message')) messageTypes.push('user message'); + } else if (msg.role === MessageRole.ASSISTANT) { + assistantMessages++; + + if (!messageTypes.includes('assistant response')) messageTypes.push('assistant response'); + } + } + + return { assistantMessages, messageTypes, totalCount: allToDelete.length, userMessages }; + } + + async regenerateMessage(messageId: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv || this.host.isChatLoadingInternal(activeConv.id)) return; + + this.host.cancelPreEncode(); + const result = this.getMessageByIdWithRole(messageId, MessageRole.ASSISTANT); + + if (!result) return; + + const { index: messageIndex } = result; + + try { + const messagesToRemove = conversationsStore.activeMessages.slice(messageIndex); + + await DatabaseService.deleteMessageCascading(activeConv.id, messagesToRemove[0].id); + conversationsStore.sliceActiveMessages(messageIndex); + conversationsStore.updateConversationTimestamp(); + this.host.setChatLoading(activeConv.id, true); + this.host.clearChatStreaming(activeConv.id); + const parentMessageId = + conversationsStore.activeMessages.length > 0 + ? conversationsStore.activeMessages[conversationsStore.activeMessages.length - 1].id + : undefined; + const assistantMessage = await this.host.createAssistantMessage(parentMessageId); + + conversationsStore.addMessageToActive(assistantMessage); + await this.host.streamChatCompletion( + conversationsStore.activeMessages.slice(0, -1), + assistantMessage + ); + } catch (error) { + if (!isAbortError(error)) console.error('Failed to regenerate message:', error); + + this.host.setChatLoading(activeConv?.id || '', false); + } + } + + async regenerateMessageWithBranching(messageId: string, modelOverride?: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv || this.host.isChatLoadingInternal(activeConv.id)) return; + + this.host.cancelPreEncode(); + try { + const idx = conversationsStore.findMessageIndex(messageId); + + if (idx === -1) return; + + const msg = conversationsStore.activeMessages[idx]; + + if (msg.role !== MessageRole.ASSISTANT) return; + + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const parentMessage = findMessageById(allMessages, msg.parent); + + if (!parentMessage) return; + + this.host.setChatLoading(activeConv.id, true); + this.host.clearChatStreaming(activeConv.id); + const newAssistantMessage = await DatabaseService.createMessageBranch( + { + children: [], + content: '', + convId: msg.convId, + model: null, + role: msg.role, + timestamp: Date.now(), + toolCalls: '', + type: msg.type + }, + parentMessage.id + ); + + await conversationsStore.updateCurrentNode(newAssistantMessage.id); + conversationsStore.updateConversationTimestamp(); + await conversationsStore.refreshActiveMessages(); + const conversationPath = filterByLeafNodeId( + allMessages, + parentMessage.id, + false + ) as DatabaseMessage[]; + const modelToUse = modelOverride || msg.model || undefined; + + await this.host.streamChatCompletion( + conversationPath, + newAssistantMessage, + undefined, + undefined, + modelToUse + ); + } catch (error) { + if (!isAbortError(error)) + console.error('Failed to regenerate message with branching:', error); + + this.host.setChatLoading(activeConv?.id || '', false); + } + } + + async updateMessage(messageId: string, newContent: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return; + + if (this.host.isChatLoadingInternal(activeConv.id)) await this.host.stopGeneration(); + + const result = this.getMessageByIdWithRole(messageId, MessageRole.USER); + + if (!result) return; + + const { index: messageIndex, message: messageToUpdate } = result; + const originalContent = messageToUpdate.content; + + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); + const isFirstUserMessage = rootMessage && messageToUpdate.parent === rootMessage.id; + + conversationsStore.updateMessageAtIndex(messageIndex, { content: newContent }); + await DatabaseService.updateMessage(messageId, { content: newContent }); + + if (isFirstUserMessage && newContent.trim()) + await conversationsStore.applyTitleFromContent(activeConv.id, newContent); + + const messagesToRemove = conversationsStore.activeMessages.slice(messageIndex + 1); + + if (messagesToRemove.length > 0) + await DatabaseService.deleteMessageCascading(activeConv.id, messagesToRemove[0].id); + + conversationsStore.sliceActiveMessages(messageIndex + 1); + conversationsStore.updateConversationTimestamp(); + this.host.setChatLoading(activeConv.id, true); + this.host.clearChatStreaming(activeConv.id); + const assistantMessage = await this.host.createAssistantMessage(); + + conversationsStore.addMessageToActive(assistantMessage); + await conversationsStore.updateCurrentNode(assistantMessage.id); + await this.host.streamChatCompletion( + conversationsStore.activeMessages.slice(0, -1), + assistantMessage, + undefined, + () => { + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(messageId), { + content: originalContent + }); + } + ); + } catch (error) { + if (!isAbortError(error)) console.error('Failed to update message:', error); + } + } + + /** + * Open a fresh assistant turn anchored at the last tool result of a resolved + * agentic round and let streamChatCompletion route through runAgenticFlow. + * Used by continueAssistantMessage when classifyContinueIntent returns + * next_turn, meaning the target assistant already has its tool_calls paired + * with trailing tool results and the next thing to generate is a brand new + * turn rather than a token level continuation. + */ + private async continueAsNextAgenticTurn(anchorIndex: number): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return; + + const anchor = conversationsStore.activeMessages[anchorIndex]; + + if (!anchor) return; + + this.host.cancelPreEncode(); + this.host.setChatLoading(activeConv.id, true); + this.host.clearChatStreaming(activeConv.id); + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const anchorMessage = findMessageById(allMessages, anchor.id); + + if (!anchorMessage) { + this.host.setChatLoading(activeConv.id, false); + + return; + } + + const newAssistantMessage = await DatabaseService.createMessageBranch( + { + children: [], + content: '', + convId: activeConv.id, + model: null, + role: MessageRole.ASSISTANT, + timestamp: Date.now(), + toolCalls: '', + type: MessageType.TEXT + }, + anchorMessage.id + ); + + await conversationsStore.updateCurrentNode(newAssistantMessage.id); + conversationsStore.updateConversationTimestamp(); + await conversationsStore.refreshActiveMessages(); + const conversationPath = filterByLeafNodeId( + allMessages, + anchorMessage.id, + false + ) as DatabaseMessage[]; + + await this.host.streamChatCompletion(conversationPath, newAssistantMessage); + } catch (error) { + if (!isAbortError(error)) console.error('Failed to continue agentic turn:', error); + + this.host.setChatLoading(activeConv.id, false); + } + } + + private async generateResponseForMessage(userMessageId: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return; + + this.host.showErrorDialog(null); + this.host.setChatLoading(activeConv.id, true); + this.host.clearChatStreaming(activeConv.id); + + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const conversationPath = filterByLeafNodeId( + allMessages, + userMessageId, + false + ) as DatabaseMessage[]; + const assistantMessage = await DatabaseService.createMessageBranch( + { + children: [], + content: '', + convId: activeConv.id, + model: null, + role: MessageRole.ASSISTANT, + timestamp: Date.now(), + toolCalls: '', + type: MessageType.TEXT + }, + userMessageId + ); + + conversationsStore.addMessageToActive(assistantMessage); + + await this.host.streamChatCompletion(conversationPath, assistantMessage); + } catch (error) { + console.error('Failed to generate response:', error); + this.host.setChatLoading(activeConv.id, false); + } + } + + private getMessageByIdWithRole( + messageId: string, + expectedRole?: MessageRole + ): { message: DatabaseMessage; index: number } | null { + const index = conversationsStore.findMessageIndex(messageId); + + if (index === -1) return null; + + const message = conversationsStore.activeMessages[index]; + + if (expectedRole && message.role !== expectedRole) return null; + + return { index, message }; + } +} diff --git a/tools/ui/src/lib/stores/chat/index.svelte.ts b/tools/ui/src/lib/stores/chat/index.svelte.ts new file mode 100644 index 0000000000..aab824fd71 --- /dev/null +++ b/tools/ui/src/lib/stores/chat/index.svelte.ts @@ -0,0 +1,1441 @@ +/** + * chatStore - Chat lifecycle, streaming and message operations + * + * Owns the active conversation's chat state: sending messages, streaming + * responses, editing/regeneration flows and per-conversation processing + * activity. Composes the stream manager, message flows, activity ledger and + * processing snapshot; persists through conversationsStore. + * + * Uses ChatService for the API layer and conversationsStore for persistence. + */ + +import { CWD_CLEARED_TEXT, SYSTEM_MESSAGE_PLACEHOLDER, TITLE_GENERATION } from '$lib/constants'; +import { + ErrorDialogType, + MessageRole, + MessageType, + ReasoningEffort, + StreamConnectionState +} from '$lib/enums'; +import { ChatService } from '$lib/services/chat.service'; +import { DatabaseService } from '$lib/services/database.service'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { agenticStore } from '$lib/stores/agentic/index.svelte'; +import { chatActivityStore } from '$lib/stores/chat/activity.svelte'; +import { type ChatFlowsHost, ChatMessageFlows } from '$lib/stores/chat/flows.svelte'; +import { chatProcessingStore } from '$lib/stores/chat/processing.svelte'; +import { type ChatStreamHost, ChatStreamManager } from '$lib/stores/chat/streams.svelte'; +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import { mcpStore } from '$lib/stores/mcp/index.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { serverStore } from '$lib/stores/server.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; +import { toolsStore } from '$lib/stores/tools.svelte'; +import type { + ApiChatMessageData, + ChatMessagePromptProgress, + ChatMessageTimings, + ChatStreamCallbacks, + DatabaseMessage, + DatabaseMessageExtra, + ErrorDialogState +} from '$lib/types'; +import { + findMessageById, + formatCwdMessage, + getConversationModel, + isAbortError, + normalizeModelName +} from '$lib/utils'; +import { SvelteMap } from 'svelte/reactivity'; + +class ChatStore implements ChatStreamHost, ChatFlowsHost { + chatReasoningStates = new SvelteMap(); + chatStreamingStates = new SvelteMap< + string, + { response: string; messageId: string; model?: string | null } + >(); + currentResponse = $state(''); + errorDialogState = $state(null); + // true while the active conversation has a local pipe (send, attach or resume-wait) + isLoading = $derived(this.activity.isLocal(conversationsStore.activeConversation?.id ?? '')); + // true while the active conversation streams reasoning content but no visible content yet + isReasoning = $derived( + this.chatReasoningStates.get(conversationsStore.activeConversation?.id ?? '') ?? false + ); + pendingEditMessageId = $state(null); + // resumable stream connection state for the active conversation + // streaming -> bytes flowing normally, resuming -> waiting on /v1/stream reconnect, lost -> unrecoverable + streamConnectionState = $state(StreamConnectionState.STREAMING); + private abortControllers = new SvelteMap(); + private addFilesHandler: ((files: File[]) => void) | null = $state(null); + // message flows: edit, regenerate, continue, delete + private flows = new ChatMessageFlows(this); + private isEditModeActive = $state(false); + private pendingDraftFiles = $state([]); + private pendingDraftMessage = $state(''); + /** Reactive: queued pending messages for non-agentic streaming */ + private pendingMessages = new SvelteMap< + string, + { content: string; extras?: DatabaseMessageExtra[] } + >(); + private preEncodeAbortController: AbortController | null = null; + + // server-side stream sessions: discovery, attach/replay, resume retry, remote sync + private streams = new ChatStreamManager(this); + + /** Conv activity (local pipe / remote session), composed here. */ + get activity() { + return chatActivityStore; + } + + /** Processing state, composed here so consumers have a single chat scope. */ + get processing() { + return chatProcessingStore; + } + + /** + * Abort the current agentic flow signal without clearing loading state. + * Used by "Send immediately" to force the agentic loop to exit so that + * the pending steering message can be re-sent. + * + * Any tool calls captured mid-stream are dropped before the abort so the + * pending message (or a manual follow-up) does not re-send a half-received + * tool call with invalid JSON arguments to the server. Mirrors what the + * Stop button already does through stopGenerationForChat. + */ + async abortCurrentFlow(convId: string): Promise { + await this.savePartialResponseIfNeeded(convId); + const c = this.abortControllers.get(convId); + + if (c) { + c.abort(); + this.abortControllers.delete(convId); + } + } + + async addMessage( + role: MessageRole, + content: string, + type: MessageType = MessageType.TEXT, + parent: string = '-1', + extras?: DatabaseMessageExtra[], + isSynthetic?: boolean + ): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) throw new Error('No active conversation'); + + let parentId: string | null = null; + + if (parent === '-1') { + const am = conversationsStore.activeMessages; + + if (am.length > 0) parentId = am[am.length - 1].id; + else { + const all = await conversationsStore.getConversationMessages(activeConv.id); + const r = all.find((m) => m.parent === null && m.type === 'root'); + + parentId = r ? r.id : await DatabaseService.createRootMessage(activeConv.id); + } + } else parentId = parent; + + const message = await DatabaseService.createMessageBranch( + { + children: [], + content, + convId: activeConv.id, + extra: extras, + isSynthetic, + role, + timestamp: Date.now(), + toolCalls: '', + type + }, + parentId + ); + + conversationsStore.addMessageToActive(message); + await conversationsStore.updateCurrentNode(message.id); + conversationsStore.updateConversationTimestamp(); + + return message; + } + async addSystemPrompt(): Promise { + let activeConv = conversationsStore.activeConversation; + + if (!activeConv) { + await conversationsStore.createConversation(); + activeConv = conversationsStore.activeConversation; + } + + if (!activeConv) return; + + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); + const rootId = rootMessage + ? rootMessage.id + : await DatabaseService.createRootMessage(activeConv.id); + const existingSystemMessage = allMessages.find( + (m) => m.role === MessageRole.SYSTEM && m.parent === rootId + ); + + if (existingSystemMessage) { + this.pendingEditMessageId = existingSystemMessage.id; + + if (!conversationsStore.activeMessages.some((m) => m.id === existingSystemMessage.id)) + conversationsStore.activeMessages.unshift(existingSystemMessage); + + return; + } + + const am = conversationsStore.activeMessages; + const firstActiveMessage = am.find((m) => m.parent === rootId); + const systemMessage = await DatabaseService.createSystemMessage( + activeConv.id, + SYSTEM_MESSAGE_PLACEHOLDER, + rootId + ); + + if (firstActiveMessage) { + await DatabaseService.updateMessage(firstActiveMessage.id, { + parent: systemMessage.id + }); + await DatabaseService.updateMessage(systemMessage.id, { + children: [firstActiveMessage.id] + }); + const updatedRootChildren = rootMessage + ? rootMessage.children.filter((id: string) => id !== firstActiveMessage.id) + : []; + + await DatabaseService.updateMessage(rootId, { + children: [ + ...updatedRootChildren.filter((id: string) => id !== systemMessage.id), + systemMessage.id + ] + }); + const firstMsgIndex = conversationsStore.findMessageIndex(firstActiveMessage.id); + + if (firstMsgIndex !== -1) + conversationsStore.updateMessageAtIndex(firstMsgIndex, { + parent: systemMessage.id + }); + } + + conversationsStore.activeMessages.unshift(systemMessage); + this.pendingEditMessageId = systemMessage.id; + conversationsStore.updateConversationTimestamp(); + } catch (error) { + console.error('Failed to add system prompt:', error); + } + } + cancelPreEncode(): void { + if (this.preEncodeAbortController) { + this.preEncodeAbortController.abort(); + this.preEncodeAbortController = null; + } + } + + /** + * Resets the loading, streaming and processing state for a conversation + * after a generation ends or errors. Shared by the flows' exit paths. + */ + cleanupStreaming(convId: string): void { + this.setChatLoading(convId, false); + this.clearChatStreaming(convId); + this.processing.setState(convId, null); + } + clearChatStreaming(convId: string, messageId?: string): void { + // session aware: a stale generation must not wipe a newer one's streaming state on the + // same conversation, that would drop the frozen stop identity and stop the wrong session + if (messageId !== undefined) { + const cur = this.chatStreamingStates.get(convId); + + if (cur && cur.messageId !== messageId) return; + } + + this.chatStreamingStates.delete(convId); + + if (convId === conversationsStore.activeConversation?.id) this.currentResponse = ''; + } + clearEditMode(): void { + this.isEditModeActive = false; + this.addFilesHandler = null; + } + + clearPendingEditMessageId(): void { + this.pendingEditMessageId = null; + } + + clearPendingMessage(convId: string): void { + this.pendingMessages.delete(convId); + } + + /** Reset per-view state when (re)mounting the empty chat screen. */ + clearUIState(): void { + this.currentResponse = ''; + } + + consumePendingDraft(): { message: string; files: ChatUploadedFile[] } | null { + if (!this.pendingDraftMessage && this.pendingDraftFiles.length === 0) return null; + + const d = { files: [...this.pendingDraftFiles], message: this.pendingDraftMessage }; + + this.pendingDraftMessage = ''; + this.pendingDraftFiles = []; + + return d; + } + + consumePendingMessage( + convId: string + ): { content: string; extras?: DatabaseMessageExtra[] } | null { + const msg = this.pendingMessages.get(convId); + + if (!msg) return null; + + this.pendingMessages.delete(convId); + + return msg; + } + + async continueAssistantMessage(messageId: string): Promise { + return this.flows.continueAssistantMessage(messageId); + } + + async createAssistantMessage(parentId?: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) throw new Error('No active conversation'); + + return await DatabaseService.createMessageBranch( + { + children: [], + content: '', + convId: activeConv.id, + model: null, + role: MessageRole.ASSISTANT, + timestamp: Date.now(), + toolCalls: '', + type: MessageType.TEXT + }, + parentId || null + ); + } + + async deleteMessage(messageId: string): Promise { + return this.flows.deleteMessage(messageId); + } + + /** + * Server-side stream sessions (discovery, attach/replay, resume retry, + * remote-running snapshot) live in ChatStreamManager. + */ + async discoverActiveStream(convId: string): Promise { + return this.streams.discoverActiveStream(convId); + } + + dismissErrorDialog(): void { + this.errorDialogState = null; + } + + async editAssistantMessage( + messageId: string, + newContent: string, + shouldBranch: boolean + ): Promise { + return this.flows.editAssistantMessage(messageId, newContent, shouldBranch); + } + + async editMessageWithBranching( + messageId: string, + newContent: string, + newExtras?: DatabaseMessageExtra[] + ): Promise { + return this.flows.editMessageWithBranching(messageId, newContent, newExtras); + } + + async editUserMessagePreserveResponses( + messageId: string, + newContent: string, + newExtras?: DatabaseMessageExtra[] + ): Promise { + return this.flows.editUserMessagePreserveResponses(messageId, newContent, newExtras); + } + + getAddFilesHandler(): ((files: File[]) => void) | null { + return this.addFilesHandler; + } + + /** Convs with any activity (local pipe or remote session), sidebar spinners. */ + getAllLoadingChats(): string[] { + return this.activity.loadingConvs; + } + + getApiOptions(): Record { + const currentConfig = settingsStore.config; + const hasValue = (value: unknown): boolean => + value !== undefined && value !== null && value !== ''; + const apiOptions: Record = { stream: true, timings_per_token: true }; + + if (serverStore.isRouterMode) { + const modelName = modelsStore.selectedModelName; + + if (modelName) apiOptions.model = modelName; + } + + if (currentConfig.systemMessage) apiOptions.systemMessage = currentConfig.systemMessage; + + if (currentConfig.disableReasoningParsing) apiOptions.disableReasoningParsing = true; + + if (currentConfig.excludeReasoningFromContext) apiOptions.excludeReasoningFromContext = true; + + // an explicit reasoning choice overrides the server default, DEFAULT sends nothing + const effort = conversationsStore.preferences.getReasoningEffort(); + + if (effort !== ReasoningEffort.DEFAULT) { + apiOptions.enableThinking = effort !== ReasoningEffort.OFF; + + if (effort !== ReasoningEffort.OFF) apiOptions.reasoningEffort = effort; + } + + if (hasValue(currentConfig.temperature)) + apiOptions.temperature = Number(currentConfig.temperature); + + if (hasValue(currentConfig.max_tokens)) + apiOptions.max_tokens = Number(currentConfig.max_tokens); + + if (hasValue(currentConfig.dynatemp_range)) + apiOptions.dynatemp_range = Number(currentConfig.dynatemp_range); + + if (hasValue(currentConfig.dynatemp_exponent)) + apiOptions.dynatemp_exponent = Number(currentConfig.dynatemp_exponent); + + if (hasValue(currentConfig.top_k)) apiOptions.top_k = Number(currentConfig.top_k); + + if (hasValue(currentConfig.top_p)) apiOptions.top_p = Number(currentConfig.top_p); + + if (hasValue(currentConfig.min_p)) apiOptions.min_p = Number(currentConfig.min_p); + + if (hasValue(currentConfig.xtc_probability)) + apiOptions.xtc_probability = Number(currentConfig.xtc_probability); + + if (hasValue(currentConfig.xtc_threshold)) + apiOptions.xtc_threshold = Number(currentConfig.xtc_threshold); + + if (hasValue(currentConfig.typ_p)) apiOptions.typ_p = Number(currentConfig.typ_p); + + if (hasValue(currentConfig.repeat_last_n)) + apiOptions.repeat_last_n = Number(currentConfig.repeat_last_n); + + if (hasValue(currentConfig.repeat_penalty)) + apiOptions.repeat_penalty = Number(currentConfig.repeat_penalty); + + if (hasValue(currentConfig.presence_penalty)) + apiOptions.presence_penalty = Number(currentConfig.presence_penalty); + + if (hasValue(currentConfig.frequency_penalty)) + apiOptions.frequency_penalty = Number(currentConfig.frequency_penalty); + + if (hasValue(currentConfig.dry_multiplier)) + apiOptions.dry_multiplier = Number(currentConfig.dry_multiplier); + + if (hasValue(currentConfig.dry_base)) apiOptions.dry_base = Number(currentConfig.dry_base); + + if (hasValue(currentConfig.dry_allowed_length)) + apiOptions.dry_allowed_length = Number(currentConfig.dry_allowed_length); + + if (hasValue(currentConfig.dry_penalty_last_n)) + apiOptions.dry_penalty_last_n = Number(currentConfig.dry_penalty_last_n); + + if (currentConfig.samplers) apiOptions.samplers = currentConfig.samplers; + + if (hasValue(currentConfig.backend_sampling)) + apiOptions.backend_sampling = currentConfig.backend_sampling; + + if (currentConfig.customJson) apiOptions.custom = currentConfig.customJson; + + return apiOptions; + } + + getChatStreaming(convId: string): { response: string; messageId: string } | undefined { + return this.getChatStreamingState(convId); + } + + async getDeletionInfo(messageId: string): Promise<{ + totalCount: number; + userMessages: number; + assistantMessages: number; + messageTypes: string[]; + }> { + return this.flows.getDeletionInfo(messageId); + } + + getOrCreateAbortController(convId: string): AbortController { + let c = this.abortControllers.get(convId); + + if (!c || c.signal.aborted) { + c = new AbortController(); + this.abortControllers.set(convId, c); + } + + return c; + } + + getPendingMessageContent(convId: string): string | null { + return this.pendingMessages.get(convId)?.content ?? null; + } + + getPendingMessageExtras(convId: string): DatabaseMessageExtra[] | undefined { + return this.pendingMessages.get(convId)?.extras; + } + + getResumeModel(convId: string): string | null { + return this.streams.getResumeModel(convId); + } + + hasPendingDraft(): boolean { + return Boolean(this.pendingDraftMessage) || this.pendingDraftFiles.length > 0; + } + + hasPendingMessage(convId: string): boolean { + return this.pendingMessages.has(convId); + } + + injectPendingMessage(convId: string, content: string, extras?: DatabaseMessageExtra[]): void { + this.pendingMessages.set(convId, { content, extras }); + } + + isChatLoading(convId: string): boolean { + return this.activity.isLocal(convId); + } + + isChatLoadingInternal(convId: string): boolean { + return this.activity.isLocal(convId) || this.chatStreamingStates.has(convId); + } + + isEditing(): boolean { + return this.isEditModeActive; + } + + /** True while the active conversation has a live streaming pipe. */ + isStreaming(): boolean { + return this.chatStreamingStates.has(conversationsStore.activeConversation?.id ?? ''); + } + + /** + * Record a working-directory change into chat history as a synthetic + * user message, so the model sees it on its next turn (the client + * sends the cwd itself via the x-tool-cwd header on tool calls). + * A plain user message is used because some chat templates reject + * tool messages without a preceding tool call. + */ + async recordCwdChange(cwd: string | null): Promise { + const content = cwd + ? formatCwdMessage(cwd, await toolsStore.resolveServerHome()) + : CWD_CLEARED_TEXT; + // Reuse the trailing cwd row when it is already the last message, so + // repeated picks update it in place instead of stacking another row. + const last = conversationsStore.activeMessages[conversationsStore.activeMessages.length - 1]; + + if (last && last.role === MessageRole.USER && last.isSynthetic === true) { + await DatabaseService.updateMessage(last.id, { content, isSynthetic: true }); + conversationsStore.updateMessageAtIndex(conversationsStore.activeMessages.length - 1, { + content, + isSynthetic: true + }); + + return; + } + + await this.addMessage(MessageRole.USER, content, MessageType.TEXT, '-1', undefined, true); + } + + async regenerateMessage(messageId: string): Promise { + return this.flows.regenerateMessage(messageId); + } + + async regenerateMessageWithBranching(messageId: string, modelOverride?: string): Promise { + return this.flows.regenerateMessageWithBranching(messageId, modelOverride); + } + + async removeSystemPromptPlaceholder(messageId: string): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return false; + + try { + const allMessages = await conversationsStore.getConversationMessages(activeConv.id); + const systemMessage = findMessageById(allMessages, messageId); + + if (!systemMessage || systemMessage.role !== MessageRole.SYSTEM) return false; + + const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); + + if (!rootMessage) return false; + + if (allMessages.length === 2 && systemMessage.children.length === 0) { + await conversationsStore.deleteConversation(activeConv.id); + + return true; + } + + for (const childId of systemMessage.children) { + await DatabaseService.updateMessage(childId, { parent: rootMessage.id }); + const childIndex = conversationsStore.findMessageIndex(childId); + + if (childIndex !== -1) + conversationsStore.updateMessageAtIndex(childIndex, { parent: rootMessage.id }); + } + await DatabaseService.updateMessage(rootMessage.id, { + children: [ + ...rootMessage.children.filter((id: string) => id !== messageId), + ...systemMessage.children + ] + }); + await DatabaseService.deleteMessage(messageId); + const systemIndex = conversationsStore.findMessageIndex(messageId); + + if (systemIndex !== -1) conversationsStore.activeMessages.splice(systemIndex, 1); + + conversationsStore.updateConversationTimestamp(); + + return false; + } catch (error) { + console.error('Failed to remove system prompt placeholder:', error); + + return false; + } + } + + savePendingDraft(message: string, files: ChatUploadedFile[]): void { + this.pendingDraftMessage = message; + this.pendingDraftFiles = [...files]; + } + async sendMessage(content: string, extras?: DatabaseMessageExtra[]): Promise { + if (!content.trim() && (!extras || extras.length === 0)) return; + + const activeConv = conversationsStore.activeConversation; + + // If agentic loop is running, inject as a steering message instead of starting a new flow + if (activeConv && agenticStore.isRunning(activeConv.id)) { + agenticStore.injectSteeringMessage(activeConv.id, content, extras); + + return; + } + + // If non-agentic streaming is active, queue as a pending message to send after completion + if (activeConv && this.isChatLoadingInternal(activeConv.id)) { + this.injectPendingMessage(activeConv.id, content, extras); + + return; + } + + // Cancel any in-flight pre-encode request + this.cancelPreEncode(); + + // Consume MCP resource attachments - converts them to extras and clears the live store + const resourceExtras = mcpStore.consumeResourceAttachmentsAsExtras(); + const allExtras = resourceExtras.length > 0 ? [...(extras || []), ...resourceExtras] : extras; + + let isNewConversation = false; + + if (!activeConv) { + await conversationsStore.createConversation(); + isNewConversation = true; + } + + const currentConv = conversationsStore.activeConversation; + + if (!currentConv) return; + + this.showErrorDialog(null); + this.setChatLoading(currentConv.id, true); + this.clearChatStreaming(currentConv.id); + try { + let parentIdForUserMessage: string | undefined; + + if (isNewConversation) { + const rootId = await DatabaseService.createRootMessage(currentConv.id); + const currentConfig = settingsStore.config; + const systemPrompt = currentConfig.systemMessage?.toString().trim(); + + let sysOrRootId = rootId; + + if (systemPrompt) { + const systemMessage = await DatabaseService.createSystemMessage( + currentConv.id, + systemPrompt, + rootId + ); + + conversationsStore.addMessageToActive(systemMessage); + sysOrRootId = systemMessage.id; + } + + // Reflect a working directory picked on the new-chat screen into + // chat history before the first user message, so the model sees + // it on its first turn. createConversation() has already threaded + // the pending pick onto the conversation. + if (currentConv.cwd) { + const cwdMessage = await this.addMessage( + MessageRole.USER, + formatCwdMessage(currentConv.cwd, await toolsStore.resolveServerHome()), + MessageType.TEXT, + sysOrRootId, + undefined, + true + ); + + parentIdForUserMessage = cwdMessage.id; + } else { + parentIdForUserMessage = sysOrRootId; + } + } + + const userMessage = await this.addMessage( + MessageRole.USER, + content, + MessageType.TEXT, + parentIdForUserMessage ?? '-1', + allExtras + ); + + if (isNewConversation && content) + await conversationsStore.applyTitleFromContent(currentConv.id, content); + + const assistantMessage = await this.createAssistantMessage(userMessage.id); + + conversationsStore.addMessageToActive(assistantMessage); + await this.streamChatCompletion( + conversationsStore.activeMessages.slice(0, -1), + assistantMessage, + undefined, + undefined, + undefined, + settingsStore.config.titleGenerationUseLLM && isNewConversation ? content : undefined + ); + } catch (error) { + if (isAbortError(error)) { + this.setChatLoading(currentConv.id, false); + + return; + } + + console.error('Failed to send message:', error); + this.setChatLoading(currentConv.id, false); + const dialogType = + error instanceof Error && error.name === 'TimeoutError' + ? ErrorDialogType.TIMEOUT + : ErrorDialogType.SERVER; + const contextInfo = ( + error as Error & { contextInfo?: { n_prompt_tokens: number; n_ctx: number } } + ).contextInfo; + + this.showErrorDialog({ + contextInfo, + message: error instanceof Error ? error.message : 'Unknown error', + type: dialogType + }); + } + } + + setChatLoading(convId: string, loading: boolean): void { + if (loading) { + this.activity.markLocal(convId); + } else { + this.activity.localEnded(convId); + this.setChatReasoning(convId, false); + } + } + + setChatReasoning(convId: string, reasoning: boolean): void { + if (reasoning) this.chatReasoningStates.set(convId, true); + else this.chatReasoningStates.delete(convId); + } + + setChatStreaming( + convId: string, + response: string, + messageId: string, + model?: string | null + ): void { + this.chatStreamingStates.set(convId, { + messageId, + model: model ?? this.chatStreamingStates.get(convId)?.model, + response + }); + + if (convId === conversationsStore.activeConversation?.id) this.currentResponse = response; + } + + setEditModeActive(handler: (files: File[]) => void): void { + this.isEditModeActive = true; + this.addFilesHandler = handler; + } + + showErrorDialog(state: ErrorDialogState | null): void { + this.errorDialogState = state; + } + + async stopGeneration(): Promise { + const activeConv = conversationsStore.activeConversation; + + if (!activeConv) return; + + await this.stopGenerationForChat(activeConv.id); + } + + async stopGenerationForChat(convId: string): Promise { + await this.savePartialResponseIfNeeded(convId); + // tell the server to stop the generation, not just drop the HTTP socket. without this the + // detached drain keeps producing tokens until eos or max_tokens. use the frozen identity + // captured when the session started, not the live dropdown + const streamStateForStop = this.chatStreamingStates.get(convId); + const modelForStop = streamStateForStop?.model ?? ChatService.getStreamState(convId)?.model; + + void ChatService.cancelServerStream(convId, modelForStop); + // an explicit stop leaves nothing to resume and kills a pending resume retry + ChatService.clearStreamState(convId); + this.streams.cancelResumeRetry(convId); + this.abortRequest(convId); + this.setChatLoading(convId, false); + this.clearChatStreaming(convId); + this.processing.setState(convId, null); + this.clearPendingMessage(convId); + } + + async streamChatCompletion( + allMessages: DatabaseMessage[], + assistantMessage: DatabaseMessage, + onComplete?: (content: string) => Promise, + onError?: (error: Error) => void, + modelOverride?: string | null, + firstUserMessageContent?: string + ): Promise { + // the ::model suffix in the stream identity is only for router mode, where it routes to the + // owning child. in single-model mode the identity stays the bare conv id so that attach, stop + // and reattach all agree, regardless of fresh send vs regenerate passing a resolved model + let effectiveModel: string | null | undefined = undefined; + + if (serverStore.isRouterMode) { + const conversationModel = getConversationModel(allMessages); + + effectiveModel = modelOverride || modelsStore.selectedModelName || conversationModel; + } + + if (serverStore.isRouterMode && effectiveModel) { + if (!modelsStore.props.getModelProps(effectiveModel)) + await modelsStore.props.fetchModelProps(effectiveModel); + } + + // Mutable state for the current message being streamed + let currentMessageId = assistantMessage.id; + let streamedContent = ''; + let streamedReasoningContent = ''; + let resolvedModel: string | null = null; + let modelPersisted = false; + + const convId = assistantMessage.convId; + + // Tracks the last message created in this flow. Used as the parent for the next + // turn's assistant message so createAssistantMessage does not have to read + // conversationsStore.activeMessages, which may belong to a different conversation + // after the user navigates while the loop is still running. + let lastCreatedInFlow = currentMessageId; + + // freeze the POST identity from t0 so a stop cancels with the exact session key, + // never a stale or empty model resolved later + this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); + + const recordModel = (modelName: string | null | undefined, persistImmediately = true): void => { + if (!modelName) return; + + const n = normalizeModelName(modelName); + + if (!n || n === resolvedModel) return; + + resolvedModel = n; + const idx = conversationsStore.findMessageIndex(currentMessageId); + + conversationsStore.updateMessageAtIndex(idx, { model: n }); + + if (persistImmediately && !modelPersisted) { + modelPersisted = true; + DatabaseService.updateMessage(currentMessageId, { model: n }).catch(() => { + modelPersisted = false; + resolvedModel = null; + }); + } + }; + + let completionIdRecorded = false; + + const recordCompletionId = (id: string): void => { + if (!id || completionIdRecorded) return; + + completionIdRecorded = true; + const idx = conversationsStore.findMessageIndex(currentMessageId); + + conversationsStore.updateMessageAtIndex(idx, { completionId: id }); + DatabaseService.updateMessage(currentMessageId, { completionId: id }).catch(() => { + completionIdRecorded = false; + }); + }; + const updateStreamingUI = () => { + this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); + const idx = conversationsStore.findMessageIndex(currentMessageId); + + conversationsStore.updateMessageAtIndex(idx, { content: streamedContent }); + }; + const cleanupStreamingState = () => { + this.setChatLoading(convId, false); + this.clearChatStreaming(convId, currentMessageId); + this.processing.setState(convId, null); + }; + + this.processing.setActiveConversation(convId); + const abortController = this.getOrCreateAbortController(convId); + const streamCallbacks: ChatStreamCallbacks = { + createAssistantMessage: async () => { + // Reset streaming state for new message + streamedContent = ''; + streamedReasoningContent = ''; + + const msg = await DatabaseService.createMessageBranch( + { + children: [], + content: '', + convId, + model: resolvedModel, + role: MessageRole.ASSISTANT, + timestamp: Date.now(), + toolCalls: '', + type: MessageType.TEXT + }, + lastCreatedInFlow + ); + + if (conversationsStore.activeConversation?.id === convId) { + conversationsStore.addMessageToActive(msg); + } + + currentMessageId = msg.id; + lastCreatedInFlow = msg.id; + + return msg; + }, + createToolResultMessage: async ( + toolCallId: string, + content: string, + extras?: DatabaseMessageExtra[], + toolCwd?: string + ) => { + const msg = await DatabaseService.createMessageBranch( + { + children: [], + content, + convId, + extra: extras, + role: MessageRole.TOOL, + timestamp: Date.now(), + toolCallId, + toolCalls: '', + toolCwd, + type: MessageType.TEXT + }, + currentMessageId + ); + + // mirror into the active store and move the node pointer only when this + // conversation is displayed; otherwise persist the node move straight to + // the db for the owning conv so a foreign conv's currNode stays untouched + if (conversationsStore.activeConversation?.id === convId) { + conversationsStore.addMessageToActive(msg); + await conversationsStore.updateCurrentNode(msg.id); + } else { + await DatabaseService.updateCurrentNode(convId, msg.id); + } + + lastCreatedInFlow = msg.id; + + return msg; + }, + onAssistantTurnComplete: async ( + content: string, + reasoningContent: string | undefined, + timings: ChatMessageTimings | undefined, + toolCalls: import('$lib/types/api').ApiChatCompletionToolCall[] | undefined + ) => { + const updateData: Record = { + content, + reasoningContent: reasoningContent || undefined, + timings, + toolCalls: toolCalls ? JSON.stringify(toolCalls) : '' + }; + + if (resolvedModel && !modelPersisted) updateData.model = resolvedModel; + + await DatabaseService.updateMessage(currentMessageId, updateData); + const idx = conversationsStore.findMessageIndex(currentMessageId); + const uiUpdate: Partial = { + content, + reasoningContent: reasoningContent || undefined, + toolCalls: toolCalls ? JSON.stringify(toolCalls) : '' + }; + + if (timings) uiUpdate.timings = timings; + + if (resolvedModel) uiUpdate.model = resolvedModel; + + // touch the active ui array and node pointer only when this conversation + // is displayed; otherwise persist the node move straight to the db so a + // foreign conv's currNode stays untouched + if (conversationsStore.activeConversation?.id === convId) { + conversationsStore.updateMessageAtIndex(idx, uiUpdate); + await conversationsStore.updateCurrentNode(currentMessageId); + } else { + await DatabaseService.updateCurrentNode(convId, currentMessageId); + } + }, + onAttachments: (messageId: string, extras: DatabaseMessageExtra[]) => { + if (!extras.length) return; + + const idx = conversationsStore.findMessageIndex(messageId); + + if (idx === -1) return; + + const msg = conversationsStore.activeMessages[idx]; + const updatedExtras = [...(msg.extra || []), ...extras]; + + conversationsStore.updateMessageAtIndex(idx, { extra: updatedExtras }); + DatabaseService.updateMessage(messageId, { extra: updatedExtras }).catch(console.error); + }, + onChunk: (chunk: string) => { + streamedContent += chunk; + updateStreamingUI(); + this.setChatReasoning(convId, false); + }, + onCompletionId: (id: string) => recordCompletionId(id), + onError: async (error: Error) => { + if (isAbortError(error)) { + cleanupStreamingState(); + // If aborted with a pending message (e.g. "Send immediately"), re-send it + const pending = this.consumePendingMessage(convId); + + if (pending) { + this.sendMessage(pending.content, pending.extras); + } + + return; + } + + console.error('Streaming error:', error); + // keep whatever was streamed so far, the message stays in memory and in DB + await this.savePartialResponseIfNeeded(convId); + cleanupStreamingState(); + this.clearPendingMessage(convId); + + const contextInfo = ( + error as Error & { contextInfo?: { n_prompt_tokens: number; n_ctx: number } } + ).contextInfo; + + this.showErrorDialog({ + contextInfo, + message: error.message, + type: error.name === 'TimeoutError' ? ErrorDialogType.TIMEOUT : ErrorDialogType.SERVER + }); + + if (onError) onError(error); + }, + onFlowComplete: (finalTimings?: ChatMessageTimings) => { + if (finalTimings) { + const idx = conversationsStore.findMessageIndex(assistantMessage.id); + + conversationsStore.updateMessageAtIndex(idx, { timings: finalTimings }); + DatabaseService.updateMessage(assistantMessage.id, { + timings: finalTimings + }).catch(console.error); + } + + cleanupStreamingState(); + + if (onComplete) onComplete(streamedContent); + + if (serverStore.isRouterMode) modelsStore.fetchRouterModels().catch(console.error); + + // Pre-encode conversation in KV cache for faster next turn + if (settingsStore.config.preEncodeConversation) { + this.triggerPreEncode( + allMessages, + assistantMessage, + streamedContent, + effectiveModel, + !!settingsStore.config.excludeReasoningFromContext + ); + } + }, + onModel: (modelName: string) => recordModel(modelName), + onReasoningChunk: (chunk: string) => { + streamedReasoningContent += chunk; + // mark streaming state so a stop mid-thinking can persist the partial reasoning + this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); + const idx = conversationsStore.findMessageIndex(currentMessageId); + + conversationsStore.updateMessageAtIndex(idx, { + reasoningContent: streamedReasoningContent + }); + this.setChatReasoning(convId, true); + }, + onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { + this.processing.applyStreamTimings(timings, promptProgress, convId); + }, + onToolCallsStreaming: (toolCalls) => { + const idx = conversationsStore.findMessageIndex(currentMessageId); + + conversationsStore.updateMessageAtIndex(idx, { + toolCalls: JSON.stringify(toolCalls) + }); + }, + onTurnComplete: (intermediateTimings: ChatMessageTimings) => { + // Update the first assistant message with cumulative agentic timings + const idx = conversationsStore.findMessageIndex(assistantMessage.id); + + conversationsStore.updateMessageAtIndex(idx, { timings: intermediateTimings }); + }, + updateToolResultMessage: async ( + messageId: string, + content: string, + extras?: DatabaseMessageExtra[] + ) => { + // Persist latest content + merged extras; mirror into the active + // store so the chat view sees live updates for streaming tools + // (e.g. exec_shell_command). The existing tool message node + // pointer stays put - the renderer is already scoped to it. + const updates: Partial = { content }; + + if (extras) { + const idx = conversationsStore.findMessageIndex(messageId); + const existing = idx >= 0 ? (conversationsStore.activeMessages[idx]?.extra ?? []) : []; + const merged = [...existing, ...extras]; + + updates.extra = merged; + } + + if (conversationsStore.activeConversation?.id === convId) { + const idx = conversationsStore.findMessageIndex(messageId); + + if (idx >= 0) conversationsStore.updateMessageAtIndex(idx, updates); + } + + await DatabaseService.updateMessage(messageId, updates); + } + }; + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); + + { + const agenticResult = await agenticStore.runAgenticFlow({ + callbacks: streamCallbacks, + conversationId: convId, + flowRootMessageId: assistantMessage.id, + messages: allMessages, + options: { + ...this.getApiOptions(), + ...(effectiveModel ? { model: effectiveModel } : {}) + }, + perChatOverrides, + signal: abortController.signal + }); + + if (agenticResult.handled) { + // Generate LLM based title for new conversations after agentic flow completes + if (firstUserMessageContent) { + await this.generateTitleWithLLM(firstUserMessageContent, streamedContent, convId); + } + + // Check if there's a pending steering message to re-send + const pending = agenticStore.consumePendingSteeringMessage(convId); + + if (pending) { + await this.sendMessage(pending.content, pending.extras); + } + + return; + } + } + + await ChatService.sendMessage( + allMessages, + { + ...this.getApiOptions(), + ...(effectiveModel ? { model: effectiveModel } : {}), + onChunk: streamCallbacks.onChunk, + onComplete: async ( + finalContent?: string, + reasoningContent?: string, + timings?: ChatMessageTimings, + toolCalls?: string + ) => { + const content = streamedContent || finalContent || ''; + const reasoning = streamedReasoningContent || reasoningContent; + const updateData: Record = { + content, + reasoningContent: reasoning || undefined, + timings, + toolCalls: toolCalls || '' + }; + + if (resolvedModel && !modelPersisted) updateData.model = resolvedModel; + + await DatabaseService.updateMessage(currentMessageId, updateData); + const idx = conversationsStore.findMessageIndex(currentMessageId); + const uiUpdate: Partial = { + content, + reasoningContent: reasoning || undefined, + toolCalls: toolCalls || '' + }; + + if (timings) uiUpdate.timings = timings; + + if (resolvedModel) uiUpdate.model = resolvedModel; + + conversationsStore.updateMessageAtIndex(idx, uiUpdate); + await conversationsStore.updateCurrentNode(currentMessageId); + cleanupStreamingState(); + + if (onComplete) await onComplete(content); + + if (serverStore.isRouterMode) modelsStore.fetchRouterModels().catch(console.error); + + // Generate LLM based title for new conversations (avoids stale reference + // issue when user switches conversations while streaming) + if (firstUserMessageContent) { + await this.generateTitleWithLLM(firstUserMessageContent, streamedContent, convId); + } + + // Check if there's a pending message queued during streaming + const pending = this.consumePendingMessage(convId); + + if (pending) { + await this.sendMessage(pending.content, pending.extras); + } + }, + onCompletionId: streamCallbacks.onCompletionId, + onConnectionState: (state: StreamConnectionState) => { + if (convId === conversationsStore.activeConversation?.id) { + this.streamConnectionState = state; + } + }, + onError: streamCallbacks.onError, + onModel: streamCallbacks.onModel, + onReasoningChunk: streamCallbacks.onReasoningChunk, + onTimings: streamCallbacks.onTimings, + stream: true + }, + convId, + abortController.signal + ); + } + + syncLoadingStateForChat(convId: string): void { + const s = this.chatStreamingStates.get(convId); + + this.currentResponse = s?.response || ''; + this.processing.setActiveConversation(convId); + + // Sync streaming content to activeMessages so UI displays current content + if (s?.response && s?.messageId) { + const idx = conversationsStore.findMessageIndex(s.messageId); + + if (idx !== -1) { + conversationsStore.updateMessageAtIndex(idx, { content: s.response }); + } + } + } + + async syncRemoteRunningStreams(): Promise { + return this.streams.syncRemoteRunningStreams(); + } + + /** + * Message flows (edit / regenerate / continue / delete) live in + * ChatMessageFlows; these delegate so consumers keep a single entry point. + */ + async updateMessage(messageId: string, newContent: string): Promise { + return this.flows.updateMessage(messageId, newContent); + } + private abortRequest(convId?: string): void { + if (convId) { + const c = this.abortControllers.get(convId); + + if (c) { + c.abort(); + this.abortControllers.delete(convId); + } + } else { + for (const c of this.abortControllers.values()) c.abort(); + this.abortControllers.clear(); + } + } + + private async generateTitleWithLLM( + userContent: string, + assistantContent: string, + convId: string + ): Promise { + const effectiveModel = + serverStore.isRouterMode && modelsStore.selectedModelName + ? modelsStore.selectedModelName + : undefined; + const configValue = settingsStore.config; + const titlePromptTemplate = + typeof configValue.titleGenerationPrompt === 'string' && + configValue.titleGenerationPrompt.trim() + ? configValue.titleGenerationPrompt + : TITLE_GENERATION.DEFAULT_PROMPT; + const titlePrompt = titlePromptTemplate + .replace('{{USER}}', String(userContent || '')) + .replace('{{ASSISTANT}}', String(assistantContent || '')); + const titleMessage: ApiChatMessageData = { + content: titlePrompt, + role: MessageRole.USER + }; + const titleResponse = await ChatService.generateTitle(titleMessage, effectiveModel); + + if (!titleResponse) { + return; + } + + let cleanTitle = titleResponse.trim(); + + cleanTitle = cleanTitle + .replace(TITLE_GENERATION.PREFIX_PATTERN, '') + .replace(TITLE_GENERATION.QUOTE_PATTERN, '') + .trim(); + + if (!cleanTitle || cleanTitle.length < TITLE_GENERATION.MIN_LENGTH) { + const firstLine = userContent.split('\n').find((l) => l.trim().length > 0); + + cleanTitle = firstLine ? firstLine.trim() : TITLE_GENERATION.FALLBACK; + } + + if (cleanTitle && cleanTitle.length >= TITLE_GENERATION.MIN_LENGTH) { + await conversationsStore.updateConversationName(convId, cleanTitle); + } + } + + private getChatStreamingState( + convId: string + ): { response: string; messageId: string } | undefined { + return this.chatStreamingStates.get(convId); + } + + private async savePartialResponseIfNeeded(convId?: string): Promise { + const conversationId = convId || conversationsStore.activeConversation?.id; + + if (!conversationId) return; + + const streamingState = this.getChatStreamingState(conversationId); + + if (!streamingState) return; + + const messages = + conversationId === conversationsStore.activeConversation?.id + ? conversationsStore.activeMessages + : await conversationsStore.getConversationMessages(conversationId); + + if (!messages.length) return; + + const lastMessage = messages[messages.length - 1]; + + if (lastMessage?.role !== MessageRole.ASSISTANT) return; + + const partialContent = streamingState.response; + const partialReasoning = lastMessage.reasoningContent || ''; + // snapshot the streamed tool calls before clearing so we still know whether + // anything was captured when deciding to skip the DB write below + const hadPartialToolCalls = !!lastMessage.toolCalls?.trim(); + + // nothing to persist when content, reasoning, and streamed tool calls are all empty + // (e.g. stop before any token). otherwise drop the partial tool call and write whatever + // was streamed: incomplete arguments (truncated JSON, missing closing quote) would + // otherwise be re-sent to the server on the next turn and rejected. + if (!partialContent.trim() && !partialReasoning.trim() && !hadPartialToolCalls) return; + + try { + const updateData: { + content?: string; + reasoningContent?: string; + toolCalls?: string; + timings?: ChatMessageTimings; + } = { + toolCalls: '' + }; + + if (partialContent.trim()) updateData.content = partialContent; + + if (partialReasoning.trim()) updateData.reasoningContent = partialReasoning; + + const lastKnownState = this.processing.getState(conversationId); + + if (lastKnownState) { + updateData.timings = { + cache_n: lastKnownState.cacheTokens || 0, + predicted_ms: + lastKnownState.tokensPerSecond && lastKnownState.tokensDecoded + ? (lastKnownState.tokensDecoded / lastKnownState.tokensPerSecond) * 1000 + : undefined, + predicted_n: lastKnownState.tokensDecoded || 0, + prompt_ms: lastKnownState.promptMs, + prompt_n: lastKnownState.promptTokens || 0 + }; + } + + await DatabaseService.updateMessage(lastMessage.id, updateData); + lastMessage.content = partialContent; + // mirror the drop into the in-memory message so the next request sent via + // sendMessage (queued pending, Send immediately, or manual follow-up) reads + // the cleared value, not whatever the streaming widget had been showing + lastMessage.toolCalls = ''; + + if (updateData.timings) lastMessage.timings = updateData.timings; + } catch (error) { + lastMessage.content = partialContent; + lastMessage.toolCalls = ''; + console.error('Failed to save partial response:', error); + } + } + + private async triggerPreEncode( + allMessages: DatabaseMessage[], + assistantMessage: DatabaseMessage, + assistantContent: string, + model?: string | null, + excludeReasoning?: boolean + ): Promise { + this.cancelPreEncode(); + this.preEncodeAbortController = new AbortController(); + + const signal = this.preEncodeAbortController.signal; + + try { + const allIdle = await ChatService.areAllSlotsIdle(model, signal); + + if (!allIdle || signal.aborted) return; + + const messagesWithAssistant: DatabaseMessage[] = [ + ...allMessages, + { ...assistantMessage, content: assistantContent } + ]; + + await ChatService.preEncode(messagesWithAssistant, model, excludeReasoning, signal); + } catch (err) { + if (!isAbortError(err)) { + console.warn('[ChatStore] Pre-encode failed:', err); + } + } + } +} + +export const chatStore = new ChatStore(); diff --git a/tools/ui/src/lib/stores/chat/processing.svelte.ts b/tools/ui/src/lib/stores/chat/processing.svelte.ts new file mode 100644 index 0000000000..69c1a69256 --- /dev/null +++ b/tools/ui/src/lib/stores/chat/processing.svelte.ts @@ -0,0 +1,188 @@ +/** + * chatProcessingStore - Per-conversation processing state + * + * Owns the live processing snapshot shown while a conversation streams: + * token counts, tokens/sec, prompt progress. Updated from stream timings, + * restored from persisted message timings when a conversation loads. + * + * Composed under chatStore.processing; not exported from the stores barrel. + */ + +import { MessageRole } from '$lib/enums'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { serverStore } from '$lib/stores/server.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; +import type { + ApiProcessingState, + ChatMessagePromptProgress, + ChatMessageTimings, + DatabaseMessage +} from '$lib/types'; +import { SvelteMap } from 'svelte/reactivity'; + +interface ProcessingTimingData { + cache_n: number; + predicted_n: number; + predicted_per_second: number; + prompt_ms?: number; + prompt_n: number; + prompt_progress?: ChatMessagePromptProgress; +} + +export class ChatProcessingStore { + private _activeConversationId = $state(null); + private states = new SvelteMap(); + + /** Processing state of the conversation currently shown in the UI. */ + activeState = $derived( + this._activeConversationId ? (this.states.get(this._activeConversationId) ?? null) : null + ); + + get activeConversationId(): string | null { + return this._activeConversationId; + } + + /** + * Applies a stream timings event (tokens/sec + token counts) to the given + * conversation's processing state. Shared by the chat and continue flows. + */ + applyStreamTimings( + timings?: ChatMessageTimings, + promptProgress?: ChatMessagePromptProgress, + conversationId?: string + ): void { + const tokensPerSecond = + timings?.predicted_ms && timings?.predicted_n + ? (timings.predicted_n / timings.predicted_ms) * 1000 + : 0; + + this.updateFromTimings( + { + cache_n: timings?.cache_n || 0, + predicted_n: timings?.predicted_n || 0, + predicted_per_second: tokensPerSecond, + prompt_ms: timings?.prompt_ms, + prompt_n: timings?.prompt_n || 0, + prompt_progress: promptProgress + }, + conversationId + ); + } + + getConversationIds(): string[] { + return Array.from(this.states.keys()); + } + + getState(conversationId: string): ApiProcessingState | null { + return this.states.get(conversationId) ?? null; + } + + restoreFromMessages(messages: DatabaseMessage[], conversationId: string): void { + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i]; + + if (message.role === MessageRole.ASSISTANT && message.timings) { + this.setState( + conversationId, + this.parseTimingData({ + cache_n: message.timings.cache_n || 0, + predicted_n: message.timings.predicted_n || 0, + predicted_per_second: + message.timings.predicted_n && message.timings.predicted_ms + ? (message.timings.predicted_n / message.timings.predicted_ms) * 1000 + : 0, + prompt_ms: message.timings.prompt_ms, + prompt_n: message.timings.prompt_n || 0 + }) + ); + + return; + } + } + } + + setActiveConversation(conversationId: string | null): void { + this._activeConversationId = conversationId; + } + + /** Passing null clears the state for the conversation. */ + setState(conversationId: string, state: ApiProcessingState | null): void { + if (state === null) this.states.delete(conversationId); + else this.states.set(conversationId, state); + } + + updateFromTimings(timingData: ProcessingTimingData, conversationId?: string): void { + const targetId = conversationId || this._activeConversationId; + + if (targetId) { + this.setState(targetId, this.parseTimingData(timingData)); + } + } + + private getContextTotal(): number | null { + const activeConvId = this._activeConversationId; + const activeState = activeConvId ? this.getState(activeConvId) : null; + + if (activeState && typeof activeState.contextTotal === 'number' && activeState.contextTotal > 0) + return activeState.contextTotal; + + if (serverStore.isRouterMode) { + const modelContextSize = modelsStore.selectedModelContextSize; + + if (typeof modelContextSize === 'number' && modelContextSize > 0) { + return modelContextSize; + } + } else { + const propsContextSize = serverStore.contextSize; + + if (typeof propsContextSize === 'number' && propsContextSize > 0) { + return propsContextSize; + } + } + + return null; + } + + private parseTimingData(timingData: ProcessingTimingData): ApiProcessingState { + const cacheTokens = timingData.cache_n || 0, + predictedTokens = timingData.predicted_n || 0, + promptMs = timingData.prompt_ms || undefined, + promptTokens = timingData.prompt_n || 0, + tokensPerSecond = timingData.predicted_per_second || 0; + const promptProgress = timingData.prompt_progress; + const contextTotal = this.getContextTotal(); + const currentConfig = settingsStore.config; + const outputTokensMax = currentConfig.max_tokens || -1; + const contextUsed = promptTokens + cacheTokens + predictedTokens, + outputTokensUsed = predictedTokens; + const progressCache = promptProgress?.cache || 0, + progressActualDone = (promptProgress?.processed ?? 0) - progressCache, + progressActualTotal = (promptProgress?.total ?? 0) - progressCache; + const progressPercent = promptProgress + ? Math.round((progressActualDone / progressActualTotal) * 100) + : undefined; + + return { + cacheTokens, + contextTotal, + contextUsed, + hasNextToken: predictedTokens > 0, + outputTokensMax, + outputTokensUsed, + progressPercent, + promptMs, + promptProgress, + promptTokens, + speculative: false, + status: predictedTokens > 0 ? 'generating' : promptProgress ? 'preparing' : 'idle', + temperature: currentConfig.temperature ?? 0.8, + tokensDecoded: predictedTokens, + tokensPerSecond, + tokensRemaining: outputTokensMax - predictedTokens, + topP: currentConfig.top_p ?? 0.95 + }; + } +} + +export const chatProcessingStore = new ChatProcessingStore(); diff --git a/tools/ui/src/lib/stores/chat/streams.svelte.ts b/tools/ui/src/lib/stores/chat/streams.svelte.ts new file mode 100644 index 0000000000..5abbc81fb6 --- /dev/null +++ b/tools/ui/src/lib/stores/chat/streams.svelte.ts @@ -0,0 +1,494 @@ +/** + * ChatStreamManager - Server-side stream sessions for conversations + * + * Owns the attach lifecycle for streams that live on the server: discovery, + * replay from byte 0, and resume retry while the owning model loads. The + * remote-running snapshot it produces feeds the chat activity ledger + * (chatStore.activity), which owns the actual running-conv state. Created + * and owned by chatStore; the host exposes the per-conversation state setters. + */ + +import { CONVERSATION_ID_SEPARATOR, STREAM_RESUME_RETRY_MS } from '$lib/constants'; +import { MessageRole, MessageType, StreamConnectionState } from '$lib/enums'; +import { ChatService } from '$lib/services/chat.service'; +import { DatabaseService } from '$lib/services/database.service'; +import type { ChatActivityStore } from '$lib/stores/chat/activity.svelte'; +import type { ChatProcessingStore } from '$lib/stores/chat/processing.svelte'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import type { ApiStreamSession, ChatMessageTimings, DatabaseMessage } from '$lib/types'; +import { streamIdentity } from '$lib/utils'; +import { SvelteMap, SvelteSet } from 'svelte/reactivity'; + +/** + * The slice of chatStore the manager drives. Kept narrow on purpose so the + * manager cannot reach around the host's full surface; chatStore implements + * this structurally. + */ +export interface ChatStreamHost { + activity: ChatActivityStore; + processing: ChatProcessingStore; + chatStreamingStates: SvelteMap< + string, + { response: string; messageId: string; model?: string | null } + >; + streamConnectionState: StreamConnectionState; + getOrCreateAbortController(convId: string): AbortController; + setChatLoading(convId: string, loading: boolean): void; + setChatStreaming( + convId: string, + response: string, + messageId: string, + model?: string | null + ): void; + clearChatStreaming(convId: string, messageId?: string): void; +} + +export class ChatStreamManager { + // in-flight discoverActiveStream guard, keyed by conv id + private discoveringConvs = new SvelteSet(); + // convs whose resume waits on a model load: their loading state belongs to the retry loop, + // so discoverActiveStream must not treat it as a live send and bail + private resumePendingConvs = new SvelteSet(); + // pending resume retry timers while an owning model loads, one per conv + private resumeRetryTimers = new SvelteMap>(); + + /** Kill a pending resume retry, e.g. on explicit stop. */ + cancelResumeRetry(convId: string): void { + const timer = this.resumeRetryTimers.get(convId); + + if (timer !== undefined) { + clearTimeout(timer); + this.resumeRetryTimers.delete(convId); + } + + this.resumePendingConvs.delete(convId); + } + + constructor(private host: ChatStreamHost) {} + + async discoverActiveStream(convId: string): Promise { + if (!convId) return; + + if (this.host.chatStreamingStates.has(convId)) return; + + if (this.host.activity.isLocal(convId) && !this.resumePendingConvs.has(convId)) return; + + // concurrency guard: another discover may already be running for this conv (typical race + // between mount and visibilitychange on tab switch). a second concurrent fetch on the same + // /v1/stream would duplicate every byte into the DB message, this guard bounces it + if (this.discoveringConvs.has(convId)) return; + + this.discoveringConvs.add(convId); + + try { + // the model is frozen at POST time, rebuild the exact conv::model identity from the + // persisted state so the lookup key matches what the server stored. null means a single + // model conv with no ::suffix, only guess from the dropdown with no persisted state + const localState = ChatService.getStreamState(convId); + const streamId = ChatService.resumeStreamIdentity( + convId, + localState, + modelsStore.selectedModelName + ); + // primary path: ask the server which sessions exist for this identity + const serverTarget = await this.probeServerStream(streamId); + + if (serverTarget) { + // pass the full server side identity (may carry a ::model suffix) so the GET routes + // straight to the owning session, no probe or fan out + await this.attachServerStream(convId, serverTarget.conversation_id); + + return; + } + + // fallback: local state remembers an interrupted byte offset for this conv, the server may + // still have a live session matching that identity (we just lost the bytes mid stream). retry + // with the frozen identity, the server probe inside attachServerStream tells us if it exists + if (!localState) { + return; + } + + // quiet status probe first: a full attach flips the loading UI on every try, probing + // keeps the retry loop invisible while the owning model is still loading (503) + const status = await ChatService.probeResumeStatus(streamId); + + if (status === 503) { + // make the wait visible: the empty assistant row persisted at send time renders + // the processing info, whose model load percentage flows from the models feed + this.resumePendingConvs.add(convId); + this.host.setChatLoading(convId, true); + + if (!this.resumeRetryTimers.has(convId)) { + this.resumeRetryTimers.set( + convId, + setTimeout(() => { + this.resumeRetryTimers.delete(convId); + void this.discoverActiveStream(convId); + }, STREAM_RESUME_RETRY_MS) + ); + } + + return; + } + + if (this.resumePendingConvs.delete(convId) && status !== 200) { + // the wait is over without a session to attach, drop the visible loading state + this.host.setChatLoading(convId, false); + } + + if (status === 0) { + // transient network failure, the next mount or visibility change retries + return; + } + + if (status !== 200) { + // the session is gone (stopped, TTL expired), nothing to resume anymore + ChatService.clearStreamState(convId); + + return; + } + + await this.attachServerStream(convId, streamId); + + // if attachServerStream failed (session gone, TTL expired), clear the local state to avoid retrying forever + if (!this.host.chatStreamingStates.has(convId) && !this.host.activity.isLocal(convId)) { + ChatService.clearStreamState(convId); + } + } finally { + this.discoveringConvs.delete(convId); + } + } + + /** + * Model frozen at send time for a stream awaiting resume, from the persisted stream state. + * The load progress indicator targets it after a reload, when the message row has no model + * yet and the dropdown selection may not be restored. + */ + getResumeModel(convId: string): string | null { + return ChatService.getStreamState(convId)?.model ?? null; + } + + /** + * Resync the activity ledger's remote set from the backend. Called by the layout at mount and + * on visibilitychange, no polling. A snapshot semantic: stale entries for sessions that + * finalized while the browser was elsewhere are dropped naturally. + */ + async syncRemoteRunningStreams(): Promise { + // the conversations store loads from IndexedDB asynchronously, the +layout onMount caller + // fires before that finishes. read ids straight from the DB so the result does not depend + // on the store init race, and the sidebar spinners light up at first paint for every conv + // the user owns even if it has not been hydrated into the store yet + let ids: string[]; + + try { + const all = await DatabaseService.getAllConversations(); + + ids = all.map((c) => c.id).filter((id) => !!id); + } catch (e) { + console.warn('syncRemoteRunningStreams DB read failed:', e); + + return; + } + + // only ask about conv ids the user already owns + if (ids.length === 0) { + this.host.activity.applyRemoteSnapshot([]); + + return; + } + + // rebuild the frozen conv::model identity per conv so a session started with a model still + // matches. the server response is mapped back to the bare id below for the sidebar set + const lookupIds = ids.map((id) => + ChatService.resumeStreamIdentity(id, ChatService.getStreamState(id), null) + ); + + let sessions: ApiStreamSession[]; + + try { + sessions = await ChatService.lookupStreamSessions(lookupIds); + } catch (e) { + console.warn('syncRemoteRunningStreams lookup failed:', e); + + return; + } + const running = new SvelteSet(); + + for (const s of sessions) { + if (s && !s.is_done && typeof s.conversation_id === 'string' && s.conversation_id) { + // strip the optional ::model suffix, the sidebar set is keyed by the bare conv id + const sepIdx = s.conversation_id.indexOf(CONVERSATION_ID_SEPARATOR); + const bareId = sepIdx === -1 ? s.conversation_id : s.conversation_id.slice(0, sepIdx); + + running.add(bareId); + } + } + this.host.activity.applyRemoteSnapshot(running); + } + + private async attachServerStream(convId: string, streamId?: string): Promise { + if (!convId) return; + + if (this.host.chatStreamingStates.has(convId)) return; + + // flip the spinner immediately, the user sees activity as soon as the conv becomes active + this.host.setChatLoading(convId, true); + + // only set the active processing conv if we are looking at it, otherwise a background + // attach would steal the indicator from the conv the user is currently viewing + if (convId === conversationsStore.activeConversation?.id) { + this.host.processing.setActiveConversation(convId); + } + + const unlock = () => { + this.host.setChatLoading(convId, false); + this.host.clearChatStreaming(convId); + }; + // fetch the replay stream from byte 0, rebuild the assistant message from scratch. + // resolve the server side identity, fall back to streamIdentity when the caller does not + // pass a streamId. probeServerStream returns the full id (with ::model suffix when present) + const id = streamId || streamIdentity(convId, modelsStore.selectedModelName); + + let response: Response; + + try { + response = await ChatService.fetchStreamReplay(id); + } catch (e) { + console.error(`attachServerStream replay failed for conv ${convId}:`, e); + unlock(); + + return; + } + + // load the target conversation messages by id, not via the active store. when multiple + // attaches run in parallel the active store may reflect another conv and writing through + // its index mixes content across convs (CoT flicker, message bleed). by going through the + // DB we stay isolated, and only mirror into the active store when the attached conv is + // the one currently displayed + let messages: DatabaseMessage[]; + + try { + messages = await DatabaseService.getConversationMessages(convId); + } catch (e) { + console.error('attachServerStream load messages failed:', e); + unlock(); + + return; + } + + // locate the slot to splice into, create a placeholder assistant message if there is none. + // we use the conv-scoped findLastAssistantIdx helpers, they only depend on the array + let targetIdx = this.findLastAssistantIdx(messages); + + if (targetIdx === -1) { + const lastUserIdx = this.findLastUserIdx(messages); + + if (lastUserIdx === -1) { + console.warn( + `attachServerStream: conv ${convId} has no user or assistant message, cannot splice` + ); + unlock(); + + return; + } + + try { + const placeholder = await DatabaseService.createMessageBranch( + { + children: [], + content: '', + convId, + parent: messages[lastUserIdx].id, + role: MessageRole.ASSISTANT, + timestamp: Date.now(), + toolCalls: '', + type: MessageType.TEXT + } as Omit, + messages[lastUserIdx].id + ); + + messages = [...messages, placeholder]; + targetIdx = messages.length - 1; + + // only push into the active store when this conv is the one displayed right now + if (convId === conversationsStore.activeConversation?.id) { + conversationsStore.addMessageToActive(placeholder); + } + } catch (e) { + console.error('attachServerStream placeholder creation failed:', e); + unlock(); + + return; + } + } + + if (targetIdx === -1) { + unlock(); + + return; + } + + const targetMessage = messages[targetIdx]; + const targetMessageId = targetMessage.id; + // when the assistant slot already has content, the running session is a continue or + // another append flow and its buffer holds only the appended deltas. preserve the prefix + // and let the replay add to it. when the slot is empty the session buffer holds the whole + // message so we wipe and rebuild from byte 0 + const existingContent = targetMessage.content ?? ''; + const existingReasoning = targetMessage.reasoningContent ?? ''; + const isAppendMode = existingContent.length > 0; + // helper: write to the active store only when the attached conv is currently displayed. + // the lookup by message id is robust to reordering of activeMessages, two parallel attaches + // can no longer step on each other's indices + const writeActive = (updates: Partial) => { + if (convId !== conversationsStore.activeConversation?.id) { + return; + } + + const liveIdx = conversationsStore.findMessageIndex(targetMessageId); + + if (liveIdx === -1) return; + + conversationsStore.updateMessageAtIndex(liveIdx, updates); + }; + + if (!isAppendMode) { + writeActive({ content: '', reasoningContent: undefined }); + } + + // extract the model suffix, the resume calls in handleStreamResponse must reuse the model + // the session was tagged with, not the live dropdown + const sepIdx = id.indexOf(CONVERSATION_ID_SEPARATOR); + const attachedModel: string | null = sepIdx === -1 ? null : id.slice(sepIdx + 2); + + this.host.setChatStreaming(convId, existingContent, targetMessageId, attachedModel); + const abortController = this.host.getOrCreateAbortController(convId); + + let streamedContent = ''; + let streamedReasoningContent = ''; + + const cleanup = () => { + unlock(); + this.host.processing.setState(convId, null); + }; + + try { + await ChatService.handleStreamResponse( + response, + (chunk: string) => { + streamedContent += chunk; + const displayed = isAppendMode ? existingContent + streamedContent : streamedContent; + + writeActive({ content: displayed }); + this.host.setChatStreaming(convId, displayed, targetMessageId); + }, + async ( + finalContent?: string, + reasoningContent?: string, + timings?: ChatMessageTimings, + toolCalls?: string + ) => { + const streamed = streamedContent || finalContent || ''; + const streamedR = streamedReasoningContent || reasoningContent || ''; + const content = isAppendMode ? existingContent + streamed : streamed; + const reasoning = isAppendMode ? existingReasoning + streamedR : streamedR; + + // the DB write is the source of truth, mirror to the active store only when + // the conv is currently displayed + await DatabaseService.updateMessage(targetMessageId, { + content, + reasoningContent: reasoning || undefined, + timings, + toolCalls: toolCalls || '' + }); + writeActive({ + content, + reasoningContent: reasoning || undefined, + timings + }); + cleanup(); + }, + (err: Error) => { + console.error('attachServerStream pipe error:', err); + cleanup(); + }, + (chunk: string) => { + streamedReasoningContent += chunk; + const displayed = isAppendMode + ? existingReasoning + streamedReasoningContent + : streamedReasoningContent; + + writeActive({ reasoningContent: displayed }); + }, + undefined, + undefined, + undefined, + undefined, + convId, + abortController.signal, + (connState: StreamConnectionState) => { + if (convId === conversationsStore.activeConversation?.id) { + this.host.streamConnectionState = connState; + } + }, + attachedModel + ); + } catch (e) { + console.error('attachServerStream pipe crashed:', e); + cleanup(); + } + } + + private findLastAssistantIdx(messages: DatabaseMessage[]): number { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === MessageRole.ASSISTANT) return i; + } + + return -1; + } + + private findLastUserIdx(messages: DatabaseMessage[]): number { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === MessageRole.USER) return i; + } + + return -1; + } + + /** + * Server side stream discovery, split in three pieces: + * + * probeServerStream(convId) -> hits POST /v1/streams/lookup with the conv id, returns the session to attach + * to or null. Pure read, no side effect, no UI lock. Safe to fire in parallel with anything. + * + * attachServerStream(convId) -> flips the spinner immediately, fetches the replay stream + * from byte 0, finds the assistant slot to splice into (creates a placeholder if the conv has + * no assistant message yet, for cross device or fresh local DB cases), and pipes the SSE bytes + * into the message via handleStreamResponse. + * + * discoverActiveStream(convId) -> probe + attach in one call. Used by callers that do not need + * to overlap the probe with other async work. + * + * The chat page in +page.svelte calls discoverActiveStream once the conversation is active + * (immediately if it already is, after loadConversation settles otherwise), and re-runs it on + * visibilitychange. Attaching only after the conversation is loaded gives the earliest + * possible time to spinner and avoids racing against an empty activeMessages array. + */ + private async probeServerStream(convId: string): Promise { + if (!convId) return null; + + let sessions: ApiStreamSession[]; + + try { + sessions = await ChatService.lookupStreamSessions([convId]); + } catch (e) { + console.warn(`probeServerStream failed for conv ${convId}:`, e); + + return null; + } + + return ChatService.selectActiveStream(sessions); + } +} diff --git a/tools/ui/src/lib/stores/conversations.svelte.ts b/tools/ui/src/lib/stores/conversations/index.svelte.ts similarity index 61% rename from tools/ui/src/lib/stores/conversations.svelte.ts rename to tools/ui/src/lib/stores/conversations/index.svelte.ts index d2184b3593..7d6dc326c4 100644 --- a/tools/ui/src/lib/stores/conversations.svelte.ts +++ b/tools/ui/src/lib/stores/conversations/index.svelte.ts @@ -1,135 +1,67 @@ /** - * conversationsStore - Reactive State Store for Conversations + * conversationsStore - Conversation lifecycle, persistence and navigation * - * Manages conversation lifecycle, persistence, navigation, and MCP server overrides. - * - * **Architecture & Relationships:** - * - **DatabaseService**: Stateless IndexedDB layer - * - **conversationsStore** (this): Reactive state + business logic - * - **chatStore**: Chat-specific state (streaming, loading) - * - * **Key Responsibilities:** - * - Conversation CRUD (create, load, delete) - * - Message management and tree navigation - * - MCP server per-chat overrides - * - Import/Export functionality - * - Title management with confirmation - * - * @see DatabaseService in services/database.ts for IndexedDB operations + * Owns conversation CRUD, message tree navigation, import/export and title + * management, persisted through DatabaseService. Per-chat options (MCP + * overrides, reasoning effort, cwd) live in ConversationPreferences, + * composed as {@link ConversationsStore.preferences}. */ import { browser } from '$app/environment'; import { goto } from '$app/navigation'; -import { REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY, ROUTES } from '$lib/constants'; -import { MessageRole, ReasoningEffort } from '$lib/enums'; +import { ROUTES } from '$lib/constants'; +import { MessageRole } from '$lib/enums'; import { ConversationTransferService } from '$lib/services/conversation-transfer.service'; import { DatabaseService } from '$lib/services/database.service'; import { MigrationService } from '$lib/services/migration.service'; import { RouterService } from '$lib/services/router.service'; // direct imports between stores, not via the barrel, to avoid circular deps -import { mcpStore } from '$lib/stores/mcp.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; -import type { McpServerOverride } from '$lib/types/database'; +import { + ConversationPreferences, + type ConversationsPreferencesHost +} from '$lib/stores/conversations/preferences.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { filterByLeafNodeId, findLeafNode, generateConversationTitle } from '$lib/utils'; import { SvelteSet } from 'svelte/reactivity'; import { toast } from 'svelte-sonner'; -class ConversationsStore { - /** - * - * - * State - * - * - */ - - /** List of all conversations */ - conversations = $state([]); - +class ConversationsStore implements ConversationsPreferencesHost { /** Currently active conversation */ activeConversation = $state(null); /** Messages in the active conversation (filtered by currNode path) */ activeMessages = $state([]); + /** List of all conversations */ + conversations = $state([]); + /** Whether the store has been initialized */ isInitialized = $state(false); - /** Global (non-conversation-specific) reasoning effort default */ - pendingReasoningEffort = $state(ConversationsStore.loadReasoningEffortDefault()); + /** Per-chat options (MCP overrides, reasoning effort, cwd), composed here. */ + private _preferences = new ConversationPreferences(this); /** - * Working directory picked on the empty new-chat screen, before any - * conversation exists. Consumed by `chatStore.sendMessage()`, which - * records it into chat history as a synthetic message on first send. - * Cleared by `loadConversation` and `clearActiveConversation` so a - * stale pick can't bleed onto an unrelated chat. + * Listeners notified with the ids of conversations that were deleted. + * Lets dependent stores (e.g. agenticStore) drop per-conversation state + * without introducing a circular import back into this store. */ - pendingCwd = $state(null); - - /** Load reasoning effort default from localStorage, DEFAULT defers to the server */ - private static loadReasoningEffortDefault(): ReasoningEffort { - if (typeof globalThis.localStorage === 'undefined') return ReasoningEffort.DEFAULT; - - try { - const raw = localStorage.getItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY); - - return (raw as ReasoningEffort) || ReasoningEffort.DEFAULT; - } catch { - return ReasoningEffort.DEFAULT; - } - } - - /** Persist reasoning effort default to localStorage */ - private saveReasoningEffortDefaults(): void { - if (typeof globalThis.localStorage === 'undefined') return; - - localStorage.setItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY, this.pendingReasoningEffort); - } + private conversationDeletionListeners = new Set<(convIds: string[]) => void>(); /** In-flight init run; shared by concurrent callers, reset on failure to allow retry */ private initPromise: Promise | null = null; /** - * - * - * Lifecycle - * - * + * Memo of the last findMessageIndex() lookup. Streaming calls it once per + * chunk for the same message, so a validated cache hit keeps that O(1) + * instead of a linear scan of activeMessages on every token. */ + private lastMessageIndex: { id: string; index: number } | null = null; - /** - * Initialize the store by loading conversations from database. - * Safe to call multiple times: concurrent callers share a single run, - * and a failed run can be retried by calling again. - */ - init(): Promise { - if (!browser) return Promise.resolve(); - - if (this.initPromise) return this.initPromise; - - this.initPromise = (async () => { - try { - await MigrationService.runAllMigrations(); - await this.loadConversations(); - this.isInitialized = true; - } catch (error) { - console.error('Failed to initialize conversations:', error); - this.initPromise = null; - } - })(); - - return this.initPromise; + get preferences() { + return this._preferences; } - /** - * - * - * Message Array Operations - * - * - */ - /** * Adds a message to the active messages array */ @@ -138,221 +70,37 @@ class ConversationsStore { } /** - * Updates a message at a specific index in active messages + * Applies a field update to a conversation row, mirroring it into both the + * conversations list and the active conversation when it is the target. + * Shared by the rename/pin/preferences flows so no caller can forget to + * mirror one side. */ - updateMessageAtIndex(index: number, updates: Partial): void { - const message = index === -1 ? undefined : this.activeMessages[index]; + applyConversationUpdate(id: string, updates: Partial): void { + const convIndex = this.conversations.findIndex((c) => c.id === id); - if (!message) return; + if (convIndex !== -1) { + const target = this.conversations[convIndex] as unknown as Record; - // Assign field by field rather than replacing the object. Replacing it - // changes the array slot, which invalidates every consumer that merely - // walks the list - notably ChatMessages.displayMessages, which rebuilds - // entries for every message in the conversation. Deep $state proxies make - // per-field writes fine-grained, so only readers of the changed field wake. - const target = message as unknown as Record; - - for (const [key, value] of Object.entries(updates)) { - if (target[key] !== value) { - target[key] = value; + for (const [key, value] of Object.entries(updates)) { + if (target[key] !== value) target[key] = value; } } - } - /** - * Finds the index of a message in active messages - */ - findMessageIndex(messageId: string): number { - return this.activeMessages.findIndex((m) => m.id === messageId); - } - - /** - * Removes messages from active messages starting at an index - */ - sliceActiveMessages(startIndex: number): void { - this.activeMessages = this.activeMessages.slice(0, startIndex); - } - - /** - * Removes a message from active messages by index - */ - removeMessageAtIndex(index: number): DatabaseMessage | undefined { - if (index !== -1) { - return this.activeMessages.splice(index, 1)[0]; - } - - return undefined; - } - - /** - * - * - * Conversation CRUD - * - * - */ - - /** - * Loads all conversations from the database - */ - async loadConversations(): Promise { - const conversations = await DatabaseService.getAllConversations(); - - this.conversations = conversations; - } - - /** - * Creates a new conversation and navigates to it - * @param name - Optional name for the conversation - * @returns The ID of the created conversation - */ - async createConversation(name?: string): Promise { - const conversationName = name || `Chat ${new Date().toLocaleString()}`; - // No MCP override list is seeded: getAllMcpServerOverrides resolves - // servers without a per-conversation override to `mcpServers[i].enabled`, - // and only explicit toggles are stored on the conversation. - // Working directory picked on the new-chat screen gets threaded in - // here too, then cleared so it doesn't bleed onto subsequent new chats. - const conversation = await DatabaseService.createConversation(conversationName, { - cwd: this.pendingCwd ?? undefined, - reasoningEffort: this.pendingReasoningEffort - }); - - this.pendingCwd = null; - - this.conversations = [conversation, ...this.conversations]; - this.activeConversation = conversation; - this.activeMessages = []; - - await goto(RouterService.chat(conversation.id)); - - return conversation.id; - } - - /** - * Loads a specific conversation and its messages - * @param convId - The conversation ID to load - * @returns True if conversation was loaded successfully - */ - async loadConversation(convId: string): Promise { - try { - const conversation = await DatabaseService.getConversation(convId); - - if (!conversation) { - return false; - } - - // Drop any cwd the user drafted on the empty new-chat screen - - // it doesn't belong to this conversation. - this.pendingCwd = null; - - this.activeConversation = conversation; - - if (conversation.currNode) { - const allMessages = await DatabaseService.getConversationMessages(convId); - const filteredMessages = filterByLeafNodeId( - allMessages, - conversation.currNode, - false - ) as DatabaseMessage[]; - - this.activeMessages = filteredMessages; - } else { - const messages = await DatabaseService.getConversationMessages(convId); - - this.activeMessages = messages; - } - - return true; - } catch (error) { - console.error('Failed to load conversation:', error); - - return false; + if (this.activeConversation?.id === id) { + this.activeConversation = { ...this.activeConversation, ...updates }; } } /** - * Clears the active conversation and messages. + * Derives a conversation title from its first message content and applies + * it, honoring the title-generation setting. Shared by every flow that + * edits or creates the first user message. */ - clearActiveConversation(): void { - this.activeConversation = null; - this.activeMessages = []; - // reload defaults so new chats inherit persisted state - this.pendingReasoningEffort = ConversationsStore.loadReasoningEffortDefault(); - this.pendingCwd = null; - } - - /** - * Deletes a conversation and all its messages - * @param convId - The conversation ID to delete - */ - async deleteConversation(convId: string, options?: { deleteWithForks?: boolean }): Promise { - try { - await DatabaseService.deleteConversation(convId, options); - - if (options?.deleteWithForks) { - // Collect all descendants recursively - const idsToRemove = new SvelteSet([convId]); - const queue = [convId]; - - while (queue.length > 0) { - const parentId = queue.pop()!; - - for (const c of this.conversations) { - if (c.forkedFromConversationId === parentId && !idsToRemove.has(c.id)) { - idsToRemove.add(c.id); - queue.push(c.id); - } - } - } - this.conversations = this.conversations.filter((c) => !idsToRemove.has(c.id)); - - if (this.activeConversation && idsToRemove.has(this.activeConversation.id)) { - this.clearActiveConversation(); - await goto(ROUTES.NEW_CHAT); - } - } else { - // Reparent direct children to deleted conv's parent (or promote to top-level) - const deletedConv = this.conversations.find((c) => c.id === convId); - const newParent = deletedConv?.forkedFromConversationId; - - this.conversations = this.conversations - .filter((c) => c.id !== convId) - .map((c) => - c.forkedFromConversationId === convId - ? { ...c, forkedFromConversationId: newParent } - : c - ); - - if (this.activeConversation?.id === convId) { - this.clearActiveConversation(); - await goto(ROUTES.NEW_CHAT); - } - } - } catch (error) { - console.error('Failed to delete conversation:', error); - } - } - - /** - * Deletes all conversations and their messages - */ - async deleteAll(): Promise { - try { - const allConversations = await DatabaseService.getAllConversations(); - - await DatabaseService.bulkDeleteConversations(allConversations.map((c) => c.id)); - - this.clearActiveConversation(); - this.conversations = []; - - toast.success('All conversations deleted'); - - await goto(ROUTES.NEW_CHAT); - } catch (error) { - console.error('Failed to delete all conversations:', error); - toast.error('Failed to delete conversations'); - } + async applyTitleFromContent(convId: string, content: string): Promise { + await this.updateConversationName( + convId, + generateConversationTitle(content, Boolean(settingsStore.config.titleGenerationUseFirstLine)) + ); } /** @@ -387,6 +135,7 @@ class ConversationsStore { await DatabaseService.bulkDeleteConversations([...idsToRemove]); this.conversations = this.conversations.filter((c) => !idsToRemove.has(c.id)); + this.notifyConversationsDeleted([...idsToRemove]); if (activeWasDeleted) { this.clearActiveConversation(); @@ -404,43 +153,6 @@ class ConversationsStore { } } - /** - * Toggles the pinned state of each conversation individually. - * Mixed-pin selections are intentionally not normalised here; the bulk - * action UI surfaces them as a disabled mixed-state instead. - * @param convIds - Conversation IDs to toggle - */ - async bulkToggleConversationPin(convIds: string[]): Promise { - if (convIds.length === 0) return; - - try { - const updates = await DatabaseService.bulkToggleConversationPins(convIds); - const activeId = this.activeConversation?.id; - - if (activeId && updates.has(activeId)) { - this.activeConversation = { - ...this.activeConversation!, - pinned: updates.get(activeId)! - }; - } - - for (let i = 0; i < this.conversations.length; i++) { - const newPinned = updates.get(this.conversations[i].id); - - if (newPinned !== undefined) this.conversations[i].pinned = newPinned; - } - - toast.success( - convIds.length === 1 - ? 'Conversation pin toggled' - : `Updated pin state for ${convIds.length} conversations` - ); - } catch (error) { - console.error('Failed to bulk toggle pin:', error); - toast.error('Failed to update pin state'); - } - } - /** * Bundles the given conversations into a single zip archive and triggers a * browser download (one JSONL file per conversation). @@ -480,418 +192,203 @@ class ConversationsStore { } /** - * - * - * Message Management - * - * + * Toggles the pinned state of each conversation individually. + * Mixed-pin selections are intentionally not normalised here; the bulk + * action UI surfaces them as a disabled mixed-state instead. + * @param convIds - Conversation IDs to toggle */ + async bulkToggleConversationPin(convIds: string[]): Promise { + if (convIds.length === 0) return; - /** - * Refreshes active messages based on currNode after branch navigation. - */ - async refreshActiveMessages(): Promise { - if (!this.activeConversation) return; - - const allMessages = await DatabaseService.getConversationMessages(this.activeConversation.id); - - if (allMessages.length === 0) { - this.activeMessages = []; - - return; - } - - const leafNodeId = - this.activeConversation.currNode || - allMessages.reduce((latest, msg) => (msg.timestamp > latest.timestamp ? msg : latest)).id; - const currentPath = filterByLeafNodeId(allMessages, leafNodeId, false) as DatabaseMessage[]; - - this.activeMessages = currentPath; - } - - /** - * Gets all messages for a specific conversation - * @param convId - The conversation ID - * @returns Array of messages - */ - async getConversationMessages(convId: string): Promise { - return await DatabaseService.getConversationMessages(convId); - } - - /** - * - * - * Title Management - * - * - */ - - /** - * Updates the name of a conversation. - * @param convId - The conversation ID to update - * @param name - The new name for the conversation - */ - async updateConversationName(convId: string, name: string): Promise { try { - await DatabaseService.updateConversation(convId, { name }); + const updates = await DatabaseService.bulkToggleConversationPins(convIds); + const activeId = this.activeConversation?.id; - const convIndex = this.conversations.findIndex((c) => c.id === convId); - - if (convIndex !== -1) { - this.conversations[convIndex].name = name; + if (activeId && updates.has(activeId)) { + this.activeConversation = { + ...this.activeConversation!, + pinned: updates.get(activeId)! + }; } - if (this.activeConversation?.id === convId) { - this.activeConversation = { ...this.activeConversation, name }; - } - } catch (error) { - console.error('Failed to update conversation name:', error); - } - } + for (let i = 0; i < this.conversations.length; i++) { + const newPinned = updates.get(this.conversations[i].id); - /** - * Toggles the pinned status of a conversation. - * @param convId - The conversation ID to toggle - * @returns The new pinned status - */ - async toggleConversationPin(convId: string): Promise { - try { - const newPinnedState = await DatabaseService.toggleConversationPin(convId); - const convIndex = this.conversations.findIndex((c) => c.id === convId); - - if (convIndex !== -1) { - this.conversations[convIndex].pinned = newPinnedState; + if (newPinned !== undefined) this.conversations[i].pinned = newPinned; } - if (this.activeConversation?.id === convId) { - this.activeConversation = { ...this.activeConversation, pinned: newPinnedState }; - } - - return newPinnedState; - } catch (error) { - console.error('Failed to toggle conversation pin:', error); - - return false; - } - } - - /** - * Marks a conversation as recently active: stamps lastModified (persisted) - * and moves it to the top of the list. Only message-activity flows call - * this; metadata updates (rename, pin, settings) do not. - * - * @param convId - Conversation that produced the activity, defaults to the active one - */ - updateConversationTimestamp(convId?: string): void { - const targetId = convId ?? this.activeConversation?.id; - - if (!targetId) return; - - const now = Date.now(); - const chatIndex = this.conversations.findIndex((c) => c.id === targetId); - - if (chatIndex !== -1) { - this.conversations[chatIndex].lastModified = now; - const updatedConv = this.conversations.splice(chatIndex, 1)[0]; - - this.conversations = [updatedConv, ...this.conversations]; - } - - if (this.activeConversation?.id === targetId) { - this.activeConversation = { ...this.activeConversation, lastModified: now }; - } - - DatabaseService.updateConversation(targetId, { lastModified: now }).catch((error) => - console.error('Failed to update conversation timestamp:', error) - ); - } - - /** - * Updates the current node of the active conversation - * @param nodeId - The new current node ID - */ - async updateCurrentNode(nodeId: string): Promise { - if (!this.activeConversation) return; - - await DatabaseService.updateCurrentNode(this.activeConversation.id, nodeId); - this.activeConversation = { ...this.activeConversation, currNode: nodeId }; - } - - /** - * - * - * Branch Navigation - * - * - */ - - /** - * Navigates to a specific sibling branch by updating currNode and refreshing messages. - * @param siblingId - The sibling message ID to navigate to - */ - async navigateToSibling(siblingId: string): Promise { - if (!this.activeConversation) return; - - const allMessages = await DatabaseService.getConversationMessages(this.activeConversation.id); - const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); - const currentFirstUserMessage = this.activeMessages.find( - (m) => m.role === MessageRole.USER && m.parent === rootMessage?.id - ); - const currentLeafNodeId = findLeafNode(allMessages, siblingId); - - await DatabaseService.updateCurrentNode(this.activeConversation.id, currentLeafNodeId); - this.activeConversation = { ...this.activeConversation, currNode: currentLeafNodeId }; - await this.refreshActiveMessages(); - - if (rootMessage && this.activeMessages.length > 0) { - const newFirstUserMessage = this.activeMessages.find( - (m) => m.role === MessageRole.USER && m.parent === rootMessage.id + toast.success( + convIds.length === 1 + ? 'Conversation pin toggled' + : `Updated pin state for ${convIds.length} conversations` ); - - if ( - newFirstUserMessage && - newFirstUserMessage.content.trim() && - (!currentFirstUserMessage || - newFirstUserMessage.id !== currentFirstUserMessage.id || - newFirstUserMessage.content.trim() !== currentFirstUserMessage.content.trim()) - ) { - await this.updateConversationName( - this.activeConversation.id, - generateConversationTitle( - newFirstUserMessage.content, - Boolean(settingsStore.config.titleGenerationUseFirstLine) - ) - ); - } + } catch (error) { + console.error('Failed to bulk toggle pin:', error); + toast.error('Failed to update pin state'); } } /** - * - * - * MCP Server Overrides - * - * + * Clears the active conversation and messages. */ - - /** - * Resolve the default enabled value for a server: its own `enabled` - * flag in `mcpServers`, so the global on/off state lives in one place. - */ - #getDefaultOverride(serverId: string): McpServerOverride | undefined { - const server = mcpStore.getServers().find((s) => s.id === serverId); - - if (!server) return undefined; - - return { enabled: server.enabled, serverId }; + clearActiveConversation(): void { + this.activeConversation = null; + this.activeMessages = []; + // reload defaults so new chats inherit persisted state + this.preferences.resetPending(); } /** - * Gets the effective MCP server override for a specific server. - * A per-conversation override wins when present; a server without one - * resolves to its `mcpServers[i].enabled` default. - * @param serverId - The server ID to check - * @returns The effective override, undefined if no matching server + * Creates a new conversation and navigates to it + * @param name - Optional name for the conversation + * @returns The ID of the created conversation */ - getMcpServerOverride(serverId: string): McpServerOverride | undefined { - const override = this.activeConversation?.mcpServerOverrides?.find( - (o: McpServerOverride) => o.serverId === serverId - ); - - if (override) return override; - - return this.#getDefaultOverride(serverId); - } - - /** - * Gets the effective override list for the current conversation: - * one entry per configured server, resolved per server. The stored - * per-conversation list is sparse and only holds explicit toggles. - */ - getAllMcpServerOverrides(): McpServerOverride[] { - const overrides = this.activeConversation?.mcpServerOverrides; - - return mcpStore.getServers().map((s) => { - const override = overrides?.find((o: McpServerOverride) => o.serverId === s.id); - - return { enabled: override?.enabled ?? s.enabled, serverId: s.id }; + async createConversation(name?: string): Promise { + const conversationName = name || `Chat ${new Date().toLocaleString()}`; + // Working directory and reasoning effort picked on the new-chat screen + // get threaded into the new conversation here, then cleared so they + // don't bleed onto subsequent new chats. + const conversation = await DatabaseService.createConversation(conversationName, { + cwd: this.preferences.pendingCwd ?? undefined, + reasoningEffort: this.preferences.pendingReasoningEffort }); + + this.preferences.pendingCwd = null; + + this.conversations = [conversation, ...this.conversations]; + this.activeConversation = conversation; + this.activeMessages = []; + + await goto(RouterService.chat(conversation.id)); + + return conversation.id; } /** - * Checks if an MCP server is enabled for the active conversation. - * @param serverId - The server ID to check - * @returns True if server is enabled for this conversation + * Deletes all conversations and their messages */ - isMcpServerEnabledForChat(serverId: string): boolean { - const override = this.getMcpServerOverride(serverId); + async deleteAll(): Promise { + try { + const allConversations = await DatabaseService.getAllConversations(); + const allIds = allConversations.map((c) => c.id); - return override?.enabled ?? false; - } + await DatabaseService.bulkDeleteConversations(allIds); - /** - * Sets or removes MCP server override for the active conversation. - * If no conversation exists, persists `enabled` onto `mcpServers[i].enabled` - * (the single source of truth for new-chat defaults). - * @param serverId - The server ID to override - * @param enabled - The enabled state, or undefined to remove per-conversation override - */ - async setMcpServerOverride(serverId: string, enabled: boolean | undefined): Promise { - if (!this.activeConversation) { - if (enabled !== undefined) { - mcpStore.updateServer(serverId, { enabled }); - } + this.clearActiveConversation(); + this.conversations = []; + this.notifyConversationsDeleted(allIds); - return; + toast.success('All conversations deleted'); + + await goto(ROUTES.NEW_CHAT); + } catch (error) { + console.error('Failed to delete all conversations:', error); + toast.error('Failed to delete conversations'); } + } - // Clone to plain objects to avoid Proxy serialization issues with IndexedDB - const currentOverrides = (this.activeConversation.mcpServerOverrides || []).map( - (o: McpServerOverride) => ({ - enabled: o.enabled, - serverId: o.serverId - }) - ); + /** + * Deletes a conversation and all its messages + * @param convId - The conversation ID to delete + */ + async deleteConversation(convId: string, options?: { deleteWithForks?: boolean }): Promise { + try { + await DatabaseService.deleteConversation(convId, options); - let newOverrides: McpServerOverride[]; + if (options?.deleteWithForks) { + // Collect all descendants recursively + const idsToRemove = new SvelteSet([convId]); + const queue = [convId]; - if (enabled === undefined) { - newOverrides = currentOverrides.filter((o: McpServerOverride) => o.serverId !== serverId); - } else { - const existingIndex = currentOverrides.findIndex( - (o: McpServerOverride) => o.serverId === serverId - ); + while (queue.length > 0) { + const parentId = queue.pop()!; - if (existingIndex >= 0) { - newOverrides = [...currentOverrides]; - newOverrides[existingIndex] = { enabled, serverId }; + for (const c of this.conversations) { + if (c.forkedFromConversationId === parentId && !idsToRemove.has(c.id)) { + idsToRemove.add(c.id); + queue.push(c.id); + } + } + } + this.conversations = this.conversations.filter((c) => !idsToRemove.has(c.id)); + + if (this.activeConversation && idsToRemove.has(this.activeConversation.id)) { + this.clearActiveConversation(); + await goto(ROUTES.NEW_CHAT); + } + + this.notifyConversationsDeleted([...idsToRemove]); } else { - newOverrides = [...currentOverrides, { enabled, serverId }]; + // Reparent direct children to deleted conv's parent (or promote to top-level) + const deletedConv = this.conversations.find((c) => c.id === convId); + const newParent = deletedConv?.forkedFromConversationId; + + this.conversations = this.conversations + .filter((c) => c.id !== convId) + .map((c) => + c.forkedFromConversationId === convId + ? { ...c, forkedFromConversationId: newParent } + : c + ); + + if (this.activeConversation?.id === convId) { + this.clearActiveConversation(); + await goto(ROUTES.NEW_CHAT); + } + + this.notifyConversationsDeleted([convId]); } - } - - await DatabaseService.updateConversation(this.activeConversation.id, { - mcpServerOverrides: newOverrides.length > 0 ? newOverrides : undefined - }); - - this.activeConversation = { - ...this.activeConversation, - mcpServerOverrides: newOverrides.length > 0 ? newOverrides : undefined - }; - - const convIndex = this.conversations.findIndex((c) => c.id === this.activeConversation!.id); - - if (convIndex !== -1) { - this.conversations[convIndex].mcpServerOverrides = - newOverrides.length > 0 ? newOverrides : undefined; + } catch (error) { + console.error('Failed to delete conversation:', error); } } /** - * Toggles MCP server enabled state for the active conversation. - * @param serverId - The server ID to toggle + * Downloads a single conversation as a JSONL file, serializing the full message tree. + * @param convId - The conversation ID to download */ - async toggleMcpServerForChat(serverId: string): Promise { - const currentEnabled = this.isMcpServerEnabledForChat(serverId); + async downloadConversation(convId: string): Promise { + const conversation = + this.activeConversation?.id === convId + ? this.activeConversation + : await DatabaseService.getConversation(convId); - await this.setMcpServerOverride(serverId, !currentEnabled); + if (!conversation) return; + + const messages = await DatabaseService.getConversationMessages(convId); + + ConversationTransferService.downloadConversationFile({ conv: conversation, messages }); } /** - * Removes MCP server override for the active conversation. - * @param serverId - The server ID to remove override for - */ - async removeMcpServerOverride(serverId: string): Promise { - await this.setMcpServerOverride(serverId, undefined); - } - - /** - * Gets the effective reasoning effort for the active conversation. - * Returns the conversation override if set, otherwise the global default. - * DEFAULT means no override is sent and the server decides. - */ - getReasoningEffort(): ReasoningEffort { - if (this.activeConversation) { - if (this.activeConversation.reasoningEffort !== undefined) { - return this.activeConversation.reasoningEffort; - } - - // conversations created before the tri-state store an explicit - // opt-out only as thinkingEnabled = false - if (this.activeConversation.thinkingEnabled === false) { - return ReasoningEffort.OFF; - } - } - - return this.pendingReasoningEffort; - } - - /** - * Sets the reasoning effort for the active conversation. - * If no conversation exists, stores the global default. - * @param effort - The effort level ('default' | 'off' | 'low' | 'medium' | 'high' | 'max') - */ - async setReasoningEffort(effort: ReasoningEffort): Promise { - if (!this.activeConversation) { - this.pendingReasoningEffort = effort; - this.saveReasoningEffortDefaults(); - - return; - } - - this.activeConversation = { - ...this.activeConversation, - reasoningEffort: effort - }; - - await DatabaseService.updateConversation(this.activeConversation.id, { - reasoningEffort: effort - }); - - const convIndex = this.conversations.findIndex((c) => c.id === this.activeConversation!.id); - - if (convIndex !== -1) { - this.conversations[convIndex].reasoningEffort = effort; - } - } - - /** - * Sets the working directory for the active conversation. Pass `null` or - * an empty string to clear it, which restores the picker's empty state. + * Finds the index of a message in active messages. * - * On the empty new-chat screen (no active conversation yet), the value - * is buffered into `pendingCwd` so the user can pick before - * sending the first message; `createConversation()` consumes it. - * - * @param value - Absolute server-side path to the working directory, or null to clear + * The last lookup is memoized and reused when it still validates against + * the current array (same id at the same position), which covers the + * streaming hot path where the same message is looked up on every chunk + * while the array itself only mutates by field. Any structural change + * (splice, reassignment, reordering) fails validation and falls back to a + * full scan. */ - async setCwd(value: string | null): Promise { - const trimmed = value?.trim() || undefined; + findMessageIndex(messageId: string): number { + const last = this.lastMessageIndex; + const messages = this.activeMessages; - // No chat yet - buffer for the first chat the user creates. - if (!this.activeConversation) { - this.pendingCwd = trimmed ?? null; - - return; + if ( + last && + last.id === messageId && + last.index >= 0 && + last.index < messages.length && + messages[last.index]?.id === messageId + ) { + return last.index; } - this.activeConversation = { - ...this.activeConversation, - cwd: trimmed - }; + const index = messages.findIndex((m) => m.id === messageId); - await DatabaseService.updateConversation(this.activeConversation.id, { - cwd: trimmed - }); + this.lastMessageIndex = { id: messageId, index }; - const convIndex = this.conversations.findIndex((c) => c.id === this.activeConversation!.id); - - if (convIndex !== -1) { - this.conversations[convIndex].cwd = trimmed; - this.conversations = [...this.conversations]; - } - - this.pendingCwd = null; + return index; } /** @@ -931,28 +428,12 @@ class ConversationsStore { } /** - * - * - * Import & Export - * - * + * Gets all messages for a specific conversation + * @param convId - The conversation ID + * @returns Array of messages */ - - /** - * Downloads a single conversation as a JSONL file, serializing the full message tree. - * @param convId - The conversation ID to download - */ - async downloadConversation(convId: string): Promise { - const conversation = - this.activeConversation?.id === convId - ? this.activeConversation - : await DatabaseService.getConversation(convId); - - if (!conversation) return; - - const messages = await DatabaseService.getConversationMessages(convId); - - ConversationTransferService.downloadConversationFile({ conv: conversation, messages }); + async getConversationMessages(convId: string): Promise { + return await DatabaseService.getConversationMessages(convId); } /** @@ -969,6 +450,280 @@ class ConversationsStore { return result; } + + /** + * Initialize the store by loading conversations from database. + * Safe to call multiple times: concurrent callers share a single run, + * and a failed run can be retried by calling again. + */ + initialize(): Promise { + if (!browser) return Promise.resolve(); + + if (this.initPromise) return this.initPromise; + + this.initPromise = (async () => { + try { + await MigrationService.runAllMigrations(); + await this.loadConversations(); + this.isInitialized = true; + } catch (error) { + console.error('Failed to initialize conversations:', error); + this.initPromise = null; + } + })(); + + return this.initPromise; + } + + /** + * Loads a specific conversation and its messages + * @param convId - The conversation ID to load + * @returns True if conversation was loaded successfully + */ + async loadConversation(convId: string): Promise { + try { + const conversation = await DatabaseService.getConversation(convId); + + if (!conversation) { + return false; + } + + // Drop any cwd the user drafted on the empty new-chat screen - + // it doesn't belong to this conversation. + this.preferences.pendingCwd = null; + + this.activeConversation = conversation; + + if (conversation.currNode) { + const allMessages = await DatabaseService.getConversationMessages(convId); + const filteredMessages = filterByLeafNodeId( + allMessages, + conversation.currNode, + false + ) as DatabaseMessage[]; + + this.activeMessages = filteredMessages; + } else { + const messages = await DatabaseService.getConversationMessages(convId); + + this.activeMessages = messages; + } + + return true; + } catch (error) { + console.error('Failed to load conversation:', error); + + return false; + } + } + + /** + * Loads all conversations from the database + */ + async loadConversations(): Promise { + const conversations = await DatabaseService.getAllConversations(); + + this.conversations = conversations; + } + + /** + * Navigates to a specific sibling branch by updating currNode and refreshing messages. + * @param siblingId - The sibling message ID to navigate to + */ + async navigateToSibling(siblingId: string): Promise { + if (!this.activeConversation) return; + + const allMessages = await DatabaseService.getConversationMessages(this.activeConversation.id); + const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null); + const currentFirstUserMessage = this.activeMessages.find( + (m) => m.role === MessageRole.USER && m.parent === rootMessage?.id + ); + const currentLeafNodeId = findLeafNode(allMessages, siblingId); + + await DatabaseService.updateCurrentNode(this.activeConversation.id, currentLeafNodeId); + this.activeConversation = { ...this.activeConversation, currNode: currentLeafNodeId }; + await this.refreshActiveMessages(); + + if (rootMessage && this.activeMessages.length > 0) { + const newFirstUserMessage = this.activeMessages.find( + (m) => m.role === MessageRole.USER && m.parent === rootMessage.id + ); + + if ( + newFirstUserMessage && + newFirstUserMessage.content.trim() && + (!currentFirstUserMessage || + newFirstUserMessage.id !== currentFirstUserMessage.id || + newFirstUserMessage.content.trim() !== currentFirstUserMessage.content.trim()) + ) { + await this.applyTitleFromContent(this.activeConversation.id, newFirstUserMessage.content); + } + } + } + + /** + * Registers a listener invoked with the ids of deleted conversations. + * Returns an unsubscribe function. + */ + onConversationsDeleted(listener: (convIds: string[]) => void): () => void { + this.conversationDeletionListeners.add(listener); + + return () => this.conversationDeletionListeners.delete(listener); + } + + /** + * Refreshes active messages based on currNode after branch navigation. + */ + async refreshActiveMessages(): Promise { + if (!this.activeConversation) return; + + const allMessages = await DatabaseService.getConversationMessages(this.activeConversation.id); + + if (allMessages.length === 0) { + this.activeMessages = []; + + return; + } + + const leafNodeId = + this.activeConversation.currNode || + allMessages.reduce((latest, msg) => (msg.timestamp > latest.timestamp ? msg : latest)).id; + const currentPath = filterByLeafNodeId(allMessages, leafNodeId, false) as DatabaseMessage[]; + + this.activeMessages = currentPath; + } + + /** + * Removes a message from active messages by index + */ + removeMessageAtIndex(index: number): DatabaseMessage | undefined { + if (index !== -1) { + return this.activeMessages.splice(index, 1)[0]; + } + + return undefined; + } + + /** + * Removes messages from active messages starting at an index + */ + sliceActiveMessages(startIndex: number): void { + this.activeMessages = this.activeMessages.slice(0, startIndex); + } + + /** + * Toggles the pinned status of a conversation. + * @param convId - The conversation ID to toggle + * @returns The new pinned status + */ + async toggleConversationPin(convId: string): Promise { + try { + const newPinnedState = await DatabaseService.toggleConversationPin(convId); + + this.applyConversationUpdate(convId, { pinned: newPinnedState }); + + return newPinnedState; + } catch (error) { + console.error('Failed to toggle conversation pin:', error); + + return false; + } + } + + /** + * Updates the name of a conversation. + * @param convId - The conversation ID to update + * @param name - The new name for the conversation + */ + async updateConversationName(convId: string, name: string): Promise { + try { + await DatabaseService.updateConversation(convId, { name }); + + this.applyConversationUpdate(convId, { name }); + } catch (error) { + console.error('Failed to update conversation name:', error); + } + } + + /** + * Marks a conversation as recently active: stamps lastModified (persisted) + * and moves it to the top of the list. Only message-activity flows call + * this; metadata updates (rename, pin, settings) do not. + * + * @param convId - Conversation that produced the activity, defaults to the active one + */ + updateConversationTimestamp(convId?: string): void { + const targetId = convId ?? this.activeConversation?.id; + + if (!targetId) return; + + const now = Date.now(); + const chatIndex = this.conversations.findIndex((c) => c.id === targetId); + + if (chatIndex !== -1) { + this.conversations[chatIndex].lastModified = now; + const updatedConv = this.conversations.splice(chatIndex, 1)[0]; + + this.conversations = [updatedConv, ...this.conversations]; + } + + if (this.activeConversation?.id === targetId) { + this.activeConversation = { ...this.activeConversation, lastModified: now }; + } + + DatabaseService.updateConversation(targetId, { lastModified: now }).catch((error) => + console.error('Failed to update conversation timestamp:', error) + ); + } + + /** + * + * + * Import & Export + * + * + */ + + /** + * Updates the current node of the active conversation + * @param nodeId - The new current node ID + */ + async updateCurrentNode(nodeId: string): Promise { + if (!this.activeConversation) return; + + await DatabaseService.updateCurrentNode(this.activeConversation.id, nodeId); + this.activeConversation = { ...this.activeConversation, currNode: nodeId }; + } + + /** + * Updates a message at a specific index in active messages + */ + updateMessageAtIndex(index: number, updates: Partial): void { + const message = index === -1 ? undefined : this.activeMessages[index]; + + if (!message) return; + + // Assign field by field rather than replacing the object. Replacing it + // changes the array slot, which invalidates every consumer that merely + // walks the list - notably ChatMessages.displayMessages, which rebuilds + // entries for every message in the conversation. Deep $state proxies make + // per-field writes fine-grained, so only readers of the changed field wake. + const target = message as unknown as Record; + + for (const [key, value] of Object.entries(updates)) { + if (target[key] !== value) { + target[key] = value; + } + } + } + + private notifyConversationsDeleted(convIds: string[]): void { + if (convIds.length === 0) return; + + for (const listener of this.conversationDeletionListeners) { + listener(convIds); + } + } } export const conversationsStore = new ConversationsStore(); diff --git a/tools/ui/src/lib/stores/conversations/preferences.svelte.ts b/tools/ui/src/lib/stores/conversations/preferences.svelte.ts new file mode 100644 index 0000000000..65a02344b6 --- /dev/null +++ b/tools/ui/src/lib/stores/conversations/preferences.svelte.ts @@ -0,0 +1,254 @@ +/** + * ConversationPreferences - Per-chat options with global fallback + * + * Owns the options that resolve per conversation: MCP server overrides, + * reasoning effort, and the working directory. Cwd and reasoning effort are + * buffered as pending state and threaded into the next created conversation + * by the host; MCP server overrides edit the sparse `mcpServerOverrides` + * list on the active row (new-chat toggles edit the server's global flag). + * Created and owned by conversationsStore; the host owns the conversation + * rows these options persist onto. + */ + +import { REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY } from '$lib/constants'; +import { ReasoningEffort } from '$lib/enums'; +import { DatabaseService } from '$lib/services/database.service'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { mcpStore } from '$lib/stores/mcp/index.svelte'; +import type { McpServerOverride } from '$lib/types/database'; + +/** Load reasoning effort default from localStorage, DEFAULT defers to the server */ +function loadReasoningEffortDefault(): ReasoningEffort { + if (typeof globalThis.localStorage === 'undefined') return ReasoningEffort.DEFAULT; + + try { + const raw = localStorage.getItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY); + + return (raw as ReasoningEffort) || ReasoningEffort.DEFAULT; + } catch { + return ReasoningEffort.DEFAULT; + } +} + +/** Persist reasoning effort default to localStorage */ +function saveReasoningEffortDefault(effort: ReasoningEffort): void { + if (typeof globalThis.localStorage === 'undefined') return; + + localStorage.setItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY, effort); +} + +/** + * The slice of conversationsStore the preferences read and write. Kept narrow + * on purpose so they cannot reach around the host's full surface; + * conversationsStore implements this structurally. + */ +export interface ConversationsPreferencesHost { + activeConversation: DatabaseConversation | null; + conversations: DatabaseConversation[]; + applyConversationUpdate(id: string, updates: Partial): void; +} + +export class ConversationPreferences { + /** + * Working directory picked on the empty new-chat screen, before any + * conversation exists. Consumed by `chatStore.sendMessage()`, which + * records it into chat history as a synthetic message on first send. + * Cleared by `loadConversation` and `clearActiveConversation` so a + * stale pick can't bleed onto an unrelated chat. + */ + pendingCwd = $state(null); + + /** Global (non-conversation-specific) reasoning effort default */ + pendingReasoningEffort = $state(loadReasoningEffortDefault()); + + constructor(private host: ConversationsPreferencesHost) {} + + /** + * Gets the effective override list for the current conversation: + * one entry per configured server, resolved per server. The stored + * per-conversation list is sparse and only holds explicit toggles. + */ + getAllMcpServerOverrides(): McpServerOverride[] { + const overrides = this.host.activeConversation?.mcpServerOverrides; + + return mcpStore.getServers().map((s) => { + const override = overrides?.find((o: McpServerOverride) => o.serverId === s.id); + + return { enabled: override?.enabled ?? s.enabled, serverId: s.id }; + }); + } + + /** + * Gets the effective MCP server override for a specific server. + * A per-conversation override wins when present; a server without one + * resolves to its `mcpServers[i].enabled` default. + */ + getMcpServerOverride(serverId: string): McpServerOverride | undefined { + const override = this.host.activeConversation?.mcpServerOverrides?.find( + (o: McpServerOverride) => o.serverId === serverId + ); + + if (override) return override; + + return this.getDefaultOverride(serverId); + } + + /** + * Gets the effective reasoning effort for the active conversation. + * Returns the conversation override if set, otherwise the global default. + * DEFAULT means no override is sent and the server decides. + */ + getReasoningEffort(): ReasoningEffort { + if (this.host.activeConversation) { + if (this.host.activeConversation.reasoningEffort !== undefined) { + return this.host.activeConversation.reasoningEffort; + } + + // conversations created before the tri-state store an explicit + // opt-out only as thinkingEnabled = false + if (this.host.activeConversation.thinkingEnabled === false) { + return ReasoningEffort.OFF; + } + } + + return this.pendingReasoningEffort; + } + + /** Checks if an MCP server is enabled for the active conversation. */ + isMcpServerEnabledForChat(serverId: string): boolean { + const override = this.getMcpServerOverride(serverId); + + return override?.enabled ?? false; + } + + /** Removes MCP server override for the active conversation. */ + async removeMcpServerOverride(serverId: string): Promise { + await this.setMcpServerOverride(serverId, undefined); + } + + /** Reload persisted defaults, e.g. when the active conversation is cleared. */ + resetPending(): void { + this.pendingReasoningEffort = loadReasoningEffortDefault(); + this.pendingCwd = null; + } + + /** + * Sets the working directory for the active conversation. Pass `null` or + * an empty string to clear it, which restores the picker's empty state. + * + * On the empty new-chat screen (no active conversation yet), the value + * is buffered into `pendingCwd` so the user can pick before + * sending the first message; `createConversation()` consumes it. + * + * @param value - Absolute server-side path to the working directory, or null to clear + */ + async setCwd(value: string | null): Promise { + const trimmed = value?.trim() || undefined; + + // No chat yet - buffer for the first chat the user creates. + if (!this.host.activeConversation) { + this.pendingCwd = trimmed ?? null; + + return; + } + + this.host.applyConversationUpdate(this.host.activeConversation.id, { + cwd: trimmed + }); + + await DatabaseService.updateConversation(this.host.activeConversation.id, { + cwd: trimmed + }); + + this.pendingCwd = null; + } + + /** + * Sets or removes MCP server override for the active conversation. + * If no conversation exists, persists `enabled` onto `mcpServers[i].enabled` + * (the single source of truth for new-chat defaults). + */ + async setMcpServerOverride(serverId: string, enabled: boolean | undefined): Promise { + if (!this.host.activeConversation) { + if (enabled !== undefined) { + mcpStore.updateServer(serverId, { enabled }); + } + + return; + } + + // Clone to plain objects to avoid Proxy serialization issues with IndexedDB + const currentOverrides = (this.host.activeConversation.mcpServerOverrides || []).map( + (o: McpServerOverride) => ({ + enabled: o.enabled, + serverId: o.serverId + }) + ); + + let newOverrides: McpServerOverride[]; + + if (enabled === undefined) { + newOverrides = currentOverrides.filter((o: McpServerOverride) => o.serverId !== serverId); + } else { + const existingIndex = currentOverrides.findIndex( + (o: McpServerOverride) => o.serverId === serverId + ); + + if (existingIndex >= 0) { + newOverrides = [...currentOverrides]; + newOverrides[existingIndex] = { enabled, serverId }; + } else { + newOverrides = [...currentOverrides, { enabled, serverId }]; + } + } + + await DatabaseService.updateConversation(this.host.activeConversation.id, { + mcpServerOverrides: newOverrides.length > 0 ? newOverrides : undefined + }); + + this.host.applyConversationUpdate(this.host.activeConversation.id, { + mcpServerOverrides: newOverrides.length > 0 ? newOverrides : undefined + }); + } + + /** + * Sets the reasoning effort for the active conversation. + * If no conversation exists, stores the global default. + * @param effort - The effort level ('default' | 'off' | 'low' | 'medium' | 'high' | 'max') + */ + async setReasoningEffort(effort: ReasoningEffort): Promise { + if (!this.host.activeConversation) { + this.pendingReasoningEffort = effort; + saveReasoningEffortDefault(effort); + + return; + } + + this.host.applyConversationUpdate(this.host.activeConversation.id, { + reasoningEffort: effort + }); + + await DatabaseService.updateConversation(this.host.activeConversation.id, { + reasoningEffort: effort + }); + } + + /** Toggles MCP server enabled state for the active conversation. */ + async toggleMcpServerForChat(serverId: string): Promise { + const currentEnabled = this.isMcpServerEnabledForChat(serverId); + + await this.setMcpServerOverride(serverId, !currentEnabled); + } + + /** + * Resolve the default enabled value for a server: its own `enabled` + * flag in `mcpServers`, so the global on/off state lives in one place. + */ + private getDefaultOverride(serverId: string): McpServerOverride | undefined { + const server = mcpStore.getServers().find((s) => s.id === serverId); + + if (!server) return undefined; + + return { enabled: server.enabled, serverId }; + } +} diff --git a/tools/ui/src/lib/stores/device.svelte.ts b/tools/ui/src/lib/stores/device.svelte.ts index 08ce2f2054..42aaf45891 100644 --- a/tools/ui/src/lib/stores/device.svelte.ts +++ b/tools/ui/src/lib/stores/device.svelte.ts @@ -34,11 +34,11 @@ class DeviceStore { readonly isIOSDevice: boolean = false; /** The Safari browser app on iOS, excluding other iOS browsers and WKWebViews. */ readonly isIOSSafari: boolean = false; + /** PWA standalone mode: the page was launched from the home screen icon. */ + isStandalone = $state(false); /** Any WKWebView context on iOS: in-app browsers, embedded web views, and the * third-party iOS browsers (all of which share the WKWebView engine). */ readonly isWKWebView: boolean = false; - /** PWA standalone mode: the page was launched from the home screen icon. */ - isStandalone = $state(false); /** OS color scheme preference; the user override lives in settingsStore. */ readonly systemTheme = $state({ isDark: false }); diff --git a/tools/ui/src/lib/stores/index.ts b/tools/ui/src/lib/stores/index.ts index 1aea8a6dab..db227158f4 100644 --- a/tools/ui/src/lib/stores/index.ts +++ b/tools/ui/src/lib/stores/index.ts @@ -18,34 +18,32 @@ */ // CHAT / MESSAGING -export { chatStore } from './chat.svelte'; +export { chatStore } from './chat/index.svelte'; -export { draftMessagesStore } from './draft-messages.svelte'; - -// AGENTIC (multi-turn tool orchestration) -export { agenticStore } from './agentic.svelte'; - -// CONVERSATIONS -export { conversationsStore } from './conversations.svelte'; +export { draftMessagesStore } from './chat/drafts.svelte'; // CONTEXT STATS (active conversation context window usage) -export { contextStatsStore } from './context-stats.svelte'; +export { contextStatsStore } from './chat/context-stats.svelte'; + +// AGENTIC (multi-turn tool orchestration) +export { agenticStore } from './agentic/index.svelte'; + +// CONVERSATIONS +export { conversationsStore } from './conversations/index.svelte'; // MCP -export { mcpStore } from './mcp.svelte'; - -export { mcpResourceStore } from './mcp-resources.svelte'; +export { mcpStore } from './mcp/index.svelte'; // MODELS -export { modelsStore } from './models.svelte'; +export { modelsStore } from './models/index.svelte'; // SERVER export { serverStore } from './server.svelte'; // SETTINGS / UI PREFERENCES -export { settingsStore } from './settings.svelte'; +export { settingsStore } from './settings/index.svelte'; -export { settingsReferrer } from './settings-referrer.svelte'; +export { settingsReferrer } from './settings/referrer.svelte'; export { permissionsStore } from './permissions.svelte'; diff --git a/tools/ui/src/lib/stores/init.ts b/tools/ui/src/lib/stores/init.ts index 1faa803033..d52c34d0fe 100644 --- a/tools/ui/src/lib/stores/init.ts +++ b/tools/ui/src/lib/stores/init.ts @@ -13,9 +13,9 @@ */ // direct imports, not via the barrel, to avoid circular deps -import { conversationsStore } from './conversations.svelte'; +import { conversationsStore } from './conversations/index.svelte'; import { permissionsStore } from './permissions.svelte'; -import { settingsStore } from './settings.svelte'; +import { settingsStore } from './settings/index.svelte'; import { toolsStore } from './tools.svelte'; import { versionStore } from './version.svelte'; import { browser } from '$app/environment'; @@ -33,7 +33,7 @@ export function initStores(): Promise { permissionsStore.initialize(); toolsStore.initialize(); void versionStore.initialize(); - void conversationsStore.init(); + void conversationsStore.initialize(); })(); return startup; diff --git a/tools/ui/src/lib/stores/mcp/health.svelte.ts b/tools/ui/src/lib/stores/mcp/health.svelte.ts new file mode 100644 index 0000000000..fffa6ea92b --- /dev/null +++ b/tools/ui/src/lib/stores/mcp/health.svelte.ts @@ -0,0 +1,298 @@ +/** + * MCPHealthCheckManager - Health checks for MCP servers + * + * Owns per-server connectivity probes: connection reuse, capability + * snapshots, and promotion of a successful check to an active connection. + * Created and owned by mcpStore; the host owns the connection registry the + * probes draw from and promote into. + */ + +import { DEFAULT_MCP_CONFIG } from '$lib/constants'; +import { HealthCheckStatus, MCPConnectionPhase, MCPLogLevel } from '$lib/enums'; +import { MCPService } from '$lib/services/mcp.service'; +import type { + ClientCapabilities, + HealthCheckParams, + HealthCheckState, + MCPCapabilitiesInfo, + MCPConnection, + MCPConnectionLog, + MCPServerConfig, + ServerCapabilities +} from '$lib/types'; +import { detectMcpTransportFromUrl } from '$lib/utils'; + +// module-level so the timestamp is not flagged as reactive state by prefer-svelte-reactivity +function createConnectionErrorLog(message: string): MCPConnectionLog { + return { + level: MCPLogLevel.ERROR, + message: `Connection failed: ${message}`, + phase: MCPConnectionPhase.ERROR, + timestamp: new Date() + }; +} + +/** + * The slice of mcpStore the probes drive. Kept narrow on purpose so the + * probes cannot reach around the host's full surface; mcpStore implements + * this structurally. + */ +export interface McpHealthHost { + autoReconnect(serverName: string): Promise; + getExistingConnection(serverId: string): MCPConnection | undefined; + getRequestTimeoutMs(): number; + promoteHealthCheckToConnection(serverId: string, connection: MCPConnection): void; + registerServerConfig(name: string, config: MCPServerConfig): void; + removeConnection(serverId: string): void; +} + +export class MCPHealthCheckManager { + private _checks = $state>({}); + + /** Raw per-server check states, for host-side capability scans. */ + get checks(): Record { + return this._checks; + } + + clear(serverId: string): void { + const { [serverId]: _removed, ...rest } = this._checks; + + this._checks = rest; + } + + constructor(private host: McpHealthHost) {} + + getState(serverId: string): HealthCheckState { + return this._checks[serverId] ?? { status: HealthCheckStatus.IDLE }; + } + + hasState(serverId: string): boolean { + return serverId in this._checks && this._checks[serverId].status !== HealthCheckStatus.IDLE; + } + + /** + * Run a health check for a server. + * If the server already has an active connection, reuses it instead of creating a new one. + * If promoteToActive is true and server is enabled, the connection will be kept + * and promoted to an active connection instead of being disconnected. + */ + async run(server: HealthCheckParams, promoteToActive = false): Promise { + const existingConnection = this.host.getExistingConnection(server.id); + + if (existingConnection) { + // Reuse existing connection - just refresh tools list + try { + const tools = await MCPService.listTools(existingConnection); + const capabilities = this.buildCapabilitiesInfo( + existingConnection.serverCapabilities, + existingConnection.clientCapabilities + ); + + this.setState(server.id, { + capabilities, + connectionTimeMs: existingConnection.connectionTimeMs, + instructions: existingConnection.instructions, + logs: [], + protocolVersion: existingConnection.protocolVersion, + serverInfo: existingConnection.serverInfo, + status: HealthCheckStatus.SUCCESS, + tools: tools.map((tool) => ({ + description: tool.description, + name: tool.name, + title: tool.title + })), + transportType: existingConnection.transportType + }); + + return; + } catch (error) { + console.warn( + `[MCPStore] Failed to reuse connection for ${server.id}, creating new one:`, + error + ); + // Connection may be stale, remove it and create new one + this.host.removeConnection(server.id); + } + } + + const trimmedUrl = server.url.trim(); + const logs: MCPConnectionLog[] = []; + + let currentPhase: MCPConnectionPhase = MCPConnectionPhase.IDLE; + + if (!trimmedUrl) { + this.setState(server.id, { + logs: [], + message: 'Please enter a server URL first.', + status: HealthCheckStatus.ERROR + }); + + return; + } + + this.setState(server.id, { + logs: [], + phase: MCPConnectionPhase.TRANSPORT_CREATING, + status: HealthCheckStatus.CONNECTING + }); + + const timeoutMs = this.host.getRequestTimeoutMs(); + const headers = this.parseHeaders(server.headers); + + try { + const serverConfig: MCPServerConfig = { + handshakeTimeoutMs: DEFAULT_MCP_CONFIG.connectionTimeoutMs, + headers, + requestTimeoutMs: timeoutMs, + transport: detectMcpTransportFromUrl(trimmedUrl), + url: trimmedUrl, + useProxy: server.useProxy + }; + + this.host.registerServerConfig(server.id, serverConfig); + + const connection = await MCPService.connect( + server.id, + serverConfig, + DEFAULT_MCP_CONFIG.clientInfo, + DEFAULT_MCP_CONFIG.capabilities, + (phase, log) => { + currentPhase = phase; + logs.push(log); + this.setState(server.id, { + logs: [...logs], + phase, + status: HealthCheckStatus.CONNECTING + }); + + if (phase === MCPConnectionPhase.DISCONNECTED && promoteToActive) { + console.log( + `[MCPStore][${server.id}] Connection lost during health check, starting auto-reconnect` + ); + this.host.autoReconnect(server.id); + } + } + ); + const tools = connection.tools.map((tool) => ({ + description: tool.description, + name: tool.name, + title: tool.title + })); + const capabilities = this.buildCapabilitiesInfo( + connection.serverCapabilities, + connection.clientCapabilities + ); + + this.setState(server.id, { + capabilities, + connectionTimeMs: connection.connectionTimeMs, + instructions: connection.instructions, + logs, + protocolVersion: connection.protocolVersion, + serverInfo: connection.serverInfo, + status: HealthCheckStatus.SUCCESS, + tools, + transportType: connection.transportType + }); + + if (promoteToActive && server.enabled) { + this.host.promoteHealthCheckToConnection(server.id, connection); + } else { + await MCPService.disconnect(connection); + } + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error occurred'; + + if (logs.at(-1)?.phase !== MCPConnectionPhase.ERROR) { + logs.push(createConnectionErrorLog(message)); + } + + this.setState(server.id, { + logs, + message, + phase: currentPhase, + status: HealthCheckStatus.ERROR + }); + } + } + + async runForServers( + servers: { + id: string; + enabled: boolean; + url: string; + headers?: string; + }[], + skipIfChecked = true, + promoteToActive = false + ): Promise { + const serversToCheck = skipIfChecked + ? servers.filter((s) => !this.hasState(s.id) && s.url.trim()) + : servers.filter((s) => s.url.trim()); + + if (serversToCheck.length === 0) { + return; + } + + const BATCH_SIZE = 5; + + for (let i = 0; i < serversToCheck.length; i += BATCH_SIZE) { + const batch = serversToCheck.slice(i, i + BATCH_SIZE); + + await Promise.allSettled(batch.map((server) => this.run(server, promoteToActive))); + } + } + + /** + * Builds capabilities info from server and client capabilities. + */ + private buildCapabilitiesInfo( + serverCaps?: ServerCapabilities, + clientCaps?: ClientCapabilities + ): MCPCapabilitiesInfo { + return { + client: { + elicitation: clientCaps?.elicitation + ? { form: !!clientCaps.elicitation.form, url: !!clientCaps.elicitation.url } + : undefined, + roots: clientCaps?.roots ? { listChanged: clientCaps.roots.listChanged } : undefined, + sampling: !!clientCaps?.sampling, + tasks: !!clientCaps?.tasks + }, + server: { + completions: !!serverCaps?.completions, + logging: !!serverCaps?.logging, + prompts: serverCaps?.prompts ? { listChanged: serverCaps.prompts.listChanged } : undefined, + resources: serverCaps?.resources + ? { + listChanged: serverCaps.resources.listChanged, + subscribe: serverCaps.resources.subscribe + } + : undefined, + tasks: !!serverCaps?.tasks, + tools: serverCaps?.tools ? { listChanged: serverCaps.tools.listChanged } : undefined + } + }; + } + + private parseHeaders(headersJson?: string): Record | undefined { + if (!headersJson?.trim()) { + return undefined; + } + + try { + const parsed = JSON.parse(headersJson); + + if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) + return parsed as Record; + } catch { + console.warn('[MCPStore] Failed to parse custom headers JSON:', headersJson); + } + + return undefined; + } + + private setState(serverId: string, state: HealthCheckState): void { + this._checks = { ...this._checks, [serverId]: state }; + } +} diff --git a/tools/ui/src/lib/stores/mcp.svelte.ts b/tools/ui/src/lib/stores/mcp/index.svelte.ts similarity index 67% rename from tools/ui/src/lib/stores/mcp.svelte.ts rename to tools/ui/src/lib/stores/mcp/index.svelte.ts index 3e0cb8e1e3..ccd53bc9d2 100644 --- a/tools/ui/src/lib/stores/mcp.svelte.ts +++ b/tools/ui/src/lib/stores/mcp/index.svelte.ts @@ -1,69 +1,38 @@ /** - * mcpStore - Reactive State Store for MCP Operations + * mcpStore - MCP host: server connections and tool operations * - * Implements the "Host" role in MCP architecture, coordinating multiple server - * connections and providing a unified interface for tool operations. - * - * **Architecture & Relationships:** - * - **MCPService**: Stateless protocol layer (transport, connect, callTool) - * - **mcpStore** (this): Reactive state + business logic - * - * **Key Responsibilities:** - * - Lifecycle management (initialize, shutdown) - * - Multi-server coordination - * - Tool name conflict detection and resolution - * - Automatic tool-to-server routing - * - Health checks - * - * MCP connection state and raw `Tool[]` per server are owned here; the - * OpenAI-compatible wire format for those tools is built in `toolsStore` - * (see {@link toolsStore.mcpEntries} / {@link toolsStore.getEnabledToolsForLLM}). - * - * @see MCPService in services/mcp.service.ts for protocol operations + * Implements the MCP "Host" role, coordinating multiple server connections + * and exposing a unified tool interface: lifecycle, name-conflict detection + * and automatic tool-to-server routing. Owns connection state and raw + * `Tool[]` per server; the OpenAI-compatible wire format is built in + * toolsStore. Composes the health-check manager; uses MCPService for the + * protocol layer. */ import type { ListChangedHandlers } from '@modelcontextprotocol/sdk/types.js'; import { browser } from '$app/environment'; import { SETTINGS_KEYS } from '$lib/constants'; -import { - CACHE, - DEFAULT_MCP_CONFIG, - EXPECTED_THEMED_ICON_PAIR_COUNT, - MCP_ALLOWED_ICON_MIME_TYPES, - MCP_RECONNECT, - MCP_SERVER_ID_PREFIX -} from '$lib/constants'; -import { - ColorMode, - HealthCheckStatus, - MCPConnectionPhase, - MCPLogLevel, - MCPRefType, - UrlProtocol -} from '$lib/enums'; +import { CACHE, DEFAULT_MCP_CONFIG, MCP_RECONNECT, MCP_SERVER_ID_PREFIX } from '$lib/constants'; +import { ColorMode, HealthCheckStatus, MCPConnectionPhase, MCPRefType } from '$lib/enums'; import { MCPService } from '$lib/services/mcp.service'; // direct imports between stores, not via the barrel, to avoid circular deps -import { mcpResourceStore } from '$lib/stores/mcp-resources.svelte'; +import { MCPHealthCheckManager, type McpHealthHost } from '$lib/stores/mcp/health.svelte'; +import { mcpResourceStore } from '$lib/stores/mcp/resources.svelte'; import { serverStore } from '$lib/stores/server.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { - ClientCapabilities, GetPromptResult, HealthCheckParams, HealthCheckState, - MCPCapabilitiesInfo, MCPClientConfig, MCPConnection, - MCPConnectionLog, MCPPromptInfo, MCPResourceAttachment, MCPResourceContent, - MCPResourceIcon, MCPServerConfig, MCPServerDisplayInfo, MCPServerSettingsEntry, MCPToolCall, - ServerCapabilities, ServerStatus, Tool, ToolExecutionResult @@ -72,484 +41,78 @@ import type { DatabaseMessageExtraMcpResource, McpServerOverride } from '$lib/ty import type { SettingsConfigType } from '$lib/types/settings'; import { detectMcpTransportFromUrl, - extractRootDomain, + getMcpIconUrl, + getMcpServerFaviconFallback, + getMcpServerLabel, parseMcpServerSettings, uuid } from '$lib/utils'; import { mode } from 'mode-watcher'; -class MCPStore { - private _isInitializing = $state(false); +class MCPStore implements McpHealthHost { private _error = $state(null); + private _isInitializing = $state(false); private _toolCount = $state(0); - private _connectedServers = $state([]); - private _healthChecks = $state>({}); - - private connections = new Map(); - private toolsIndex = new Map(); - private serverConfigs = new Map(); // Store configs for reconnection - private reconnectingServers = new Set(); // Guard against concurrent reconnections - private configSignature: string | null = null; - private initPromise: Promise | null = null; private activeFlowCount = 0; - get isProxyAvailable(): boolean { - return serverStore.props?.cors_proxy_enabled ?? false; + private configSignature: string | null = null; + private connectedServers = $state([]); + private connections = new Map(); + // health checks: per-server connectivity probes with optional promotion to active connections + private health = new MCPHealthCheckManager(this); + private initPromise: Promise | null = null; + private reconnectingServers = new Set(); // Guard against concurrent reconnections + private serverConfigs = new Map(); // Store configs for reconnection + private serversCache: { raw: unknown; servers: MCPServerSettingsEntry[] } | null = null; + private toolsIndex = new Map(); + + get availableTools(): string[] { + return Array.from(this.toolsIndex.keys()); } - /** - * Generates a unique server ID from an optional ID string or index. - */ - #generateServerId(id: unknown, index: number): string { - if (typeof id === 'string' && id.trim()) { - return id.trim(); - } - - return `${MCP_SERVER_ID_PREFIX}-${index + 1}`; + get connectedServerCount(): number { + return this.connectedServers.length; } - /** - * Parses raw server settings from config into MCPServerSettingsEntry array. - */ - #parseServerSettings(rawServers: unknown): MCPServerSettingsEntry[] { - if (!rawServers) { - return []; - } - - let parsed: unknown; - - if (typeof rawServers === 'string') { - const trimmed = rawServers.trim(); - - if (!trimmed) { - return []; - } - - try { - parsed = JSON.parse(trimmed); - } catch (error) { - console.warn('[MCP] Failed to parse mcpServers JSON:', error); - - return []; - } - } else { - parsed = rawServers; - } - - if (!Array.isArray(parsed)) { - return []; - } - - return parsed.map((entry, index) => { - const url = typeof entry?.url === 'string' ? entry.url.trim() : ''; - const headers = typeof entry?.headers === 'string' ? entry.headers.trim() : undefined; - - return { - displayName: (entry as { displayName?: string })?.displayName, - enabled: Boolean((entry as { enabled?: unknown })?.enabled), - headers: headers || undefined, - id: this.#generateServerId((entry as { id?: unknown })?.id, index), - name: (entry as { name?: string })?.name, - url, - useProxy: Boolean((entry as { useProxy?: unknown })?.useProxy) - } satisfies MCPServerSettingsEntry; - }); - } - - /** - * Request timeout in milliseconds, read live from the global setting - * so a change in Settings applies to every server immediately. - */ - #requestTimeoutMs(): number { - const seconds = - Number(settingsStore.config.mcpRequestTimeoutSeconds) || - DEFAULT_MCP_CONFIG.requestTimeoutSeconds; - - return Math.round(seconds * 1000); - } - - /** - * Builds server configuration from a settings entry. - */ - #buildServerConfig( - entry: MCPServerSettingsEntry, - connectionTimeoutMs = DEFAULT_MCP_CONFIG.connectionTimeoutMs - ): MCPServerConfig | undefined { - if (!entry?.url) { - return undefined; - } - - let headers: Record | undefined; - - if (entry.headers) { - try { - const parsed = JSON.parse(entry.headers); - - if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) - headers = parsed as Record; - } catch { - console.warn('[MCP] Failed to parse custom headers JSON:', entry.headers); - } - } - - return { - handshakeTimeoutMs: connectionTimeoutMs, - headers, - requestTimeoutMs: this.#requestTimeoutMs(), - transport: detectMcpTransportFromUrl(entry.url), - url: entry.url, - useProxy: entry.useProxy - }; - } - - /** - * Checks if a server is enabled for a given chat. - * A per-chat override wins when present; a server without one resolves - * to its own `enabled` flag in `mcpServers`. - */ - #checkServerEnabled( - server: MCPServerSettingsEntry, - perChatOverrides?: McpServerOverride[] - ): boolean { - // Per-chat overrides win when present; missing entries inherit the - // server's own `enabled` flag so partial override lists are not all - // treated as disabled. - const override = perChatOverrides?.find((o) => o.serverId === server.id); - - return override?.enabled ?? server.enabled; - } - - /** - * Builds MCP client configuration from settings. - */ - #buildMcpClientConfig( - cfg: SettingsConfigType, - perChatOverrides?: McpServerOverride[] - ): MCPClientConfig | undefined { - const rawServers = this.#parseServerSettings(cfg.mcpServers); - - if (!rawServers.length) { - return undefined; - } - - const servers: Record = {}; - - for (const [index, entry] of rawServers.entries()) { - if (!this.#checkServerEnabled(entry, perChatOverrides)) continue; - - const normalized = this.#buildServerConfig(entry); - - if (normalized) servers[this.#generateServerId(entry.id, index)] = normalized; - } - - if (Object.keys(servers).length === 0) { - return undefined; - } - - return { - capabilities: DEFAULT_MCP_CONFIG.capabilities, - clientInfo: DEFAULT_MCP_CONFIG.clientInfo, - protocolVersion: DEFAULT_MCP_CONFIG.protocolVersion, - requestTimeoutMs: this.#requestTimeoutMs(), - servers - }; - } - - /** - * Builds capabilities info from server and client capabilities. - */ - #buildCapabilitiesInfo( - serverCaps?: ServerCapabilities, - clientCaps?: ClientCapabilities - ): MCPCapabilitiesInfo { - return { - client: { - elicitation: clientCaps?.elicitation - ? { form: !!clientCaps.elicitation.form, url: !!clientCaps.elicitation.url } - : undefined, - roots: clientCaps?.roots ? { listChanged: clientCaps.roots.listChanged } : undefined, - sampling: !!clientCaps?.sampling, - tasks: !!clientCaps?.tasks - }, - server: { - completions: !!serverCaps?.completions, - logging: !!serverCaps?.logging, - prompts: serverCaps?.prompts ? { listChanged: serverCaps.prompts.listChanged } : undefined, - resources: serverCaps?.resources - ? { - listChanged: serverCaps.resources.listChanged, - subscribe: serverCaps.resources.subscribe - } - : undefined, - tasks: !!serverCaps?.tasks, - tools: serverCaps?.tools ? { listChanged: serverCaps.tools.listChanged } : undefined - } - }; - } - - get isInitializing(): boolean { - return this._isInitializing; - } - - get isInitialized(): boolean { - return this.connections.size > 0; + get connectedServerNames(): string[] { + return this.connectedServers; } get error(): string | null { return this._error; } - get toolCount(): number { - return this._toolCount; - } - - get connectedServerCount(): number { - return this._connectedServers.length; - } - - get connectedServerNames(): string[] { - return this._connectedServers; - } - get isEnabled(): boolean { - const mcpConfig = this.#buildMcpClientConfig(settingsStore.config); + const mcpConfig = this.buildMcpClientConfig(settingsStore.config); return ( mcpConfig !== null && mcpConfig !== undefined && Object.keys(mcpConfig.servers).length > 0 ); } - get availableTools(): string[] { - return Array.from(this.toolsIndex.keys()); + get isInitialized(): boolean { + return this.connections.size > 0; } - private updateState(state: { - isInitializing?: boolean; - error?: string | null; - toolCount?: number; - connectedServers?: string[]; - }): void { - if (state.isInitializing !== undefined) { - this._isInitializing = state.isInitializing; - } - - if (state.error !== undefined) { - this._error = state.error; - } - - if (state.toolCount !== undefined) { - this._toolCount = state.toolCount; - } - - if (state.connectedServers !== undefined) { - this._connectedServers = state.connectedServers; - } + get isInitializing(): boolean { + return this._isInitializing; } - updateHealthCheck(serverId: string, state: HealthCheckState): void { - this._healthChecks = { ...this._healthChecks, [serverId]: state }; + get isProxyAvailable(): boolean { + return serverStore.props?.cors_proxy_enabled ?? false; } - getHealthCheckState(serverId: string): HealthCheckState { - return this._healthChecks[serverId] ?? { status: HealthCheckStatus.IDLE }; + /** Resource state, composed here so consumers have a single MCP scope. */ + get resources() { + return mcpResourceStore; } - hasHealthCheck(serverId: string): boolean { - return ( - serverId in this._healthChecks && - this._healthChecks[serverId].status !== HealthCheckStatus.IDLE - ); + get toolCount(): number { + return this._toolCount; } - clearHealthCheck(serverId: string): void { - const { [serverId]: _removed, ...rest } = this._healthChecks; - - this._healthChecks = rest; - } - - clearAllHealthChecks(): void { - this._healthChecks = {}; - } - - clearError(): void { - this._error = null; - } - - getServers(): MCPServerSettingsEntry[] { - return parseMcpServerSettings(settingsStore.config.mcpServers); - } - - /** - * Get all active MCP connections. - * @returns Map of server names to connections - */ - getConnections(): Map { - return this.connections; - } - - /** - * Resolves the raw label for a server: user-defined display name first, - * then server-reported title or name when the health check succeeded, - * then the configured name (admin baseline or legacy data), then URL. - */ - #serverBaseLabel(server: MCPServerDisplayInfo): string { - if (server.displayName) return server.displayName; - - const healthState = this.getHealthCheckState(server.id); - - if (healthState?.status === HealthCheckStatus.SUCCESS) - return ( - healthState.serverInfo?.title || healthState.serverInfo?.name || server.name || server.url - ); - - return server.name || server.url; - } - - /** - * Returns the display label for a server, suffixed with a positional - * counter when several configured servers resolve to the same base label - * (e.g. two endpoints of the same host reporting an identical name). - * Numbering follows config order, so it is stable across renders. - */ - getServerLabel(server: MCPServerDisplayInfo): string { - const label = this.#serverBaseLabel(server); - const twins = this.getServers().filter((s) => this.#serverBaseLabel(s) === label); - - if (twins.length < 2) return label; - - const position = twins.findIndex((s) => s.id === server.id); - - return position < 0 ? label : `${label} (${position + 1})`; - } - - getServerById(serverId: string): MCPServerSettingsEntry | undefined { - return this.getServers().find((s) => s.id === serverId); - } - - /** - * Get display name for an MCP server by its ID. - * Falls back to the server ID if server is not found. - */ - getServerDisplayName(serverId: string): string { - const server = this.getServerById(serverId); - - return server ? this.getServerLabel(server) : serverId; - } - - /** - * Validates that an icon URI uses a safe scheme (https: or data:). - */ - #isValidIconUri(src: string): boolean { - try { - if (src.startsWith(UrlProtocol.DATA)) return true; - - const url = new URL(src); - - return url.protocol === UrlProtocol.HTTPS; - } catch { - return false; - } - } - - /** - * Selects the best icon URL from an MCP icons array. - * Follows security guidelines from the MCP specification: - * - Only allows https: and data: URIs - * - Filters to supported MIME types - * - * Selection priority: - * 1. Icon matching the current color scheme (dark/light) - * 2. Universal icon (no theme specified); if exactly 2, assumes [0]=light, [1]=dark - * 3. First valid icon as last resort - */ - #getMcpIconUrl(icons: MCPResourceIcon[] | undefined, isDark = false): string | null { - if (!icons?.length) return null; - - const validIcons = icons.filter((icon) => { - if (!icon.src || !this.#isValidIconUri(icon.src)) return false; - - if (icon.mimeType && !MCP_ALLOWED_ICON_MIME_TYPES.has(icon.mimeType)) return false; - - return true; - }); - - if (validIcons.length === 0) return null; - - const preferredTheme = isDark ? ColorMode.DARK : ColorMode.LIGHT; - // 1. Prefer icon explicitly matching the current color scheme - const themedIcon = validIcons.find((icon) => icon.theme === preferredTheme); - - if (themedIcon) return themedIcon.src; - - // 2. Handle universal icons (no theme specified) - const universalIcons = validIcons.filter((icon) => !icon.theme); - - if (universalIcons.length === EXPECTED_THEMED_ICON_PAIR_COUNT) { - // Heuristic: two theme-less icons → assume [0] = light, [1] = dark - return universalIcons[isDark ? 1 : 0].src; - } - - if (universalIcons.length > 0) { - return universalIcons[0].src; - } - - // 3. Last resort: use opposite-theme icon - return validIcons[0].src; - } - - /** - * Get icon URL for an MCP server by its ID. - * Returns the best icon from the MCP server's `icons` array - * (see MCP spec: spec.modelcontextprotocol.io). - * Returns null if no icon is available. - */ - getServerFavicon(serverId: string): string | null { - const server = this.getServerById(serverId); - - if (!server) { - return null; - } - - const isDark = mode.current === ColorMode.DARK; - const healthState = this.getHealthCheckState(serverId); - - if (healthState.status === HealthCheckStatus.SUCCESS && healthState.serverInfo?.icons) { - const mcpIconUrl = this.#getMcpIconUrl(healthState.serverInfo.icons, isDark); - - if (mcpIconUrl) { - return mcpIconUrl; - } - } - - return this.#getServerFaviconFallback(server.url); - } - - /** - * Construct a fallback favicon URL from the MCP server URL. - * e.g. https://mcp.example.com/sse -> https://example.com/favicon.ico - */ - #getServerFaviconFallback(serverUrl: string): string | null { - try { - const url = new URL(serverUrl); - const rootDomain = extractRootDomain(url); - - if (!rootDomain) return null; - - const origin = `${url.protocol}//${rootDomain}`; - const candidates = ['favicon.ico', 'favicon.png']; - - for (const path of candidates) { - const faviconUrl = `${origin}/${path}`; - - if (this.#isValidIconUri(faviconUrl)) { - return faviconUrl; - } - } - } catch { - // Invalid URL, return null - } - - return null; + acquireConnection(): void { + this.activeFlowCount++; } addServer( @@ -571,321 +134,40 @@ class MCPStore { return newServer; } - updateServer(id: string, updates: Partial): void { - const servers = this.getServers(); + /** + * Add a resource as attachment to chat context. + * Automatically fetches content if not cached. + */ + async attachResource(uri: string): Promise { + const resourceInfo = mcpResourceStore.findResourceByUri(uri); - settingsStore.updateConfig( - SETTINGS_KEYS.MCP_SERVERS, - JSON.stringify( - servers.map((server) => (server.id === id ? { ...server, ...updates } : server)) - ) - ); - } + if (!resourceInfo) { + console.error(`[MCPStore] Resource not found: ${uri}`); - removeServer(id: string): void { - const servers = this.getServers(); - - settingsStore.updateConfig( - SETTINGS_KEYS.MCP_SERVERS, - JSON.stringify(servers.filter((s) => s.id !== id)) - ); - this.clearHealthCheck(id); - } - - hasAvailableServers(): boolean { - return parseMcpServerSettings(settingsStore.config.mcpServers).some( - (s) => s.enabled && s.url.trim() - ); - } - hasEnabledServers(perChatOverrides?: McpServerOverride[]): boolean { - return Boolean(this.#buildMcpClientConfig(settingsStore.config, perChatOverrides)); - } - - getEnabledServersForConversation( - perChatOverrides?: McpServerOverride[] - ): MCPServerSettingsEntry[] { - return this.getServers().filter((server) => { - return this.#checkServerEnabled(server, perChatOverrides); - }); - } - - async ensureInitialized(perChatOverrides?: McpServerOverride[]): Promise { - if (!browser) { - return false; + return null; } - const mcpConfig = this.#buildMcpClientConfig(settingsStore.config, perChatOverrides); - const signature = mcpConfig ? JSON.stringify(mcpConfig) : null; - - if (!signature) { - await this.shutdown(); - - return false; + if (mcpResourceStore.isAttached(uri)) { + return null; } - if (this.isInitialized && this.configSignature === signature) { - return true; - } + const attachment = mcpResourceStore.addAttachment(resourceInfo); - if (this.initPromise && this.configSignature === signature) { - return this.initPromise; - } + try { + const content = await this.readResource(uri); - if (this.connections.size > 0 || this.initPromise) await this.shutdown(); - - return this.initialize(signature, mcpConfig!); - } - - private async initialize(signature: string, mcpConfig: MCPClientConfig): Promise { - this.updateState({ error: null, isInitializing: true }); - this.configSignature = signature; - - const serverEntries = Object.entries(mcpConfig.servers); - - if (serverEntries.length === 0) { - this.updateState({ connectedServers: [], isInitializing: false, toolCount: 0 }); - - return false; - } - - this.initPromise = this.doInitialize(signature, mcpConfig, serverEntries); - - return this.initPromise; - } - - private async doInitialize( - signature: string, - mcpConfig: MCPClientConfig, - serverEntries: [string, MCPClientConfig['servers'][string]][] - ): Promise { - const clientInfo = mcpConfig.clientInfo ?? DEFAULT_MCP_CONFIG.clientInfo; - const capabilities = mcpConfig.capabilities ?? DEFAULT_MCP_CONFIG.capabilities; - const results = await Promise.allSettled( - serverEntries.map(async ([name, serverConfig]) => { - // Store config for reconnection - this.serverConfigs.set(name, serverConfig); - - const listChangedHandlers = this.createListChangedHandlers(name); - const connection = await MCPService.connect( - name, - serverConfig, - clientInfo, - capabilities, - (phase) => { - // Handle WebSocket disconnection - if (phase === MCPConnectionPhase.DISCONNECTED) { - console.log(`[MCPStore][${name}] Connection lost, starting auto-reconnect`); - this.autoReconnect(name); - } - }, - listChangedHandlers - ); - - return { connection, name }; - }) - ); - - if (this.configSignature !== signature) { - for (const result of results) { - if (result.status === 'fulfilled') - await MCPService.disconnect(result.value.connection).catch(console.warn); - } - - return false; - } - - for (const result of results) { - if (result.status === 'fulfilled') { - const { connection, name } = result.value; - - this.connections.set(name, connection); - - for (const tool of connection.tools) { - if (this.toolsIndex.has(tool.name)) - console.warn( - `[MCPStore] Tool name conflict: "${tool.name}" exists in "${this.toolsIndex.get(tool.name)}" and "${name}". Using tool from "${name}".` - ); - - this.toolsIndex.set(tool.name, name); - } + if (content) { + mcpResourceStore.updateAttachmentContent(attachment.id, content); } else { - console.error(`[MCPStore] Failed to connect:`, result.reason); + mcpResourceStore.updateAttachmentError(attachment.id, 'Failed to read resource'); } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + + mcpResourceStore.updateAttachmentError(attachment.id, message); } - const successCount = this.connections.size; - - if (successCount === 0 && serverEntries.length > 0) { - this.updateState({ - connectedServers: [], - error: 'All MCP server connections failed', - isInitializing: false, - toolCount: 0 - }); - this.initPromise = null; - - return false; - } - - this.updateState({ - connectedServers: Array.from(this.connections.keys()), - error: null, - isInitializing: false, - toolCount: this.toolsIndex.size - }); - this.initPromise = null; - - return true; - } - - private createListChangedHandlers(serverName: string): ListChangedHandlers { - return { - prompts: { - onChanged: (error: Error | null) => { - if (error) { - console.warn(`[MCPStore][${serverName}] Prompts list changed error:`, error); - - return; - } - } - }, - tools: { - onChanged: (error: Error | null, tools: Tool[] | null) => { - if (error) { - console.warn(`[MCPStore][${serverName}] Tools list changed error:`, error); - - return; - } - - this.handleToolsListChanged(serverName, tools ?? []); - } - } - }; - } - - private handleToolsListChanged(serverName: string, tools: Tool[]): void { - const connection = this.connections.get(serverName); - - if (!connection) { - return; - } - - for (const [toolName, ownerServer] of this.toolsIndex.entries()) { - if (ownerServer === serverName) this.toolsIndex.delete(toolName); - } - - connection.tools = tools; - - for (const tool of tools) { - if (this.toolsIndex.has(tool.name)) - console.warn( - `[MCPStore] Tool name conflict after list change: "${tool.name}" exists in "${this.toolsIndex.get(tool.name)}" and "${serverName}". Using tool from "${serverName}".` - ); - - this.toolsIndex.set(tool.name, serverName); - } - this.updateState({ toolCount: this.toolsIndex.size }); - } - - acquireConnection(): void { - this.activeFlowCount++; - } - - /** - * Release a connection reference. - * By default, keeps connections alive for reuse (shutdownIfUnused=false). - * MCP spec encourages long-lived sessions to avoid reconnection overhead. - */ - async releaseConnection(shutdownIfUnused = false): Promise { - this.activeFlowCount = Math.max(0, this.activeFlowCount - 1); - - if (shutdownIfUnused && this.activeFlowCount === 0) { - await this.shutdown(); - } - } - - getActiveFlowCount(): number { - return this.activeFlowCount; - } - - async shutdown(): Promise { - if (this.initPromise) { - await this.initPromise.catch(() => {}); - this.initPromise = null; - } - - if (this.connections.size === 0) { - return; - } - - await Promise.all( - Array.from(this.connections.values()).map((conn) => - MCPService.disconnect(conn).catch((error) => - console.warn(`[MCPStore] Error disconnecting ${conn.serverName}:`, error) - ) - ) - ); - - this.connections.clear(); - this.toolsIndex.clear(); - this.serverConfigs.clear(); - this.configSignature = null; - this.updateState({ - connectedServers: [], - error: null, - isInitializing: false, - toolCount: 0 - }); - } - - /** - * Immediately reconnect to a server by creating a fresh transport and session. - * Used when a session-expired error (HTTP 404) is detected during tool execution. - * Per MCP spec 2025-11-25: client MUST discard session ID and re-initialize. - * - * Unlike autoReconnect (which uses exponential backoff for connectivity issues), - * this performs a single immediate reconnection attempt since the server is known - * to be reachable (it responded with 404). - */ - private async reconnectServer(serverName: string): Promise { - const serverConfig = this.serverConfigs.get(serverName); - - if (!serverConfig) { - throw new Error(`[MCPStore] No config found for ${serverName}, cannot reconnect`); - } - - // Disconnect stale connection (clears old transport + session ID) - const oldConnection = this.connections.get(serverName); - - if (oldConnection) { - await MCPService.disconnect(oldConnection).catch(console.warn); - this.connections.delete(serverName); - } - - console.log(`[MCPStore][${serverName}] Session expired, reconnecting with fresh session...`); - - const listChangedHandlers = this.createListChangedHandlers(serverName); - const connection = await MCPService.connect( - serverName, - serverConfig, - DEFAULT_MCP_CONFIG.clientInfo, - DEFAULT_MCP_CONFIG.capabilities, - (phase) => { - if (phase === MCPConnectionPhase.DISCONNECTED) { - console.log(`[MCPStore][${serverName}] Connection lost, starting auto-reconnect`); - this.autoReconnect(serverName); - } - }, - listChangedHandlers - ); - - // Replace connection and rebuild tool index for this server - this.connections.set(serverName, connection); - for (const tool of connection.tools) { - this.toolsIndex.set(tool.name, serverName); - } - - console.log(`[MCPStore][${serverName}] Session recovered successfully`); + return mcpResourceStore.getAttachment(attachment.id) ?? null; } /** @@ -901,7 +183,7 @@ class MCPStore { * set inside the phase callback and honoured in the `finally` block after * the guard entry has been removed. */ - private async autoReconnect(serverName: string): Promise { + async autoReconnect(serverName: string): Promise { // Guard against concurrent reconnections if (this.reconnectingServers.has(serverName)) { console.log(`[MCPStore][${serverName}] Reconnection already in progress, skipping`); @@ -968,13 +250,10 @@ class MCPStore { ); const connection = await Promise.race([connectPromise, timeoutPromise]); - // Replace old connection with new one this.connections.set(serverName, connection); // Rebuild tool index for this server - for (const tool of connection.tools) { - this.toolsIndex.set(tool.name, serverName); - } + this.indexServerTools(serverName, connection.tools); console.log(`[MCPStore][${serverName}] Reconnected successfully`); @@ -998,182 +277,68 @@ class MCPStore { } } - getToolNames(): string[] { - return Array.from(this.toolsIndex.keys()); + clearError(): void { + this._error = null; } - hasTool(toolName: string): boolean { - return this.toolsIndex.has(toolName); - } - - getToolServer(toolName: string): string | undefined { - return this.toolsIndex.get(toolName); + clearHealthCheck(serverId: string): void { + this.health.clear(serverId); } /** - * Resolve which configured MCP server owns a given tool name. Looks at - * active connections first (fast path), then falls back to per-server - * health-check data so server-side MCP proxies (where llama-server - * executes MCP tools but the browser does not hold a direct connection) - * still resolve tool names to their owning server. + * Clear all resource attachments. */ - findServerForTool(toolName: string): string | undefined { - const fromIndex = this.toolsIndex.get(toolName); - - if (fromIndex) return fromIndex; - - for (const server of this.getServers()) { - const health = this._healthChecks[server.id]; - - if (!health || health.status !== HealthCheckStatus.SUCCESS) continue; - - if (health.tools.some((tool) => tool.name === toolName)) { - return server.id; - } - } - - return undefined; + clearResourceAttachments(): void { + mcpResourceStore.clearAttachments(); } /** - * Resolve the favicon URL for an MCP server by one of its tool names. - * Returns `null` if the tool is not provided by any configured MCP server, - * or if the owning server has no icon to show. - * Pair with {@link getServerFavicon} for direct server-id lookup. + * Convert current resource attachments to DatabaseMessageExtra[] and clear them. + * Called during message send to persist resources with the user message. */ - getServerFaviconForTool(toolName: string | undefined): string | null { - if (!toolName) return null; + consumeResourceAttachmentsAsExtras(): DatabaseMessageExtraMcpResource[] { + const extras = mcpResourceStore.toMessageExtras(); - const serverId = this.findServerForTool(toolName); - - if (!serverId) return null; - - return this.getServerFavicon(serverId); - } - - hasPromptsSupport(): boolean { - for (const connection of this.connections.values()) { - if (connection.serverCapabilities?.prompts) { - return true; - } + if (extras.length > 0) { + mcpResourceStore.clearAttachments(); } - return false; + return extras; } - /** - * Check if any enabled server with successful health check supports prompts. - * Uses health check state since servers may not have active connections until - * the user actually sends a message or uses prompts. - * @param perChatOverrides - Per-chat server overrides to filter by enabled servers. - * If provided (even empty array), only checks enabled servers. - * If undefined, falls back to each server's own `enabled` flag. - */ - hasPromptsCapability(perChatOverrides?: McpServerOverride[]): boolean { - let enabledServerIds: Set; - - if (perChatOverrides !== undefined) { - enabledServerIds = new Set(perChatOverrides.filter((o) => o.enabled).map((o) => o.serverId)); - } else { - enabledServerIds = new Set( - this.getServers() - .filter((s) => s.enabled) - .map((s) => s.id) - ); - } - - if (enabledServerIds.size === 0) { + async ensureInitialized(perChatOverrides?: McpServerOverride[]): Promise { + if (!browser) { return false; } - for (const [serverId, state] of Object.entries(this._healthChecks)) { - if (!enabledServerIds.has(serverId)) continue; + const mcpConfig = this.buildMcpClientConfig(settingsStore.config, perChatOverrides); + const signature = mcpConfig ? JSON.stringify(mcpConfig) : null; - if ( - state.status === HealthCheckStatus.SUCCESS && - state.capabilities?.server?.prompts !== undefined - ) { - return true; - } + if (!signature) { + await this.shutdown(); + + return false; } - for (const [serverName, connection] of this.connections) { - if (!enabledServerIds.has(serverName)) continue; - - if (connection.serverCapabilities?.prompts) { - return true; - } + if (this.isInitialized && this.configSignature === signature) { + return true; } - return false; - } - - async getAllPrompts(): Promise { - const results: MCPPromptInfo[] = []; - - for (const [serverName, connection] of this.connections) { - if (!connection.serverCapabilities?.prompts) continue; - - const prompts = await MCPService.listPrompts(connection); - - for (const prompt of prompts) { - results.push({ - arguments: prompt.arguments?.map((arg) => ({ - description: arg.description, - name: arg.name, - required: arg.required - })), - description: prompt.description, - name: prompt.name, - serverName, - title: prompt.title - }); - } + if (this.initPromise && this.configSignature === signature) { + return this.initPromise; } - return results; - } + if (this.connections.size > 0 || this.initPromise) await this.shutdown(); - async getPrompt( - serverName: string, - promptName: string, - args?: Record - ): Promise { - const connection = this.connections.get(serverName); - - if (!connection) throw new Error(`Server "${serverName}" not found for prompt "${promptName}"`); - - return MCPService.getPrompt(connection, promptName, args); + return this.initialize(signature, mcpConfig!); } async executeTool(toolCall: MCPToolCall, signal?: AbortSignal): Promise { - const toolName = toolCall.function.name; - const serverName = this.toolsIndex.get(toolName); - - if (!serverName) throw new Error(`Unknown tool: ${toolName}`); - - const connection = this.connections.get(serverName); - - if (!connection) throw new Error(`Server "${serverName}" is not connected`); - - const args = this.parseToolArguments(toolCall.function.arguments); - - try { - return await MCPService.callTool(connection, { arguments: args, name: toolName }, signal); - } catch (error) { - // Session expired (server restarted) - reconnect and retry once - if (MCPService.isSessionExpiredError(error)) { - await this.reconnectServer(serverName); - - const newConnection = this.connections.get(serverName); - - if (!newConnection) throw new Error(`Failed to reconnect to "${serverName}"`); - - return MCPService.callTool(newConnection, { arguments: args, name: toolName }, signal); - } - - throw error; - } + return this.executeToolByName( + toolCall.function.name, + this.parseToolArguments(toolCall.function.arguments), + signal + ); } async executeToolByName( @@ -1206,514 +371,6 @@ class MCPStore { } } - private parseToolArguments(args: string | Record): Record { - if (typeof args === 'string') { - const trimmed = args.trim(); - - if (trimmed === '') { - return {}; - } - - try { - const parsed = JSON.parse(trimmed); - - if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) - throw new Error( - `Tool arguments must be an object, got ${Array.isArray(parsed) ? 'array' : typeof parsed}` - ); - - return parsed as Record; - } catch (error) { - throw new Error(`Failed to parse tool arguments as JSON: ${(error as Error).message}`); - } - } - - if (typeof args === 'object' && args !== null && !Array.isArray(args)) { - return args; - } - - throw new Error(`Invalid tool arguments type: ${typeof args}`); - } - - async getPromptCompletions( - serverName: string, - promptName: string, - argumentName: string, - argumentValue: string - ): Promise<{ values: string[]; total?: number; hasMore?: boolean } | null> { - const connection = this.connections.get(serverName); - - if (!connection) { - console.warn(`[MCPStore] Server "${serverName}" is not connected`); - - return null; - } - - if (!connection.serverCapabilities?.completions) { - return null; - } - - return MCPService.complete( - connection, - { name: promptName, type: MCPRefType.PROMPT }, - { name: argumentName, value: argumentValue } - ); - } - - /** - * Get completions for a resource template argument. - * Uses the MCP Completion API with ref/resource. - */ - async getResourceCompletions( - serverName: string, - uriTemplate: string, - argumentName: string, - argumentValue: string - ): Promise<{ values: string[]; total?: number; hasMore?: boolean } | null> { - const connection = this.connections.get(serverName); - - if (!connection) { - console.warn(`[MCPStore] Server "${serverName}" is not connected`); - - return null; - } - - if (!connection.serverCapabilities?.completions) { - return null; - } - - return MCPService.complete( - connection, - { type: MCPRefType.RESOURCE, uri: uriTemplate }, - { name: argumentName, value: argumentValue } - ); - } - - /** - * Read a resource by an arbitrary URI (e.g., one expanded from a template). - * Unlike readResource(), this does not require the URI to be in the resources list. - */ - async readResourceByUri(serverName: string, uri: string): Promise { - const connection = this.connections.get(serverName); - - if (!connection) { - console.error(`[MCPStore] No connection found for server: ${serverName}`); - - return null; - } - - try { - const result = await MCPService.readResource(connection, uri); - - return result.contents; - } catch (error) { - console.error(`[MCPStore] Failed to read resource ${uri}:`, error); - - return null; - } - } - - private parseHeaders(headersJson?: string): Record | undefined { - if (!headersJson?.trim()) { - return undefined; - } - - try { - const parsed = JSON.parse(headersJson); - - if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) - return parsed as Record; - } catch { - console.warn('[MCPStore] Failed to parse custom headers JSON:', headersJson); - } - - return undefined; - } - - async runHealthChecksForServers( - servers: { - id: string; - enabled: boolean; - url: string; - headers?: string; - }[], - skipIfChecked = true, - promoteToActive = false - ): Promise { - const serversToCheck = skipIfChecked - ? servers.filter((s) => !this.hasHealthCheck(s.id) && s.url.trim()) - : servers.filter((s) => s.url.trim()); - - if (serversToCheck.length === 0) { - return; - } - - const BATCH_SIZE = 5; - - for (let i = 0; i < serversToCheck.length; i += BATCH_SIZE) { - const batch = serversToCheck.slice(i, i + BATCH_SIZE); - - await Promise.allSettled(batch.map((server) => this.runHealthCheck(server, promoteToActive))); - } - } - - /** - * Check if a server already has an active connection that can be reused. - * Returns the existing connection if available. - */ - getExistingConnection(serverId: string): MCPConnection | undefined { - return this.connections.get(serverId); - } - - /** - * Run a health check for a server. - * If the server already has an active connection, reuses it instead of creating a new one. - * If promoteToActive is true and server is enabled, the connection will be kept - * and promoted to an active connection instead of being disconnected. - */ - async runHealthCheck(server: HealthCheckParams, promoteToActive = false): Promise { - // Check if we already have an active connection for this server - const existingConnection = this.connections.get(server.id); - - if (existingConnection) { - // Reuse existing connection - just refresh tools list - try { - const tools = await MCPService.listTools(existingConnection); - const capabilities = this.#buildCapabilitiesInfo( - existingConnection.serverCapabilities, - existingConnection.clientCapabilities - ); - - this.updateHealthCheck(server.id, { - capabilities, - connectionTimeMs: existingConnection.connectionTimeMs, - instructions: existingConnection.instructions, - logs: [], - protocolVersion: existingConnection.protocolVersion, - serverInfo: existingConnection.serverInfo, - status: HealthCheckStatus.SUCCESS, - tools: tools.map((tool) => ({ - description: tool.description, - name: tool.name, - title: tool.title - })), - transportType: existingConnection.transportType - }); - - return; - } catch (error) { - console.warn( - `[MCPStore] Failed to reuse connection for ${server.id}, creating new one:`, - error - ); - // Connection may be stale, remove it and create new one - this.connections.delete(server.id); - } - } - - const trimmedUrl = server.url.trim(); - const logs: MCPConnectionLog[] = []; - - let currentPhase: MCPConnectionPhase = MCPConnectionPhase.IDLE; - - if (!trimmedUrl) { - this.updateHealthCheck(server.id, { - logs: [], - message: 'Please enter a server URL first.', - status: HealthCheckStatus.ERROR - }); - - return; - } - - this.updateHealthCheck(server.id, { - logs: [], - phase: MCPConnectionPhase.TRANSPORT_CREATING, - status: HealthCheckStatus.CONNECTING - }); - - const timeoutMs = this.#requestTimeoutMs(); - const headers = this.parseHeaders(server.headers); - - try { - const serverConfig: MCPServerConfig = { - handshakeTimeoutMs: DEFAULT_MCP_CONFIG.connectionTimeoutMs, - headers, - requestTimeoutMs: timeoutMs, - transport: detectMcpTransportFromUrl(trimmedUrl), - url: trimmedUrl, - useProxy: server.useProxy - }; - - // Store config for reconnection - this.serverConfigs.set(server.id, serverConfig); - - const connection = await MCPService.connect( - server.id, - serverConfig, - DEFAULT_MCP_CONFIG.clientInfo, - DEFAULT_MCP_CONFIG.capabilities, - (phase, log) => { - currentPhase = phase; - logs.push(log); - this.updateHealthCheck(server.id, { - logs: [...logs], - phase, - status: HealthCheckStatus.CONNECTING - }); - - // Handle WebSocket disconnection - if (phase === MCPConnectionPhase.DISCONNECTED && promoteToActive) { - console.log( - `[MCPStore][${server.id}] Connection lost during health check, starting auto-reconnect` - ); - this.autoReconnect(server.id); - } - } - ); - const tools = connection.tools.map((tool) => ({ - description: tool.description, - name: tool.name, - title: tool.title - })); - const capabilities = this.#buildCapabilitiesInfo( - connection.serverCapabilities, - connection.clientCapabilities - ); - - this.updateHealthCheck(server.id, { - capabilities, - connectionTimeMs: connection.connectionTimeMs, - instructions: connection.instructions, - logs, - protocolVersion: connection.protocolVersion, - serverInfo: connection.serverInfo, - status: HealthCheckStatus.SUCCESS, - tools, - transportType: connection.transportType - }); - - // Promote to active connection or disconnect - if (promoteToActive && server.enabled) { - this.promoteHealthCheckToConnection(server.id, connection); - } else { - await MCPService.disconnect(connection); - } - } catch (error) { - const message = error instanceof Error ? error.message : 'Unknown error occurred'; - - if (logs.at(-1)?.phase !== MCPConnectionPhase.ERROR) { - logs.push({ - level: MCPLogLevel.ERROR, - message: `Connection failed: ${message}`, - phase: MCPConnectionPhase.ERROR, - timestamp: new Date() - }); - } - - this.updateHealthCheck(server.id, { - logs, - message, - phase: currentPhase, - status: HealthCheckStatus.ERROR - }); - } - } - - /** - * Promote a health check connection to an active connection. - * This avoids the need to reconnect when the server is needed for agentic flows. - */ - private promoteHealthCheckToConnection(serverId: string, connection: MCPConnection): void { - // Register tools from the connection - for (const tool of connection.tools) { - if (this.toolsIndex.has(tool.name)) { - console.warn( - `[MCPStore] Tool name conflict during promotion: "${tool.name}" exists in "${this.toolsIndex.get(tool.name)}" and "${serverId}". Using tool from "${serverId}".` - ); - } - - this.toolsIndex.set(tool.name, serverId); - } - - // Add to active connections - this.connections.set(serverId, connection); - - // Update state - this.updateState({ - connectedServers: Array.from(this.connections.keys()), - toolCount: this.toolsIndex.size - }); - } - - getServersStatus(): ServerStatus[] { - const statuses: ServerStatus[] = []; - - for (const [name, connection] of this.connections) { - statuses.push({ - error: undefined, - isConnected: true, - name, - toolCount: connection.tools.length - }); - } - - return statuses; - } - - /** - * Get aggregated server instructions from all connected servers. - * Returns an array of { serverName, serverTitle, instructions } objects. - */ - getServerInstructions(): Array<{ - serverName: string; - serverTitle?: string; - instructions: string; - }> { - const results: Array<{ serverName: string; serverTitle?: string; instructions: string }> = []; - - for (const [serverName, connection] of this.connections) { - if (connection.instructions) { - results.push({ - instructions: connection.instructions, - serverName, - serverTitle: connection.serverInfo?.title || connection.serverInfo?.name - }); - } - } - - return results; - } - - /** - * Get server instructions from health check results (for display before active connection). - * Useful for showing instructions in settings UI. - */ - getHealthCheckInstructions(): Array<{ - serverId: string; - serverTitle?: string; - instructions: string; - }> { - const results: Array<{ serverId: string; serverTitle?: string; instructions: string }> = []; - - for (const [serverId, state] of Object.entries(this._healthChecks)) { - if (state.status === HealthCheckStatus.SUCCESS && state.instructions) { - results.push({ - instructions: state.instructions, - serverId, - serverTitle: state.serverInfo?.title || state.serverInfo?.name - }); - } - } - - return results; - } - - /** - * Check if any connected server has instructions. - */ - hasServerInstructions(): boolean { - for (const connection of this.connections.values()) { - if (connection.instructions) { - return true; - } - } - - return false; - } - - /** - * - * - * Resources Operations - * - * - */ - - /** - * Check if any enabled server with successful health check supports resources. - * Uses health check state since servers may not have active connections until - * the user actually sends a message or uses prompts. - * @param perChatOverrides - Per-chat server overrides to filter by enabled servers. - * If provided (even empty array), only checks enabled servers. - * If undefined, falls back to each server's own `enabled` flag. - */ - hasResourcesCapability(perChatOverrides?: McpServerOverride[]): boolean { - let enabledServerIds: Set; - - if (perChatOverrides !== undefined) { - enabledServerIds = new Set(perChatOverrides.filter((o) => o.enabled).map((o) => o.serverId)); - } else { - enabledServerIds = new Set( - this.getServers() - .filter((s) => s.enabled) - .map((s) => s.id) - ); - } - - if (enabledServerIds.size === 0) { - return false; - } - - for (const [serverId, state] of Object.entries(this._healthChecks)) { - if (!enabledServerIds.has(serverId)) continue; - - if ( - state.status === HealthCheckStatus.SUCCESS && - state.capabilities?.server?.resources !== undefined - ) { - return true; - } - } - - for (const [serverName, connection] of this.connections) { - if (!enabledServerIds.has(serverName)) continue; - - if (MCPService.supportsResources(connection)) { - return true; - } - } - - return false; - } - - /** - * Get list of enabled servers that support resources. - * Checks active connections first, then health check state as fallback. - */ - getServersWithResources(): string[] { - const enabledServerIds = new Set( - this.getServers() - .filter((s) => s.enabled) - .map((s) => s.id) - ); - const servers: string[] = []; - - // Check active connections - for (const [name, connection] of this.connections) { - if (!enabledServerIds.has(name)) continue; - - if (MCPService.supportsResources(connection) && !servers.includes(name)) { - servers.push(name); - } - } - - // Also check health check states for servers not yet connected - for (const [serverId, state] of Object.entries(this._healthChecks)) { - if (!enabledServerIds.has(serverId)) continue; - - if ( - !servers.includes(serverId) && - state.status === HealthCheckStatus.SUCCESS && - state.capabilities?.server?.resources !== undefined - ) { - servers.push(serverId); - } - } - - return servers; - } - /** * Fetch resources from all connected servers that support them. * Updates mcpResourceStore with the results. @@ -1793,12 +450,506 @@ class MCPStore { } } + /** + * Resolve which configured MCP server owns a given tool name. Looks at + * active connections first (fast path), then falls back to per-server + * health-check data so server-side MCP proxies (where llama-server + * executes MCP tools but the browser does not hold a direct connection) + * still resolve tool names to their owning server. + */ + findServerForTool(toolName: string): string | undefined { + const fromIndex = this.toolsIndex.get(toolName); + + if (fromIndex) return fromIndex; + + for (const server of this.getServers()) { + const health = this.health.checks[server.id]; + + if (!health || health.status !== HealthCheckStatus.SUCCESS) continue; + + if (health.tools.some((tool) => tool.name === toolName)) { + return server.id; + } + } + + return undefined; + } + getActiveFlowCount(): number { + return this.activeFlowCount; + } + + async getAllPrompts(): Promise { + const results: MCPPromptInfo[] = []; + + for (const [serverName, connection] of this.connections) { + if (!connection.serverCapabilities?.prompts) continue; + + const prompts = await MCPService.listPrompts(connection); + + for (const prompt of prompts) { + results.push({ + arguments: prompt.arguments?.map((arg) => ({ + description: arg.description, + name: arg.name, + required: arg.required + })), + description: prompt.description, + name: prompt.name, + serverName, + title: prompt.title + }); + } + } + + return results; + } + + /** + * Get all active MCP connections. + * @returns Map of server names to connections + */ + getConnections(): Map { + return this.connections; + } + + getEnabledServersForConversation( + perChatOverrides?: McpServerOverride[] + ): MCPServerSettingsEntry[] { + return this.getServers().filter((server) => { + return this.checkServerEnabled(server, perChatOverrides); + }); + } + + /** + * Check if a server already has an active connection that can be reused. + * Returns the existing connection if available. + */ + getExistingConnection(serverId: string): MCPConnection | undefined { + return this.connections.get(serverId); + } + + /** + * Get server instructions from health check results (for display before active connection). + * Useful for showing instructions in settings UI. + */ + getHealthCheckInstructions(): Array<{ + serverId: string; + serverTitle?: string; + instructions: string; + }> { + const results: Array<{ serverId: string; serverTitle?: string; instructions: string }> = []; + + for (const [serverId, state] of Object.entries(this.health.checks)) { + if (state.status === HealthCheckStatus.SUCCESS && state.instructions) { + results.push({ + instructions: state.instructions, + serverId, + serverTitle: state.serverInfo?.title || state.serverInfo?.name + }); + } + } + + return results; + } + + /** + * Health checks live in MCPHealthCheckManager; these delegate so + * consumers keep a single entry point. + */ + getHealthCheckState(serverId: string): HealthCheckState { + return this.health.getState(serverId); + } + + async getPrompt( + serverName: string, + promptName: string, + args?: Record + ): Promise { + const connection = this.connections.get(serverName); + + if (!connection) throw new Error(`Server "${serverName}" not found for prompt "${promptName}"`); + + return MCPService.getPrompt(connection, promptName, args); + } + + async getPromptCompletions( + serverName: string, + promptName: string, + argumentName: string, + argumentValue: string + ): Promise<{ values: string[]; total?: number; hasMore?: boolean } | null> { + const connection = this.connections.get(serverName); + + if (!connection) { + console.warn(`[MCPStore] Server "${serverName}" is not connected`); + + return null; + } + + if (!connection.serverCapabilities?.completions) { + return null; + } + + return MCPService.complete( + connection, + { name: promptName, type: MCPRefType.PROMPT }, + { name: argumentName, value: argumentValue } + ); + } + + /** + * Request timeout in milliseconds, read live from the global setting + * so a change in Settings applies to every server immediately. + */ + getRequestTimeoutMs(): number { + const seconds = + Number(settingsStore.config.mcpRequestTimeoutSeconds) || + DEFAULT_MCP_CONFIG.requestTimeoutSeconds; + + return Math.round(seconds * 1000); + } + + /** + * Get completions for a resource template argument. + * Uses the MCP Completion API with ref/resource. + */ + async getResourceCompletions( + serverName: string, + uriTemplate: string, + argumentName: string, + argumentValue: string + ): Promise<{ values: string[]; total?: number; hasMore?: boolean } | null> { + const connection = this.connections.get(serverName); + + if (!connection) { + console.warn(`[MCPStore] Server "${serverName}" is not connected`); + + return null; + } + + if (!connection.serverCapabilities?.completions) { + return null; + } + + return MCPService.complete( + connection, + { type: MCPRefType.RESOURCE, uri: uriTemplate }, + { name: argumentName, value: argumentValue } + ); + } + + /** + * Get formatted resource context for chat. + */ + getResourceContextForChat(): string { + return mcpResourceStore.formatAttachmentsForContext(); + } + + getServerById(serverId: string): MCPServerSettingsEntry | undefined { + return this.getServers().find((s) => s.id === serverId); + } + + /** + * Get display name for an MCP server by its ID. + * Falls back to the server ID if server is not found. + */ + getServerDisplayName(serverId: string): string { + const server = this.getServerById(serverId); + + return server ? this.getServerLabel(server) : serverId; + } + + /** + * Get icon URL for an MCP server by its ID. + * Returns the best icon from the MCP server's `icons` array + * (see MCP spec: spec.modelcontextprotocol.io). + * Returns null if no icon is available. + */ + getServerFavicon(serverId: string): string | null { + const server = this.getServerById(serverId); + + if (!server) { + return null; + } + + const isDark = mode.current === ColorMode.DARK; + const healthState = this.health.getState(serverId); + + if (healthState.status === HealthCheckStatus.SUCCESS && healthState.serverInfo?.icons) { + const mcpIconUrl = getMcpIconUrl(healthState.serverInfo.icons, isDark); + + if (mcpIconUrl) { + return mcpIconUrl; + } + } + + return getMcpServerFaviconFallback(server.url); + } + + /** + * Resolve the favicon URL for an MCP server by one of its tool names. + * Returns `null` if the tool is not provided by any configured MCP server, + * or if the owning server has no icon to show. + * Pair with {@link getServerFavicon} for direct server-id lookup. + */ + getServerFaviconForTool(toolName: string | undefined): string | null { + if (!toolName) return null; + + const serverId = this.findServerForTool(toolName); + + if (!serverId) return null; + + return this.getServerFavicon(serverId); + } + + /** + * Get aggregated server instructions from all connected servers. + * Returns an array of { serverName, serverTitle, instructions } objects. + */ + getServerInstructions(): Array<{ + serverName: string; + serverTitle?: string; + instructions: string; + }> { + const results: Array<{ serverName: string; serverTitle?: string; instructions: string }> = []; + + for (const [serverName, connection] of this.connections) { + if (connection.instructions) { + results.push({ + instructions: connection.instructions, + serverName, + serverTitle: connection.serverInfo?.title || connection.serverInfo?.name + }); + } + } + + return results; + } + + getServerLabel(server: MCPServerDisplayInfo): string { + return getMcpServerLabel(server, this.getServers(), this.health.checks); + } + + getServers(): MCPServerSettingsEntry[] { + const raw = settingsStore.config.mcpServers; + + // cache the parse: the config string rarely changes and getServers is + // called from hot paths (per-tool display lookups, capability checks) + if (this.serversCache && this.serversCache.raw === raw) { + return this.serversCache.servers; + } + + const servers = parseMcpServerSettings(raw); + + this.serversCache = { raw, servers }; + + return servers; + } + + getServersStatus(): ServerStatus[] { + const statuses: ServerStatus[] = []; + + for (const [name, connection] of this.connections) { + statuses.push({ + error: undefined, + isConnected: true, + name, + toolCount: connection.tools.length + }); + } + + return statuses; + } + + /** + * Get list of enabled servers that support resources. + * Checks active connections first, then health check state as fallback. + */ + getServersWithResources(): string[] { + const enabledServerIds = new Set( + this.getServers() + .filter((s) => s.enabled) + .map((s) => s.id) + ); + const servers: string[] = []; + + for (const [name, connection] of this.connections) { + if (!enabledServerIds.has(name)) continue; + + if (MCPService.supportsResources(connection) && !servers.includes(name)) { + servers.push(name); + } + } + + // Also check health check states for servers not yet connected + for (const [serverId, state] of Object.entries(this.health.checks)) { + if (!enabledServerIds.has(serverId)) continue; + + if ( + !servers.includes(serverId) && + state.status === HealthCheckStatus.SUCCESS && + state.capabilities?.server?.resources !== undefined + ) { + servers.push(serverId); + } + } + + return servers; + } + + getToolNames(): string[] { + return Array.from(this.toolsIndex.keys()); + } + + getToolServer(toolName: string): string | undefined { + return this.toolsIndex.get(toolName); + } + + hasAvailableServers(): boolean { + return parseMcpServerSettings(settingsStore.config.mcpServers).some( + (s) => s.enabled && s.url.trim() + ); + } + + hasEnabledServers(perChatOverrides?: McpServerOverride[]): boolean { + return Boolean(this.buildMcpClientConfig(settingsStore.config, perChatOverrides)); + } + + /** + * Check if any enabled server with successful health check supports prompts. + * Uses health check state since servers may not have active connections until + * the user actually sends a message or uses prompts. + */ + hasPromptsCapability(perChatOverrides?: McpServerOverride[]): boolean { + let enabledServerIds: Set; + + if (perChatOverrides !== undefined) { + enabledServerIds = new Set(perChatOverrides.filter((o) => o.enabled).map((o) => o.serverId)); + } else { + enabledServerIds = new Set( + this.getServers() + .filter((s) => s.enabled) + .map((s) => s.id) + ); + } + + if (enabledServerIds.size === 0) { + return false; + } + + for (const [serverId, state] of Object.entries(this.health.checks)) { + if (!enabledServerIds.has(serverId)) continue; + + if ( + state.status === HealthCheckStatus.SUCCESS && + state.capabilities?.server?.prompts !== undefined + ) { + return true; + } + } + + for (const [serverName, connection] of this.connections) { + if (!enabledServerIds.has(serverName)) continue; + + if (connection.serverCapabilities?.prompts) { + return true; + } + } + + return false; + } + + hasPromptsSupport(): boolean { + for (const connection of this.connections.values()) { + if (connection.serverCapabilities?.prompts) { + return true; + } + } + + return false; + } + + /** + * Check if any enabled server with successful health check supports resources. + * Uses health check state since servers may not have active connections until + * the user actually sends a message or uses prompts. + */ + hasResourcesCapability(perChatOverrides?: McpServerOverride[]): boolean { + let enabledServerIds: Set; + + if (perChatOverrides !== undefined) { + enabledServerIds = new Set(perChatOverrides.filter((o) => o.enabled).map((o) => o.serverId)); + } else { + enabledServerIds = new Set( + this.getServers() + .filter((s) => s.enabled) + .map((s) => s.id) + ); + } + + if (enabledServerIds.size === 0) { + return false; + } + + for (const [serverId, state] of Object.entries(this.health.checks)) { + if (!enabledServerIds.has(serverId)) continue; + + if ( + state.status === HealthCheckStatus.SUCCESS && + state.capabilities?.server?.resources !== undefined + ) { + return true; + } + } + + for (const [serverName, connection] of this.connections) { + if (!enabledServerIds.has(serverName)) continue; + + if (MCPService.supportsResources(connection)) { + return true; + } + } + + return false; + } + + /** + * Check if any connected server has instructions. + */ + hasServerInstructions(): boolean { + for (const connection of this.connections.values()) { + if (connection.instructions) { + return true; + } + } + + return false; + } + + hasTool(toolName: string): boolean { + return this.toolsIndex.has(toolName); + } + + /** + * Promote a health check connection to an active connection. + * This avoids the need to reconnect when the server is needed for agentic flows. + */ + promoteHealthCheckToConnection(serverId: string, connection: MCPConnection): void { + this.indexServerTools(serverId, connection.tools); + + this.connections.set(serverId, connection); + + this.updateState({ + connectedServers: Array.from(this.connections.keys()), + toolCount: this.toolsIndex.size + }); + } + /** * Read resource content from a server. * Caches the result in mcpResourceStore. */ async readResource(uri: string): Promise { - // Check cache first const cached = mcpResourceStore.getCachedContent(uri); if (cached) { @@ -1838,6 +989,120 @@ class MCPStore { } } + /** + * Read a resource by an arbitrary URI (e.g., one expanded from a template). + * Unlike readResource(), this does not require the URI to be in the resources list. + */ + async readResourceByUri(serverName: string, uri: string): Promise { + const connection = this.connections.get(serverName); + + if (!connection) { + console.error(`[MCPStore] No connection found for server: ${serverName}`); + + return null; + } + + try { + const result = await MCPService.readResource(connection, uri); + + return result.contents; + } catch (error) { + console.error(`[MCPStore] Failed to read resource ${uri}:`, error); + + return null; + } + } + + /** Store a server config so auto-reconnect can rebuild the session. */ + registerServerConfig(name: string, config: MCPServerConfig): void { + this.serverConfigs.set(name, config); + } + + /** + * Release a connection reference. + * By default, keeps connections alive for reuse (shutdownIfUnused=false). + * MCP spec encourages long-lived sessions to avoid reconnection overhead. + */ + async releaseConnection(shutdownIfUnused = false): Promise { + this.activeFlowCount = Math.max(0, this.activeFlowCount - 1); + + if (shutdownIfUnused && this.activeFlowCount === 0) { + await this.shutdown(); + } + } + + /** + * Drop a connection without disconnecting, e.g. when a health check finds + * it stale and recreates it. + */ + removeConnection(serverId: string): void { + this.connections.delete(serverId); + } + + /** + * Remove a resource attachment from chat context. + */ + removeResourceAttachment(attachmentId: string): void { + mcpResourceStore.removeAttachment(attachmentId); + } + + removeServer(id: string): void { + const servers = this.getServers(); + + settingsStore.updateConfig( + SETTINGS_KEYS.MCP_SERVERS, + JSON.stringify(servers.filter((s) => s.id !== id)) + ); + this.clearHealthCheck(id); + } + + async runHealthCheck(server: HealthCheckParams, promoteToActive = false): Promise { + return this.health.run(server, promoteToActive); + } + + async runHealthChecksForServers( + servers: { + id: string; + enabled: boolean; + url: string; + headers?: string; + }[], + skipIfChecked = true, + promoteToActive = false + ): Promise { + return this.health.runForServers(servers, skipIfChecked, promoteToActive); + } + + async shutdown(): Promise { + if (this.initPromise) { + await this.initPromise.catch(() => {}); + this.initPromise = null; + } + + if (this.connections.size === 0) { + return; + } + + await Promise.all( + Array.from(this.connections.values()).map((conn) => + MCPService.disconnect(conn).catch((error) => + console.warn(`[MCPStore] Error disconnecting ${conn.serverName}:`, error) + ) + ) + ); + + this.connections.clear(); + this.toolsIndex.clear(); + this.serverConfigs.clear(); + this.configSignature = null; + this.updateState({ + connectedServers: [], + error: null, + isInitializing: false, + toolCount: 0 + }); + } + /** * Subscribe to resource updates. */ @@ -1906,78 +1171,366 @@ class MCPStore { } } + updateServer(id: string, updates: Partial): void { + const servers = this.getServers(); + + settingsStore.updateConfig( + SETTINGS_KEYS.MCP_SERVERS, + JSON.stringify( + servers.map((server) => (server.id === id ? { ...server, ...updates } : server)) + ) + ); + } + /** - * Add a resource as attachment to chat context. - * Automatically fetches content if not cached. + * Builds MCP client configuration from settings. */ - async attachResource(uri: string): Promise { - const resourceInfo = mcpResourceStore.findResourceByUri(uri); + private buildMcpClientConfig( + cfg: SettingsConfigType, + perChatOverrides?: McpServerOverride[] + ): MCPClientConfig | undefined { + const rawServers = parseMcpServerSettings(cfg.mcpServers); - if (!resourceInfo) { - console.error(`[MCPStore] Resource not found: ${uri}`); - - return null; + if (!rawServers.length) { + return undefined; } - // Check if already attached - if (mcpResourceStore.isAttached(uri)) { - return null; + const servers: Record = {}; + + for (const [index, entry] of rawServers.entries()) { + if (!this.checkServerEnabled(entry, perChatOverrides)) continue; + + const normalized = this.buildServerConfig(entry); + + if (normalized) servers[this.generateServerId(entry.id, index)] = normalized; } - // Add attachment (initially loading) - const attachment = mcpResourceStore.addAttachment(resourceInfo); + if (Object.keys(servers).length === 0) { + return undefined; + } - // Fetch content - try { - const content = await this.readResource(uri); + return { + capabilities: DEFAULT_MCP_CONFIG.capabilities, + clientInfo: DEFAULT_MCP_CONFIG.clientInfo, + protocolVersion: DEFAULT_MCP_CONFIG.protocolVersion, + requestTimeoutMs: this.getRequestTimeoutMs(), + servers + }; + } - if (content) { - mcpResourceStore.updateAttachmentContent(attachment.id, content); - } else { - mcpResourceStore.updateAttachmentError(attachment.id, 'Failed to read resource'); + /** + * Builds server configuration from a settings entry. + */ + private buildServerConfig( + entry: MCPServerSettingsEntry, + connectionTimeoutMs = DEFAULT_MCP_CONFIG.connectionTimeoutMs + ): MCPServerConfig | undefined { + if (!entry?.url) { + return undefined; + } + + let headers: Record | undefined; + + if (entry.headers) { + try { + const parsed = JSON.parse(entry.headers); + + if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) + headers = parsed as Record; + } catch { + console.warn('[MCP] Failed to parse custom headers JSON:', entry.headers); } - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - - mcpResourceStore.updateAttachmentError(attachment.id, message); } - return mcpResourceStore.getAttachment(attachment.id) ?? null; + return { + handshakeTimeoutMs: connectionTimeoutMs, + headers, + requestTimeoutMs: this.getRequestTimeoutMs(), + transport: detectMcpTransportFromUrl(entry.url), + url: entry.url, + useProxy: entry.useProxy + }; } /** - * Remove a resource attachment from chat context. + * Checks if a server is enabled for a given chat. + * A per-chat override wins when present; a server without one resolves + * to its own `enabled` flag in `mcpServers`. */ - removeResourceAttachment(attachmentId: string): void { - mcpResourceStore.removeAttachment(attachmentId); + private checkServerEnabled( + server: MCPServerSettingsEntry, + perChatOverrides?: McpServerOverride[] + ): boolean { + const override = perChatOverrides?.find((o) => o.serverId === server.id); + + return override?.enabled ?? server.enabled; } - /** - * Clear all resource attachments. - */ - clearResourceAttachments(): void { - mcpResourceStore.clearAttachments(); + private createListChangedHandlers(serverName: string): ListChangedHandlers { + return { + prompts: { + onChanged: (error: Error | null) => { + if (error) { + console.warn(`[MCPStore][${serverName}] Prompts list changed error:`, error); + + return; + } + } + }, + tools: { + onChanged: (error: Error | null, tools: Tool[] | null) => { + if (error) { + console.warn(`[MCPStore][${serverName}] Tools list changed error:`, error); + + return; + } + + this.handleToolsListChanged(serverName, tools ?? []); + } + } + }; } - /** - * Get formatted resource context for chat. - */ - getResourceContextForChat(): string { - return mcpResourceStore.formatAttachmentsForContext(); - } + private async doInitialize( + signature: string, + mcpConfig: MCPClientConfig, + serverEntries: [string, MCPClientConfig['servers'][string]][] + ): Promise { + const clientInfo = mcpConfig.clientInfo ?? DEFAULT_MCP_CONFIG.clientInfo; + const capabilities = mcpConfig.capabilities ?? DEFAULT_MCP_CONFIG.capabilities; + const results = await Promise.allSettled( + serverEntries.map(async ([name, serverConfig]) => { + this.serverConfigs.set(name, serverConfig); - /** - * Convert current resource attachments to DatabaseMessageExtra[] and clear them. - * Called during message send to persist resources with the user message. - */ - consumeResourceAttachmentsAsExtras(): DatabaseMessageExtraMcpResource[] { - const extras = mcpResourceStore.toMessageExtras(); + const listChangedHandlers = this.createListChangedHandlers(name); + const connection = await MCPService.connect( + name, + serverConfig, + clientInfo, + capabilities, + (phase) => { + if (phase === MCPConnectionPhase.DISCONNECTED) { + console.log(`[MCPStore][${name}] Connection lost, starting auto-reconnect`); + this.autoReconnect(name); + } + }, + listChangedHandlers + ); - if (extras.length > 0) { - mcpResourceStore.clearAttachments(); + return { connection, name }; + }) + ); + + if (this.configSignature !== signature) { + for (const result of results) { + if (result.status === 'fulfilled') + await MCPService.disconnect(result.value.connection).catch(console.warn); + } + + return false; } - return extras; + for (const result of results) { + if (result.status === 'fulfilled') { + const { connection, name } = result.value; + + this.connections.set(name, connection); + + this.indexServerTools(name, connection.tools); + } else { + console.error(`[MCPStore] Failed to connect:`, result.reason); + } + } + + const successCount = this.connections.size; + + if (successCount === 0 && serverEntries.length > 0) { + this.updateState({ + connectedServers: [], + error: 'All MCP server connections failed', + isInitializing: false, + toolCount: 0 + }); + this.initPromise = null; + + return false; + } + + this.updateState({ + connectedServers: Array.from(this.connections.keys()), + error: null, + isInitializing: false, + toolCount: this.toolsIndex.size + }); + this.initPromise = null; + + return true; + } + + /** + * Generates a unique server ID from an optional ID string or index. + */ + private generateServerId(id: unknown, index: number): string { + if (typeof id === 'string' && id.trim()) { + return id.trim(); + } + + return `${MCP_SERVER_ID_PREFIX}-${index + 1}`; + } + + private handleToolsListChanged(serverName: string, tools: Tool[]): void { + const connection = this.connections.get(serverName); + + if (!connection) { + return; + } + + for (const [toolName, ownerServer] of this.toolsIndex.entries()) { + if (ownerServer === serverName) this.toolsIndex.delete(toolName); + } + + connection.tools = tools; + + for (const tool of tools) { + if (this.toolsIndex.has(tool.name)) + console.warn( + `[MCPStore] Tool name conflict after list change: "${tool.name}" exists in "${this.toolsIndex.get(tool.name)}" and "${serverName}". Using tool from "${serverName}".` + ); + + this.toolsIndex.set(tool.name, serverName); + } + this.updateState({ toolCount: this.toolsIndex.size }); + } + + /** + * Registers the tools exposed by a server into the global name->server index, + * warning on conflicts. Shared by connect, reconnect and auto-reconnect. + */ + private indexServerTools(serverName: string, tools: Tool[]): void { + for (const tool of tools) { + if (this.toolsIndex.has(tool.name)) + console.warn( + `[MCPStore] Tool name conflict: "${tool.name}" exists in "${this.toolsIndex.get(tool.name)}" and "${serverName}". Using tool from "${serverName}".` + ); + + this.toolsIndex.set(tool.name, serverName); + } + } + + private async initialize(signature: string, mcpConfig: MCPClientConfig): Promise { + this.updateState({ error: null, isInitializing: true }); + this.configSignature = signature; + + const serverEntries = Object.entries(mcpConfig.servers); + + if (serverEntries.length === 0) { + this.updateState({ connectedServers: [], isInitializing: false, toolCount: 0 }); + + return false; + } + + this.initPromise = this.doInitialize(signature, mcpConfig, serverEntries); + + return this.initPromise; + } + + private parseToolArguments(args: string | Record): Record { + if (typeof args === 'string') { + const trimmed = args.trim(); + + if (trimmed === '') { + return {}; + } + + try { + const parsed = JSON.parse(trimmed); + + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) + throw new Error( + `Tool arguments must be an object, got ${Array.isArray(parsed) ? 'array' : typeof parsed}` + ); + + return parsed as Record; + } catch (error) { + throw new Error(`Failed to parse tool arguments as JSON: ${(error as Error).message}`); + } + } + + if (typeof args === 'object' && args !== null && !Array.isArray(args)) { + return args; + } + + throw new Error(`Invalid tool arguments type: ${typeof args}`); + } + + /** + * Immediately reconnect to a server by creating a fresh transport and session. + * Used when a session-expired error (HTTP 404) is detected during tool execution. + * Per MCP spec 2025-11-25: client MUST discard session ID and re-initialize. + * + * Unlike autoReconnect (which uses exponential backoff for connectivity issues), + * this performs a single immediate reconnection attempt since the server is known + * to be reachable (it responded with 404). + */ + private async reconnectServer(serverName: string): Promise { + const serverConfig = this.serverConfigs.get(serverName); + + if (!serverConfig) { + throw new Error(`[MCPStore] No config found for ${serverName}, cannot reconnect`); + } + + // Disconnect stale connection (clears old transport + session ID) + const oldConnection = this.connections.get(serverName); + + if (oldConnection) { + await MCPService.disconnect(oldConnection).catch(console.warn); + this.connections.delete(serverName); + } + + console.log(`[MCPStore][${serverName}] Session expired, reconnecting with fresh session...`); + + const listChangedHandlers = this.createListChangedHandlers(serverName); + const connection = await MCPService.connect( + serverName, + serverConfig, + DEFAULT_MCP_CONFIG.clientInfo, + DEFAULT_MCP_CONFIG.capabilities, + (phase) => { + if (phase === MCPConnectionPhase.DISCONNECTED) { + console.log(`[MCPStore][${serverName}] Connection lost, starting auto-reconnect`); + this.autoReconnect(serverName); + } + }, + listChangedHandlers + ); + + this.connections.set(serverName, connection); + this.indexServerTools(serverName, connection.tools); + + console.log(`[MCPStore][${serverName}] Session recovered successfully`); + } + + private updateState(state: { + isInitializing?: boolean; + error?: string | null; + toolCount?: number; + connectedServers?: string[]; + }): void { + if (state.isInitializing !== undefined) { + this._isInitializing = state.isInitializing; + } + + if (state.error !== undefined) { + this._error = state.error; + } + + if (state.toolCount !== undefined) { + this._toolCount = state.toolCount; + } + + if (state.connectedServers !== undefined) { + this.connectedServers = state.connectedServers; + } } } diff --git a/tools/ui/src/lib/stores/mcp-resources.svelte.ts b/tools/ui/src/lib/stores/mcp/resources.svelte.ts similarity index 96% rename from tools/ui/src/lib/stores/mcp-resources.svelte.ts rename to tools/ui/src/lib/stores/mcp/resources.svelte.ts index b68def89f5..79ff2c2092 100644 --- a/tools/ui/src/lib/stores/mcp-resources.svelte.ts +++ b/tools/ui/src/lib/stores/mcp/resources.svelte.ts @@ -38,32 +38,40 @@ function generateAttachmentId(): string { } class MCPResourceStore { - private _serverResources = $state>(new SvelteMap()); - private _cachedResources = $state>(new SvelteMap()); - private _subscriptions = $state>(new SvelteMap()); private _attachments = $state([]); + private _cachedResources = $state>(new SvelteMap()); private _isLoading = $state(false); + private _serverResources = $state>(new SvelteMap()); + private _subscriptions = $state>(new SvelteMap()); - get serverResources(): Map { - return this._serverResources; - } - - get cachedResources(): Map { - return this._cachedResources; - } - - get subscriptions(): Map { - return this._subscriptions; + get attachmentCount(): number { + return this._attachments.length; } get attachments(): MCPResourceAttachment[] { return this._attachments; } + get cachedResources(): Map { + return this._cachedResources; + } + + get hasAttachments(): boolean { + return this._attachments.length > 0; + } + get isLoading(): boolean { return this._isLoading; } + get serverResources(): Map { + return this._serverResources; + } + + get subscriptions(): Map { + return this._subscriptions; + } + get totalResourceCount(): number { let count = 0; @@ -84,86 +92,183 @@ class MCPResourceStore { return count; } - get attachmentCount(): number { - return this._attachments.length; - } + /** + * Add a resource attachment to the current chat context + */ + addAttachment(resource: MCPResourceInfo): MCPResourceAttachment { + const attachment: MCPResourceAttachment = { + id: generateAttachmentId(), + loading: true, + resource + }; - get hasAttachments(): boolean { - return this._attachments.length > 0; + this._attachments = [...this._attachments, attachment]; + console.log(`[MCPResources] Added attachment: ${resource.uri}`); + + return attachment; } /** - * - * - * Server Resources Management - * - * + * Register a subscription for a resource */ - - /** - * Set resources for a server (called after listResources) - */ - setServerResources( - serverName: string, - resources: MCPResource[], - templates: MCPResourceTemplate[] - ): void { - this._serverResources.set(serverName, { - error: undefined, - lastFetched: new Date(), - loading: false, - resources, + addSubscription(uri: string, serverName: string): void { + this._subscriptions.set(uri, { serverName, - templates + subscribedAt: new Date(), + uri }); - console.log( - `[MCPResources][${serverName}] Set ${resources.length} resources, ${templates.length} templates` - ); - } - /** - * Set loading state for a server's resources - */ - setServerLoading(serverName: string, loading: boolean): void { - const existing = this._serverResources.get(serverName); + const cached = this._cachedResources.get(uri); - if (existing) { - this._serverResources.set(serverName, { ...existing, loading }); - } else { - this._serverResources.set(serverName, { - error: undefined, - loading, - resources: [], - serverName, - templates: [] - }); + if (cached) { + this._cachedResources.set(uri, { ...cached, subscribed: true }); } + + console.log(`[MCPResources] Added subscription: ${uri}`); } /** - * Set error state for a server's resources + * Cache resource content after reading */ - setServerError(serverName: string, error: string): void { - const existing = this._serverResources.get(serverName); + cacheResourceContent(resource: MCPResourceInfo, content: MCPResourceContent[]): void { + // Enforce cache size limit + if (this._cachedResources.size >= MCP_RESOURCE_CACHE.MAX_ENTRIES) { + const oldestKey = this._cachedResources.keys().next().value; - if (existing) { - this._serverResources.set(serverName, { ...existing, error, loading: false }); - } else { - this._serverResources.set(serverName, { - error, - loading: false, - resources: [], - serverName, - templates: [] - }); + if (oldestKey) { + this._cachedResources.delete(oldestKey); + } } + + this._cachedResources.set(resource.uri, { + content, + fetchedAt: new Date(), + resource, + subscribed: this._subscriptions.has(resource.uri) + }); + console.log(`[MCPResources] Cached content for: ${resource.uri}`); } /** - * Get resources for a specific server + * Clear all state (e.g., on full reset) */ - getServerResources(serverName: string): MCPServerResources | undefined { - return this._serverResources.get(serverName); + clear(): void { + this._serverResources.clear(); + this._cachedResources.clear(); + this._subscriptions.clear(); + this._attachments = []; + this._isLoading = false; + console.log(`[MCPResources] Cleared all state`); + } + + /** + * Clear all attachments + */ + clearAttachments(): void { + this._attachments = []; + console.log(`[MCPResources] Cleared all attachments`); + } + + /** + * Clear all cached content + */ + clearCache(): void { + this._cachedResources.clear(); + console.log(`[MCPResources] Cleared all cached content`); + } + + /** + * Clear resources for a server (e.g., when disconnected) + */ + clearServerResources(serverName: string): void { + this._serverResources.delete(serverName); + + for (const [uri, cached] of this._cachedResources) { + if (cached.resource.serverName === serverName) { + this._cachedResources.delete(uri); + } + } + + for (const [uri, sub] of this._subscriptions) { + if (sub.serverName === serverName) { + this._subscriptions.delete(uri); + } + } + + console.log(`[MCPResources][${serverName}] Cleared all resources`); + } + + /** + * Find resource info by URI across all servers + */ + findResourceByUri(uri: string): MCPResourceInfo | undefined { + const normalizedUri = normalizeResourceUri(uri); + + for (const [serverName, serverRes] of this._serverResources) { + const resource = + serverRes.resources.find((r) => r.uri === uri) ?? + serverRes.resources.find((r) => normalizeResourceUri(r.uri) === normalizedUri); + + if (resource) { + return { + annotations: resource.annotations, + description: resource.description, + icons: resource.icons, + mimeType: resource.mimeType, + name: resource.name, + serverName, + title: resource.title, + uri: resource.uri + }; + } + } + + return undefined; + } + + /** + * Find server name for a resource URI + */ + findServerForUri(uri: string): string | undefined { + for (const [serverName, serverRes] of this._serverResources) { + if (serverRes.resources.some((r) => r.uri === uri)) { + return serverName; + } + } + + return undefined; + } + + /** + * Get resource content as text for chat context + * Formats content for inclusion in LLM prompts + */ + formatAttachmentsForContext(): string { + if (this._attachments.length === 0) return ''; + + const parts: string[] = []; + + for (const attachment of this._attachments) { + if (attachment.error) continue; + + if (!attachment.content || attachment.content.length === 0) continue; + + const resourceName = attachment.resource.title || attachment.resource.name; + const serverName = attachment.resource.serverName; + + for (const content of attachment.content) { + if ('text' in content && content.text) { + parts.push(`\n\n--- Resource: ${resourceName} (from ${serverName}) ---\n${content.text}`); + } else if ('blob' in content && content.blob) { + // For binary content, just note it exists + parts.push( + `\n\n--- Resource: ${resourceName} (from ${serverName}) ---\n[${BINARY_CONTENT_LABEL}: ${content.mimeType || RESOURCE_UNKNOWN_TYPE}]` + ); + } + } + } + + return parts.join(''); } /** @@ -215,57 +320,10 @@ class MCPResourceStore { } /** - * Clear resources for a server (e.g., when disconnected) + * Get attachment by ID */ - clearServerResources(serverName: string): void { - this._serverResources.delete(serverName); - - // Also clear cached content for this server's resources - for (const [uri, cached] of this._cachedResources) { - if (cached.resource.serverName === serverName) { - this._cachedResources.delete(uri); - } - } - - // Clear subscriptions for this server - for (const [uri, sub] of this._subscriptions) { - if (sub.serverName === serverName) { - this._subscriptions.delete(uri); - } - } - - console.log(`[MCPResources][${serverName}] Cleared all resources`); - } - - /** - * - * - * Resource Content Caching - * - * - */ - - /** - * Cache resource content after reading - */ - cacheResourceContent(resource: MCPResourceInfo, content: MCPResourceContent[]): void { - // Enforce cache size limit - if (this._cachedResources.size >= MCP_RESOURCE_CACHE.MAX_ENTRIES) { - // Remove oldest entry - const oldestKey = this._cachedResources.keys().next().value; - - if (oldestKey) { - this._cachedResources.delete(oldestKey); - } - } - - this._cachedResources.set(resource.uri, { - content, - fetchedAt: new Date(), - resource, - subscribed: this._subscriptions.has(resource.uri) - }); - console.log(`[MCPResources] Cached content for: ${resource.uri}`); + getAttachment(attachmentId: string): MCPResourceAttachment | undefined { + return this._attachments.find((att) => att.id === attachmentId); } /** @@ -276,7 +334,6 @@ class MCPResourceStore { if (!cached) return undefined; - // Check if cache is still valid const age = Date.now() - cached.fetchedAt.getTime(); if (age > MCP_RESOURCE_CACHE.TTL_MS && !cached.subscribed) { @@ -290,100 +347,22 @@ class MCPResourceStore { } /** - * Invalidate cached content for a resource (e.g., on update notification) + * Get resources for a specific server */ - invalidateCache(uri: string): void { - this._cachedResources.delete(uri); - console.log(`[MCPResources] Invalidated cache for: ${uri}`); - } - - /** - * Clear all cached content - */ - clearCache(): void { - this._cachedResources.clear(); - console.log(`[MCPResources] Cleared all cached content`); - } - - /** - * - * - * Subscriptions - * - * - */ - - /** - * Register a subscription for a resource - */ - addSubscription(uri: string, serverName: string): void { - this._subscriptions.set(uri, { - serverName, - subscribedAt: new Date(), - uri - }); - - // Update cached resource if exists - const cached = this._cachedResources.get(uri); - - if (cached) { - this._cachedResources.set(uri, { ...cached, subscribed: true }); - } - - console.log(`[MCPResources] Added subscription: ${uri}`); - } - - /** - * Remove a subscription for a resource - */ - removeSubscription(uri: string): void { - this._subscriptions.delete(uri); - - // Update cached resource if exists - const cached = this._cachedResources.get(uri); - - if (cached) { - this._cachedResources.set(uri, { ...cached, subscribed: false }); - } - - console.log(`[MCPResources] Removed subscription: ${uri}`); - } - - /** - * Check if a resource is subscribed - */ - isSubscribed(uri: string): boolean { - return this._subscriptions.has(uri); - } - - /** - * Handle resource update notification - */ - handleResourceUpdate(uri: string): void { - // Invalidate cache so next read gets fresh content - this.invalidateCache(uri); - - // Update subscription last update time - const sub = this._subscriptions.get(uri); - - if (sub) { - this._subscriptions.set(uri, { ...sub, lastUpdate: new Date() }); - } - - console.log(`[MCPResources] Resource updated: ${uri}`); + getServerResources(serverName: string): MCPServerResources | undefined { + return this._serverResources.get(serverName); } /** * Handle resources list changed notification */ handleResourcesListChanged(serverName: string): void { - // Mark server resources as needing refresh const existing = this._serverResources.get(serverName); if (existing) { this._serverResources.set(serverName, { ...existing, - lastFetched: undefined // Mark as stale + lastFetched: undefined }); } @@ -399,60 +378,27 @@ class MCPResourceStore { */ /** - * Add a resource attachment to the current chat context + * Handle resource update notification */ - addAttachment(resource: MCPResourceInfo): MCPResourceAttachment { - const attachment: MCPResourceAttachment = { - id: generateAttachmentId(), - loading: true, - resource - }; + handleResourceUpdate(uri: string): void { + // Invalidate cache so next read gets fresh content + this.invalidateCache(uri); - this._attachments = [...this._attachments, attachment]; - console.log(`[MCPResources] Added attachment: ${resource.uri}`); + const sub = this._subscriptions.get(uri); - return attachment; + if (sub) { + this._subscriptions.set(uri, { ...sub, lastUpdate: new Date() }); + } + + console.log(`[MCPResources] Resource updated: ${uri}`); } /** - * Update attachment with fetched content + * Invalidate cached content for a resource (e.g., on update notification) */ - updateAttachmentContent(attachmentId: string, content: MCPResourceContent[]): void { - this._attachments = this._attachments.map((att) => - att.id === attachmentId ? { ...att, content, error: undefined, loading: false } : att - ); - } - - /** - * Update attachment with error - */ - updateAttachmentError(attachmentId: string, error: string): void { - this._attachments = this._attachments.map((att) => - att.id === attachmentId ? { ...att, error, loading: false } : att - ); - } - - /** - * Remove an attachment - */ - removeAttachment(attachmentId: string): void { - this._attachments = this._attachments.filter((att) => att.id !== attachmentId); - console.log(`[MCPResources] Removed attachment: ${attachmentId}`); - } - - /** - * Clear all attachments - */ - clearAttachments(): void { - this._attachments = []; - console.log(`[MCPResources] Cleared all attachments`); - } - - /** - * Get attachment by ID - */ - getAttachment(attachmentId: string): MCPResourceAttachment | undefined { - return this._attachments.find((att) => att.id === attachmentId); + invalidateCache(uri: string): void { + this._cachedResources.delete(uri); + console.log(`[MCPResources] Invalidated cache for: ${uri}`); } /** @@ -467,12 +413,34 @@ class MCPResourceStore { } /** - * - * - * Utility Methods - * - * + * Check if a resource is subscribed */ + isSubscribed(uri: string): boolean { + return this._subscriptions.has(uri); + } + + /** + * Remove an attachment + */ + removeAttachment(attachmentId: string): void { + this._attachments = this._attachments.filter((att) => att.id !== attachmentId); + console.log(`[MCPResources] Removed attachment: ${attachmentId}`); + } + + /** + * Remove a subscription for a resource + */ + removeSubscription(uri: string): void { + this._subscriptions.delete(uri); + + const cached = this._cachedResources.get(uri); + + if (cached) { + this._cachedResources.set(uri, { ...cached, subscribed: false }); + } + + console.log(`[MCPResources] Removed subscription: ${uri}`); + } /** * Set global loading state @@ -482,88 +450,62 @@ class MCPResourceStore { } /** - * Find resource info by URI across all servers + * Set error state for a server's resources */ - findResourceByUri(uri: string): MCPResourceInfo | undefined { - const normalizedUri = normalizeResourceUri(uri); + setServerError(serverName: string, error: string): void { + const existing = this._serverResources.get(serverName); - for (const [serverName, serverRes] of this._serverResources) { - const resource = - serverRes.resources.find((r) => r.uri === uri) ?? - serverRes.resources.find((r) => normalizeResourceUri(r.uri) === normalizedUri); - - if (resource) { - return { - annotations: resource.annotations, - description: resource.description, - icons: resource.icons, - mimeType: resource.mimeType, - name: resource.name, - serverName, - title: resource.title, - uri: resource.uri - }; - } + if (existing) { + this._serverResources.set(serverName, { ...existing, error, loading: false }); + } else { + this._serverResources.set(serverName, { + error, + loading: false, + resources: [], + serverName, + templates: [] + }); } - - return undefined; } /** - * Find server name for a resource URI + * Set loading state for a server's resources */ - findServerForUri(uri: string): string | undefined { - for (const [serverName, serverRes] of this._serverResources) { - if (serverRes.resources.some((r) => r.uri === uri)) { - return serverName; - } - } + setServerLoading(serverName: string, loading: boolean): void { + const existing = this._serverResources.get(serverName); - return undefined; + if (existing) { + this._serverResources.set(serverName, { ...existing, loading }); + } else { + this._serverResources.set(serverName, { + error: undefined, + loading, + resources: [], + serverName, + templates: [] + }); + } } /** - * Clear all state (e.g., on full reset) + * Set resources for a server (called after listResources) */ - clear(): void { - this._serverResources.clear(); - this._cachedResources.clear(); - this._subscriptions.clear(); - this._attachments = []; - this._isLoading = false; - console.log(`[MCPResources] Cleared all state`); - } - - /** - * Get resource content as text for chat context - * Formats content for inclusion in LLM prompts - */ - formatAttachmentsForContext(): string { - if (this._attachments.length === 0) return ''; - - const parts: string[] = []; - - for (const attachment of this._attachments) { - if (attachment.error) continue; - - if (!attachment.content || attachment.content.length === 0) continue; - - const resourceName = attachment.resource.title || attachment.resource.name; - const serverName = attachment.resource.serverName; - - for (const content of attachment.content) { - if ('text' in content && content.text) { - parts.push(`\n\n--- Resource: ${resourceName} (from ${serverName}) ---\n${content.text}`); - } else if ('blob' in content && content.blob) { - // For binary content, just note it exists - parts.push( - `\n\n--- Resource: ${resourceName} (from ${serverName}) ---\n[${BINARY_CONTENT_LABEL}: ${content.mimeType || RESOURCE_UNKNOWN_TYPE}]` - ); - } - } - } - - return parts.join(''); + setServerResources( + serverName: string, + resources: MCPResource[], + templates: MCPResourceTemplate[] + ): void { + this._serverResources.set(serverName, { + error: undefined, + lastFetched: new Date(), + loading: false, + resources, + serverName, + templates + }); + console.log( + `[MCPResources][${serverName}] Set ${resources.length} resources, ${templates.length} templates` + ); } /** @@ -605,6 +547,24 @@ class MCPResourceStore { return extras; } + + /** + * Update attachment with fetched content + */ + updateAttachmentContent(attachmentId: string, content: MCPResourceContent[]): void { + this._attachments = this._attachments.map((att) => + att.id === attachmentId ? { ...att, content, error: undefined, loading: false } : att + ); + } + + /** + * Update attachment with error + */ + updateAttachmentError(attachmentId: string, error: string): void { + this._attachments = this._attachments.map((att) => + att.id === attachmentId ? { ...att, error, loading: false } : att + ); + } } export const mcpResourceStore = new MCPResourceStore(); diff --git a/tools/ui/src/lib/stores/models.svelte.ts b/tools/ui/src/lib/stores/models.svelte.ts deleted file mode 100644 index c741d144c4..0000000000 --- a/tools/ui/src/lib/stores/models.svelte.ts +++ /dev/null @@ -1,1077 +0,0 @@ -import { FAVORITE_MODELS_LOCALSTORAGE_KEY, MODEL_PROPS_CACHE } from '$lib/constants'; -import { - FileTypeCategory, - ModelModality, - ServerModelsSseEventType, - ServerModelStatus -} from '$lib/enums'; -import { ModelsService } from '$lib/services/models.service'; -import { PropsService } from '$lib/services/props.service'; -// direct imports between stores, not via the barrel, to avoid circular deps -import { conversationsStore } from '$lib/stores/conversations.svelte'; -import { serverStore } from '$lib/stores/server.svelte'; -// deep imports, not the '$lib/utils' barrel: it re-exports modules that reach back -// into the stores, and going through it here would read a half-built module -import { TTLCache } from '$lib/utils/cache-ttl'; -import { - detectThinkingSupport, - detectThinkingSupportWithReason -} from '$lib/utils/chat-template-thinking-detector'; -import { getConversationModel } from '$lib/utils/conversation-utils'; -import { SvelteMap, SvelteSet } from 'svelte/reactivity'; -import { toast } from 'svelte-sonner'; - -/** - * modelsStore - Reactive store for model management in both MODEL and ROUTER modes. - * - * **Architecture & Relationships:** - * - **ModelsService**: Stateless service for model API communication - * - **PropsService**: Stateless service for props/modalities fetching - * - **modelsStore** (this class): Reactive store for model state - * - **conversationsStore**: Tracks which conversations use which models - * - * **API Inconsistency Workaround:** - * In MODEL mode, `/props` returns modalities for the single model. - * In ROUTER mode, `/props` has no modalities — must use `/props?model=` per model. - * This store normalizes this behavior so consumers don't need to know the server mode. - */ -class ModelsStore { - /** - * - * - * State - * - * - */ - - models = $state([]); - routerModels = $state([]); - loading = $state(false); - updating = $state(false); - error = $state(null); - selectedModelId = $state(null); - selectedModelName = $state(null); - - // Dedup concurrent fetch() callers — all awaiters share the same inflight promise. - // Without this, ?model= URL handler races an in-progress fetch and sees an empty list. - private inflightFetch: Promise | null = null; - - private modelUsage = $state>>(new Map()); - private modelLoadingStates = new SvelteMap(); - - // /models/sse feed state, the single source of truth for status and load progress - private statusAbort: AbortController | null = null; - private statusReaderActive = false; - private loadProgress = new SvelteMap(); - private statusWaiters = new Map< - string, - { target: ServerModelStatus; resolve: () => void; reject: (e: Error) => void } - >(); - - favoriteModelIds = $state>(this.loadFavoritesFromStorage()); - - /** - * Model-specific props cache with TTL. - * Key: modelId, Value: props data including modalities. - * TTL: 10 minutes — props don't change frequently. - */ - private modelPropsCache = new TTLCache({ - maxEntries: MODEL_PROPS_CACHE.MAX_ENTRIES, - ttlMs: MODEL_PROPS_CACHE.TTL_MS - }); - private modelPropsFetching = $state>(new Set()); - - /** - * Version counter for props cache — used to trigger reactivity when props are updated. - */ - propsCacheVersion = $state(0); - - /** - * - * - * Computed Getters - * - * - */ - - get selectedModel(): ModelOption | null { - if (!this.selectedModelId) return null; - - return this.models.find((m) => m.id === this.selectedModelId) ?? null; - } - - get loadedModelIds(): string[] { - return this.routerModels - .filter( - (m) => - m.status.value === ServerModelStatus.LOADED || - m.status.value === ServerModelStatus.SLEEPING - ) - .map((m) => m.id); - } - - get loadingModelIds(): string[] { - return Array.from(this.modelLoadingStates.entries()) - .filter(([, loading]) => loading) - .map(([id]) => id); - } - - /** - * Get model name in MODEL mode (single model). - * Extracts from model_path or model_alias from server props. - * In ROUTER mode, returns null (model is per-conversation). - */ - get singleModelName(): string | null { - if (serverStore.isRouterMode) return null; - - const props = serverStore.props; - - if (props?.model_alias) return props.model_alias; - - if (!props?.model_path) return null; - - return props.model_path.split(/(\\|\/)/).pop() || null; - } - - /** - * Model the active conversation view resolves to. Router mode: the user's - * selection first, then the conversation's own model. Otherwise the single - * served model, from the models list or the server props as a fallback. - */ - get activeModelId(): string | null { - if (!serverStore.isRouterMode) { - return this.models.length > 0 ? this.models[0].model : this.singleModelName; - } - - if (this.selectedModelId) { - const selected = this.models.find((m) => m.id === this.selectedModelId); - - if (selected) return selected.model; - } - - const conversationModel = getConversationModel(conversationsStore.activeMessages); - - if (conversationModel) { - const model = this.models.find((m) => m.model === conversationModel); - - if (model) return model.model; - } - - return null; - } - - get selectedModelContextSize(): number | null { - if (!this.selectedModelName) return null; - - return this.getModelContextSize(this.selectedModelName); - } - - /** - * - * - * Modalities - * - * - */ - - getModelModalities(modelId: string): ModelModalities | null { - if (!serverStore.isRouterMode && serverStore.props?.modalities) { - return this.buildModalities(serverStore.props.modalities); - } - - const model = this.models.find((m) => m.model === modelId || m.id === modelId); - - if (model?.modalities) { - return model.modalities; - } - - const props = this.modelPropsCache.get(modelId); - - if (props?.modalities) { - return this.buildModalities(props.modalities); - } - - return null; - } - - modelSupportsVision(modelId: string): boolean { - return this.getModelModalities(modelId)?.vision ?? false; - } - - modelSupportsAudio(modelId: string): boolean { - return this.getModelModalities(modelId)?.audio ?? false; - } - - modelSupportsVideo(modelId: string): boolean { - return this.getModelModalities(modelId)?.video ?? false; - } - - getModelModalitiesArray(modelId: string): ModelModality[] { - const modalities = this.getModelModalities(modelId); - - if (!modalities) return []; - - const result: ModelModality[] = []; - - if (modalities.vision) result.push(ModelModality.VISION); - - if (modalities.audio) result.push(ModelModality.AUDIO); - - if (modalities.video) result.push(ModelModality.VIDEO); - - return result; - } - - getModelProps(modelId: string): ApiLlamaCppServerProps | null { - return this.modelPropsCache.get(modelId); - } - - getModelContextSize(modelId: string): number | null { - const props = this.getModelProps(modelId); - const nCtx = props?.default_generation_settings?.n_ctx; - - return typeof nCtx === 'number' ? nCtx : null; - } - - isModelPropsFetching(modelId: string): boolean { - return this.modelPropsFetching.has(modelId); - } - - /** - * - * - * Status Queries - * - * - */ - - isModelLoaded(modelId: string): boolean { - const model = this.routerModels.find((m) => m.id === modelId); - - return ( - model?.status.value === ServerModelStatus.LOADED || - model?.status.value === ServerModelStatus.SLEEPING - ); - } - - isModelOperationInProgress(modelId: string): boolean { - return this.modelLoadingStates.get(modelId) ?? false; - } - - getModelStatus(modelId: string): ServerModelStatus | null { - const model = this.routerModels.find((m) => m.id === modelId); - - return model?.status.value ?? null; - } - - getModelUsage(modelId: string): SvelteSet { - return this.modelUsage.get(modelId) ?? new SvelteSet(); - } - - isModelInUse(modelId: string): boolean { - const usage = this.modelUsage.get(modelId); - - return usage !== undefined && usage.size > 0; - } - // - // Thinking Support Detection - // - - /** - * Whether the selected model's chat template supports thinking/reasoning. - * Uses heuristic detection on the model's chat_template from /props. - * - * - MODEL mode: the global /props already describes the single loaded model, - * so its chat_template is used directly and no per-model cache is involved - * - ROUTER mode: fetches /props?model= for the selected model (cached), - * triggering an async fetch if not yet cached - */ - get supportsThinking(): boolean { - if (!serverStore.isRouterMode) { - return detectThinkingSupport(serverStore.props?.chat_template ?? ''); - } - - const modelId = this.selectedModelName; - - if (!modelId) return false; - - if (!this.modelPropsCache.get(modelId)) { - this.fetchModelProps(modelId); - } - - const props = this.getModelProps(modelId); - - return detectThinkingSupport(props?.chat_template ?? ''); - } - - /** - * Check if a specific model supports thinking. - * In MODEL mode the global /props describes the single loaded model. - * In ROUTER mode, fetches model props if not cached. - */ - checkModelSupportsThinking(modelId: string): boolean { - if (!serverStore.isRouterMode) { - return detectThinkingSupport(serverStore.props?.chat_template ?? ''); - } - - if (!modelId) return false; - - if (!this.modelPropsCache.get(modelId)) { - this.fetchModelProps(modelId); - } - - const props = this.getModelProps(modelId); - - return detectThinkingSupport(props?.chat_template ?? ''); - } - - /** - * Detailed thinking support detection result with reason for debugging/UI. - */ - get thinkingSupportDetails(): { supported: boolean; reason: string } { - if (!serverStore.isRouterMode) { - return detectThinkingSupportWithReason(serverStore.props?.chat_template ?? ''); - } - - const modelId = this.selectedModelName; - - if (!modelId) { - return { reason: 'No model selected', supported: false }; - } - - if (!this.modelPropsCache.get(modelId)) { - this.fetchModelProps(modelId); - } - - const props = this.getModelProps(modelId); - - return detectThinkingSupportWithReason(props?.chat_template ?? ''); - } - - /** - * - * - * Data Fetching - * - * - */ - - /** - * Fetch list of models from server and detect server role. - * Also fetches modalities for MODEL mode (single model). - */ - async fetch(force = false): Promise { - if (this.inflightFetch) return this.inflightFetch; - - if (this.models.length > 0 && !force) return; - - this.inflightFetch = this.runFetch(); - try { - await this.inflightFetch; - } finally { - this.inflightFetch = null; - } - } - - private async runFetch(): Promise { - this.loading = true; - this.error = null; - - try { - if (!serverStore.props) { - await serverStore.fetch(); - } - - const router = serverStore.isRouterMode; - - if (router) { - const response = await ModelsService.listRouter(); - - this.routerModels = response.data; - this.models = this.buildModelOptions(response); - - await this.fetchModalitiesForLoadedModels(); - - const visible = this.getVisibleModels(); - - if (visible.length === 1 && this.isModelLoaded(visible[0].model)) { - this.selectModelById(visible[0].id); - } - } else { - this.models = await this.fetchModelModeInternal(); - } - } catch (error) { - this.models = []; - this.error = error instanceof Error ? error.message : 'Failed to load models'; - - throw error; - } finally { - this.loading = false; - } - } - - /** Fetch models in MODEL mode (single model, standard OpenAI-compatible). */ - private async fetchModelModeInternal(): Promise { - const response = await ModelsService.list(); - - return this.buildModelOptions(response); - } - - /** - * Build ModelOption[] from an API response. - * Both MODEL and ROUTER modes share the same mapping logic; - * they differ only in which endpoint is called. - */ - private buildModelOptions( - response: ApiModelListResponse | ApiRouterModelsListResponse - ): ModelOption[] { - return response.data.map((item: ApiModelDataEntry, index: number) => { - const details = response.models?.[index]; - const rawCapabilities = Array.isArray(details?.capabilities) ? details?.capabilities : []; - const displayNameSource = - details?.name && details.name.trim().length > 0 ? details.name : item.id; - const modelId = details?.model || item.id; - - return { - aliases: item.aliases ?? [], - capabilities: rawCapabilities.filter((value: unknown): value is string => Boolean(value)), - description: details?.description, - details: details?.details, - id: item.id, - meta: item.meta ?? null, - modalities: this.buildArchitectureModalities(item.architecture), - model: modelId, - name: this.toDisplayName(displayNameSource), - parsedId: ModelsService.parseModelId(modelId), - tags: item.tags ?? [] - }; - }); - } - - /** - * Fetch router models with full metadata (ROUTER mode only). - * No-op in router mode — fetch() already calls listRouter() internally. - * Kept for API compatibility (e.g. handleOpenChange dropdown open handler). - */ - async fetchRouterModels(): Promise { - if (!serverStore.isRouterMode) return; - - try { - const response = await ModelsService.listRouter(); - - this.routerModels = response.data; - await this.fetchModalitiesForLoadedModels(); - - const visible = this.getVisibleModels(); - - if (visible.length === 1 && this.isModelLoaded(visible[0].model)) { - this.selectModelById(visible[0].id); - } - } catch (error) { - console.warn('Failed to fetch router models:', error); - this.routerModels = []; - } - } - - /** - * Fetch props for a specific model from /props endpoint. - * Uses caching to avoid redundant requests. - * - * In ROUTER mode, this only fetches props if the model is loaded, - * since unloaded models return 400 from /props endpoint. - * - * @param modelId - Model identifier to fetch props for - * @returns Props data or null if fetch failed or model not loaded - */ - async fetchModelProps(modelId: string): Promise { - const cached = this.modelPropsCache.get(modelId); - - if (cached) return cached; - - if (serverStore.isRouterMode && !this.isModelLoaded(modelId)) { - return null; - } - - if (this.modelPropsFetching.has(modelId)) return null; - - this.modelPropsFetching.add(modelId); - - try { - const props = await PropsService.fetchForModel(modelId); - - this.modelPropsCache.set(modelId, props); - this.propsCacheVersion++; - - return props; - } catch (error) { - console.warn(`Failed to fetch props for model ${modelId}:`, error); - - return null; - } finally { - this.modelPropsFetching.delete(modelId); - } - } - - /** Fetch modalities for all loaded models from /props endpoint. */ - async fetchModalitiesForLoadedModels(): Promise { - const loadedModelIds = this.loadedModelIds; - - if (loadedModelIds.length === 0) return; - - const propsPromises = loadedModelIds.map((modelId) => this.fetchModelProps(modelId)); - - try { - const results = await Promise.all(propsPromises); - - this.models = this.models.map((model) => { - const modelIndex = loadedModelIds.indexOf(model.model); - - if (modelIndex === -1) return model; - - const props = results[modelIndex]; - - if (!props?.modalities) return model; - - return { ...model, modalities: this.buildModalities(props.modalities) }; - }); - - this.propsCacheVersion++; - } catch (error) { - console.warn('Failed to fetch modalities for loaded models:', error); - } - } - - /** - * Update modalities for a specific model. - * Called when a model is loaded or when we need fresh modality data. - */ - async updateModelModalities(modelId: string): Promise { - const props = await this.fetchModelProps(modelId); - - if (!props?.modalities) return; - - this.models = this.models.map((model) => - model.model === modelId - ? { ...model, modalities: this.buildModalities(props.modalities!) } - : model - ); - - this.propsCacheVersion++; - } - - /** - * Filter to models visible in the UI (ui !== false). - */ - private getVisibleModels(): ModelOption[] { - return this.models.filter((option) => this.getModelProps(option.model)?.ui !== false); - } - - /** - * Gets the model name from the last assistant message in the active conversation. - * Used by both the chat page and settings page to maintain model consistency. - */ - getModelFromLastAssistantResponse(): string | null { - const messages = conversationsStore.activeMessages; - - if (!messages || messages.length === 0) return null; - - for (let i = messages.length - 1; i >= 0; i--) { - if (messages[i].model) { - return messages[i].model; - } - } - - return null; - } - - /** - * Auto-selects the model from the last assistant response if available and loaded. - * Returns true if a model was selected, false otherwise. - */ - async selectModelFromLastAssistantResponse(): Promise { - const lastModel = this.getModelFromLastAssistantResponse(); - - if (!lastModel || this.selectedModelName === lastModel) return false; - - const matchingModel = this.models.find((option) => option.model === lastModel); - - if (!matchingModel || !this.isModelLoaded(lastModel)) return false; - - try { - await this.selectModelById(matchingModel.id); - console.log(`[modelsStore] Automatically selected model: ${lastModel} from last message`); - - return true; - } catch (error) { - console.warn('[modelsStore] Failed to automatically select model from last message:', error); - - return false; - } - } - - /** - * Auto-selects the first available model if none is selected. - * Prioritizes: - * 1. Model from active conversation's last assistant response (if loaded) - * 2. Model from active conversation's last assistant response (if not loaded) - * 3. First loaded model (not from active conversation) - * 4. A favorite model - * 5. First available model - */ - async ensureFirstModelSelected(): Promise { - if (this.selectedModelName) return; - - const availableModels = this.getVisibleModels(); - - if (availableModels.length === 0) return; - - // Try to select model from last assistant response first - const lastModel = this.getModelFromLastAssistantResponse(); - - if (lastModel) { - const lastModelOption = availableModels.find((m) => m.model === lastModel); - - if (lastModelOption) { - await this.selectModelById(lastModelOption.id); - - if (this.isModelLoaded(lastModel)) { - await this.fetchModelProps(lastModel); - } - - return; - } - } - - // Try a loaded model first - const loadedModel = availableModels.find((m) => this.isModelLoaded(m.model)); - - if (loadedModel) { - await this.selectModelById(loadedModel.id); - await this.fetchModelProps(loadedModel.model); - - return; - } - - // Try loading a favorite model - const favorite = this.favoriteModelIds.values().next()?.value; - - if (favorite) { - await this.selectModelById(favorite); - - return; - } - - // Fall back to the first available model - await this.selectModelById(availableModels[0].id); - } - - /** - * - * - * Model Selection - * - * - */ - - async selectModelById(modelId: string): Promise { - if (!modelId || this.updating) return; - - if (this.selectedModelId === modelId) return; - - const option = this.models.find((model) => model.id === modelId); - - if (!option) throw new Error('Selected model is not available'); - - this.updating = true; - this.error = null; - - try { - this.selectedModelId = option.id; - this.selectedModelName = option.model; - } finally { - this.updating = false; - } - } - - /** - * Select a model by its model name (used for syncing with conversation model). - */ - selectModelByName(modelName: string): void { - const option = this.models.find((model) => model.model === modelName); - - if (option) { - this.selectedModelId = option.id; - this.selectedModelName = option.model; - } - } - - clearSelection(): void { - this.selectedModelId = null; - this.selectedModelName = null; - } - - findModelByName(modelName: string): ModelOption | null { - return ( - this.models.find( - (model) => - model.model === modelName || model.id === modelName || model.aliases?.includes(modelName) - ) ?? null - ); - } - - findModelById(modelId: string): ModelOption | null { - return this.models.find((model) => model.id === modelId) ?? null; - } - - hasModel(modelName: string): boolean { - return this.models.some((model) => model.model === modelName); - } - - /** - * - * - * Loading / Unloading Models - * - * - */ - - // reconnect delay after the feed drops or the server is not ready yet - /** - * Open the /models/sse feed and keep it live with auto reconnect. - * Idempotent and router mode only. The feed drives status and progress, - * so it replaces any post-operation polling. - */ - subscribeStatus(): void { - if (this.statusReaderActive) return; - - if (!serverStore.isRouterMode) return; - - this.statusReaderActive = true; - this.statusAbort = new AbortController(); - void this.runStatusReader(this.statusAbort.signal); - } - - /** - * Close the /models/sse feed and drop transient progress. - */ - unsubscribeStatus(): void { - this.statusReaderActive = false; - this.statusAbort?.abort(); - this.statusAbort = null; - this.loadProgress.clear(); - } - - /** - * Current load progress for a model, or null when not loading. - */ - getLoadProgress(modelId: string): ModelLoadProgress | null { - return this.loadProgress.get(modelId) ?? null; - } - - /** - * Read the feed and reconnect until unsubscribed. - */ - private async runStatusReader(signal: AbortSignal): Promise { - await ModelsService.watchModelEvents(signal, (event) => this.applyStatusEvent(event)); - } - - /** - * Route one feed record by event kind. Only the status_* events carry a - * status payload, models_reload triggers a list refresh, model_remove drops - * the row, download_* belong to the download surface, not here. - */ - private applyStatusEvent(event: ApiModelsSseEvent): void { - switch (event.event) { - case ServerModelsSseEventType.STATUS_CHANGE: - case ServerModelsSseEventType.MODEL_STATUS: - case ServerModelsSseEventType.STATUS_UPDATE: - this.applyModelStatus(event); - - break; - case ServerModelsSseEventType.MODELS_RELOAD: - void this.fetchRouterModels(); - - break; - case ServerModelsSseEventType.MODEL_REMOVE: - this.removeRouterModel(event.model); - - break; - case ServerModelsSseEventType.DOWNLOAD_PROGRESS: - break; - } - } - - /** - * Apply a status envelope: update the model row, track or clear progress, - * settle any pending load or unload awaiter. - */ - private applyModelStatus(event: ApiModelsSseEvent): void { - const model = event.model; - const data = event.data; - - if (!model || !data?.status) return; - - const status = data.status; - - this.setRouterModelStatus(model, status); - - if (status === ServerModelStatus.LOADING) { - if (data.progress) this.loadProgress.set(model, data.progress); - } else { - this.loadProgress.delete(model); - } - - if (status === ServerModelStatus.LOADED) { - void this.updateModelModalities(model); - } - - const failed = - status === ServerModelStatus.FAILED || - (status === ServerModelStatus.UNLOADED && (data.exit_code ?? 0) !== 0); - - if (failed) { - this.rejectStatus(model, new Error(`Model failed: ${this.toDisplayName(model)}`)); - - return; - } - - this.settleStatus(model, status); - } - - /** - * Drop a model row reported gone by the feed and settle its awaiters. - */ - private removeRouterModel(modelId: string): void { - if (this.routerModels.findIndex((m) => m.id === modelId) === -1) return; - - this.routerModels = this.routerModels.filter((m) => m.id !== modelId); - this.loadProgress.delete(modelId); - this.rejectStatus(modelId, new Error(`Model removed: ${this.toDisplayName(modelId)}`)); - } - - /** - * Update one model row status in place, reassigning to trigger reactivity. - */ - private setRouterModelStatus(modelId: string, status: ServerModelStatus): void { - const idx = this.routerModels.findIndex((m) => m.id === modelId); - - if (idx === -1) return; - - const current = this.routerModels[idx]; - - if (current.status.value === status) return; - - const next = [...this.routerModels]; - - next[idx] = { ...current, status: { ...current.status, value: status } }; - this.routerModels = next; - } - - /** - * Register an awaiter that resolves when the feed reports target status. - * One operation runs per model at a time, so one awaiter per model is kept. - */ - private waitForStatus(modelId: string, target: ServerModelStatus): Promise { - return new Promise((resolve, reject) => { - this.statusWaiters.set(modelId, { reject, resolve, target }); - }); - } - - /** - * Resolve and drop the awaiter when the model reaches its target status. - */ - private settleStatus(modelId: string, status: ServerModelStatus): void { - const waiter = this.statusWaiters.get(modelId); - - if (waiter && waiter.target === status) { - this.statusWaiters.delete(modelId); - waiter.resolve(); - } - } - - /** - * Reject and drop the awaiter for a model. - */ - private rejectStatus(modelId: string, error: Error): void { - const waiter = this.statusWaiters.get(modelId); - - if (waiter) { - this.statusWaiters.delete(modelId); - waiter.reject(error); - } - } - - async loadModel(modelId: string): Promise { - if (this.isModelLoaded(modelId)) return; - - if (this.modelLoadingStates.get(modelId)) return; - - this.modelLoadingStates.set(modelId, true); - this.error = null; - - // the feed drives completion, so it must be live before the request - this.subscribeStatus(); - - const reachedLoaded = this.waitForStatus(modelId, ServerModelStatus.LOADED); - - reachedLoaded.catch(() => {}); - - try { - await ModelsService.load(modelId); - await reachedLoaded; - toast.success(`Model loaded: ${this.toDisplayName(modelId)}`); - } catch (error) { - this.rejectStatus(modelId, error instanceof Error ? error : new Error('load failed')); - this.error = error instanceof Error ? error.message : 'Failed to load model'; - toast.error(`Failed to load model: ${this.toDisplayName(modelId)}`); - - throw error; - } finally { - this.modelLoadingStates.set(modelId, false); - } - } - - async unloadModel(modelId: string): Promise { - if (!this.isModelLoaded(modelId)) return; - - if (this.modelLoadingStates.get(modelId)) return; - - this.modelLoadingStates.set(modelId, true); - this.error = null; - - this.subscribeStatus(); - - const reachedUnloaded = this.waitForStatus(modelId, ServerModelStatus.UNLOADED); - - reachedUnloaded.catch(() => {}); - - try { - await ModelsService.unload(modelId); - await reachedUnloaded; - toast.info(`Model unloaded: ${this.toDisplayName(modelId)}`); - } catch (error) { - this.rejectStatus(modelId, error instanceof Error ? error : new Error('unload failed')); - this.error = error instanceof Error ? error.message : 'Failed to unload model'; - toast.error(`Failed to unload model: ${this.toDisplayName(modelId)}`); - - throw error; - } finally { - this.modelLoadingStates.set(modelId, false); - } - } - - async ensureModelLoaded(modelId: string): Promise { - if (this.isModelLoaded(modelId)) return; - - await this.loadModel(modelId); - } - - /** - * - * - * Favorites - * - * - */ - - isFavorite(modelId: string): boolean { - return this.favoriteModelIds.has(modelId); - } - - toggleFavorite(modelId: string): void { - const next = new SvelteSet(this.favoriteModelIds); - - if (next.has(modelId)) { - next.delete(modelId); - } else { - next.add(modelId); - } - - this.favoriteModelIds = next; - - try { - localStorage.setItem(FAVORITE_MODELS_LOCALSTORAGE_KEY, JSON.stringify([...next])); - } catch { - toast.error('Failed to save favorite models to local storage'); - } - } - - private loadFavoritesFromStorage(): Set { - try { - const raw = localStorage.getItem(FAVORITE_MODELS_LOCALSTORAGE_KEY); - - return raw ? new Set(JSON.parse(raw) as string[]) : new Set(); - } catch { - toast.error('Failed to load favorite models from local storage'); - - return new Set(); - } - } - - /** - * - * - * Utilities - * - * - */ - - private toDisplayName(id: string): string { - const segments = id.split(/\\|\//); - const candidate = segments.pop(); - - return candidate && candidate.trim().length > 0 ? candidate : id; - } - - private buildModalities( - modalities: NonNullable - ): ModelModalities { - return { - audio: modalities.audio ?? false, - video: modalities.video ?? false, - vision: modalities.vision ?? false - }; - } - - /** Map the router modalities, the only source available while a model is not loaded. */ - private buildArchitectureModalities( - architecture: ApiModelDataEntry['architecture'] - ): ModelModalities | undefined { - if (!architecture) return undefined; - - const inputs = architecture.input_modalities; - - return { - audio: inputs.includes(FileTypeCategory.AUDIO), - video: inputs.includes(FileTypeCategory.VIDEO), - vision: inputs.includes(FileTypeCategory.IMAGE) - }; - } - - clear(): void { - this.unsubscribeStatus(); - this.statusWaiters.forEach((waiter) => waiter.reject(new Error('Models store cleared'))); - this.statusWaiters.clear(); - this.models = []; - this.routerModels = []; - this.loading = false; - this.updating = false; - this.error = null; - this.selectedModelId = null; - this.selectedModelName = null; - this.modelUsage.clear(); - this.modelLoadingStates.clear(); - this.modelPropsCache.clear(); - this.modelPropsFetching.clear(); - } - - /** - * Prune expired entries from caches. - * Call periodically for proactive memory cleanup. - */ - pruneExpiredCache(): number { - return this.modelPropsCache.prune(); - } -} - -export const modelsStore = new ModelsStore(); diff --git a/tools/ui/src/lib/stores/models/index.svelte.ts b/tools/ui/src/lib/stores/models/index.svelte.ts new file mode 100644 index 0000000000..90d6fe76b7 --- /dev/null +++ b/tools/ui/src/lib/stores/models/index.svelte.ts @@ -0,0 +1,451 @@ +/** + * modelsStore - Model management for MODEL and ROUTER modes + * + * Owns model lists, selection, favorites and load/unload state. Composes the + * per-model props cache (modalities, thinking detection) as + * {@link ModelsStore.props} and the /models/sse status feed as + * {@link ModelsStore.status}; tracks which conversations use which models. + */ + +import { FAVORITE_MODELS_LOCALSTORAGE_KEY } from '$lib/constants'; +import { ServerModelStatus } from '$lib/enums'; +import { ModelsService } from '$lib/services/models.service'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import { type ModelPropsHost, ModelPropsManager } from '$lib/stores/models/props.svelte'; +import { type ModelStatusHost, ModelStatusManager } from '$lib/stores/models/status.svelte'; +import { serverStore } from '$lib/stores/server.svelte'; +import { getConversationModel } from '$lib/utils/conversation-utils'; +import { SvelteSet } from 'svelte/reactivity'; +import { toast } from 'svelte-sonner'; + +class ModelsStore implements ModelPropsHost, ModelStatusHost { + error = $state(null); + favoriteModelIds = $state>(this.loadFavoritesFromStorage()); + loading = $state(false); + models = $state([]); + routerModels = $state([]); + selectedModelId = $state(null); + selectedModelName = $state(null); + + updating = $state(false); + + /** Per-model props cache, modalities and thinking detection, composed here. */ + private _props = new ModelPropsManager(this); + + /** Load/unload operations and the /models/sse status feed, composed here. */ + private _status = new ModelStatusManager(this); + + // Dedup concurrent fetch() callers — all awaiters share the same inflight promise. + // Without this, ?model= URL handler races an in-progress fetch and sees an empty list. + private inflightFetch: Promise | null = null; + + /** + * Model the active conversation view resolves to. Router mode: the user's + * selection first, then the conversation's own model. Otherwise the single + * served model, from the models list or the server props as a fallback. + */ + get activeModelId(): string | null { + if (!serverStore.isRouterMode) { + return this.models.length > 0 ? this.models[0].model : this.singleModelName; + } + + if (this.selectedModelId) { + const selected = this.models.find((m) => m.id === this.selectedModelId); + + if (selected) return selected.model; + } + + const conversationModel = getConversationModel(conversationsStore.activeMessages); + + if (conversationModel) { + const model = this.models.find((m) => m.model === conversationModel); + + if (model) return model.model; + } + + return null; + } + + get loadedModelIds(): string[] { + return this.routerModels + .filter( + (m) => + m.status.value === ServerModelStatus.LOADED || + m.status.value === ServerModelStatus.SLEEPING + ) + .map((m) => m.id); + } + + get props() { + return this._props; + } + + get selectedModel(): ModelOption | null { + if (!this.selectedModelId) return null; + + return this.models.find((m) => m.id === this.selectedModelId) ?? null; + } + + get selectedModelContextSize(): number | null { + if (!this.selectedModelName) return null; + + return this.props.getModelContextSize(this.selectedModelName); + } + + /** + * Get model name in MODEL mode (single model). + * Extracts from model_path or model_alias from server props. + * In ROUTER mode, returns null (model is per-conversation). + */ + get singleModelName(): string | null { + if (serverStore.isRouterMode) return null; + + const props = serverStore.props; + + if (props?.model_alias) return props.model_alias; + + if (!props?.model_path) return null; + + return props.model_path.split(/(\\|\/)/).pop() || null; + } + + get status() { + return this._status; + } + + clearSelection(): void { + this.selectedModelId = null; + this.selectedModelName = null; + } + + /** + * Auto-selects the first available model if none is selected. + * Prioritizes: + * 1. Model from active conversation's last assistant response (if loaded) + * 2. Model from active conversation's last assistant response (if not loaded) + * 3. First loaded model (not from active conversation) + * 4. A favorite model + * 5. First available model + */ + async ensureFirstModelSelected(): Promise { + if (this.selectedModelName) return; + + const availableModels = this.getVisibleModels(); + + if (availableModels.length === 0) return; + + // Try to select model from last assistant response first + const lastModel = this.getModelFromLastAssistantResponse(); + + if (lastModel) { + const lastModelOption = availableModels.find((m) => m.model === lastModel); + + if (lastModelOption) { + await this.selectModelById(lastModelOption.id); + + if (this.isModelLoaded(lastModel)) { + await this.props.fetchModelProps(lastModel); + } + + return; + } + } + + // Try a loaded model first + const loadedModel = availableModels.find((m) => this.isModelLoaded(m.model)); + + if (loadedModel) { + await this.selectModelById(loadedModel.id); + await this.props.fetchModelProps(loadedModel.model); + + return; + } + + // Try loading a favorite model + const favorite = this.favoriteModelIds.values().next()?.value; + + if (favorite) { + await this.selectModelById(favorite); + + return; + } + + // Fall back to the first available model + await this.selectModelById(availableModels[0].id); + } + + /** + * Fetch list of models from server and detect server role. + * Also fetches modalities for MODEL mode (single model). + */ + async fetch(force = false): Promise { + if (this.inflightFetch) return this.inflightFetch; + + if (this.models.length > 0 && !force) return; + + this.inflightFetch = this.runFetch(); + try { + await this.inflightFetch; + } finally { + this.inflightFetch = null; + } + } + + /** + * Fetch router models with full metadata (ROUTER mode only). + * No-op in router mode — fetch() already calls listRouter() internally. + * Kept for API compatibility (e.g. handleOpenChange dropdown open handler). + */ + async fetchRouterModels(): Promise { + if (!serverStore.isRouterMode) return; + + try { + const response = await ModelsService.listRouter(); + + this.routerModels = response.data; + await this.props.fetchModalitiesForLoadedModels(); + + const visible = this.getVisibleModels(); + + if (visible.length === 1 && this.isModelLoaded(visible[0].model)) { + this.selectModelById(visible[0].id); + } + } catch (error) { + console.warn('Failed to fetch router models:', error); + this.routerModels = []; + } + } + + findModelById(modelId: string): ModelOption | null { + return this.models.find((model) => model.id === modelId) ?? null; + } + + findModelByName(modelName: string): ModelOption | null { + return ( + this.models.find( + (model) => + model.model === modelName || model.id === modelName || model.aliases?.includes(modelName) + ) ?? null + ); + } + + /** + * Gets the model name from the last assistant message in the active conversation. + * Used by both the chat page and settings page to maintain model consistency. + */ + getModelFromLastAssistantResponse(): string | null { + const messages = conversationsStore.activeMessages; + + if (!messages || messages.length === 0) return null; + + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].model) { + return messages[i].model; + } + } + + return null; + } + + getModelStatus(modelId: string): ServerModelStatus | null { + const model = this.routerModels.find((m) => m.id === modelId); + + return model?.status.value ?? null; + } + + hasModel(modelName: string): boolean { + return this.models.some((model) => model.model === modelName); + } + + isFavorite(modelId: string): boolean { + return this.favoriteModelIds.has(modelId); + } + + isModelLoaded(modelId: string): boolean { + const model = this.routerModels.find((m) => m.id === modelId); + + return ( + model?.status.value === ServerModelStatus.LOADED || + model?.status.value === ServerModelStatus.SLEEPING + ); + } + + async selectModelById(modelId: string): Promise { + if (!modelId || this.updating) return; + + if (this.selectedModelId === modelId) return; + + const option = this.models.find((model) => model.id === modelId); + + if (!option) throw new Error('Selected model is not available'); + + this.updating = true; + this.error = null; + + try { + this.selectedModelId = option.id; + this.selectedModelName = option.model; + } finally { + this.updating = false; + } + } + + /** + * Select a model by its model name (used for syncing with conversation model). + */ + selectModelByName(modelName: string): void { + const option = this.models.find((model) => model.model === modelName); + + if (option) { + this.selectedModelId = option.id; + this.selectedModelName = option.model; + } + } + + /** + * Auto-selects the model from the last assistant response if available and loaded. + * Returns true if a model was selected, false otherwise. + */ + async selectModelFromLastAssistantResponse(): Promise { + const lastModel = this.getModelFromLastAssistantResponse(); + + if (!lastModel || this.selectedModelName === lastModel) return false; + + const matchingModel = this.models.find((option) => option.model === lastModel); + + if (!matchingModel || !this.isModelLoaded(lastModel)) return false; + + try { + await this.selectModelById(matchingModel.id); + console.log(`[modelsStore] Automatically selected model: ${lastModel} from last message`); + + return true; + } catch (error) { + console.warn('[modelsStore] Failed to automatically select model from last message:', error); + + return false; + } + } + + toDisplayName(id: string): string { + const segments = id.split(/\\|\//); + const candidate = segments.pop(); + + return candidate && candidate.trim().length > 0 ? candidate : id; + } + + toggleFavorite(modelId: string): void { + const next = new SvelteSet(this.favoriteModelIds); + + if (next.has(modelId)) { + next.delete(modelId); + } else { + next.add(modelId); + } + + this.favoriteModelIds = next; + + try { + localStorage.setItem(FAVORITE_MODELS_LOCALSTORAGE_KEY, JSON.stringify([...next])); + } catch { + toast.error('Failed to save favorite models to local storage'); + } + } + + /** + * Build ModelOption[] from an API response. + * Both MODEL and ROUTER modes share the same mapping logic; + * they differ only in which endpoint is called. + */ + private buildModelOptions( + response: ApiModelListResponse | ApiRouterModelsListResponse + ): ModelOption[] { + return response.data.map((item: ApiModelDataEntry, index: number) => { + const details = response.models?.[index]; + const rawCapabilities = Array.isArray(details?.capabilities) ? details?.capabilities : []; + const displayNameSource = + details?.name && details.name.trim().length > 0 ? details.name : item.id; + const modelId = details?.model || item.id; + + return { + aliases: item.aliases ?? [], + capabilities: rawCapabilities.filter((value: unknown): value is string => Boolean(value)), + description: details?.description, + details: details?.details, + id: item.id, + meta: item.meta ?? null, + modalities: this.props.buildArchitectureModalities(item.architecture), + model: modelId, + name: this.toDisplayName(displayNameSource), + parsedId: ModelsService.parseModelId(modelId), + tags: item.tags ?? [] + }; + }); + } + + /** Fetch models in MODEL mode (single model, standard OpenAI-compatible). */ + private async fetchModelModeInternal(): Promise { + const response = await ModelsService.list(); + + return this.buildModelOptions(response); + } + + /** + * Filter to models visible in the UI (ui !== false). + */ + private getVisibleModels(): ModelOption[] { + return this.models.filter((option) => this.props.getModelProps(option.model)?.ui !== false); + } + + private loadFavoritesFromStorage(): Set { + try { + const raw = localStorage.getItem(FAVORITE_MODELS_LOCALSTORAGE_KEY); + + return raw ? new Set(JSON.parse(raw) as string[]) : new Set(); + } catch { + toast.error('Failed to load favorite models from local storage'); + + return new Set(); + } + } + + private async runFetch(): Promise { + this.loading = true; + this.error = null; + + try { + if (!serverStore.props) { + await serverStore.fetch(); + } + + const router = serverStore.isRouterMode; + + if (router) { + const response = await ModelsService.listRouter(); + + this.routerModels = response.data; + this.models = this.buildModelOptions(response); + + await this.props.fetchModalitiesForLoadedModels(); + + const visible = this.getVisibleModels(); + + if (visible.length === 1 && this.isModelLoaded(visible[0].model)) { + this.selectModelById(visible[0].id); + } + } else { + this.models = await this.fetchModelModeInternal(); + } + } catch (error) { + this.models = []; + this.error = error instanceof Error ? error.message : 'Failed to load models'; + + throw error; + } finally { + this.loading = false; + } + } +} + +export const modelsStore = new ModelsStore(); diff --git a/tools/ui/src/lib/stores/models/props.svelte.ts b/tools/ui/src/lib/stores/models/props.svelte.ts new file mode 100644 index 0000000000..9d2d817acb --- /dev/null +++ b/tools/ui/src/lib/stores/models/props.svelte.ts @@ -0,0 +1,273 @@ +/** + * ModelPropsManager - Per-model props cache, modalities and thinking detection + * + * Owns the /props?model= cache with TTL, the modality views over it, + * and chat-template thinking detection. Created and owned by modelsStore; + * the host owns the model lists that fetched modalities are mirrored onto. + * + * **API Inconsistency Workaround:** + * In MODEL mode, `/props` returns modalities for the single model. + * In ROUTER mode, `/props` has no modalities - must use `/props?model=` per model. + */ + +import { MODEL_PROPS_CACHE } from '$lib/constants'; +import { FileTypeCategory, ModelModality } from '$lib/enums'; +import { PropsService } from '$lib/services/props.service'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { serverStore } from '$lib/stores/server.svelte'; +// deep imports, not the '$lib/utils' barrel: it re-exports modules that reach back +// into the stores, and going through it here would read a half-built module +import { TTLCache } from '$lib/utils/cache-ttl'; +import { detectThinkingSupport } from '$lib/utils/chat-template-thinking-detector'; +import { SvelteSet } from 'svelte/reactivity'; + +/** + * The slice of modelsStore the manager reads. Kept narrow on purpose so it + * cannot reach around the host's full surface; modelsStore implements this + * structurally. + */ +export interface ModelPropsHost { + /** Model rows the manager mirrors fetched modalities onto. */ + models: ModelOption[]; + readonly selectedModelName: string | null; + readonly loadedModelIds: string[]; + isModelLoaded(modelId: string): boolean; +} + +export class ModelPropsManager { + /** Version counter for the cache - bumped on writes so $derived consumers recompute. */ + cacheVersion = $state(0); + /** + * Model-specific props cache with TTL. + * Key: modelId, Value: props data including modalities. + */ + private cache = new TTLCache({ + maxEntries: MODEL_PROPS_CACHE.MAX_ENTRIES, + ttlMs: MODEL_PROPS_CACHE.TTL_MS + }); + private fetching = new SvelteSet(); + + /** + * Whether the selected model's chat template supports thinking/reasoning. + * Uses heuristic detection on the model's chat_template from /props. + * + * - MODEL mode: the global /props already describes the single loaded model, + * so its chat_template is used directly and no per-model cache is involved + * - ROUTER mode: fetches /props?model= for the selected model (cached), + * triggering an async fetch if not yet cached + */ + get supportsThinking(): boolean { + if (!serverStore.isRouterMode) { + return detectThinkingSupport(serverStore.props?.chat_template ?? ''); + } + + const modelId = this.host.selectedModelName; + + if (!modelId) return false; + + if (!this.cache.get(modelId)) { + this.fetchModelProps(modelId); + } + + const props = this.getModelProps(modelId); + + return detectThinkingSupport(props?.chat_template ?? ''); + } + + /** Map the router modalities, the only source available while a model is not loaded. */ + buildArchitectureModalities( + architecture: ApiModelDataEntry['architecture'] + ): ModelModalities | undefined { + if (!architecture) return undefined; + + const inputs = architecture.input_modalities; + + return { + audio: inputs.includes(FileTypeCategory.AUDIO), + video: inputs.includes(FileTypeCategory.VIDEO), + vision: inputs.includes(FileTypeCategory.IMAGE) + }; + } + + /** + * Check if a specific model supports thinking. + * In MODEL mode the global /props describes the single loaded model. + * In ROUTER mode, fetches model props if not cached. + */ + checkModelSupportsThinking(modelId: string): boolean { + if (!serverStore.isRouterMode) { + return detectThinkingSupport(serverStore.props?.chat_template ?? ''); + } + + if (!modelId) return false; + + if (!this.cache.get(modelId)) { + this.fetchModelProps(modelId); + } + + const props = this.getModelProps(modelId); + + return detectThinkingSupport(props?.chat_template ?? ''); + } + + constructor(private host: ModelPropsHost) {} + + /** Fetch modalities for all loaded models from /props endpoint. */ + async fetchModalitiesForLoadedModels(): Promise { + const loadedModelIds = this.host.loadedModelIds; + + if (loadedModelIds.length === 0) return; + + const propsPromises = loadedModelIds.map((modelId) => this.fetchModelProps(modelId)); + + try { + const results = await Promise.all(propsPromises); + + this.host.models = this.host.models.map((model) => { + const modelIndex = loadedModelIds.indexOf(model.model); + + if (modelIndex === -1) return model; + + const props = results[modelIndex]; + + if (!props?.modalities) return model; + + return { ...model, modalities: this.buildModalities(props.modalities) }; + }); + + this.cacheVersion++; + } catch (error) { + console.warn('Failed to fetch modalities for loaded models:', error); + } + } + + /** + * Fetch props for a specific model from /props endpoint. + * Uses caching to avoid redundant requests. + * + * In ROUTER mode, this only fetches props if the model is loaded, + * since unloaded models return 400 from /props endpoint. + * + * @param modelId - Model identifier to fetch props for + * @returns Props data or null if fetch failed or model not loaded + */ + async fetchModelProps(modelId: string): Promise { + const cached = this.cache.get(modelId); + + if (cached) return cached; + + if (serverStore.isRouterMode && !this.host.isModelLoaded(modelId)) { + return null; + } + + if (this.fetching.has(modelId)) return null; + + this.fetching.add(modelId); + + try { + const props = await PropsService.fetchForModel(modelId); + + this.cache.set(modelId, props); + this.cacheVersion++; + + return props; + } catch (error) { + console.warn(`Failed to fetch props for model ${modelId}:`, error); + + return null; + } finally { + this.fetching.delete(modelId); + } + } + + getModelContextSize(modelId: string): number | null { + const props = this.getModelProps(modelId); + const nCtx = props?.default_generation_settings?.n_ctx; + + return typeof nCtx === 'number' ? nCtx : null; + } + + getModelModalities(modelId: string): ModelModalities | null { + if (!serverStore.isRouterMode && serverStore.props?.modalities) { + return this.buildModalities(serverStore.props.modalities); + } + + const model = this.host.models.find((m) => m.model === modelId || m.id === modelId); + + if (model?.modalities) { + return model.modalities; + } + + const props = this.cache.get(modelId); + + if (props?.modalities) { + return this.buildModalities(props.modalities); + } + + return null; + } + + getModelModalitiesArray(modelId: string): ModelModality[] { + const modalities = this.getModelModalities(modelId); + + if (!modalities) return []; + + const result: ModelModality[] = []; + + if (modalities.vision) result.push(ModelModality.VISION); + + if (modalities.audio) result.push(ModelModality.AUDIO); + + if (modalities.video) result.push(ModelModality.VIDEO); + + return result; + } + + getModelProps(modelId: string): ApiLlamaCppServerProps | null { + return this.cache.get(modelId); + } + + isModelPropsFetching(modelId: string): boolean { + return this.fetching.has(modelId); + } + + modelSupportsAudio(modelId: string): boolean { + return this.getModelModalities(modelId)?.audio ?? false; + } + + modelSupportsVideo(modelId: string): boolean { + return this.getModelModalities(modelId)?.video ?? false; + } + + modelSupportsVision(modelId: string): boolean { + return this.getModelModalities(modelId)?.vision ?? false; + } + + /** + * Update modalities for a specific model. + * Called when a model is loaded or when we need fresh modality data. + */ + async updateModelModalities(modelId: string): Promise { + const props = await this.fetchModelProps(modelId); + + if (!props?.modalities) return; + + this.host.models = this.host.models.map((model) => + model.model === modelId + ? { ...model, modalities: this.buildModalities(props.modalities!) } + : model + ); + + this.cacheVersion++; + } + + private buildModalities( + modalities: NonNullable + ): ModelModalities { + return { + audio: modalities.audio ?? false, + video: modalities.video ?? false, + vision: modalities.vision ?? false + }; + } +} diff --git a/tools/ui/src/lib/stores/models/status.svelte.ts b/tools/ui/src/lib/stores/models/status.svelte.ts new file mode 100644 index 0000000000..d0160aa4da --- /dev/null +++ b/tools/ui/src/lib/stores/models/status.svelte.ts @@ -0,0 +1,278 @@ +/** + * ModelStatusManager - Model load/unload operations and the /models/sse feed + * + * Owns the status feed subscription, load progress tracking, and the + * awaiters that settle load/unload operations. The feed drives status and + * progress, so it replaces any post-operation polling. Created and owned by + * modelsStore; the host owns the router model rows the feed updates. + */ + +import { ServerModelsSseEventType, ServerModelStatus } from '$lib/enums'; +import { ModelsService } from '$lib/services/models.service'; +import type { ModelPropsManager } from '$lib/stores/models/props.svelte'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { serverStore } from '$lib/stores/server.svelte'; +import { SvelteMap } from 'svelte/reactivity'; +import { toast } from 'svelte-sonner'; + +/** + * The slice of modelsStore the manager drives. Kept narrow on purpose so it + * cannot reach around the host's full surface; modelsStore implements this + * structurally. + */ +export interface ModelStatusHost { + error: string | null; + readonly props: ModelPropsManager; + /** Router model rows the status feed updates. */ + routerModels: ApiModelDataEntry[]; + fetchRouterModels(): Promise; + isModelLoaded(modelId: string): boolean; + toDisplayName(id: string): string; +} + +export class ModelStatusManager { + private loadingStates = new SvelteMap(); + private loadProgress = new SvelteMap(); + // /models/sse feed state, the single source of truth for status and load progress + private statusAbort: AbortController | null = null; + private statusReaderActive = false; + private statusWaiters = new SvelteMap< + string, + { target: ServerModelStatus; resolve: () => void; reject: (e: Error) => void } + >(); + + constructor(private host: ModelStatusHost) {} + + async ensureLoaded(modelId: string): Promise { + if (this.host.isModelLoaded(modelId)) return; + + await this.load(modelId); + } + + /** + * Current load progress for a model, or null when not loading. + */ + getLoadProgress(modelId: string): ModelLoadProgress | null { + return this.loadProgress.get(modelId) ?? null; + } + + isOperationInProgress(modelId: string): boolean { + return this.loadingStates.get(modelId) ?? false; + } + + async load(modelId: string): Promise { + if (this.host.isModelLoaded(modelId)) return; + + if (this.loadingStates.get(modelId)) return; + + this.loadingStates.set(modelId, true); + this.host.error = null; + + // the feed drives completion, so it must be live before the request + this.subscribe(); + + const reachedLoaded = this.waitForStatus(modelId, ServerModelStatus.LOADED); + + reachedLoaded.catch(() => {}); + + try { + await ModelsService.load(modelId); + await reachedLoaded; + toast.success(`Model loaded: ${this.host.toDisplayName(modelId)}`); + } catch (error) { + this.rejectStatus(modelId, error instanceof Error ? error : new Error('load failed')); + this.host.error = error instanceof Error ? error.message : 'Failed to load model'; + toast.error(`Failed to load model: ${this.host.toDisplayName(modelId)}`); + + throw error; + } finally { + this.loadingStates.set(modelId, false); + } + } + + /** + * Open the /models/sse feed and keep it live with auto reconnect. + * Idempotent and router mode only. + */ + subscribe(): void { + if (this.statusReaderActive) return; + + if (!serverStore.isRouterMode) return; + + this.statusReaderActive = true; + this.statusAbort = new AbortController(); + void this.runStatusReader(this.statusAbort.signal); + } + + async unload(modelId: string): Promise { + if (!this.host.isModelLoaded(modelId)) return; + + if (this.loadingStates.get(modelId)) return; + + this.loadingStates.set(modelId, true); + this.host.error = null; + + this.subscribe(); + + const reachedUnloaded = this.waitForStatus(modelId, ServerModelStatus.UNLOADED); + + reachedUnloaded.catch(() => {}); + + try { + await ModelsService.unload(modelId); + await reachedUnloaded; + toast.info(`Model unloaded: ${this.host.toDisplayName(modelId)}`); + } catch (error) { + this.rejectStatus(modelId, error instanceof Error ? error : new Error('unload failed')); + this.host.error = error instanceof Error ? error.message : 'Failed to unload model'; + toast.error(`Failed to unload model: ${this.host.toDisplayName(modelId)}`); + + throw error; + } finally { + this.loadingStates.set(modelId, false); + } + } + + /** + * Close the /models/sse feed and drop transient progress. + */ + unsubscribe(): void { + this.statusReaderActive = false; + this.statusAbort?.abort(); + this.statusAbort = null; + this.loadProgress.clear(); + } + + /** + * Apply a status envelope: update the model row, track or clear progress, + * settle any pending load or unload awaiter. + */ + private applyModelStatus(event: ApiModelsSseEvent): void { + const model = event.model; + const data = event.data; + + if (!model || !data?.status) return; + + const status = data.status; + + this.setRouterModelStatus(model, status); + + if (status === ServerModelStatus.LOADING) { + if (data.progress) this.loadProgress.set(model, data.progress); + } else { + this.loadProgress.delete(model); + } + + if (status === ServerModelStatus.LOADED) { + void this.host.props.updateModelModalities(model); + } + + const failed = + status === ServerModelStatus.FAILED || + (status === ServerModelStatus.UNLOADED && (data.exit_code ?? 0) !== 0); + + if (failed) { + this.rejectStatus(model, new Error(`Model failed: ${this.host.toDisplayName(model)}`)); + + return; + } + + this.settleStatus(model, status); + } + + /** + * Route one feed record by event kind. Only the status_* events carry a + * status payload, models_reload triggers a list refresh, model_remove drops + * the row, download_* belong to the download surface, not here. + */ + private applyStatusEvent(event: ApiModelsSseEvent): void { + switch (event.event) { + case ServerModelsSseEventType.STATUS_CHANGE: + case ServerModelsSseEventType.MODEL_STATUS: + case ServerModelsSseEventType.STATUS_UPDATE: + this.applyModelStatus(event); + + break; + case ServerModelsSseEventType.MODELS_RELOAD: + void this.host.fetchRouterModels(); + + break; + case ServerModelsSseEventType.MODEL_REMOVE: + this.removeRouterModel(event.model); + + break; + case ServerModelsSseEventType.DOWNLOAD_PROGRESS: + break; + } + } + + /** + * Reject and drop the awaiter for a model. + */ + private rejectStatus(modelId: string, error: Error): void { + const waiter = this.statusWaiters.get(modelId); + + if (waiter) { + this.statusWaiters.delete(modelId); + waiter.reject(error); + } + } + + /** + * Drop a model row reported gone by the feed and settle its awaiters. + */ + private removeRouterModel(modelId: string): void { + if (this.host.routerModels.findIndex((m) => m.id === modelId) === -1) return; + + this.host.routerModels = this.host.routerModels.filter((m) => m.id !== modelId); + this.loadProgress.delete(modelId); + this.rejectStatus(modelId, new Error(`Model removed: ${this.host.toDisplayName(modelId)}`)); + } + + /** + * Read the feed and reconnect until unsubscribed. + */ + private async runStatusReader(signal: AbortSignal): Promise { + await ModelsService.watchModelEvents(signal, (event) => this.applyStatusEvent(event)); + } + + /** + * Update one model row status in place, reassigning to trigger reactivity. + */ + private setRouterModelStatus(modelId: string, status: ServerModelStatus): void { + const idx = this.host.routerModels.findIndex((m) => m.id === modelId); + + if (idx === -1) return; + + const current = this.host.routerModels[idx]; + + if (current.status.value === status) return; + + const next = [...this.host.routerModels]; + + next[idx] = { ...current, status: { ...current.status, value: status } }; + this.host.routerModels = next; + } + + /** + * Resolve and drop the awaiter when the model reaches its target status. + */ + private settleStatus(modelId: string, status: ServerModelStatus): void { + const waiter = this.statusWaiters.get(modelId); + + if (waiter && waiter.target === status) { + this.statusWaiters.delete(modelId); + waiter.resolve(); + } + } + + /** + * Register an awaiter that resolves when the feed reports target status. + * One operation runs per model at a time, so one awaiter per model is kept. + */ + private waitForStatus(modelId: string, target: ServerModelStatus): Promise { + return new Promise((resolve, reject) => { + this.statusWaiters.set(modelId, { reject, resolve, target }); + }); + } +} diff --git a/tools/ui/src/lib/stores/permissions.svelte.ts b/tools/ui/src/lib/stores/permissions.svelte.ts index 3e83538e95..f4eae4b7e6 100644 --- a/tools/ui/src/lib/stores/permissions.svelte.ts +++ b/tools/ui/src/lib/stores/permissions.svelte.ts @@ -1,3 +1,11 @@ +/** + * permissionsStore - Allowed tool permissions + * + * Owns the set of tools the user has permanently allowed, persisted to + * localStorage. The agentic loop's permission gates consult it to run a + * tool without prompting. + */ + import { browser } from '$app/environment'; import { ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY } from '$lib/constants'; import { SvelteSet } from 'svelte/reactivity'; @@ -5,6 +13,24 @@ import { SvelteSet } from 'svelte/reactivity'; class PermissionsStore { private _tools = $state(new SvelteSet()); + get tools(): ReadonlySet { + return this._tools; + } + + allowTool(key: string): void { + this._tools.add(key); + this.persist(); + } + + allowTools(keys: string[]): void { + for (const key of keys) this._tools.add(key); + this.persist(); + } + + hasTool(key: string): boolean { + return this._tools.has(key); + } + /** * Load persisted permissions. Called by initStores() after migrations * have run. @@ -29,30 +55,12 @@ class PermissionsStore { } } - get tools(): ReadonlySet { - return this._tools; - } - - hasTool(key: string): boolean { - return this._tools.has(key); - } - - allowTool(key: string): void { - this._tools.add(key); - this._persist(); - } - - allowTools(keys: string[]): void { - for (const key of keys) this._tools.add(key); - this._persist(); - } - revokeTool(key: string): void { this._tools.delete(key); - this._persist(); + this.persist(); } - private _persist(): void { + private persist(): void { try { localStorage.setItem(ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY, JSON.stringify([...this._tools])); } catch (err) { diff --git a/tools/ui/src/lib/stores/server.svelte.ts b/tools/ui/src/lib/stores/server.svelte.ts index 7de5850b9e..e145e2891d 100644 --- a/tools/ui/src/lib/stores/server.svelte.ts +++ b/tools/ui/src/lib/stores/server.svelte.ts @@ -1,79 +1,57 @@ +/** + * serverStore - Server connection state, configuration and role detection + * + * Owns the connection state and properties fetched from /props, plus MODEL + * vs ROUTER role detection and server-wide generation defaults. Uses + * PropsService for the /props fetch. + */ + import { ServerRole } from '$lib/enums'; import { PropsService } from '$lib/services/props.service'; import { ApiError } from '$lib/utils'; const LOADING_RETRY_INTERVAL_MS = 1000; -/** - * serverStore - Server connection state, configuration, and role detection - * - * This store manages the server connection state and properties fetched from `/props`. - * It provides reactive state for server configuration and role detection. - * - * **Architecture & Relationships:** - * - **PropsService**: Stateless service for fetching `/props` data - * - **serverStore** (this class): Reactive store for server state - * - **modelsStore**: Independent store for model management (uses PropsService directly) - * - * **Key Features:** - * - **Server State**: Connection status, loading, error handling - * - **Role Detection**: MODEL (single model) vs ROUTER (multi-model) - * - **Default Params**: Server-wide generation defaults - */ class ServerStore { - /** - * - * - * State - * - * - */ - - props = $state(null); - loading = $state(false); error = $state(null); - status = $state(null); + loading = $state(false); + props = $state(null); role = $state(null); + status = $state(null); private fetchPromise: Promise | null = null; private retryTimer: ReturnType | null = null; - /** - * - * - * Getters - * - * - */ - - get defaultParams(): ApiLlamaCppServerProps['default_generation_settings']['params'] | null { - return this.props?.default_generation_settings?.params || null; - } - get contextSize(): number | null { const nCtx = this.props?.default_generation_settings?.n_ctx; return typeof nCtx === 'number' ? nCtx : null; } - get uiSettings(): Record | undefined { - return this.props?.ui_settings ?? this.props?.webui_settings; - } - - get isRouterMode(): boolean { - return this.role === ServerRole.ROUTER; + get defaultParams(): ApiLlamaCppServerProps['default_generation_settings']['params'] | null { + return this.props?.default_generation_settings?.params || null; } get isModelMode(): boolean { return this.role === ServerRole.MODEL; } - /** - * - * - * Data Handling - * - * - */ + get isRouterMode(): boolean { + return this.role === ServerRole.ROUTER; + } + + get uiSettings(): Record | undefined { + return this.props?.ui_settings ?? this.props?.webui_settings; + } + + clear(): void { + this.clearRetryTimer(); + this.props = null; + this.error = null; + this.status = null; + this.loading = false; + this.role = null; + this.fetchPromise = null; + } /** * @param background - Set by the automatic "still loading" poll. Skips the @@ -124,14 +102,20 @@ class ServerStore { await fetchPromise; } - clear(): void { - this.clearRetryTimer(); - this.props = null; - this.error = null; - this.status = null; - this.loading = false; - this.role = null; - this.fetchPromise = null; + private clearRetryTimer(): void { + if (this.retryTimer) { + clearTimeout(this.retryTimer); + this.retryTimer = null; + } + } + + private detectRole(props: ApiLlamaCppServerProps): void { + const newRole = props?.role === ServerRole.ROUTER ? ServerRole.ROUTER : ServerRole.MODEL; + + if (this.role !== newRole) { + this.role = newRole; + console.info(`Server running in ${newRole === ServerRole.ROUTER ? 'ROUTER' : 'MODEL'} mode`); + } } private scheduleRetry(): void { @@ -142,30 +126,6 @@ class ServerStore { this.fetch({ background: true }); }, LOADING_RETRY_INTERVAL_MS); } - - private clearRetryTimer(): void { - if (this.retryTimer) { - clearTimeout(this.retryTimer); - this.retryTimer = null; - } - } - - /** - * - * - * Utilities - * - * - */ - - private detectRole(props: ApiLlamaCppServerProps): void { - const newRole = props?.role === ServerRole.ROUTER ? ServerRole.ROUTER : ServerRole.MODEL; - - if (this.role !== newRole) { - this.role = newRole; - console.info(`Server running in ${newRole === ServerRole.ROUTER ? 'ROUTER' : 'MODEL'} mode`); - } - } } export const serverStore = new ServerStore(); diff --git a/tools/ui/src/lib/stores/settings.svelte.ts b/tools/ui/src/lib/stores/settings/index.svelte.ts similarity index 89% rename from tools/ui/src/lib/stores/settings.svelte.ts rename to tools/ui/src/lib/stores/settings/index.svelte.ts index f23f6953ae..0373ade420 100644 --- a/tools/ui/src/lib/stores/settings.svelte.ts +++ b/tools/ui/src/lib/stores/settings/index.svelte.ts @@ -1,34 +1,10 @@ /** * settingsStore - Application configuration and theme management * - * This store manages all application settings including AI model parameters, UI preferences, - * and theme configuration. It provides persistent storage through localStorage with reactive - * state management using Svelte 5 runes. - * - * **Architecture & Relationships:** - * - **settingsStore** (this class): Configuration state management - * - Manages AI model parameters (temperature, max tokens, etc.) - * - Handles theme switching and persistence - * - Provides localStorage synchronization - * - Offers reactive configuration access - * - * - **ChatService**: Reads model parameters for API requests - * - **UI Components**: Subscribe to theme and configuration changes - * - * **Key Features:** - * - **Model Parameters**: Temperature, max tokens, top-p, top-k, repeat penalty - * - **Theme Management**: Auto, light, dark theme switching - * - **Persistence**: Automatic localStorage synchronization - * - **Reactive State**: Svelte 5 runes for automatic UI updates - * - **Default Handling**: Graceful fallback to defaults for missing settings - * - **Batch Updates**: Efficient multi-setting updates - * - **Reset Functionality**: Restore defaults for individual or all settings - * - * **Configuration Categories:** - * - Generation parameters (temperature, tokens, sampling) - * - UI preferences (theme, display options) - * - System settings (model selection, prompts) - * - Advanced options (seed, penalties, context handling) + * Owns generation parameters, UI preferences and theme, persisted to + * localStorage with Svelte 5 runes. Applies the admin's server ui_settings + * as defaults on first visit; sampling parameters sync with the server via + * ParameterSyncService. */ import { browser } from '$app/environment'; @@ -53,14 +29,6 @@ import { import { setMode } from 'mode-watcher'; class SettingsStore { - /** - * - * - * State - * - * - */ - config = $state({ ...SETTING_CONFIG_DEFAULT }); isInitialized = $state(false); userOverrides = $state>(new Set()); @@ -69,29 +37,182 @@ class SettingsStore { // application of server ui_settings defaults for new users. private isFirstVisit = false; + canSyncParameter(key: string): boolean { + return ParameterSyncService.canSyncParameter(key); + } /** - * - * - * Utilities (private helpers) - * - * + * Clear all user overrides (for debugging) */ - - /** - * Helper method to get server defaults with null safety - * Centralizes the pattern of getting and extracting server defaults - */ - private getServerDefaults(): Record { - return ParameterSyncService.extractServerDefaults(serverStore.defaultParams); + clearAllUserOverrides(): void { + this.userOverrides.clear(); + this.saveConfig(); + console.log('Cleared all user overrides'); } /** - * - * - * Lifecycle - * - * + * Export all settings as a versioned JSON-compatible object. + * The export captures the full config (excluding sensitive values like API key) + * and user overrides. Sensitive fields are filtered out for security by default. + * @param includeSensitiveData - If true, include sensitive fields (apiKey, MCP server headers) in export */ + exportSettings(includeSensitiveData: boolean = false): SettingsExportType { + // Build config excluding sensitive data unless user opts in + const configToExport: Record = + includeSensitiveData + ? { ...this.config } + : Object.fromEntries(Object.entries(this.config).filter(([key]) => key !== 'apiKey')); + + // Handle MCP servers: exclude custom headers unless user opts in + if ('mcpServers' in configToExport && !includeSensitiveData) { + try { + const mcpServers = JSON.parse(configToExport.mcpServers as string) as Array< + Record + >; + const safeServers = mcpServers.map((server) => { + delete server.headers; + + return server; + }); + + configToExport.mcpServers = JSON.stringify(safeServers); + } catch { + // If parsing fails, just exclude the entire mcpServers field + delete (configToExport as Record).mcpServers; + } + } + + return { + config: configToExport, + timestamp: Date.now(), + userOverrides: Array.from(this.userOverrides), + version: 1 + }; + } + + /** + * Reset all parameters to their default values (from props) + * This is used by the "Reset to Default" functionality + * Prioritizes Server defaults from /props, falls back to UI defaults + */ + forceSyncWithServerDefaults(): void { + const propsDefaults = this.getServerDefaults(); + const uiSettings = serverStore.uiSettings; + + for (const key of ParameterSyncService.getSyncableParameterKeys()) { + if (uiSettings && key in uiSettings) { + // UI setting from admin config: write actual value + setConfigValue(this.config, key, uiSettings[key]); + } else if (propsDefaults[key] !== undefined) { + // sampling param: clear it, let server decide + setConfigValue(this.config, key, ''); + } else if (key in SETTING_CONFIG_DEFAULT) { + setConfigValue(this.config, key, getConfigValue(SETTING_CONFIG_DEFAULT, key)); + } + + this.userOverrides.delete(key); + } + + // Non-syncable keys: reset is a full return to the instance state, the + // admin baseline value when defined, the factory default otherwise. + for (const key of Object.keys(SETTING_CONFIG_DEFAULT)) { + if (ParameterSyncService.canSyncParameter(key)) { + continue; + } + + const value = + uiSettings && key in uiSettings && uiSettings[key] !== undefined + ? uiSettings[key] + : getConfigValue(SETTING_CONFIG_DEFAULT, key); + + setConfigValue(this.config, key, value); + + if (key === SETTINGS_KEYS.THEME) { + setMode(value as ColorMode); + } + + this.userOverrides.delete(key); + } + + this.saveConfig(); + } + + /** + * Get the entire configuration object + * @returns The complete configuration object + */ + getAllConfig(): SettingsConfigType { + return { ...this.config }; + } + + /** + * Get a specific configuration value + * @param key - The configuration key to get + * @returns The configuration value + */ + getConfig(key: K): SettingsConfigType[K] { + return this.config[key]; + } + + /** + * Get diff between current settings and server defaults + */ + getParameterDiff() { + const serverDefaults = this.getServerDefaults(); + + if (Object.keys(serverDefaults).length === 0) return {}; + + const configAsRecord = configToParameterRecord( + this.config, + ParameterSyncService.getSyncableParameterKeys() + ); + + return ParameterSyncService.createParameterDiff(configAsRecord, serverDefaults); + } + + /** + * Get parameter information including source for a specific parameter + */ + getParameterInfo(key: string) { + const propsDefaults = this.getServerDefaults(); + const currentValue = getConfigValue(this.config, key); + + return ParameterSyncService.getParameterInfo( + key, + currentValue ?? '', + propsDefaults, + this.userOverrides + ); + } + + /** + * Import settings from a previously exported object. + * Restores config (including theme) and user overrides. + * @param data - The exported settings object + */ + importSettings(data: SettingsExportType): void { + if (!browser) return; + + if (!data || !data.config) { + throw new Error('Invalid settings data: missing config'); + } + + // Restore config (theme is included in config) + this.config = { + ...SETTING_CONFIG_DEFAULT, + ...data.config + }; + + // Restore user overrides (derived state — may be stale if server defaults differ) + this.userOverrides = new Set(data.userOverrides ?? []); + + // Persist to localStorage + this.saveConfig(); + + // Apply theme for immediate visual feedback + setMode(this.config[SETTINGS_KEYS.THEME] as ColorMode); + + console.log('Settings imported successfully'); + } /** * Initialize the settings store by loading from localStorage. @@ -111,6 +232,201 @@ class SettingsStore { } } + /** + * Reset all settings to defaults. + */ + resetAll() { + this.resetConfig(); + + this.resetTheme(); + } + + /** + * Reset configuration to defaults + */ + resetConfig() { + this.config = { ...SETTING_CONFIG_DEFAULT }; + + this.saveConfig(); + } + + /** + * Reset a parameter to Server default (or UI default if no Server default) + */ + resetParameterToServerDefault(key: string): void { + const serverDefaults = this.getServerDefaults(); + const uiSettings = serverStore.uiSettings; + + if (uiSettings && key in uiSettings) { + // UI setting from admin config: write actual value + setConfigValue(this.config, key, uiSettings[key]); + } else if (serverDefaults[key] !== undefined) { + // sampling param known by server: clear it, let server decide + setConfigValue(this.config, key, ''); + } else if (key in SETTING_CONFIG_DEFAULT) { + setConfigValue(this.config, key, getConfigValue(SETTING_CONFIG_DEFAULT, key)); + } + + this.userOverrides.delete(key); + this.saveConfig(); + } + + /** + * Reset theme to default value. + * Theme is now stored inside the config object. + */ + resetTheme() { + this.updateConfig(SETTINGS_KEYS.THEME, SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.THEME]); + + setMode(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.THEME] as ColorMode); + } + + /** + * Initialize settings with props defaults when server properties are first loaded + * This sets up the default values from /props endpoint + */ + syncWithServerDefaults(): void { + const propsDefaults = this.getServerDefaults(); + + if (Object.keys(propsDefaults).length === 0) return; + + const uiSettings = serverStore.uiSettings; + const uiSettingsKeys = new Set(uiSettings ? Object.keys(uiSettings) : []); + + for (const [key, propsValue] of Object.entries(propsDefaults)) { + const currentValue = getConfigValue(this.config, key); + const normalizedCurrent = normalizeFloatingPoint(currentValue); + const normalizedDefault = normalizeFloatingPoint(propsValue); + + // if user value matches server, it's not a real override + if (normalizedCurrent === normalizedDefault) { + this.userOverrides.delete(key); + + if (!uiSettingsKeys.has(key) && getConfigValue(SETTING_CONFIG_DEFAULT, key) === undefined) { + setConfigValue(this.config, key, undefined); + } + } + } + + // UI settings are the admin's defaults for new users: applied once on + // the first visit, never on later loads, so the user's config can + // diverge. "Reset to Default" is the explicit way back to the baseline. + // A first visit config carries factory values only, so a key that + // already diverges here was set by the user before the baseline could + // be reached, through the API key splash, and stays theirs. + if (uiSettings && this.isFirstVisit) { + this.isFirstVisit = false; + + for (const [key, value] of Object.entries(uiSettings)) { + if (value === undefined || this.userOverrides.has(key)) continue; + + if (getConfigValue(this.config, key) !== getConfigValue(SETTING_CONFIG_DEFAULT, key)) { + continue; + } + + setConfigValue(this.config, key, value); + + // theme lives in mode-watcher, not just in config -> propagate + if (key === SETTINGS_KEYS.THEME) { + setMode(value as ColorMode); + } + } + } + + this.saveConfig(); + console.log('User overrides after sync:', Array.from(this.userOverrides)); + } + + /** + * Update a specific configuration setting + * @param key - The configuration key to update + * @param value - The new value for the configuration key + */ + updateConfig(key: K, value: SettingsConfigType[K]): void { + this.config[key] = value; + + if (ParameterSyncService.canSyncParameter(key as string)) { + const propsDefaults = this.getServerDefaults(); + const propsDefault = propsDefaults[key as string]; + + if (propsDefault !== undefined) { + const normalizedValue = normalizeFloatingPoint(value); + const normalizedDefault = normalizeFloatingPoint(propsDefault); + + if (normalizedValue === normalizedDefault) { + this.userOverrides.delete(key as string); + } else { + this.userOverrides.add(key as string); + } + } + } + + this.saveConfig(); + } + + /** + * + * + * Import / Export + * + * + */ + + /** + * Update multiple configuration settings at once + * @param updates - Object containing the configuration updates + */ + updateMultipleConfig(updates: Partial) { + Object.assign(this.config, updates); + + const propsDefaults = this.getServerDefaults(); + + for (const [key, value] of Object.entries(updates)) { + if (ParameterSyncService.canSyncParameter(key)) { + const propsDefault = propsDefaults[key]; + + if (propsDefault !== undefined) { + const normalizedValue = normalizeFloatingPoint(value); + const normalizedDefault = normalizeFloatingPoint(propsDefault); + + if (normalizedValue === normalizedDefault) { + this.userOverrides.delete(key); + } else { + this.userOverrides.add(key); + } + } + } + } + + this.saveConfig(); + } + + /** + * Update the theme setting. + * @param newTheme - The new theme value + */ + updateTheme(newTheme: string) { + this.updateConfig(SETTINGS_KEYS.THEME, newTheme); + + setMode(newTheme as ColorMode); + } + + /** + * + * + * Utilities (private helpers) + * + * + */ + + /** + * Helper method to get server defaults with null safety + * Centralizes the pattern of getting and extracting server defaults + */ + private getServerDefaults(): Record { + return ParameterSyncService.extractServerDefaults(serverStore.defaultParams); + } + /** * Load configuration from localStorage * Returns default values for missing keys to prevent breaking changes @@ -171,69 +487,6 @@ class SettingsStore { setMode(legacyTheme as ColorMode); } } - /** - * - * - * Config Updates - * - * - */ - - /** - * Update a specific configuration setting - * @param key - The configuration key to update - * @param value - The new value for the configuration key - */ - updateConfig(key: K, value: SettingsConfigType[K]): void { - this.config[key] = value; - - if (ParameterSyncService.canSyncParameter(key as string)) { - const propsDefaults = this.getServerDefaults(); - const propsDefault = propsDefaults[key as string]; - - if (propsDefault !== undefined) { - const normalizedValue = normalizeFloatingPoint(value); - const normalizedDefault = normalizeFloatingPoint(propsDefault); - - if (normalizedValue === normalizedDefault) { - this.userOverrides.delete(key as string); - } else { - this.userOverrides.add(key as string); - } - } - } - - this.saveConfig(); - } - - /** - * Update multiple configuration settings at once - * @param updates - Object containing the configuration updates - */ - updateMultipleConfig(updates: Partial) { - Object.assign(this.config, updates); - - const propsDefaults = this.getServerDefaults(); - - for (const [key, value] of Object.entries(updates)) { - if (ParameterSyncService.canSyncParameter(key)) { - const propsDefault = propsDefaults[key]; - - if (propsDefault !== undefined) { - const normalizedValue = normalizeFloatingPoint(value); - const normalizedDefault = normalizeFloatingPoint(propsDefault); - - if (normalizedValue === normalizedDefault) { - this.userOverrides.delete(key); - } else { - this.userOverrides.add(key); - } - } - } - } - - this.saveConfig(); - } /** * Save the current configuration to localStorage @@ -252,331 +505,6 @@ class SettingsStore { console.error('Failed to save config to localStorage:', error); } } - - /** - * Update the theme setting. - * @param newTheme - The new theme value - */ - updateTheme(newTheme: string) { - this.updateConfig(SETTINGS_KEYS.THEME, newTheme); - - setMode(newTheme as ColorMode); - } - - /** - * - * - * Reset - * - * - */ - - /** - * Reset configuration to defaults - */ - resetConfig() { - this.config = { ...SETTING_CONFIG_DEFAULT }; - - this.saveConfig(); - } - - /** - * Reset theme to default value. - * Theme is now stored inside the config object. - */ - resetTheme() { - this.updateConfig(SETTINGS_KEYS.THEME, SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.THEME]); - - setMode(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.THEME] as ColorMode); - } - - /** - * Reset all settings to defaults. - */ - resetAll() { - this.resetConfig(); - - this.resetTheme(); - } - - /** - * Reset a parameter to Server default (or UI default if no Server default) - */ - resetParameterToServerDefault(key: string): void { - const serverDefaults = this.getServerDefaults(); - const uiSettings = serverStore.uiSettings; - - if (uiSettings && key in uiSettings) { - // UI setting from admin config: write actual value - setConfigValue(this.config, key, uiSettings[key]); - } else if (serverDefaults[key] !== undefined) { - // sampling param known by server: clear it, let server decide - setConfigValue(this.config, key, ''); - } else if (key in SETTING_CONFIG_DEFAULT) { - setConfigValue(this.config, key, getConfigValue(SETTING_CONFIG_DEFAULT, key)); - } - - this.userOverrides.delete(key); - this.saveConfig(); - } - - /** - * - * - * Server Sync - * - * - */ - - /** - * Initialize settings with props defaults when server properties are first loaded - * This sets up the default values from /props endpoint - */ - syncWithServerDefaults(): void { - const propsDefaults = this.getServerDefaults(); - - if (Object.keys(propsDefaults).length === 0) return; - - const uiSettings = serverStore.uiSettings; - const uiSettingsKeys = new Set(uiSettings ? Object.keys(uiSettings) : []); - - for (const [key, propsValue] of Object.entries(propsDefaults)) { - const currentValue = getConfigValue(this.config, key); - const normalizedCurrent = normalizeFloatingPoint(currentValue); - const normalizedDefault = normalizeFloatingPoint(propsValue); - - // if user value matches server, it's not a real override - if (normalizedCurrent === normalizedDefault) { - this.userOverrides.delete(key); - - if (!uiSettingsKeys.has(key) && getConfigValue(SETTING_CONFIG_DEFAULT, key) === undefined) { - setConfigValue(this.config, key, undefined); - } - } - } - - // UI settings are the admin's defaults for new users: applied once on - // the first visit, never on later loads, so the user's config can - // diverge. "Reset to Default" is the explicit way back to the baseline. - // A first visit config carries factory values only, so a key that - // already diverges here was set by the user before the baseline could - // be reached, through the API key splash, and stays theirs. - if (uiSettings && this.isFirstVisit) { - this.isFirstVisit = false; - - for (const [key, value] of Object.entries(uiSettings)) { - if (value === undefined || this.userOverrides.has(key)) continue; - - if (getConfigValue(this.config, key) !== getConfigValue(SETTING_CONFIG_DEFAULT, key)) { - continue; - } - - setConfigValue(this.config, key, value); - - // theme lives in mode-watcher, not just in config -> propagate - if (key === SETTINGS_KEYS.THEME) { - setMode(value as ColorMode); - } - } - } - - this.saveConfig(); - console.log('User overrides after sync:', Array.from(this.userOverrides)); - } - - /** - * Reset all parameters to their default values (from props) - * This is used by the "Reset to Default" functionality - * Prioritizes Server defaults from /props, falls back to UI defaults - */ - forceSyncWithServerDefaults(): void { - const propsDefaults = this.getServerDefaults(); - const uiSettings = serverStore.uiSettings; - - for (const key of ParameterSyncService.getSyncableParameterKeys()) { - if (uiSettings && key in uiSettings) { - // UI setting from admin config: write actual value - setConfigValue(this.config, key, uiSettings[key]); - } else if (propsDefaults[key] !== undefined) { - // sampling param: clear it, let server decide - setConfigValue(this.config, key, ''); - } else if (key in SETTING_CONFIG_DEFAULT) { - setConfigValue(this.config, key, getConfigValue(SETTING_CONFIG_DEFAULT, key)); - } - - this.userOverrides.delete(key); - } - - // Non-syncable keys: reset is a full return to the instance state, the - // admin baseline value when defined, the factory default otherwise. - for (const key of Object.keys(SETTING_CONFIG_DEFAULT)) { - if (ParameterSyncService.canSyncParameter(key)) { - continue; - } - - const value = - uiSettings && key in uiSettings && uiSettings[key] !== undefined - ? uiSettings[key] - : getConfigValue(SETTING_CONFIG_DEFAULT, key); - - setConfigValue(this.config, key, value); - - if (key === SETTINGS_KEYS.THEME) { - setMode(value as ColorMode); - } - - this.userOverrides.delete(key); - } - - this.saveConfig(); - } - - /** - * - * - * Utilities - * - * - */ - - /** - * Get a specific configuration value - * @param key - The configuration key to get - * @returns The configuration value - */ - getConfig(key: K): SettingsConfigType[K] { - return this.config[key]; - } - - /** - * Get the entire configuration object - * @returns The complete configuration object - */ - getAllConfig(): SettingsConfigType { - return { ...this.config }; - } - - canSyncParameter(key: string): boolean { - return ParameterSyncService.canSyncParameter(key); - } - - /** - * Get parameter information including source for a specific parameter - */ - getParameterInfo(key: string) { - const propsDefaults = this.getServerDefaults(); - const currentValue = getConfigValue(this.config, key); - - return ParameterSyncService.getParameterInfo( - key, - currentValue ?? '', - propsDefaults, - this.userOverrides - ); - } - - /** - * Get diff between current settings and server defaults - */ - getParameterDiff() { - const serverDefaults = this.getServerDefaults(); - - if (Object.keys(serverDefaults).length === 0) return {}; - - const configAsRecord = configToParameterRecord( - this.config, - ParameterSyncService.getSyncableParameterKeys() - ); - - return ParameterSyncService.createParameterDiff(configAsRecord, serverDefaults); - } - - /** - * Clear all user overrides (for debugging) - */ - clearAllUserOverrides(): void { - this.userOverrides.clear(); - this.saveConfig(); - console.log('Cleared all user overrides'); - } - - /** - * - * - * Import / Export - * - * - */ - - /** - * Export all settings as a versioned JSON-compatible object. - * The export captures the full config (excluding sensitive values like API key) - * and user overrides. Sensitive fields are filtered out for security by default. - * @param includeSensitiveData - If true, include sensitive fields (apiKey, MCP server headers) in export - */ - exportSettings(includeSensitiveData: boolean = false): SettingsExportType { - // Build config excluding sensitive data unless user opts in - const configToExport: Record = - includeSensitiveData - ? { ...this.config } - : Object.fromEntries(Object.entries(this.config).filter(([key]) => key !== 'apiKey')); - - // Handle MCP servers: exclude custom headers unless user opts in - if ('mcpServers' in configToExport && !includeSensitiveData) { - try { - const mcpServers = JSON.parse(configToExport.mcpServers as string) as Array< - Record - >; - const safeServers = mcpServers.map((server) => { - delete server.headers; - - return server; - }); - - configToExport.mcpServers = JSON.stringify(safeServers); - } catch { - // If parsing fails, just exclude the entire mcpServers field - delete (configToExport as Record).mcpServers; - } - } - - return { - config: configToExport, - timestamp: Date.now(), - userOverrides: Array.from(this.userOverrides), - version: 1 - }; - } - - /** - * Import settings from a previously exported object. - * Restores config (including theme) and user overrides. - * @param data - The exported settings object - */ - importSettings(data: SettingsExportType): void { - if (!browser) return; - - if (!data || !data.config) { - throw new Error('Invalid settings data: missing config'); - } - - // Restore config (theme is included in config) - this.config = { - ...SETTING_CONFIG_DEFAULT, - ...data.config - }; - - // Restore user overrides (derived state — may be stale if server defaults differ) - this.userOverrides = new Set(data.userOverrides ?? []); - - // Persist to localStorage - this.saveConfig(); - - // Apply theme for immediate visual feedback - setMode(this.config[SETTINGS_KEYS.THEME] as ColorMode); - - console.log('Settings imported successfully'); - } } export const settingsStore = new SettingsStore(); diff --git a/tools/ui/src/lib/stores/settings-referrer.svelte.ts b/tools/ui/src/lib/stores/settings/referrer.svelte.ts similarity index 50% rename from tools/ui/src/lib/stores/settings-referrer.svelte.ts rename to tools/ui/src/lib/stores/settings/referrer.svelte.ts index 297a0d6a45..9679049df9 100644 --- a/tools/ui/src/lib/stores/settings-referrer.svelte.ts +++ b/tools/ui/src/lib/stores/settings/referrer.svelte.ts @@ -1,3 +1,10 @@ +/** + * settingsReferrer - Remembers the settings route to return to after exit + * + * Tracks the last settings section the user was on so the app can return + * there after a fallback exit. Standalone reactive value, no host. + */ + import { SETTINGS_FALLBACK_EXIT_ROUTE } from '$lib/constants'; let _url = $state(SETTINGS_FALLBACK_EXIT_ROUTE); diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.ts index 9f044c83e6..e255b8a43e 100644 --- a/tools/ui/src/lib/stores/tools.svelte.ts +++ b/tools/ui/src/lib/stores/tools.svelte.ts @@ -1,3 +1,12 @@ +/** + * toolsStore - Tool registry and enablement + * + * Owns the server tool listing (with working-directory resolution), built-in + * browser tools, MCP tools and per-tool enablement, exposed as a unified + * tool set for the LLM and the tools UI. Consumed by the agentic loop and + * the chat flows. + */ + import { browser } from '$app/environment'; import { buildBrowserInfoToolDefinition, @@ -18,9 +27,9 @@ import { } from '$lib/enums'; import { ToolsService } from '$lib/services/tools.service'; // direct imports between stores, not via the barrel, to avoid circular deps -import { mcpStore } from '$lib/stores/mcp.svelte'; -import { modelsStore } from '$lib/stores/models.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { mcpStore } from '$lib/stores/mcp/index.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { OpenAIToolDefinition, ToolEntry, ToolGroup } from '$lib/types'; import { buildSandboxToolDefinition } from '$lib/utils'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; @@ -28,273 +37,18 @@ import { SvelteMap, SvelteSet } from 'svelte/reactivity'; /** Stable selection identity for a tool, shared by the disabled set and the permission store */ class ToolsStore { - private _serverTools = $state([]); - private _loading = $state(false); - private _error = $state(null); private _disabledTools = $state(new SvelteSet()); + private _error = $state(null); + private _loading = $state(false); + private _serverHome = $state(undefined); + private _serverTools = $state([]); + private _toolsEndpointUnreachable = $state(false); // server tools that resolve their paths against the working directory, // as declared by the server in its `/tools` listing - private _cwdAwareTools = $state(new SvelteSet()); - private _toolsEndpointUnreachable = $state(false); - private _serverHome = $state(undefined); + private cwdAwareTools = $state(new SvelteSet()); - /** - * Load persisted disabled tools and fetch the builtin tool list. - * Called by initStores() after migrations have run. - */ - initialize(): void { - // browser-only init: skip on SSR to avoid localStorage/fetch side effects - if (!browser) return; - - try { - const stored = localStorage.getItem(DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY); - - if (stored) { - const parsed = JSON.parse(stored); - - if (Array.isArray(parsed)) { - for (const key of parsed) { - if (typeof key === 'string') this._disabledTools.add(key); - } - } - } - } catch (err) { - console.error('[ToolsStore] Failed to load disabled tools from localStorage:', err); - } - - this.fetchServerTools(); - } - - private persistDisabledTools(): void { - try { - localStorage.setItem( - DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY, - JSON.stringify([...this._disabledTools]) - ); - } catch { - // ignore storage errors - } - } - - private toolKey(source: ToolSource, name: string, serverId?: string): string { - switch (source) { - case ToolSource.MCP: - return serverId ? `mcp-${serverId}:${name}` : `mcp:${name}`; - case ToolSource.CUSTOM: - return `custom:${name}`; - case ToolSource.BROWSER: - return `browser:${name}`; - default: - return `server:${name}`; - } - } - - private inferTypeFromDefault(value: unknown): string | undefined { - if (typeof value === 'string') return 'string'; - - if (typeof value === 'boolean') return 'boolean'; - - if (typeof value === 'number') return Number.isInteger(value) ? 'integer' : 'number'; - - if (Array.isArray(value)) return 'array'; - - if (value !== null && typeof value === 'object') return 'object'; - - return undefined; - } - - /** - * Recursively normalize a JSON Schema object: infers `type` from `default` - * for properties / items that omit it, and descends into nested `properties` - * and `items`. Returns a new object -- does not mutate the input. - */ - private normalizeJsonSchema(schema: Record): Record { - if (!schema || typeof schema !== 'object') return schema; - - const normalized: Record = { ...schema }; - - if (normalized.properties && typeof normalized.properties === 'object') { - const props = normalized.properties as Record>; - const normalizedProps: Record> = {}; - - for (const [key, prop] of Object.entries(props)) { - if (!prop || typeof prop !== 'object') { - normalizedProps[key] = prop; - - continue; - } - - const normalizedProp: Record = { ...prop }; - - if (!normalizedProp.type && normalizedProp.default !== undefined) { - const inferred = this.inferTypeFromDefault(normalizedProp.default); - - if (inferred) normalizedProp.type = inferred; - } - - if (normalizedProp.properties) { - Object.assign( - normalizedProp, - this.normalizeJsonSchema(normalizedProp as Record) - ); - } - - if (normalizedProp.items && typeof normalizedProp.items === 'object') { - normalizedProp.items = this.normalizeJsonSchema( - normalizedProp.items as Record - ); - } - - normalizedProps[key] = normalizedProp; - } - normalized.properties = normalizedProps; - } - - return normalized; - } - - private mcpDefinition( - name: string, - description: string | undefined, - schema?: Record - ): OpenAIToolDefinition { - return { - function: { - description, - name, - parameters: schema ?? { properties: {}, required: [], type: JsonSchemaType.OBJECT } - }, - type: ToolCallType.FUNCTION - }; - } - - get serverTools(): OpenAIToolDefinition[] { - return this._serverTools; - } - - get serverHome(): string | null { - return this._serverHome ?? null; - } - - get mcpTools(): OpenAIToolDefinition[] { - return this.mcpEntries().map((e) => e.definition); - } - - get browserTools(): OpenAIToolDefinition[] { - const tools: OpenAIToolDefinition[] = [buildGetDatetimeToolDefinition()]; - - if (settingsStore.config.jsSandboxEnabled) { - tools.push(buildSandboxToolDefinition(!!settingsStore.config.symbolicMathEnabled)); - } - - const readMedia = this.readMediaTool(); - - if (readMedia) tools.push(readMedia); - - // provide browser's get_info tool if server doesn't provide one - if (!this.hasServerTool(BuiltInTool.SERVER_GET_INFO)) { - tools.push(buildBrowserInfoToolDefinition()); - } - - return tools; - } - - private hasServerTool(name: BuiltInTool): boolean { - return this._serverTools.some((def) => def.function.name === name); - } - - /** - * `read_media` runs in the browser on top of the server's `read_file`, so it - * exists only when that tool is served and the active model can perceive the - * bytes. The server cannot make this call - it does not know which model the - * conversation uses. - */ - private readMediaTool(): OpenAIToolDefinition | null { - if (!this.hasServerTool(BuiltInTool.SERVER_READ_FILE)) return null; - - const model = modelsStore.selectedModelName ?? modelsStore.models[0]?.model ?? ''; - - if (!model) return null; - - const vision = modelsStore.modelSupportsVision(model); - const audio = modelsStore.modelSupportsAudio(model); - - if (!vision && !audio) return null; - - return buildReadMediaToolDefinition(vision, audio); - } - - get customTools(): OpenAIToolDefinition[] { - const raw = settingsStore.config.customJson; - - if (!raw || typeof raw !== 'string') return []; - - try { - const parsed = JSON.parse(raw); - - if (!Array.isArray(parsed)) return []; - - return parsed.filter( - (t: unknown): t is OpenAIToolDefinition => - typeof t === 'object' && - t !== null && - 'type' in t && - (t as OpenAIToolDefinition).type === 'function' && - 'function' in t && - typeof (t as OpenAIToolDefinition).function?.name === 'string' - ); - } catch { - return []; - } - } - - /** Normalize MCP tools from live connections when available, fall back to health check data */ - private mcpEntries(): { - serverId: string; - serverName: string; - definition: OpenAIToolDefinition; - }[] { - const out: { serverId: string; serverName: string; definition: OpenAIToolDefinition }[] = []; - const connections = mcpStore.getConnections(); - - if (connections.size > 0) { - for (const [serverId, connection] of connections) { - const serverName = mcpStore.getServerDisplayName(serverId); - - for (const tool of connection.tools) { - const rawSchema = (tool.inputSchema as Record) ?? { - properties: {}, - required: [], - type: JsonSchemaType.OBJECT - }; - - out.push({ - definition: { - function: { - description: tool.description, - name: tool.name, - parameters: this.normalizeJsonSchema(rawSchema) - }, - type: ToolCallType.FUNCTION - }, - serverId, - serverName - }); - } - } - } else { - for (const { serverId, serverName, tools } of this.getMcpToolsFromHealthChecks()) { - for (const tool of tools) { - out.push({ - definition: this.mcpDefinition(tool.name, tool.description), - serverId, - serverName - }); - } - } - } - - return out; + get allToolDefinitions(): OpenAIToolDefinition[] { + return this.allTools.map((t) => t.definition); } /** Canonical flat list of tool entries with source metadata and stable keys, deduped by key */ @@ -353,6 +107,97 @@ class ToolsStore { return entries; } + get browserTools(): OpenAIToolDefinition[] { + const tools: OpenAIToolDefinition[] = [buildGetDatetimeToolDefinition()]; + + if (settingsStore.config.jsSandboxEnabled) { + tools.push(buildSandboxToolDefinition(!!settingsStore.config.symbolicMathEnabled)); + } + + const readMedia = this.readMediaTool(); + + if (readMedia) tools.push(readMedia); + + // provide browser's get_info tool if server doesn't provide one + if (!this.hasServerTool(BuiltInTool.SERVER_GET_INFO)) { + tools.push(buildBrowserInfoToolDefinition()); + } + + return tools; + } + + get customTools(): OpenAIToolDefinition[] { + const raw = settingsStore.config.customJson; + + if (!raw || typeof raw !== 'string') return []; + + try { + const parsed = JSON.parse(raw); + + if (!Array.isArray(parsed)) return []; + + return parsed.filter( + (t: unknown): t is OpenAIToolDefinition => + typeof t === 'object' && + t !== null && + 'type' in t && + (t as OpenAIToolDefinition).type === 'function' && + 'function' in t && + typeof (t as OpenAIToolDefinition).function?.name === 'string' + ); + } catch { + return []; + } + } + + get disabledTools(): SvelteSet { + return this._disabledTools; + } + + get error(): string | null { + return this._error; + } + + /** + * Check if a working directory is worth setting: at least one server tool + * that reads it is both served and left enabled by the user. + */ + get hasEnabledCwdTools(): boolean { + return this._serverTools.some((def) => { + const name = def.function.name; + + return ( + this.cwdAwareTools.has(name) && + !this._disabledTools.has(this.toolKey(ToolSource.SERVER, name)) + ); + }); + } + + /** Check if there are any enabled tools available (server, MCP, or custom) */ + get hasEnabledTools(): boolean { + return this.getEnabledToolsForLLM().length > 0; + } + + get isToolsEndpointUnreachable(): boolean { + return this._toolsEndpointUnreachable; + } + + get loading(): boolean { + return this._loading; + } + + get mcpTools(): OpenAIToolDefinition[] { + return this.mcpEntries().map((e) => e.definition); + } + + get serverHome(): string | null { + return this._serverHome ?? null; + } + + get serverTools(): OpenAIToolDefinition[] { + return this._serverTools; + } + /** Tools grouped by category for tree display, derived from the canonical entries */ get toolGroups(): ToolGroup[] { const groups: ToolGroup[] = []; @@ -382,16 +227,47 @@ class ToolsStore { return groups; } - private groupLabel(entry: ToolEntry): string { - switch (entry.source) { - case ToolSource.MCP: - return entry.serverName ?? ''; - case ToolSource.CUSTOM: - return TOOL_GROUP_LABELS[ToolSource.CUSTOM]; - case ToolSource.BROWSER: - return TOOL_GROUP_LABELS[ToolSource.BROWSER]; - default: - return TOOL_GROUP_LABELS[ToolSource.SERVER]; + /** Enable all tools belonging to a specific MCP server */ + enableAllToolsForServer(serverId: string): void { + const connection = mcpStore.getConnections().get(serverId); + + if (!connection) return; + + for (const tool of connection.tools) { + this._disabledTools.delete(this.toolKey(ToolSource.MCP, tool.name, serverId)); + } + this.persistDisabledTools(); + } + + async fetchServerTools(): Promise { + if (this._loading) return; + + this._loading = true; + this._error = null; + this._toolsEndpointUnreachable = false; + + try { + const toolInfos = await ToolsService.list(); + + this._serverTools = toolInfos.map((info) => info.definition); + this.cwdAwareTools = new SvelteSet( + toolInfos.filter((info) => info.uses_cwd).map((info) => info.tool) + ); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : String(err); + + this._error = errorMessage; + + // 403 from /tools means the server was started without --tools + // TODO: check status code instead of relying on message + if (errorMessage.includes('this feature is disabled')) { + this._toolsEndpointUnreachable = true; + console.info('[ToolsStore] Server tools are disabled on the server'); + } else { + console.error('[ToolsStore] Failed to fetch server tools:', err); + } + } finally { + this._loading = false; } } @@ -430,112 +306,9 @@ class ToolsStore { return result; } - get allToolDefinitions(): OpenAIToolDefinition[] { - return this.allTools.map((t) => t.definition); - } - - get loading(): boolean { - return this._loading; - } - - get error(): string | null { - return this._error; - } - - get isToolsEndpointUnreachable(): boolean { - return this._toolsEndpointUnreachable; - } - - get disabledTools(): SvelteSet { - return this._disabledTools; - } - - isToolEnabled(key: string): boolean { - return !this._disabledTools.has(key); - } - - toggleTool(key: string): void { - if (this._disabledTools.has(key)) { - this._disabledTools.delete(key); - } else { - this._disabledTools.add(key); - } - - this.persistDisabledTools(); - } - - setToolEnabled(key: string, enabled: boolean): void { - if (enabled) { - this._disabledTools.delete(key); - } else { - this._disabledTools.add(key); - } - } - - /** Enable all tools belonging to a specific MCP server */ - enableAllToolsForServer(serverId: string): void { - const connection = mcpStore.getConnections().get(serverId); - - if (!connection) return; - - for (const tool of connection.tools) { - this._disabledTools.delete(this.toolKey(ToolSource.MCP, tool.name, serverId)); - } - this.persistDisabledTools(); - } - - toggleGroup(group: ToolGroup): void { - const allEnabled = group.tools.every((t) => this.isToolEnabled(t.key)); - const target = !allEnabled; - - for (const tool of group.tools) { - if (target) this._disabledTools.delete(tool.key); - else this._disabledTools.add(tool.key); - } - this.persistDisabledTools(); - } - - isGroupFullyEnabled(group: ToolGroup): boolean { - return group.tools.length > 0 && group.tools.every((t) => this.isToolEnabled(t.key)); - } - - /** Get MCP tools from health check data, used when live connections aren't established yet */ - private getMcpToolsFromHealthChecks(): { - serverId: string; - serverName: string; - tools: { name: string; description?: string }[]; - }[] { - const result: ReturnType = []; - - for (const server of mcpStore.getServers()) { - if (!server.enabled) continue; - - const health = mcpStore.getHealthCheckState(server.id); - - if (health.status === HealthCheckStatus.SUCCESS && health.tools.length > 0) { - result.push({ - serverId: server.id, - serverName: mcpStore.getServerLabel(server), - tools: health.tools - }); - } - } - - return result; - } - - /** First canonical entry matching a tool name, runtime tool calls resolve by name */ - private findEntryByName(toolName: string): ToolEntry | null { - for (const entry of this.allTools) { - if (entry.definition.function.name === toolName) return entry; - } - - return null; - } - - /** Determine the source of a tool by its name */ - getToolSource(toolName: string): ToolSource | null { - return this.findEntryByName(toolName)?.source ?? null; + /** Permission key for a tool name, identical to the selection key */ + getPermissionKey(toolName: string): string | null { + return this.findEntryByName(toolName)?.key ?? null; } /** Get the display label for the server that owns a given tool */ @@ -555,61 +328,44 @@ class ToolsStore { return ''; } - /** Permission key for a tool name, identical to the selection key */ - getPermissionKey(toolName: string): string | null { - return this.findEntryByName(toolName)?.key ?? null; - } - - /** Check if there are any enabled tools available (server, MCP, or custom) */ - get hasEnabledTools(): boolean { - return this.getEnabledToolsForLLM().length > 0; + /** Determine the source of a tool by its name */ + getToolSource(toolName: string): ToolSource | null { + return this.findEntryByName(toolName)?.source ?? null; } /** - * Check if a working directory is worth setting: at least one server tool - * that reads it is both served and left enabled by the user. + * Load persisted disabled tools and fetch the builtin tool list. + * Called by initStores() after migrations have run. */ - get hasEnabledCwdTools(): boolean { - return this._serverTools.some((def) => { - const name = def.function.name; - - return ( - this._cwdAwareTools.has(name) && - !this._disabledTools.has(this.toolKey(ToolSource.SERVER, name)) - ); - }); - } - - async fetchServerTools(): Promise { - if (this._loading) return; - - this._loading = true; - this._error = null; - this._toolsEndpointUnreachable = false; + initialize(): void { + // browser-only init: skip on SSR to avoid localStorage/fetch side effects + if (!browser) return; try { - const toolInfos = await ToolsService.list(); + const stored = localStorage.getItem(DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY); - this._serverTools = toolInfos.map((info) => info.definition); - this._cwdAwareTools = new SvelteSet( - toolInfos.filter((info) => info.uses_cwd).map((info) => info.tool) - ); - } catch (err) { - const errorMessage = err instanceof Error ? err.message : String(err); + if (stored) { + const parsed = JSON.parse(stored); - this._error = errorMessage; - - // 403 from /tools means the server was started without --tools - // TODO: check status code instead of relying on message - if (errorMessage.includes('this feature is disabled')) { - this._toolsEndpointUnreachable = true; - console.info('[ToolsStore] Server tools are disabled on the server'); - } else { - console.error('[ToolsStore] Failed to fetch server tools:', err); + if (Array.isArray(parsed)) { + for (const key of parsed) { + if (typeof key === 'string') this._disabledTools.add(key); + } + } } - } finally { - this._loading = false; + } catch (err) { + console.error('[ToolsStore] Failed to load disabled tools from localStorage:', err); } + + this.fetchServerTools(); + } + + isGroupFullyEnabled(group: ToolGroup): boolean { + return group.tools.length > 0 && group.tools.every((t) => this.isToolEnabled(t.key)); + } + + isToolEnabled(key: string): boolean { + return !this._disabledTools.has(key); } /** @@ -637,6 +393,259 @@ class ToolsStore { return this._serverHome; } + + setToolEnabled(key: string, enabled: boolean): void { + if (enabled) { + this._disabledTools.delete(key); + } else { + this._disabledTools.add(key); + } + } + + toggleGroup(group: ToolGroup): void { + const allEnabled = group.tools.every((t) => this.isToolEnabled(t.key)); + const target = !allEnabled; + + for (const tool of group.tools) { + if (target) this._disabledTools.delete(tool.key); + else this._disabledTools.add(tool.key); + } + this.persistDisabledTools(); + } + + toggleTool(key: string): void { + if (this._disabledTools.has(key)) { + this._disabledTools.delete(key); + } else { + this._disabledTools.add(key); + } + + this.persistDisabledTools(); + } + + /** First canonical entry matching a tool name, runtime tool calls resolve by name */ + private findEntryByName(toolName: string): ToolEntry | null { + for (const entry of this.allTools) { + if (entry.definition.function.name === toolName) return entry; + } + + return null; + } + + /** Get MCP tools from health check data, used when live connections aren't established yet */ + private getMcpToolsFromHealthChecks(): { + serverId: string; + serverName: string; + tools: { name: string; description?: string }[]; + }[] { + const result: ReturnType = []; + + for (const server of mcpStore.getServers()) { + if (!server.enabled) continue; + + const health = mcpStore.getHealthCheckState(server.id); + + if (health.status === HealthCheckStatus.SUCCESS && health.tools.length > 0) { + result.push({ + serverId: server.id, + serverName: mcpStore.getServerLabel(server), + tools: health.tools + }); + } + } + + return result; + } + + private groupLabel(entry: ToolEntry): string { + switch (entry.source) { + case ToolSource.MCP: + return entry.serverName ?? ''; + case ToolSource.CUSTOM: + return TOOL_GROUP_LABELS[ToolSource.CUSTOM]; + case ToolSource.BROWSER: + return TOOL_GROUP_LABELS[ToolSource.BROWSER]; + default: + return TOOL_GROUP_LABELS[ToolSource.SERVER]; + } + } + + private hasServerTool(name: BuiltInTool): boolean { + return this._serverTools.some((def) => def.function.name === name); + } + + private inferTypeFromDefault(value: unknown): string | undefined { + if (typeof value === 'string') return 'string'; + + if (typeof value === 'boolean') return 'boolean'; + + if (typeof value === 'number') return Number.isInteger(value) ? 'integer' : 'number'; + + if (Array.isArray(value)) return 'array'; + + if (value !== null && typeof value === 'object') return 'object'; + + return undefined; + } + + private mcpDefinition( + name: string, + description: string | undefined, + schema?: Record + ): OpenAIToolDefinition { + return { + function: { + description, + name, + parameters: schema ?? { properties: {}, required: [], type: JsonSchemaType.OBJECT } + }, + type: ToolCallType.FUNCTION + }; + } + + /** Normalize MCP tools from live connections when available, fall back to health check data */ + private mcpEntries(): { + serverId: string; + serverName: string; + definition: OpenAIToolDefinition; + }[] { + const out: { serverId: string; serverName: string; definition: OpenAIToolDefinition }[] = []; + const connections = mcpStore.getConnections(); + + if (connections.size > 0) { + for (const [serverId, connection] of connections) { + const serverName = mcpStore.getServerDisplayName(serverId); + + for (const tool of connection.tools) { + const rawSchema = (tool.inputSchema as Record) ?? { + properties: {}, + required: [], + type: JsonSchemaType.OBJECT + }; + + out.push({ + definition: { + function: { + description: tool.description, + name: tool.name, + parameters: this.normalizeJsonSchema(rawSchema) + }, + type: ToolCallType.FUNCTION + }, + serverId, + serverName + }); + } + } + } else { + for (const { serverId, serverName, tools } of this.getMcpToolsFromHealthChecks()) { + for (const tool of tools) { + out.push({ + definition: this.mcpDefinition(tool.name, tool.description), + serverId, + serverName + }); + } + } + } + + return out; + } + + /** + * Recursively normalize a JSON Schema object: infers `type` from `default` + * for properties / items that omit it, and descends into nested `properties` + * and `items`. Returns a new object -- does not mutate the input. + */ + private normalizeJsonSchema(schema: Record): Record { + if (!schema || typeof schema !== 'object') return schema; + + const normalized: Record = { ...schema }; + + if (normalized.properties && typeof normalized.properties === 'object') { + const props = normalized.properties as Record>; + const normalizedProps: Record> = {}; + + for (const [key, prop] of Object.entries(props)) { + if (!prop || typeof prop !== 'object') { + normalizedProps[key] = prop; + + continue; + } + + const normalizedProp: Record = { ...prop }; + + if (!normalizedProp.type && normalizedProp.default !== undefined) { + const inferred = this.inferTypeFromDefault(normalizedProp.default); + + if (inferred) normalizedProp.type = inferred; + } + + if (normalizedProp.properties) { + Object.assign( + normalizedProp, + this.normalizeJsonSchema(normalizedProp as Record) + ); + } + + if (normalizedProp.items && typeof normalizedProp.items === 'object') { + normalizedProp.items = this.normalizeJsonSchema( + normalizedProp.items as Record + ); + } + + normalizedProps[key] = normalizedProp; + } + normalized.properties = normalizedProps; + } + + return normalized; + } + + private persistDisabledTools(): void { + try { + localStorage.setItem( + DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY, + JSON.stringify([...this._disabledTools]) + ); + } catch { + // ignore storage errors + } + } + + /** + * `read_media` runs in the browser on top of the server's `read_file`, so it + * exists only when that tool is served and the active model can perceive the + * bytes. The server cannot make this call - it does not know which model the + * conversation uses. + */ + private readMediaTool(): OpenAIToolDefinition | null { + if (!this.hasServerTool(BuiltInTool.SERVER_READ_FILE)) return null; + + const model = modelsStore.selectedModelName ?? modelsStore.models[0]?.model ?? ''; + + if (!model) return null; + + const vision = modelsStore.props.modelSupportsVision(model); + const audio = modelsStore.props.modelSupportsAudio(model); + + if (!vision && !audio) return null; + + return buildReadMediaToolDefinition(vision, audio); + } + + private toolKey(source: ToolSource, name: string, serverId?: string): string { + switch (source) { + case ToolSource.MCP: + return serverId ? `mcp-${serverId}:${name}` : `mcp:${name}`; + case ToolSource.CUSTOM: + return `custom:${name}`; + case ToolSource.BROWSER: + return `browser:${name}`; + default: + return `server:${name}`; + } + } } export const toolsStore = new ToolsStore(); diff --git a/tools/ui/src/lib/types/agentic.d.ts b/tools/ui/src/lib/types/agentic.d.ts index e7c6d34e1d..1a604476bf 100644 --- a/tools/ui/src/lib/types/agentic.d.ts +++ b/tools/ui/src/lib/types/agentic.d.ts @@ -205,7 +205,7 @@ export interface AgenticSection { /** ID of the model-side tool call (matches tool_calls[i].id). Lets * downstream consumers correlate a section with the agentic loop's * currently-executing tool, e.g. to drive live-streaming UI state - * by matching against agenticStore.executingToolCallId. */ + * by matching against agenticStore.getExecutingToolCallId. */ toolCallId?: string; wasInterrupted?: boolean; } diff --git a/tools/ui/src/lib/utils/api-fetch.ts b/tools/ui/src/lib/utils/api-fetch.ts index 65e1129def..2059200049 100644 --- a/tools/ui/src/lib/utils/api-fetch.ts +++ b/tools/ui/src/lib/utils/api-fetch.ts @@ -1,7 +1,6 @@ import { getAuthHeaders, getJsonHeaders } from './api-headers'; import { base } from '$app/paths'; -import { ERROR_MESSAGES, HTTP_CODE_TO_STRING } from '$lib/constants'; -import { UrlProtocol } from '$lib/enums'; +import { API_ABSOLUTE_URL_PROTOCOLS, ERROR_MESSAGES, HTTP_CODE_TO_STRING } from '$lib/constants'; /** * API Fetch Utilities @@ -63,10 +62,8 @@ export async function apiFetch(path: string, options: ApiFetchOptions = {}): const { authOnly = false, headers: customHeaders, ...fetchOptions } = options; const baseHeaders = authOnly ? getAuthHeaders() : getJsonHeaders(); const headers = { ...baseHeaders, ...customHeaders }; - const url = - path.startsWith(UrlProtocol.HTTP) || path.startsWith(UrlProtocol.HTTPS) - ? path - : `${base}${path}`; + // absolute URLs with an allowed protocol pass through untouched; relative paths get the base prefix + const url = API_ABSOLUTE_URL_PROTOCOLS.some((p) => path.startsWith(p)) ? path : `${base}${path}`; let response; @@ -117,28 +114,7 @@ export async function apiFetchWithParams( } } - const { authOnly = false, headers: customHeaders, ...fetchOptions } = options; - const baseHeaders = authOnly ? getAuthHeaders() : getJsonHeaders(); - const headers = { ...baseHeaders, ...customHeaders }; - - let response; - - try { - response = await fetch(url.toString(), { - ...fetchOptions, - headers - }); - } catch (e) { - throw new Error(beautifyNetworkError(e)); - } - - if (!response.ok) { - const errorMessage = await parseErrorMessage(response); - - throw new ApiError(errorMessage, response.status); - } - - return response.json() as Promise; + return apiFetch(url.toString(), options); } /** diff --git a/tools/ui/src/lib/utils/api-headers.ts b/tools/ui/src/lib/utils/api-headers.ts index 4b2b19d442..49d56d0619 100644 --- a/tools/ui/src/lib/utils/api-headers.ts +++ b/tools/ui/src/lib/utils/api-headers.ts @@ -1,7 +1,7 @@ import { redactValue } from './redact'; import { CORS_PROXY, HEADERS } from '$lib/constants'; import { MimeTypeApplication } from '$lib/enums'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; /** * Get authorization headers for API requests diff --git a/tools/ui/src/lib/utils/api-key-validation.ts b/tools/ui/src/lib/utils/api-key-validation.ts index 8cde154fd2..187199afc2 100644 --- a/tools/ui/src/lib/utils/api-key-validation.ts +++ b/tools/ui/src/lib/utils/api-key-validation.ts @@ -3,7 +3,7 @@ import { browser } from '$app/environment'; import { base } from '$app/paths'; import { HEADERS } from '$lib/constants'; import { MimeTypeApplication } from '$lib/enums'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; /** * Validates API key by making a request to the server props endpoint diff --git a/tools/ui/src/lib/utils/audio-recording.ts b/tools/ui/src/lib/utils/audio-recording.ts index 1241d6e055..4cfe173784 100644 --- a/tools/ui/src/lib/utils/audio-recording.ts +++ b/tools/ui/src/lib/utils/audio-recording.ts @@ -14,10 +14,37 @@ import { MimeTypeAudio } from '$lib/enums'; * - Proper cleanup and resource management */ export class AudioRecorder { - private mediaRecorder: MediaRecorder | null = null; private audioChunks: Blob[] = []; - private stream: MediaStream | null = null; + private mediaRecorder: MediaRecorder | null = null; private recordingState: boolean = false; + private stream: MediaStream | null = null; + + cancelRecording(): void { + const recorder = this.mediaRecorder; + const stream = this.stream; + + this.mediaRecorder = null; + this.audioChunks = []; + this.stream = null; + this.recordingState = false; + + if (recorder && recorder.state !== 'inactive') { + // Drop the original handlers so the pending stop event does not touch the instance + recorder.onstop = null; + recorder.onerror = null; + recorder.stop(); + } + + if (stream) { + for (const track of stream.getTracks()) { + track.stop(); + } + } + } + + isRecording(): boolean { + return this.recordingState; + } async startRecording(): Promise { try { @@ -90,33 +117,6 @@ export class AudioRecorder { }); } - isRecording(): boolean { - return this.recordingState; - } - - cancelRecording(): void { - const recorder = this.mediaRecorder; - const stream = this.stream; - - this.mediaRecorder = null; - this.audioChunks = []; - this.stream = null; - this.recordingState = false; - - if (recorder && recorder.state !== 'inactive') { - // Drop the original handlers so the pending stop event does not touch the instance - recorder.onstop = null; - recorder.onerror = null; - recorder.stop(); - } - - if (stream) { - for (const track of stream.getTracks()) { - track.stop(); - } - } - } - private initializeRecorder(stream: MediaStream): void { const options: MediaRecorderOptions = {}; diff --git a/tools/ui/src/lib/utils/cache-ttl.ts b/tools/ui/src/lib/utils/cache-ttl.ts index bb0100755b..bec40989c4 100644 --- a/tools/ui/src/lib/utils/cache-ttl.ts +++ b/tools/ui/src/lib/utils/cache-ttl.ts @@ -31,9 +31,29 @@ interface CacheEntry { export class TTLCache { private cache = new Map>(); - private readonly ttlMs: number; private readonly maxEntries: number; private readonly onEvict?: (key: string, value: unknown) => void; + private readonly ttlMs: number; + + /** + * Get the number of entries (including potentially expired ones). + */ + get size(): number { + return this.cache.size; + } + + /** + * Clear all entries from cache. + */ + clear(): void { + if (this.onEvict) { + for (const [key, entry] of this.cache) { + this.onEvict(key, entry.value); + } + } + + this.cache.clear(); + } constructor(options: TTLCacheOptions = {}) { this.ttlMs = options.ttlMs ?? CACHE.DEFAULT_TTL_MS; @@ -41,6 +61,19 @@ export class TTLCache { this.onEvict = options.onEvict; } + /** + * Delete a specific key from cache. + */ + delete(key: K): boolean { + const entry = this.cache.get(key); + + if (entry && this.onEvict) { + this.onEvict(key, entry.value); + } + + return this.cache.delete(key); + } + /** * Get a value from cache. Returns null if expired or not found. */ @@ -61,25 +94,6 @@ export class TTLCache { return entry.value; } - /** - * Set a value in cache with TTL. - */ - set(key: K, value: V, customTtlMs?: number): void { - // Evict oldest entries if at capacity - if (this.cache.size >= this.maxEntries && !this.cache.has(key)) { - this.evictOldest(); - } - - const ttl = customTtlMs ?? this.ttlMs; - const now = Date.now(); - - this.cache.set(key, { - expiresAt: now + ttl, - lastAccessed: now, - value - }); - } - /** * Check if key exists and is not expired. */ @@ -98,36 +112,19 @@ export class TTLCache { } /** - * Delete a specific key from cache. + * Get all valid (non-expired) keys. */ - delete(key: K): boolean { - const entry = this.cache.get(key); + keys(): K[] { + const now = Date.now(); + const validKeys: K[] = []; - if (entry && this.onEvict) { - this.onEvict(key, entry.value); - } - - return this.cache.delete(key); - } - - /** - * Clear all entries from cache. - */ - clear(): void { - if (this.onEvict) { - for (const [key, entry] of this.cache) { - this.onEvict(key, entry.value); + for (const [key, entry] of this.cache) { + if (now <= entry.expiresAt) { + validKeys.push(key); } } - this.cache.clear(); - } - - /** - * Get the number of entries (including potentially expired ones). - */ - get size(): number { - return this.cache.size; + return validKeys; } /** @@ -150,38 +147,22 @@ export class TTLCache { } /** - * Get all valid (non-expired) keys. + * Set a value in cache with TTL. */ - keys(): K[] { + set(key: K, value: V, customTtlMs?: number): void { + // Evict oldest entries if at capacity + if (this.cache.size >= this.maxEntries && !this.cache.has(key)) { + this.evictOldest(); + } + + const ttl = customTtlMs ?? this.ttlMs; const now = Date.now(); - const validKeys: K[] = []; - for (const [key, entry] of this.cache) { - if (now <= entry.expiresAt) { - validKeys.push(key); - } - } - - return validKeys; - } - - /** - * Evict the oldest (least recently accessed) entry. - */ - private evictOldest(): void { - let oldestKey: K | null = null; - let oldestTime = Infinity; - - for (const [key, entry] of this.cache) { - if (entry.lastAccessed < oldestTime) { - oldestTime = entry.lastAccessed; - oldestKey = key; - } - } - - if (oldestKey !== null) { - this.delete(oldestKey); - } + this.cache.set(key, { + expiresAt: now + ttl, + lastAccessed: now, + value + }); } /** @@ -205,6 +186,25 @@ export class TTLCache { return true; } + + /** + * Evict the oldest (least recently accessed) entry. + */ + private evictOldest(): void { + let oldestKey: K | null = null; + let oldestTime = Infinity; + + for (const [key, entry] of this.cache) { + if (entry.lastAccessed < oldestTime) { + oldestTime = entry.lastAccessed; + oldestKey = key; + } + } + + if (oldestKey !== null) { + this.delete(oldestKey); + } + } } /** @@ -213,14 +213,26 @@ export class TTLCache { */ export class ReactiveTTLMap { private entries = $state>>(new Map()); - private readonly ttlMs: number; private readonly maxEntries: number; + private readonly ttlMs: number; + + get size(): number { + return this.entries.size; + } + + clear(): void { + this.entries.clear(); + } constructor(options: TTLCacheOptions = {}) { this.ttlMs = options.ttlMs ?? CACHE.DEFAULT_TTL_MS; this.maxEntries = options.maxEntries ?? CACHE.DEFAULT_MAX_ENTRIES; } + delete(key: K): boolean { + return this.entries.delete(key); + } + get(key: K): V | null { const entry = this.entries.get(key); @@ -237,21 +249,6 @@ export class ReactiveTTLMap { return entry.value; } - set(key: K, value: V, customTtlMs?: number): void { - if (this.entries.size >= this.maxEntries && !this.entries.has(key)) { - this.evictOldest(); - } - - const ttl = customTtlMs ?? this.ttlMs; - const now = Date.now(); - - this.entries.set(key, { - expiresAt: now + ttl, - lastAccessed: now, - value - }); - } - has(key: K): boolean { const entry = this.entries.get(key); @@ -266,18 +263,6 @@ export class ReactiveTTLMap { return true; } - delete(key: K): boolean { - return this.entries.delete(key); - } - - clear(): void { - this.entries.clear(); - } - - get size(): number { - return this.entries.size; - } - prune(): number { const now = Date.now(); @@ -293,6 +278,21 @@ export class ReactiveTTLMap { return pruned; } + set(key: K, value: V, customTtlMs?: number): void { + if (this.entries.size >= this.maxEntries && !this.entries.has(key)) { + this.evictOldest(); + } + + const ttl = customTtlMs ?? this.ttlMs; + const now = Date.now(); + + this.entries.set(key, { + expiresAt: now + ttl, + lastAccessed: now, + value + }); + } + private evictOldest(): void { let oldestKey: K | null = null; let oldestTime = Infinity; diff --git a/tools/ui/src/lib/utils/chat-form-input-rich-tokenizer.ts b/tools/ui/src/lib/utils/chat-form-input-rich-tokenizer.ts index c09afd018b..626b10b29b 100644 --- a/tools/ui/src/lib/utils/chat-form-input-rich-tokenizer.ts +++ b/tools/ui/src/lib/utils/chat-form-input-rich-tokenizer.ts @@ -38,7 +38,7 @@ import { SETTINGS_KEYS } from '$lib/constants'; import { BooleanString, ChatFormInputRichTokenKind } from '$lib/enums'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { toolsStore } from '$lib/stores/tools.svelte'; import type { ChatFormInputRichToken } from '$lib/types/chat-form-input-rich'; diff --git a/tools/ui/src/lib/utils/convert-files-to-extra.ts b/tools/ui/src/lib/utils/convert-files-to-extra.ts index e348f25fe9..735e91c44a 100644 --- a/tools/ui/src/lib/utils/convert-files-to-extra.ts +++ b/tools/ui/src/lib/utils/convert-files-to-extra.ts @@ -4,8 +4,8 @@ import { isLikelyTextFile, readFileAsText } from './text-files'; import { isWebpMimeType, webpBase64UrlToPngDataURL } from './webp-to-png'; import { SETTINGS_KEYS } from '$lib/constants'; import { AttachmentType, FileTypeCategory, SpecialFileType } from '$lib/enums'; -import { modelsStore } from '$lib/stores/models.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { ChatUploadedFile, DatabaseMessageExtra, FileProcessingResult } from '$lib/types'; import { getFileTypeCategory } from '$lib/utils'; import { toast } from 'svelte-sonner'; @@ -112,7 +112,7 @@ export async function parseFilesToMessageExtras( const currentConfig = settingsStore.config; // Use per-model vision check for router mode const hasVisionSupport = activeModelId - ? modelsStore.modelSupportsVision(activeModelId) + ? modelsStore.props.modelSupportsVision(activeModelId) : false; // Force PDF-to-text for non-vision models diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.ts index 239f9f5724..079cdc871c 100644 --- a/tools/ui/src/lib/utils/index.ts +++ b/tools/ui/src/lib/utils/index.ts @@ -130,7 +130,7 @@ export { getImageErrorFallbackHtml } from './image-error-fallback'; // SSE-with-JSON stream iterator (used by server tool streaming, decoupled // from chat.service.ts which embeds its own SSE parser for resume support) -export { parseSseJsonStream } from './sse'; +export { extractSseDataPayload, parseSseJsonStream, splitSseRecords } from './sse'; // Stream session identity (conversation-id based) export { streamIdentity } from './stream-identity'; @@ -150,7 +150,10 @@ export { getResourceIcon, getResourceTextContent, getResourceBlobContent, - downloadResourceContent + downloadResourceContent, + getMcpIconUrl, + getMcpServerFaviconFallback, + getMcpServerLabel } from './mcp'; // URI Template utilities diff --git a/tools/ui/src/lib/utils/mcp.ts b/tools/ui/src/lib/utils/mcp.ts index 61d5f8a9a5..c60a59e80e 100644 --- a/tools/ui/src/lib/utils/mcp.ts +++ b/tools/ui/src/lib/utils/mcp.ts @@ -1,3 +1,4 @@ +import { extractRootDomain } from './url'; import { AlertTriangle, Code, @@ -12,8 +13,10 @@ import { CODE_FILE_EXTENSION_REGEX, DEFAULT_RESOURCE_FILENAME, DISPLAY_NAME_SEPARATOR_REGEX, + EXPECTED_THEMED_ICON_PAIR_COUNT, FILE_EXTENSION_REGEX, IMAGE_FILE_EXTENSION_REGEX, + MCP_ALLOWED_ICON_MIME_TYPES, MCP_SERVER_ID_PREFIX, MCP_SSE, MIME_TYPE_PREFIXES, @@ -24,8 +27,22 @@ import { TEXT_FILE_EXTENSION_REGEX, URI_PATTERNS } from '$lib/constants'; -import { MCPLogLevel, MCPTransportType, MimeTypeText, UrlProtocol } from '$lib/enums'; -import type { MCPResourceContent, MCPResourceInfo, MCPServerSettingsEntry } from '$lib/types'; +import { + ColorMode, + HealthCheckStatus, + MCPLogLevel, + MCPTransportType, + MimeTypeText, + UrlProtocol +} from '$lib/enums'; +import type { + HealthCheckState, + MCPResourceContent, + MCPResourceIcon, + MCPResourceInfo, + MCPServerDisplayInfo, + MCPServerSettingsEntry +} from '$lib/types'; import type { MimeTypeUnion } from '$lib/types/common'; import type { Component } from 'svelte'; @@ -316,3 +333,132 @@ export function downloadResourceContent( document.body.removeChild(a); URL.revokeObjectURL(url); } + +/** + * Validates that an icon URI uses a safe scheme (https: or data:). + */ +function isValidMcpIconUri(src: string): boolean { + try { + if (src.startsWith(UrlProtocol.DATA)) return true; + + const url = new URL(src); + + return url.protocol === UrlProtocol.HTTPS; + } catch { + return false; + } +} + +/** + * Selects the best icon URL from an MCP icons array. + * Follows security guidelines from the MCP specification: + * - Only allows https: and data: URIs + * - Filters to supported MIME types + * + * Selection priority: + * 1. Icon matching the current color scheme (dark/light) + * 2. Universal icon (no theme specified); if exactly 2, assumes [0]=light, [1]=dark + * 3. First valid icon as last resort + */ +export function getMcpIconUrl(icons: MCPResourceIcon[] | undefined, isDark = false): string | null { + if (!icons?.length) return null; + + const validIcons = icons.filter((icon) => { + if (!icon.src || !isValidMcpIconUri(icon.src)) return false; + + if (icon.mimeType && !MCP_ALLOWED_ICON_MIME_TYPES.has(icon.mimeType)) return false; + + return true; + }); + + if (validIcons.length === 0) return null; + + const preferredTheme = isDark ? ColorMode.DARK : ColorMode.LIGHT; + // 1. Prefer icon explicitly matching the current color scheme + const themedIcon = validIcons.find((icon) => icon.theme === preferredTheme); + + if (themedIcon) return themedIcon.src; + + // 2. Handle universal icons (no theme specified) + const universalIcons = validIcons.filter((icon) => !icon.theme); + + if (universalIcons.length === EXPECTED_THEMED_ICON_PAIR_COUNT) { + // Heuristic: two theme-less icons → assume [0] = light, [1] = dark + return universalIcons[isDark ? 1 : 0].src; + } + + if (universalIcons.length > 0) { + return universalIcons[0].src; + } + + // 3. Last resort: use opposite-theme icon + return validIcons[0].src; +} + +/** + * Construct a fallback favicon URL from the MCP server URL. + * e.g. https://mcp.example.com/sse -> https://example.com/favicon.ico + */ +export function getMcpServerFaviconFallback(serverUrl: string): string | null { + try { + const url = new URL(serverUrl); + const rootDomain = extractRootDomain(url); + + if (!rootDomain) return null; + + const origin = `${url.protocol}//${rootDomain}`; + const candidates = ['favicon.ico', 'favicon.png']; + + for (const path of candidates) { + const faviconUrl = `${origin}/${path}`; + + if (isValidMcpIconUri(faviconUrl)) { + return faviconUrl; + } + } + } catch { + // Invalid URL, return null + } + + return null; +} + +/** + * Resolves the raw label for a server: user-defined display name first, + * then server-reported title or name when the health check succeeded, + * then the configured name (admin baseline or legacy data), then URL. + */ +function getMcpServerBaseLabel( + server: MCPServerDisplayInfo, + healthState?: HealthCheckState +): string { + if (server.displayName) return server.displayName; + + if (healthState?.status === HealthCheckStatus.SUCCESS) + return ( + healthState.serverInfo?.title || healthState.serverInfo?.name || server.name || server.url + ); + + return server.name || server.url; +} + +/** + * Returns the display label for a server, suffixed with a positional + * counter when several configured servers resolve to the same base label + * (e.g. two endpoints of the same host reporting an identical name). + * Numbering follows config order, so it is stable across renders. + */ +export function getMcpServerLabel( + server: MCPServerDisplayInfo, + servers: MCPServerDisplayInfo[], + healthChecks: Record +): string { + const label = getMcpServerBaseLabel(server, healthChecks[server.id]); + const twins = servers.filter((s) => getMcpServerBaseLabel(s, healthChecks[s.id]) === label); + + if (twins.length < 2) return label; + + const position = twins.findIndex((s) => s.id === server.id); + + return position < 0 ? label : `${label} (${position + 1})`; +} diff --git a/tools/ui/src/lib/utils/process-uploaded-files.ts b/tools/ui/src/lib/utils/process-uploaded-files.ts index 49bdd2412f..e71371345c 100644 --- a/tools/ui/src/lib/utils/process-uploaded-files.ts +++ b/tools/ui/src/lib/utils/process-uploaded-files.ts @@ -4,8 +4,8 @@ import { isSvgMimeType, svgBase64UrlToPngDataURL } from './svg-to-png'; import { isWebpMimeType, webpBase64UrlToPngDataURL } from './webp-to-png'; import { SETTINGS_KEYS } from '$lib/constants'; import { FileTypeCategory } from '$lib/enums'; -import { modelsStore } from '$lib/stores/models.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { getFileTypeCategory } from '$lib/utils'; import { toast } from 'svelte-sonner'; @@ -108,7 +108,7 @@ export async function processFilesToChatUploaded( // Show suggestion toast if vision model is available but PDF as image is disabled const hasVisionSupport = activeModelId - ? modelsStore.modelSupportsVision(activeModelId) + ? modelsStore.props.modelSupportsVision(activeModelId) : false; const currentConfig = settingsStore.config; diff --git a/tools/ui/src/lib/utils/source-history.ts b/tools/ui/src/lib/utils/source-history.ts index 32995ae034..6228ae7e49 100644 --- a/tools/ui/src/lib/utils/source-history.ts +++ b/tools/ui/src/lib/utils/source-history.ts @@ -12,9 +12,9 @@ export interface SourceHistoryEntry { } export class SourceHistory { - private undoStack: SourceHistoryEntry[] = []; - private redoStack: SourceHistoryEntry[] = []; private lastPush = 0; + private redoStack: SourceHistoryEntry[] = []; + private undoStack: SourceHistoryEntry[] = []; constructor( private limit = 100, @@ -32,17 +32,6 @@ export class SourceHistory { this.redoStack = []; } - undo(current: SourceHistoryEntry): SourceHistoryEntry | null { - const entry = this.undoStack.pop(); - - if (!entry) return null; - - this.redoStack.push(current); - this.lastPush = 0; // the next edit after an undo starts a new group - - return entry; - } - redo(current: SourceHistoryEntry): SourceHistoryEntry | null { const entry = this.redoStack.pop(); @@ -53,4 +42,15 @@ export class SourceHistory { return entry; } + + undo(current: SourceHistoryEntry): SourceHistoryEntry | null { + const entry = this.undoStack.pop(); + + if (!entry) return null; + + this.redoStack.push(current); + this.lastPush = 0; // the next edit after an undo starts a new group + + return entry; + } } diff --git a/tools/ui/src/lib/utils/sse.ts b/tools/ui/src/lib/utils/sse.ts index 41d9a1152a..c984e77ee6 100644 --- a/tools/ui/src/lib/utils/sse.ts +++ b/tools/ui/src/lib/utils/sse.ts @@ -25,6 +25,30 @@ export interface SseJsonEvent { data: T; } +/** + * Splits a raw SSE byte buffer into complete records on the blank-line + * boundary, returning the leftover partial record separately. Shared by the + * record-based consumers (parseSseJsonStream, models.service). + */ +export function splitSseRecords(buffer: string): { records: string[]; rest: string } { + const parts = buffer.split(SSE_RECORD_SEPARATOR); + + return { records: parts.slice(0, -1), rest: parts[parts.length - 1] ?? '' }; +} + +/** + * Extracts the joined `data:` payload from one SSE record (the data lines + * concatenated with a newline), or an empty string when the record carries + * no data lines. Used by models.service to parse status envelopes. + */ +export function extractSseDataPayload(record: string): string { + return record + .split(SSE_LINE_SEPARATOR) + .filter((line) => line.startsWith(SSE_DATA_PREFIX)) + .map((line) => line.slice(SSE_DATA_PREFIX.length).trim()) + .join(SSE_LINE_SEPARATOR); +} + export async function* parseSseJsonStream( response: Response, signal?: AbortSignal @@ -46,9 +70,9 @@ export async function* parseSseJsonStream( if (done) break; buffer += decoder.decode(value, { stream: true }); - const records = buffer.split(SSE_RECORD_SEPARATOR); + const { records, rest } = splitSseRecords(buffer); - buffer = records.pop() ?? ''; + buffer = rest; for (const record of records) { if (!record) continue; diff --git a/tools/ui/src/routes/(chat)/+page.svelte b/tools/ui/src/routes/(chat)/+page.svelte index 224d264c43..de8574e352 100644 --- a/tools/ui/src/routes/(chat)/+page.svelte +++ b/tools/ui/src/routes/(chat)/+page.svelte @@ -47,8 +47,8 @@ serverStore.isRouterMode && !modelsStore.isModelLoaded(model.id) ) { - modelsStore - .loadModel(model.id) + modelsStore.status + .load(model.id) .catch((error) => console.error('Failed to load model:', error)); } } catch (error) { @@ -77,7 +77,7 @@ onMount(async () => { if (!conversationsStore.isInitialized) { - await conversationsStore.init(); + await conversationsStore.initialize(); } conversationsStore.clearActiveConversation(); diff --git a/tools/ui/src/routes/+layout.svelte b/tools/ui/src/routes/+layout.svelte index 8314cd2a2d..f87bbe26a2 100644 --- a/tools/ui/src/routes/+layout.svelte +++ b/tools/ui/src/routes/+layout.svelte @@ -216,11 +216,11 @@ if (!serverStore.isRouterMode) return; untrack(() => { - modelsStore.subscribeStatus(); + modelsStore.status.subscribe(); }); return () => { - modelsStore.unsubscribeStatus(); + modelsStore.status.unsubscribe(); }; }); diff --git a/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts b/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts index 0b06d57a5b..b4d6df4538 100644 --- a/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts +++ b/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts @@ -14,7 +14,7 @@ import { perfState } from './components/agentic-perf-state.svelte'; import AgenticPerfWrapper from './components/AgenticPerfWrapper.svelte'; import ChatMessagesPerfWrapper from './components/ChatMessagesPerfWrapper.svelte'; import { MessageRole } from '$lib/enums'; -import { conversationsStore } from '$lib/stores/conversations.svelte'; +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; import type { DatabaseMessage } from '$lib/types'; import { tick } from 'svelte'; import { describe, it } from 'vitest'; diff --git a/tools/ui/tests/client/apikey-splash.svelte.test.ts b/tools/ui/tests/client/apikey-splash.svelte.test.ts index bad7f6ccb0..b2705dd8ca 100644 --- a/tools/ui/tests/client/apikey-splash.svelte.test.ts +++ b/tools/ui/tests/client/apikey-splash.svelte.test.ts @@ -1,5 +1,5 @@ import { CONFIG_LOCALSTORAGE_KEY } from '$lib/constants'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { validateApiKey } from '$lib/utils/api-key-validation'; import { beforeEach, describe, expect, it } from 'vitest'; diff --git a/tools/ui/tests/client/chat-form-enter-code-block.svelte.test.ts b/tools/ui/tests/client/chat-form-enter-code-block.svelte.test.ts index 485dc39655..3454170b6b 100644 --- a/tools/ui/tests/client/chat-form-enter-code-block.svelte.test.ts +++ b/tools/ui/tests/client/chat-form-enter-code-block.svelte.test.ts @@ -7,7 +7,7 @@ import ChatFormTestWrapper from './components/ChatFormTestWrapper.svelte'; import { SETTINGS_KEYS } from '$lib/constants'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { tick } from 'svelte'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { userEvent } from 'vitest/browser'; diff --git a/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte b/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte index 504f685973..ab5cc38bc9 100644 --- a/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte +++ b/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte @@ -4,7 +4,7 @@ // toolMessages array) rather than a single message subtree. import ChatMessages from '$lib/components/app/chat/ChatMessages/ChatMessages.svelte'; import * as Tooltip from '$lib/components/ui/tooltip'; - import { conversationsStore } from '$lib/stores/conversations.svelte'; + import { conversationsStore } from '$lib/stores/conversations/index.svelte'; diff --git a/tools/ui/tests/client/mcp-display-name.svelte.test.ts b/tools/ui/tests/client/mcp-display-name.svelte.test.ts index f17e08cf1b..7db0ffd42e 100644 --- a/tools/ui/tests/client/mcp-display-name.svelte.test.ts +++ b/tools/ui/tests/client/mcp-display-name.svelte.test.ts @@ -1,6 +1,6 @@ import { McpServerForm } from '$lib/components/app/mcp'; -import { mcpStore } from '$lib/stores/mcp.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { mcpStore } from '$lib/stores/mcp/index.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { beforeEach, describe, expect, it } from 'vitest'; import { render } from 'vitest-browser-svelte'; diff --git a/tools/ui/tests/client/sandbox.service.svelte.test.ts b/tools/ui/tests/client/sandbox.service.svelte.test.ts index 7c0d7926f8..547e3ac1f2 100644 --- a/tools/ui/tests/client/sandbox.service.svelte.test.ts +++ b/tools/ui/tests/client/sandbox.service.svelte.test.ts @@ -10,7 +10,7 @@ const run = (code: string, timeoutMs?: number) => describe('sandbox service', () => { beforeEach(async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.config = { ...settingsStore.config, diff --git a/tools/ui/tests/client/settings-registry-invariants.svelte.test.ts b/tools/ui/tests/client/settings-registry-invariants.svelte.test.ts index 45af7e0d15..0ed6996b53 100644 --- a/tools/ui/tests/client/settings-registry-invariants.svelte.test.ts +++ b/tools/ui/tests/client/settings-registry-invariants.svelte.test.ts @@ -1,7 +1,7 @@ import { CONFIG_LOCALSTORAGE_KEY, SETTING_CONFIG_DEFAULT } from '$lib/constants'; import { ParameterSyncService } from '$lib/services/parameter-sync.service'; import { serverStore } from '$lib/stores/server.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { SettingsConfigType } from '$lib/types'; import { beforeEach, describe, expect, it } from 'vitest'; diff --git a/tools/ui/tests/client/settings-render-keys-migration.svelte.test.ts b/tools/ui/tests/client/settings-render-keys-migration.svelte.test.ts index 32f4ff3dd4..ce65aeb700 100644 --- a/tools/ui/tests/client/settings-render-keys-migration.svelte.test.ts +++ b/tools/ui/tests/client/settings-render-keys-migration.svelte.test.ts @@ -6,7 +6,7 @@ import { CONFIG_LOCALSTORAGE_KEY } from '$lib/constants'; import { MigrationService } from '$lib/services/migration.service'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { beforeEach, describe, expect, it } from 'vitest'; const RENDER_KEYS_MIGRATION_ID = 'render-keys-unfold-v1'; diff --git a/tools/ui/tests/client/ui-settings-sync.svelte.test.ts b/tools/ui/tests/client/ui-settings-sync.svelte.test.ts index 6dca891c85..ca9268e2e0 100644 --- a/tools/ui/tests/client/ui-settings-sync.svelte.test.ts +++ b/tools/ui/tests/client/ui-settings-sync.svelte.test.ts @@ -1,6 +1,6 @@ import { CONFIG_LOCALSTORAGE_KEY } from '$lib/constants'; import { serverStore } from '$lib/stores/server.svelte'; -import { settingsStore } from '$lib/stores/settings.svelte'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; import { beforeEach, describe, expect, it } from 'vitest'; function mockProps(uiSettings: Record) { diff --git a/tools/ui/tests/client/update-message-in-place.svelte.test.ts b/tools/ui/tests/client/update-message-in-place.svelte.test.ts index 65298b44b8..ea3b65d0cc 100644 --- a/tools/ui/tests/client/update-message-in-place.svelte.test.ts +++ b/tools/ui/tests/client/update-message-in-place.svelte.test.ts @@ -8,7 +8,7 @@ // -> 3.07ms at 40). Mutating in place keeps it flat. import { MessageRole } from '$lib/enums'; -import { conversationsStore } from '$lib/stores/conversations.svelte'; +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; import type { DatabaseMessage } from '$lib/types'; import { describe, expect, it } from 'vitest'; diff --git a/tools/ui/tests/stories/ChatMessage.stories.svelte b/tools/ui/tests/stories/ChatMessage.stories.svelte index 84fee2ea1c..e9bf7a6f6f 100644 --- a/tools/ui/tests/stories/ChatMessage.stories.svelte +++ b/tools/ui/tests/stories/ChatMessage.stories.svelte @@ -105,7 +105,7 @@ message: userMessage }} play={async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.updateConfig('showRawOutputSwitch', false); }} @@ -118,7 +118,7 @@ message: assistantMessage }} play={async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.updateConfig('showRawOutputSwitch', false); }} @@ -131,7 +131,7 @@ message: assistantWithReasoning }} play={async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.updateConfig('showRawOutputSwitch', false); }} @@ -144,7 +144,7 @@ message: rawOutputMessage }} play={async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.updateConfig('showRawOutputSwitch', true); }} @@ -157,7 +157,7 @@ }} asChild play={async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.updateConfig('showRawOutputSwitch', false); // Phase 1: Stream reasoning content in chunks @@ -213,11 +213,11 @@ message: processingMessage }} play={async () => { - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); settingsStore.updateConfig('showRawOutputSwitch', false); // Import the chat store to simulate loading state - const { chatStore } = await import('$lib/stores/chat.svelte'); + const { chatStore } = await import('$lib/stores/chat/index.svelte'); // Set loading state to true to trigger the processing UI chatStore.isLoading = true; diff --git a/tools/ui/tests/stories/ModelsSelector.stories.svelte b/tools/ui/tests/stories/ModelsSelector.stories.svelte index d63300cb21..7018d09e7b 100644 --- a/tools/ui/tests/stories/ModelsSelector.stories.svelte +++ b/tools/ui/tests/stories/ModelsSelector.stories.svelte @@ -4,7 +4,7 @@ import ModelsSelectorOption from '$lib/components/app/models/ModelsSelectorOption.svelte'; import type { GroupedModelOptions, ModelItem } from '$lib/components/app/models/utils'; import { ServerModelStatus } from '$lib/enums'; - import { modelsStore } from '$lib/stores/models.svelte'; + import { modelsStore } from '$lib/stores/models/index.svelte'; const { Story } = defineMeta({ parameters: { diff --git a/tools/ui/tests/stories/SidebarNavigation.stories.svelte b/tools/ui/tests/stories/SidebarNavigation.stories.svelte index 6359926012..ddaa90485d 100644 --- a/tools/ui/tests/stories/SidebarNavigation.stories.svelte +++ b/tools/ui/tests/stories/SidebarNavigation.stories.svelte @@ -53,7 +53,7 @@ asChild name="Default" play={async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); waitFor(() => setTimeout(() => { @@ -71,7 +71,7 @@ asChild name="SearchActive" play={async ({ userEvent }) => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); waitFor(() => setTimeout(() => { @@ -98,7 +98,7 @@ name="Empty" play={async () => { // Mock empty conversations store - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.conversations = []; }} diff --git a/tools/ui/tests/stories/fixtures/storybook-mocks.ts b/tools/ui/tests/stories/fixtures/storybook-mocks.ts index 7366746904..ac9fb63cd0 100644 --- a/tools/ui/tests/stories/fixtures/storybook-mocks.ts +++ b/tools/ui/tests/stories/fixtures/storybook-mocks.ts @@ -1,4 +1,4 @@ -import { modelsStore } from '$lib/stores/models.svelte'; +import { modelsStore } from '$lib/stores/models/index.svelte'; import { serverStore } from '$lib/stores/server.svelte'; /** diff --git a/tools/ui/tests/unit/chat-activity.test.ts b/tools/ui/tests/unit/chat-activity.test.ts new file mode 100644 index 0000000000..051648ead6 --- /dev/null +++ b/tools/ui/tests/unit/chat-activity.test.ts @@ -0,0 +1,77 @@ +import { ChatActivityStore } from '$lib/stores/chat/activity.svelte'; +import { beforeEach, describe, expect, it } from 'vitest'; + +describe('ChatActivityStore', () => { + let store: ChatActivityStore; + + beforeEach(() => { + store = new ChatActivityStore(); + }); + + it('starts with no local or remote activity', () => { + expect(store.loadingConvs).toEqual([]); + expect(store.isLocal('a')).toBe(false); + expect(store.isRemote('a')).toBe(false); + }); + + it('markLocal adds a conv to the local set and the loading union', () => { + store.markLocal('a'); + + expect(store.isLocal('a')).toBe(true); + expect(store.isRemote('a')).toBe(false); + expect(store.loadingConvs).toEqual(['a']); + }); + + it('localEnded removes a local conv', () => { + store.markLocal('a'); + store.localEnded('a'); + + expect(store.isLocal('a')).toBe(false); + expect(store.loadingConvs).toEqual([]); + }); + + it('localEnded also drops a stale remote hint for the same conv', () => { + store.markLocal('a'); + store.applyRemoteSnapshot(['a']); + expect(store.isRemote('a')).toBe(true); + + store.localEnded('a'); + + expect(store.isLocal('a')).toBe(false); + expect(store.isRemote('a')).toBe(false); + expect(store.loadingConvs).toEqual([]); + }); + + it('applyRemoteSnapshot adds remote convs and unions them with local', () => { + store.markLocal('local'); + store.applyRemoteSnapshot(['remote']); + + expect(store.isRemote('remote')).toBe(true); + expect(store.loadingConvs).toEqual(['local', 'remote']); + }); + + it('applyRemoteSnapshot removes remote convs missing from the snapshot', () => { + store.applyRemoteSnapshot(['a', 'b']); + store.applyRemoteSnapshot(['a']); + + expect(store.isRemote('a')).toBe(true); + expect(store.isRemote('b')).toBe(false); + expect(store.loadingConvs).toEqual(['a']); + }); + + it('applyRemoteSnapshot keeps local convs absent from the snapshot', () => { + store.markLocal('local'); + store.applyRemoteSnapshot(['remote']); + store.applyRemoteSnapshot([]); + + expect(store.isLocal('local')).toBe(true); + expect(store.loadingConvs).toEqual(['local']); + }); + + it('loadingConvs does not duplicate a conv that is both local and remote', () => { + store.markLocal('a'); + store.applyRemoteSnapshot(['a']); + + expect(store.loadingConvs).toEqual(['a']); + }); +}); diff --git a/tools/ui/tests/unit/mcp-override-fallback.test.ts b/tools/ui/tests/unit/mcp-override-fallback.test.ts index 47d6ac2536..12ed6e4c4b 100644 --- a/tools/ui/tests/unit/mcp-override-fallback.test.ts +++ b/tools/ui/tests/unit/mcp-override-fallback.test.ts @@ -46,7 +46,7 @@ describe('conversationsStore MCP override resolution', () => { // The settings store constructor bails in node env (no `browser`), // so seed the config directly. The shape mirrors what `loadConfig` // would build from localStorage. - const { settingsStore } = await import('$lib/stores/settings.svelte'); + const { settingsStore } = await import('$lib/stores/settings/index.svelte'); const raw = localStorage.getItem(CONFIG_LOCALSTORAGE_KEY) ?? '{}'; const saved = JSON.parse(raw) as Record; @@ -73,77 +73,77 @@ describe('conversationsStore MCP override resolution', () => { } it('inherits server.enabled when no conversation is active', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.activeConversation = null; - expect(conversationsStore.isMcpServerEnabledForChat('alpha')).toBe(false); - expect(conversationsStore.isMcpServerEnabledForChat('bravo')).toBe(true); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('alpha')).toBe(false); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('bravo')).toBe(true); }); it('inherits server.enabled on a newly created chat with no overrides', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.activeConversation = makeConversation(); // Empty override list: must fall back to global server.enabled, not all-off. - expect(conversationsStore.isMcpServerEnabledForChat('alpha')).toBe(false); - expect(conversationsStore.isMcpServerEnabledForChat('bravo')).toBe(true); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('alpha')).toBe(false); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('bravo')).toBe(true); }); it('inherits server.enabled on a newly created chat when overrides is undefined', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.activeConversation = makeConversation(undefined); - expect(conversationsStore.isMcpServerEnabledForChat('alpha')).toBe(false); - expect(conversationsStore.isMcpServerEnabledForChat('bravo')).toBe(true); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('alpha')).toBe(false); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('bravo')).toBe(true); }); it('uses explicit per-chat overrides, with defaults for non-overridden servers', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); // Override flips bravo off for this chat, alpha keeps its global default. conversationsStore.activeConversation = makeConversation([ { enabled: false, serverId: 'bravo' } ]); - expect(conversationsStore.isMcpServerEnabledForChat('alpha')).toBe(false); - expect(conversationsStore.isMcpServerEnabledForChat('bravo')).toBe(false); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('alpha')).toBe(false); + expect(conversationsStore.preferences.isMcpServerEnabledForChat('bravo')).toBe(false); }); it('getAllMcpServerOverrides returns a complete list merged from defaults', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.activeConversation = makeConversation([ { enabled: true, serverId: 'alpha' } ]); - expect(conversationsStore.getAllMcpServerOverrides()).toEqual([ + expect(conversationsStore.preferences.getAllMcpServerOverrides()).toEqual([ { enabled: true, serverId: 'alpha' }, { enabled: true, serverId: 'bravo' } ]); }); it('getAllMcpServerOverrides falls back to defaults when there are no explicit overrides', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.activeConversation = makeConversation(); - expect(conversationsStore.getAllMcpServerOverrides()).toEqual([ + expect(conversationsStore.preferences.getAllMcpServerOverrides()).toEqual([ { enabled: false, serverId: 'alpha' }, { enabled: true, serverId: 'bravo' } ]); }); it('getMcpServerOverride returns the global default when the server has no explicit override', async () => { - const { conversationsStore } = await import('$lib/stores/conversations.svelte'); + const { conversationsStore } = await import('$lib/stores/conversations/index.svelte'); conversationsStore.activeConversation = makeConversation([ { enabled: true, serverId: 'alpha' } ]); - expect(conversationsStore.getMcpServerOverride('bravo')).toEqual({ + expect(conversationsStore.preferences.getMcpServerOverride('bravo')).toEqual({ enabled: true, serverId: 'bravo' }); diff --git a/tools/ui/tests/unit/stream-resume.test.ts b/tools/ui/tests/unit/stream-resume.test.ts index 43d89272ef..ce4eee9aa7 100644 --- a/tools/ui/tests/unit/stream-resume.test.ts +++ b/tools/ui/tests/unit/stream-resume.test.ts @@ -92,6 +92,67 @@ describe('ChatService stream resume', () => { expect(ChatService.getStreamState('conv-a')!.model).toBe('model-y'); }); + describe('throttled saves (per-chunk path)', () => { + // unique conversation ids: the throttle tracker is module state and + // outlives beforeEach's localStorage.clear() + let counter = 0; + + const freshConv = () => `conv-throttle-${++counter}`; + + it('writes immediately when no write was recorded for the conversation', () => { + const conv = freshConv(); + + ChatService.saveStreamStateThrottled(conv, 100); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(100); + }); + + it('holds a save pending when it lands inside the interval, flush forces it out', () => { + const conv = freshConv(); + + ChatService.saveStreamStateThrottled(conv, 100); + ChatService.saveStreamStateThrottled(conv, 200); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(100); + + ChatService.flushStreamState(conv); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(200); + }); + + it('flush is a no-op when nothing is pending', () => { + const conv = freshConv(); + + ChatService.saveStreamStateThrottled(conv, 100); + ChatService.flushStreamState(conv); + ChatService.flushStreamState(conv); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(100); + }); + + it('an immediate save resets the throttle window', () => { + const conv = freshConv(); + + ChatService.saveStreamStateThrottled(conv, 100); + ChatService.saveStreamState(conv, 150); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(150); + + ChatService.saveStreamStateThrottled(conv, 200); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(150); + + ChatService.flushStreamState(conv); + expect(ChatService.getStreamState(conv)!.bytesReceived).toBe(200); + }); + + it('clearStreamState drops the pending throttled state', () => { + const conv = freshConv(); + + ChatService.saveStreamStateThrottled(conv, 100); + ChatService.saveStreamStateThrottled(conv, 200); + ChatService.clearStreamState(conv); + expect(ChatService.getStreamState(conv)).toBeNull(); + + ChatService.flushStreamState(conv); + expect(ChatService.getStreamState(conv)).toBeNull(); + }); + }); + describe('resumeStreamIdentity', () => { it('appends the persisted model so the resume key matches the frozen POST identity', () => { ChatService.saveStreamState('conv-a', 10, 'model-x'); From 6b4fa88a6ce2429958ea4ee7c0223928e0979fa2 Mon Sep 17 00:00:00 2001 From: lhez Date: Thu, 20 Aug 2026 10:52:07 -0700 Subject: [PATCH 34/36] opencl: fix local size for norm (#27339) --- ggml/src/ggml-opencl/ggml-opencl.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index fbf7dadb90..d169f33896 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -12830,7 +12830,10 @@ static void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - const int nth = MIN(64, ne00); + int nth = 1; + while (nth < ne00 && nth < 64) { + nth *= 2; + } cl_kernel kernel = backend_ctx->kernel_norm; From 6503355df0eb4f65875012523263c302fe0088c1 Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Thu, 20 Aug 2026 10:58:35 -0700 Subject: [PATCH 35/36] opencl: fix q6_K flat mul_mat for Adreno A6x/A7x GPUs with older E031 compilers (#26476) * opencl: decline KV-convert flash_attn variants on Adreno A7X (compiler SIGSEGV) The Adreno 740 (A7X) compiler E031.41 crashes inside clBuildProgram when building the flash_attn programs whose KV path is mixed-type or dequantized: flash_attn_f32_f16, flash_attn_f32_q8_0, flash_attn_f32_q4_0. It is a driver crash rather than a compile-error return, so build_program_from_source_ex() cannot catch it. The uniform f32 and f16 programs build correctly. Decline the three KV-convert variants on the A7X in supports_op so they never lazy-compile; those attention layers run on the CPU backend instead. Same idiom as the existing Intel DK=512 and X1E carve-outs. test-backend-ops FLASH_ATTN_EXT on the 740: 226 OK / 0 FAIL, previously exit 139. Other parts are unaffected - the gate is dead code there. * opencl: fix q6_K flat mul_mat on older Adreno E031 compilers, gated kernel_mul_mv_q6_K_f32_flat produces ~10x-wrong output on the older Adreno E031 compilers while q4_K and q5_K are correct. Four codegen defects, each confirmed on-device against the CPU reference: 1. 64-bit ulong arithmetic is miscompiled, so every weight and scale read hit the wrong address - the primary cause, and why q5_K (int offsets) was unaffected. The block index is computed in int and widened only inside the pointer expression. 2. The vectorized dequant (int4/float4 bit-ops, convert_*4, dot()) is miscompiled; the 6-bit weights are reconstructed and the dot done scalar. 3. vload4 of the f32 activations is miscompiled; replaced by a scalar-indexed load. 4. The accumulation is miscompiled unless a side effect forces the partial sums to materialize. A printf under a guard the compiler cannot prove false acts as a zero-cost optimizer barrier; its placement is load-bearing. The defect tracks the compiler, not the GPU generation: it reproduces on E031.38 (Adreno 642L) and E031.41 (Adreno 740) and is fixed by E031.45 (Adreno 619), so the workarounds are gated on the compiler version. Where they are not needed they cost real throughput - 42.4 -> 35.1 GFLOPS on an Adreno 840 q6_K GEMV. The explicit compiler-type check is required, not redundant: newer_than_or_same() is false for every non-E031 compiler, so negating it alone would enable the workarounds on E17 and DX. test-backend-ops MUL_MAT is 919/919 on the Adreno 740, 642L, 619, 840 and 850; the 740 and 642L were 909/919 before. The 642L additionally needs the A6X per-kernel-program support to reach these tests at all. --- ggml/src/ggml-opencl/ggml-opencl.cpp | 42 +++++++- .../kernels/mul_mv_q6_k_f32_flat.cl | 96 ++++++++++++++++++- 2 files changed, 136 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index d169f33896..49cd9fd355 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -582,6 +582,8 @@ struct ggml_backend_opencl_context { bool adreno_use_bin_kernels; get_adreno_bin_kernel_func_t get_adreno_bin_kernel_func = nullptr; ggml_cl_compiler_version adreno_cl_compiler_version; + // The q6_K flat mul_mat codegen workarounds are needed by old E031 compilers only. + bool q6_k_flat_old_compiler; std::string kernel_compile_opts; // cached for lazy-compiled kernels. @@ -1931,8 +1933,14 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { #else const std::string kernel_src = read_file("mul_mv_q6_k_f32_flat.cl"); #endif + // The codegen workarounds in this kernel are a measured 13-20% loss on + // compilers that do not need them, so only the affected ones build them; + // everyone else gets the original source. + const std::string q6k_opts = backend_ctx->q6_k_flat_old_compiler + ? compile_opts + " -DADRENO_OLD_COMPILER=1" + : compile_opts; cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), q6k_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q6_K_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); @@ -5917,6 +5925,16 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) { (backend_ctx->adreno_cl_compiler_version.type == E031 && backend_ctx->adreno_cl_compiler_version.major >= 47) || (backend_ctx->adreno_cl_compiler_version.type == DX && backend_ctx->adreno_cl_compiler_version.major >= 17); + // The q6_K flat mul_mat miscompile is a defect of the older E031 compilers, not a + // property of any GPU generation: it reproduces on E031.38 (Adreno 642L) and E031.41 + // (Adreno 740) and is fixed by E031.45 (Adreno 619). Gate on the compiler so parts + // that do not need the workarounds do not pay for them. The explicit type check is + // required: newer_than_or_same() is false for every non-E031 compiler, so negating it + // alone would enable the workarounds on E17/DX. + backend_ctx->q6_k_flat_old_compiler = + backend_ctx->adreno_cl_compiler_version.type == E031 && + !backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 45, 0, 0); + size_t ext_str_size; clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size); char *ext_buffer = (char *)alloca(ext_str_size + 1); @@ -7496,6 +7514,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16; const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32; + const bool is_f32_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0 && op->type == GGML_TYPE_F32 && dk % 32 == 0 && dv % 32 == 0; @@ -7503,6 +7522,21 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te v->type == GGML_TYPE_Q4_0 && op->type == GGML_TYPE_F32 && dk % 32 == 0 && dv % 32 == 0; + // A7X (Adreno 740, compiler E031.41) SIGSEGVs inside clBuildProgram + // building the flash_attn programs whose KV path is mixed-type or + // dequantized — f32_f16, q8_0, q4_0 (reproduced at DK=40 and DK=64; it + // is DK-independent). It is a driver crash, not codegen-wrong-output, so + // it cannot be caught in-process (fatal=false only handles clean compile + // errors). The uniform f16_f16 / f32_f32 programs compile fine on this + // compiler, so decline only the KV-convert variants; ggml then runs + // those (f16-KV / quant-KV) attention layers on the CPU backend. + // Negative compiler carve-out, same idiom as the Intel DK=512 decline + // below and the X1E driver-quirk guards. + if (backend_ctx && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && + (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) { + return false; + } + // Asymmetric KV: host-dequants both sides to F32, uses f32 kernel. auto is_kv_type_ok = [](ggml_type t) { return t == GGML_TYPE_F16 || t == GGML_TYPE_F32 || @@ -20583,6 +20617,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3)); + // The optimizer-barrier arg exists only in the ADRENO_OLD_COMPILER build of + // this kernel; conformant compilers get the original 17-arg signature. + if (backend_ctx->q6_k_flat_old_compiler) { + cl_uchar q6k_mask = 0xFF; // never 0xFE in prod; see the kernel note + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_uchar), &q6k_mask)); + } #else kernel = backend_ctx->kernel_mul_mv_q6_K_f32; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl index 57b90c05ae..2cca5335dd 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl @@ -28,6 +28,13 @@ #define QK_K 256 +// ADRENO_OLD_COMPILER is defined by the host (-D) only for the Adreno E031 +// compilers older than E031.45, which miscompile several constructs this kernel +// used (confirmed on E031.38 and E031.41; E031.45 is clean). Every other +// compiler -- newer E031, E17, DX, Intel, and every non-Adreno device that +// builds this program -- takes the #else branches, which are the original +// source: the workarounds below cost ~13% on the q6_K flat n=1 GEMV where they +// are not needed. inline float block_q_6_K_dot_y_flat( global uchar * blk_ql, global uchar * blk_qh, @@ -37,6 +44,9 @@ inline float block_q_6_K_dot_y_flat( int ip, int is, int l0, +#if defined(ADRENO_OLD_COMPILER) + int dbg, +#endif float4 y0, float4 y1, float4 y2, @@ -48,10 +58,40 @@ inline float block_q_6_K_dot_y_flat( global uchar * q1 = blk_ql + ib*128 + q_offset_l; global uchar * q2 = q1 + QK_K/8; global uchar * qh = blk_qh + ib*64 + q_offset_h; - global char * sc = blk_scales + ib*16 + is; float dall = blk_d[ib]; +#if defined(ADRENO_OLD_COMPILER) + // The vectorized dequant (int4/float4 bit-ops, convert_*4, dot()) and vload4 + // are miscompiled here -> garbage weights. Reconstruct the 6-bit weights and + // take the dot product scalar. q4_K/q5_K flat already use scalar paths, which + // is why q6_K was the only flat GEMV that failed. + // Scales are SIGNED int8; read as uchar and sign-extend arithmetically so the + // result does not depend on whether the compiler treats `char` as signed. + global uchar * sc = (global uchar *)(blk_scales + ib*16 + is); + + int s0 = (int)sc[0] - 256*(sc[0] >> 7); + int s2 = (int)sc[2] - 256*(sc[2] >> 7); + int s4 = (int)sc[4] - 256*(sc[4] >> 7); + int s6 = (int)sc[6] - 256*(sc[6] >> 7); + + // one 6-bit weight: low/high nibble of a ql byte OR'd with a 2-bit qh plane + // (plane p in {0,1,2,3} selects qh bits 2p..2p+1) placed at bits 4-5, minus 32. + #define Q6W(qb, sh, hb, p) ((float)((((int)(qb) >> (sh)) & 15) | ((((int)(hb) >> (2*(p))) & 3) << 4)) - 32.f) + + float d0 = y0.s0*Q6W(q1[0],0,qh[0],0) + y0.s1*Q6W(q1[1],0,qh[1],0) + y0.s2*Q6W(q1[2],0,qh[2],0) + y0.s3*Q6W(q1[3],0,qh[3],0); + float d1 = y1.s0*Q6W(q2[0],0,qh[0],1) + y1.s1*Q6W(q2[1],0,qh[1],1) + y1.s2*Q6W(q2[2],0,qh[2],1) + y1.s3*Q6W(q2[3],0,qh[3],1); + float d2 = y2.s0*Q6W(q1[0],4,qh[0],2) + y2.s1*Q6W(q1[1],4,qh[1],2) + y2.s2*Q6W(q1[2],4,qh[2],2) + y2.s3*Q6W(q1[3],4,qh[3],2); + float d3 = y3.s0*Q6W(q2[0],4,qh[0],3) + y3.s1*Q6W(q2[1],4,qh[1],3) + y3.s2*Q6W(q2[2],4,qh[2],3) + y3.s3*Q6W(q2[3],4,qh[3],3); + #undef Q6W + + if (dbg) printf("HELPER dall=%f s=[%d %d %d %d] d=[%f %f %f %f] ql0=%d qh0=%d y00=%f\n", + dall, s0, s2, s4, s6, d0, d1, d2, d3, (int)q1[0], (int)qh[0], y0.s0); + + return dall * (d0 * s0 + d1 * s2 + d2 * s4 + d3 * s6); +#else + global char * sc = blk_scales + ib*16 + is; + // Vectorized loads: 3 uchar4 weight loads instead of 12 scalar byte reads. // q_offset_l/h are 4-aligned, so these are aligned vector loads. uchar4 q1v = vload4(0, q1); @@ -72,6 +112,7 @@ inline float block_q_6_K_dot_y_flat( return dall * (dot(y0, w0) * sc[0] + dot(y1, w1) * sc[2] + dot(y2, w2) * sc[4] + dot(y3, w3) * sc[6]); +#endif } #undef N_DST @@ -113,6 +154,11 @@ kernel void kernel_mul_mv_q6_K_f32_flat( int ne1, int r2, int r3 +#if defined(ADRENO_OLD_COMPILER) + , + uchar q6k_mask // runtime 0xFF; the host passes it so the compiler cannot + // constant-fold the printf guards below into nothing +#endif ) { src1 = (global float*)((global char*)src1 + offset1); dst = (global float*)((global char*)dst + offsetd); @@ -128,6 +174,22 @@ kernel void kernel_mul_mv_q6_K_f32_flat( int first_row = (N_SIMDGROUP * r0 + get_sub_group_id()) * N_DST; +#if defined(ADRENO_OLD_COMPILER) + // 64-bit `ulong` integer arithmetic is miscompiled here -> the base-pointer byte + // offsets came out wrong, so EVERY weight/scale read hit the wrong address. This + // was the primary cause of the q6_K flat failure (q5_K uses int offsets and is + // unaffected). Compute the block index in `int` and widen to `ulong` only inside + // the pointer expression: the byte offset stays 64-bit, but there is no ulong + // arithmetic chain to miscompile. The int index would overflow past ~2^31 blocks, + // which no realistic weight reaches -- but that is a narrowing, so keep it off the + // conformant path, which retains full ulong arithmetic. + int offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + + global uchar * blk_ql = (global uchar *) src0_ql + (ulong)offset_src0 * 128; + global uchar * blk_qh = (global uchar *) src0_qh + (ulong)offset_src0 * 64; + global char * blk_scales = (global char *) src0_s + (ulong)offset_src0 * 16; + global half * blk_d = (global half *) src0_d + offset_src0; +#else ulong offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); ulong offset_src0_ql = offset_src0 * 128; ulong offset_src0_qh = offset_src0 * 64; @@ -138,6 +200,7 @@ kernel void kernel_mul_mv_q6_K_f32_flat( global uchar * blk_qh = (global uchar *) src0_qh + offset_src0_qh; global char * blk_scales = (global char *) src0_s + offset_src0_s; global half * blk_d = (global half *) src0_d + offset_src0_d; +#endif global float * yy = (global float *) src1 + r1*ne10 + im*ne00*ne1; int tid = get_sub_group_local_id()%(N_SIMDWIDTH/BLOCK_STRIDE); // within-super-block part, 0..15 @@ -155,24 +218,55 @@ kernel void kernel_mul_mv_q6_K_f32_flat( for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { global float * y = yy + ib * QK_K + 128*ip + l0; +#if defined(ADRENO_OLD_COMPILER) + // vload4 of f32 is miscompiled here; index the lanes scalar instead. + float4 y0 = (float4)(y[ 0], y[ 1], y[ 2], y[ 3]); + float4 y1 = (float4)(y[32], y[33], y[34], y[35]); + float4 y2 = (float4)(y[64], y[65], y[66], y[67]); + float4 y3 = (float4)(y[96], y[97], y[98], y[99]); +#else float4 y0 = vload4(0, y + 0); float4 y1 = vload4(0, y + 32); float4 y2 = vload4(0, y + 64); float4 y3 = vload4(0, y + 96); +#endif for (int row = 0; row < N_DST; row++) { if (first_row + row < ne01) { +#if defined(ADRENO_OLD_COMPILER) + int dbg = (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ib==0 && + ne00==256 && ne01==16 && get_sub_group_local_id()==0) ? 1 : 0; + sumf[row] += block_q_6_K_dot_y_flat( + blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb, + ib, ip, is, l0, dbg, y0, y1, y2, y3); +#else sumf[row] += block_q_6_K_dot_y_flat( blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb, ib, ip, is, l0, y0, y1, y2, y3); +#endif } } } +#if defined(ADRENO_OLD_COMPILER) + // Optimizer barrier. This compiler drops the sumf partials unless a side effect + // forces them to materialize. q6k_mask is a kernel arg the compiler cannot prove + // is never 0xFE (the host always passes 0xFF), so the printf survives compilation + // but never executes. FRAGILE: the exact set and placement of these guarded + // printfs is load-bearing on E031.41 -- removing any one re-breaks q6_K. + if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && ne00==256 && ne01==16 && get_sub_group_local_id()<16) { + printf("Q6KLANE lane=%d ip=%d il=%d is=%d l0=%d sumf0=%f\n", + get_sub_group_local_id(), ip, il, is, l0, sumf[0]); + } +#endif for (int row = 0; row < N_DST; row++) { float tot = sub_group_reduce_add(sumf[row]); if (get_sub_group_local_id() == 0 && first_row + row < ne01) { dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot; +#if defined(ADRENO_OLD_COMPILER) + if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ne00==256 && ne01==16) + printf("Q6KTOT tot=%f\n", tot); +#endif } } } From a30273376ef669023334fc20ad02ae4ed8196a65 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 21:31:29 +0300 Subject: [PATCH 36/36] metal : clamp K extent in tensor API mat-mat kernel for K not a multiple of 32 (#27450) The Tensor API mat-mat path of kernel_mul_mm (GGML_METAL_HAS_TENSOR) fed a static K=32 tile to the matmul2d op on every iteration. On the last, partial K tile (ne00 % 32 != 0) the src1 slice extends past the K extent of the tensor, and the op reads those out-of-bounds elements (undefined behavior per the MSL specification, section 2.22.2). Depending on stale memory contents, this corrupted the result or produced NaN. Make the matmul2d op use dynamic_extent for K, and clamp the K extent of both operand tensor views to the remaining valid K range (min(32, K - loop_k)) per iteration, so the op reads exactly the valid K range on every iteration (mirroring the tail handling of the MPP matmul2d examples). On K-aligned inputs the clamp degenerates to the full 32-wide tile: the only difference from the static-K op is that the dynamic-K op derives K from the operand extents and edge-checks the tile against the tensor extents (a handful of integer ops per iteration). Add test-backend-ops MUL_MAT cases with K not a multiple of 32 to exercise the unaligned K path. Assisted-by: pi:llama.cpp/Qwen3.8-27B --- ggml/src/ggml-metal/ggml-metal.metal | 15 +++++++++++---- tests/test-backend-ops.cpp | 8 ++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 949931c8dc..27f97b5e07 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -10365,9 +10365,12 @@ kernel void kernel_mul_mm( auto tB = tensor(ptrB, dextents(K, N), array({1, strideB})); // Configure matmul operation + // note: K is dynamic_extent (clamped to the valid range in PHASE 2), since a static + // N_MM_NK_TOTAL K tile would read src1 out of bounds when K % N_MM_NK_TOTAL != 0 + // ref: https://github.com/ggml-org/llama.cpp/pull/27064 mpp::tensor_ops::matmul2d< mpp::tensor_ops::matmul2d_descriptor( - NRB, NRA, N_MM_NK_TOTAL, false, true, true, + NRB, NRA, static_cast(dynamic_extent), false, true, true, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), execution_simdgroups> mm; @@ -10419,10 +10422,14 @@ kernel void kernel_mul_mm( threadgroup_barrier(mem_flags::mem_threadgroup); // === PHASE 2: Tensor matmul === - auto mA = tA.slice(0, 0); - auto mB = tB.slice(loop_k, rb); + // Clamp the K extent of both operand tensors to the remaining valid K range so + // the dynamic-K op never reads past the K extent of src1 (or the staged A tile). + const int kExt = min(N_MM_NK_TOTAL, K - loop_k); - mm.run(mB, mA, cT); + auto tAv = tensor(sa, dextents(kExt, NRA), array({1, N_MM_NK_TOTAL})); + auto tBv = tensor(ptrB + loop_k + rb * strideB, dextents(kExt, N - rb), array({1, strideB})); + + mm.run(tBv, tAv, cT); threadgroup_barrier(mem_flags::mem_threadgroup); } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 17098825bc..89b954a7d1 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9298,6 +9298,14 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 6, 4096, 5120, {1, 1}, {1, 1})); + // K not a multiple of 32 + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 65, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 80, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 32, 80, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 32, 80, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 588, {1, 1}, {1, 1})); // 14*14*3, e.g. conv_2d im2col + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 80, {4, 1}, {1, 1})); + #if 0 // test the mat-mat path for Metal for (int k = 1; k < 512; ++k) {