diff --git a/Makefile b/Makefile index 9244e852a..d078fe7bc 100644 --- a/Makefile +++ b/Makefile @@ -719,7 +719,7 @@ otherarch/sdcpp/thirdparty/zip.o: otherarch/sdcpp/thirdparty/zip.c OBJS_SDTYPE := otherarch/sdcpp/sdtype_adapter.o $(OBJS_SDCOMMON) -LLAMASERVER_SRCS := tools/server/main.cpp tools/server/server.cpp tools/server/server-schema.cpp tools/server/server-chat.cpp tools/server/server-common.cpp tools/server/server-context.cpp tools/server/server-http.cpp tools/server/server-models.cpp tools/server/server-queue.cpp tools/server/server-task.cpp tools/server/server-tools.cpp tools/server/ui.cpp tools/server/server-stream.cpp +LLAMASERVER_SRCS := tools/server/main.cpp tools/server/server.cpp tools/server/server-schema.cpp tools/server/server-chat.cpp tools/server/server-common.cpp tools/server/server-context.cpp tools/server/server-http.cpp tools/server/server-models.cpp tools/server/server-queue.cpp tools/server/server-task.cpp tools/server/server-tools.cpp tools/server/server-mcp.cpp tools/server/ui.cpp tools/server/server-stream.cpp COMMON_DOWNLOAD_SRCS := common/download.cpp common/hf-cache.cpp vendor/cpp-httplib/httplib.cpp LLAMASERVER_COMMON_SRCS := common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) LLAMASERVER_CXXFLAGS := -I./tools/mtmd diff --git a/common/arg.cpp b/common/arg.cpp index ea74adbe9..2482a6d65 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -851,8 +851,9 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context params.kv_overrides.back().key[0] = 0; } - if (!params.server_tools.empty() && !params.cors_origins_explicit) { - LOG_WRN("server tools are enabled, using localhost as default CORS origin (change via --cors-origins)\n"); + const bool mcp_enabled = !params.mcp_servers_config.empty() || !params.mcp_servers_json.empty(); + if ((!params.server_tools.empty() || mcp_enabled) && !params.cors_origins_explicit) { + LOG_WRN("server tools or MCP servers are enabled, using localhost as default CORS origin (change via --cors-origins)\n"); params.cors_origins = "localhost"; } @@ -3262,6 +3263,22 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.server_tools = parse_csv_row(value); } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS")); + add_opt(common_arg( + {"--mcp-servers-config"}, "PATH", + "experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)\n" + "note: for security reasons, this will limit --cors-origins to localhost by default", + [](common_params & params, const std::string & value) { + params.mcp_servers_config = value; + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MCP_SERVERS_CONFIG")); + add_opt(common_arg( + {"--mcp-servers-json"}, "JSON", + "experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)\n" + "note: for security reasons, this will limit --cors-origins to localhost by default", + [](common_params & params, const std::string & value) { + params.mcp_servers_json = value; + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MCP_SERVERS_JSON")); add_opt(common_arg( {"-ag", "--agent"}, {"-no-ag", "--no-agent"}, diff --git a/common/chat.cpp b/common/chat.cpp index c7a1638f0..de27e8e7e 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -574,6 +574,7 @@ std::vector common_chat_tools_parse_oaicompat(const json & too } #include "common/unicode.h" +#include "trie.cpp" #include "peg-parser.cpp" #include "chat-peg-parser.cpp" @@ -1037,7 +1038,7 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_ data.supports_thinking = true; data.thinking_start_tag = "[THINK]"; - data.thinking_end_tag = "[/THINK]"; + data.thinking_end_tags = {"[/THINK]"}; data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages); data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages); data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; @@ -1163,6 +1164,9 @@ static common_chat_params common_chat_params_init_gpt_oss(const common_chat_temp data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; data.supports_thinking = true; + data.thinking_start_tag = "<|channel|>analysis<|message|>"; + data.thinking_end_tags = {"<|end|>"}; + // These special tokens are required to parse properly, so we include them // even if parse_tool_calls is false. data.preserved_tokens = { @@ -1307,7 +1311,7 @@ static common_chat_params common_chat_params_init_gemma4(const common_chat_templ data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4; data.supports_thinking = true; data.thinking_start_tag = "<|channel>thought"; - data.thinking_end_tag = ""; + data.thinking_end_tags = {""}; data.preserved_tokens = { "<|channel>", @@ -1582,7 +1586,7 @@ static common_chat_params common_chat_params_init_kimi_k2(const common_chat_temp const std::string GEN_PROMPT = "<|im_assistant|>assistant<|im_middle|>"; data.thinking_start_tag = THINK_START; - data.thinking_end_tag = THINK_END; + data.thinking_end_tags = {THINK_END}; if (inputs.has_continuation()) { const auto & msg = inputs.continue_msg; @@ -1716,7 +1720,7 @@ static common_chat_params common_chat_params_init_lfm2(const common_chat_templat } data.thinking_start_tag = THINK_START; - data.thinking_end_tag = THINK_END; + data.thinking_end_tags = {THINK_END}; auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); @@ -1956,7 +1960,7 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; data.supports_thinking = true; data.thinking_start_tag = ""; - data.thinking_end_tag = ""; + data.thinking_end_tags = {""}; data.preserved_tokens = { "|DSML|", "", @@ -2173,7 +2177,7 @@ static common_chat_params common_chat_params_init_cohere2moe(const common_chat_t data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; data.supports_thinking = true; data.thinking_start_tag = THINK_START; - data.thinking_end_tag = THINK_END; + data.thinking_end_tags = {THINK_END}; data.preserved_tokens = { TURN_START, TURN_END, CHATBOT, USER, SYSTEM, THINK_START, THINK_END, @@ -2523,7 +2527,7 @@ static common_chat_params common_chat_params_init_minicpm5(const common_chat_tem }; data.thinking_start_tag = ""; - data.thinking_end_tag = ""; + data.thinking_end_tags = {""}; data.message_delimiters = { { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, @@ -2879,7 +2883,10 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ auto_params.supports_thinking = autoparser.reasoning.mode != autoparser::reasoning_mode::NONE; if (auto_params.supports_thinking) { auto_params.thinking_start_tag = trim_whitespace(autoparser.reasoning.start); - auto_params.thinking_end_tag = trim_whitespace(autoparser.reasoning.end); + auto end_tag = trim_whitespace(autoparser.reasoning.end); + if (!end_tag.empty()) { + auto_params.thinking_end_tags = {std::move(end_tag)}; + } } common_peg_arena arena; arena.load(auto_params.parser); diff --git a/common/chat.h b/common/chat.h index 7898f1623..d79f4ecd7 100644 --- a/common/chat.h +++ b/common/chat.h @@ -274,7 +274,7 @@ struct common_chat_params { std::string generation_prompt; bool supports_thinking = false; std::string thinking_start_tag; // e.g., "" - std::string thinking_end_tag; // e.g., "" + std::vector thinking_end_tags; // e.g., "" std::vector grammar_triggers; std::vector preserved_tokens; std::vector additional_stops; diff --git a/common/common.h b/common/common.h index c741c60aa..dc3b34c20 100644 --- a/common/common.h +++ b/common/common.h @@ -285,12 +285,12 @@ struct common_params_sampling { // reasoning budget sampler parameters // these are populated by the server/CLI based on chat template params - int32_t reasoning_budget_tokens = -1; // -1 = disabled, >= 0 = token budget - std::vector reasoning_budget_start; // start tag token sequence - std::vector reasoning_budget_end; // end tag token sequence - std::vector reasoning_budget_forced; // forced sequence (message + end tag) - std::string reasoning_budget_message; // message injected before end tag when budget exhausted - bool reasoning_control = false; // create the budget sampler on demand so reasoning can be ended at runtime + int32_t reasoning_budget_tokens = -1; // -1 = disabled, >= 0 = token budget + std::vector reasoning_budget_start; // start tag token sequence + std::vector reasoning_budget_end; // end tag token sequences; the first tag is used as the forcing sequence + std::vector reasoning_budget_forced; // forced sequence (message + first end tag) + std::string reasoning_budget_message; // message injected before end tag when budget exhausted + bool reasoning_control = false; // create the budget sampler on demand so reasoning can be ended at runtime bool backend_sampling = false; @@ -669,6 +669,10 @@ struct common_params { // enable built-in tools std::vector server_tools; + // MCP server configs (Cursor-compatible JSON) + std::string mcp_servers_config; // path to JSON file with MCP server definitions + std::string mcp_servers_json; // inline JSON with MCP server definitions + // router server configs std::string models_dir = ""; // directory containing models for the router server std::string models_preset = ""; // directory containing model presets for the router server diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 807e952d9..ef290ed7c 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -3,10 +3,10 @@ #include "common.h" #include "json-schema-to-grammar.h" #include "log.h" +#include "trie.h" #include "unicode.h" #include -#include #include #include #include @@ -32,154 +32,6 @@ static bool is_hex_digit(const char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } -// Trie for matching multiple literals. -// This is used in common_peg_until_parser and to build a GBNF exclusion grammar -struct trie { - struct node { - std::map children; // Use uint32_t to store Unicode codepoints - bool is_word; - }; - - std::vector nodes; - - trie(const std::vector & words) { - create_node(); // root node - for (const auto & w : words) { - insert(w); - } - } - - enum match_result { NO_MATCH, PARTIAL_MATCH, COMPLETE_MATCH }; - - // Check if a delimiter starts at the given position - match_result check_at(std::string_view sv, size_t start_pos) const { - size_t current = 0; // Start at root - size_t pos = start_pos; - - // LOG_DBG("%s: checking at pos %zu, sv='%s'\n", __func__, start_pos, std::string(sv).c_str()); - - while (pos < sv.size()) { - auto result = common_parse_utf8_codepoint(sv, pos); - if (result.status != utf8_parse_result::SUCCESS) { - break; - } - - auto it = nodes[current].children.find(result.codepoint); - if (it == nodes[current].children.end()) { - // Can't continue matching - return match_result{match_result::NO_MATCH}; - } - - current = it->second; - pos += result.bytes_consumed; - - // Check if we've matched a complete word - if (nodes[current].is_word) { - return match_result{match_result::COMPLETE_MATCH}; - } - } - - // Reached end of input while still in the trie (not at root) - if (current != 0) { - // We're in the middle of a potential match - return match_result{match_result::PARTIAL_MATCH}; - } - - // Reached end at root (no match) - return match_result{match_result::NO_MATCH}; - } - - private: - size_t create_node() { - size_t index = nodes.size(); - nodes.emplace_back(); - return index; - } - - void insert(const std::string & word) { - size_t current = 0; - size_t pos = 0; - while (pos < word.length()) { - auto result = common_parse_utf8_codepoint(word, pos); - if (result.status != utf8_parse_result::SUCCESS) { - break; - } - - uint32_t ch = result.codepoint; - pos += result.bytes_consumed; - - auto it = nodes[current].children.find(ch); - if (it == nodes[current].children.end()) { - size_t child = create_node(); - nodes[current].children[ch] = child; - current = child; - } else { - current = it->second; - } - } - nodes[current].is_word = true; - } -}; - -// Aho-Corasick automaton -struct aho_corasick { - trie t; - std::vector fail; // failure links - std::vector order; // states in BFS order - std::vector terminal; // match states (directly or via a suffix link) - std::set alphabet; // every character with a transition - - aho_corasick(const std::vector & strings) : t(strings) { - const auto & nodes = t.nodes; - const size_t n = nodes.size(); - - fail.assign(n, 0); - order.reserve(n); - - std::deque queue{ 0 }; - while (!queue.empty()) { - size_t u = queue.front(); - queue.pop_front(); - order.push_back(u); - for (const auto & [ch, v] : nodes[u].children) { - if (u != 0) { - size_t f = fail[u]; - while (f && nodes[f].children.find(ch) == nodes[f].children.end()) { - f = fail[f]; - } - auto it = nodes[f].children.find(ch); - fail[v] = (it != nodes[f].children.end() && it->second != v) ? it->second : 0; - } - queue.push_back(v); - } - } - - terminal.assign(n, false); - for (size_t u : order) { - terminal[u] = nodes[u].is_word || (u != 0 && terminal[fail[u]]); - } - - for (const auto & node : nodes) { - for (const auto & [ch, v] : node.children) { - alphabet.insert(ch); - } - } - } - - size_t num_states() const { return t.nodes.size(); } - bool is_terminal(size_t s) const { return terminal[s]; } - - // follow failure links until a transition on `ch` exists. - size_t next(size_t state, uint32_t ch) const { - const auto & nodes = t.nodes; - while (state && nodes[state].children.find(ch) == nodes[state].children.end()) { - state = fail[state]; - } - auto it = nodes[state].children.find(ch); - return it != nodes[state].children.end() ? it->second : 0; - } -}; - static std::pair parse_hex_escape(const std::string & str, size_t pos, int hex_count) { if (pos + hex_count > str.length()) { return {0, 0}; @@ -797,7 +649,7 @@ struct parser_executor { } common_peg_parse_result operator()(const common_peg_until_parser & p) const { - trie matcher(p.delimiters); + common_trie matcher(p.delimiters); // Scan input and check for delimiters size_t pos = start_pos; @@ -824,12 +676,12 @@ struct parser_executor { // Check if a delimiter starts at this position auto match = matcher.check_at(ctx.input, pos); - if (match == trie::COMPLETE_MATCH) { + if (match == common_trie::COMPLETE_MATCH) { // Found a complete delimiter, return everything before it return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } - if (match == trie::PARTIAL_MATCH) { + if (match == common_trie::PARTIAL_MATCH) { // Found a partial match extending to end of input, return everything before it return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } @@ -1559,7 +1411,7 @@ static std::string gbnf_ac_grammar( const std::map> &, const std::vector &, const std::function &)> & build_rule) { - aho_corasick ac(strings); + common_aho_corasick ac(strings); auto state_name = [&](size_t s) -> std::string { if (s == 0) { diff --git a/common/preset.cpp b/common/preset.cpp index 4362c0621..eb0c60b09 100644 --- a/common/preset.cpp +++ b/common/preset.cpp @@ -330,6 +330,10 @@ common_presets common_preset_context::load_from_ini(const std::string & path, co } } + if (preset.name == COMMON_PRESET_DEFAULT_NAME && preset.options.empty()) { + continue; + } + if (preset.name == "*") { // handle global preset global = preset; diff --git a/common/reasoning-budget.cpp b/common/reasoning-budget.cpp index 7da0bb1c5..1fe242d06 100644 --- a/common/reasoning-budget.cpp +++ b/common/reasoning-budget.cpp @@ -1,39 +1,52 @@ #include "reasoning-budget.h" #include "common.h" +#include "trie.h" #include "unicode.h" #include "log.h" +#include #include #include #include #include struct token_matcher { - std::vector tokens; - size_t pos = 0; + std::vector seqs; + common_aho_corasick ac; + size_t state = 0; - bool advance(llama_token token) { - if (tokens.empty()) { - return false; - } + token_matcher(const std::vector & seqs) : seqs(collect(seqs)), ac(build_trie(this->seqs)) {} - if (token == tokens[pos]) { - pos++; - if (pos >= tokens.size()) { - pos = 0; - return true; - } - } else { - pos = 0; - if (token == tokens[0]) { - pos = 1; + static std::vector collect(const std::vector & seqs) { + std::vector res; + for (const auto & seq : seqs) { + if (!seq.empty() && std::find(res.begin(), res.end(), seq) == res.end()) { + res.push_back(seq); } } - return false; + return res; } - void reset() { pos = 0; } + static common_trie build_trie(const std::vector & seqs) { + common_trie t; + for (const auto & seq : seqs) { + t.insert(std::vector(seq.begin(), seq.end())); + } + return t; + } + + // returns the index into seqs of the longest sequence ending at this token, or -1 + int32_t advance(llama_token token) { + state = ac.next(state, (uint32_t) token); + const int32_t p = ac.match_pattern(state); + if (p >= 0) { + state = 0; + } + return p; + } + + void reset() { state = 0; } }; struct common_reasoning_budget_ctx { @@ -41,7 +54,7 @@ struct common_reasoning_budget_ctx { token_matcher start_matcher; token_matcher end_matcher; - std::vector forced_tokens; + llama_tokens forced_tokens; int32_t budget; // maximum tokens in reasoning block int32_t remaining; // tokens remaining in budget @@ -50,6 +63,8 @@ struct common_reasoning_budget_ctx { // for forcing size_t force_pos; // next position in forced_tokens to force + + int32_t end_match; // index into end_matcher.seqs of the sequence that transitioned to DONE, -1 if none }; static const char * common_reasoning_budget_name(const struct llama_sampler * /*smpl*/) { @@ -62,7 +77,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to switch (ctx->state) { case REASONING_BUDGET_IDLE: { - if (ctx->start_matcher.advance(token)) { + if (ctx->start_matcher.advance(token) >= 0) { ctx->state = REASONING_BUDGET_COUNTING; ctx->remaining = ctx->budget; COM_TRC("activated, budget=%d tokens\n", ctx->budget); @@ -78,8 +93,10 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to case REASONING_BUDGET_COUNTING: case REASONING_BUDGET_WAITING_UTF8: { - if (ctx->end_matcher.advance(token)) { + const int32_t match = ctx->end_matcher.advance(token); + if (match >= 0) { ctx->state = REASONING_BUDGET_DONE; + ctx->end_match = match; COM_TRC("%s", "deactivated (natural end)\n"); break; } @@ -115,19 +132,25 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to break; } case REASONING_BUDGET_FORCING: + { + // track the end sequence within forced_tokens so it is also reported on DONE + const int32_t match = ctx->end_matcher.advance(token); ctx->force_pos++; if (ctx->force_pos >= ctx->forced_tokens.size()) { ctx->state = REASONING_BUDGET_DONE; + ctx->end_match = match; COM_TRC("%s", "forced sequence complete, done\n"); } break; + } case REASONING_BUDGET_DONE: // Re-arm on a new start tag: some models emit multiple blocks // per response, and each should get a fresh budget window. - if (ctx->start_matcher.advance(token)) { + if (ctx->start_matcher.advance(token) >= 0) { ctx->state = REASONING_BUDGET_COUNTING; ctx->remaining = ctx->budget; ctx->end_matcher.reset(); + ctx->end_match = -1; COM_TRC("re-activated on new start tag, budget=%d tokens\n", ctx->budget); if (ctx->remaining <= 0) { @@ -169,11 +192,12 @@ static void common_reasoning_budget_reset(struct llama_sampler * smpl) { ctx->start_matcher.reset(); ctx->end_matcher.reset(); ctx->force_pos = 0; + ctx->end_match = -1; } static struct llama_sampler * common_reasoning_budget_init_state( - const struct llama_vocab * vocab, const std::vector & start_tokens, - const std::vector & end_tokens, const std::vector & forced_tokens, + const struct llama_vocab * vocab, const std::vector & start_seqs, + const std::vector & end_seqs, const llama_tokens & forced_tokens, int32_t budget, common_reasoning_budget_state initial_state); static struct llama_sampler * common_reasoning_budget_clone(const struct llama_sampler * smpl); @@ -205,12 +229,12 @@ static struct llama_sampler * common_reasoning_budget_clone(const struct llama_s } static struct llama_sampler * common_reasoning_budget_init_state( - const struct llama_vocab * vocab, - const std::vector & start_tokens, - const std::vector & end_tokens, - const std::vector & forced_tokens, - int32_t budget, - common_reasoning_budget_state initial_state) { + const struct llama_vocab * vocab, + const std::vector & start_seqs, + const std::vector & end_seqs, + const llama_tokens & forced_tokens, + int32_t budget, + common_reasoning_budget_state initial_state) { // promote COUNTING with budget <= 0 to FORCING if (initial_state == REASONING_BUDGET_COUNTING && budget <= 0) { initial_state = REASONING_BUDGET_FORCING; @@ -220,25 +244,26 @@ static struct llama_sampler * common_reasoning_budget_init_state( /* .iface = */ &common_reasoning_budget_i, /* .ctx = */ new common_reasoning_budget_ctx { /* .vocab = */ vocab, - /* .start_matcher = */ { start_tokens, 0 }, - /* .end_matcher = */ { end_tokens, 0 }, + /* .start_matcher = */ token_matcher(start_seqs), + /* .end_matcher = */ token_matcher(end_seqs), /* .forced_tokens = */ forced_tokens, /* .budget = */ budget, /* .remaining = */ budget, /* .state = */ initial_state, /* .force_pos = */ 0, + /* .end_match = */ -1, } ); } struct llama_sampler * common_reasoning_budget_init( - const struct llama_vocab * vocab, - const std::vector & start_tokens, - const std::vector & end_tokens, - const std::vector & forced_tokens, - int32_t budget, - common_reasoning_budget_state initial_state) { - return common_reasoning_budget_init_state(vocab, start_tokens, end_tokens, forced_tokens, budget, initial_state); + const struct llama_vocab * vocab, + const std::vector & start_seqs, + const std::vector & end_seqs, + const llama_tokens & forced_tokens, + int32_t budget, + common_reasoning_budget_state initial_state) { + return common_reasoning_budget_init_state(vocab, start_seqs, end_seqs, forced_tokens, budget, initial_state); } common_reasoning_budget_state common_reasoning_budget_get_state(const struct llama_sampler * smpl) { @@ -248,6 +273,19 @@ common_reasoning_budget_state common_reasoning_budget_get_state(const struct lla return ((const common_reasoning_budget_ctx *)smpl->ctx)->state; } +const llama_tokens * common_reasoning_budget_get_end_match(const struct llama_sampler * smpl) { + if (!smpl) { + return nullptr; + } + + const auto * ctx = (const common_reasoning_budget_ctx *) smpl->ctx; + if (ctx->end_match < 0) { + return nullptr; + } + + return &ctx->end_matcher.seqs[ctx->end_match]; +} + bool common_reasoning_budget_force(struct llama_sampler * smpl) { if (!smpl) { return false; diff --git a/common/reasoning-budget.h b/common/reasoning-budget.h index 0cf689a56..1b89a04c4 100644 --- a/common/reasoning-budget.h +++ b/common/reasoning-budget.h @@ -2,6 +2,8 @@ #include "llama.h" +#include "common.h" + #include #include @@ -17,30 +19,34 @@ enum common_reasoning_budget_state { // reasoning block (e.g. between and ). // // State machine: IDLE -> COUNTING -> WAITING_UTF8 -> FORCING -> DONE -// IDLE: passthrough, watching for start_tokens sequence -// COUNTING: counting down remaining tokens, watching for natural end_tokens +// IDLE: passthrough, watching for a start sequence +// COUNTING: counting down remaining tokens, watching for a natural end sequence // WAITING_UTF8: budget exhausted, allowing tokens to complete a UTF-8 sequence // FORCING: forces forced_tokens token-by-token (all other logits -> -inf) // DONE: passthrough forever // // Parameters: // vocab - vocabulary (used for UTF-8 boundary detection; can be nullptr) -// start_tokens - token sequence that activates counting -// end_tokens - token sequence for natural deactivation +// start_seqs - token sequences, any of which activates counting +// end_seqs - token sequences, any of which naturally deactivates // forced_tokens - token sequence forced when budget expires // budget - max tokens allowed in the reasoning block // initial_state - initial state // struct llama_sampler * common_reasoning_budget_init( - const struct llama_vocab * vocab, - const std::vector & start_tokens, - const std::vector & end_tokens, - const std::vector & forced_tokens, - int32_t budget, - common_reasoning_budget_state initial_state = REASONING_BUDGET_IDLE); + const struct llama_vocab * vocab, + const std::vector & start_seqs, + const std::vector & end_seqs, + const llama_tokens & forced_tokens, + int32_t budget, + common_reasoning_budget_state initial_state = REASONING_BUDGET_IDLE); common_reasoning_budget_state common_reasoning_budget_get_state(const struct llama_sampler * smpl); +// The end sequence that transitioned the sampler to DONE, or nullptr if none +// was recorded. Cleared when a new start sequence re-arms the sampler. +const llama_tokens * common_reasoning_budget_get_end_match(const struct llama_sampler * smpl); + // Manually transition the reasoning budget sampler into the FORCING state. // Returns true if the transition occurred. bool common_reasoning_budget_force(struct llama_sampler * smpl); diff --git a/common/sampling.cpp b/common/sampling.cpp index 75a299e23..7b241e34f 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -299,7 +299,7 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, st if (!params.reasoning_budget_start.empty() && !params.reasoning_budget_end.empty() && (params.grammar_lazy || params.reasoning_budget_tokens >= 0 || params.reasoning_control)) { rbudget = common_reasoning_budget_init( vocab, - params.reasoning_budget_start, + {params.reasoning_budget_start}, params.reasoning_budget_end, params.reasoning_budget_forced, params.reasoning_budget_tokens < 0 ? INT_MAX : params.reasoning_budget_tokens); @@ -453,6 +453,17 @@ void common_sampler_accept(struct common_sampler * gsmpl, llama_token token, boo if (gsmpl->rbudget && is_generated) { llama_sampler_accept(gsmpl->rbudget, token); + + // if done, replay end sequence which may contain a grammar trigger + const bool is_done = common_reasoning_budget_get_state(gsmpl->rbudget) == REASONING_BUDGET_DONE; + if (gsmpl->grmr && !accept_grammar && is_done) { + const llama_tokens * end_seq = common_reasoning_budget_get_end_match(gsmpl->rbudget); + if (end_seq) { + for (const llama_token end_token : *end_seq) { + llama_sampler_accept(gsmpl->grmr, end_token); + } + } + } } if (gsmpl->grmr && accept_grammar) { diff --git a/common/trie.cpp b/common/trie.cpp new file mode 100644 index 000000000..b5c9666ba --- /dev/null +++ b/common/trie.cpp @@ -0,0 +1,123 @@ +#include "trie.h" + +#include "unicode.h" + +#include + +common_trie::match_result common_trie::check_at(std::string_view sv, size_t start_pos) const { + size_t current = 0; // Start at root + size_t pos = start_pos; + + // LOG_DBG("%s: checking at pos %zu, sv='%s'\n", __func__, start_pos, std::string(sv).c_str()); + + while (pos < sv.size()) { + auto result = common_parse_utf8_codepoint(sv, pos); + if (result.status != utf8_parse_result::SUCCESS) { + break; + } + + auto it = nodes[current].children.find(result.codepoint); + if (it == nodes[current].children.end()) { + // Can't continue matching + return match_result{match_result::NO_MATCH}; + } + + current = it->second; + pos += result.bytes_consumed; + + // Check if we've matched a complete word + if (nodes[current].pattern >= 0) { + return match_result{match_result::COMPLETE_MATCH}; + } + } + + // Reached end of input while still in the trie (not at root) + if (current != 0) { + // We're in the middle of a potential match + return match_result{match_result::PARTIAL_MATCH}; + } + + // Reached end at root (no match) + return match_result{match_result::NO_MATCH}; +} + +int32_t common_trie::insert(const std::string & word) { + std::vector symbols; + size_t pos = 0; + while (pos < word.length()) { + auto result = common_parse_utf8_codepoint(word, pos); + if (result.status != utf8_parse_result::SUCCESS) { + break; + } + + symbols.push_back(result.codepoint); + pos += result.bytes_consumed; + } + return insert(symbols); +} + +int32_t common_trie::insert(const std::vector & symbols) { + size_t current = 0; + for (uint32_t ch : symbols) { + auto it = nodes[current].children.find(ch); + if (it == nodes[current].children.end()) { + size_t child = create_node(); + nodes[current].children[ch] = child; + current = child; + } else { + current = it->second; + } + } + if (nodes[current].pattern < 0) { + nodes[current].pattern = n_patterns++; + } + return nodes[current].pattern; +} + +common_aho_corasick::common_aho_corasick(common_trie trie) : t(std::move(trie)) { + const auto & nodes = t.nodes; + const size_t n = nodes.size(); + + fail.assign(n, 0); + order.reserve(n); + + std::deque queue{ 0 }; + while (!queue.empty()) { + size_t u = queue.front(); + queue.pop_front(); + order.push_back(u); + for (const auto & [ch, v] : nodes[u].children) { + if (u != 0) { + size_t f = fail[u]; + while (f && nodes[f].children.find(ch) == nodes[f].children.end()) { + f = fail[f]; + } + auto it = nodes[f].children.find(ch); + fail[v] = (it != nodes[f].children.end() && it->second != v) ? it->second : 0; + } + queue.push_back(v); + } + } + + // fail[u] points to a strictly shorter suffix, so the first pattern found on + // the fail chain (including u itself) is the longest pattern ending at u + match.assign(n, -1); + for (size_t u : order) { + match[u] = nodes[u].pattern >= 0 ? nodes[u].pattern : (u != 0 ? match[fail[u]] : -1); + } + + for (const auto & node : nodes) { + for (const auto & [ch, v] : node.children) { + alphabet.insert(ch); + } + } +} + +size_t common_aho_corasick::next(size_t state, uint32_t ch) const { + const auto & nodes = t.nodes; + while (state && nodes[state].children.find(ch) == nodes[state].children.end()) { + state = fail[state]; + } + auto it = nodes[state].children.find(ch); + return it != nodes[state].children.end() ? it->second : 0; +} diff --git a/common/trie.h b/common/trie.h new file mode 100644 index 000000000..0f7b16a36 --- /dev/null +++ b/common/trie.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +// Trie for matching multiple literals. +// This is used in common_peg_until_parser and to build a GBNF exclusion grammar +struct common_trie { + struct node { + std::map children; // Use uint32_t to store Unicode codepoints + int32_t pattern = -1; // index of the pattern ending at this node, -1 if none + }; + + std::vector nodes; + + common_trie() { + create_node(); // root node + } + + common_trie(const std::vector & words) : common_trie() { + for (const auto & w : words) { + insert(w); + } + } + + enum match_result { NO_MATCH, PARTIAL_MATCH, COMPLETE_MATCH }; + + // Check if a delimiter starts at the given position + match_result check_at(std::string_view sv, size_t start_pos) const; + + // Insert a word as a sequence of Unicode codepoints, returns its pattern index + int32_t insert(const std::string & word); + + // Insert a raw symbol sequence, returns its pattern index (insertion order, + // duplicates keep the first index) + int32_t insert(const std::vector & symbols); + + private: + int32_t n_patterns = 0; + + size_t create_node() { + size_t index = nodes.size(); + nodes.emplace_back(); + return index; + } +}; + +// Aho-Corasick automaton +struct common_aho_corasick { + common_trie t; + std::vector fail; // failure links + std::vector order; // states in BFS order + std::vector match; // longest pattern ending at each state (directly or via a suffix link), -1 if none + std::set alphabet; // every character with a transition + + common_aho_corasick(common_trie trie); + + common_aho_corasick(const std::vector & strings) + : common_aho_corasick(common_trie(strings)) {} + + size_t num_states() const { return t.nodes.size(); } + bool is_terminal(size_t s) const { return match[s] >= 0; } + + // index of the longest pattern ending at this state, -1 if none + int32_t match_pattern(size_t s) const { return match[s]; } + + // follow failure links until a transition on `ch` exists. + size_t next(size_t state, uint32_t ch) const; +}; diff --git a/conversion/glm.py b/conversion/glm.py index 895cefc22..d85268a62 100644 --- a/conversion/glm.py +++ b/conversion/glm.py @@ -237,6 +237,9 @@ class GlmMoeDsaModel(DeepseekV2Model): self.gguf_writer.add_indexer_head_count(self.hparams["index_n_heads"]) self.gguf_writer.add_indexer_key_length(self.hparams["index_head_dim"]) self.gguf_writer.add_indexer_top_k(self.hparams["index_topk"]) + if (indexer_types := self.hparams.get("indexer_types")) is not None: + indexer_types = [t == "full" for t in indexer_types] + self.gguf_writer.add_indexer_types(indexer_types) @ModelBase.register("SolarOpenForCausalLM") diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index da215eafe..39e9a2a21 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -1468,7 +1468,7 @@ void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const vo struct gguf_writer_base { size_t written_bytes {0u}; - ~gguf_writer_base(void) = default; + virtual ~gguf_writer_base(void) = default; // we bet on devirtualization virtual void write(int8_t val) = 0; diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 4391bd11d..d55253e0e 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -200,6 +200,7 @@ class Keys: HEAD_COUNT = "{arch}.attention.indexer.head_count" KEY_LENGTH = "{arch}.attention.indexer.key_length" TOP_K = "{arch}.attention.indexer.top_k" + TYPES = "{arch}.attention.indexer.types" class HyperConnection: COUNT = "{arch}.hyper_connection.count" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 1e277f068..bb2159670 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -793,6 +793,10 @@ class GGUFWriter: def add_indexer_top_k(self, top_k: int) -> None: self.add_uint32(Keys.Attention.Indexer.TOP_K.format(arch=self.arch), top_k) + def add_indexer_types(self, value: Sequence[bool]) -> None: + key = Keys.Attention.Indexer.TYPES.format(arch=self.arch) + self.add_array(key, value) + def add_max_alibi_bias(self, bias: float) -> None: self.add_float32(Keys.Attention.MAX_ALIBI_BIAS.format(arch=self.arch), bias) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index e5a9a29c7..9aa3dace5 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -253,6 +253,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, "%s.attention.indexer.head_count" }, { LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, "%s.attention.indexer.key_length" }, { LLM_KV_ATTENTION_INDEXER_TOP_K, "%s.attention.indexer.top_k" }, + { LLM_KV_ATTENTION_INDEXER_TYPES, "%s.attention.indexer.types" }, { LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, "%s.attention.output_group_count" }, { LLM_KV_ATTENTION_OUTPUT_LORA_RANK, "%s.attention.output_lora_rank" }, { LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, "%s.attention.compress_rope_freq_base" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 0ef95b01b..39c55a66a 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -258,6 +258,7 @@ enum llm_kv { LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, LLM_KV_ATTENTION_INDEXER_TOP_K, + LLM_KV_ATTENTION_INDEXER_TYPES, LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, LLM_KV_ATTENTION_OUTPUT_LORA_RANK, LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, diff --git a/src/llama-grammar.cpp b/src/llama-grammar.cpp index a56f2c5b6..1f14d0f6b 100644 --- a/src/llama-grammar.cpp +++ b/src/llama-grammar.cpp @@ -1199,6 +1199,18 @@ struct llama_grammar * llama_grammar_init_impl( vec_rules[i].push_back({LLAMA_GRETYPE_END, 0}); } + // Validate that all rule references point to valid rules + for (size_t i = 0; i < n_rules; i++) { + for (const auto & elem : vec_rules[i]) { + if (elem.type == LLAMA_GRETYPE_RULE_REF) { + if (elem.value >= n_rules || vec_rules[elem.value].empty()) { + LLAMA_LOG_ERROR("invalid grammar: rule %zu references undefined rule %u\n", i, elem.value); + return nullptr; + } + } + } + } + // Check for left recursion std::vector rules_visited(n_rules); std::vector rules_in_progress(n_rules); diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 9d0683d2f..846d4c69a 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -248,6 +248,14 @@ bool llama_hparams::is_mla() const { return n_embd_head_k_mla_impl != 0 && n_embd_head_v_mla_impl != 0; } +bool llama_hparams::is_indexer_full(uint32_t il) const { + if (il < n_layer()) { + return is_indexer_full_impl[il]; + } + + GGML_ABORT("%s: il (%u) out of bounds (n_layer: %u)\n", __func__, il, n_layer()); +} + uint32_t llama_hparams::n_embd_head_k_mla() const { return is_mla() ? n_embd_head_k_mla_impl : n_embd_head_k(); } diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 8be5f28f3..747754fc0 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -227,6 +227,10 @@ struct llama_hparams { uint32_t indexer_head_size = 0; uint32_t indexer_top_k = 0; + // Indexer is "full" (1) or "shared" (0) + // Shared indexers reuse top-k from previous full layer + std::array is_indexer_full_impl; + // DeepSeek-V4 uint32_t dsv4_o_group_count = 0; uint32_t dsv4_o_lora_rank = 0; @@ -302,6 +306,8 @@ struct llama_hparams { bool is_swa(uint32_t il) const; + bool is_indexer_full(uint32_t il) const; + void set_recr_pattern(uint32_t n_pattern, bool dense_first = false); // whether or not the given layer is recurrent (for hybrid models) diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 064912a44..cc9c01c67 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -323,7 +323,7 @@ llama_kv_cache::llama_kv_cache( hparams.n_embd_head_k() % 64 == 0; // always create Hadamard rotation tensors for DeepSeek lightning indexers - if ((model.arch == LLM_ARCH_DEEPSEEK32 || model.arch == LLM_ARCH_DEEPSEEK4) && + if ((model.arch == LLM_ARCH_DEEPSEEK32 || model.arch == LLM_ARCH_DEEPSEEK4 || model.arch == LLM_ARCH_GLM_DSA) && hparams.n_embd_head_k_full == hparams.indexer_head_size) { attn_rot_k = true; } @@ -2059,7 +2059,12 @@ void llama_kv_cache::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama bool res = true; res = res && state_read_meta(io, strm, cell_count, sinfo, seq_id); - res = res && state_read_data(io, strm, cell_count, sinfo); + + try { + res = res && state_read_data(io, strm, cell_count, sinfo); + } catch (...) { + res = false; + } if (!res) { if (seq_id == -1) { diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cpp index 0d05e64b1..8e8d654b5 100644 --- a/src/llama-memory-recurrent.cpp +++ b/src/llama-memory-recurrent.cpp @@ -819,7 +819,12 @@ void llama_memory_recurrent::state_read(llama_io_read_i & io, llama_seq_id seq_i bool res = true; res = res && state_read_meta(io, cell_count, seq_id); - res = res && state_read_data(io, cell_count); + + try { + res = res && state_read_data(io, cell_count); + } catch (...) { + res = false; + } if (!res) { if (seq_id == -1) { diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 867e1fe4d..d26e2ff7a 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -281,6 +281,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k); + add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, true); add_kv(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, true); const float rope_scaling_factor = hparams.rope_freq_scale_train == 1.0f ? 0.0f : 1.0f/hparams.rope_freq_scale_train; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 682103911..b4f484334 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1223,6 +1223,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false); ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false); ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer_all); + GGML_ASSERT(hparams.n_layer_all > 0 && hparams.n_layer_all <= LLAMA_MAX_LAYERS); ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false); ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false); ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false); @@ -1268,6 +1269,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { std::fill(hparams.rope_sections.begin(), hparams.rope_sections.end(), 0); std::fill(hparams.is_swa_impl.begin(), hparams.is_swa_impl.end(), 0); std::fill(hparams.is_recr_impl.begin(), hparams.is_recr_impl.end(), llm_arch_is_recurrent(ml.get_arch()) ? 1 : 0); + std::fill(hparams.is_indexer_full_impl.begin(), hparams.is_indexer_full_impl.end(), 0); std::fill(hparams.xielu_alpha_n.begin(), hparams.xielu_alpha_n.end(), 0.0f); std::fill(hparams.xielu_alpha_p.begin(), hparams.xielu_alpha_p.end(), 0.0f); @@ -2204,6 +2206,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = nullptr; } break; case LLM_ARCH_DEEPSEEK32: + case LLM_ARCH_GLM_DSA: { res = new llama_kv_cache_dsa( *this, diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index d00d377cd..0228389b7 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -1357,6 +1357,7 @@ llama_model * llama_quant_model_from_metadata(const llama_quant_model_desc * des model->hparams.n_embd_head_k_full = desc->n_embd_head_k; model->hparams.n_embd_head_v_full = desc->n_embd_head_v; model->hparams.n_layer_all = desc->n_layer; + GGML_ASSERT(desc->n_layer > 0 && desc->n_layer <= LLAMA_MAX_LAYERS); model->hparams.n_expert = desc->n_expert; for (uint32_t i = 0; i < desc->n_layer; i++) { diff --git a/src/llama-sampler.cpp b/src/llama-sampler.cpp index 2370e91a1..6520e4181 100644 --- a/src/llama-sampler.cpp +++ b/src/llama-sampler.cpp @@ -263,6 +263,10 @@ static void llama_log_softmax(float * array, size_t size) { */ static void llama_sampler_temp_impl(llama_token_data_array * cur_p, float temp) { + if (cur_p->size == 0) { + return; + } + if (temp <= 0.0f) { // find the token with the highest logit and set the rest to -inf size_t max_i = 0; diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 682764a29..089139a49 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -1556,6 +1556,9 @@ struct llm_tokenizer_rwkv_session { token_id = node->value; token_length = position + 1; } + if (position + 1 >= text.size()) { + break; + } node = node->traverse(text[++position]); } @@ -3117,6 +3120,11 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { LLAMA_LOG_INFO("%s: printing all EOG tokens:\n", __func__); for (auto tid : special_eog_ids) { + if (tid < 0 || tid >= (llama_token) id_to_token.size()) { + LLAMA_LOG_WARN("%s: EOG token id %d is out of range (vocab size %zu), skipping\n", + __func__, tid, id_to_token.size()); + continue; + } auto & text = id_to_token[tid].text; LLAMA_LOG_INFO("%s: - %d ('%s')\n", __func__, tid, text.c_str()); @@ -3151,6 +3159,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { llama_token s_id = LLAMA_TOKEN_NULL; for (auto tid : special_eog_ids) { + if (tid < 0 || tid >= (llama_token) id_to_token.size()) { + continue; + } const auto & text = id_to_token[tid].text; if (text == "<|tool_response>") { has_tool_response = true; @@ -4329,7 +4340,11 @@ int llama_vocab::find_bpe_rank(const std::string & token_left, const std::string } std::vector llama_vocab::get_bpe_merges() const { - std::vector result(pimpl->bpe_ranks.size()); + int max_rank = -1; + for (const auto & pair : pimpl->bpe_ranks) { + max_rank = std::max(max_rank, pair.second); + } + std::vector result(max_rank + 1); for (const auto & pair : pimpl->bpe_ranks) { result[pair.second] = pair.first.first + " " + pair.first.second; diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp index 32fe6def6..df190e1f6 100644 --- a/src/models/glm-dsa.cpp +++ b/src/models/glm-dsa.cpp @@ -1,5 +1,31 @@ #include "models.h" +#include "llama-kv-cache-dsa.h" + +// https://huggingface.co/zai-org/GLM-5.2/blob/main/config.json#L26 +const std::array GLM_5_2_DEFAULT_INDEXER_TYPES = { + 1, 1, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, +}; + void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -34,10 +60,19 @@ void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) { // NextN/MTP parameters ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false); - GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl"); + GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all"); + + // BC for GLM 5, 5.1 (full indexers) without indexer_types metadata + const bool is_pre_5_2 = hparams.n_ctx_train < 1048576; + if (is_pre_5_2) { + std::fill(hparams.is_indexer_full_impl.begin(), hparams.is_indexer_full_impl.end(), 1); + } else { + hparams.is_indexer_full_impl = GLM_5_2_DEFAULT_INDEXER_TYPES; + } + ml.get_key_or_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, hparams.n_layer(), false); switch (hparams.n_layer()) { - case 79: type = LLM_TYPE_744B_A40B; break; + case 78: type = LLM_TYPE_744B_A40B; break; default: type = LLM_TYPE_UNKNOWN; } } @@ -150,3 +185,361 @@ std::unique_ptr llama_model_glm_dsa::build_arch_graph(const l return std::make_unique(*this, params); } +llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + const bool is_mla = hparams.is_mla(); + GGML_ASSERT(is_mla); + + // note: these are the actual head sizes you get when treating as MHA or after "decompression" using wv_b for MLA + const int64_t n_embd_head_k = hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_v = hparams.n_embd_head_v_mla(); + GGML_UNUSED(n_embd_head_v); + + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k - n_embd_head_qk_rope; + + const int64_t n_indexer_head = hparams.indexer_n_head; + const int64_t n_embd_indexer_head = hparams.indexer_head_size; + const int64_t n_embd_indexer_head_rope = hparams.n_rot(); + const int64_t n_embd_indexer_head_nope = n_embd_indexer_head - n_embd_indexer_head_rope; + const uint32_t n_indexer_top_k = hparams.indexer_top_k; + + const uint32_t kv_lora_rank = hparams.n_lora_kv; + + // We have to pre-scale kq_scale and attn_factor to make the YaRN RoPE work correctly. + // See https://github.com/ggml-org/llama.cpp/discussions/7416 for detailed explanation. + // And also: https://github.com/ggml-org/llama.cpp/pull/17945 [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX] + + // first cancel the adjustment from llama_hparams::yarn_attn_factor_adjust to get the original attn_factor + GGML_ASSERT(ext_factor >= 0.0f); + const float attn_factor_org = attn_factor * (1.0f + 0.1f * logf(1.0f / freq_scale)); + + // use the original attn_factor to pre-scale the kq_scale + const float mscale = attn_factor_org * (1.0f + 0.1f * hparams.rope_yarn_log_mul * logf(1.0f / freq_scale)); + const float kq_scale = 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k)); + + ggml_tensor * cur; + ggml_tensor * inpL; + + // {n_embd, n_tokens} + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + llm_graph_input_attn_k_dsa * inp_attn_dsa = build_attn_inp_k_dsa(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // Difference vs Deepseek 3.2: shared indexer layers reuse the top_k from the previous full indexer layers + // See https://huggingface.co/zai-org/GLM-5.2/blob/main/config.json#L30 + ggml_tensor * prev_top_k = nullptr; + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self_attention + { + ggml_tensor * qr = ggml_mul_mat(ctx0, model.layers[il].wq_a, cur); + cb(qr, "qr", il); + + qr = build_norm(qr, model.layers[il].attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + cb(qr, "qr", il); + + ggml_tensor * top_k = nullptr; + + // lightning indexer + if (hparams.is_indexer_full(il)) { + // "full" layer + ggml_tensor * indexer_q = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_q_b, qr); + cb(indexer_q, "indexer_q", il); + + // split into {n_embd_indexer_head_rope, n_indexer_head, n_tokens} + ggml_tensor * indexer_q_pe = + ggml_view_3d(ctx0, indexer_q, n_embd_indexer_head_rope, n_indexer_head, n_tokens, + ggml_row_size(indexer_q->type, n_embd_indexer_head), + ggml_row_size(indexer_q->type, n_embd_indexer_head) * n_indexer_head, 0); + cb(indexer_q_pe, "indexer_q_pe", il); + + // and {n_embd_indexer_head_nope, n_indexer_head, n_tokens} + ggml_tensor * indexer_q_nope = + ggml_view_3d(ctx0, indexer_q, n_embd_indexer_head_nope, n_indexer_head, n_tokens, + ggml_row_size(indexer_q->type, n_embd_indexer_head), + ggml_row_size(indexer_q->type, n_embd_indexer_head) * n_indexer_head, + ggml_row_size(indexer_q->type, n_embd_indexer_head_nope)); + cb(indexer_q_nope, "indexer_q_nope", il); + + indexer_q_pe = ggml_rope_ext(ctx0, indexer_q_pe, inp_pos, nullptr, n_rot, + LLAMA_ROPE_TYPE_NORM, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(indexer_q_pe, "indexer_q_pe", il); + + // {n_embd_indexer_head_rope + n_embd_indexer_head_nope, n_head, n_tokens} + indexer_q = ggml_concat(ctx0, indexer_q_pe, indexer_q_nope, 0); + cb(indexer_q, "indexer_q", il); + + ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur); + cb(indexer_k, "indexer_k", il); + + indexer_k = build_norm(indexer_k, model.layers[il].indexer_k_norm, model.layers[il].indexer_k_norm_b, LLM_NORM, il); + cb(indexer_k, "indexer_k", il); + + // split into {n_embd_indexer_head_rope, 1, n_tokens} + ggml_tensor * indexer_k_pe = + ggml_view_3d(ctx0, indexer_k, n_embd_indexer_head_rope, 1, n_tokens, + ggml_row_size(indexer_k->type, n_embd_indexer_head), + ggml_row_size(indexer_k->type, n_embd_indexer_head) * 1, 0); + cb(indexer_k_pe, "indexer_k_pe", il); + + // and {n_embd_indexer_head_nope, 1, n_tokens} + ggml_tensor * indexer_k_nope = + ggml_view_3d(ctx0, indexer_k, n_embd_indexer_head_nope, 1, n_tokens, + ggml_row_size(indexer_k->type, n_embd_indexer_head), + ggml_row_size(indexer_k->type, n_embd_indexer_head) * 1, + ggml_row_size(indexer_k->type, n_embd_indexer_head_nope)); + cb(indexer_k_nope, "indexer_k_nope", il); + + indexer_k_pe = ggml_rope_ext(ctx0, indexer_k_pe, inp_pos, nullptr, n_rot, + LLAMA_ROPE_TYPE_NORM, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(indexer_k_pe, "indexer_k_pe", il); + + // {n_embd_indexer_head_rope + n_embd_indexer_head_nope, 1, n_tokens} + indexer_k = ggml_concat(ctx0, indexer_k_pe, indexer_k_nope, 0); + cb(indexer_k, "indexer_k", il); + + // perform Hadamard transform on indexer q and k + indexer_q = ggml_mul_mat(ctx0, inp_attn_dsa->self_k_rot_lid, indexer_q); + cb(indexer_q, "indexer_q", il); + indexer_k = ggml_mul_mat(ctx0, inp_attn_dsa->self_k_rot_lid, indexer_k); + cb(indexer_k, "indexer_k", il); + + // store indexer keys to KV cache + const auto * mctx_lid = inp_attn_dsa->mctx->get_lid(); + const auto & k_idxs_lid = inp_attn_dsa->get_k_idxs_lid(); + ggml_build_forward_expand(gf, mctx_lid->cpy_k(ctx0, indexer_k, k_idxs_lid, il)); + + // prepare indexer weights + ggml_tensor * indexer_weights = ggml_mul_mat(ctx0, model.layers[il].indexer_proj, cur); + cb(indexer_weights, "indexer_weights", il); + + // get cached indexer keys + indexer_k = mctx_lid->get_k(ctx0, il); + + // split the batch into streams if needed + const auto n_stream = indexer_k->ne[3]; + indexer_q = ggml_view_4d(ctx0, indexer_q, indexer_q->ne[0], indexer_q->ne[1], indexer_q->ne[2]/n_stream, n_stream, indexer_q->nb[1], indexer_q->nb[2], indexer_q->nb[3]/n_stream, 0); + indexer_weights = ggml_view_4d(ctx0, indexer_weights, indexer_weights->ne[0], indexer_weights->ne[1]/n_stream, indexer_weights->ne[2], n_stream, indexer_weights->nb[1], indexer_weights->nb[2]/n_stream, indexer_weights->nb[3]/n_stream, 0); + + // pre-scale weights to avoid scaling operations on huge indexer_score tensor + indexer_weights = ggml_scale(ctx0, indexer_weights, 1.0f / sqrtf(float(n_embd_indexer_head * n_indexer_head))); + cb(indexer_weights, "indexer_weights", il); + + ggml_tensor * indexer_score = nullptr; + if (cparams.fused_lid) { + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->get_kq_mask_lid()); + cb(indexer_score, "indexer_score", il); + res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); + } else { + // calculate indexer kq + indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3); + cb(indexer_q, "indexer_q", il); + indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3); + cb(indexer_k, "indexer_k", il); + + ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q); + cb(indexer_kq, "indexer_kq", il); + + // ReLU requires contiguous tensors + indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3)); + cb(indexer_kq, "indexer_kq", il); + + // apply ReLU + indexer_score = ggml_relu(ctx0, indexer_kq); + cb(indexer_score, "indexer_score", il); + + // multiply scores by indexer weights + indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights); + cb(indexer_score, "indexer_score", il); + + // sum by q n_indexer_head dimension + indexer_score = ggml_sum_rows(ctx0, indexer_score); + cb(indexer_score, "indexer_score", il); + + // permute result to match KQ mask + indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3)); + cb(indexer_score, "indexer_score", il); + + // mask indexer scores + ggml_tensor * indexer_kq_mask = inp_attn_dsa->get_kq_mask_lid(); + indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); + cb(indexer_score, "indexer_score", il); + } + + // get indices of top k indexer scores + uint32_t n_top_k = indexer_score->ne[0] < n_indexer_top_k ? indexer_score->ne[0] : n_indexer_top_k; + top_k = ggml_cont(ctx0, ggml_top_k(ctx0, indexer_score, n_top_k)); + prev_top_k = top_k; + cb(top_k, "top_k", il); + } else { + // "shared" indexer layer - reuse top-k from a previous full layer + GGML_ASSERT(prev_top_k != nullptr && "shared indexer layer must follow a previous full indexer layer"); + top_k = prev_top_k; + cb(top_k, "top_k", il); + } + + ggml_tensor * q = ggml_mul_mat(ctx0, model.layers[il].wq_b, qr); + cb(q, "q", il); + + // split into {n_embd_head_qk_nope, n_head, n_tokens} + ggml_tensor * q_nope = + ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k), + ggml_row_size(q->type, n_embd_head_k) * n_head, 0); + cb(q_nope, "q_nope", il); + + // and {n_embd_head_qk_rope, n_head, n_tokens} + ggml_tensor * q_pe = ggml_view_3d( + ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k), + ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope)); + cb(q_pe, "q_pe", il); + + ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur); + cb(kv_cmpr_pe, "kv_cmpr_pe", il); + + // split into {kv_lora_rank, n_tokens} + ggml_tensor * kv_cmpr = + ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0); + cb(kv_cmpr, "kv_cmpr", il); + + // and {n_embd_head_qk_rope, 1, n_tokens} + ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); + cb(k_pe, "k_pe", il); + + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(q_pe, "q_pe", il); + + k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(k_pe, "k_pe", il); + + kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); + cb(kv_cmpr, "kv_cmpr", il); + + // MLA attention + { + // {n_embd_head_qk_nope, n_tokens, n_head} + q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); + cb(q_nope, "q_nope_perm", il); + + // {n_embd_head_qk_nope, kv_lora_rank, n_head} x {n_embd_head_qk_nope, n_tokens, n_head} + ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, model.layers[il].wk_b, q_nope); + cb(q_nope_absorbed, "q_nope_absorbed", il); + + // {kv_lora_rank, n_head, n_tokens} + q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3); + cb(q_nope_absorbed, "q_nope_absorbed_perm", il); + + // {n_embd_head_qk_rope + kv_lora_rank, n_head, n_tokens} + // note: rope must go first for in-place context shifting in build_rope_shift() + ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); + cb(Qcur, "Qcur", il); + + kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens); + cb(kv_cmpr, "kv_cmpr_reshape", il); + + // {n_embd_head_qk_rope + kv_lora_rank, 1, n_tokens} + ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0); + cb(Kcur, "Kcur", il); + + // {kv_lora_rank, 1, n_tokens} + ggml_tensor * Vcur = kv_cmpr; + cb(Vcur, "Vcur", il); + + // note: MLA with the absorption optimization converts into MQA (ie: GQA with 1 group) + cur = build_attn(inp_attn_dsa, + model.layers[il].wo, NULL, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, top_k, kq_scale, il); + } + } + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } else { + // MoE branch + ggml_tensor * moe_out = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + model.layers[il].ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, + model.layers[il].ffn_gate_up_exps, + model.layers[il].ffn_up_exps_s, + model.layers[il].ffn_gate_exps_s, + model.layers[il].ffn_down_exps_s); + cb(moe_out, "ffn_moe_out", il); + + // FFN shared expert + { + ggml_tensor * ffn_shexp = + build_ffn(cur, + model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s, + model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s, + model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } + } + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + cur = inpL; + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = ggml_mul_mat(ctx0, model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/models.h b/src/models/models.h index 916b83929..76daa8cc1 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1217,7 +1217,9 @@ struct llama_model_glm_dsa : public llama_model_base { void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; - using graph = llama_model_deepseek2::graph; + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; diff --git a/tools/server/README-dev.md b/tools/server/README-dev.md index e81336e5e..b4ec9f17d 100644 --- a/tools/server/README-dev.md +++ b/tools/server/README-dev.md @@ -189,7 +189,7 @@ This endpoint is intended to be used internally by the Web UI and subject to cha Get a list of tools, each tool has these fields: - `tool` (string): the ID name of the tool, to be used in POST call. Example: `read_file` - `display_name` (string): the name to be displayed on UI. Example: `Read file` -- `type` (string): always be `"builtin"` for now +- `type` (string): `"builtin"` for a built-in tool, or `"mcp"` for a tool exposed by an MCP server - `permissions` (object): a mapping string --> boolean that indicates the permission required by this tool. This is useful for the UI to ask the user before calling the tool. For now, the only permission supported is `"write"` - `definition` (object): the OAI-compat definition of this tool @@ -199,7 +199,7 @@ Invoke a tool call, request body is a JSON object with: - `tool` (string): the name of the tool - `params` (object): a mapping from argument name (string) to argument value -Returns JSON object. There are two response formats: +Returns JSON object. There are two response formats (MCP tools use the same two formats: their result content is concatenated into `plain_text_response`, and RPC or tool errors are surfaced as the `error` string): Format 1: Plain text. The text will be placed into a field called `plain_text_response`, example: diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index 31f94e023..0322e54cc 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -283,6 +283,15 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) { chatcmpl_body["max_tokens"] = response_body["max_output_tokens"]; } + if (response_body.contains("reasoning")) { + // Only "effort" is handled so far + const json & reasoning = response_body.at("reasoning"); + if (reasoning.contains("effort")) { + chatcmpl_body["reasoning_effort"] = reasoning.at("effort"); + } + chatcmpl_body.erase("reasoning"); + } + return chatcmpl_body; } diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 78b5c6819..c9109fc96 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1086,6 +1086,14 @@ json oaicompat_chat_params_parse( throw std::invalid_argument("invalid type for \"enable_thinking\" (expected boolean, got string)"); } + // Parse also the OAI "reasoning_effort": "none" specific value + if (body.contains("reasoning_effort")) { + auto reasoning_effort = json_value(body, "reasoning_effort", std::string("")); + if (reasoning_effort == "none") { + inputs.enable_thinking = false; + } // other reasoning_effort values are model-specific and not yet handled + } + inputs.force_pure_content = opt.force_pure_content; // Apply chat template to the list of messages @@ -1123,10 +1131,10 @@ json oaicompat_chat_params_parse( reasoning_budget = opt.reasoning_budget; } - if (!chat_params.thinking_end_tag.empty()) { + if (!chat_params.thinking_end_tags.empty()) { llama_params["reasoning_budget_tokens"] = reasoning_budget; llama_params["reasoning_budget_start_tag"] = chat_params.thinking_start_tag; - llama_params["reasoning_budget_end_tag"] = chat_params.thinking_end_tag; + llama_params["reasoning_budget_end_tags"] = chat_params.thinking_end_tags; llama_params["reasoning_budget_message"] = json_value(body, "reasoning_budget_message", opt.reasoning_budget_message); llama_params["reasoning_control"] = json_value(body, "reasoning_control", false); } diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 583736638..6ef797ebb 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -9,9 +9,15 @@ #define JSON_ASSERT GGML_ASSERT #include +#include +#include +#include +#include +#include +#include +#include #include #include -#include using json = nlohmann::ordered_json; @@ -376,3 +382,67 @@ server_tokens format_prompt_rerank( mtmd_context * mctx, const std::string & query, const std::string & doc); + +// simple implementation of a pipe +// used for streaming data between threads +template +struct server_pipe { + std::mutex mutex; + std::condition_variable cv; + std::queue queue; + std::atomic writer_closed{false}; + std::atomic reader_closed{false}; + + // 0 = unbounded (default) + // > 0, write() drops the oldest item once the queue is full + size_t max_size = 0; + + void close_write() { + writer_closed.store(true, std::memory_order_relaxed); + cv.notify_all(); + } + + void close_read() { + reader_closed.store(true, std::memory_order_relaxed); + cv.notify_all(); + } + + // close_on_stop = true: should_stop means the reader is gone for good, so the writer is told the pipe is broken. + // close_on_stop = false: should_stop is a per-read deadline and further reads still come, so the pipe stays usable. + bool read(T & output, const std::function & should_stop, bool close_on_stop = true) { + std::unique_lock lk(mutex); + constexpr auto poll_interval = std::chrono::milliseconds(500); + while (true) { + if (!queue.empty()) { + output = std::move(queue.front()); + queue.pop(); + return true; + } + if (writer_closed.load()) { + return false; // clean EOF + } + if (should_stop && should_stop()) { // a null should_stop means "never stop" + if (close_on_stop) { + close_read(); // signal broken pipe to writer + } + return false; // cancelled / deadline reached + } + cv.wait_for(lk, poll_interval); + } + } + + bool write(T && data) { + std::lock_guard lk(mutex); + if (reader_closed.load()) { + return false; // broken pipe + } + if (max_size > 0) { + while (queue.size() >= max_size) { + queue.pop(); // drop oldest to stay bounded + } + } + queue.push(std::move(data)); + cv.notify_one(); + return true; + } +}; diff --git a/tools/server/server-mcp.cpp b/tools/server/server-mcp.cpp new file mode 100644 index 000000000..aa87afeb0 --- /dev/null +++ b/tools/server/server-mcp.cpp @@ -0,0 +1,836 @@ +#include "server-mcp.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# include +# include +#else +# include +# include +# include +# include +extern char ** environ; +#endif + +// read NDJSON lines from a child pipe, calling on_line per line until `running` clears, EOF/error, or on_line returns false. +// polled, not blocking: a grandchild can inherit the pipe's write end and hold it open (terminate() kills only the direct child), so a blocking read would hang teardown on an EOF that never comes. +static void mcp_pump_ndjson(FILE * f, std::atomic & running, + const std::function & on_line) { + if (!f) { + return; + } + const int poll_ms = 50; + const size_t max_line = 8 * 1024 * 1024; // drop any single NDJSON line larger than this, so a child that never emits '\n' can't grow buf without bound +#if defined(_WIN32) + HANDLE h = (HANDLE) _get_osfhandle(_fileno(f)); +#else + int fd = fileno(f); + int fl = fcntl(fd, F_GETFL, 0); + if (fl >= 0) { + fcntl(fd, F_SETFL, fl | O_NONBLOCK); + } +#endif + std::string buf; + bool skipping = false; // discarding an over-long line until its terminating newline + char chunk[4096]; + while (running.load()) { + size_t n = 0; +#if defined(_WIN32) + DWORD avail = 0; + if (!PeekNamedPipe(h, NULL, 0, NULL, &avail, NULL)) { + break; // pipe broken / child gone + } + if (avail == 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(poll_ms)); + continue; + } + DWORD to_read = avail < (DWORD) sizeof(chunk) ? avail : (DWORD) sizeof(chunk); + DWORD got = 0; + if (!ReadFile(h, chunk, to_read, &got, NULL) || got == 0) { + break; + } + n = (size_t) got; +#else + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + pfd.revents = 0; + int pr = poll(&pfd, 1, poll_ms); + if (pr < 0) { + if (errno == EINTR) { + continue; + } + break; + } + if (pr == 0) { + continue; // timeout -> re-check running + } + if (pfd.revents & (POLLERR | POLLNVAL)) { + break; + } + ssize_t r = read(fd, chunk, sizeof(chunk)); + if (r < 0) { + if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { + continue; + } + break; + } + if (r == 0) { + break; // EOF: child (and any pipe writers) closed the stream + } + n = (size_t) r; +#endif + buf.append(chunk, n); + + // resync after an over-long, unterminated line: discard bytes until the next newline + if (skipping) { + size_t nl = buf.find('\n'); + if (nl == std::string::npos) { + if (buf.size() > max_line) { + buf.clear(); // stay bounded while waiting for a terminator + } + continue; + } + buf.erase(0, nl + 1); + skipping = false; + } + + size_t pos; + while ((pos = buf.find('\n')) != std::string::npos) { + std::string line = buf.substr(0, pos); + buf.erase(0, pos + 1); + if (!line.empty() && line.back() == '\r') { + line.pop_back(); + } + if (line.empty()) { + continue; + } + if (!on_line(std::move(line))) { + return; + } + } + + // a partial line already larger than the cap and still no newline: drop it to avoid unbounded growth + if (buf.size() > max_line) { + SRV_WRN("MCP: dropping oversized line (> %zu bytes) from child pipe\n", max_line); + buf.clear(); + skipping = true; + } + } +} + +// +// server_mcp_server_config +// + +std::vector server_mcp_server_config::parse_from_json(const std::string & json_str) { + return parse_cursor_format(json::parse(json_str)); +} + +std::vector server_mcp_server_config::parse_cursor_format(const json & j) { + std::vector result; + + if (!j.contains("mcpServers") || !j.at("mcpServers").is_object()) { + return result; + } + + for (const auto & [name, cfg] : j.at("mcpServers").items()) { + server_mcp_server_config sc; + sc.name = name; + sc.command = cfg.value("command", std::string()); + sc.cwd = cfg.value("cwd", std::string()); + sc.timeout_ms = cfg.value("timeout_ms", sc.timeout_ms); + + if (cfg.contains("args") && cfg.at("args").is_array()) { + for (const auto & a : cfg.at("args")) { + sc.args.push_back(a.get()); + } + } + if (cfg.contains("env") && cfg.at("env").is_object()) { + for (const auto & [k, v] : cfg.at("env").items()) { + sc.env[k] = v.get(); + } + } + + if (sc.command.empty()) { + SRV_WRN("MCP server '%s' has no command, skipping\n", name.c_str()); + continue; + } + result.push_back(std::move(sc)); + } + + return result; +} + + +// +// server_mcp_transport +// + +static constexpr const char * MCP_PROTOCOL_VERSION = "2024-11-05"; + +static std::string rpc_error_message(const json & resp) { + if (resp.contains("error")) { + const json & e = resp.at("error"); + if (e.is_object()) { + return e.value("message", "unknown error"); + } + if (e.is_string()) { + return e.get(); + } + } + return "unknown error"; +} + +// normalize an MCP tools/call result to the /tools contract (see README-dev.md): +// concat text parts of result.content[], and surface an isError result +static json mcp_result_to_response(const json & result) { + std::string text; + if (result.contains("content") && result.at("content").is_array()) { + for (const auto & part : result.at("content")) { + if (part.is_object() && part.value("type", "") == "text") { + if (!text.empty()) { + text += "\n"; + } + text += part.value("text", ""); + } + } + } + if (result.is_object() && result.value("isError", false)) { + return {{"error", text.empty() ? "MCP tool returned an error" : text}}; + } + return {{"plain_text_response", text}}; +} + +json server_mcp_transport::send_rpc(const json & request, const std::function & should_stop) { + if (!to_server.write(request.dump())) { + return {{"error", {{"code", -32603}, {"message", "transport closed"}}}}; + } + + const bool has_id = request.contains("id"); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms); + auto stop = [&]() { + return (should_stop && should_stop()) || std::chrono::steady_clock::now() >= deadline; + }; + + std::string frame; + while (from_server.read(frame, stop, false)) { + json reply; + try { + reply = json::parse(frame); + } catch (...) { + if (std::chrono::steady_clock::now() >= deadline) { + break; + } + continue; // skip malformed frame + } + // no id: a notification. mismatched id: a stale reply from a timed-out request (ids are monotonic, never a future one) + if (!has_id || (reply.contains("id") && reply.at("id") == request.at("id"))) { + return reply; + } + if (std::chrono::steady_clock::now() >= deadline) { + break; // a flood of notifications must not outrun the deadline + } + } + + if (should_stop && should_stop()) { + return {{"error", {{"code", -32603}, {"message", "cancelled"}}}}; + } + if (std::chrono::steady_clock::now() >= deadline) { + return {{"error", {{"code", -32603}, {"message", "request timed out"}}}}; + } + return {{"error", {{"code", -32603}, {"message", "transport closed"}}}}; +} + +bool server_mcp_transport::ensure_init(const std::function & should_stop) { + if (initialized) { + return true; + } + + json init_req = { + {"jsonrpc", "2.0"}, + {"id", next_id++}, + {"method", "initialize"}, + {"params", { + {"protocolVersion", MCP_PROTOCOL_VERSION}, + {"capabilities", json::object()}, + {"clientInfo", {{"name", "llama.cpp"}, {"version", "1.0"}}}, + }}, + }; + json resp = send_rpc(init_req, should_stop); + if (!resp.contains("result")) { + last_error = "initialize failed: " + rpc_error_message(resp); + return false; + } + + // notifications/initialized: no id, no reply expected + json notif = {{"jsonrpc", "2.0"}, {"method", "notifications/initialized"}}; + to_server.write(notif.dump()); + + initialized = true; + return true; +} + +std::vector server_mcp_transport::list_tools(const std::function & should_stop) { + std::lock_guard lock(rpc_mutex); + if (!ensure_init(should_stop)) { + return {}; + } + if (!tools.empty()) { + return tools; + } + + json req = {{"jsonrpc", "2.0"}, {"id", next_id++}, {"method", "tools/list"}}; + json resp = send_rpc(req, should_stop); + if (!resp.contains("result")) { + last_error = "tools/list failed: " + rpc_error_message(resp); + return {}; + } + + const json & result = resp.at("result"); + if (result.contains("tools") && result.at("tools").is_array()) { + for (const auto & t : result.at("tools")) { + server_mcp_tool_def def; + def.server_name = name; + def.name = t.value("name", ""); + def.description = t.value("description", ""); + if (t.contains("inputSchema")) { + def.input_schema = t.at("inputSchema"); + } + tools.push_back(std::move(def)); + } + } + return tools; +} + +json server_mcp_transport::call_tool(const std::string & tool_name, + const json & arguments, + const std::function & should_stop) { + std::lock_guard lock(rpc_mutex); + if (!ensure_init(should_stop)) { + return {{"error", last_error}}; + } + + json req = { + {"jsonrpc", "2.0"}, + {"id", next_id++}, + {"method", "tools/call"}, + {"params", {{"name", tool_name}, {"arguments", arguments}}}, + }; + json resp = send_rpc(req, should_stop); + if (resp.contains("error")) { + return {{"error", rpc_error_message(resp)}}; + } + if (resp.contains("result")) { + return mcp_result_to_response(resp.at("result")); + } + return {{"error", "invalid response from MCP server"}}; +} + +// +// server_mcp_stdio +// + +struct server_mcp_stdio::process_handle { + subprocess_s sp; + FILE * in = nullptr; // child stdin + FILE * out = nullptr; // child stdout + FILE * err = nullptr; // child stderr +}; + +#if defined(_WIN32) +// config strings are UTF-8 (from JSON) and subprocess.h converts them with CP_UTF8, so inputs must be UTF-8, not the active code page +static std::wstring windows_utf8_to_wide(const std::string & s) { + if (s.empty()) { + return std::wstring(); + } + int n = MultiByteToWideChar(CP_UTF8, 0, s.data(), (int) s.size(), NULL, 0); + if (n <= 0) { + return std::wstring(); + } + std::wstring w((size_t) n, L'\0'); + MultiByteToWideChar(CP_UTF8, 0, s.data(), (int) s.size(), &w[0], n); + return w; +} + +static std::string windows_wide_to_utf8(const wchar_t * s, int len /* -1 for NUL-terminated */) { + int n = WideCharToMultiByte(CP_UTF8, 0, s, len, NULL, 0, NULL, NULL); + if (n <= 0) { + return std::string(); + } + std::string out((size_t) n, '\0'); + WideCharToMultiByte(CP_UTF8, 0, s, len, &out[0], n, NULL, NULL); + if (len == -1 && !out.empty() && out.back() == '\0') { + out.pop_back(); // drop the terminator WideCharToMultiByte counts for -1 + } + return out; +} +#endif + +static std::string mcp_resolve_command(const std::string & command) { +#if defined(_WIN32) + // For Windows: make sure we handle ".exe" correctly, as well as UTF-8 + std::wstring wcmd = windows_utf8_to_wide(command); + wchar_t buf[MAX_PATH * 4]; + const DWORD cap = (DWORD) (sizeof(buf) / sizeof(buf[0])); + + auto search = [&](const wchar_t * ext) -> std::string { + DWORD n = SearchPathW(NULL, wcmd.c_str(), ext, cap, buf, NULL); + return (n > 0 && n < cap) ? windows_wide_to_utf8(buf, (int) n) : std::string(); + }; + + std::string found = search(NULL); // exact path / already-extensioned / .exe on PATH + if (!found.empty()) { + return found; + } + + std::wstring pathext; + DWORD need = GetEnvironmentVariableW(L"PATHEXT", NULL, 0); + if (need > 0) { + pathext.resize(need); + DWORD got = GetEnvironmentVariableW(L"PATHEXT", &pathext[0], need); + pathext.resize(got); + } + if (pathext.empty()) { + pathext = L".COM;.EXE;.BAT;.CMD"; + } + for (size_t start = 0; start <= pathext.size();) { + size_t sep = pathext.find(L';', start); + std::wstring ext = pathext.substr(start, sep == std::wstring::npos ? std::wstring::npos : sep - start); + if (!ext.empty()) { + found = search(ext.c_str()); + if (!found.empty()) { + return found; + } + } + if (sep == std::wstring::npos) { + break; + } + start = sep + 1; + } + return command; // give up and let subprocess.h report the spawn error +#else + return command; +#endif // _WIN32 +} + +static std::vector mcp_parent_env() { + std::vector env; +#if defined(_WIN32) + LPWCH block = GetEnvironmentStringsW(); + if (block) { + for (LPWCH e = block; *e; e += wcslen(e) + 1) { + env.emplace_back(windows_wide_to_utf8(e, -1)); + } + FreeEnvironmentStringsW(block); + } +#else + if (environ) { + for (char ** e = environ; *e; ++e) { + env.emplace_back(*e); + } + } +#endif + return env; +} + +// parent env with the config overrides applied, in "KEY=VALUE" form +static std::vector mcp_build_env(const std::map & overrides) { + std::vector env; + for (auto & e : mcp_parent_env()) { + size_t eq = e.find('='); + std::string key = eq == std::string::npos ? e : e.substr(0, eq); + if (overrides.find(key) == overrides.end()) { + env.push_back(e); + } + } + for (auto & [k, v] : overrides) { + env.push_back(k + "=" + v); + } + return env; +} + +server_mcp_stdio::server_mcp_stdio(const server_mcp_server_config & config) : config(config) { + name = config.name; + timeout_ms = config.timeout_ms; + // bound the reply queue: send_rpc only drains during a call, so unsolicited notifications would otherwise grow it without limit + from_server.max_size = 65536; +} + +server_mcp_stdio::~server_mcp_stdio() { + join_pumps(); +} + +bool server_mcp_stdio::start() { + std::vector argv_s; + argv_s.push_back(mcp_resolve_command(config.command)); + argv_s.insert(argv_s.end(), config.args.begin(), config.args.end()); + + int options = subprocess_option_no_window | subprocess_option_search_user_path; + std::vector envp_s; + if (config.env.empty()) { + options |= subprocess_option_inherit_environment; + } else { + envp_s = mcp_build_env(config.env); + } + + auto to_ptrs = [](std::vector & v) { + std::vector p; + p.reserve(v.size() + 1); + for (auto & s : v) { + p.push_back(s.c_str()); + } + p.push_back(nullptr); + return p; + }; + auto argv = to_ptrs(argv_s); + auto envp = to_ptrs(envp_s); + + auto handle = std::make_unique(); + int rc = subprocess_create_ex(argv.data(), options, + config.env.empty() ? nullptr : envp.data(), + config.cwd.empty() ? nullptr : config.cwd.c_str(), + &handle->sp); + if (rc != 0) { + SRV_WRN("MCP '%s': failed to spawn '%s'\n", config.name.c_str(), config.command.c_str()); + return false; + } + handle->in = subprocess_stdin(&handle->sp); + handle->out = subprocess_stdout(&handle->sp); + handle->err = subprocess_stderr(&handle->sp); + + proc = std::move(handle); + running.store(true); + reader = std::thread([this] { reader_loop(); }); + writer = std::thread([this] { writer_loop(); }); + errlog = std::thread([this] { errlog_loop(); }); + return true; +} + +void server_mcp_stdio::close() { + join_pumps(); +} + +bool server_mcp_stdio::is_alive() const { + return running.load(); +} + +std::string server_mcp_stdio::diagnostics() { + std::string out; + { + std::lock_guard lock(rpc_mutex); // last_error is written by send_rpc's callers + out = last_error; + } + std::lock_guard lk(err_mu); + if (!err_tail.empty()) { + if (!out.empty()) { + out += "; "; + } + out += "last stderr: " + err_tail; + } + return out; +} + +void server_mcp_stdio::reader_loop() { + mcp_pump_ndjson(proc->out, running, [this](std::string && line) { + return from_server.write(std::move(line)); // false => consumer gone, stop + }); + running.store(false); + to_server.close_write(); // stop the writer + from_server.close_write(); // EOF to any waiting caller +} + +// write all of `data` to child stdin, non-blocking and polled so teardown never hangs (a grandchild can hold the read end of a full pipe open). returns false on error/close/shutdown. +static bool mcp_write_all(FILE * f, const std::string & data, std::atomic & running) { + if (!f) { + return false; + } + size_t total = 0; +#if defined(_WIN32) + HANDLE h = (HANDLE) _get_osfhandle(_fileno(f)); + DWORD nowait = PIPE_NOWAIT; + SetNamedPipeHandleState(h, &nowait, NULL, NULL); + while (total < data.size() && running.load()) { + DWORD written = 0; + BOOL ok = WriteFile(h, data.data() + total, (DWORD) (data.size() - total), &written, NULL); + if (ok && written > 0) { + total += written; + continue; + } + if (!ok) { + DWORD err = GetLastError(); + if (err != ERROR_NO_DATA && err != ERROR_PIPE_BUSY) { + return false; + } + } + // backpressure (pipe full) is rare for small JSON-RPC frames; sleep rather than spin. + // no writable-wait exists for a PIPE_NOWAIT anonymous pipe, so this polls like the POSIX poll() path. + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } +#else + int fd = fileno(f); + int fl = fcntl(fd, F_GETFL, 0); + if (fl >= 0) { + fcntl(fd, F_SETFL, fl | O_NONBLOCK); + } + while (total < data.size() && running.load()) { + ssize_t n = write(fd, data.data() + total, data.size() - total); + if (n > 0) { + total += (size_t) n; + continue; + } + if (n == 0) { + return false; + } + if (errno == EINTR) { + continue; + } + if (errno != EAGAIN && errno != EWOULDBLOCK) { + return false; + } + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLOUT; + pfd.revents = 0; + int pr = poll(&pfd, 1, 50); + if (pr < 0) { + if (errno == EINTR) { + continue; + } + return false; + } + if (pfd.revents & (POLLERR | POLLNVAL | POLLHUP)) { + return false; + } + } +#endif + return total == data.size(); +} + +void server_mcp_stdio::writer_loop() { + auto should_stop = [this] { return !running.load(); }; + std::string msg; + while (to_server.read(msg, should_stop)) { + msg.push_back('\n'); + if (!mcp_write_all(proc->in, msg, running)) { + break; // child gone or shutting down + } + } + running.store(false); + to_server.close_read(); // fail fast on any further send_rpc write + from_server.close_write(); // wake any caller waiting for a reply +} + +void server_mcp_stdio::errlog_loop() { + static constexpr size_t ERR_TAIL_MAX = 4096; + // drain stderr (an undrained pipe blocks the child): + // log it, and keep a bounded tail for reporting when the server dies + mcp_pump_ndjson(proc->err, running, [this](std::string && line) { + SRV_DBG("MCP '%s' stderr: %s\n", name.c_str(), line.c_str()); + std::lock_guard lk(err_mu); + err_tail += line; + err_tail += '\n'; + if (err_tail.size() > ERR_TAIL_MAX) { + err_tail.erase(0, err_tail.size() - ERR_TAIL_MAX); + } + return true; + }); +} + +void server_mcp_stdio::join_pumps() { + if (!proc) { + return; + } + running.store(false); + to_server.close_write(); // wake the writer if it waits for a message + from_server.close_write(); // wake any caller waiting for a reply + + subprocess_terminate(&proc->sp); // child death unblocks the blocked fread/fwrite + + if (writer.joinable()) writer.join(); + if (reader.joinable()) reader.join(); + if (errlog.joinable()) errlog.join(); + + subprocess_join(&proc->sp, nullptr); // reap the child: destroy() never waits, so the pid would stay a zombie for the process lifetime + subprocess_destroy(&proc->sp); // safe now: no thread touches the FILE* anymore + proc.reset(); +} + + +// +// server_mcp +// + +static constexpr int MCP_COOLDOWN_SECONDS = 5; +static constexpr int MCP_WARMUP_TIMEOUT_SECONDS = 10; // cap per-server tool discovery at startup + +server_mcp::~server_mcp() { + shutdown(); + + std::vector> to_close; + { + std::lock_guard lock(mutex); + for (auto & [name, t] : transports) { + to_close.push_back(std::move(t)); + } + transports.clear(); + } + for (auto & t : to_close) { + t->close(); + } +} + +std::shared_ptr server_mcp::create_transport(const server_mcp_server_config & cfg) { + return std::make_shared(cfg); +} + +void server_mcp::shutdown() { + stopping.store(true); +} + +const server_mcp_server_config * server_mcp::find_config(const std::string & name) const { + for (const auto & c : configs) { + if (c.name == name) { + return &c; + } + } + return nullptr; +} + +void server_mcp::start(const common_params & params) { + auto append = [this](const std::string & json_str) { + try { + auto parsed = server_mcp_server_config::parse_from_json(json_str); + if (parsed.empty()) { + SRV_WRN("%s", "MCP config: no servers found in JSON\n"); + } + for (auto & p : parsed) { + // names must be unique across both config sources: get_or_create / find_config key on the name + if (find_config(p.name)) { + SRV_WRN("MCP config: duplicate server name '%s', skipping\n", p.name.c_str()); + continue; + } + configs.push_back(std::move(p)); + } + } catch (const std::exception & e) { + throw std::runtime_error(std::string("failed to parse MCP config JSON: ") + e.what()); + } + }; + if (!params.mcp_servers_config.empty()) { + std::ifstream f = fs_open_ifstream(params.mcp_servers_config, std::ios::in); + if (!f) { + throw std::runtime_error("failed to open MCP config file: " + params.mcp_servers_config); + } + std::stringstream ss; + ss << f.rdbuf(); + append(ss.str()); + } + if (!params.mcp_servers_json.empty()) { + append(params.mcp_servers_json); + } + + if (configs.empty()) { + return; + } + + std::vector discovered; + for (const auto & cfg : configs) { + auto t = create_transport(cfg); + if (!t->start()) { + SRV_WRN("MCP warmup: failed to spawn '%s': %s\n", cfg.name.c_str(), t->diagnostics().c_str()); + continue; + } + // bound warmup per server so an unresponsive one can't stall startup for the full per-call timeout + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(MCP_WARMUP_TIMEOUT_SECONDS); + auto should_stop = [this, deadline]() { + return stopping.load() || std::chrono::steady_clock::now() >= deadline; + }; + auto tools = t->list_tools(should_stop); + SRV_INF("MCP warmup: '%s' discovered %zu tools\n", cfg.name.c_str(), tools.size()); + discovered.insert(discovered.end(), tools.begin(), tools.end()); + t->close(); + } + + std::lock_guard lock(mutex); + registry.swap(discovered); +} + +std::vector server_mcp::list_tools() const { + std::lock_guard lock(mutex); + return registry; +} + +json server_mcp::call_tool(const std::string & server_name, + const std::string & tool_name, + const json & arguments, + const std::function & should_stop) { + auto transport = get_or_create(server_name); + if (!transport) { + return {{"error", "MCP server unavailable: " + server_name}}; + } + + auto stop = [this, &should_stop]() { + return stopping.load() || (should_stop && should_stop()); + }; + return transport->call_tool(tool_name, arguments, stop); +} + +std::shared_ptr server_mcp::get_or_create(const std::string & name) { + std::vector> to_close; // closed after unlock + std::shared_ptr result; + + { + std::lock_guard lock(mutex); + if (stopping.load()) { + return nullptr; + } + + auto now = std::chrono::steady_clock::now(); + auto dead_it = dead_servers.find(name); + if (dead_it != dead_servers.end()) { + if (now < dead_it->second) { + return nullptr; + } + dead_servers.erase(dead_it); + } + + auto it = transports.find(name); + if (it != transports.end()) { + if (it->second->is_alive()) { + return it->second; + } + SRV_WRN("MCP '%s' is no longer alive: %s\n", name.c_str(), it->second->diagnostics().c_str()); + to_close.push_back(std::move(it->second)); + transports.erase(it); + } + + const server_mcp_server_config * cfg = find_config(name); + if (cfg) { + auto fresh = create_transport(*cfg); + if (fresh->start() && fresh->is_alive()) { + transports[name] = fresh; + result = fresh; + } else { + SRV_WRN("MCP '%s': failed to start: %s\n", name.c_str(), fresh->diagnostics().c_str()); + to_close.push_back(std::move(fresh)); + dead_servers[name] = now + std::chrono::seconds(MCP_COOLDOWN_SECONDS); + } + } + } + + for (auto & t : to_close) { + t->close(); // blocking call, no leaks + } + + return result; +} + diff --git a/tools/server/server-mcp.h b/tools/server/server-mcp.h new file mode 100644 index 000000000..c7f33a3f7 --- /dev/null +++ b/tools/server/server-mcp.h @@ -0,0 +1,176 @@ +#pragma once + +#include "server-common.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// +// Configuration (Cursor-compatible "mcpServers" JSON) +// + +struct server_mcp_server_config { + std::string name; // config key, e.g. "filesystem" + std::string command; + std::vector args; + std::map env; // merged over the parent env + std::string cwd; + int timeout_ms = 30000; // per-tool-call timeout + + // throw on parse errors; missing "mcpServers" yields an empty list; entries without a "command" are skipped + static std::vector parse_from_json(const std::string & json_str); + static std::vector parse_cursor_format(const json & j); +}; + +// a tool advertised by an MCP server +struct server_mcp_tool_def { + std::string server_name; + std::string name; // bare tool name, no "_" prefix + std::string description; + json input_schema; // JSON Schema for the arguments, or null +}; + +// +// server_mcp_transport: one MCP server session. +// +// caller --send_rpc--> to_server --[writer]--> framing --> server +// caller <--send_rpc-- from_server <--[reader]-- framing <-- server +// +// each queue item is one complete serialized JSON message. +// subclass owns byte I/O and framing; base owns JSON and the JSON-RPC session (handshake, id correlation). +// + +struct server_mcp_transport { + std::string name; + int timeout_ms = 30000; + + server_pipe to_server; // serialized messages we send to the server + server_pipe from_server; // serialized messages read from the server + + virtual ~server_mcp_transport() = default; + + virtual bool start() = 0; + virtual void close() = 0; // blocking and idempotent + virtual bool is_alive() const = 0; // never blocks behind an in-flight send_rpc() + + // human-readable diagnostics for logging when the transport fails/dies + // (example: last RPC error, plus any transport-specific detail) + // may run on a different thread than send_rpc(), so last_error is read under rpc_mutex + virtual std::string diagnostics() { + std::lock_guard lock(rpc_mutex); + return last_error; + } + + std::vector list_tools(const std::function & should_stop); + + json call_tool(const std::string & tool_name, + const json & arguments, + const std::function & should_stop); + +protected: + // per-transport: send_rpc() holds it across the reply wait, so sharing it would stall every server behind one slow call. guards all members below. + std::mutex rpc_mutex; + uint64_t next_id = 1; // reset to 1 per (re)spawn + bool initialized = false; + std::string last_error; + std::vector tools; + + // both assume rpc_mutex is already held by the public caller + bool ensure_init(const std::function & should_stop); // initialize handshake, once + json send_rpc(const json & request, const std::function & should_stop); // returns the reply or an {"error": ...} +}; + +// +// server_mcp_stdio: child process, NDJSON JSON-RPC over stdio (stderr drained to the debug log) +// + +struct server_mcp_stdio : server_mcp_transport { + explicit server_mcp_stdio(const server_mcp_server_config & config); + ~server_mcp_stdio() override; + + bool start() override; + void close() override; + bool is_alive() const override; + std::string diagnostics() override; + +private: + server_mcp_server_config config; + + // defined in the .cpp so stays out of this header + struct process_handle; + std::unique_ptr proc; + + std::thread reader; // child stdout -> NDJSON de-framing -> from_server + std::thread writer; // to_server -> NDJSON framing -> child stdin + std::thread errlog; // child stderr -> debug log (must be drained or the child blocks) + + // cleared by close() or by the reader on stdout EOF; read without rpc_mutex + std::atomic running{false}; + + // bounded tail of the child's stderr, for diagnostics when it dies + std::mutex err_mu; + std::string err_tail; + + void reader_loop(); + void writer_loop(); + void errlog_loop(); + void join_pumps(); +}; + +// +// server_mcp +// declare before the HTTP context so it outlives every /tools handler. +// + +class server_mcp { +public: + server_mcp() = default; + ~server_mcp(); + + // parse the MCP config from params (file and/or inline JSON), + // then spawn each server once, list its tools, and shut it down + // throws on config parse errors; spawn failures are logged. + void start(const common_params & params); + + // true until start() has parsed at least one server from the config + bool empty() const { return configs.empty(); } + + std::vector list_tools() const; + + // lazily (re)spawns the transport. returns the MCP result or an {"error": ...}. should_stop is OR-ed with the manager's cancel flag. + json call_tool(const std::string & server_name, + const std::string & tool_name, + const json & arguments, + const std::function & should_stop = nullptr); + + // flip the cancel flag so in-flight calls return; blocking teardown is in the destructor. call before the HTTP server drains. + // note: multiple calls are idempotent + void shutdown(); + +private: + std::vector configs; + + mutable std::mutex mutex; // guards transports, dead_servers, registry + + // shared_ptr: call_tool() hands a transport to the caller and drops the lock for the blocking RPC, so a concurrent evict/respawn must not destroy it mid-call + std::map> transports; + std::map dead_servers; // spawn-failure cooldown + std::vector registry; + + std::atomic stopping{false}; + + const server_mcp_server_config * find_config(const std::string & name) const; + + // the only place that names a concrete transport + std::shared_ptr create_transport(const server_mcp_server_config & cfg); + + // nullptr during cooldown or shutdown + std::shared_ptr get_or_create(const std::string & name); +}; diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 9eac58e9d..ba6378814 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1944,53 +1944,6 @@ void server_models_routes::init_routes() { // server_http_proxy // -// simple implementation of a pipe -// used for streaming data between threads -template -struct pipe_t { - std::mutex mutex; - std::condition_variable cv; - std::queue queue; - std::atomic writer_closed{false}; - std::atomic reader_closed{false}; - void close_write() { - writer_closed.store(true, std::memory_order_relaxed); - cv.notify_all(); - } - void close_read() { - reader_closed.store(true, std::memory_order_relaxed); - cv.notify_all(); - } - bool read(T & output, const std::function & should_stop) { - std::unique_lock lk(mutex); - constexpr auto poll_interval = std::chrono::milliseconds(500); - while (true) { - if (!queue.empty()) { - output = std::move(queue.front()); - queue.pop(); - return true; - } - if (writer_closed.load()) { - return false; // clean EOF - } - if (should_stop()) { - close_read(); // signal broken pipe to writer - return false; // cancelled / reader no longer alive - } - cv.wait_for(lk, poll_interval); - } - } - bool write(T && data) { - std::lock_guard lk(mutex); - if (reader_closed.load()) { - return false; // broken pipe - } - queue.push(std::move(data)); - cv.notify_one(); - return true; - } -}; - static std::string to_lower_copy(const std::string & value) { std::string lowered(value.size(), '\0'); std::transform(value.begin(), value.end(), lowered.begin(), [](unsigned char c) { return std::tolower(c); }); @@ -2100,7 +2053,7 @@ server_http_proxy::server_http_proxy( ) { // shared between reader and writer threads auto cli = std::make_shared(host, port); - auto pipe = std::make_shared>(); + auto pipe = std::make_shared>(); if (scheme == "https") { #ifdef CPPHTTPLIB_OPENSSL_SUPPORT diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index 89026eb4e..e880f4ca7 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -390,21 +390,40 @@ std::vector> make_llama_cmpl_schema(const common_params & ctx.params.sampling.reasoning_budget_start = common_tokenize(ctx.vocab, data.at("reasoning_budget_start_tag").get(), false, true); })); - add((new field_str("reasoning_budget_end_tag")) - ->set_desc("Token string marking the end of the reasoning budget section") + add((new field_json("reasoning_budget_end_tags")) + ->add_alias("reasoning_budget_end_tag") + ->set_desc("Token strings marking the end of the reasoning budget section; the first is forced when the budget expires") ->set_handler([&](field_eval_context & ctx, const json & data) { GGML_ASSERT(ctx.vocab != nullptr); - std::string end_tag = data.at("reasoning_budget_end_tag").get(); - ctx.params.sampling.reasoning_budget_end = common_tokenize(ctx.vocab, end_tag, false, true); + ctx.params.sampling.reasoning_budget_end.clear(); + if (data.contains("reasoning_budget_end_tags")) { + for (const auto & t : data.at("reasoning_budget_end_tags")) { + std::string tag = t.get(); + if (!tag.empty()) { + ctx.params.sampling.reasoning_budget_end.push_back(common_tokenize(ctx.vocab, tag, false, true)); + } + } + } else if (data.contains("reasoning_budget_end_tag")) { + std::string tag = data.at("reasoning_budget_end_tag").get(); + if (!tag.empty()) { + ctx.params.sampling.reasoning_budget_end.push_back(common_tokenize(ctx.vocab, tag, false, true)); + } + } })); add((new field_str("reasoning_budget_message")) ->set_desc("Message to prepend to the reasoning budget end tag when forcing it") ->set_handler([&](field_eval_context & ctx, const json & data) { GGML_ASSERT(ctx.vocab != nullptr); - std::string end_tag = json_value(data, "reasoning_budget_end_tag", std::string()); - std::string message = data.at("reasoning_budget_message").get(); - ctx.params.sampling.reasoning_budget_forced = common_tokenize(ctx.vocab, message + end_tag, false, true); + if (!ctx.params.sampling.reasoning_budget_end.empty()) { + llama_tokens end_tag = ctx.params.sampling.reasoning_budget_end.front(); + std::string message = json_value(data, "reasoning_budget_message", std::string()); + if (!message.empty()) { + llama_tokens message_tokens = common_tokenize(ctx.vocab, message, false, true); + end_tag.insert(end_tag.begin(), message_tokens.begin(), message_tokens.end()); + } + ctx.params.sampling.reasoning_budget_forced = std::move(end_tag); + } })); add((new field_json("logit_bias")) @@ -546,7 +565,7 @@ task_params eval_llama_cmpl_schema( // debugging { auto budget = params.sampling.reasoning_budget_tokens; - SRV_DBG("reasoning budget: tokens=%d, generation_prompt='%s', start=%zu toks, end=%zu toks, forced=%zu toks\n", + SRV_DBG("reasoning budget: tokens=%d, generation_prompt='%s', start=%zu toks, end=%zu seqs, forced=%zu toks\n", budget, params.sampling.generation_prompt.c_str(), params.sampling.reasoning_budget_start.size(), params.sampling.reasoning_budget_end.size(), diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index f01780100..1fd7cce27 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -63,6 +63,8 @@ json task_params::to_json(bool only_metrics) const { {"mirostat", sampling.mirostat}, {"mirostat_tau", sampling.mirostat_tau}, {"mirostat_eta", sampling.mirostat_eta}, + {"adaptive_target", sampling.adaptive_target}, + {"adaptive_decay", sampling.adaptive_decay}, {"max_tokens", n_predict}, {"n_predict", n_predict}, // TODO: deduplicate? {"n_keep", n_keep}, @@ -114,6 +116,8 @@ json task_params::to_json(bool only_metrics) const { {"mirostat", sampling.mirostat}, {"mirostat_tau", sampling.mirostat_tau}, {"mirostat_eta", sampling.mirostat_eta}, + {"adaptive_target", sampling.adaptive_target}, + {"adaptive_decay", sampling.adaptive_decay}, {"stop", antiprompt}, {"max_tokens", n_predict}, {"n_predict", n_predict}, // TODO: deduplicate? diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index 9eb57abae..2af44e49b 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -7,12 +7,13 @@ #include #include #include +#include #include #include -#include #include #include #include +#include namespace fs = std::filesystem; @@ -24,7 +25,7 @@ json server_tool::to_json() const { return { {"display_name", display_name}, {"tool", name}, - {"type", "builtin"}, + {"type", type()}, {"permissions", json{ {"write", permission_write} }}, @@ -1048,16 +1049,42 @@ struct server_tool_get_datetime : server_tool { {"type", "function"}, {"function", { {"name", name}, - {"description", "Returns the current date and time"}, + {"description", "Returns the current date and time in UTC"}, + {"parameters", { + {"type", "object"}, + {"properties", { + {"format", { + {"type", "string"}, + {"description", + "strftime()-style format string for the output (default: \"%Y-%m-%dT%H:%M:%SZ\", " + "e.g. ISO 8601). Choose your own format if you need something else, " + "e.g. \"%A, %B %d %Y\" for a human-readable date."}, + }}, + }}, + }}, }}, }; } - json invoke(json, server_tool::stream *) const override { - auto now = std::chrono::system_clock::now(); - auto time = std::chrono::system_clock::to_time_t(now); + json invoke(json params, server_tool::stream *) const override { + std::string format = json_value(params, "format", std::string("%Y-%m-%dT%H:%M:%SZ")); - return {{"result", std::ctime(&time)}}; + auto now = std::chrono::system_clock::now(); + auto time = std::chrono::system_clock::to_time_t(now); + std::tm tm_utc; +#ifdef _WIN32 + gmtime_s(&tm_utc, &time); +#else + gmtime_r(&time, &tm_utc); +#endif + + char buf[256]; + size_t len = std::strftime(buf, sizeof(buf), format.c_str(), &tm_utc); + if (len == 0) { + return {{"error", "invalid format string"}}; + } + + return {{"result", std::string(buf, len)}}; } }; @@ -1102,6 +1129,49 @@ struct server_tools_res : server_http_res { } }; +// +// server_mcp_tool: exposes one tool from a running MCP server as a server_tool. +// +struct server_mcp_tool : server_tool { + std::string server_name; + std::string tool_name; + server_mcp_tool_def def; + server_mcp & mcp_mgr; + + server_mcp_tool(server_mcp_tool_def d, server_mcp & mgr) + : server_name(d.server_name) + , tool_name(d.name) + , def(std::move(d)) + , mcp_mgr(mgr) + { + name = server_name + "_" + tool_name; + display_name = name; + permission_write = false; + support_stream = false; + } + + std::string type() const override { return "mcp"; } + + json get_definition() const override { + json schema = def.input_schema; + if (schema.is_null() || !schema.is_object()) { + schema = json::object(); + } + return { + {"type", "function"}, + {"function", { + {"name", name}, + {"description", def.description}, + {"parameters", schema}, + }}, + }; + } + + json invoke(json params, server_tool::stream *) const override { + return mcp_mgr.call_tool(server_name, tool_name, params); + } +}; + static server_tool & find_tool(std::vector> & tools, const std::string & name, bool require_stream) { for (auto & t : tools) { if (t->name == name) { @@ -1130,7 +1200,8 @@ static std::vector> build_tools() { return tools; } -void server_tools::setup(const std::vector & enabled_tools) { +void server_tools::setup(const std::vector & enabled_tools, + server_mcp & mcp_mgr) { if (!enabled_tools.empty()) { std::unordered_set enabled_set(enabled_tools.begin(), enabled_tools.end()); auto all_tools = build_tools(); @@ -1161,6 +1232,29 @@ void server_tools::setup(const std::vector & enabled_tools) { } } + // append MCP tools, skipping any that collide with a built-in or another MCP tool of the same "_" name + if (!mcp_mgr.empty()) { + std::unordered_set seen_names; + for (auto & t : tools) { + seen_names.insert(t->name); + } + size_t n_added = 0; + for (const auto & def : mcp_mgr.list_tools()) { + std::string mcp_name = def.server_name + "_" + def.name; + if (seen_names.count(mcp_name)) { + SRV_WRN("MCP tool \"%s\" from server \"%s\" collides with an existing tool, skipping\n", + mcp_name.c_str(), def.server_name.c_str()); + continue; + } + seen_names.insert(mcp_name); + tools.push_back(std::make_unique(def, mcp_mgr)); + n_added++; + } + if (n_added > 0) { + SRV_INF("Added %zu MCP tools\n", n_added); + } + } + handle_get = [this](const server_http_req &) -> server_http_res_ptr { auto res = std::make_unique(); try { diff --git a/tools/server/server-tools.h b/tools/server/server-tools.h index 6f6528f48..601399ee9 100644 --- a/tools/server/server-tools.h +++ b/tools/server/server-tools.h @@ -3,9 +3,11 @@ #include "server-common.h" #include "server-http.h" #include "server-queue.h" +#include "server-mcp.h" #include #include +#include struct server_tool { std::string name; @@ -15,6 +17,7 @@ struct server_tool { virtual ~server_tool() = default; virtual json get_definition() const = 0; + virtual std::string type() const { return "builtin"; } struct stream { server_response & qr; @@ -34,7 +37,8 @@ struct server_tools { server_response queue_res; std::atomic res_id{0}; - void setup(const std::vector & enabled_tools); + void setup(const std::vector & enabled_tools, + server_mcp & mcp_mgr); server_http_context::handler_t handle_get; server_http_context::handler_t handle_post; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 20effbb14..b6fef99e8 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -88,6 +88,11 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t int llama_server(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); +#ifndef _WIN32 + // Ignore SIGPIPE so the server does not crash if an MCP child exits while we are writing to its stdin + signal(SIGPIPE, SIG_IGN); +#endif + // own arguments required by this example common_params params; @@ -157,6 +162,9 @@ int llama_server(common_params & params, int argc, char ** argv) { params.model_alias.insert(model_name); } + // note: this is guaranteed to out-live ctx_http and tools + server_mcp mcp_mgr; + // struct that contains llama context and inference server_context ctx_server; @@ -326,17 +334,28 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_http.post("/cors-proxy", ex_wrapper(res_403)); } - // EXPERIMENTAL built-in tools - if (!params.server_tools.empty()) { + try { + mcp_mgr.start(params); + } catch (const std::exception & e) { + SRV_ERR("MCP starting failed: %s\n", e.what()); + return 1; + } + + if (!params.server_tools.empty() || !mcp_mgr.empty()) { try { - tools.setup(params.server_tools); + tools.setup(params.server_tools, mcp_mgr); } catch (const std::exception & e) { SRV_ERR("tools setup failed: %s\n", e.what()); return 1; } ctx_http.get ("/tools", ex_wrapper(tools.handle_get)); ctx_http.post("/tools", ex_wrapper(tools.handle_post)); - warn_names.push_back("built-in tools (experimental)"); + if (!params.server_tools.empty()) { + warn_names.push_back("built-in tools (experimental)"); + } + if (!mcp_mgr.empty()) { + warn_names.push_back("MCP servers (experimental)"); + } } else { ctx_http.get ("/tools", ex_wrapper(res_403)); ctx_http.post("/tools", ex_wrapper(res_403)); @@ -378,7 +397,7 @@ int llama_server(common_params & params, int argc, char ** argv) { if (is_router_server) { SRV_INF("%s", "starting server in router mode. models will be automatically loaded on-demand\n"); - clean_up = [&models_routes]() { + clean_up = [&models_routes, &mcp_mgr]() { SRV_INF("%s: cleaning up before exit...\n", __func__); // stop the session GC first, it finalizes live sessions and wakes pending readers server_stream_session_manager_stop(); @@ -386,6 +405,7 @@ int llama_server(common_params & params, int argc, char ** argv) { models_routes->stopping.store(true); // maybe redundant, but just to be safe models_routes->models.unload_all(); } + mcp_mgr.shutdown(); llama_backend_free(); }; @@ -401,17 +421,19 @@ int llama_server(common_params & params, int argc, char ** argv) { // important to disconnect any SSE clients models_routes->stopping.store(true); } + mcp_mgr.shutdown(); ctx_http.stop(); }; } else { // setup clean up function, to be called before exit - clean_up = [&ctx_http, &ctx_server]() { + clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() { SRV_INF("%s: cleaning up before exit...\n", __func__); // stop the session GC first, it finalizes live sessions and wakes pending readers server_stream_session_manager_stop(); ctx_http.stop(); ctx_server.terminate(); + mcp_mgr.shutdown(); llama_backend_free(); }; @@ -444,6 +466,7 @@ int llama_server(common_params & params, int argc, char ** argv) { SRV_INF("%s", "model loaded\n"); shutdown_handler = [&](int) { + mcp_mgr.shutdown(); // this will unblock start_loop() ctx_server.terminate(); }; diff --git a/tools/server/tests/fixtures/mcp_burst_server.py b/tools/server/tests/fixtures/mcp_burst_server.py new file mode 100644 index 000000000..22892d9a1 --- /dev/null +++ b/tools/server/tests/fixtures/mcp_burst_server.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +""" +Minimal MCP server that writes notification + response in a single write() with no flush. +This reproduces the buffering bug where read_message() can strand the response. +""" +import json +import sys +import os + +TOOLS = [ + { + "name": "echo", + "description": "Echo back the input message", + "inputSchema": { + "type": "object", + "properties": { + "message": {"type": "string"} + }, + "required": ["message"] + } + } +] + +def handle_initialize(params, req_id): + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "burst-test", "version": "1.0"} + } + } + +def handle_tools_list(params, req_id): + return { + "jsonrpc": "2.0", + "id": req_id, + "result": {"tools": TOOLS} + } + +def handle_tools_call(params, req_id): + tool_name = params.get("name") + arguments = params.get("arguments", {}) + + if tool_name == "echo": + message = arguments.get("message", "") + notif = { + "jsonrpc": "2.0", + "method": "notifications/progress", + "params": {"progress": 50, "total": 100} + } + response = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [{"type": "text", "text": f"echo: {message}"}] + } + } + # Single os.write() call: both lines land in one pipe packet atomically. + # This is the key difference from mcp_malformed_server.py which flushes between writes. + data = (json.dumps(notif) + "\n" + json.dumps(response) + "\n").encode("utf-8") + os.write(sys.stdout.fileno(), data) + return None # already written + else: + response = { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32602, "message": f"Unknown tool: {tool_name}"} + } + return response + +HANDLERS = { + "initialize": handle_initialize, + "tools/list": handle_tools_list, + "tools/call": handle_tools_call, +} + +def main(): + # Use line-buffered text mode for regular responses, but the burst write + # uses os.write() directly to guarantee a single kernel write(). + sys.stdout = os.fdopen(sys.stdout.fileno(), "w", buffering=1) + sys.stderr = os.fdopen(sys.stderr.fileno(), "w", buffering=1) + + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + request = json.loads(line) + except json.JSONDecodeError: + continue + + method = request.get("method") + req_id = request.get("id") + params = request.get("params", {}) + + # JSON-RPC 2.0: a message without an id is a notification and must not receive a response + if req_id is None: + continue + + handler = HANDLERS.get(method) + if handler: + response = handler(params, req_id) + if response is not None: + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + else: + response = { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32601, "message": f"Method not found: {method}"} + } + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + +if __name__ == "__main__": + main() diff --git a/tools/server/tests/fixtures/mcp_crash_server.py b/tools/server/tests/fixtures/mcp_crash_server.py new file mode 100644 index 000000000..8dffdc61c --- /dev/null +++ b/tools/server/tests/fixtures/mcp_crash_server.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +""" +MCP server that crashes after receiving a specific tool call. +""" +import json +import sys +import os + +def handle_initialize(params, req_id): + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "crash-test", "version": "1.0"} + } + } + +def handle_tools_list(params, req_id): + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "tools": [ + { + "name": "echo", + "description": "Echo back the input message", + "inputSchema": { + "type": "object", + "properties": { + "message": {"type": "string"} + } + } + }, + { + "name": "crash", + "description": "Crash the server", + "inputSchema": { + "type": "object", + "properties": {} + } + } + ] + } + } + +def handle_tools_call(params, req_id): + tool_name = params.get("name") + arguments = params.get("arguments", {}) + + if tool_name == "echo": + message = arguments.get("message", "") + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [{"type": "text", "text": f"echo: {message}"}] + } + } + elif tool_name == "crash": + # Send a partial response then exit + sys.stdout.write(json.dumps({"jsonrpc": "2.0", "id": req_id, "result": {"content": [{"type": "text", "text": "crashing..."}]}}) + "\n") + sys.stdout.flush() + os._exit(1) + else: + return { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32602, "message": f"Unknown tool: {tool_name}"} + } + +HANDLERS = { + "initialize": handle_initialize, + "tools/list": handle_tools_list, + "tools/call": handle_tools_call, +} + +def main(): + sys.stdout = os.fdopen(sys.stdout.fileno(), "w", buffering=1) + sys.stderr = os.fdopen(sys.stderr.fileno(), "w", buffering=1) + + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + request = json.loads(line) + except json.JSONDecodeError: + continue + + method = request.get("method") + req_id = request.get("id") + params = request.get("params", {}) + + # JSON-RPC 2.0: a message without an id is a notification and must not receive a response + if req_id is None: + continue + + handler = HANDLERS.get(method) + if handler: + response = handler(params, req_id) + else: + response = { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32601, "message": f"Method not found: {method}"} + } + + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + +if __name__ == "__main__": + main() diff --git a/tools/server/tests/fixtures/mcp_echo_server.py b/tools/server/tests/fixtures/mcp_echo_server.py new file mode 100755 index 000000000..7acfb3588 --- /dev/null +++ b/tools/server/tests/fixtures/mcp_echo_server.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +""" +Minimal MCP server for testing. +Implements JSON-RPC 2.0 over stdio (line-delimited JSON). +""" +import json +import sys +import os + +# Ensure we use python3 from the current environment +if sys.platform == "win32": + # On Windows, we need to use the same python interpreter + pass + +TOOLS = [ + { + "name": "echo", + "description": "Echo back the input message", + "inputSchema": { + "type": "object", + "properties": { + "message": {"type": "string", "description": "Message to echo"} + }, + "required": ["message"] + } + }, + { + "name": "add", + "description": "Add two numbers", + "inputSchema": { + "type": "object", + "properties": { + "a": {"type": "number"}, + "b": {"type": "number"} + }, + "required": ["a", "b"] + } + }, + { + "name": "fail_once", + "description": "Fails on first call, succeeds on subsequent calls", + "inputSchema": { + "type": "object", + "properties": {} + } + } +] + +_state = {"fail_once_called": False} + +def handle_initialize(params, req_id): + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "echo-test", "version": "1.0"} + } + } + +def handle_tools_list(params, req_id): + return { + "jsonrpc": "2.0", + "id": req_id, + "result": {"tools": TOOLS} + } + +def handle_tools_call(params, req_id): + tool_name = params.get("name") + arguments = params.get("arguments", {}) + + if tool_name == "echo": + message = arguments.get("message", "") + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [{"type": "text", "text": f"echo: {message}"}] + } + } + elif tool_name == "add": + a = arguments.get("a", 0) + b = arguments.get("b", 0) + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [{"type": "text", "text": str(a + b)}] + } + } + elif tool_name == "fail_once": + if not _state["fail_once_called"]: + _state["fail_once_called"] = True + return { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32000, "message": "transient error"} + } + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [{"type": "text", "text": "ok"}] + } + } + else: + return { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32602, "message": f"Unknown tool: {tool_name}"} + } + +def handle_ping(params, req_id): + return { + "jsonrpc": "2.0", + "id": req_id, + "result": {} + } + +HANDLERS = { + "initialize": handle_initialize, + "tools/list": handle_tools_list, + "tools/call": handle_tools_call, + "ping": handle_ping, +} + +def main(): + # Use unbuffered output + sys.stdout = os.fdopen(sys.stdout.fileno(), "w", buffering=1) + sys.stderr = os.fdopen(sys.stderr.fileno(), "w", buffering=1) + + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + request = json.loads(line) + except json.JSONDecodeError: + continue + + method = request.get("method") + req_id = request.get("id") + params = request.get("params", {}) + + # JSON-RPC 2.0: a message without an id is a notification and must not receive a response + if req_id is None: + continue + + handler = HANDLERS.get(method) + if handler: + response = handler(params, req_id) + else: + response = { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32601, "message": f"Method not found: {method}"} + } + + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + +if __name__ == "__main__": + main() diff --git a/tools/server/tests/fixtures/mcp_grandchild_server.py b/tools/server/tests/fixtures/mcp_grandchild_server.py new file mode 100644 index 000000000..2604a77ee --- /dev/null +++ b/tools/server/tests/fixtures/mcp_grandchild_server.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +""" +MCP server (NDJSON JSON-RPC over stdio) that spawns a long-lived grandchild which inherits +this process's stdin/stdout/stderr and keeps them open. + +This reproduces the reader-teardown deadlock: killing the direct MCP child (SIGKILL, which is +all subprocess_terminate() does) does NOT close the stdout/stderr pipe write ends, because the +grandchild still holds them. A server that reads those pipes with a blocking read would then +wait forever for an EOF that never arrives, hanging teardown (both warmup shutdown at startup +and process shutdown). The polled, running-aware reader must exit regardless. +""" +import json +import os +import subprocess +import sys + +# Spawn a grandchild that inherits our std handles (fds 0/1/2 = the MCP pipes) and lives well +# past any teardown in the tests. We do NOT redirect its stdio, so it keeps the pipe write ends +# open even after this process is killed. +subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) + +TOOLS = [ + { + "name": "echo", + "description": "Echo back the input message", + "inputSchema": { + "type": "object", + "properties": {"message": {"type": "string", "description": "Message to echo"}}, + "required": ["message"], + }, + } +] + + +def handle_initialize(params, req_id): + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "grandchild-test", "version": "1.0"}, + }, + } + + +def handle_tools_list(params, req_id): + return {"jsonrpc": "2.0", "id": req_id, "result": {"tools": TOOLS}} + + +def handle_tools_call(params, req_id): + if params.get("name") == "echo": + message = params.get("arguments", {}).get("message", "") + return { + "jsonrpc": "2.0", + "id": req_id, + "result": {"content": [{"type": "text", "text": f"echo: {message}"}]}, + } + return {"jsonrpc": "2.0", "id": req_id, "error": {"code": -32602, "message": "Unknown tool"}} + + +HANDLERS = { + "initialize": handle_initialize, + "tools/list": handle_tools_list, + "tools/call": handle_tools_call, +} + + +def main(): + sys.stdout = os.fdopen(sys.stdout.fileno(), "w", buffering=1) + sys.stderr = os.fdopen(sys.stderr.fileno(), "w", buffering=1) + + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + request = json.loads(line) + except json.JSONDecodeError: + continue + + method = request.get("method") + req_id = request.get("id") + params = request.get("params", {}) + + if req_id is None: + continue # notification, no response + + handler = HANDLERS.get(method) + if handler: + response = handler(params, req_id) + else: + response = {"jsonrpc": "2.0", "id": req_id, "error": {"code": -32601, "message": f"Method not found: {method}"}} + + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + + +if __name__ == "__main__": + main() diff --git a/tools/server/tests/fixtures/mcp_malformed_server.py b/tools/server/tests/fixtures/mcp_malformed_server.py new file mode 100644 index 000000000..743333c5f --- /dev/null +++ b/tools/server/tests/fixtures/mcp_malformed_server.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +""" +MCP server that sends malformed responses and notifications during requests. +""" +import json +import sys +import os + +def handle_initialize(params, req_id): + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "malformed-test", "version": "1.0"} + } + } + +def handle_tools_list(params, req_id): + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "tools": [ + { + "name": "echo", + "description": "Echo back the input message", + "inputSchema": { + "type": "object", + "properties": { + "message": {"type": "string"} + } + } + } + ] + } + } + +def handle_tools_call(params, req_id): + tool_name = params.get("name") + arguments = params.get("arguments", {}) + + if tool_name == "echo": + message = arguments.get("message", "") + # Send a notification first (no id field) + notif = { + "jsonrpc": "2.0", + "method": "notifications/progress", + "params": {"progress": 50, "total": 100} + } + sys.stdout.write(json.dumps(notif) + "\n") + sys.stdout.flush() + # Then send the actual response + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [{"type": "text", "text": f"echo: {message}"}] + } + } + else: + return { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32602, "message": f"Unknown tool: {tool_name}"} + } + +HANDLERS = { + "initialize": handle_initialize, + "tools/list": handle_tools_list, + "tools/call": handle_tools_call, +} + +def main(): + sys.stdout = os.fdopen(sys.stdout.fileno(), "w", buffering=1) + sys.stderr = os.fdopen(sys.stderr.fileno(), "w", buffering=1) + + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + request = json.loads(line) + except json.JSONDecodeError: + # Send malformed JSON response + sys.stdout.write("THIS IS NOT JSON\n") + sys.stdout.flush() + continue + + method = request.get("method") + req_id = request.get("id") + params = request.get("params", {}) + + # JSON-RPC 2.0: a message without an id is a notification and must not receive a response + if req_id is None: + continue + + handler = HANDLERS.get(method) + if handler: + response = handler(params, req_id) + else: + response = { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32601, "message": f"Method not found: {method}"} + } + + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + +if __name__ == "__main__": + main() diff --git a/tools/server/tests/fixtures/mcp_slow_server.py b/tools/server/tests/fixtures/mcp_slow_server.py new file mode 100644 index 000000000..7f8e67835 --- /dev/null +++ b/tools/server/tests/fixtures/mcp_slow_server.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +""" +MCP server that sleeps before responding, for timeout testing. +""" +import json +import sys +import os +import time +import argparse + +TOOLS = [ + { + "name": "sleep", + "description": "Sleep for a given number of seconds", + "inputSchema": { + "type": "object", + "properties": { + "seconds": {"type": "number", "description": "Seconds to sleep"} + }, + "required": ["seconds"] + } + } +] + +def handle_initialize(params, req_id): + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "slow-test", "version": "1.0"} + } + } + +def handle_tools_list(params, req_id): + return { + "jsonrpc": "2.0", + "id": req_id, + "result": {"tools": TOOLS} + } + +def handle_tools_call(params, req_id): + tool_name = params.get("name") + arguments = params.get("arguments", {}) + + if tool_name == "sleep": + seconds = arguments.get("seconds", 1) + time.sleep(seconds) + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [{"type": "text", "text": f"slept {seconds}s"}] + } + } + else: + return { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32602, "message": f"Unknown tool: {tool_name}"} + } + +HANDLERS = { + "initialize": handle_initialize, + "tools/list": handle_tools_list, + "tools/call": handle_tools_call, +} + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--delay", type=float, default=5.0, help="Delay in seconds for sleep tool") + args = parser.parse_args() + + # Override the sleep duration + global handle_tools_call + def handle_tools_call(params, req_id): + tool_name = params.get("name") + arguments = params.get("arguments", {}) + + if tool_name == "sleep": + seconds = arguments.get("seconds", args.delay) + time.sleep(seconds) + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [{"type": "text", "text": f"slept {seconds}s"}] + } + } + else: + return { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32602, "message": f"Unknown tool: {tool_name}"} + } + + sys.stdout = os.fdopen(sys.stdout.fileno(), "w", buffering=1) + sys.stderr = os.fdopen(sys.stderr.fileno(), "w", buffering=1) + + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + request = json.loads(line) + except json.JSONDecodeError: + continue + + method = request.get("method") + req_id = request.get("id") + params = request.get("params", {}) + + # JSON-RPC 2.0: a message without an id is a notification and must not receive a response + if req_id is None: + continue + + handler = HANDLERS.get(method) + if handler: + response = handler(params, req_id) + else: + response = { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32601, "message": f"Method not found: {method}"} + } + + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + +if __name__ == "__main__": + main() diff --git a/tools/server/tests/unit/test_completion.py b/tools/server/tests/unit/test_completion.py index 1e0891987..9375e0110 100644 --- a/tools/server/tests/unit/test_completion.py +++ b/tools/server/tests/unit/test_completion.py @@ -66,6 +66,8 @@ def test_completion_stream(prompt: str, n_predict: int, re_content: str, n_promp assert server.n_predict is not None assert data["generation_settings"]["n_predict"] == min(n_predict, server.n_predict) assert data["generation_settings"]["seed"] == server.seed + assert "adaptive_target" in data["generation_settings"] + assert "adaptive_decay" in data["generation_settings"] assert match_regex(re_content, content) else: assert len(data["tokens"]) > 0 diff --git a/tools/server/tests/unit/test_mcp_servers.py b/tools/server/tests/unit/test_mcp_servers.py new file mode 100644 index 000000000..9ad2241bd --- /dev/null +++ b/tools/server/tests/unit/test_mcp_servers.py @@ -0,0 +1,718 @@ +#!/usr/bin/env python3 +""" +Tests for MCP server integration via the /tools endpoint. + +Invariants verified: +1. MCP tools appear in /tools listing when configured +2. MCP tools use _ naming +3. MCP tools can be invoked and return correct results +4. Misconfigured MCP servers do not crash the server +5. Multiple MCP servers can be configured simultaneously +6. Warmup populates the tool list at startup +""" +import json +import os +import sys +import tempfile +import time + +import pytest + +from utils import * + +# Path to the test MCP server fixture +FIXTURES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "fixtures") +MCP_ECHO_SERVER = os.path.join(FIXTURES_DIR, "mcp_echo_server.py") + +server: ServerProcess + + +def _mcp_config_json(servers: dict) -> str: + """Create a JSON config string for --mcp-servers-json.""" + return json.dumps({"mcpServers": servers}) + + +def _start_server_with_mcp(mcp_json: str, **kwargs) -> ServerProcess: + """Helper to start a router server with MCP config.""" + srv = ServerPreset.router() + srv.server_tools = "all" + srv.no_ui = True + srv.server_port = 8085 # avoid conflict with load_all() which uses 8080 + srv.mcp_servers_json = mcp_json + for k, v in kwargs.items(): + setattr(srv, k, v) + srv.start() + return srv + + +def test_mcp_tools_listed_in_tools_endpoint(): + """MCP tools should appear in GET /tools with server:tool naming.""" + global server + mcp_json = _mcp_config_json({ + "echo": { + "command": sys.executable, + "args": [MCP_ECHO_SERVER], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + res = server.make_request("GET", "/tools") + assert res.status_code == 200, res.body + + tools = res.body + assert isinstance(tools, list), f"Expected list, got {type(tools)}" + + # Find MCP tools - name is in "tool" field or definition.function.name + def get_tool_name(t): + return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "") + + mcp_tools = [t for t in tools if get_tool_name(t).startswith("echo_")] + assert len(mcp_tools) >= 2, f"Expected at least 2 echo_ tools, got {len(mcp_tools)}: {mcp_tools}" + + tool_names = {get_tool_name(t) for t in mcp_tools} + assert "echo_echo" in tool_names + assert "echo_add" in tool_names + + # Verify tool structure + echo_tool = next(t for t in mcp_tools if get_tool_name(t) == "echo_echo") + assert "description" in echo_tool or "definition" in echo_tool + finally: + server.stop() + + +def test_mcp_tool_invocation(): + """MCP tools should be callable via POST /tools and return correct results.""" + global server + mcp_json = _mcp_config_json({ + "echo": { + "command": sys.executable, + "args": [MCP_ECHO_SERVER], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + # Call echo_echo + res = server.make_request("POST", "/tools", data={ + "tool": "echo_echo", + "params": {"message": "hello world"} + }) + assert res.status_code == 200, res.body + body = res.body + assert "error" not in body, body + # The result format depends on the tool implementation + # For MCP tools, it should contain the tool result + assert "plain_text_response" in body or "result" in body or "content" in body, body + + # Call echo_add + res = server.make_request("POST", "/tools", data={ + "tool": "echo_add", + "params": {"a": 3, "b": 5} + }) + assert res.status_code == 200, res.body + body = res.body + assert "error" not in body, body + finally: + server.stop() + + +def test_mcp_bad_command_does_not_crash(): + """A misconfigured MCP server should not crash the llama-server.""" + global server + mcp_json = _mcp_config_json({ + "nonexistent": { + "command": "this_executable_does_not_exist_12345", + "args": [], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + # Server should still be healthy + res = server.make_request("GET", "/health") + assert res.status_code == 200, res.body + + # Builtin tools should still work + res = server.make_request("GET", "/tools") + assert res.status_code == 200, res.body + tools = res.body + # Should have builtin tools but no MCP tools from the bad server + mcp_tools = [t for t in tools if t.get("name", "").startswith("nonexistent_")] + assert len(mcp_tools) == 0, f"Expected no nonexistent_ tools, got {mcp_tools}" + finally: + server.stop() + + +def test_mcp_multiple_servers(): + """Multiple MCP servers can be configured simultaneously.""" + global server + mcp_json = _mcp_config_json({ + "echo": { + "command": sys.executable, + "args": [MCP_ECHO_SERVER], + }, + "echo2": { + "command": sys.executable, + "args": [MCP_ECHO_SERVER], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + res = server.make_request("GET", "/tools") + assert res.status_code == 200, res.body + + tools = res.body + + def get_tool_name(t): + return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "") + + echo_tools = [t for t in tools if get_tool_name(t).startswith("echo_")] + echo2_tools = [t for t in tools if get_tool_name(t).startswith("echo2_")] + + assert len(echo_tools) >= 2, f"Expected echo_ tools, got {echo_tools}" + assert len(echo2_tools) >= 2, f"Expected echo2_ tools, got {echo2_tools}" + finally: + server.stop() + + +def test_mcp_tools_not_listed_when_not_configured(): + """Without MCP config, no MCP tools should appear.""" + global server + server = ServerPreset.router() + server.server_tools = "all" + server.no_ui = True + server.server_port = 8085 + server.start() + + try: + res = server.make_request("GET", "/tools") + assert res.status_code == 200, res.body + + tools = res.body + + def get_tool_name(t): + return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "") + + # Should only have builtin tools, no server: prefixed tools + mcp_tools = [t for t in tools if ":" in get_tool_name(t)] + assert len(mcp_tools) == 0, f"Expected no MCP tools, got {mcp_tools}" + finally: + server.stop() + + +def test_mcp_fail_once_tool_eventual_success(): + """Test that a tool that fails once eventually succeeds (tests instance respawn).""" + global server + mcp_json = _mcp_config_json({ + "echo": { + "command": sys.executable, + "args": [MCP_ECHO_SERVER], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + # First call should succeed (warmup already spawned and shut down the instance, + # but the first actual tool call will spawn a fresh instance) + res = server.make_request("POST", "/tools", data={ + "tool": "echo_fail_once", + "params": {} + }) + # It might fail on first call if the warmup instance was shut down + # and a new instance is spawned. The fail_once state is per-process, + # so a fresh process will fail once then succeed. + # Actually, warmup spawns, lists, then shuts down. So the first tool call + # spawns a new process which will fail once. + assert res.status_code in (200, 500), res.body + finally: + server.stop() + + +def test_mcp_tools_via_json_config_file(): + """Test that --mcp-servers-config (file) works as well as --mcp-servers-json.""" + global server + config = { + "mcpServers": { + "echo": { + "command": sys.executable, + "args": [MCP_ECHO_SERVER], + } + } + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(config, f) + config_path = f.name + + try: + server = ServerPreset.router() + server.server_tools = "all" + server.no_ui = True + server.server_port = 8085 + server.mcp_servers_config = config_path + server.start() + + res = server.make_request("GET", "/tools") + assert res.status_code == 200, res.body + + tools = res.body + + def get_tool_name(t): + return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "") + + mcp_tools = [t for t in tools if get_tool_name(t).startswith("echo_")] + assert len(mcp_tools) >= 2, f"Expected echo_ tools, got {mcp_tools}" + finally: + os.unlink(config_path) + server.stop() + + +def test_mcp_tools_slot_independent(): + """MCP tools should work without any slot concept; /tools is slot-independent.""" + global server + mcp_json = _mcp_config_json({ + "echo": { + "command": sys.executable, + "args": [MCP_ECHO_SERVER], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + # Call /tools without any slot binding - should succeed + res = server.make_request("POST", "/tools", data={ + "tool": "echo_echo", + "params": {"message": "hello"} + }) + assert res.status_code == 200, res.body + body = res.body + assert "error" not in body, body + finally: + server.stop() + + +def test_mcp_concurrent_tool_calls(): + """Concurrent POST /tools to same MCP server should all succeed.""" + global server + mcp_json = _mcp_config_json({ + "echo": { + "command": sys.executable, + "args": [MCP_ECHO_SERVER], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + def call_tool(): + return server.make_request("POST", "/tools", data={ + "tool": "echo_echo", + "params": {"message": "hi"} + }) + + with ThreadPoolExecutor(max_workers=10) as executor: + futures = [executor.submit(call_tool) for _ in range(10)] + results = [f.result() for f in futures] + + for res in results: + assert res.status_code == 200, res.body + assert "error" not in res.body, res.body + finally: + server.stop() + + +def test_mcp_tool_timeout(): + """Tool call should timeout if MCP server is too slow.""" + global server + MCP_SLOW_SERVER = os.path.join(FIXTURES_DIR, "mcp_slow_server.py") + mcp_json = _mcp_config_json({ + "slow": { + "command": sys.executable, + "args": [MCP_SLOW_SERVER, "--delay", "5"], + "timeout_ms": 500 + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + res = server.make_request("POST", "/tools", data={ + "tool": "slow_sleep", + "params": {"seconds": 5} + }) + assert res.status_code == 200, res.body + body = res.body + assert "error" in body, body + finally: + server.stop() + + +def test_mcp_warmup_partial_failure(): + """Good server's tools should appear even if bad server fails warmup.""" + global server + mcp_json = _mcp_config_json({ + "good": { + "command": sys.executable, + "args": [MCP_ECHO_SERVER], + }, + "bad": { + "command": "nonexistent", + "args": [] + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + res = server.make_request("GET", "/tools") + assert res.status_code == 200, res.body + tools = res.body + + def get_tool_name(t): + return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "") + + # good server tools should be present + assert any("good_" in get_tool_name(t) for t in tools), f"Expected good: tools in {tools}" + finally: + server.stop() + + +def test_mcp_notification_during_request(): + """Notification during request should not be returned as response.""" + global server + MCP_MALFORMED_SERVER = os.path.join(FIXTURES_DIR, "mcp_malformed_server.py") + mcp_json = _mcp_config_json({ + "notifying": { + "command": sys.executable, + "args": [MCP_MALFORMED_SERVER], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + res = server.make_request("POST", "/tools", data={ + "tool": "notifying_echo", + "params": {"message": "hi"} + }) + assert res.status_code == 200, res.body + body = res.body + assert "error" not in body, body + finally: + server.stop() + + +def test_mcp_instance_respawn_after_crash(): + """Tool call after process crash should respawn and succeed.""" + global server + MCP_CRASH_SERVER = os.path.join(FIXTURES_DIR, "mcp_crash_server.py") + mcp_json = _mcp_config_json({ + "crash": { + "command": sys.executable, + "args": [MCP_CRASH_SERVER], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + # First call succeeds + res1 = server.make_request("POST", "/tools", data={ + "tool": "crash_echo", + "params": {"message": "hi"} + }) + assert res1.status_code == 200, res1.body + assert "error" not in res1.body, res1.body + + # Second call should also succeed (respawned instance) + res2 = server.make_request("POST", "/tools", data={ + "tool": "crash_echo", + "params": {"message": "hi2"} + }) + assert res2.status_code == 200, res2.body + assert "error" not in res2.body, res2.body + finally: + server.stop() + + + + +def test_mcp_fail_once_eventual_success_verified(): + """Verify that fail_once tool eventually succeeds after respawn.""" + global server + mcp_json = _mcp_config_json({ + "echo": { + "command": sys.executable, + "args": [MCP_ECHO_SERVER], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + # First call may fail (fresh process) + res1 = server.make_request("POST", "/tools", data={ + "tool": "echo_fail_once", + "params": {} + }) + # Second call should succeed + res2 = server.make_request("POST", "/tools", data={ + "tool": "echo_fail_once", + "params": {} + }) + assert res2.status_code == 200, res2.body + assert "error" not in res2.body, res2.body + finally: + server.stop() + + +def test_mcp_config_file_errors(): + """Invalid JSON config and missing file should cause server to fail to start.""" + # Invalid JSON - server should fail to start + server = ServerPreset.router() + server.server_tools = "all" + server.no_ui = True + server.server_port = 8085 + server.mcp_servers_json = "not valid json" + try: + server.start() + assert False, "Server should not have started with invalid MCP JSON config" + except RuntimeError: + pass # Expected: server process dies due to bad config + + # Missing file - server should fail to start + server = ServerPreset.router() + server.server_tools = "all" + server.no_ui = True + server.server_port = 8085 + server.mcp_servers_config = "/nonexistent/path.json" + try: + server.start() + assert False, "Server should not have started with missing config file" + except RuntimeError: + pass # Expected: server process dies due to missing config + + +def test_mcp_empty_tool_list(): + """MCP server reporting zero tools should result in empty tool list.""" + global server + # Create a minimal server that returns empty tools list + empty_server = os.path.join(FIXTURES_DIR, "_empty_mcp_server.py") + with open(empty_server, "w") as f: + f.write('''#!/usr/bin/env python3 +import json, sys, os +def main(): + sys.stdout = os.fdopen(sys.stdout.fileno(), "w", buffering=1) + for line in sys.stdin: + line = line.strip() + if not line: continue + try: request = json.loads(line) + except: continue + method = request.get("method") + req_id = request.get("id") + if method == "initialize": + resp = {"jsonrpc": "2.0", "id": req_id, "result": {"protocolVersion": "2024-11-05", "capabilities": {"tools": {}}, "serverInfo": {"name": "empty", "version": "1.0"}}} + elif method == "tools/list": + resp = {"jsonrpc": "2.0", "id": req_id, "result": {"tools": []}} + else: + resp = {"jsonrpc": "2.0", "id": req_id, "error": {"code": -32601, "message": "Method not found"}} + sys.stdout.write(json.dumps(resp) + "\\n") + sys.stdout.flush() +if __name__ == "__main__": + main() +''') + try: + mcp_json = _mcp_config_json({ + "empty": { + "command": sys.executable, + "args": [empty_server], + } + }) + server = _start_server_with_mcp(mcp_json) + res = server.make_request("GET", "/tools") + assert res.status_code == 200, res.body + tools = res.body + def get_tool_name(t): + return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "") + mcp_tools = [t for t in tools if get_tool_name(t).startswith("empty:")] + assert len(mcp_tools) == 0, f"Expected no empty: tools, got {mcp_tools}" + finally: + os.unlink(empty_server) + server.stop() + + +def test_mcp_rapid_succession_calls(): + """Many rapid calls should increment next_id correctly and correlate responses.""" + global server + mcp_json = _mcp_config_json({ + "echo": { + "command": sys.executable, + "args": [MCP_ECHO_SERVER], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + for i in range(20): + res = server.make_request("POST", "/tools", data={ + "tool": "echo_echo", + "params": {"message": f"msg{i}"} + }) + assert res.status_code == 200, res.body + assert "error" not in res.body, res.body + finally: + server.stop() + + +def test_mcp_notification_burst(): + """Notification + response in a single write() with no flush should not strand the response.""" + global server + MCP_BURST_SERVER = os.path.join(FIXTURES_DIR, "mcp_burst_server.py") + mcp_json = _mcp_config_json({ + "burst": { + "command": sys.executable, + "args": [MCP_BURST_SERVER], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + res = server.make_request("POST", "/tools", data={ + "tool": "burst_echo", + "params": {"message": "burst test"} + }) + assert res.status_code == 200, res.body + body = res.body + assert "error" not in body, body + finally: + server.stop() + + +def test_mcp_tool_definition_shape_via_chat_completions(): + """MCP tool definitions returned by GET /tools should have the correct shape for chat/completions.""" + global server + mcp_json = _mcp_config_json({ + "echo": { + "command": sys.executable, + "args": [MCP_ECHO_SERVER], + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + # Get MCP tool definitions + res = server.make_request("GET", "/tools") + assert res.status_code == 200, res.body + tools = res.body + + def get_tool_name(t): + return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "") + + echo_tools = [t for t in tools if get_tool_name(t).startswith("echo_")] + assert len(echo_tools) >= 2, f"Expected echo_ tools, got {echo_tools}" + + echo_tool = next(t for t in echo_tools if get_tool_name(t) == "echo_echo") + definition = echo_tool.get("definition", echo_tool) + + # Verify the definition has the standard function-calling shape + assert definition.get("type") == "function", f"Expected type=function, got {definition.get('type')}" + func = definition.get("function", {}) + assert "name" in func, "Missing function.name" + assert "description" in func, "Missing function.description" + assert "parameters" in func, f"Missing function.parameters, got keys: {list(func.keys())}" + params = func["parameters"] + assert params.get("type") == "object", f"Expected parameters.type=object, got {params.get('type')}" + assert "properties" in params, "Missing parameters.properties" + finally: + server.stop() + + +def test_mcp_slow_tool_call_slot_release(): + """A slow tool call should not stall server shutdown for the full I/O timeout.""" + global server + MCP_SLOW_SERVER = os.path.join(FIXTURES_DIR, "mcp_slow_server.py") + mcp_json = _mcp_config_json({ + "slow": { + "command": sys.executable, + "args": [MCP_SLOW_SERVER, "--delay", "10"], + "timeout_ms": 30000 + } + }) + server = _start_server_with_mcp(mcp_json) + + try: + # Start a slow tool call in a background thread + def slow_call(): + return server.make_request("POST", "/tools", data={ + "tool": "slow_sleep", + "params": {"seconds": 10} + }) + + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(slow_call) + + # Wait a moment for the call to start + time.sleep(2) + + # Stop the server while the tool call is in progress. + # With global MCP instances, close_all() is called explicitly at shutdown + # (not from slot release), so shutdown should complete promptly. + start_time = time.time() + server.stop() + elapsed = time.time() - start_time + + # The server should stop quickly, not wait for the full 30s I/O timeout. + # With the terminating flag, send_rpc() bails out within one select() + # slice (~50ms). This threshold MUST stay below the 5s force-kill + # fallback in ServerProcess.stop(): without the flag, shutdown stalls + # on the instance mutex and only completes when stop() sends SIGKILL + # at ~5s -- which any threshold above 5 would still accept. + assert elapsed < 3, f"Server stop took {elapsed:.1f}s, expected < 3s" + + # Wait for the future to complete (it will get an error response or timeout) + try: + res = future.result(timeout=5) + # If we got a response, it should be an error since the server stopped + if hasattr(res, 'status_code'): + assert res.status_code in (200, 500, 502, 503, 504), f"Unexpected status: {res.status_code}" + except Exception: + # Thread may have raised due to connection error - that's acceptable + pass + finally: + server.stop() + + +def test_mcp_grandchild_holding_pipes_does_not_deadlock(): + """An MCP server that leaves a grandchild inheriting its stdout/stderr must not deadlock + teardown. + + subprocess_terminate() only SIGKILLs the direct MCP child, so the inherited pipe write ends + stay open and a blocking read on them would never see EOF. That hung both warmup shutdown + (the server would never reach "ready") and process shutdown. The polled, running-aware reader + must exit regardless, so the server both starts and stops promptly here. + """ + global server + MCP_GRANDCHILD_SERVER = os.path.join(FIXTURES_DIR, "mcp_grandchild_server.py") + mcp_json = _mcp_config_json({ + "gc": { + "command": sys.executable, + "args": [MCP_GRANDCHILD_SERVER], + } + }) + + # If warmup teardown deadlocked, the server would never become ready and start() would time out. + server = _start_server_with_mcp(mcp_json) + + try: + # invoking the tool spawns a live transport whose reader thread holds the inherited pipe + res = server.make_request("POST", "/tools", data={ + "tool": "gc_echo", + "params": {"message": "hello"} + }) + assert res.status_code == 200, res.body + assert "error" not in res.body, res.body + + # shutdown must be prompt: a deadlocked reader-join would stall until the 5s SIGKILL + # fallback in ServerProcess.stop(), so the threshold has to stay below that + start = time.time() + server.stop() + elapsed = time.time() - start + assert elapsed < 3, f"server shutdown took {elapsed:.1f}s (expected < 3s) — teardown likely deadlocked" + finally: + server.stop() diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index 5d5c873ac..ae56bc70a 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -115,6 +115,8 @@ class ServerProcess: backend_sampling: bool = False gcp_compat: bool = False server_tools: str | None = None + mcp_servers_config: str | None = None + mcp_servers_json: str | None = None cors_origins: str | None = None # session variables @@ -265,6 +267,10 @@ class ServerProcess: server_args.append("--ui-mcp-proxy") if self.server_tools: server_args.extend(["--tools", self.server_tools]) + if self.mcp_servers_config: + server_args.extend(["--mcp-servers-config", self.mcp_servers_config]) + if self.mcp_servers_json: + server_args.extend(["--mcp-servers-json", self.mcp_servers_json]) if self.backend_sampling: server_args.append("--backend_sampling") if self.gcp_compat: diff --git a/tools/ui/src/lib/actions/fade-in-view.svelte.ts b/tools/ui/src/lib/actions/fade-in-view.svelte.ts deleted file mode 100644 index 9a5918131..000000000 --- a/tools/ui/src/lib/actions/fade-in-view.svelte.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { isElementInViewport } from '$lib/utils/viewport'; - -/** - * Svelte action that fades in an element when it enters the viewport. - * Uses IntersectionObserver for efficient viewport detection. - * - * If skipIfVisible is set and the element is already visible in the viewport - * when the action attaches (e.g. a markdown block promoted from unstable - * during streaming), the fade is skipped entirely to avoid a flash. - */ -export function fadeInView( - node: HTMLElement, - options: { duration?: number; y?: number; delay?: number; skipIfVisible?: boolean } = {} -) { - const { duration = 300, y = 0, delay = 0, skipIfVisible = false } = options; - - if (skipIfVisible && isElementInViewport(node)) { - return; - } - - node.style.opacity = '0'; - node.style.transform = `translateY(${y}px)`; - node.style.transition = `opacity ${duration}ms ease-out, transform ${duration}ms ease-out`; - - $effect(() => { - const observer = new IntersectionObserver( - (entries) => { - for (const entry of entries) { - if (entry.isIntersecting) { - setTimeout(() => { - requestAnimationFrame(() => { - node.style.opacity = '1'; - node.style.transform = 'translateY(0)'; - }); - }, delay); - observer.disconnect(); - } - } - }, - { threshold: 0.05 } - ); - - observer.observe(node); - - return () => { - observer.disconnect(); - }; - }); -} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte index 9b2077b8d..85683908c 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -25,6 +25,7 @@ SpecialFileType } from '$lib/enums'; import { config } from '$lib/stores/settings.svelte'; + import ContextGaugePopup from './ChatFormContextGauge/ContextGaugePopup.svelte'; import { modelOptions, selectedModelId } from '$lib/stores/models.svelte'; import { isRouterMode } from '$lib/stores/server.svelte'; import { chatStore } from '$lib/stores/chat.svelte'; @@ -556,6 +557,8 @@ /> + +
- {#each toolsPanel.activeGroups as group (group.label)} + {#each toolsPanel.activeGroups as group (group.key)} {@const checked = toolsPanel.isGroupChecked(group)} {@const enabledCount = toolsPanel.getEnabledToolCount(group)} {@const favicon = toolsPanel.getFavicon(group)} @@ -307,7 +307,7 @@ {/each} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte index 9ca9360c5..4473c29a3 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte @@ -64,14 +64,14 @@ {/if} {:else}
- {#each toolsPanel.activeGroups as group (group.label)} - {@const isExpanded = toolsPanel.expandedGroups.has(group.label)} + {#each toolsPanel.activeGroups as group (group.key)} + {@const isExpanded = toolsPanel.expandedGroups.has(group.key)} {@const checked = toolsPanel.isGroupChecked(group)} {@const favicon = toolsPanel.getFavicon(group)} toolsPanel.toggleGroupExpanded(group.label)} + onOpenChange={() => toolsPanel.toggleGroupExpanded(group.key)} >
toolsPanel.toggleGroupByLabel(group.label)} + onCheckedChange={() => toolsPanel.toggleGroupByKey(group.key)} class="mr-2 {ICON_CLASS_DEFAULT} shrink-0" /> {/snippet} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte index 855cf6ce7..ff6d39fdd 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte @@ -1,14 +1,16 @@ - - - - - - -
-
- Context - · - - {formatParameters(gauge.contextUsed)} - / {gauge.contextTotal !== null ? formatParameters(gauge.contextTotal) : '-'} - -
- - {#if gauge.activeModelId !== null && !gauge.isActiveModelLoaded} - - {:else if showProgressBar} -
-
-
- -
- - {gauge.contextPercent}% used - - - {formatParameters((gauge.contextTotal ?? 0) - gauge.contextUsed)} remaining - -
- {:else} -
No context info available
- {/if} - - {#if gauge.hasAnyUsage} - - {/if} -
-
-
+
+ +
diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte new file mode 100644 index 000000000..81acdbaf7 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte @@ -0,0 +1,106 @@ + + +{#if gaugePopup.open} +
+
+
+ Context + · + + {formatParameters(gauge.contextUsed)} + / {gauge.contextTotal !== null ? formatParameters(gauge.contextTotal) : '-'} + +
+ + {#if gauge.activeModelId !== null && !gauge.isActiveModelLoaded} + + {:else if showProgressBar} +
+
+
+ +
+ + {gauge.contextPercent}% used + + + {formatParameters((gauge.contextTotal ?? 0) - gauge.contextUsed)} remaining + +
+ {:else} +
No context info available
+ {/if} + + {#if gauge.hasAnyUsage} + + {/if} +
+
+{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte index 560bf73ab..8e8a14ac3 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte @@ -7,7 +7,6 @@ import { SYSTEM_MESSAGE_PLACEHOLDER } from '$lib/constants'; import { REASONING_TAGS } from '$lib/constants/agentic'; import { MessageRole, AttachmentType, AgenticSectionType } from '$lib/enums'; - import { fadeInView } from '$lib/actions/fade-in-view.svelte'; import { ChatMessageAssistant, ChatMessageUser, @@ -328,7 +327,7 @@ } -
+
{#if message.role === MessageRole.SYSTEM} {/if}
+ + diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserPending.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserPending.svelte index 58b3a42e0..1cc79fe6b 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserPending.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserPending.svelte @@ -1,6 +1,5 @@
- import { onMount } from 'svelte'; - import { beforeNavigate, afterNavigate } from '$app/navigation'; import { ChatMessage, ChatMessageUserPending } from '$lib/components/app'; import { setChatActionsContext } from '$lib/contexts'; import { MessageRole } from '$lib/enums'; @@ -35,9 +33,6 @@ let { messages = [], onUserAction, onMessagesReady }: Props = $props(); let allConversationMessages = $state([]); - let isVisible = $state(false); - let previousConversationId = $state(null); - let previousRouteId = $state(null); const currentConfig = config(); @@ -123,26 +118,10 @@ } } - // Track conversation changes to trigger transition even on same route + // Refresh messages whenever the active conversation changes $effect(() => { - const conversation = activeConversation(); - const currentId = conversation?.id ?? null; - - if (currentId !== previousConversationId && previousConversationId !== null) { - // Conversation changed - trigger fade out/in - isVisible = false; - requestAnimationFrame(() => { - refreshAllMessages(); - previousConversationId = currentId; - requestAnimationFrame(() => { - isVisible = true; - }); - }); - } else { - previousConversationId = currentId; - if (conversation) { - refreshAllMessages(); - } + if (activeConversation()) { + refreshAllMessages(); } }); @@ -152,23 +131,6 @@ onMessagesReady?.(displayMessages.length); }); - onMount(() => { - requestAnimationFrame(() => { - isVisible = true; - }); - }); - - beforeNavigate((navigation) => { - isVisible = false; - previousRouteId = navigation.from?.route.id ?? null; - }); - - afterNavigate(() => { - requestAnimationFrame(() => { - isVisible = true; - }); - }); - let siblingInfoByMessageId = $derived(buildSiblingInfoMap(allConversationMessages)); let displayMessages = $derived.by(() => { @@ -272,11 +234,7 @@ }); -
+
{#each displayMessages as { message, toolMessages, isLastAssistantMessage, isLastUserMessage, nextAssistantMessage, siblingInfo } (message.id)} import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes'; import { AlertTriangle, Loader2, RefreshCw } from '@lucide/svelte'; - import { fadeInView } from '$lib/actions/fade-in-view.svelte'; import * as Alert from '$lib/components/ui/alert'; import { serverError, serverLoading, serverStatus, serverStore } from '$lib/stores/server.svelte'; @@ -10,10 +9,7 @@ {#if hasError} -
+
{#if isLoadingModel} diff --git a/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte b/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte index ea7fe7b95..ad7032261 100644 --- a/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte +++ b/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte @@ -89,10 +89,15 @@ -
-
- {@render children()} + + {#if open} +
+
+ {@render children()} +
-
+ {/if} diff --git a/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte b/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte index 4370aea42..5cbe003bd 100644 --- a/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte +++ b/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte @@ -90,8 +90,12 @@ -
- {@render children()} -
+ + {#if open} +
+ {@render children()} +
+ {/if}
diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte index 8ac7f9448..fc7e31412 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte @@ -78,7 +78,6 @@ import { createAutoScrollController } from '$lib/hooks/use-auto-scroll.svelte'; import type { DatabaseMessageExtra } from '$lib/types/database'; import { config } from '$lib/stores/settings.svelte'; - import { fadeInView } from '$lib/actions/fade-in-view.svelte'; interface Props { attachments?: DatabaseMessageExtra[]; @@ -108,6 +107,15 @@ return null; }); const liveSvgHtml = $derived(streamingSvgCode !== null ? sanitizeSvg(streamingSvgCode) : ''); + + // Derived rather than called inline in the template so it only recomputes when + // the block actually changes. Auto-detection is disabled while streaming: it + // costs ~38ms a call and re-guesses the language on every chunk. + const streamingCodeHtml = $derived( + incompleteCodeBlock + ? highlightCode(incompleteCodeBlock.code, incompleteCodeBlock.language || 'text', false) + : '' + ); let previewDialogOpen = $state(false); let previewCode = $state(''); let previewLanguage = $state('text'); @@ -828,7 +836,7 @@ : ''}" > {#each renderedBlocks as block (block.id)} -
+
{@html block.html}
{/each} @@ -904,10 +912,7 @@ >
{@html highlightCode(
-								incompleteCodeBlock.code,
-								incompleteCodeBlock.language || 'text'
-							)}{@html streamingCodeHtml}
diff --git a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte index fa10d5d79..9ec57a558 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte @@ -15,6 +15,7 @@ REDACTED_HEADERS } from '$lib/constants'; import { browser } from '$app/environment'; + import { HealthCheckStatus } from '$lib/enums'; interface Props { open: boolean; @@ -24,6 +25,16 @@ let { open = $bindable(), onOpenChange }: Props = $props(); let newServerUrl = $state(''); + let newServerName = $state(''); + let nameAutoFilled = $state(''); + let nameTouched = $state(false); + + let previewRun = 0; + + function handleNameChange(value: string) { + newServerName = value; + nameTouched = true; + } let newServerHeaders = $state(''); let newServerUseProxy = $state(false); @@ -115,6 +126,48 @@ } }); + // Debounced preview handshake: once the URL is valid and stable, fetch the + // server-reported name to prefill the display name field. A manual edit + // freezes the autofill for good, and failures stay silent. + $effect(() => { + const url = newServerUrl.trim(); + const headers = newServerHeaders.trim(); + const useProxy = newServerUseProxy; + + if (!open || newServerUrlError || !url) return; + + const run = ++previewRun; + // One throwaway id per run: concurrent previews (URL typed, then the + // bearer token pasted) would poison each other's shared health state. + const previewId = `${MCP_SERVER_ID_PREFIX}-preview-${run}`; + const timer = setTimeout(async () => { + await mcpStore.runHealthCheck({ + id: previewId, + enabled: false, + url, + headers: headers || undefined, + useProxy + }); + + const state = mcpStore.getHealthCheckState(previewId); + + mcpStore.clearHealthCheck(previewId); + + if (run !== previewRun) return; + + if (state.status !== HealthCheckStatus.SUCCESS) return; + + const autoName = state.serverInfo?.title || state.serverInfo?.name || ''; + + if (autoName && !nameTouched) { + newServerName = autoName; + nameAutoFilled = autoName; + } + }, 600); + + return () => clearTimeout(timer); + }); + let hasSelection = $derived(selectedRecommendationId !== null); let unconfiguredRecommendations = $derived.by(() => { @@ -146,6 +199,10 @@ function handleOpenChange(value: boolean) { if (!value) { newServerUrl = ''; + newServerName = ''; + nameAutoFilled = ''; + nameTouched = false; + previewRun++; newServerHeaders = ''; newServerUseProxy = false; newServerWantsAuthorization = false; @@ -163,6 +220,12 @@ id: newServerId, enabled: true, url: newServerUrl.trim(), + // A name equal to the autofilled server-reported one is not a + // customization: keep following the automatic label. + displayName: + newServerName.trim() && newServerName.trim() !== nameAutoFilled.trim() + ? newServerName.trim() + : undefined, headers: newServerHeaders.trim() || undefined, useProxy: newServerUseProxy }); @@ -210,6 +273,8 @@
(newServerUrl = v)} diff --git a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCard.svelte b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCard.svelte index cf171dcc9..5d4c89209 100644 --- a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCard.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCard.svelte @@ -70,7 +70,12 @@ async function startEditing() { isEditing = true; await tick(); - editFormRef?.setInitialValues(server.url, server.headers || '', server.useProxy || false); + editFormRef?.setInitialValues( + server.url, + server.headers || '', + server.useProxy || false, + displayName + ); } function cancelEditing() { @@ -81,9 +86,12 @@ } } - function saveEditing(url: string, headers: string, useProxy: boolean) { + function saveEditing(url: string, headers: string, useProxy: boolean, name?: string) { onUpdate({ url: url, + // undefined = prefill untouched, keep any existing custom name; + // empty string = field cleared, back to the automatic label + displayName: name === undefined ? server.displayName : name.trim() || undefined, headers: headers || undefined, useProxy: useProxy }); @@ -106,6 +114,7 @@ serverId={server.id} serverUrl={server.url} serverUseProxy={server.useProxy} + serverLabel={displayName} onSave={saveEditing} onCancel={cancelEditing} /> diff --git a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte index 8ed4ee8b8..19778f95b 100644 --- a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte @@ -7,13 +7,23 @@ serverId: string; serverUrl: string; serverUseProxy?: boolean; - onSave: (url: string, headers: string, useProxy: boolean) => void; + /** Current automatic label, prefilled so the user can customize it. */ + serverLabel?: string; + onSave: (url: string, headers: string, useProxy: boolean, name?: string) => void; onCancel: () => void; } - let { serverId, serverUrl, serverUseProxy = false, onSave, onCancel }: Props = $props(); + let { + serverId, + serverUrl, + serverUseProxy = false, + serverLabel = '', + onSave, + onCancel + }: Props = $props(); let editUrl = $derived(serverUrl); + let editName = $derived(serverLabel); let editHeaders = $state(''); let editUseProxy = $derived(serverUseProxy); @@ -34,7 +44,12 @@ function handleSave() { if (!canSave) return; - onSave(editUrl.trim(), editHeaders.trim(), editUseProxy); + + // An unchanged prefill keeps following the automatic label; only an + // actual edit becomes a persisted custom display name. + const name = editName.trim() !== serverLabel.trim() ? editName.trim() : undefined; + + onSave(editUrl.trim(), editHeaders.trim(), editUseProxy, name); } function handleSubmit(event: SubmitEvent) { @@ -42,10 +57,11 @@ handleSave(); } - export function setInitialValues(url: string, headers: string, useProxy: boolean) { + export function setInitialValues(url: string, headers: string, useProxy: boolean, name = '') { editUrl = url; editHeaders = headers; editUseProxy = useProxy; + editName = name; } @@ -55,6 +71,8 @@ (editName = v)} headers={editHeaders} useProxy={editUseProxy} onUrlChange={(v) => (editUrl = v)} diff --git a/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte b/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte index a7472add3..2b8e1226b 100644 --- a/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte @@ -17,6 +17,10 @@ interface Props { url: string; headers: string; + name?: string; + onNameChange?: (name: string) => void; + /** Shown in the empty display name field, e.g. the current automatic label. */ + namePlaceholder?: string; useProxy?: boolean; onUrlChange: (url: string) => void; onHeadersChange: (headers: string) => void; @@ -44,6 +48,9 @@ let { url, headers, + name = '', + onNameChange, + namePlaceholder = 'Name reported by the server', useProxy = false, onUrlChange, onHeadersChange, @@ -156,6 +163,20 @@ {/if}
+
+ + + onNameChange?.(e.currentTarget.value)} + /> +
+