Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 17:24:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

big wip

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-21 22:22:14 +02:00
parent d1a9e5a8e6
commit ea4b4b2862
25 changed files with 265 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-chat.cpp
+1 -1
View File
@@ -153,7 +153,7 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) {
prev_msg["content"] = json::array();
}
auto & prev_content = prev_msg["content"];
prev_content.insert(prev_content.end(), chatcmpl_content.begin(), chatcmpl_content.end());
prev_content.insert(chatcmpl_content);
} else {
item.erase("status");
item.erase("type");
tools/server/server-common.cpp
+4 -6
View File
@@ -9,8 +9,6 @@
#include "server-common.h"
// the chat API is not migrated yet, so this file still needs the bridge
#include <nlohmann/json.hpp>
#include <random>
#include <sstream>
@@ -1261,8 +1259,8 @@ json oaicompat_chat_params_parse(
auto caps = common_chat_templates_get_caps(opt.tmpls.get());
common_chat_templates_inputs inputs;
inputs.messages = common_chat_msgs_parse_oaicompat(common_json_raw<nlohmann::ordered_json>(messages));
inputs.tools = common_chat_tools_parse_oaicompat(common_json_raw<nlohmann::ordered_json>(tools));
inputs.messages = common_chat_msgs_parse_oaicompat(messages);
inputs.tools = common_chat_tools_parse_oaicompat(tools);
inputs.tool_choice = common_chat_tool_choice_parse_oaicompat(tool_choice);
inputs.json_schema = json_schema.is_null() ? "" : json_schema.dump();
inputs.grammar = grammar;
@@ -1270,7 +1268,7 @@ json oaicompat_chat_params_parse(
inputs.parallel_tool_calls = json_value(body, "parallel_tool_calls", caps["supports_parallel_tool_calls"]);
inputs.add_generation_prompt = json_value(body, "add_generation_prompt", true);
inputs.continue_final_message = body.contains("continue_final_message") ?
common_chat_continuation_parse(common_json_raw<nlohmann::ordered_json>(body.at("continue_final_message"))) :
common_chat_continuation_parse(body.at("continue_final_message")) :
COMMON_CHAT_CONTINUATION_NONE;
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_NONE && opt.prefill_assistant
&& !inputs.messages.empty() && inputs.messages.back().role == "assistant") {
@@ -1351,7 +1349,7 @@ json oaicompat_chat_params_parse(
llama_params["chat_parser"] = chat_params.parser;
}
llama_params["message_delimiters"] = common_json_from_raw(chat_params.message_delimiters.to_json());
llama_params["message_delimiters"] = chat_params.message_delimiters.to_json();
// Reasoning budget: pass parameters through to sampling layer
{
tools/server/server-context.cpp
+16 -16
View File
@@ -35,7 +35,6 @@
#include <windows.h>
#endif
using json = nlohmann::ordered_json;
constexpr int HTTP_POLLING_SECONDS = 1;
@@ -4220,7 +4219,8 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
// tasks.reserve(inputs.size()); // TODO: this is inaccurate due to child tasks
// message delimiters for checkpointing
auto delimiters = common_chat_msg_delimiters_parse(json_value(data, "message_delimiters", json::array()));
json delims = json_value(data, "message_delimiters", json::array());
auto delimiters = common_chat_msg_delimiters_parse(delims);
delimiters.tokenize(ctx_server.vocab);
for (size_t i = 0; i < inputs.size(); i++) {
@@ -4483,8 +4483,8 @@ static json get_res_model_info(const server_context_meta & meta) {
static json get_res_models(const server_context_meta & meta) {
// note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep
return {
{"models", {
return json{
{"models", json::array({
{
{"name", meta.model_name},
{"model", meta.model_name},
@@ -4493,23 +4493,23 @@ static json get_res_models(const server_context_meta & meta) {
{"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash
{"type", "model"},
{"description", ""},
{"tags", {""}},
{"capabilities", meta.has_mtmd ? json({"completion","multimodal"}) : json({"completion"})},
{"tags", json::array({""})},
{"capabilities", meta.has_mtmd ? json::array({"completion","multimodal"}) : json::array({"completion"})},
{"parameters", ""},
{"details", {
{"parent_model", ""},
{"format", "gguf"},
{"family", ""},
{"families", {""}},
{"families", json::array({""})},
{"parameter_size", ""},
{"quantization_level", ""}
}}
}
}},
})},
{"object", "list"},
{"data", {
{"data", json::array({
get_res_model_info(meta),
}}
})}
};
}
@@ -5045,7 +5045,7 @@ void server_routes::init_routes() {
std::string content;
if (body.count("tokens") != 0) {
const llama_tokens tokens = body.at("tokens");
const llama_tokens tokens = body.at("tokens").get<llama_tokens>();
content = tokens_to_str(ctx_server.vocab, tokens);
}
@@ -5103,7 +5103,7 @@ void server_routes::init_routes() {
std::vector<server_task> tasks;
tasks.reserve(documents.size());
for (size_t i = 0; i < documents.size(); i++) {
auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query, documents[i]);
auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query.get<std::string>(), documents[i]);
server_task task = server_task(SERVER_TASK_TYPE_RERANK);
task.id = rd.get_new_id();
task.tokens = std::move(tmp);
@@ -5207,7 +5207,7 @@ void server_routes::init_routes() {
std::unique_ptr<server_res_generator> server_routes::handle_slots_save(const server_http_req & req, int id_slot) {
auto res = create_response();
const json request_data = json::parse(req.body);
std::string filename = request_data.at("filename");
std::string filename = request_data.at("filename").get<std::string>();
if (!fs_validate_filename(filename)) {
res->error(format_error_response("Invalid filename", ERROR_TYPE_INVALID_REQUEST));
return res;
@@ -5243,7 +5243,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_slots_save(const ser
std::unique_ptr<server_res_generator> server_routes::handle_slots_restore(const server_http_req & req, int id_slot) {
auto res = create_response();
const json request_data = json::parse(req.body);
std::string filename = request_data.at("filename");
std::string filename = request_data.at("filename").get<std::string>();
if (!fs_validate_filename(filename)) {
res->error(format_error_response("Invalid filename", ERROR_TYPE_INVALID_REQUEST));
return res;
@@ -5332,7 +5332,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons
bool use_base64 = false;
if (body.count("encoding_format") != 0) {
const std::string & format = body.at("encoding_format");
const std::string format = body.at("encoding_format").get<std::string>();
if (format == "base64") {
use_base64 = true;
} else if (format != "float") {
@@ -5352,7 +5352,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons
int embd_normalize = params.embd_normalize;
if (body.count("embd_normalize") != 0) {
embd_normalize = body.at("embd_normalize");
embd_normalize = body.at("embd_normalize").get<int>();
if (meta->pooling_type == LLAMA_POOLING_TYPE_NONE) {
SRV_DBG("embd_normalize is not supported by pooling type %d, ignoring it\n", meta->pooling_type);
}
tools/server/server-models.cpp
+1 -1
View File
@@ -2462,7 +2462,7 @@ server_http_proxy::server_http_proxy(
bool has_files = !files.empty();
if (has_files) {
json form_fields = json::parse(body, nullptr, false);
json form_fields = json::parse_no_throw(body);
if (!form_fields.is_discarded()) {
auto boundary = generate_multipart_boundary();
effective_body = build_multipart_body(form_fields, files, boundary);
tools/server/server-schema.cpp
+4 -5
View File
@@ -258,7 +258,7 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
try {
auto schema = json_value(data, "json_schema", json::object());
SRV_DBG("JSON schema: %s\n", schema.dump(2).c_str());
std::string grammar_str = json_schema_to_grammar(common_json_from_raw(schema));
std::string grammar_str = json_schema_to_grammar(schema);
SRV_DBG("Converted grammar: %s\n", grammar_str.c_str());
params.sampling.grammar = {COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, std::move(grammar_str)};
} catch (const std::exception & e) {
@@ -487,7 +487,7 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
const auto & stop = data.at("stop");
if (stop.is_array()) {
for (const auto & word : stop) {
if (!word.empty()) ctx.params.antiprompt.push_back(word);
if (!word.empty()) ctx.params.antiprompt.push_back(word.get<std::string>());
}
} else if (stop.is_string()) {
ctx.params.antiprompt.push_back(stop.get<std::string>());
@@ -503,7 +503,7 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
->set_handler([&](field_eval_context & ctx, const json & data) {
const auto & samplers = data.at("samplers");
if (samplers.is_array()) {
ctx.params.sampling.samplers = common_sampler_types_from_names(samplers);
ctx.params.sampling.samplers = common_sampler_types_from_names(samplers.get<std::vector<std::string>>());
} else if (samplers.is_string()) {
ctx.params.sampling.samplers = common_sampler_types_from_chars(samplers.get<std::string>());
}
@@ -580,8 +580,7 @@ static void handle_with_catch(const char * name, std::function<void()> func) {
// treat a null value as absent so clients can send null to request the server default
static bool has_value(const json & data, const char * n) {
auto it = data.find(n);
return it != data.end() && !it->is_null();
return data.contains(n) && !data.at(n).is_null();
}
template <typename T>
tools/server/server-task.cpp
+14 -14
View File
@@ -1,5 +1,6 @@
#include "server-task.h"
#include "build-info.h"
#include "server-chat.h"
#include "chat.h"
@@ -12,7 +13,6 @@
#include <sstream>
using json = nlohmann::ordered_json;
//
// task_params
@@ -304,7 +304,7 @@ json completion_token_output::probs_vector_to_json(const std::vector<completion_
}
float completion_token_output::logarithm(float x) {
// nlohmann::json converts -inf to null, so we need to prevent that
// the JSON library converts -inf to null, so we need to prevent that
return x == 0.0f ? std::numeric_limits<float>::lowest() : std::log(x);
}
@@ -407,7 +407,7 @@ json server_task_result_cmpl_final::to_json_oaicompat() {
res["__verbose"] = to_json_non_oaicompat();
}
if (stats.is_set()) {
res.push_back({"timings", stats.to_json()});
res["timings"] = stats.to_json();
}
return res;
@@ -455,7 +455,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() {
res["__verbose"] = to_json_non_oaicompat();
}
if (stats.is_set()) {
res.push_back({"timings", stats.to_json()});
res["timings"] = stats.to_json();
}
return res;
@@ -516,7 +516,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
}
if (stats.is_set()) {
deltas.back().push_back({"timings", stats.to_json()});
deltas.back()["timings"] = stats.to_json();
}
// extra fields for debugging purposes
@@ -709,7 +709,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
});
if (stats.is_set()) {
server_sent_events.back().at("data").push_back({"timings", stats.to_json()});
server_sent_events.back().at("data")["timings"] = stats.to_json();
}
return server_sent_events;
@@ -1061,10 +1061,10 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() {
};
// populate the timings object when needed (usually for the last response or with timings_per_token enabled)
if (stats.is_set()) {
res.push_back({"timings", stats.to_json()});
res["timings"] = stats.to_json();
}
if (is_progress) {
res.push_back({"prompt_progress", progress.to_json()});
res["prompt_progress"] = progress.to_json();
}
if (!prob_output.probs.empty()) {
res["completion_probabilities"] = completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs);
@@ -1101,10 +1101,10 @@ json server_task_result_cmpl_partial::to_json_oaicompat() {
res["__verbose"] = to_json_non_oaicompat();
}
if (stats.is_set()) {
res.push_back({"timings", stats.to_json()});
res["timings"] = stats.to_json();
}
if (is_progress) {
res.push_back({"prompt_progress", progress.to_json()});
res["prompt_progress"] = progress.to_json();
}
return res;
@@ -1155,10 +1155,10 @@ json server_task_result_cmpl_partial::to_json_oaicompat_chat() {
}
if (stats.is_set()) {
last_json.push_back({"timings", stats.to_json()});
last_json["timings"] = stats.to_json();
}
if (is_progress) {
last_json.push_back({"prompt_progress", progress.to_json()});
last_json["prompt_progress"] = progress.to_json();
}
}
@@ -1305,10 +1305,10 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
if (!events.empty()) {
json & data = events.back().at("data");
if (stats.is_set()) {
data.push_back({"timings", stats.to_json()});
data["timings"] = stats.to_json();
}
if (is_progress) {
data.push_back({"prompt_progress", progress.to_json()});
data["prompt_progress"] = progress.to_json();
}
}
tools/server/server-tools.cpp
+1 -1
View File
@@ -2156,7 +2156,7 @@ void server_tools::setup(const std::vector<std::string> & enabled_tools,
res->status = 200;
res->data = safe_json_to_str(result);
}
} catch (const json::exception & e) {
} catch (const common_json_error & e) {
res->status = 400;
res->data = safe_json_to_str(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST));
} catch (const std::invalid_argument & e) {