From 9f710a6e777202de9d2e29f05d0b041c3bedd92f Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 19 Aug 2026 12:29:19 +0200 Subject: [PATCH] add cached responses --- tools/server/server-context.cpp | 227 +++++++++++++++++--------------- tools/server/server-context.h | 7 + 2 files changed, 130 insertions(+), 104 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 842e4203cd..c669cf5f81 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4433,6 +4433,109 @@ server_routes::server_routes(const common_params & params, server_context & ctx_ init_routes(); } +static json get_res_models(const server_context_meta & meta) { + // note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep + + json model_info = { + {"id", meta.model_name}, + {"aliases", meta.model_aliases}, + {"tags", meta.model_tags}, + {"object", "model"}, + {"created", std::time(0)}, + {"owned_by", "llamacpp"}, + {"meta", { + {"vocab_type", meta.model_vocab_type}, + {"n_vocab", meta.model_vocab_n_tokens}, + {"n_ctx", meta.slot_n_ctx}, + {"n_ctx_train", meta.model_n_ctx_train}, + {"n_embd", meta.model_n_embd_inp}, + {"n_params", meta.model_n_params}, + {"size", meta.model_size}, + {"ftype", meta.model_ftype}, + }}, + }; + + return { + {"models", { + { + {"name", meta.model_name}, + {"model", meta.model_name}, + {"modified_at", ""}, + {"size", ""}, + {"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash + {"type", "model"}, + {"description", ""}, + {"tags", {""}}, + {"capabilities", meta.has_mtmd ? json({"completion","multimodal"}) : json({"completion"})}, + {"parameters", ""}, + {"details", { + {"parent_model", ""}, + {"format", "gguf"}, + {"family", ""}, + {"families", {""}}, + {"parameter_size", ""}, + {"quantization_level", ""} + }} + } + }}, + {"object", "list"}, + {"data", { + model_info, + }} + }; +} + +static json get_res_props(const server_context_meta & meta, const common_params & params, bool is_sleeping) { + // note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep + + task_params tparams; + tparams.sampling = params.sampling; + json default_generation_settings_for_props = json { + { "params", tparams.to_json(true) }, + { "n_ctx", meta.slot_n_ctx }, + }; + + std::string tmpl_default = common_chat_templates_source(meta.chat_params.tmpls.get(), ""); + std::string tmpl_tools = common_chat_templates_source(meta.chat_params.tmpls.get(), "tool_use"); + + json props = { + { "default_generation_settings", default_generation_settings_for_props }, + { "total_slots", params.n_parallel }, + { "model_alias", meta.model_name }, + { "model_ftype", meta.model_ftype }, + { "model_path", meta.model_path }, + { "modalities", json { + {"vision", meta.has_inp_image}, + {"video", meta.has_inp_video}, + {"audio", meta.has_inp_audio}, + } }, + { "media_marker", get_media_marker() }, + { "endpoint_slots", params.endpoint_slots }, + { "endpoint_props", params.endpoint_props }, + { "endpoint_metrics", params.endpoint_metrics }, + { "ui", params.ui }, + { "ui_settings", meta.json_ui_settings }, + { "chat_template", tmpl_default }, + { "chat_template_caps", meta.chat_template_caps }, + { "bos_token", meta.bos_token_str }, + { "eos_token", meta.eos_token_str }, + { "build_info", meta.build_info }, + { "is_sleeping", is_sleeping }, + { "cors_proxy_enabled", params.ui_mcp_proxy }, + }; + if (params.use_jinja) { + if (!tmpl_tools.empty()) { + props["chat_template_tool_use"] = tmpl_tools; + } + } + + return props; +} + +json server_routes::get_model_info() const { + return get_res_models(*meta).at("data"); +} + void server_routes::init_routes() { // IMPORTANT: all lambda functions must start with create_response() // this is to ensure that the server_res_generator can handle sleeping case correctly @@ -4566,53 +4669,10 @@ void server_routes::init_routes() { this->get_props = [this](const server_http_req &) { auto res = create_response(true); - - // this endpoint can be accessed during sleeping - // the next LOC is to avoid someone accidentally use ctx_server - bool ctx_server; // do NOT delete this line - GGML_UNUSED(ctx_server); - - task_params tparams; - tparams.sampling = params.sampling; - json default_generation_settings_for_props = json { - { "params", tparams.to_json(true) }, - { "n_ctx", meta->slot_n_ctx }, - }; - - std::string tmpl_default = common_chat_templates_source(meta->chat_params.tmpls.get(), ""); - std::string tmpl_tools = common_chat_templates_source(meta->chat_params.tmpls.get(), "tool_use"); - - json props = { - { "default_generation_settings", default_generation_settings_for_props }, - { "total_slots", params.n_parallel }, - { "model_alias", meta->model_name }, - { "model_ftype", meta->model_ftype }, - { "model_path", meta->model_path }, - { "modalities", json { - {"vision", meta->has_inp_image}, - {"video", meta->has_inp_video}, - {"audio", meta->has_inp_audio}, - } }, - { "media_marker", get_media_marker() }, - { "endpoint_slots", params.endpoint_slots }, - { "endpoint_props", params.endpoint_props }, - { "endpoint_metrics", params.endpoint_metrics }, - { "ui", params.ui }, - { "ui_settings", meta->json_ui_settings }, - { "chat_template", tmpl_default }, - { "chat_template_caps", meta->chat_template_caps }, - { "bos_token", meta->bos_token_str }, - { "eos_token", meta->eos_token_str }, - { "build_info", meta->build_info }, - { "is_sleeping", queue_tasks.is_sleeping() }, - { "cors_proxy_enabled", params.ui_mcp_proxy }, - }; - if (params.use_jinja) { - if (!tmpl_tools.empty()) { - props["chat_template_tool_use"] = tmpl_tools; - } - } - res->ok(props); + // note: do NOT use ctx_server here, this endpoint must be accessible during sleep + res->ok(!cached_props.is_null() + ? cached_props + : get_res_props(*meta, params, queue_tasks.is_sleeping())); return res; }; @@ -4874,42 +4934,10 @@ void server_routes::init_routes() { this->get_models = [this](const server_http_req &) { auto res = create_response(true); - - // this endpoint can be accessed during sleeping - // the next LOC is to avoid someone accidentally use ctx_server - bool ctx_server; // do NOT delete this line - GGML_UNUSED(ctx_server); - - json models = { - {"models", { - { - {"name", meta->model_name}, - {"model", meta->model_name}, - {"modified_at", ""}, - {"size", ""}, - {"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash - {"type", "model"}, - {"description", ""}, - {"tags", {""}}, - {"capabilities", meta->has_mtmd ? json({"completion","multimodal"}) : json({"completion"})}, - {"parameters", ""}, - {"details", { - {"parent_model", ""}, - {"format", "gguf"}, - {"family", ""}, - {"families", {""}}, - {"parameter_size", ""}, - {"quantization_level", ""} - }} - } - }}, - {"object", "list"}, - {"data", { - get_model_info(), - }} - }; - - res->ok(models); + // note: do NOT use ctx_server here, this endpoint must be accessible during sleep + res->ok(!cached_models.is_null() + ? cached_models + : get_res_models(*meta)); return res; }; @@ -5119,27 +5147,6 @@ void server_routes::init_routes() { }; } -json server_routes::get_model_info() const { - return json { - {"id", meta->model_name}, - {"aliases", meta->model_aliases}, - {"tags", meta->model_tags}, - {"object", "model"}, - {"created", std::time(0)}, - {"owned_by", "llamacpp"}, - {"meta", { - {"vocab_type", meta->model_vocab_type}, - {"n_vocab", meta->model_vocab_n_tokens}, - {"n_ctx", meta->slot_n_ctx}, - {"n_ctx_train", meta->model_n_ctx_train}, - {"n_embd", meta->model_n_embd_inp}, - {"n_params", meta->model_n_params}, - {"size", meta->model_size}, - {"ftype", meta->model_ftype}, - }}, - }; -} - std::unique_ptr server_routes::handle_slots_save(const server_http_req & req, int id_slot) { auto res = create_response(); const json request_data = json::parse(req.body); @@ -5388,3 +5395,15 @@ std::unique_ptr server_routes::handle_count_tokens(const l res->ok(response); return res; } + +void server_routes::update_cached_responses(bool enabled) { + if (enabled) { + cached_models = get_res_models(*meta); + cached_props = get_res_props(*meta, params, queue_tasks.is_sleeping()); + cached_metrics = "TODO"; + } else { + cached_models = nullptr; + cached_props = nullptr; + cached_metrics = ""; + } +} diff --git a/tools/server/server-context.h b/tools/server/server-context.h index f9ab1132b1..0120ebc8af 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -179,4 +179,11 @@ private: server_queue & queue_tasks; server_response & queue_results; std::unique_ptr create_response(bool bypass_sleep = false); + + // cached responses, to be used during sleep + json cached_models = nullptr; + json cached_props = nullptr; + std::string cached_metrics = ""; + // call right before sleep to update the cached responses + void update_cached_responses(bool enabled); };