mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-19 17:25:07 +02:00
Merge commit '9d5d882d8cd0f0a9283d87ed5e6fe3ee0d925fb1' into concedo_experimental
# Conflicts: # .github/labeler.yml # app/CMakeLists.txt # app/llama.cpp # build-xcframework.sh # common/CMakeLists.txt # common/download.h # docs/backend/SYCL.md # docs/backend/snapdragon/CMakeUserPresets.json # docs/speculative.md # ggml/CMakeLists.txt # ggml/include/ggml-sycl.h # ggml/src/ggml-hexagon/CMakeLists.txt # ggml/src/ggml-hexagon/ggml-hexagon.cpp # ggml/src/ggml-hexagon/htp/CMakeLists.txt # ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake # ggml/src/ggml-hexagon/htp/flash-attn-ops.c # ggml/src/ggml-hexagon/htp/hex-dma.h # ggml/src/ggml-hexagon/htp/hex-utils.h # ggml/src/ggml-hexagon/htp/hmx-flash-attn-ops.c # ggml/src/ggml-hexagon/htp/htp-ctx.h # ggml/src/ggml-hexagon/htp/htp-ops.h # ggml/src/ggml-hexagon/htp/htp_iface.idl # ggml/src/ggml-hexagon/htp/hvx-base.h # ggml/src/ggml-hexagon/htp/main.c # ggml/src/ggml-hexagon/htp/matmul-ops.c # ggml/src/ggml-hexagon/libggml-htp.inf # ggml/src/ggml-opencl/ggml-opencl.cpp # ggml/src/ggml-opencl/kernels/norm.cl # ggml/src/ggml-sycl/conv3d.cpp # ggml/src/ggml-sycl/ggml-sycl.cpp # scripts/snapdragon/adb/run-completion.sh # scripts/snapdragon/adb/run-tool.sh # scripts/snapdragon/ggml-hexagon-profile.py # tests/CMakeLists.txt # tests/test-backend-ops.cpp # tests/test-thread-safety.cpp # tools/llama-bench/llama-bench.cpp # tools/mtmd/CMakeLists.txt # tools/mtmd/tests/test-deepseek-ocr.py
This commit is contained in:
@@ -223,8 +223,8 @@ void server_model_meta::update_caps() {
|
||||
"LLAMA_ARG_HF_REPO_FILE",
|
||||
});
|
||||
params.offline = true;
|
||||
// params.skip_download = true; // TODO: ideally, we should validate the model here, but it takes too much time
|
||||
common_params_handle_models(params, LLAMA_EXAMPLE_SERVER, {});
|
||||
common_models_handler handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
|
||||
common_models_handler_apply(handler, params); // note: this won't download the model because offline=true
|
||||
if (params.mmproj.path.empty()) {
|
||||
multimodal = { false, false };
|
||||
} else {
|
||||
@@ -1393,9 +1393,8 @@ struct server_download_state : public common_download_callback {
|
||||
|
||||
bool run(common_params & params) {
|
||||
try {
|
||||
common_params_handle_models_params p;
|
||||
p.callback = this;
|
||||
common_params_handle_models(params, LLAMA_EXAMPLE_SERVER, p);
|
||||
common_models_handler handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
|
||||
common_models_handler_apply(handler, params, this);
|
||||
is_ok = true;
|
||||
} catch (const std::exception & e) {
|
||||
auto model_name = params.model.get_name();
|
||||
@@ -1768,23 +1767,14 @@ void server_models_routes::init_routes() {
|
||||
throw std::invalid_argument("model must be a non-empty string");
|
||||
}
|
||||
|
||||
common_params_model model;
|
||||
common_download_opts opts;
|
||||
common_params p;
|
||||
p.model.hf_repo = name;
|
||||
p.hf_token = params.hf_token;
|
||||
|
||||
model.hf_repo = name;
|
||||
opts.bearer_token = params.hf_token;
|
||||
// note: we only check main model, no need sidecar here
|
||||
opts.download_mmproj = false;
|
||||
opts.download_mtp = false;
|
||||
|
||||
// first, only check if the model is valid and can be downloaded
|
||||
opts.skip_download = true;
|
||||
// validate by fetching metadata
|
||||
bool ok = false;
|
||||
try {
|
||||
auto validation = common_download_model(model, opts);
|
||||
ok = !validation.model_path.empty();
|
||||
} catch (const common_skip_download_exception &) {
|
||||
// model is valid and will be downloaded
|
||||
common_models_handler_init(p, LLAMA_EXAMPLE_SERVER);
|
||||
ok = true;
|
||||
} catch (...) {
|
||||
SRV_ERR("unknown error while validating model '%s'\n", name.c_str());
|
||||
|
||||
+35
-9
@@ -89,15 +89,16 @@ int llama_server(int argc, char ** argv) {
|
||||
llama_backend_init();
|
||||
llama_numa_init(params.numa);
|
||||
|
||||
// note: router mode also accepts -hf remote-preset, so we need to check that first
|
||||
if (!params.model.hf_repo.empty()) {
|
||||
try {
|
||||
common_params_handle_models_params handle_params;
|
||||
handle_params.preset_only = true;
|
||||
common_params_handle_models(params, LLAMA_EXAMPLE_SERVER, handle_params);
|
||||
} catch (const std::exception & e) {
|
||||
// ignored for now
|
||||
common_models_handler models_handler;
|
||||
try {
|
||||
models_handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
|
||||
if (common_models_handler_is_preset_repo(models_handler)) {
|
||||
// apply the preset and start the server in router mode
|
||||
common_models_handler_apply(models_handler, params);
|
||||
}
|
||||
} catch (const std::exception & e) {
|
||||
SRV_ERR("failed to fetch model metadata: %s\n", e.what());
|
||||
return 1;
|
||||
}
|
||||
|
||||
// router server never loads a model and must not touch the GPU
|
||||
@@ -241,6 +242,19 @@ int llama_server(int argc, char ** argv) {
|
||||
// Google Cloud Platform (Vertex AI) compat
|
||||
ctx_http.register_gcp_compat();
|
||||
|
||||
// return 403 for disabled features
|
||||
server_http_context::handler_t res_403 = [](const server_http_req &) {
|
||||
auto res = std::make_unique<server_http_res>();
|
||||
res->status = 403;
|
||||
res->data = safe_json_to_str({
|
||||
{"error", {
|
||||
{"message", "this feature is disabled"},
|
||||
{"type", "feature_disabled"},
|
||||
}}
|
||||
});
|
||||
return res;
|
||||
};
|
||||
|
||||
// CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)
|
||||
if (params.ui_mcp_proxy) {
|
||||
SRV_WRN("%s", "-----------------\n");
|
||||
@@ -249,7 +263,11 @@ int llama_server(int argc, char ** argv) {
|
||||
SRV_WRN("%s", "-----------------\n");
|
||||
ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get));
|
||||
ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post));
|
||||
} else {
|
||||
ctx_http.get ("/cors-proxy", ex_wrapper(res_403));
|
||||
ctx_http.post("/cors-proxy", ex_wrapper(res_403));
|
||||
}
|
||||
|
||||
// EXPERIMENTAL built-in tools
|
||||
if (!params.server_tools.empty()) {
|
||||
try {
|
||||
@@ -264,6 +282,9 @@ int llama_server(int argc, char ** argv) {
|
||||
SRV_WRN("%s", "-----------------\n");
|
||||
ctx_http.get ("/tools", ex_wrapper(tools.handle_get));
|
||||
ctx_http.post("/tools", ex_wrapper(tools.handle_post));
|
||||
} else {
|
||||
ctx_http.get ("/tools", ex_wrapper(res_403));
|
||||
ctx_http.post("/tools", ex_wrapper(res_403));
|
||||
}
|
||||
|
||||
//
|
||||
@@ -274,7 +295,12 @@ int llama_server(int argc, char ** argv) {
|
||||
return child.run_download(params);
|
||||
} else if (!is_router_server) {
|
||||
// single-model mode (NOT spawned by router)
|
||||
common_params_handle_models(params, LLAMA_EXAMPLE_SERVER, {});
|
||||
try {
|
||||
common_models_handler_apply(models_handler, params);
|
||||
} catch (const std::exception & e) {
|
||||
SRV_ERR("failed to download model: %s\n", e.what());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
@@ -16,7 +16,7 @@ def test_mcp_no_proxy():
|
||||
server.start()
|
||||
|
||||
res = server.make_request("GET", "/cors-proxy")
|
||||
assert res.status_code == 404
|
||||
assert res.status_code == 403
|
||||
|
||||
|
||||
def test_mcp_proxy():
|
||||
|
||||
Reference in New Issue
Block a user