Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 17:25:07 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '961e4b26a7dd0e01e20599b27d709a74788ecb55' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/hip-quality-check.yml
#	AGENTS.md
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp/CMakeLists.txt
#	ggml/src/ggml-hexagon/htp/concat-ops.c
#	ggml/src/ggml-hexagon/htp/flash-attn-ops.c
#	ggml/src/ggml-hexagon/htp/flash-attn-ops.h
#	ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h
#	ggml/src/ggml-hexagon/htp/hmx-queue.c
#	ggml/src/ggml-hexagon/htp/hmx-queue.h
#	ggml/src/ggml-hexagon/htp/hmx-utils.h
#	ggml/src/ggml-hexagon/htp/htp-ctx.h
#	ggml/src/ggml-hexagon/htp/hvx-utils.h
#	ggml/src/ggml-hexagon/htp/main.c
#	ggml/src/ggml-hexagon/htp/matmul-ops.c
#	ggml/src/ggml-hexagon/htp/rope-ops.c
#	ggml/src/ggml-hexagon/htp/unary-ops.c
#	ggml/src/ggml-hexagon/htp/worker-pool.c
#	ggml/src/ggml-hexagon/htp/worker-pool.h
#	ggml/src/ggml-hip/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl
#	ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl
#	ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl
#	ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl
#	ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl
#	ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl
#	ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl
#	ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl
#	ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl
#	ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl
#	ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl
#	ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
#	ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl
#	tests/CMakeLists.txt
#	tests/test-backend-ops.cpp
#	tools/cli/CMakeLists.txt
#	tools/cli/cli.cpp
#	tools/llama-bench/llama-bench.cpp
This commit is contained in:
Concedo
2026-07-11 11:13:27 +08:00
parent 0f0245161e 961e4b26a7
commit 9d8a50378a
42 changed files with 2045 additions and 5613 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+16 -10
View File
@@ -1444,6 +1444,7 @@ private:
// populate chat template params
{
common_chat_templates_ptr chat_templates;
bool enable_thinking = false;
try {
chat_templates = common_chat_templates_init(model_tgt, params_base.chat_template);
@@ -1451,6 +1452,12 @@ private:
SRV_TRC("%s: chat template, example_format: '%s'\n", __func__,
common_chat_format_example(chat_templates.get(), params_base.use_jinja, params_base.default_template_kwargs).c_str());
// thinking is enabled if:
// 1. It's not explicitly disabled via --reasoning off
// 2. The chat template supports it
const bool template_supports_thinking = params_base.use_jinja && common_chat_templates_support_enable_thinking(chat_templates.get());
enable_thinking = params_base.enable_reasoning != 0 && template_supports_thinking;
SRV_TRC("%s: chat template, thinking = %d\n", __func__, enable_thinking);
} catch (const std::exception & e) {
SRV_ERR("%s: chat template parsing error: %s\n", __func__, e.what());
SRV_ERR("%s: please consider disabling jinja via --no-jinja, or use a custom chat template via --chat-template\n", __func__);
@@ -1458,13 +1465,6 @@ private:
return false;
}
// thinking is enabled if:
// 1. It's not explicitly disabled via --reasoning off
// 2. The chat template supports it
const bool template_supports_thinking = params_base.use_jinja && common_chat_templates_support_enable_thinking(chat_templates.get());
const bool enable_thinking = params_base.enable_reasoning != 0 && template_supports_thinking;
SRV_TRC("%s: chat template, thinking = %d\n", __func__, enable_thinking);
// IMPORTANT: chat_params is reused across sleeping / resuming states,
// never store llama_context/llama_model pointers in chat_params,
// as they may be invalidated after sleeping
@@ -3435,9 +3435,14 @@ private:
slot.n_prompt_tokens_processed++;
// stop the prompt batch exactly before a user message
if (spans.is_user_start(slot.prompt.n_tokens())) {
break;
// break at the last user message, or at user messages at least min step past the last checkpoint
if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) {
const auto pos = slot.prompt.n_tokens();
const auto & checkpoints = slot.prompt.checkpoints;
if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) {
break;
}
}
// process the last few tokens of the prompt separately in order to allow for a checkpoint to be created.
@@ -4521,6 +4526,7 @@ void server_routes::init_routes() {
{ "default_generation_settings", default_generation_settings_for_props },
{ "total_slots", params.n_parallel },
{ "model_alias", meta->model_name },
{ "model_ftype", meta->model_ftype },
{ "model_path", meta->model_path },
{ "modalities", json {
{"vision", meta->has_inp_image},
tools/server/server-models.cpp
+3 -69
View File
@@ -7,6 +7,7 @@
#include "build-info.h"
#include "preset.h"
#include "download.h"
#include "http.h"
#include <cpp-httplib/httplib.h> // TODO: remove this once we use HTTP client from download.h
#include <optional>
@@ -28,14 +29,7 @@
#include <sstream>
#include <cstring>
#ifdef _WIN32
#include <winsock2.h>
#include <windows.h>
#else
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#ifndef _WIN32
extern char **environ;
#endif
@@ -716,66 +710,6 @@ std::optional<server_model_meta> server_models::get_meta(const std::string & nam
return std::nullopt;
}
static int get_free_port() {
#ifdef _WIN32
WSADATA wsaData;
if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) {
return -1;
}
typedef SOCKET native_socket_t;
#define INVALID_SOCKET_VAL INVALID_SOCKET
#define CLOSE_SOCKET(s) closesocket(s)
#else
typedef int native_socket_t;
#define INVALID_SOCKET_VAL -1
#define CLOSE_SOCKET(s) close(s)
#endif
native_socket_t sock = socket(AF_INET, SOCK_STREAM, 0);
if (sock == INVALID_SOCKET_VAL) {
#ifdef _WIN32
WSACleanup();
#endif
return -1;
}
struct sockaddr_in serv_addr;
std::memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
serv_addr.sin_port = htons(0);
if (bind(sock, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) != 0) {
CLOSE_SOCKET(sock);
#ifdef _WIN32
WSACleanup();
#endif
return -1;
}
#ifdef _WIN32
int namelen = sizeof(serv_addr);
#else
socklen_t namelen = sizeof(serv_addr);
#endif
if (getsockname(sock, (struct sockaddr*)&serv_addr, &namelen) != 0) {
CLOSE_SOCKET(sock);
#ifdef _WIN32
WSACleanup();
#endif
return -1;
}
int port = ntohs(serv_addr.sin_port);
CLOSE_SOCKET(sock);
#ifdef _WIN32
WSACleanup();
#endif
return port;
}
// helper to convert vector<string> to char **
// pointers are only valid as long as the original vector is valid
static std::vector<char *> to_char_ptr_array(const std::vector<std::string> & vec) {
@@ -879,7 +813,7 @@ void server_models::load(const std::string & name, const load_options & opts) {
// prepare new instance info
instance_t inst;
inst.meta = meta;
inst.meta.port = get_free_port();
inst.meta.port = common_http_get_free_port();
inst.meta.status = SERVER_MODEL_STATUS_LOADING;
inst.meta.loaded_info = json{};
inst.meta.last_used = ggml_time_ms();
tools/server/server.cpp
+46 -23
View File
@@ -36,6 +36,19 @@ static inline void signal_handler(int signal) {
shutdown_handler(signal);
}
// satisfies -Wmissing-declarations (used by llama command)
int llama_server(int argc, char ** argv);
// to be used via CLI (argc / argv are used by router mode only)
int llama_server(common_params & params, int argc, char ** argv);
void llama_server_terminate();
void llama_server_terminate() {
if (shutdown_handler) {
shutdown_handler(0);
}
}
// wrapper function that handles exceptions and logs errors
// this is to make sure handler_t never throws exceptions; instead, it returns an error response
static server_http_context::handler_t ex_wrapper(server_http_context::handler_t func) {
@@ -72,9 +85,6 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t
};
}
// satisfies -Wmissing-declarations
int llama_server(int argc, char ** argv);
int llama_server(int argc, char ** argv) {
std::setlocale(LC_NUMERIC, "C");
@@ -94,16 +104,26 @@ int llama_server(int argc, char ** argv) {
llama_backend_init();
llama_numa_init(params.numa);
return llama_server(params, argc, argv);
}
int llama_server(common_params & params, int argc, char ** argv) {
bool is_run_by_cli = (argv == nullptr);
common_models_handler models_handler;
try {
models_handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
if (common_models_handler_is_preset_repo(models_handler)) {
// apply the preset and start the server in router mode
common_models_handler_apply(models_handler, params);
// note: router mode also accepts -hf remote-preset, so we need to check that first
if (!is_run_by_cli && !params.model.hf_repo.empty()) {
try {
models_handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
if (common_models_handler_is_preset_repo(models_handler)) {
// apply the preset and start the server in router mode
common_models_handler_apply(models_handler, params);
}
} catch (const std::exception & e) {
SRV_ERR("failed to fetch model metadata: %s\n", e.what());
return 1;
}
} catch (const std::exception & e) {
SRV_ERR("failed to fetch model metadata: %s\n", e.what());
return 1;
}
// router server never loads a model and must not touch the GPU
@@ -321,8 +341,9 @@ int llama_server(int argc, char ** argv) {
if (child.is_child() && child.get_mode() == SERVER_CHILD_MODE_DOWNLOAD) {
return child.run_download(params);
} else if (!is_router_server) {
} else if (!is_router_server && !is_run_by_cli) {
// single-model mode (NOT spawned by router)
// if this is invoked by CLI, model downloading should be already handled
try {
common_models_handler_apply(models_handler, params);
} catch (const std::exception & e) {
@@ -411,20 +432,22 @@ int llama_server(int argc, char ** argv) {
};
}
// TODO: refactor in common/console
// register signal handler if not running by CLI
if (!is_run_by_cli) {
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
struct sigaction sigint_action;
sigint_action.sa_handler = signal_handler;
sigemptyset (&sigint_action.sa_mask);
sigint_action.sa_flags = 0;
sigaction(SIGINT, &sigint_action, NULL);
sigaction(SIGTERM, &sigint_action, NULL);
struct sigaction sigint_action;
sigint_action.sa_handler = signal_handler;
sigemptyset (&sigint_action.sa_mask);
sigint_action.sa_flags = 0;
sigaction(SIGINT, &sigint_action, NULL);
sigaction(SIGTERM, &sigint_action, NULL);
#elif defined (_WIN32)
auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;
};
SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;
};
SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
#endif
}
SRV_INF("listening on %s\n", ctx_http.listening_address.c_str());