Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-20 01:31:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '34af94cd9ab277632e27caeec2d41de2fd091b31' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/docker.yml
#	.github/workflows/make-release.yml
#	.github/workflows/release.yml
#	.pi/gg/SYSTEM.md
#	CMakeLists.txt
#	build-xcframework.sh
#	docs/development/HOWTO-add-model.md
#	docs/ops.md
#	docs/ops/SYCL.csv
#	docs/speculative.md
#	examples/sycl/update-ops-doc.sh
#	ggml/CMakeLists.txt
#	ggml/src/ggml-sycl/cpy.cpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	scripts/make-release-checks.sh
#	scripts/sync-ggml.last
#	tests/test-chat-auto-parser.cpp
#	tests/test-chat.cpp
#	tests/test-jinja.cpp
#	tests/test-llama-archs.cpp
#	tests/testing.h
#	tools/llama-bench/llama-bench.cpp
#	tools/server/README.md
This commit is contained in:
Concedo
2026-08-20 17:08:46 +08:00
parent 1fc6e11ab8 34af94cd9a
commit f0cd0225aa
135 changed files with 3116 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+17 -8
View File
@@ -2947,8 +2947,10 @@ private:
});
// generate the actual drafts (if any)
{
common_speculative_draft(spec.get());
if (!drafting.empty()) {
queue_tasks.yield_to_queue([&]() {
common_speculative_draft(spec.get());
});
}
// make checkpoints if needed
@@ -3578,8 +3580,8 @@ private:
has_output |= batch.tokens[i].output;
}
// decode on the worker thread, so we can still handle metrics tasks while waiting
// note: the sync is done here too, so that the wait also happens off the main thread
// yield to the queue, so we can still handle metrics tasks while decoding
// note: the sync is done here too, so that the wait is also covered by the yield
int ret = 0;
queue_tasks.yield_to_queue([&]() {
ret = llama_decode(ctx_tgt, batch_view);
@@ -3644,11 +3646,18 @@ private:
// TODO: avoid restoring the draft context and re-evaluating the drafted tokens when not needed [TAG_SPEC_AVOID_DRAFT_REEVAL]
// for now, always re-evaluate for simplicity
// ref: https://github.com/ggml-org/llama.cpp/pull/22728#issuecomment-4400925384
if (!common_speculative_process(spec.get(), batch_view)) {
SRV_ERR("%s", "failed to process speculative batch\n");
if (spec) {
bool ok = true;
queue_tasks.yield_to_queue([&]() {
ok = common_speculative_process(spec.get(), batch_view);
});
// TODO: handle error
throw std::runtime_error("failed to process speculative batch");
if (!ok) {
SRV_ERR("%s", "failed to process speculative batch\n");
// TODO: handle error
throw std::runtime_error("failed to process speculative batch");
}
}
// handle `n_cmpl > 1` tasks - when the main prompt is processed, activate all child tasks too
tools/server/server-queue.cpp
+54 -32
View File
@@ -150,31 +150,46 @@ bool server_queue::process_new_tasks(bool is_yielding) {
void server_queue::worker_loop() {
while (true) {
std::function<void()> work;
{
std::unique_lock<std::mutex> lock(mutex_tasks);
// wait on busy instead of yielding - busy stays set even when the yield already ended
worker.cv.wait(lock, [&]{
return worker.stop || worker.work != nullptr;
return worker.stop || worker.busy;
});
if (worker.stop) {
return;
}
work = std::move(worker.work);
worker.work = nullptr;
}
// note: do not hold any lock here, work() may post new tasks
std::exception_ptr exception;
try {
work();
} catch (...) {
exception = std::current_exception();
// process tasks while the yield is active
while (true) {
bool terminated = false;
try {
// note: do not hold any lock here, the callback may post new tasks
terminated = process_new_tasks(true);
} catch (...) {
std::unique_lock<std::mutex> lock(mutex_tasks);
worker.exception = std::current_exception();
break;
}
std::unique_lock<std::mutex> lock(mutex_tasks);
if (terminated || worker.stop || !worker.yielding) {
break;
}
if (!queue_tasks.empty()) {
continue; // a new task arrived in the meantime
}
condition_tasks.wait(lock, [&]{
return worker.stop || !running || !worker.yielding || !queue_tasks.empty();
});
}
// signal completion to yield_to_queue()
std::unique_lock<std::mutex> lock(mutex_tasks);
worker.exception = std::move(exception);
worker.busy = false;
// signal to yield_to_queue() that no more tasks will be processed
{
std::unique_lock<std::mutex> lock(mutex_tasks);
worker.busy = false;
}
condition_tasks.notify_all();
}
}
@@ -188,6 +203,7 @@ void server_queue::worker_stop() {
worker.stop = true;
}
worker.cv.notify_one();
condition_tasks.notify_all();
worker.thread.join();
}
@@ -199,29 +215,29 @@ void server_queue::yield_to_queue(std::function<void()> && work) {
{
std::unique_lock<std::mutex> lock(mutex_tasks);
GGML_ASSERT(!worker.busy && "yield_to_queue() cannot be nested");
worker.busy = true;
worker.work = std::move(work);
worker.busy = true;
worker.yielding = true;
}
worker.cv.notify_one();
while (true) {
// note: on terminate this is a no-op, but we still wait for the work to finish
process_new_tasks(true);
std::unique_lock<std::mutex> lock(mutex_tasks);
// declined tasks are moved to queue_tasks_unhandled, so a non-empty queue always has something new
condition_tasks.wait(lock, [&]{
return !worker.busy || (running && !queue_tasks.empty());
});
if (!worker.busy) {
break;
}
// run the work on the current thread, so that all ggml compute stays on the same thread
std::exception_ptr exception;
try {
work();
} catch (...) {
exception = std::current_exception();
}
std::exception_ptr exception;
{
std::unique_lock<std::mutex> lock(mutex_tasks);
// the yield is over, wait for the worker to finish its current task
worker.yielding = false;
condition_tasks.notify_all();
condition_tasks.wait(lock, [&]{
return !worker.busy;
});
// put the declined tasks back, keeping their order
while (!queue_tasks_unhandled.empty()) {
queue_tasks.push_front(std::move(queue_tasks_unhandled.back()));
@@ -231,8 +247,12 @@ void server_queue::yield_to_queue(std::function<void()> && work) {
// make sure to avoid idle timeout here
time_last_task = ggml_time_ms();
// the worker is idle now, take the exception it may have left behind
std::swap(exception, worker.exception);
// an exception from work() takes precedence over the one from the worker
if (!exception) {
std::swap(exception, worker.exception);
} else {
worker.exception = nullptr;
}
}
QUE_DBG("%s", "done yielding to queue\n");
@@ -249,7 +269,9 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
// spawn the worker thread used by yield_to_queue()
GGML_ASSERT(!worker.thread.joinable() && "start_loop() is already running");
worker.stop = false;
worker.stop = false;
worker.busy = false;
worker.yielding = false;
worker.thread = std::thread([this]() { worker_loop(); });
constexpr auto max_wait_time = std::chrono::seconds(1);
tools/server/server-queue.h
+7 -6
View File
@@ -33,11 +33,11 @@ private:
// used by yield_to_queue, all fields are guarded by mutex_tasks
struct worker_t {
std::thread thread;
std::condition_variable cv; // the worker sleeps on this until there is work
std::function<void()> work; // pending work, picked up by the thread
std::exception_ptr exception; // exception thrown by work(), if any
bool stop = false;
bool busy = false;
std::condition_variable cv; // the worker sleeps on this until a yield starts
std::exception_ptr exception; // exception thrown while processing tasks, if any
bool stop = false;
bool busy = false; // set by yield_to_queue(), cleared by the worker once it is done processing tasks
bool yielding = false; // work() is still running on the start_loop() thread
};
worker_t worker;
@@ -93,7 +93,7 @@ public:
*/
void start_loop(int64_t idle_sleep_ms = -1);
// run work() on a separate thread, while the current thread calls process_new_tasks
// while waiting for work() to finish, run process_new_tasks on the worker thread
// returns once work() is done (may throw exceptions)
// must be called from start_loop() thread (ideally inside callback_update_slots)
// use case: return metrics while encode/decode is running
@@ -116,6 +116,7 @@ public:
// the second argument tells whether the queue is currently yielding (see yield_to_queue)
// only then may the callback return false to decline the task, and it must leave it
// untouched, so that it can be put back in the queue later
// note: while yielding, the callback runs on worker thread, not main thread
void on_new_task(std::function<bool(server_task &&, bool)> callback) {
callback_new_task = std::move(callback);
}
tools/server/server-tools.cpp
+4 -57
View File
@@ -8,7 +8,6 @@
#include <regex>
#include <thread>
#include <chrono>
#include <ctime>
#include <atomic>
#include <cstring>
#include <cctype>
@@ -1692,61 +1691,6 @@ private:
}
};
//
// get_datetime: returns the current date and time
//
struct server_tool_get_datetime : server_tool {
server_tool_get_datetime() {
name = "get_datetime";
display_name = "Get Date & Time";
permission_write = false;
}
json get_definition() const override {
return {
{"type", "function"},
{"function", {
{"name", name},
{"description", "Returns the current date and time in UTC"},
{"parameters", {
{"type", "object"},
{"properties", {
{"format", {
{"type", "string"},
{"description",
"strftime()-style format string for the output (default: \"%Y-%m-%dT%H:%M:%SZ\", "
"e.g. ISO 8601). Choose your own format if you need something else, "
"e.g. \"%A, %B %d %Y\" for a human-readable date."},
}},
}},
}},
}},
};
}
json invoke(json params, server_tool::stream *) const override {
std::string format = json_value(params, "format", std::string("%Y-%m-%dT%H:%M:%SZ"));
auto now = std::chrono::system_clock::now();
auto time = std::chrono::system_clock::to_time_t(now);
std::tm tm_utc;
#ifdef _WIN32
gmtime_s(&tm_utc, &time);
#else
gmtime_r(&time, &tm_utc);
#endif
char buf[256];
size_t len = std::strftime(buf, sizeof(buf), format.c_str(), &tm_utc);
if (len == 0) {
return {{"error", "invalid format string"}};
}
return {{"result", std::string(buf, len)}};
}
};
//
// get_info: returns runtime info (OS name/version and cwd)
//
@@ -2005,6 +1949,10 @@ static server_tool & find_tool(std::vector<std::unique_ptr<server_tool>> & tools
//
static std::vector<std::unique_ptr<server_tool>> build_tools() {
// IMPORTANT: for contributors, please keep this array of tools as minimal as possible
// we only accept minimal i/o and shell command tools here
// for example, do not add: web search, get date time, etc.
// high-level functionality should be added either via MCP or web UI
std::vector<std::unique_ptr<server_tool>> tools;
tools.push_back(std::make_unique<server_tool_read_file>());
tools.push_back(std::make_unique<server_tool_file_glob_search>());
@@ -2012,7 +1960,6 @@ static std::vector<std::unique_ptr<server_tool>> build_tools() {
tools.push_back(std::make_unique<server_tool_exec_shell_command>());
tools.push_back(std::make_unique<server_tool_write_file>());
tools.push_back(std::make_unique<server_tool_edit_file>());
tools.push_back(std::make_unique<server_tool_get_datetime>());
tools.push_back(std::make_unique<server_tool_get_info>());
return tools;
}