Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 01:04:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

server: keep a queued model out of the victim pool until its waiters leave

Browse Source 
A waiter that gave up while its model was still loading left the
model idle with no request behind it, and nothing recounted the free
slots, so a second request queued behind it stayed queued forever.
tick() was only driven by requests: join, claim and the end of a
proxied request.

Keep the queue entry alive after a successful claim so the model
coming up is never picked as a victim before its waiters use it, and
recount the slots on every status change and whenever a waiter
abandons the queue. The model is then evicted as soon as it comes up
with nobody left to serve.
This commit is contained in:
Pascal
2026-09-07 13:39:02 +02:00
parent f2dc72b866
commit eee2fa9c79
1 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-models.cpp
+9 -9
View File
@@ -145,16 +145,16 @@ struct server_lru_sched {
return true;
}
// ok means the model is up: drop the entry, the other waiters just watch its status now
// on failure the entry is back in line; on success it stays until its waiters leave,
// so the model coming up is never picked as a victim before they use it
void claim_done(std::unique_lock<std::mutex> & lk, const std::string & model_id, bool ok) {
check_lock(lk);
if (ok) {
return;
}
for (auto it = queue.begin(); it != queue.end(); ++it) {
if (it->model_id == model_id) {
if (ok) {
queue.erase(it);
} else {
it->loading = false;
}
it->loading = false;
return;
}
}
@@ -1220,6 +1220,8 @@ void server_models::update_status(const std::string & name, const update_status_
if (!args.progress.is_null()) {
meta.progress = args.progress;
}
// a model that comes up idle or goes down changes the slot count for queued requests
sched->tick(lk);
}
// broadcast status change to SSE
{
@@ -1445,9 +1447,6 @@ bool server_models::ensure_model_ready(const std::string & name, const std::func
lk.lock();
sched->claim_done(lk, name, ok);
sched->tick(lk);
if (ok) {
queued = false; // entry is gone, the other waiters watch the status now
}
continue;
}
@@ -1455,6 +1454,7 @@ bool server_models::ensure_model_ready(const std::string & name, const std::func
}
} catch (...) {
leave_queue();
sched->tick(lk); // a slot freed for this waiter goes to the next one
throw;
}
leave_queue();