address review comments

This commit is contained in:
Xuan Son Nguyen
2026-08-07 14:36:59 +02:00
committed by Xuan-Son Nguyen
parent 3954a9c39b
commit fbb29b3f98
4 changed files with 10 additions and 10 deletions
+4 -4
View File
@@ -70,8 +70,8 @@ struct server_subproc {
}
};
struct lru_sched {
lru_sched(server_models & models) : models(models) {}
struct server_lru_sched {
server_lru_sched(server_models & models) : models(models) {}
bool has_capacity(std::unique_lock<std::mutex> & lk) {
check_lock(lk);
@@ -412,7 +412,7 @@ server_models::server_models(
base_params(params),
base_env(get_environment()),
base_preset(ctx_preset.load_from_args(argc, argv)),
sched(std::make_unique<lru_sched>(*this)) {
sched(std::make_unique<server_lru_sched>(*this)) {
// clean up base preset
unset_reserved_args(base_preset, true);
// set binary path
@@ -424,7 +424,7 @@ server_models::server_models(
LOG_WRN("using original argv[0] as fallback: %s\n", argv[0]);
}
load_models();
debug_fake_timing = !common_get_env("LLAMA_DEBUG_FAKE_TIMING").empty();
debug_fake_timing = !common_get_env("LLAMA_SERVER_DEBUG_FAKE_TIMING").empty();
}
server_models::~server_models() = default;
+4 -4
View File
@@ -106,12 +106,12 @@ struct server_model_meta {
};
struct server_models_routes;
struct server_subproc; // defined in server-models.cpp
struct lru_sched; // defined in server-models.cpp
struct server_subproc; // defined in server-models.cpp
struct server_lru_sched; // defined in server-models.cpp
struct server_models {
friend struct server_models_routes;
friend struct lru_sched;
friend struct server_lru_sched;
private:
struct instance_t {
@@ -198,7 +198,7 @@ private:
common_preset base_preset; // base preset from llama-server CLI args
// queue of requests waiting for a models_max slot
std::unique_ptr<lru_sched> sched;
std::unique_ptr<server_lru_sched> sched;
// if true, add some delay to simulate works (useful for testing)
bool debug_fake_timing = false;
+1 -1
View File
@@ -145,7 +145,7 @@ def test_router_models_max_evicts_lru():
assert _get_model_status(first) == "unloaded"
# lru_sched tests (relying on LLAMA_DEBUG_FAKE_TIMING)
# server_lru_sched tests (relying on LLAMA_SERVER_DEBUG_FAKE_TIMING)
MODEL_A = "ggml-org/tinygemma3-GGUF:Q8_0"
MODEL_B = "ggml-org/test-model-stories260K:F32"
+1 -1
View File
@@ -134,7 +134,7 @@ class ServerProcess:
def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None:
env = {
**os.environ,
"LLAMA_DEBUG_FAKE_TIMING": "1",
"LLAMA_SERVER_DEBUG_FAKE_TIMING": "1",
}
if "LLAMA_CACHE" not in os.environ:
env["LLAMA_CACHE"] = "tmp"