mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-09-06 13:01:08 +02:00
address review comments
This commit is contained in:
committed by
Xuan-Son Nguyen
parent
3954a9c39b
commit
fbb29b3f98
@@ -70,8 +70,8 @@ struct server_subproc {
|
||||
}
|
||||
};
|
||||
|
||||
struct lru_sched {
|
||||
lru_sched(server_models & models) : models(models) {}
|
||||
struct server_lru_sched {
|
||||
server_lru_sched(server_models & models) : models(models) {}
|
||||
|
||||
bool has_capacity(std::unique_lock<std::mutex> & lk) {
|
||||
check_lock(lk);
|
||||
@@ -412,7 +412,7 @@ server_models::server_models(
|
||||
base_params(params),
|
||||
base_env(get_environment()),
|
||||
base_preset(ctx_preset.load_from_args(argc, argv)),
|
||||
sched(std::make_unique<lru_sched>(*this)) {
|
||||
sched(std::make_unique<server_lru_sched>(*this)) {
|
||||
// clean up base preset
|
||||
unset_reserved_args(base_preset, true);
|
||||
// set binary path
|
||||
@@ -424,7 +424,7 @@ server_models::server_models(
|
||||
LOG_WRN("using original argv[0] as fallback: %s\n", argv[0]);
|
||||
}
|
||||
load_models();
|
||||
debug_fake_timing = !common_get_env("LLAMA_DEBUG_FAKE_TIMING").empty();
|
||||
debug_fake_timing = !common_get_env("LLAMA_SERVER_DEBUG_FAKE_TIMING").empty();
|
||||
}
|
||||
|
||||
server_models::~server_models() = default;
|
||||
|
||||
@@ -106,12 +106,12 @@ struct server_model_meta {
|
||||
};
|
||||
|
||||
struct server_models_routes;
|
||||
struct server_subproc; // defined in server-models.cpp
|
||||
struct lru_sched; // defined in server-models.cpp
|
||||
struct server_subproc; // defined in server-models.cpp
|
||||
struct server_lru_sched; // defined in server-models.cpp
|
||||
|
||||
struct server_models {
|
||||
friend struct server_models_routes;
|
||||
friend struct lru_sched;
|
||||
friend struct server_lru_sched;
|
||||
|
||||
private:
|
||||
struct instance_t {
|
||||
@@ -198,7 +198,7 @@ private:
|
||||
common_preset base_preset; // base preset from llama-server CLI args
|
||||
|
||||
// queue of requests waiting for a models_max slot
|
||||
std::unique_ptr<lru_sched> sched;
|
||||
std::unique_ptr<server_lru_sched> sched;
|
||||
|
||||
// if true, add some delay to simulate works (useful for testing)
|
||||
bool debug_fake_timing = false;
|
||||
|
||||
@@ -145,7 +145,7 @@ def test_router_models_max_evicts_lru():
|
||||
assert _get_model_status(first) == "unloaded"
|
||||
|
||||
|
||||
# lru_sched tests (relying on LLAMA_DEBUG_FAKE_TIMING)
|
||||
# server_lru_sched tests (relying on LLAMA_SERVER_DEBUG_FAKE_TIMING)
|
||||
|
||||
MODEL_A = "ggml-org/tinygemma3-GGUF:Q8_0"
|
||||
MODEL_B = "ggml-org/test-model-stories260K:F32"
|
||||
|
||||
@@ -134,7 +134,7 @@ class ServerProcess:
|
||||
def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None:
|
||||
env = {
|
||||
**os.environ,
|
||||
"LLAMA_DEBUG_FAKE_TIMING": "1",
|
||||
"LLAMA_SERVER_DEBUG_FAKE_TIMING": "1",
|
||||
}
|
||||
if "LLAMA_CACHE" not in os.environ:
|
||||
env["LLAMA_CACHE"] = "tmp"
|
||||
|
||||
Reference in New Issue
Block a user