mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-08-27 15:41:19 +02:00
Xuan-Son Nguyen
947fd9bb2b
* add cached responses * refactor on_sleeping_state * allow accessing metrics during sleep * metrics task should not reset timer * updated docs * fix * fix get_res_model_info * add test * fix a race condition * split metrics and slots tasks / results * should_reset_buckets
128 lines
3.7 KiB
Python
128 lines
3.7 KiB
Python
import pytest
|
|
import time
|
|
from utils import *
|
|
|
|
server = ServerPreset.tinyllama2()
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def create_server():
|
|
global server
|
|
server = ServerPreset.tinyllama2()
|
|
|
|
|
|
def is_sleeping(server: ServerProcess) -> bool:
|
|
res = server.make_request("GET", "/props")
|
|
assert res.status_code == 200
|
|
return res.body["is_sleeping"]
|
|
|
|
|
|
def wait_for_sleep(server: ServerProcess, timeout: float = 10.0):
|
|
start = time.time()
|
|
while time.time() - start < timeout:
|
|
if is_sleeping(server):
|
|
return
|
|
time.sleep(0.1)
|
|
raise TimeoutError("server did not go to sleep")
|
|
|
|
|
|
def fetch_metrics(server: ServerProcess) -> str:
|
|
res = server.make_request("GET", "/metrics")
|
|
assert res.status_code == 200
|
|
assert isinstance(res.body, str)
|
|
return res.body
|
|
|
|
|
|
def get_metric(text: str, name: str) -> float:
|
|
prefix = f"llamacpp:{name} "
|
|
values = [ln for ln in text.splitlines() if ln.startswith(prefix)]
|
|
assert len(values) == 1, f"{name} not found in metrics"
|
|
return float(values[0][len(prefix):])
|
|
|
|
|
|
def test_server_sleep():
|
|
global server
|
|
server.sleep_idle_seconds = 1
|
|
server.start()
|
|
|
|
# wait a bit so that server can go to sleep
|
|
time.sleep(2)
|
|
|
|
# make sure these endpoints are still responsive after sleep
|
|
res = server.make_request("GET", "/health")
|
|
assert res.status_code == 200
|
|
res = server.make_request("GET", "/props")
|
|
assert res.status_code == 200
|
|
assert res.body["is_sleeping"] == True
|
|
res = server.make_request("GET", "/models")
|
|
assert res.status_code == 200
|
|
assert len(res.body["data"]) == 1
|
|
assert res.body["data"][0]["id"] == server.model_alias
|
|
|
|
# make a generation request to wake up the server
|
|
res = server.make_request("POST", "/completion", data={
|
|
"n_predict": 1,
|
|
"prompt": "Hello",
|
|
})
|
|
assert res.status_code == 200
|
|
|
|
# it should no longer be sleeping
|
|
res = server.make_request("GET", "/props")
|
|
assert res.status_code == 200
|
|
assert res.body["is_sleeping"] == False
|
|
|
|
|
|
def test_server_sleep_read_only_endpoints():
|
|
global server
|
|
server.sleep_idle_seconds = 1
|
|
server.server_metrics = True
|
|
server.start()
|
|
|
|
res = server.make_request("POST", "/completion", data={
|
|
"n_predict": 4,
|
|
"prompt": "Hello",
|
|
})
|
|
assert res.status_code == 200
|
|
|
|
# the first scrape resets the throughput buckets, so that the second one reports
|
|
# the same zero rates as the snapshot taken on entering sleep
|
|
fetch_metrics(server)
|
|
metrics_awake = fetch_metrics(server)
|
|
assert get_metric(metrics_awake, "tokens_predicted_total") > 0
|
|
|
|
wait_for_sleep(server)
|
|
|
|
# during sleep, metrics are served from the snapshot taken right before sleeping
|
|
assert fetch_metrics(server) == metrics_awake
|
|
|
|
# scraping /metrics must not wake the server up
|
|
assert is_sleeping(server)
|
|
|
|
|
|
def test_server_sleep_metrics_buckets():
|
|
global server
|
|
server.sleep_idle_seconds = 1
|
|
server.server_metrics = True
|
|
server.start()
|
|
|
|
res = server.make_request("POST", "/completion", data={
|
|
"n_predict": 8,
|
|
"prompt": "Hello",
|
|
})
|
|
assert res.status_code == 200
|
|
|
|
wait_for_sleep(server)
|
|
|
|
# the first scrape reports the throughput of the last generation
|
|
assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") > 0
|
|
|
|
# nothing runs while sleeping, so the next scrapes report an empty window
|
|
assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0
|
|
assert is_sleeping(server)
|
|
|
|
# waking up must not report the buckets again
|
|
res = server.make_request("POST", "/tokenize", data={"content": "Hello"})
|
|
assert res.status_code == 200
|
|
assert is_sleeping(server) == False
|
|
assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0
|