diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 35b9355700..6202e2aad3 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -685,11 +685,10 @@ void server_models::load_models() { models_to_load.size(), base_params.models_max)); } + // to be lazy-loaded after main() setup phase is completed + startup_models = std::move(models_to_load); + lk.unlock(); - for (const auto & name : models_to_load) { - SRV_INF("(startup) loading model %s\n", name.c_str()); - load(name); - } } else { // RELOAD: diff the new preset list against the current mapping and reconcile is_reloading = true; @@ -877,6 +876,19 @@ void server_models::load_models() { } } +void server_models::load_startup_models() { + std::vector to_load; + { + std::lock_guard lk(mutex); + to_load = std::move(startup_models); + startup_models.clear(); + } + for (const auto & name : to_load) { + SRV_INF("(startup) loading model %s\n", name.c_str()); + load(name); + } +} + void server_models::update_meta(const std::string & name, const server_model_meta & meta) { std::lock_guard lk(mutex); auto it = mapping.find(name); diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 79b231cba8..0a3e3b3a42 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -136,6 +136,9 @@ private: // if true, the next get_meta() will trigger a reload of model list bool need_reload = false; + // models marked with load-on-startup + std::vector startup_models; + // conv_id -> model name that currently serves its stream session, lets the resumable stream // routes go straight to the owning child instead of polling every one. populated when // proxy_request forwards a POST carrying an X-Conversation-Id. best effort: a stale entry just @@ -231,6 +234,9 @@ public: // - if a model is not running, it will be added or updated according to the source void load_models(); + // lazy-load startup_models, to be called after main() setup phase + void load_startup_models(); + // check if a model instance exists (thread-safe) bool has_model(const std::string & name); diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 01cc6633a3..d1538b7a3d 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -423,6 +423,18 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_http.stop(); }; + try { + models_routes->models.load_startup_models(); + } catch (const std::exception & e) { + SRV_ERR("failed to load models on startup: %s\n", e.what()); + ctx_http.stop(); + if (ctx_http.thread.joinable()) { + ctx_http.thread.join(); + } + clean_up(); + return 1; + } + } else { // setup clean up function, to be called before exit clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() {