server: add --sleep-mode rst

common/arg.cpp

@@ -3757,6 +3757,22 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.sleep_idle_seconds = value;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"--sleep-mode"}, "MODE",
+        "what to release when the server sleeps:\n"
+        "- 'free' frees context and model memory\n"
+        "- 'rst' restarts the whole process, may help reset memory to zero on certain backend\n"
+        "(default: free)",
+        [](common_params & params, const std::string & value) {
+            if (value == "free") {
+                params.sleep_mode = COMMON_SLEEP_MODE_FREE;
+            } else if (value == "rst") {
+                params.sleep_mode = COMMON_SLEEP_MODE_RST;
+            } else {
+                throw std::invalid_argument("invalid value: " + value);
+            }
+        }
+    ).set_examples({LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"--simple-io"},
         "use basic IO for better compatibility in subprocesses and limited consoles",

common/common.h

@@ -408,6 +408,11 @@ struct common_params_diffusion {
 
 // reasoning API response format (not to be confused as chat template's reasoning format)
 // only used by server
+enum common_sleep_mode {
+    COMMON_SLEEP_MODE_FREE, // free context and model memory
+    COMMON_SLEEP_MODE_RST, // also restart the process, releasing all backend resources
+};
+
 enum common_reasoning_format {
     COMMON_REASONING_FORMAT_NONE,
     COMMON_REASONING_FORMAT_AUTO,            // Same as deepseek, using `message.reasoning_content`
@@ -633,6 +638,7 @@ struct common_params {
     int enable_reasoning = -1; // -1 = auto, 0 = disable, 1 = enable
     bool prefill_assistant = true; // if true, any trailing assistant message will be prefilled into the response
     int sleep_idle_seconds = -1;   // if >0, server will sleep after this many seconds of idle time
+    common_sleep_mode sleep_mode = COMMON_SLEEP_MODE_FREE;
 
     std::vector<std::string> api_keys;
 

tools/server/README.md

@@ -196,11 +196,11 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG) |
 | `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG_FILE) |
 | `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)<br/>(env: LLAMA_ARG_UI_MCP_PROXY) |
-| `--tools TOOL1,TOOL2,...` | experimental: whether to enable server tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
+| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
 | `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)<br/>available options:<br/>  'docker:<image>', 'podman:<image>': spin up a new container and reuse it for all invocations, clean up on server exit<br/>  'docker-container:<id>', 'podman-container:<id>': use an existing container by ID, won't stop on server exit<br/>  'ssh:<target>': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required<br/><br/>(env: LLAMA_ARG_TOOLS_RUNTIME) |
 | `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_CONFIG) |
 | `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_JSON) |
-| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all server tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) |
+| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) |
 | `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)<br/>(env: LLAMA_ARG_UI) |
 | `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)<br/>(env: LLAMA_ARG_EMBEDDINGS) |
 | `--rerank, --reranking` | enable reranking endpoint on server (default: disabled)<br/>(env: LLAMA_ARG_RERANKING) |
@@ -237,6 +237,7 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `-sps, --slot-prompt-similarity SIMILARITY` | how much the prompt of a request must match the prompt of a slot in order to use that slot (default: 0.10, 0.0 = disabled) |
 | `--lora-init-without-apply` | load LoRA adapters without applying them (apply later via POST /lora-adapters) (default: disabled) |
 | `--sleep-idle-seconds SECONDS` | number of seconds of idleness after which the server will sleep (default: -1; -1 = disabled) |
+| `--sleep-mode MODE` | what to release when the server sleeps:<br/>- 'free' frees context and model memory<br/>- 'rst' restarts the whole process, may help reset memory to zero on certain backend<br/>(default: free) |
 | `--log-prompts-dir PATH` | Log prompts to directory (auto-created if not present; only used for debugging, default: disabled) |
 | `--spec-draft-hf, -hfd, -hfrd, --hf-repo-draft <user>/<model>[:quant]` | Same as --hf-repo, but for the draft model (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_HF_REPO) |
 | `--spec-draft-threads, -td, --threads-draft N` | number of threads to use during generation (default: same as --threads) |
@@ -2073,6 +2074,8 @@ Note that the following endpoints are exempt from being considered as incoming t
 - `GET /models`
 - `GET /metrics`
 
+Some backends keep memory allocated even after the model is unloaded, for example a CUDA context stays on the GPU. To also release that memory, use `--sleep-mode rst`, which restarts the server process upon sleeping. The process keeps the same PID and port, and the responses of the endpoints listed above are preserved across the restart. This mode is not supported on Windows.
+
 ## More examples
 
 ### Interactive mode

tools/server/server-common.cpp

@@ -16,6 +16,18 @@
 #include <cstring>
 #include <type_traits>
 
+#if !defined(_WIN32)
+#include <unistd.h>
+#include <limits.h>
+#include <cerrno>
+#include <fcntl.h>
+#include <sys/resource.h>
+#endif
+
+#if defined(__APPLE__)
+#include <mach-o/dyld.h>
+#endif
+
 json format_error_response(const std::string & message, const enum error_type type) {
     std::string type_str;
     int code = 500;
@@ -1877,3 +1889,135 @@ server_tokens format_prompt_rerank(
 
     return result;
 }
+
+
+//
+// server_sleep_rst
+//
+
+#if !defined(_WIN32)
+static std::string server_proc_exe_path(char ** argv) {
+    char buf[PATH_MAX];
+#if defined(__linux__)
+    const ssize_t len = readlink("/proc/self/exe", buf, sizeof(buf) - 1);
+    if (len > 0) {
+        buf[len] = '\0';
+        return buf;
+    }
+#elif defined(__APPLE__)
+    uint32_t size = sizeof(buf);
+    if (_NSGetExecutablePath(buf, &size) == 0) {
+        return buf;
+    }
+#endif
+    return argv[0];
+}
+
+// exec() keeps the file descriptors open, so mark them all to be closed instead
+// this releases the listening port and the backend devices, and makes child processes see EOF
+static void server_proc_close_fds_on_exec() {
+    int n_fd = 4096;
+
+    struct rlimit lim;
+    if (getrlimit(RLIMIT_NOFILE, &lim) == 0 && lim.rlim_cur != RLIM_INFINITY) {
+        n_fd = std::min<int>(lim.rlim_cur, 65536);
+    }
+
+    // skip stdin/stdout/stderr, they are used to communicate with the router
+    for (int fd = 3; fd < n_fd; fd++) {
+        const int flags = fcntl(fd, F_GETFD);
+        if (flags != -1) {
+            fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
+        }
+    }
+}
+#endif
+
+static void server_proc_restart(char ** argv, const char * env_name, const std::string & env_value) {
+#if defined(_WIN32) || defined(__EMSCRIPTEN__)
+    GGML_UNUSED(argv);
+    GGML_UNUSED(env_name);
+    GGML_UNUSED(env_value);
+    SRV_ERR("%s", "restarting the process is not supported on this platform\n");
+#else
+    GGML_ASSERT(argv != nullptr);
+
+    // exec() rejects an env var larger than MAX_ARG_STRLEN (128 kB on linux)
+    if (env_value.size() > 64*1024) {
+        SRV_ERR("cannot restart the process, '%s' is too large (%zu bytes)\n", env_name, env_value.size());
+        return;
+    }
+
+    setenv(env_name, env_value.c_str(), 1);
+
+    const std::string exe = server_proc_exe_path(argv);
+    SRV_INF("restarting the process, exe = '%s'\n", exe.c_str());
+
+    server_proc_close_fds_on_exec();
+
+    // the log worker thread does not survive exec(), flush it while we still can
+    common_log_pause(common_log_main());
+    fflush(stdout);
+    fflush(stderr);
+
+    execv(exe.c_str(), argv);
+
+    // exec() only returns on error, the server can no longer serve requests at this point
+    GGML_ABORT("execv() failed: %s", strerror(errno));
+#endif
+}
+
+static const char * SLEEP_STATE_ENV = "LLAMA_SERVER_SLEEP_STATE";
+
+void server_sleep_rst::init(int argc, char ** argv) {
+    GGML_ASSERT(argv == nullptr || argc > 0);
+
+    this->argv = argv;
+
+    const char * state = std::getenv(SLEEP_STATE_ENV);
+    if (state == nullptr) {
+        return;
+    }
+
+    try {
+        boot_state = json::parse(state);
+    } catch (const std::exception & e) {
+        SRV_ERR("failed to read the state left by the previous process: %s\n", e.what());
+    }
+
+#if defined(_WIN32)
+    _putenv_s(SLEEP_STATE_ENV, "");
+#else
+    // clear it now, so that child processes do not inherit it
+    unsetenv(SLEEP_STATE_ENV);
+#endif
+}
+
+void server_sleep_rst::enable(common_params & params) {
+    if (params.sleep_mode != COMMON_SLEEP_MODE_RST) {
+        boot_state = json();
+        return;
+    }
+
+    if (argv == nullptr) {
+        // exec() can only restart a standalone process
+        SRV_WRN("%s", "--sleep-mode rst is not supported in this mode, using --sleep-mode free\n");
+        params.sleep_mode = COMMON_SLEEP_MODE_FREE;
+        boot_state = json();
+        return;
+    }
+
+    if (params.sleep_idle_seconds < 0) {
+        SRV_WRN("%s", "--sleep-mode has no effect without --sleep-idle-seconds\n");
+    }
+
+    enabled = true;
+}
+
+void server_sleep_rst::restart() const {
+    if (!enabled) {
+        return;
+    }
+
+    server_proc_restart(argv, SLEEP_STATE_ENV, safe_json_to_str(state_provider ? state_provider() : json()));
+}

tools/server/server-common.h

@@ -494,7 +494,7 @@ struct server_metrics {
         n_prompt_cached += n_tokens;
     }
 
-    // used to keep the metrics across a process restart, see --sleep-mode all
+    // used to keep the metrics across a process restart, see --sleep-mode rst
     json to_json() const;
     void from_json(const json & data);
 };
@@ -611,3 +611,39 @@ struct server_pipe {
         return true;
     }
 };
+
+//
+// server_sleep_rst
+// this allow --sleep-mode rst to reset the whole process, but still preserve metrics and props data
+// see README-dev.md for more info
+//
+
+struct server_sleep_rst {
+    // remember argv and read the state left by the previous process
+    // must be called once at startup, before spawning any thread or child process
+    void init(int argc, char ** argv);
+
+    // enable the restart upon sleeping, warns and falls back to --sleep-mode free if not possible
+    void enable(common_params & params);
+
+    // true if the process was restarted by a previous instance
+    // in this case, the model is only loaded upon the first request
+    bool is_boot_to_sleep() const { return !boot_state.is_null(); }
+
+    // state left by the previous process, only valid if is_boot_to_sleep()
+    const json & get_boot_state() const { return boot_state; }
+
+    // set the state to be preserved across the restart
+    void set_state_provider(std::function<json()> provider) { state_provider = std::move(provider); }
+
+    // restart the process, does nothing if not enabled, does not return on success
+    void restart() const;
+
+private:
+    bool    enabled = false;
+    char ** argv    = nullptr;
+
+    json boot_state;
+
+    std::function<json()> state_provider;
+};

tools/server/server-context.cpp

@@ -798,6 +798,15 @@ public:
     mtmd_context * mctx = nullptr;
     const llama_vocab * vocab = nullptr;
 
+    server_sleep_rst sleep_rst;
+
+    server_metrics metrics;
+
+    // called each time the model is loaded, used by server_routes to refresh its metadata
+    void on_model_loaded(std::function<void()> callback) {
+        callback_model_loaded = std::move(callback);
+    }
+
     server_queue    queue_tasks;
     server_response queue_results;
 
@@ -818,14 +827,6 @@ public:
         }
     }
 
-    server_metrics get_metrics() const {
-        return metrics;
-    }
-
-    void reset_metrics_bucket() {
-        metrics.reset_bucket();
-    }
-
 private:
     // note: accessing these fields outside of this class is not thread-safe
     // use server_context methods instead
@@ -866,8 +867,6 @@ private:
 
     std::unique_ptr<server_prompt_cache> prompt_cache;
 
-    server_metrics metrics;
-
     // queued prompt stats - llama_decode() is async, so the timing is only valid after a sync
     // note: kept out of server_metrics, which is copied as-is into the task result
     int64_t  t_decode_start  = 0; // start of the last submitted decode
@@ -885,6 +884,13 @@ private:
 
     bool sleeping = false;
 
+    // set once init() has run, which requires a loaded model
+    bool initialized = false;
+
+    bool queue_initialized = false;
+
+    std::function<void()> callback_model_loaded;
+
     int64_t t_last_load_progress_ms = 0;
 
     void destroy() {
@@ -912,11 +918,15 @@ private:
             }
             SRV_INF("%s", "server is entering sleeping state\n");
             destroy();
-        } else {
-            SRV_INF("%s", "server is exiting sleeping state\n");
-            if (!load_model(params_base)) {
-                GGML_ABORT("failed to reload model after sleeping");
-            }
+            sleeping = new_state;
+            // everything is released, the process can now restart itself
+            sleep_rst.restart();
+            return;
+        }
+
+        SRV_INF("%s", "server is exiting sleeping state\n");
+        if (!load_model(params_base)) {
+            GGML_ABORT("failed to reload model after sleeping");
         }
         sleeping = new_state;
     }
@@ -956,6 +966,15 @@ private:
     // load the model and initialize llama_context
     // this may also be called to resume from sleeping state
     bool load_model(common_params & params) {
+        if (!initialized && !sleeping) {
+            sleep_rst.enable(params);
+
+            if (sleep_rst.is_boot_to_sleep()) {
+                init_sleeping(params);
+                return true;
+            }
+        }
+
         load_progress_data load_progress_text  (this, "text_model");
         load_progress_data load_progress_mmproj(this, "mmproj_model");
         load_progress_data load_progress_spec  (this, "spec_model");
@@ -1349,25 +1368,34 @@ private:
         // propagate new defaults back to caller
         params = params_base;
 
-        if (!is_resume) {
-            return init();
+        // a process restarted into sleeping state loads the model here for the first time
+        if (!initialized) {
+            initialized = true;
+            if (!init()) {
+                return false;
+            }
         }
 
-        if (callback_state) {
+        SRV_INF("%s", "model loaded\n");
+
+        if (callback_model_loaded) {
+            callback_model_loaded();
+        }
+
+        if (is_resume && callback_state) {
             callback_state(SERVER_STATE_READY, {});
         }
 
         return true;
     }
 
-    // unlike load_model(), this is only called once during initialization
-    bool init() {
-        GGML_ASSERT(ctx_tgt   != nullptr);
-        GGML_ASSERT(model_tgt != nullptr);
+    // wiring up server queues, must be done once before start_loop()
+    void init_queue() {
+        if (queue_initialized) {
+            return; // already done by init_sleeping()
+        }
+        queue_initialized = true;
 
-        GGML_ASSERT(!sleeping);
-
-        // wiring up server queues
         queue_tasks.on_new_task([this](server_task && task, bool is_yielding) {
             return process_single_task(std::move(task), is_yielding);
         });
@@ -1379,6 +1407,28 @@ private:
         });
 
         metrics.init();
+    }
+
+    // enter sleeping state without a model, the model is loaded upon leaving that state
+    void init_sleeping(common_params & params) {
+        GGML_ASSERT(!initialized);
+
+        params_base = params;
+
+        init_queue();
+
+        sleeping = true;
+        queue_tasks.init_sleeping();
+
+        SRV_INF("%s", "restarted in sleeping state, the model will be loaded upon the first request\n");
+    }
+
+    // unlike load_model(), this is only called once during initialization
+    bool init() {
+        GGML_ASSERT(ctx_tgt   != nullptr);
+        GGML_ASSERT(model_tgt != nullptr);
+
+        init_queue();
 
         if (params_base.cache_idle_slots) {
             if (params_base.cache_ram_mib == 0) {
@@ -4074,6 +4124,10 @@ private:
 server_context::server_context() : impl(new server_context_impl()) {}
 server_context::~server_context() = default;
 
+void server_context::init(int argc, char ** argv) {
+    impl->sleep_rst.init(argc, argv);
+}
+
 bool server_context::load_model(common_params & params) {
     return impl->load_model(params);
 }
@@ -4454,6 +4508,21 @@ server_routes::server_routes(const common_params & params, server_context & ctx_
     queue_tasks.on_sleeping_state([this](bool is_sleeping) {
         update_cached_responses(is_sleeping);
     });
+
+    // meta is only available once the model is loaded, which may happen upon leaving sleeping state
+    this->ctx_server.on_model_loaded([this, &ctx_server]() {
+        update_meta(ctx_server);
+    });
+
+    // set the hook to allow sleep_rst to capture the state BEFORE resetting the process
+    this->ctx_server.sleep_rst.set_state_provider([this]() {
+        return cache_to_json();
+    });
+
+    // reverse of above: if we just booted AFTER sleep_rst reset the process, we restore it
+    if (this->ctx_server.sleep_rst.is_boot_to_sleep()) {
+        cache_from_json(this->ctx_server.sleep_rst.get_boot_state());
+    }
 }
 
 static json get_res_model_info(const server_context_meta & meta) {
@@ -5459,7 +5528,7 @@ void server_routes::update_cached_responses(bool is_sleeping) {
     if (is_sleeping) {
         cached_models  = get_res_models(*meta);
         cached_props   = get_res_props(*meta, params, true);
-        cached_metrics = ctx_server.get_metrics();
+        cached_metrics = ctx_server.metrics;
 
         should_reset_buckets = false;
 
@@ -5467,7 +5536,7 @@ void server_routes::update_cached_responses(bool is_sleeping) {
 
     } else if (should_reset_buckets) {
         // a scrape during sleep already reported these buckets
-        ctx_server.reset_metrics_bucket();
+        ctx_server.metrics.reset_bucket();
 
         should_reset_buckets = false;
     }
@@ -5495,5 +5564,9 @@ bool server_routes::cache_from_json(const json & data) {
         return false;
     }
 
+    // keep counting from where the previous process stopped
+    ctx_server.metrics = cached_metrics;
+
     return true;
 }
+

tools/server/server-context.h

@@ -87,8 +87,14 @@ struct server_context {
     server_context();
     ~server_context();
 
+    // remember the command line, needed to restart the process, see --sleep-mode rst
+    // must be called once at startup, before spawning any thread or child process
+    void init(int argc, char ** argv);
+
     // load the model and initialize llama_context
     // returns true on success
+    // note: if the process was restarted into sleeping state, no model is loaded and it
+    //       returns true right away, the model is then loaded upon the first request
     bool load_model(common_params & params);
 
     // this function will block main thread until termination
@@ -158,7 +164,7 @@ struct server_routes {
     // to be used in router mode
     json get_model_info() const;
 
-    // save / restore the cached responses across a process restart, see --sleep-mode all
+    // save / restore the cached responses across a process restart, see --sleep-mode rst
     // only valid while sleeping, as the cache is only updated upon entering that state
     json cache_to_json();
     bool cache_from_json(const json & data);
@@ -196,3 +202,4 @@ private:
     // call right before sleep to update the cached responses
     void update_cached_responses(bool is_sleeping);
 };
+

tools/server/server-queue.cpp

@@ -286,6 +286,28 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
     worker.yielding = false;
     worker.thread = std::thread([this]() { worker_loop(); });
 
+    // the process may start already sleeping, see init_sleeping()
+    {
+        std::unique_lock<std::mutex> lock(mutex_tasks);
+        if (sleeping) {
+            QUE_INF("%s", "starting in sleeping state\n");
+            condition_tasks.wait(lock, [&]{
+                return (!running || req_stop_sleeping);
+            });
+            if (running) {
+                QUE_INF("%s", "exiting sleeping state\n");
+                req_stop_sleeping = false;
+                // Call order cb{N} -> cb1 -> cb0
+                for (size_t i = callback_sleeping_state.size(); i > 0; i--) {
+                    callback_sleeping_state[i - 1](false);
+                }
+                sleeping = false;
+                condition_tasks.notify_all(); // notify wait_until_no_sleep()
+            }
+            time_last_task = ggml_time_ms();
+        }
+    }
+
     constexpr auto max_wait_time = std::chrono::seconds(1);
     auto should_sleep = [&]() -> bool {
         // caller must hold mutex_tasks

tools/server/server-queue.h

@@ -74,6 +74,13 @@ public:
         return sleeping;
     }
 
+    // enter sleeping state before start_loop(), for a process restarted into that state
+    // note: callback_sleeping_state(true) is not called, the state is already known
+    void init_sleeping() {
+        std::unique_lock<std::mutex> lock(mutex_tasks);
+        sleeping = true;
+    }
+
     // end the start_loop routine
     void terminate();
 

tools/server/server.cpp

@@ -167,6 +167,7 @@ int llama_server(common_params & params, int argc, char ** argv) {
 
     // struct that contains llama context and inference
     server_context ctx_server;
+    ctx_server.init(argc, argv);
 
     server_http_context ctx_http;
     if (!ctx_http.init(params)) {
@@ -458,11 +459,8 @@ int llama_server(common_params & params, int argc, char ** argv) {
             return 1;
         }
 
-        routes.update_meta(ctx_server);
         ctx_http.is_ready.store(true);
 
-        SRV_INF("%s", "model loaded\n");
-
         shutdown_handler = [&](int) {
             mcp_mgr.shutdown();
             // this will unblock start_loop()
@@ -513,7 +511,10 @@ int llama_server(common_params & params, int argc, char ** argv) {
         std::thread monitor_thread;
         if (child.is_child()) {
             monitor_thread = child.setup(shutdown_handler);
-            child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info());
+            // if no model is loaded, the process restarted into sleeping state and the router knows it
+            if (ctx_server.get_llama_context() != nullptr) {
+                child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info());
+            }
         }
 
         // this call blocks the main thread until queue_tasks.terminate() is called