Merge remote-tracking branch 'upstream/master' into xsn/server_docker_isolate

tools/server/README.md

@@ -132,14 +132,14 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) |
 | `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) |
 | `--typical, --typical-p N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
-| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled, -1 = ctx_size) |
+| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled) |
 | `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) |
 | `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) |
 | `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled) |
 | `--dry-multiplier N` | set DRY sampling multiplier (default: 0.00, 0.0 = disabled) |
 | `--dry-base N` | set DRY sampling base value (default: 1.75) |
 | `--dry-allowed-length N` | set allowed length for DRY sampling (default: 2) |
-| `--dry-penalty-last-n N` | set DRY penalty for the last n tokens (default: -1, 0 = disable, -1 = context size) |
+| `--dry-penalty-last-n N` | set DRY penalty for the last n tokens (default: 64, 0 = disable) |
 | `--dry-sequence-breaker STRING` | add sequence breaker for DRY sampling, clearing out default breakers ('\n', ':', '"', '*') in the process; use "none" to not use any sequence breakers |
 | `--adaptive-target N` | adaptive-p: select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) (default: -1.00)<br/>[(more info)](https://github.com/ggml-org/llama.cpp/pull/17927) |
 | `--adaptive-decay N` | adaptive-p: decay rate for target adaptation over time. lower values are more reactive, higher values are more stable.<br/>(valid range 0.0 to 0.99) (default: 0.90) |
@@ -477,7 +477,7 @@ These words will not be included in the completion, so make sure to add them to
 
 `repeat_penalty`: Control the repetition of token sequences in the generated text. Default: `1.1`
 
-`repeat_last_n`: Last n tokens to consider for penalizing repetition. Default: `64`, where `0` is disabled and `-1` is ctx-size.
+`repeat_last_n`: Last n tokens to consider for penalizing repetition. Default: `64`, where `0` is disabled.
 
 `presence_penalty`: Repeat alpha presence penalty. Default: `0.0`, which is disabled.
 
@@ -489,7 +489,7 @@ These words will not be included in the completion, so make sure to add them to
 
 `dry_allowed_length`: Tokens that extend repetition beyond this receive exponentially increasing penalty: multiplier * base ^ (length of repeating sequence before token - allowed length). Default: `2`
 
-`dry_penalty_last_n`: How many tokens to scan for repetitions. Default: `-1`, where `0` is disabled and `-1` is context size.
+`dry_penalty_last_n`: How many tokens to scan for repetitions. Default: `64`, where `0` is disabled.
 
 `dry_sequence_breakers`: Specify an array of sequence breakers for DRY sampling. Only a JSON array of strings is accepted. Default: `['\n', ':', '"', '*']`
 
@@ -797,7 +797,7 @@ By default, it is read-only. To make POST request to change global properties, y
       "dry_multiplier": 0.0,
       "dry_base": 1.75,
       "dry_allowed_length": 2,
-      "dry_penalty_last_n": -1,
+      "dry_penalty_last_n": 64,
       "dry_sequence_breakers": [
         "\n",
         ":",
@@ -1077,6 +1077,10 @@ In *router mode* the query param `?model={model_id}` has to be set. This endpoin
 | `llamacpp:n_tokens_max` | Counter | High watermark of the context size observed. |
 | `llamacpp:n_decode_total` | Counter | Total Number of llama_decode() calls. |
 | `llamacpp:n_busy_slots_per_decode` | Gauge | Average number of busy slots per llama_decode() call. |
+| `llamacpp:spec_decode_num_draft_tokens_total` | Counter | Total draft tokens generated (0 when spec-decode is off). |
+| `llamacpp:spec_decode_num_accepted_tokens_total` | Counter | Total draft tokens accepted by the target model (0 when spec-decode is off). |
+| `llamacpp:spec_decode_num_drafts_total` | Counter | Total speculative decoding verification steps (0 when spec-decode is off). |
+| `llamacpp:spec_decode_num_accepted_tokens_per_pos_total` | Counter | Accepted tokens per draft position (labeled `position="N"`; absent when spec-decode is off or before the first completed speculative request). |
 
 ### POST `/slots/{id_slot}?action=save`: Save the prompt cache of the specified slot to a file.
 

tools/server/server-context.cpp

@@ -845,6 +845,11 @@ struct server_metrics {
     uint64_t n_decode_total     = 0;
     uint64_t n_busy_slots_total = 0;
 
+    uint64_t n_draft_tokens_total      = 0;
+    uint64_t n_draft_accepted_total    = 0;
+    uint64_t n_draft_verif_steps_total = 0;
+    std::vector<uint64_t> n_accepted_per_pos_total;
+
     void init() {
         t_start = ggml_time_us();
     }
@@ -863,6 +868,17 @@ struct server_metrics {
         n_tokens_predicted         += slot.n_decoded;
         t_tokens_generation        += slot.t_token_generation;
         t_tokens_generation_total  += slot.t_token_generation;
+
+        n_draft_tokens_total      += slot.n_draft_total;
+        n_draft_accepted_total    += slot.n_draft_accepted;
+        n_draft_verif_steps_total += slot.n_draft_verif_steps;
+
+        if (n_accepted_per_pos_total.size() < slot.n_accepted_per_pos.size()) {
+            n_accepted_per_pos_total.resize(slot.n_accepted_per_pos.size(), 0);
+        }
+        for (size_t i = 0; i < slot.n_accepted_per_pos.size(); i++) {
+            n_accepted_per_pos_total[i] += slot.n_accepted_per_pos[i];
+        }
     }
 
     void on_decoded(const std::vector<server_slot> & slots) {
@@ -1807,8 +1823,7 @@ private:
         // initialize samplers
         if (task.need_sampling()) {
             try {
-                slot.smpl.reset(common_sampler_init(
-                        model_tgt, task.params.sampling, (int32_t) llama_n_ctx(ctx_tgt)));
+                slot.smpl.reset(common_sampler_init(model_tgt, task.params.sampling));
             } catch (std::exception & e) {
                 std::string err_msg = std::string("Failed to initialize samplers: ") + e.what();
                 send_error(task, err_msg, ERROR_TYPE_INVALID_REQUEST);
@@ -2553,6 +2568,11 @@ private:
                     res->n_decode_total          = metrics.n_decode_total;
                     res->n_busy_slots_total      = metrics.n_busy_slots_total;
 
+                    res->n_draft_tokens_total      = metrics.n_draft_tokens_total;
+                    res->n_draft_accepted_total    = metrics.n_draft_accepted_total;
+                    res->n_draft_verif_steps_total = metrics.n_draft_verif_steps_total;
+                    res->n_accepted_per_pos_total  = metrics.n_accepted_per_pos_total;
+
                     if (task.metrics_reset_bucket) {
                         metrics.reset_bucket();
                     }
@@ -4148,7 +4168,6 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
             task.params = server_schema::eval_llama_cmpl_schema(
                     ctx_server.vocab,
                     params,
-                    meta->slot_n_ctx,
                     meta->logit_bias_eog,
                     data);
 
@@ -4442,6 +4461,18 @@ void server_routes::init_routes() {
                     {"name",  "n_tokens_max"},
                     {"help",  "Largest observed n_tokens."},
                     {"value",  res_task->n_tokens_max}
+            }, {
+                    {"name",  "spec_decode_num_draft_tokens_total"},
+                    {"help",  "Total draft tokens generated"},
+                    {"value",  res_task->n_draft_tokens_total}
+            }, {
+                    {"name",  "spec_decode_num_accepted_tokens_total"},
+                    {"help",  "Total draft tokens accepted by the target model"},
+                    {"value",  res_task->n_draft_accepted_total}
+            }, {
+                    {"name",  "spec_decode_num_drafts_total"},
+                    {"help",  "Total speculative decoding verification steps"},
+                    {"value",  res_task->n_draft_verif_steps_total}
             }}},
             {"gauge", {{
                     {"name",  "prompt_tokens_seconds"},
@@ -4483,6 +4514,17 @@ void server_routes::init_routes() {
             }
         }
 
+        // labeled counter: one time series per draft position
+        if (!res_task->n_accepted_per_pos_total.empty()) {
+            prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total"
+                          " Accepted tokens per draft position\n"
+                       << "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n";
+            for (size_t i = 0; i < res_task->n_accepted_per_pos_total.size(); i++) {
+                prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\""
+                           << i << "\"} " << res_task->n_accepted_per_pos_total[i] << "\n";
+            }
+        }
+
         res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->t_start);
         res->content_type = "text/plain; version=0.0.4";
         res->status = 200;

tools/server/server-models.cpp

@@ -70,6 +70,188 @@ struct server_subproc {
     }
 };
 
+struct server_lru_sched {
+    server_lru_sched(server_models & models) : models(models) {}
+
+    bool has_capacity(std::unique_lock<std::mutex> & lk) {
+        check_lock(lk);
+        return models.base_params.models_max <= 0
+            || count_running() < (size_t) models.base_params.models_max;
+    }
+
+    // returns "" if no model can be given up
+    std::string pick_victim(std::unique_lock<std::mutex> & lk, const std::string & exclude) {
+        check_lock(lk);
+        std::string victim;
+        int64_t victim_last_used = 0;
+        for (const auto & m : models.mapping) {
+            if (m.first == exclude) {
+                continue;
+            }
+            // a busy model is mid-request, one still coming up has no request to finish
+            if (m.second.req_count != 0 || !m.second.meta.is_ready_or_sleep()) {
+                continue;
+            }
+            if (victim.empty() || m.second.meta.last_used < victim_last_used) {
+                victim           = m.first;
+                victim_last_used = m.second.meta.last_used;
+            }
+        }
+        return victim;
+    }
+
+    // requests wanting the same model share one entry, so they all need only one slot
+    // and all get unblocked by the single load that entry performs
+    void join(std::unique_lock<std::mutex> & lk, const std::string & model_id) {
+        check_lock(lk);
+        if (entry_t * e = find(model_id)) {
+            e->n_waiters++;
+            SRV_INF("request for name=%s joined the queue, %d waiting\n", model_id.c_str(), e->n_waiters);
+            return;
+        }
+        queue.push_back({ model_id, 1, false, false });
+        SRV_INF("models_max reached, request for name=%s queued at position %zu\n",
+                model_id.c_str(), queue.size());
+    }
+
+    void leave(std::unique_lock<std::mutex> & lk, const std::string & model_id) {
+        check_lock(lk);
+        for (auto it = queue.begin(); it != queue.end(); ++it) {
+            if (it->model_id == model_id) {
+                if (--it->n_waiters <= 0) {
+                    queue.erase(it); // last one waiting for this model went away
+                }
+                return;
+            }
+        }
+    }
+
+    bool queue_empty(std::unique_lock<std::mutex> & lk) {
+        check_lock(lk);
+        return queue.empty();
+    }
+
+    // true if it is this model's turn to load, and nobody is loading it yet
+    bool try_claim(std::unique_lock<std::mutex> & lk, const std::string & model_id) {
+        check_lock(lk);
+        if (queue.empty() || queue.front().model_id != model_id || queue.front().loading) {
+            return false;
+        }
+        if (!has_capacity(lk)) {
+            return false;
+        }
+        queue.front().loading = true;
+        return true;
+    }
+
+    // ok means the model is up: drop the entry, the other waiters just watch its status now
+    void claim_done(std::unique_lock<std::mutex> & lk, const std::string & model_id, bool ok) {
+        check_lock(lk);
+        for (auto it = queue.begin(); it != queue.end(); ++it) {
+            if (it->model_id == model_id) {
+                if (ok) {
+                    queue.erase(it);
+                } else {
+                    it->loading = false;
+                }
+                return;
+            }
+        }
+    }
+
+    // a model is on its way out for this entry, so other requests do not also give up one
+    void mark_slot_pending(std::unique_lock<std::mutex> & lk, const std::string & model_id) {
+        check_lock(lk);
+        if (entry_t * e = find(model_id)) {
+            e->slot_pending = true;
+        }
+    }
+
+    // model_id went idle: give up its slot if a queued request needs one
+    // thread-safe, caller must NOT hold models.mutex
+    void on_model_idle(const std::string & model_id) {
+        if (models.base_params.models_max <= 0) {
+            return; // no limit, nothing is ever queued
+        }
+        {
+            std::unique_lock<std::mutex> lk(models.mutex);
+            if (queue.empty()) {
+                return;
+            }
+            size_t promised     = 0;
+            bool   has_unserved = false;
+            for (const auto & e : queue) {
+                if (e.needs_slot()) {
+                    has_unserved = true;
+                } else {
+                    promised++;
+                }
+            }
+            if (!has_unserved) {
+                return;
+            }
+            if ((int) count_running() - (int) promised < models.base_params.models_max) {
+                return; // a slot is already on its way
+            }
+            // never give up a model that a queued request wants
+            for (const auto & e : queue) {
+                if (e.model_id == model_id) {
+                    return;
+                }
+            }
+            auto it = models.mapping.find(model_id);
+            if (it == models.mapping.end() || it->second.req_count != 0 || !it->second.meta.is_ready_or_sleep()) {
+                return;
+            }
+            for (auto & e : queue) {
+                if (!e.slot_pending) {
+                    e.slot_pending = true;
+                    break;
+                }
+            }
+        }
+        SRV_INF("model name=%s went idle, giving up its slot to a queued request\n", model_id.c_str());
+        models.unload(model_id);
+    }
+
+  private:
+    struct entry_t {
+        std::string model_id;
+        int  n_waiters;    // requests waiting for this model
+        bool slot_pending; // a model is already being evicted for this entry
+        bool loading;      // one of the waiters is doing the load right now
+
+        // a slot is already coming, or already taken by the load in flight
+        bool needs_slot() const { return !slot_pending && !loading; }
+    };
+
+    entry_t * find(const std::string & model_id) {
+        for (auto & e : queue) {
+            if (e.model_id == model_id) {
+                return &e;
+            }
+        }
+        return nullptr;
+    }
+
+    void check_lock(std::unique_lock<std::mutex> & lk) {
+        GGML_ASSERT(lk.owns_lock() && lk.mutex() == &models.mutex);
+    }
+
+    size_t count_running() {
+        size_t count = 0;
+        for (const auto & m : models.mapping) {
+            if (m.second.meta.is_running()) {
+                count++;
+            }
+        }
+        return count;
+    }
+
+    server_models & models;
+    std::deque<entry_t> queue;
+};
+
 // short loopback budget for the resumable stream router to child JSON calls (probe, lookup,
 // delete). distinct from params.timeout_read/write which only applies to the generation proxy
 static constexpr int STREAM_LOOKUP_TIMEOUT_MS = 250;
@@ -229,7 +411,8 @@ server_models::server_models(
             : ctx_preset(LLAMA_EXAMPLE_SERVER),
               base_params(params),
               base_env(get_environment()),
-              base_preset(ctx_preset.load_from_args(argc, argv)) {
+              base_preset(ctx_preset.load_from_args(argc, argv)),
+              sched(std::make_unique<server_lru_sched>(*this)) {
     // clean up base preset
     unset_reserved_args(base_preset, true);
     // set binary path
@@ -241,8 +424,11 @@ server_models::server_models(
         LOG_WRN("using original argv[0] as fallback: %s\n", argv[0]);
     }
     load_models();
+    debug_fake_timing = !common_get_env("LLAMA_SERVER_DEBUG_FAKE_TIMING").empty();
 }
 
+server_models::~server_models() = default;
+
 void server_models::add_model(server_model_meta && meta) {
     if (mapping.find(meta.name) != mapping.end()) {
         throw std::runtime_error(string_format("model '%s' appears multiple times", meta.name.c_str()));
@@ -713,22 +899,15 @@ void server_models::unload_lru() {
         return; // no limit
     }
     // remove one of the servers if we passed the models_max (least recently used - LRU)
-    std::string lru_model_name = "";
-    int64_t lru_last_used = ggml_time_ms();
-    size_t count_active = 0;
+    std::string lru_model_name;
     {
         std::unique_lock<std::mutex> lk(mutex);
-        for (const auto & m : mapping) {
-            if (m.second.meta.is_running()) {
-                count_active++;
-                if (m.second.meta.last_used < lru_last_used) {
-                    lru_model_name = m.first;
-                    lru_last_used = m.second.meta.last_used;
-                }
-            }
+        if (sched->has_capacity(lk)) {
+            return;
         }
+        lru_model_name = sched->pick_victim(lk, "");
     }
-    if (!lru_model_name.empty() && count_active >= (size_t)base_params.models_max) {
+    if (!lru_model_name.empty()) {
         SRV_INF("models_max limit reached, removing LRU name=%s\n", lru_model_name.c_str());
         unload(lru_model_name);
         // wait for unload to complete
@@ -746,6 +925,11 @@ void server_models::load(const std::string & name) {
 }
 
 void server_models::load(const std::string & name, const load_options & opts) {
+    if (debug_fake_timing) {
+        // do not hold the mutex here, other requests must keep making progress
+        std::this_thread::sleep_for(std::chrono::seconds(2));
+    }
+
     if (!opts.custom_meta.has_value()) {
         if (!has_model(name)) {
             throw std::runtime_error("model name=" + name + " is not found");
@@ -1138,7 +1322,7 @@ void server_models::wait(std::unique_lock<std::mutex> & lk, const std::string &
     });
 }
 
-bool server_models::ensure_model_ready(const std::string & name) {
+bool server_models::ensure_model_ready(const std::string & name, const std::function<bool()> & should_stop) {
     auto meta = get_meta(name);
     if (!meta.has_value()) {
         throw std::runtime_error("model name=" + name + " is not found");
@@ -1149,25 +1333,112 @@ bool server_models::ensure_model_ready(const std::string & name) {
     if (meta->status == SERVER_MODEL_STATUS_SLEEPING) {
         return false; // child is sleeping but still running; new request will wake it up
     }
-    if (meta->status == SERVER_MODEL_STATUS_UNLOADED) {
-        SRV_INF("model name=%s is not loaded, loading...\n", name.c_str());
-        load(name);
-    }
 
-    // wait for loading to complete
-    SRV_INF("waiting until model name=%s is fully loaded...\n", name.c_str());
-    wait(name, [&meta](const server_model_meta & new_meta) {
-        if (new_meta.status != SERVER_MODEL_STATUS_LOADING) {
-            meta = new_meta; // update meta for final check after wait
-            return true;
+    bool queued   = false;
+    bool did_load = false;
+    std::string victim;
+    {
+        std::unique_lock<std::mutex> lk(mutex);
+        auto it = mapping.find(name);
+        if (it != mapping.end() && it->second.meta.status == SERVER_MODEL_STATUS_UNLOADED) {
+            bool has_capacity = sched->has_capacity(lk);
+            if (has_capacity && sched->queue_empty(lk)) {
+                lk.unlock();
+                SRV_INF("model name=%s is not loaded, loading...\n", name.c_str());
+                load(name);
+                did_load = true;
+            } else {
+                // also queue when a slot looks free but others wait already, else they starve
+                sched->join(lk, name);
+                queued = true;
+                if (!has_capacity) {
+                    // an idle model may sit here right now, do not wait for a request to end
+                    victim = sched->pick_victim(lk, name);
+                    if (!victim.empty()) {
+                        sched->mark_slot_pending(lk, name);
+                    }
+                }
+            }
         }
-        return false;
-    });
-
-    // check final status
-    if (!meta.has_value() || meta->is_failed()) {
-        throw std::runtime_error("model name=" + name + " failed to load");
     }
+    if (!victim.empty()) {
+        SRV_INF("evicting idle LRU name=%s to make room for name=%s\n", victim.c_str(), name.c_str());
+        unload(victim);
+    }
+
+    // while queued, this is also where the load happens: the head of the queue does it
+    SRV_INF("waiting until model name=%s is fully loaded...\n", name.c_str());
+    std::unique_lock<std::mutex> lk(mutex);
+    auto leave_queue = [this, &queued, &lk, &name]() {
+        if (queued) {
+            sched->leave(lk, name);
+            queued = false;
+        }
+    };
+
+    try {
+        bool saw_loading = false;
+        while (true) {
+            auto it = mapping.find(name);
+            if (it == mapping.end()) {
+                break; // removed by another code path, nothing to wait for
+            }
+            const server_model_status status = it->second.meta.status;
+
+            if (status == SERVER_MODEL_STATUS_LOADED || status == SERVER_MODEL_STATUS_SLEEPING) {
+                break;
+            }
+            if (status == SERVER_MODEL_STATUS_DOWNLOADING || status == SERVER_MODEL_STATUS_DOWNLOADED) {
+                break; // do not wait on a download child
+            }
+            if (status == SERVER_MODEL_STATUS_LOADING) {
+                saw_loading = true;
+            } else if (status == SERVER_MODEL_STATUS_UNLOADED) {
+                if (did_load || saw_loading) {
+                    // a spawn happened and the instance came back down
+                    if (it->second.meta.is_failed()) {
+                        throw std::runtime_error("model name=" + name + " failed to load");
+                    }
+                    break; // unloaded by another code path, caller reports "not running"
+                }
+                if (!queued) {
+                    break; // not queued, and the load someone else started fell over
+                }
+            }
+
+            if (should_stop && should_stop()) {
+                // if a model was evicted for us, the free slot goes to the next waiter
+                throw std::runtime_error("request cancelled while waiting for model name=" + name);
+            }
+
+            // our turn: our model is at the head, and a slot really did free up
+            if (status == SERVER_MODEL_STATUS_UNLOADED && sched->try_claim(lk, name)) {
+                lk.unlock();
+                bool ok = true;
+                try {
+                    SRV_INF("slot available, loading queued model name=%s\n", name.c_str());
+                    load(name);
+                    did_load = true;
+                } catch (const std::exception & e) {
+                    // lost a race for the slot, stay in line and retry
+                    SRV_WRN("queued load of name=%s did not go through: %s\n", name.c_str(), e.what());
+                    ok = false;
+                }
+                lk.lock();
+                sched->claim_done(lk, name, ok);
+                if (ok) {
+                    queued = false; // entry is gone, the other waiters watch the status now
+                }
+                continue;
+            }
+
+            cv.wait_for(lk, std::chrono::milliseconds(200));
+        }
+    } catch (...) {
+        leave_queue();
+        throw;
+    }
+    leave_queue();
 
     return true;
 }
@@ -1180,9 +1451,16 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co
     if (!meta->is_running()) {
         throw std::invalid_argument("model name=" + name + " is not running");
     }
-    if (update_last_used) {
+    {
         std::unique_lock<std::mutex> lk(mutex);
-        mapping[name].meta.last_used = ggml_time_ms();
+        if (update_last_used) {
+            mapping[name].meta.last_used = ggml_time_ms();
+        }
+        mapping[name].req_count++;
+    }
+    if (debug_fake_timing) {
+        // sleep after req_count++, so the model counts as busy while we wait here
+        std::this_thread::sleep_for(std::chrono::seconds(2));
     }
     SRV_INF("proxying request to model %s on port %d\n", name.c_str(), meta->port);
     std::string proxy_path = req.path;
@@ -1198,13 +1476,29 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co
             req.headers,
             req.body,
             req.files,
-            // a detached request belongs to a replay session that outlives the client socket:
-            // it reaches the child even when the downstream died during the load wait, the
-            // session buffer is the recipient and DELETE remains the stop
-            detached ? std::function<bool()>([]() { return false; }) : req.should_stop,
+            // a detached request belongs to a replay session
+            detached
+                ? std::function<bool()>([]() { return false; })
+                : req.should_stop,
             base_params.timeout_read,
             base_params.timeout_write
             );
+
+    proxy->cleanup = [this, name]() {
+        bool went_idle = false;
+        {
+            std::unique_lock<std::mutex> lk(mutex);
+            auto it = mapping.find(name);
+            if (it != mapping.end() && it->second.req_count > 0) {
+                it->second.req_count--;
+                went_idle = it->second.req_count == 0;
+            }
+        }
+        if (went_idle) {
+            sched->on_model_idle(name);
+        }
+    };
+
     return proxy;
 }
 
@@ -1568,7 +1862,7 @@ void server_models_routes::init_routes() {
             return error_res;
         }
         if (autoload) {
-            models.ensure_model_ready(name);
+            models.ensure_model_ready(name, req.should_stop);
         }
         return models.proxy_request(req, method, name, false);
     };
@@ -1588,7 +1882,9 @@ void server_models_routes::init_routes() {
         // this request instead of leaving an orphan generation
         std::string conv_id = server_stream_conv_id_from_headers(req.headers);
         uint64_t ticket = models.conv_models.remember(conv_id, name);
-        bool waited = autoload && models.ensure_model_ready(name);
+        // a dead socket must not cancel a session request, only a stop does (checked right below)
+        auto should_stop = ticket == 0 ? req.should_stop : nullptr;
+        bool waited = autoload && models.ensure_model_ready(name, should_stop);
         if (ticket != 0 && !models.conv_models.alive(conv_id, ticket)) {
             SRV_INF("request for conv_id=%s cancelled while model name=%s was loading\n",
                     conv_id.c_str(), name.c_str());
@@ -2064,7 +2360,7 @@ server_http_proxy::server_http_proxy(
     cli->set_write_timeout(timeout_read, 0); // reversed for cli (client) vs srv (server)
     cli->set_read_timeout(timeout_write, 0);
     this->status = 500; // to be overwritten upon response
-    this->cleanup = [pipe]() {
+    this->cleanup_pipes = [pipe]() {
         pipe->close_read();
         pipe->close_write();
     };
@@ -2079,9 +2375,8 @@ server_http_proxy::server_http_proxy(
         return has_next; // false if EOF or pipe broken
     };
 
-    // wire up the HTTP client
-    // note: do NOT capture `this` pointer, as it may be destroyed before the thread ends
-    httplib::ResponseHandler response_handler = [pipe, cli](const httplib::Response & response) {
+    // build the header message forwarded to the reader thread, stripping internal proxy headers
+    auto make_header_msg = [](const httplib::Response & response) {
         msg_t msg;
         msg.status = response.status;
         for (const auto & [key, value] : response.headers) {
@@ -2095,7 +2390,17 @@ server_http_proxy::server_http_proxy(
             }
             msg.headers[key] = value;
         }
-        return pipe->write(std::move(msg)); // send headers first
+        return msg;
+    };
+
+    // true once response_handler has already forwarded the headers
+    auto headers_sent = std::make_shared<std::atomic<bool>>(false);
+
+    // wire up the HTTP client
+    // note: do NOT capture `this` pointer, as it may be destroyed before the thread ends
+    httplib::ResponseHandler response_handler = [pipe, headers_sent, make_header_msg](const httplib::Response & response) {
+        headers_sent->store(true);
+        return pipe->write(make_header_msg(response)); // send headers first
     };
     httplib::ContentReceiverWithProgress content_receiver = [pipe](const char * data, size_t data_length, size_t, size_t) {
         // send data chunks
@@ -2169,13 +2474,16 @@ server_http_proxy::server_http_proxy(
 
     // start the proxy thread
     SRV_DBG("start proxy thread %s %s\n", req.method.c_str(), req.path.c_str());
-    this->thread = std::thread([cli, pipe, req]() {
+    this->thread = std::thread([cli, pipe, req, headers_sent, make_header_msg]() {
         auto result = cli->send(std::move(req));
         if (result.error() != httplib::Error::Success) {
             auto err_str = httplib::to_string(result.error());
             SRV_ERR("http client error: %s\n", err_str.c_str());
             pipe->write({{}, 500, "", ""}); // header
             pipe->write({{}, 0, "proxy error: " + err_str, ""}); // body
+        } else if (!headers_sent->load()) {
+            // httplib skips response_handler for bodyless statuses like 204, send headers here instead
+            pipe->write(make_header_msg(*result));
         }
         pipe->close_write(); // signal EOF to reader
         SRV_DBG("%s", "client request thread ended\n");

tools/server/server-models.h

@@ -84,7 +84,6 @@ struct server_model_meta {
     int exit_code = 0; // exit code of the model instance process (only valid if status == FAILED)
     int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdown
     mtmd_caps multimodal; // multimodal capabilities
-    // bool need_download = false; // whether the model needs to be downloaded before loading // TODO @ngxson: implement this
 
     bool is_ready() const {
         return status == SERVER_MODEL_STATUS_LOADED;
@@ -94,6 +93,10 @@ struct server_model_meta {
         return status == SERVER_MODEL_STATUS_LOADED || status == SERVER_MODEL_STATUS_LOADING || status == SERVER_MODEL_STATUS_SLEEPING;
     }
 
+    bool is_ready_or_sleep() const {
+        return status == SERVER_MODEL_STATUS_LOADED || status == SERVER_MODEL_STATUS_SLEEPING;
+    }
+
     bool is_failed() const {
         return status == SERVER_MODEL_STATUS_UNLOADED && exit_code != 0;
     }
@@ -103,16 +106,19 @@ struct server_model_meta {
 };
 
 struct server_models_routes;
-struct server_subproc; // defined in server-models.cpp
+struct server_subproc;   // defined in server-models.cpp
+struct server_lru_sched; // defined in server-models.cpp
 
 struct server_models {
     friend struct server_models_routes;
+    friend struct server_lru_sched;
 
 private:
     struct instance_t {
         std::shared_ptr<server_subproc> subproc; // shared between main thread and monitoring thread
         std::thread th;
         server_model_meta meta;
+        int req_count = 0; // number of active proxy requests
     };
 
     std::mutex mutex;
@@ -191,6 +197,12 @@ private:
     std::vector<std::string> base_env;
     common_preset base_preset; // base preset from llama-server CLI args
 
+    // queue of requests waiting for a models_max slot
+    std::unique_ptr<server_lru_sched> sched;
+
+    // if true, add some delay to simulate works (useful for testing)
+    bool debug_fake_timing = false;
+
     void update_meta(const std::string & name, const server_model_meta & meta);
 
     // unload least recently used models if the limit is reached
@@ -207,6 +219,7 @@ public:
     conv_model_tracker conv_models;
 
     server_models(const common_params & params, int argc, char ** argv);
+    ~server_models();
 
     server_response sse; // for real-time updates via SSE endpoint
 
@@ -263,7 +276,9 @@ public:
     // ensure the model is in ready state (thread-safe)
     // return false if model is ready
     // otherwise, load the model and blocking wait until it's ready, then return true (meta may need to be refreshed)
-    bool ensure_model_ready(const std::string & name);
+    // if models_max is reached, the request waits in a queue until a slot frees up
+    // throws if the load fails, or if should_stop fires while waiting
+    bool ensure_model_ready(const std::string & name, const std::function<bool()> & should_stop = nullptr);
 
     // proxy an HTTP request to the model instance
     server_http_res_ptr proxy_request(const server_http_req & req, const std::string & method, const std::string & name, bool update_last_used, bool detached = false);
@@ -343,7 +358,6 @@ struct server_models_routes {
  */
 struct server_http_proxy : server_http_res {
     std::function<void()> cleanup = nullptr;
-public:
     server_http_proxy(const std::string & method,
                       const std::string & scheme,
                       const std::string & host,
@@ -357,11 +371,15 @@ public:
                       int32_t timeout_write
                       );
     ~server_http_proxy() {
+        if (cleanup_pipes) {
+            cleanup_pipes();
+        }
         if (cleanup) {
             cleanup();
         }
     }
 private:
+    std::function<void()> cleanup_pipes = nullptr;
     std::thread thread;
     struct msg_t {
         std::map<std::string, std::string> headers;

tools/server/server-schema.cpp

@@ -124,8 +124,8 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
         ->set_desc("Dynamic temperature exponent, controls how entropy maps to temperature"));
 
     add((new field_num("repeat_last_n", params.sampling.penalty_last_n))
-        ->set_hard_limits(-1, INT32_MAX)
-        ->set_desc("Last n tokens to consider for penalizing repetition (0 = disabled, -1 = ctx-size)"));
+        ->set_hard_limits(0, INT32_MAX)
+        ->set_desc("Last n tokens to consider for penalizing repetition (0 = disabled)"));
 
     add((new field_num("repeat_penalty", params.sampling.penalty_repeat))
         ->set_desc("Control the repetition of token sequences in the generated text (1.0 = disabled)"));
@@ -151,8 +151,8 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
         ->set_desc("Tokens that extend repetition beyond this length receive exponentially increasing penalty: multiplier * base ^ (sequence_length - allowed_length)"));
 
     add((new field_num("dry_penalty_last_n", params.sampling.dry_penalty_last_n))
-        ->set_hard_limits(-1, INT32_MAX)
-        ->set_desc("How many tokens to scan for repetitions (0 = disabled, -1 = context size)"));
+        ->set_hard_limits(0, INT32_MAX)
+        ->set_desc("How many tokens to scan for repetitions (0 = disabled)"));
 
     add((new field_num("mirostat", params.sampling.mirostat))
         ->set_limits(0, 2)
@@ -515,12 +515,11 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
 task_params eval_llama_cmpl_schema(
                 const llama_vocab * vocab,
                 const common_params & params_base,
-                const int n_ctx_slot,
                 const std::vector<llama_logit_bias> & logit_bias_eog,
                 const json & data) {
     task_params params;
 
-    // Sampling parameter defaults are loaded from the global server context (but individual requests can still them)
+    // Sampling parameter defaults are loaded from the global server context (but individual requests can still override them)
     params.sampling      = params_base.sampling;
     params.speculative   = params_base.speculative;
     params.n_keep        = params_base.n_keep;
@@ -549,15 +548,6 @@ task_params eval_llama_cmpl_schema(
 
     // post-processing
     {
-        if (params.sampling.penalty_last_n == -1) {
-            // note: should be the slot's context and not the full context, but it's ok
-            params.sampling.penalty_last_n = n_ctx_slot;
-        }
-
-        if (params.sampling.dry_penalty_last_n == -1) {
-            params.sampling.dry_penalty_last_n = n_ctx_slot;
-        }
-
         // if "reasoning_format" is not provided, its handler will not be called, we will need to handle it here
         auto reasoning_format = params.chat_parser_params.reasoning_format;
         params.chat_parser_params.reasoning_in_content = params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);

tools/server/server-schema.h

@@ -98,7 +98,6 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(
 task_params eval_llama_cmpl_schema(
                     const llama_vocab * vocab,
                     const common_params & params_base,
-                    const int n_ctx_slot,
                     const std::vector<llama_logit_bias> & logit_bias_eog,
                     const json & data);
 

tools/server/server-task.cpp

@@ -1560,6 +1560,11 @@ json server_task_result_metrics::to_json() {
         { "n_decode_total",                  n_decode_total },
         { "n_busy_slots_total",              n_busy_slots_total },
 
+        { "n_draft_tokens_total",            n_draft_tokens_total },
+        { "n_draft_accepted_total",          n_draft_accepted_total },
+        { "n_draft_verif_steps_total",       n_draft_verif_steps_total },
+        { "n_accepted_per_pos_total",        n_accepted_per_pos_total },
+
         { "slots",                           slots_data },
     };
 }

tools/server/server-task.h

@@ -532,6 +532,11 @@ struct server_task_result_metrics : server_task_result {
     uint64_t n_decode_total     = 0;
     uint64_t n_busy_slots_total = 0;
 
+    uint64_t n_draft_tokens_total      = 0;
+    uint64_t n_draft_accepted_total    = 0;
+    uint64_t n_draft_verif_steps_total = 0;
+    std::vector<uint64_t> n_accepted_per_pos_total;
+
     // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
     // therefore, we use json to temporarily store the slot.to_json() result
     json slots_data = json::array();

tools/server/server-tools.cpp

@@ -11,18 +11,60 @@
 #include <atomic>
 #include <cstring>
 #include <cstdint>
+#include <cstdlib>
 #include <algorithm>
 #include <unordered_set>
+#include <tuple>
 #include <functional>
 #include <memory>
 #include <mutex>
 
+#if defined(_WIN32)
+#   ifndef NOMINMAX
+#       define NOMINMAX
+#   endif
+#   include <windows.h>
+#endif
+
 namespace fs = std::filesystem;
 
 //
 // internal helpers
 //
 
+// a child process writes in the OEM code page, so accented output would reach
+// the JSON layer as invalid bytes. run() spawns without a console, so the
+// console code page never applies
+static std::string console_output_to_utf8(const std::string & text) {
+#if defined(_WIN32)
+    // a chunk can end mid sequence, so the incomplete tail is dropped first
+    if (text.empty() || is_valid_utf8(text.substr(0, validate_utf8(text)))) {
+        // never decode twice a child that already emits UTF-8
+        return text;
+    }
+
+    const UINT cp = GetOEMCP();
+
+    // fail rather than emit replacement characters when the code page is wrong
+    const int wide_len = MultiByteToWideChar(cp, MB_ERR_INVALID_CHARS, text.data(), (int) text.size(), nullptr, 0);
+    if (wide_len <= 0) {
+        return text;
+    }
+    std::wstring wide(wide_len, L'\0');
+    MultiByteToWideChar(cp, MB_ERR_INVALID_CHARS, text.data(), (int) text.size(), wide.data(), wide_len);
+
+    const int utf8_len = WideCharToMultiByte(CP_UTF8, 0, wide.data(), wide_len, nullptr, 0, nullptr, nullptr);
+    if (utf8_len <= 0) {
+        return text;
+    }
+    std::string utf8(utf8_len, '\0');
+    WideCharToMultiByte(CP_UTF8, 0, wide.data(), wide_len, utf8.data(), utf8_len, nullptr, nullptr);
+    return utf8;
+#else
+    return text;
+#endif
+}
+
 json server_tool::to_json() const {
     return {
         {"display_name", display_name},
@@ -36,7 +78,56 @@ json server_tool::to_json() const {
 }
 
 static constexpr size_t SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT = 8 * 1024 * 1024; // 8 MB
-static constexpr int SERVER_TOOL_GIT_LS_FILES_TIMEOUT = 15; // seconds
+// budget for one listing call, shared by the git and walker paths
+static constexpr int SERVER_TOOL_LIST_ENTRIES_TIMEOUT = 15; // seconds
+
+// entry kinds a directory listing may return
+enum class list_kind {
+    files, // regular files only
+    dirs,  // directories only
+    all,   // both
+};
+
+// a narrow path uses the active code page on Windows, so every crossing between
+// a std::string (always UTF-8 here) and fs::path is converted explicitly
+static fs::path path_from_utf8(const std::string & s) {
+    return fs::u8path(s);
+}
+
+// '/' separators on every platform: Windows accepts them, the web UI needs them
+static std::string path_to_utf8(const fs::path & p) {
+    const auto s = p.generic_u8string();
+    return std::string(s.begin(), s.end());
+}
+
+// home directory, read once at first use (getenv is not thread safe against setenv)
+static const std::string & home_dir() {
+    static const std::string home = [] {
+#ifdef _WIN32
+        // the narrow getenv would return the profile path in the active code page
+        const wchar_t * w = _wgetenv(L"HOME");
+        if (w == nullptr) w = _wgetenv(L"USERPROFILE");
+        return w ? path_to_utf8(fs::path(w)) : std::string();
+#else
+        const char * h = getenv("HOME");
+        return h ? std::string(h) : std::string();
+#endif
+    }();
+    return home;
+}
+
+static std::string expand_home(const std::string & path) {
+    if (path.empty() || path[0] != '~') return path;
+    if (path.size() > 1 && path[1] != '/' && path[1] != '\\') return path;
+    const std::string & home = home_dir();
+    if (home.empty()) return path;
+    return home + path.substr(1);
+}
+
+// depth of a '/'-separated relative path: "a/b/c" is 3
+static int entry_depth(const std::string & rel) {
+    return 1 + (int) std::count(rel.begin(), rel.end(), '/');
+}
 
 class tools_io {
 public:
@@ -53,8 +144,20 @@ public:
     virtual bool file_size(const std::string & path, uintmax_t & out_size) const = 0;
     virtual bool read_file(const std::string & path, std::string & out) const = 0;
     virtual bool write_file(const std::string & path, const std::string & content) const = 0;
-    // paths relative to `base`, '/'-separated; sets `err` if `base` isn't a directory
-    virtual std::vector<std::string> list_files(const std::string & base, std::string & err) const = 0;
+    // resolve `path` against the IO's working directory; absolute paths are returned unchanged
+    virtual std::string resolve(const std::string & path) const = 0;
+    struct list_entry {
+        std::string rel; // '/'-separated, relative to `base`
+        bool is_dir = false;
+    };
+    struct list_result {
+        std::vector<list_entry> entries;
+        std::string err;        // set when `base` is not a directory
+        bool truncated = false; // set when the walk could not see everything
+    };
+    // entries relative to `base`, which must already be resolved (absolute)
+    // max_depth == 0 means unlimited, 1 means direct children of `base` only
+    virtual list_result list_entries(const std::string & base, int max_depth, list_kind kind) const = 0;
     // on_chunk, if set, is called with each chunk of output as it is read (before truncation cuts in);
     // returning false terminates the process early (e.g. the client disconnected)
     virtual exec_result run(
@@ -114,14 +217,14 @@ static tools_io::exec_result run_subprocess(
                 size_t len = strlen(buf);
                 if (output.size() + len <= max_output) {
                     output.append(buf, len);
-                    if (on_chunk && !on_chunk(std::string(buf, len))) {
+                    if (on_chunk && !on_chunk(console_output_to_utf8(std::string(buf, len)))) {
                         proc.terminate();
                         break;
                     }
                 } else {
                     size_t remaining = max_output - output.size();
                     output.append(buf, remaining);
-                    if (on_chunk && remaining > 0) on_chunk(std::string(buf, remaining));
+                    if (on_chunk && remaining > 0) on_chunk(console_output_to_utf8(std::string(buf, remaining)));
                     truncated = true;
                 }
             }
@@ -135,7 +238,7 @@ static tools_io::exec_result run_subprocess(
 
     res.exit_code = proc.join();
 
-    res.output    = output;
+    res.output    = console_output_to_utf8(output);
     res.timed_out = timed_out.load();
     if (truncated) {
         res.output += "\n[output truncated]";
@@ -148,24 +251,50 @@ public:
     // cwd, if non-empty, is used to resolve relative paths and as the working directory for run()
     explicit tools_io_basic(std::string cwd = "") : cwd(std::move(cwd)) {}
 
+    // expands a leading `~`, then resolves `path` against `cwd` (or the server
+    // working directory when `cwd` is unset); the result is always absolute
+    std::string resolve(const std::string & path) const override {
+        const std::string p = expand_home(path);
+
+        fs::path full = path_from_utf8(p);
+        if (!full.is_absolute()) {
+            if (cwd.empty()) {
+                std::error_code ec;
+                const fs::path cur = fs::current_path(ec);
+                if (ec) return p;
+                full = cur / full;
+            } else {
+                full = path_from_utf8(cwd) / full;
+            }
+        }
+
+        // drop "." and ".." so they never reach git or the client
+        full = full.lexically_normal();
+        // a trailing ".." normalizes to a path that ends with a separator
+        if (!full.has_filename() && full != full.root_path()) {
+            full = full.parent_path();
+        }
+        return path_to_utf8(full);
+    }
+
     bool is_directory(const std::string & path) const override {
         std::error_code ec;
-        return fs::is_directory(resolve(path), ec) && !ec;
+        return fs::is_directory(path_from_utf8(resolve(path)), ec) && !ec;
     }
 
     bool is_regular_file(const std::string & path) const override {
         std::error_code ec;
-        return fs::is_regular_file(resolve(path), ec) && !ec;
+        return fs::is_regular_file(path_from_utf8(resolve(path)), ec) && !ec;
     }
 
     bool file_size(const std::string & path, uintmax_t & out_size) const override {
         std::error_code ec;
-        out_size = fs::file_size(resolve(path), ec);
+        out_size = fs::file_size(path_from_utf8(resolve(path)), ec);
         return !ec;
     }
 
     bool read_file(const std::string & path, std::string & out) const override {
-        std::ifstream f(resolve(path), std::ios::binary);
+        std::ifstream f(path_from_utf8(resolve(path)), std::ios::binary);
         if (!f) return false;
         std::ostringstream ss;
         ss << f.rdbuf();
@@ -175,7 +304,7 @@ public:
 
     bool write_file(const std::string & path, const std::string & content) const override {
         std::error_code ec;
-        fs::path fpath(resolve(path));
+        fs::path fpath = path_from_utf8(resolve(path));
         if (fpath.has_parent_path()) {
             fs::create_directories(fpath.parent_path(), ec);
             if (ec) return false;
@@ -186,34 +315,41 @@ public:
         return (bool) f;
     }
 
-    std::vector<std::string> list_files(const std::string & base, std::string & err) const override {
-        err.clear();
-        std::string abs_base = resolve(base);
-        if (!is_directory(base)) {
-            err = "path does not exist or is not a directory: " + base;
-            return {};
+    list_result list_entries(const std::string & base, int max_depth, list_kind kind) const override {
+        list_result out;
+
+        std::error_code ec;
+        if (!fs::is_directory(base, ec) || ec) {
+            out.err = "path does not exist or is not a directory";
+            return out;
         }
 
-        auto res = run(
-            {"git", "-C", abs_base, "ls-files", "--cached", "--others", "--exclude-standard"},
-            SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, SERVER_TOOL_GIT_LS_FILES_TIMEOUT);
+        const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(SERVER_TOOL_LIST_ENTRIES_TIMEOUT);
 
-        if (res.exit_code == 0 && !res.timed_out) {
-            std::vector<std::string> result;
-            std::istringstream iss(res.output);
-            std::string line;
-            while (std::getline(iss, line)) {
-                if (!line.empty() && line.back() == '\r') line.pop_back();
-                if (line.empty()) continue;
-                std::replace(line.begin(), line.end(), '\\', '/');
-                if (is_regular_file((fs::path(base) / line).string())) {
-                    result.push_back(line);
+        // git ls-files cannot list directories; use the walker when they are requested
+        if (kind == list_kind::files) {
+            auto res = run(
+                {"git", "-C", base, "ls-files", "--cached", "--others", "--exclude-standard"},
+                SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, SERVER_TOOL_LIST_ENTRIES_TIMEOUT);
+
+            if (res.exit_code == 0 && !res.timed_out) {
+                std::istringstream iss(res.output);
+                std::string line;
+                while (std::getline(iss, line)) {
+                    if (!line.empty() && line.back() == '\r') line.pop_back();
+                    if (line.empty()) continue;
+                    std::replace(line.begin(), line.end(), '\\', '/');
+                    if (max_depth > 0 && entry_depth(line) > max_depth) continue;
+                    if (is_regular_file(path_to_utf8(path_from_utf8(base) / path_from_utf8(line)))) {
+                        out.entries.push_back({line, false});
+                    }
                 }
+                return out;
             }
-            return result;
         }
 
-        return list_files_fallback(abs_base);
+        out.entries = list_entries_fallback(base, max_depth, kind, deadline, out.truncated);
+        return out;
     }
 
     exec_result run(
@@ -227,12 +363,40 @@ public:
 private:
     std::string cwd;
 
-    // resolves `path` against `cwd` if `path` is relative and `cwd` is set; otherwise returns `path` unchanged
-    std::string resolve(const std::string & path) const {
-        if (cwd.empty() || fs::path(path).is_absolute()) {
-            return path;
+    // a link can point back to an ancestor and loop forever, so it is never walked
+    static bool is_link(const fs::directory_entry & entry) {
+        std::error_code ec;
+        if (entry.is_symlink(ec) || ec) {
+            return true;
         }
-        return (fs::path(cwd) / path).string();
+#if defined(_WIN32)
+        // a junction looks like a plain directory to std::filesystem, so read the reparse tag
+        WIN32_FIND_DATAW data;
+        const HANDLE h = FindFirstFileW(entry.path().c_str(), &data);
+        if (h == INVALID_HANDLE_VALUE) {
+            return false;
+        }
+        FindClose(h);
+        if ((data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0) {
+            return false;
+        }
+        // other reparse points (cloud placeholder, dedup stub) are real directories
+        return data.dwReserved0 == IO_REPARSE_TAG_SYMLINK || data.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT;
+#else
+        return false;
+#endif
+    }
+
+    // NTFS is case insensitive, so Build and build are the same directory
+    static std::string get_effective_name(const std::string & fname) {
+#if defined(_WIN32)
+        std::string lowered = fname;
+        std::transform(lowered.begin(), lowered.end(), lowered.begin(),
+                       [](unsigned char c) { return (char) std::tolower(c); });
+        return lowered;
+#else
+        return fname;
+#endif
     }
 
     static const std::unordered_set<std::string> & junk_dir_names() {
@@ -243,28 +407,57 @@ private:
         return names;
     }
 
-    std::vector<std::string> list_files_fallback(const std::string & base) const {
-        std::vector<std::string> result;
-        std::error_code ec;
+    std::vector<list_entry> list_entries_fallback(const std::string & base, int max_depth, list_kind kind,
+                                                  std::chrono::steady_clock::time_point deadline, bool & truncated) const {
+        std::vector<list_entry> result;
 
-        std::vector<std::pair<fs::path, fs::path>> stack;
-        stack.emplace_back(fs::path(base), fs::path());
+        std::vector<std::tuple<fs::path, fs::path, int>> stack;
+        stack.emplace_back(path_from_utf8(base), fs::path(), 0);
 
         while (!stack.empty()) {
-            auto [dir, rel_dir] = stack.back();
+            if (std::chrono::steady_clock::now() >= deadline) {
+                truncated = true;
+                return result;
+            }
+
+            auto [dir, rel_dir, depth] = std::move(stack.back());
             stack.pop_back();
 
-            for (const auto & entry : fs::directory_iterator(dir, fs::directory_options::skip_permission_denied, ec)) {
-                if (ec) break;
-                std::string fname = entry.path().filename().string();
+            std::error_code ec;
+            // step the iterator by hand: the throwing increment escapes on a directory that goes away
+            fs::directory_iterator it(dir, fs::directory_options::skip_permission_denied, ec);
+            // permission errors are skipped above, so this is a subtree the caller never sees
+            if (ec) {
+                truncated = true;
+                continue;
+            }
+            for (const fs::directory_iterator end; it != end; it.increment(ec)) {
+                if (ec) {
+                    truncated = true;
+                    break;
+                }
+                if (std::chrono::steady_clock::now() >= deadline) {
+                    truncated = true;
+                    return result;
+                }
+                const fs::directory_entry & entry = *it;
+                const fs::path fname = entry.path().filename();
                 std::error_code tec;
-                if (entry.is_directory(tec)) {
-                    if (junk_dir_names().count(fname) > 0) continue;
-                    stack.emplace_back(entry.path(), rel_dir / fname);
+                const bool is_dir = entry.is_directory(tec);
+                if (tec) continue;
+                if (is_dir) {
+                    if (kind == list_kind::dirs || kind == list_kind::all) {
+                        result.push_back({path_to_utf8(rel_dir / fname), true});
+                    }
+                    // junk directories stay selectable but are never walked: they can be enormous
+                    if (junk_dir_names().count(get_effective_name(path_to_utf8(fname))) > 0) continue;
+                    if (!is_link(entry) && (max_depth == 0 || depth + 1 < max_depth)) {
+                        stack.emplace_back(entry.path(), rel_dir / fname, depth + 1);
+                    }
                 } else if (entry.is_regular_file(tec)) {
-                    std::string rel = (rel_dir / fname).string();
-                    std::replace(rel.begin(), rel.end(), '\\', '/');
-                    result.push_back(rel);
+                    if (kind == list_kind::files || kind == list_kind::all) {
+                        result.push_back({path_to_utf8(rel_dir / fname), false});
+                    }
                 }
             }
         }
@@ -347,35 +540,42 @@ public:
         return ok;
     }
 
-    std::vector<std::string> list_files(const std::string & base, std::string & err) const override {
-        err.clear();
-        std::string abs_base = resolve(base);
+    list_result list_entries(const std::string & base, int max_depth, list_kind kind) const override {
+        list_result out;
+
+        const std::string abs_base = resolve(base);
         if (!is_directory(base)) {
-            err = "path does not exist or is not a directory: " + base;
-            return {};
+            out.err = "path does not exist or is not a directory";
+            return out;
         }
 
-        auto res = exec(
-            {"sh", "-c", "cd \"$1\" && git ls-files --cached --others --exclude-standard", "_", abs_base},
-            SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, true);
+        // git ls-files cannot list directories; use the walker when they are requested
+        if (kind == list_kind::files) {
+            auto res = exec(
+                {"sh", "-c", "cd \"$1\" && git ls-files --cached --others --exclude-standard", "_", abs_base},
+                SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, true);
 
-        if (res.exit_code == 0 && !res.timed_out) {
-            return split_lines(res.output, /*strip_dot_slash=*/false);
+            if (res.exit_code == 0 && !res.timed_out) {
+                for (const auto & rel : split_lines(res.output, /*strip_dot_slash=*/false)) {
+                    if (max_depth > 0 && entry_depth(rel) > max_depth) continue;
+                    out.entries.push_back({rel, false});
+                }
+                return out;
+            }
         }
 
-        static const char * prune_names[] = {
-            ".git", ".svn", ".hg", "node_modules", "__pycache__",
-            ".venv", "venv", "dist", "build", "target", ".cache", ".idea", ".vscode",
-        };
-        std::string prune_expr;
-        for (const char * n : prune_names) {
-            if (!prune_expr.empty()) prune_expr += " -o ";
-            prune_expr += std::string("-name ") + n;
+        if (kind == list_kind::dirs || kind == list_kind::all) {
+            for (auto & rel : find_entries(abs_base, max_depth, /*dirs=*/true, out.truncated)) {
+                out.entries.push_back({std::move(rel), true});
+            }
+        }
+        if (kind == list_kind::files || kind == list_kind::all) {
+            for (auto & rel : find_entries(abs_base, max_depth, /*dirs=*/false, out.truncated)) {
+                out.entries.push_back({std::move(rel), false});
+            }
         }
-        std::string find_cmd = "cd \"$1\" && find . \\( " + prune_expr + " \\) -prune -o -type f -print";
-        auto find_res = exec({"sh", "-c", find_cmd, "_", abs_base}, SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, true);
 
-        return split_lines(find_res.output, /*strip_dot_slash=*/true);
+        return out;
     }
 
     // wraps the command with an in-container `timeout`, since killing the local `docker exec` client
@@ -440,6 +640,32 @@ private:
         }
         return result;
     }
+
+    // one `find` pass in the container. junk directories stay selectable but are never descended into,
+    // and -mindepth/-maxdepth keep a busybox image working as well as a GNU one
+    std::vector<std::string> find_entries(const std::string & abs_base, int max_depth, bool dirs, bool & truncated) const {
+        static const char * junk_names[] = {
+            ".git", ".svn", ".hg", "node_modules", "__pycache__",
+            ".venv", "venv", "dist", "build", "target", ".cache", ".idea", ".vscode",
+        };
+
+        std::string prune_expr;
+        for (const char * n : junk_names) {
+            if (!prune_expr.empty()) prune_expr += " -o ";
+            prune_expr += std::string("-name ") + n;
+        }
+
+        std::string cmd = "cd \"$1\" && find . -mindepth 1";
+        if (max_depth > 0) {
+            cmd += " -maxdepth " + std::to_string(max_depth);
+        }
+        cmd += " \\( " + prune_expr + " \\) -prune";
+        cmd += dirs ? " -print -o -type d -print" : " -o -type f -print";
+
+        auto res = exec({"sh", "-c", cmd, "_", abs_base}, SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, true);
+        truncated = truncated || res.timed_out;
+        return split_lines(res.output, /*strip_dot_slash=*/true);
+    }
 };
 
 static std::unique_ptr<tools_io> make_tools_io(const json & params) {
@@ -453,7 +679,7 @@ static std::unique_ptr<tools_io> make_tools_io(const json & params) {
 // no '/' in pattern -> match basename at any depth; else match full relative path
 static bool path_glob_match(const std::string & pattern, const std::string & rel_path) {
     if (pattern.find('/') == std::string::npos) {
-        return glob_match(pattern, fs::path(rel_path).filename().string());
+        return glob_match(pattern, path_to_utf8(path_from_utf8(rel_path).filename()));
     }
     if (pattern == "**" || pattern.rfind("**/", 0) == 0 || pattern.rfind('/', 0) == 0) {
         return glob_match(pattern, rel_path);
@@ -550,7 +776,10 @@ struct server_tool_read_file : server_tool {
 // file_glob_search: find files matching a glob pattern under a base directory
 //
 
-static constexpr size_t SERVER_TOOL_FILE_SEARCH_MAX_RESULTS = 100;
+static constexpr int SERVER_TOOL_FILE_SEARCH_MAX_RESULTS = 100;
+static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_FILE = "file";
+static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_DIR  = "dir";
+static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_ALL  = "all";
 
 struct server_tool_file_glob_search : server_tool {
     server_tool_file_glob_search() {
@@ -570,13 +799,18 @@ struct server_tool_file_glob_search : server_tool {
                     "and common junk directories (.git, node_modules, build, dist, etc.) otherwise. "
                     "A pattern with no '/' (e.g. \"*.cpp\") matches the file's basename at any depth. "
                     "A pattern containing '/' matches the full relative path; unless already anchored with "
-                    "\"**/\" or a leading '/', it is automatically prefixed with \"**/\"."},
+                    "\"**/\" or a leading '/', it is automatically prefixed with \"**/\". "
+                    "Use type=\"dir\" or \"all\" to also list directories; directory entries are suffixed with '/' in the output. "
+                    "Note: directory listings do not apply .gitignore filtering."},
                 {"parameters", {
                     {"type", "object"},
                     {"properties", {
-                        {"path",    {{"type", "string"}, {"description", "Base directory to search in"}}},
-                        {"include", {{"type", "string"}, {"description", "Glob pattern for files to include (e.g. \"*.cpp\" or \"src/**/*.cpp\"). Default: **"}}},
-                        {"exclude", {{"type", "string"}, {"description", "Glob pattern for files to exclude"}}},
+                        {"path",      {{"type", "string"},  {"description", "Base directory to search in"}}},
+                        {"include",   {{"type", "string"},  {"description", "Glob pattern for files to include (e.g. \"*.cpp\" or \"src/**/*.cpp\"). Default: **"}}},
+                        {"exclude",   {{"type", "string"},  {"description", "Glob pattern for files to exclude"}}},
+                        {"type",      {{"type", "string"},  {"description", "Entry type to return: \"file\" (default), \"dir\" or \"all\""}}},
+                        {"max_depth", {{"type", "integer"}, {"description", "Maximum depth to descend into subdirectories (default: 0 = unlimited; 1 = direct children only)"}}},
+                        {"limit",     {{"type", "integer"}, {"description", string_format("Maximum number of results to return, capped at %d (default %d)", SERVER_TOOL_FILE_SEARCH_MAX_RESULTS, SERVER_TOOL_FILE_SEARCH_MAX_RESULTS)}}},
                     }},
                     {"required", json::array({"path"})},
                 }},
@@ -585,30 +819,55 @@ struct server_tool_file_glob_search : server_tool {
     }
 
     json invoke(json params, server_tool::stream *) const override {
-        std::string base    = params.at("path").get<std::string>();
-        std::string include = json_value(params, "include", std::string("**"));
-        std::string exclude = json_value(params, "exclude", std::string(""));
-
         auto io = make_tools_io(params);
-        std::string err;
-        auto files = io->list_files(base, err);
-        if (!err.empty()) {
-            return {{"error", err}};
+
+        const std::string path = params.at("path").get<std::string>();
+
+        std::string base      = io->resolve(path);
+        std::string include   = json_value(params, "include", std::string("**"));
+        std::string exclude   = json_value(params, "exclude", std::string(""));
+        std::string type      = json_value(params, "type",    std::string("file"));
+        int         max_depth = std::max(0, json_value(params, "max_depth", 0));
+        const int   limit_req = json_value(params, "limit", SERVER_TOOL_FILE_SEARCH_MAX_RESULTS);
+        if (limit_req < 1) {
+            return {{"error", "invalid limit: " + std::to_string(limit_req) + " (expected 1 or more)"}};
+        }
+        const int   limit     = std::min(limit_req, SERVER_TOOL_FILE_SEARCH_MAX_RESULTS);
+
+        list_kind kind;
+        if (type == SERVER_TOOL_FILE_SEARCH_TYPE_FILE) {
+            kind = list_kind::files;
+        } else if (type == SERVER_TOOL_FILE_SEARCH_TYPE_DIR) {
+            kind = list_kind::dirs;
+        } else if (type == SERVER_TOOL_FILE_SEARCH_TYPE_ALL) {
+            kind = list_kind::all;
+        } else {
+            return {{"error", "invalid type: " + type + " (expected \"file\", \"dir\" or \"all\")"}};
         }
 
-        std::vector<std::string> matches;
-        for (const auto & rel : files) {
-            if (!path_glob_match(include, rel)) continue;
-            if (!exclude.empty() && path_glob_match(exclude, rel)) continue;
-            matches.push_back(rel);
+        const auto listing = io->list_entries(base, max_depth, kind);
+        if (!listing.err.empty()) {
+            return {{"error", listing.err + ": " + path}};
+        }
+
+        std::vector<tools_io::list_entry> matches;
+        for (const auto & entry : listing.entries) {
+            if (!path_glob_match(include, entry.rel)) continue;
+            if (!exclude.empty() && path_glob_match(exclude, entry.rel)) continue;
+            matches.push_back(entry);
         }
 
         size_t total = matches.size();
-        size_t shown = std::min(total, SERVER_TOOL_FILE_SEARCH_MAX_RESULTS);
+        size_t shown = std::min(total, (size_t) limit);
 
         std::ostringstream output_text;
+        json entries_json = json::array();
         for (size_t i = 0; i < shown; i++) {
-            output_text << matches[i] << "\n";
+            output_text << matches[i].rel << (matches[i].is_dir ? "/" : "") << "\n";
+            entries_json.push_back({
+                {"path", matches[i].rel},
+                {"type", matches[i].is_dir ? "dir" : "file"},
+            });
         }
 
         output_text << "\n---\nTotal matches: " << total << "\n";
@@ -617,8 +876,16 @@ struct server_tool_file_glob_search : server_tool {
                 "[%zu results limit reached (%zu total matches). Refine the glob pattern to narrow the search.]\n",
                 shown, total);
         }
+        if (listing.truncated) {
+            output_text << "[results truncated: time budget or unreadable directory]\n";
+        }
 
-        return {{"plain_text_response", output_text.str()}};
+        // `base` is always absolute (resolve falls back to the server cwd), so
+        // API clients (e.g. the web UI picker) can join the relative entries
+        // into absolute paths. `plain_text_response` is what the model sees;
+        // `entries` is the same data as structured JSON for the UI picker,
+        // which reads `entries`/`base` instead of re-parsing the text.
+        return {{"plain_text_response", output_text.str()}, {"entries", entries_json}, {"base", base}};
     }
 };
 
@@ -701,18 +968,18 @@ struct server_tool_grep_search : server_tool {
         // collect (absolute_path, display_path) pairs to search
         std::vector<std::pair<std::string, std::string>> files;
 
-        if (io->is_regular_file(path)) {
-            files.emplace_back(path, path);
-        } else if (io->is_directory(path)) {
-            std::string err;
-            auto candidates = io->list_files(path, err);
-            if (!err.empty()) {
-                return {{"error", err}};
+        const std::string abs_path = io->resolve(path);
+        if (io->is_regular_file(abs_path)) {
+            files.emplace_back(abs_path, path);
+        } else if (io->is_directory(abs_path)) {
+            const auto listing = io->list_entries(abs_path, 0, list_kind::files);
+            if (!listing.err.empty()) {
+                return {{"error", listing.err + ": " + path}};
             }
-            for (const auto & rel : candidates) {
-                if (!path_glob_match(include, rel)) continue;
-                if (!exclude.empty() && path_glob_match(exclude, rel)) continue;
-                files.emplace_back((fs::path(path) / rel).string(), rel);
+            for (const auto & entry : listing.entries) {
+                if (!path_glob_match(include, entry.rel)) continue;
+                if (!exclude.empty() && path_glob_match(exclude, entry.rel)) continue;
+                files.emplace_back(path_to_utf8(path_from_utf8(abs_path) / path_from_utf8(entry.rel)), entry.rel);
             }
         } else {
             return {{"error", "path does not exist: " + path}};
@@ -1285,6 +1552,9 @@ struct server_tool_get_datetime : server_tool {
 // get_info: returns runtime info (OS name/version and cwd)
 //
 
+static constexpr size_t SERVER_TOOL_GET_INFO_MAX_OUTPUT = 4096;
+static constexpr int    SERVER_TOOL_GET_INFO_TIMEOUT    = 5; // seconds
+
 struct server_tool_get_info : server_tool {
     server_tool_get_info() {
         name = "get_info";
@@ -1318,13 +1588,13 @@ struct server_tool_get_info : server_tool {
         std::vector<std::string> args = {"uname", "-a"};
 #endif
 
-        auto res = io->run(args, 4096, 5);
+        auto res = io->run(args, SERVER_TOOL_GET_INFO_MAX_OUTPUT, SERVER_TOOL_GET_INFO_TIMEOUT);
         std::string os_info = res.exit_code == 0 && !res.timed_out ? string_strip(res.output) : "unknown";
 
         std::string cwd = json_value(params, "cwd", std::string());
         if (cwd.empty()) {
             std::error_code ec;
-            cwd = fs::current_path(ec).string();
+            cwd = path_to_utf8(fs::current_path(ec));
         }
 
         return {

tools/server/tests/conftest.py

@@ -15,7 +15,7 @@ def stop_server_after_each_test():
         server.stop()
 
 
-@pytest.fixture(scope="module", autouse=True)
-def do_something():
+@pytest.fixture(scope="session", autouse=True)
+def load_server_presets():
     # this will be run once per test session, before any tests
     ServerPreset.load_all()

tools/server/tests/tests.sh

@@ -14,10 +14,10 @@ fi
 if [ $# -lt 1 ]
 then
     if [[ "${SLOW_TESTS:-0}" == 1 ]]; then
-        pytest -v -x
+        pytest --durations=30 -v -x
     else
-        pytest -v -x -m "not slow"
+        pytest --durations=30 -v -x -m "not slow"
     fi
 else
-    pytest "$@"
+    pytest --durations=30 "$@"
 fi

tools/server/tests/unit/test_proxy.py

@@ -1,5 +1,7 @@
 import pytest
 from utils import *
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 server = ServerPreset.tinyllama2()
 
@@ -39,3 +41,31 @@ def test_mcp_proxy_custom_port():
     res = server.make_request("GET", f"/cors-proxy?url=http://{server.server_host}:{server.server_port}/models")
     assert res.status_code == 200
     assert "data" in res.body
+
+
+def test_mcp_proxy_no_content():
+    # note: see issue #26598
+    class NoContentHandler(BaseHTTPRequestHandler):
+        def do_POST(self):
+            self.send_response(204)
+            self.end_headers()
+
+        def log_message(self, format, *args):
+            pass
+
+    target = ThreadingHTTPServer(("127.0.0.1", 0), NoContentHandler)
+    target_thread = threading.Thread(target=target.serve_forever, daemon=True)
+    target_thread.start()
+
+    try:
+        global server
+        server.ui_mcp_proxy = True
+        server.start()
+
+        res = server.make_request("POST", f"/cors-proxy?url=http://127.0.0.1:{target.server_port}/", data={})
+        assert res.status_code == 204
+        assert res.body in (None, b"", "")
+    finally:
+        target.shutdown()
+        target.server_close()
+

tools/server/tests/unit/test_router.py

@@ -85,7 +85,7 @@ def _wait_for_model_status(model_id: str, desired: set[str], timeout: int = 60)
         last_status = _get_model_status(model_id)
         if last_status in desired:
             return last_status
-        time.sleep(1)
+        time.sleep(0.01)
     raise AssertionError(
         f"Timed out waiting for {model_id} to reach {desired}, last status: {last_status}"
     )
@@ -145,6 +145,156 @@ def test_router_models_max_evicts_lru():
     assert _get_model_status(first) == "unloaded"
 
 
+# server_lru_sched tests (relying on LLAMA_SERVER_DEBUG_FAKE_TIMING)
+
+MODEL_A = "ggml-org/tinygemma3-GGUF:Q8_0"
+MODEL_B = "ggml-org/test-model-stories260K:F32"
+MODEL_C = "ggml-org/test-model-stories260K-infill:F32"
+
+
+def _tokenize(model_id: str, timeout: float | None = DEFAULT_REQUEST_TIMEOUT) -> ServerResponse:
+    return server.make_request(
+        "POST", "/tokenize", data={"model": model_id, "content": "hello world"}, timeout=timeout
+    )
+
+
+class _Bg:
+    """runs one request in a thread, keeps its result, error and finish time"""
+
+    def __init__(self, fn):
+        self.result = None
+        self.error: Exception | None = None
+        self.done_at: float = 0.0
+        self._thread = threading.Thread(target=self._run, args=(fn,), daemon=True)
+
+    def _run(self, fn):
+        try:
+            self.result = fn()
+        except Exception as e:
+            self.error = e
+        self.done_at = time.time()
+
+    def start(self):
+        self._thread.start()
+        return self
+
+    def join(self, timeout: int = 180):
+        self._thread.join(timeout)
+        assert not self._thread.is_alive(), "background request did not finish in time"
+        return self
+
+    def assert_ok(self, what: str):
+        assert self.error is None, f"{what} raised {self.error!r}"
+        assert self.result is not None and self.result.status_code == 200, \
+            f"{what} failed: {self.result.status_code if self.result else None} {self.result.body if self.result else None}"
+
+
+def test_router_queue_does_not_evict_busy_model():
+    """a request that finds no free slot waits, and the model serving a request survives it"""
+    global server
+    server.models_max = 1
+    server.start()
+
+    _load_model_and_wait(MODEL_A, timeout=120)
+
+    busy = _Bg(lambda: _tokenize(MODEL_A)).start()
+    time.sleep(0.5)  # let the request reach the child and take the only slot
+
+    # no slot free and MODEL_A is busy, so this queues instead of evicting mid-request
+    queued = _Bg(lambda: _tokenize(MODEL_B)).start()
+
+    busy.join()
+    queued.join()
+
+    # had MODEL_A been evicted while serving, its own request would have died
+    busy.assert_ok("request against the busy model")
+    queued.assert_ok("queued request")
+
+    _wait_for_model_status(MODEL_B, {"loaded"}, timeout=120)
+    assert _get_model_status(MODEL_A) == "unloaded"
+
+
+def test_router_queue_coalesces_requests_for_same_model():
+    """many requests for one missing model share a slot, so only one model is given up"""
+    global server
+    server.models_max = 2
+    server.start()
+
+    _load_model_and_wait(MODEL_A, timeout=120)
+    _load_model_and_wait(MODEL_B, timeout=120)
+
+    # keep MODEL_A busy so MODEL_B is the only model that can be given up
+    busy = _Bg(lambda: _tokenize(MODEL_A)).start()
+    time.sleep(0.5)
+
+    waiters = [_Bg(lambda: _tokenize(MODEL_C)).start() for _ in range(3)]
+
+    busy.join()
+    for w in waiters:
+        w.join()
+
+    busy.assert_ok("request against the busy model")
+    for i, w in enumerate(waiters):
+        w.assert_ok(f"queued request {i}")
+
+    _wait_for_model_status(MODEL_C, {"loaded"}, timeout=120)
+    # one entry for 3 requests means one eviction: MODEL_B goes, MODEL_A is left alone.
+    # without coalescing the leftover entries still ask for a slot,
+    # and MODEL_A is taken too as soon as it goes idle
+    assert _get_model_status(MODEL_A) == "loaded"
+    assert _get_model_status(MODEL_B) == "unloaded"
+
+
+def test_router_queue_client_disconnect_keeps_model():
+    """a client that leaves while queued must not cost a running model its slot"""
+    global server
+    server.models_max = 1
+    server.start()
+
+    _load_model_and_wait(MODEL_A, timeout=120)
+
+    busy = _Bg(lambda: _tokenize(MODEL_A)).start()
+    time.sleep(0.5)
+
+    # queues behind MODEL_A, then gives up long before MODEL_A goes idle
+    with pytest.raises(requests.exceptions.RequestException):
+        _tokenize(MODEL_B, timeout=1)
+
+    busy.join()
+    busy.assert_ok("request against the busy model")
+
+    # nobody is waiting anymore, so MODEL_A keeps its slot
+    time.sleep(3)
+    assert _get_model_status(MODEL_A) == "loaded"
+    assert _get_model_status(MODEL_B) == "unloaded"
+
+
+def test_router_queue_is_fifo():
+    """the queue is served in arrival order"""
+    global server
+    server.models_max = 1
+    server.start()
+
+    _load_model_and_wait(MODEL_A, timeout=120)
+
+    busy = _Bg(lambda: _tokenize(MODEL_A)).start()
+    time.sleep(0.5)
+
+    first = _Bg(lambda: _tokenize(MODEL_B)).start()
+    time.sleep(1)  # keep the arrival order unambiguous
+    second = _Bg(lambda: _tokenize(MODEL_C)).start()
+
+    busy.join()
+    first.join()
+    second.join()
+
+    busy.assert_ok("request against the busy model")
+    first.assert_ok("first queued request")
+    second.assert_ok("second queued request")
+
+    assert first.done_at < second.done_at, "queue was not served in arrival order"
+
+
 def test_router_no_models_autoload():
     global server
     server.no_models_autoload = True
@@ -310,7 +460,7 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: i
     while time.time() < deadline:
         if any(e.get("event") == event_type and e.get("model") == model for e in collected):
             return True
-        time.sleep(0.5)
+        time.sleep(0.01)
     return False
 
 

tools/server/tests/unit/test_tools_builtin.py

@@ -243,3 +243,122 @@ def test_tools_builtin_edit_file_rejects_overlapping_edits():
     finally:
         if os.path.exists(log_path):
             os.remove(log_path)
+
+
+def test_tools_builtin_file_glob_search_type_dir(tmp_path):
+    global server
+    server.start()
+
+    (tmp_path / "project-alpha" / "src").mkdir(parents=True)
+    (tmp_path / "project-alpha" / "README.md").write_text("alpha")
+    (tmp_path / "project-alpha" / "src" / "main.cpp").write_text("int main() {}")
+    (tmp_path / "project-beta").mkdir()
+    (tmp_path / "project-beta" / "notes.txt").write_text("beta")
+
+    res = call_tool("file_glob_search", {"path": str(tmp_path), "type": "dir"})
+    text = res["plain_text_response"]
+    assert "project-alpha/" in text
+    assert "project-beta/" in text
+    assert "project-alpha/src/" in text
+    assert "README.md" not in text
+    types = {e["path"]: e["type"] for e in res["entries"]}
+    assert types["project-alpha"] == "dir"
+    assert types["project-alpha/src"] == "dir"
+
+    res_all = call_tool("file_glob_search", {"path": str(tmp_path), "type": "all", "include": "*proj*"})
+    paths = [e["path"] for e in res_all["entries"]]
+    assert "project-alpha" in paths
+    assert "project-beta" in paths
+
+
+def test_tools_builtin_file_glob_search_max_depth_and_limit(tmp_path):
+    global server
+    server.start()
+
+    (tmp_path / "a" / "b" / "c").mkdir(parents=True)
+    (tmp_path / "top.txt").write_text("top")
+    (tmp_path / "a" / "mid.txt").write_text("mid")
+    (tmp_path / "a" / "b" / "deep.txt").write_text("deep")
+
+    res = call_tool("file_glob_search", {"path": str(tmp_path), "max_depth": 1})
+    assert "top.txt" in res["plain_text_response"]
+    assert "mid.txt" not in res["plain_text_response"]
+
+    res = call_tool("file_glob_search", {"path": str(tmp_path), "max_depth": 2})
+    assert "mid.txt" in res["plain_text_response"]
+    assert "deep.txt" not in res["plain_text_response"]
+
+    res = call_tool("file_glob_search", {"path": str(tmp_path), "limit": 1})
+    assert len(res["entries"]) == 1
+    assert "Total matches: 3" in res["plain_text_response"]
+
+
+def test_tools_builtin_file_glob_search_junk_dirs(tmp_path):
+    global server
+    server.start()
+
+    (tmp_path / "build" / "nested").mkdir(parents=True)
+    (tmp_path / "build" / "artifact.txt").write_text("built")
+    (tmp_path / "src").mkdir()
+    (tmp_path / "src" / "main.cpp").write_text("int main() {}")
+
+    # a junk directory stays selectable as a working directory
+    res = call_tool("file_glob_search", {"path": str(tmp_path), "type": "dir", "max_depth": 1})
+    assert "build" in [e["path"] for e in res["entries"]]
+
+    # but it is never walked, so nothing inside it shows up
+    res = call_tool("file_glob_search", {"path": str(tmp_path), "type": "all"})
+    paths = [e["path"] for e in res["entries"]]
+    assert "src/main.cpp" in paths
+    assert "build/artifact.txt" not in paths
+    assert "build/nested" not in paths
+
+
+def test_tools_builtin_file_glob_search_rejects_invalid_type(tmp_path):
+    global server
+    server.start()
+
+    err = call_tool_expect_error("file_glob_search", {"path": str(tmp_path), "type": "bogus"})
+    assert "invalid type" in err
+
+
+def test_tools_builtin_cwd_header_overrides_model_param(tmp_path):
+    global server
+    server.start()
+
+    workdir = tmp_path / "workdir"
+    workdir.mkdir()
+    (workdir / "marker.txt").write_text("marker")
+
+    # a model-provided "cwd" in the params is overridden by the x-tool-cwd header
+    res = call_tool("read_file", {"path": "marker.txt", "cwd": "/definitely/not/a/real/path"},
+                    headers={"x-tool-cwd": str(workdir)})
+    assert "marker" in res["plain_text_response"]
+
+
+def test_tools_builtin_cwd_relative_paths(tmp_path):
+    global server
+    server.start()
+
+    workdir = tmp_path / "workdir"
+    workdir.mkdir()
+    (workdir / "rel.txt").write_text("relative-content")
+
+    headers = {"x-tool-cwd": str(workdir)}
+
+    # relative paths in file tools resolve against the header cwd
+    res = call_tool("read_file", {"path": "rel.txt"}, headers=headers)
+    assert "relative-content" in res["plain_text_response"]
+
+    res = call_tool("write_file", {"path": "sub/out.txt", "content": "written"}, headers=headers)
+    assert (workdir / "sub" / "out.txt").read_text() == "written"
+
+    res = call_tool("file_glob_search", {"path": ".", "include": "*.txt"}, headers=headers)
+    assert "rel.txt" in res["plain_text_response"]
+
+    # absolute paths are unaffected by the cwd
+    other = tmp_path / "other"
+    other.mkdir()
+    (other / "abs.txt").write_text("absolute-content")
+    res = call_tool("read_file", {"path": str(other / "abs.txt")}, headers=headers)
+    assert "absolute-content" in res["plain_text_response"]

tools/server/tests/utils.py

@@ -133,7 +133,10 @@ class ServerProcess:
         self.external_server = "DEBUG_EXTERNAL" in os.environ
 
     def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None:
-        env = {**os.environ}
+        env = {
+            **os.environ,
+            "LLAMA_SERVER_DEBUG_FAKE_TIMING": "1",
+        }
         if "LLAMA_CACHE" not in os.environ:
             env["LLAMA_CACHE"] = "tmp"
         if self.external_server:
@@ -306,6 +309,7 @@ class ServerProcess:
 
         # wait for server to start
         start_time = time.time()
+        last_print_time = start_time
         while time.time() - start_time < timeout_seconds:
             try:
                 response = self.make_request("GET", "/health", headers={
@@ -320,8 +324,10 @@ class ServerProcess:
             if self.process.poll() is not None:
                 raise RuntimeError(f"Server process died with return code {self.process.returncode}")
 
-            print(f"Waiting for server to start...")
-            time.sleep(0.5)
+            if time.time() - last_print_time >= 1.0:
+                print(f"Waiting for server to start...")
+                last_print_time = time.time()
+            time.sleep(0.01)
         raise TimeoutError(f"Server did not start within {timeout_seconds} seconds")
 
     def stop(self) -> None: