Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-20 01:31:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '873e5d8e39feb34a376e0efd01bf3f665dfffeb5' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/build-cmake-pkg.yml
#	.github/workflows/build-cpu.yml
#	.github/workflows/make-release.yml
#	.github/workflows/release.yml
#	.pi/gg/SYSTEM.md
#	CMakeLists.txt
#	cmake/arm64-windows-llvm.cmake
#	docs/backend/ET.md
#	docs/build.md
#	docs/development/HOWTO-add-model.md
#	ggml/CMakeLists.txt
#	ggml/src/CMakeLists.txt
#	ggml/src/ggml-cpu/CMakeLists.txt
#	ggml/src/ggml-cpu/kleidiai/kernels.cpp
#	ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp/rope-ops.c
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl
#	ggml/src/ggml-opencl/kernels/rope.cl
#	ggml/src/ggml-sycl/dmmv.cpp
#	ggml/src/ggml-sycl/dpct/helper.hpp
#	ggml/src/ggml-sycl/element_wise.cpp
#	ggml/src/ggml-sycl/esimd.hpp
#	ggml/src/ggml-sycl/fattn-mkl.cpp
#	ggml/src/ggml-sycl/fattn-onednn.cpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-sycl/im2col.cpp
#	ggml/src/ggml-sycl/norm.cpp
#	ggml/src/ggml-sycl/rope.cpp
#	ggml/src/ggml-sycl/set_rows.cpp
#	ggml/src/ggml-vulkan/CMakeLists.txt
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl
#	ggml/src/ggml-zendnn/CMakeLists.txt
#	scripts/make-release-desc.sh
#	scripts/sync-ggml.last
#	tests/test-backend-ops.cpp
#	tests/test-json-schema-to-grammar.cpp
#	tools/cli/README.md
#	tools/server/README.md
#	tools/ui/src/lib/constants/settings.constants.ts
#	tools/ui/src/lib/services/chat.service.ts
This commit is contained in:
Concedo
2026-08-24 21:15:18 +08:00
parent 047b233a9c 873e5d8e39
commit bfd6500450
172 changed files with 12496 additions and 13725 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+1
View File
@@ -998,6 +998,7 @@ private:
mtmd_context_params mparams = mtmd_context_params_default();
if (has_mmproj) {
mparams.use_gpu = params_base.mmproj_use_gpu;
mparams.device = params_base.mmproj_device;
mparams.print_timings = false;
mparams.n_threads = params_base.cpuparams.n_threads;
mparams.flash_attn_type = params_base.flash_attn_type;
tools/server/server-models.cpp
+35 -35
View File
@@ -672,24 +672,26 @@ void server_models::load_models() {
apply_hidden();
log_available_models();
std::vector<std::string> models_to_load;
for (const auto & [name, inst] : mapping) {
std::string val;
if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
models_to_load.push_back(name);
// skipped on reload, see startup_models
if (startup_models.has_value()) {
std::vector<std::string> models_to_load;
for (const auto & [name, inst] : mapping) {
std::string val;
if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
models_to_load.push_back(name);
}
}
}
if ((int)models_to_load.size() > base_params.models_max) {
throw std::runtime_error(string_format(
"number of models to load on startup (%zu) exceeds models_max (%d)",
models_to_load.size(), base_params.models_max));
if ((int)models_to_load.size() > base_params.models_max) {
throw std::runtime_error(string_format(
"number of models to load on startup (%zu) exceeds models_max (%d)",
models_to_load.size(), base_params.models_max));
}
// to be lazy-loaded after main() setup phase is completed
startup_models = std::move(models_to_load);
}
lk.unlock();
for (const auto & name : models_to_load) {
SRV_INF("(startup) loading model %s\n", name.c_str());
load(name);
}
} else {
// RELOAD: diff the new preset list against the current mapping and reconcile
is_reloading = true;
@@ -819,8 +821,8 @@ void server_models::load_models() {
inst.meta.update_caps();
}
// add models that are new in this reload
std::vector<std::string> newly_added;
// add models that are new in this reload, load-on-startup is not honored here since a
// reload never spawns an instance
for (const auto & [name, preset] : final_presets) {
if (mapping.find(name) == mapping.end()) {
server_model_meta meta{
@@ -841,42 +843,40 @@ void server_models::load_models() {
// /* need_download */ false,
};
add_model(std::move(meta));
newly_added.push_back(name);
}
}
apply_stop_timeout();
apply_hidden();
// clear reload flag before unlocking for autoload - load() blocks on !is_reloading,
// so clearing it here (while still locked) prevents a deadlock in the autoload calls below
// clear reload flag under the lock, this releases the load() calls waiting on !is_reloading
is_reloading = false;
cv.notify_all();
log_available_models();
// collect autoload candidates while still under the lock
std::vector<std::string> to_autoload;
for (const auto & name : newly_added) {
auto it = mapping.find(name);
if (it != mapping.end()) {
std::string val;
if (it->second.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
to_autoload.push_back(name);
}
}
}
lk.unlock();
for (const auto & name : to_autoload) {
SRV_INF("(reload) loading new model %s\n", name.c_str());
load(name);
}
notify_sse("models_reload", "*");
}
}
void server_models::load_startup_models() {
std::vector<std::string> to_load;
{
std::lock_guard<std::mutex> lk(mutex);
if (!startup_models.has_value()) {
return; // already drained
}
to_load = std::move(*startup_models);
startup_models.reset();
}
for (const auto & name : to_load) {
SRV_INF("(startup) loading model %s\n", name.c_str());
load(name);
}
}
void server_models::update_meta(const std::string & name, const server_model_meta & meta) {
std::lock_guard<std::mutex> lk(mutex);
auto it = mapping.find(name);
tools/server/server-models.h
+7
View File
@@ -136,6 +136,10 @@ private:
// if true, the next get_meta() will trigger a reload of model list
bool need_reload = false;
// models marked with load-on-startup, unset once load_startup_models() drains it
// no value means the startup phase is over, so a reload must not queue anything
std::optional<std::vector<std::string>> startup_models{std::in_place};
// conv_id -> model name that currently serves its stream session, lets the resumable stream
// routes go straight to the owning child instead of polling every one. populated when
// proxy_request forwards a POST carrying an X-Conversation-Id. best effort: a stale entry just
@@ -231,6 +235,9 @@ public:
// - if a model is not running, it will be added or updated according to the source
void load_models();
// lazy-load startup_models, to be called after main() setup phase
void load_startup_models();
// check if a model instance exists (thread-safe)
bool has_model(const std::string & name);
tools/server/server.cpp
+14 -1
View File
@@ -133,7 +133,8 @@ int llama_server(common_params & params, int argc, char ** argv) {
// router server never loads a model and must not touch the GPU
const bool is_router_server = params.model.path.empty()
&& params.model.hf_repo.empty();
&& params.model.hf_repo.empty()
&& params.model.docker_repo.empty();
// skip device enumeration so the CUDA primary context stays uncreated
common_params_print_info(params, !is_router_server);
@@ -423,6 +424,18 @@ int llama_server(common_params & params, int argc, char ** argv) {
ctx_http.stop();
};
try {
models_routes->models.load_startup_models();
} catch (const std::exception & e) {
SRV_ERR("failed to load models on startup: %s\n", e.what());
ctx_http.stop();
if (ctx_http.thread.joinable()) {
ctx_http.thread.join();
}
clean_up();
return 1;
}
} else {
// setup clean up function, to be called before exit
clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() {