Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 01:05:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/build-cache.yml
#	.github/workflows/build-cmake-pkg.yml
#	.github/workflows/build-cpu.yml
#	.github/workflows/build-cuda-windows.yml
#	.github/workflows/build-sanitize.yml
#	.github/workflows/release.yml
#	.github/workflows/winget.yml
#	CMakeLists.txt
#	README.md
#	app/llama.cpp
#	cmake/llama-config.cmake.in
#	cmake/llama.pc.in
#	common/CMakeLists.txt
#	common/build-info.h
#	docs/backend/OPENVINO.md
#	docs/backend/SYCL.md
#	docs/preset.md
#	ggml/CMakeLists.txt
#	ggml/cmake/ggml-config.cmake.in
#	ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
#	ggml/src/ggml-hip/CMakeLists.txt
#	ggml/src/ggml-openvino/ggml-decoder.cpp
#	ggml/src/ggml-openvino/ggml-decoder.h
#	ggml/src/ggml-openvino/ggml-openvino-extra.cpp
#	ggml/src/ggml-openvino/ggml-openvino-extra.h
#	ggml/src/ggml-openvino/ggml-openvino.cpp
#	ggml/src/ggml-openvino/ggml-quants.cpp
#	ggml/src/ggml-openvino/ggml-quants.h
#	ggml/src/ggml-openvino/openvino/decoder.h
#	ggml/src/ggml-openvino/openvino/node_context.h
#	ggml/src/ggml-openvino/openvino/op/cpy.cpp
#	ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
#	ggml/src/ggml-openvino/openvino/op/get_rows.cpp
#	ggml/src/ggml-openvino/openvino/op/l2_norm.cpp
#	ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp
#	ggml/src/ggml-openvino/openvino/op/repeat.cpp
#	ggml/src/ggml-openvino/openvino/op/reshape.cpp
#	ggml/src/ggml-openvino/openvino/op/rms_norm.cpp
#	ggml/src/ggml-openvino/openvino/op/rope.cpp
#	ggml/src/ggml-openvino/openvino/op/scale.cpp
#	ggml/src/ggml-openvino/openvino/op/set_rows.cpp
#	ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp
#	ggml/src/ggml-openvino/openvino/op/view.cpp
#	ggml/src/ggml-openvino/openvino/op_table.cpp
#	ggml/src/ggml-openvino/openvino/op_table.h
#	ggml/src/ggml-openvino/openvino/translate_session.cpp
#	ggml/src/ggml-openvino/openvino/utils.cpp
#	ggml/src/ggml-openvino/openvino/utils.h
#	ggml/src/ggml-openvino/utils.cpp
#	ggml/src/ggml-openvino/utils.h
#	ggml/src/ggml-sycl/common.hpp
#	ggml/src/ggml-sycl/concat.cpp
#	ggml/src/ggml-sycl/dmmv.cpp
#	ggml/src/ggml-sycl/element_wise.cpp
#	ggml/src/ggml-sycl/element_wise.hpp
#	ggml/src/ggml-sycl/fusion.cpp
#	ggml/src/ggml-sycl/fusion.hpp
#	ggml/src/ggml-sycl/gated_delta_net.cpp
#	ggml/src/ggml-sycl/gated_delta_net.hpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-sycl/mmvq.cpp
#	ggml/src/ggml-sycl/mmvq.hpp
#	scripts/sync-ggml.last
#	src/CMakeLists.txt
#	tests/test-backend-ops.cpp
#	tests/test-chat.cpp
#	tests/test-gguf.cpp
#	tests/test-quantize-stats.cpp
#	tools/cvector-generator/cvector-generator.cpp
#	tools/mtmd/CMakeLists.txt
#	tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte
This commit is contained in:
Concedo
2026-08-14 18:10:18 +08:00
parent da91226fbc 885c5bbe8e
commit 886a42d446
380 changed files with 5639 additions and 4009 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/common.cpp
+123 -10
View File
@@ -1025,20 +1025,21 @@ std::string fs_get_cache_directory() {
std::string cache_directory = "";
auto ensure_trailing_slash = [](std::string p) {
// Make sure to add trailing slash
if (p.back() != DIRECTORY_SEPARATOR) {
if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
p += DIRECTORY_SEPARATOR;
}
return p;
};
if (getenv("LLAMA_CACHE")) {
cache_directory = std::getenv("LLAMA_CACHE");
} else {
cache_directory = common_get_env("LLAMA_CACHE");
if (cache_directory.empty()) {
#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
defined(__OpenBSD__) || defined(__NetBSD__)
if (std::getenv("XDG_CACHE_HOME")) {
cache_directory = std::getenv("XDG_CACHE_HOME");
} else if (std::getenv("HOME")) {
cache_directory = std::getenv("HOME") + std::string("/.cache/");
const std::string xdg_cache_home = common_get_env("XDG_CACHE_HOME");
const std::string home = common_get_env("HOME");
if (!xdg_cache_home.empty()) {
cache_directory = xdg_cache_home;
} else if (!home.empty()) {
cache_directory = home + "/.cache/";
} else {
#if defined(__linux__)
/* no $HOME is defined, fallback to getpwuid */
@@ -1053,9 +1054,16 @@ std::string fs_get_cache_directory() {
#endif /* defined(__linux__) */
}
#elif defined(__APPLE__)
cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");
cache_directory = common_get_env("HOME");
if (cache_directory.empty()) {
throw std::runtime_error("Failed to find $HOME directory");
}
cache_directory += "/Library/Caches/";
#elif defined(_WIN32)
cache_directory = std::getenv("LOCALAPPDATA");
cache_directory = common_get_env("LOCALAPPDATA");
if (cache_directory.empty()) {
throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
}
#elif defined(__EMSCRIPTEN__)
GGML_ABORT("not implemented on this platform");
#else
@@ -1067,6 +1075,51 @@ std::string fs_get_cache_directory() {
return ensure_trailing_slash(cache_directory);
}
std::string fs_get_config_directory() {
std::string config_directory = "";
auto ensure_trailing_slash = [](std::string p) {
if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
p += DIRECTORY_SEPARATOR;
}
return p;
};
#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)
const std::string xdg_config_home = common_get_env("XDG_CONFIG_HOME");
const std::string home = common_get_env("HOME");
if (!xdg_config_home.empty()) {
config_directory = xdg_config_home;
} else if (!home.empty()) {
config_directory = home + "/.config/";
} else {
#if defined(__linux__)
/* no $HOME is defined, fallback to getpwuid */
struct passwd *pw = getpwuid(getuid());
if ((!pw) || (!pw->pw_dir)) {
throw std::runtime_error("Failed to find $HOME directory");
}
config_directory = std::string(pw->pw_dir) + std::string("/.config/");
#else
throw std::runtime_error("Failed to find $HOME directory");
#endif
}
#elif defined(_WIN32)
config_directory = common_get_env("APPDATA");
if (config_directory.empty()) {
throw std::runtime_error("Failed to find %APPDATA% directory");
}
#elif defined(__EMSCRIPTEN__)
// caller decides what to do when there is no config directory
throw std::runtime_error("not implemented on this platform");
#else
# error Unknown architecture
#endif
config_directory = ensure_trailing_slash(config_directory);
config_directory += "llama.cpp";
return ensure_trailing_slash(config_directory);
}
std::string fs_get_cache_file(const std::string & filename) {
GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);
std::string cache_directory = fs_get_cache_directory();
@@ -1228,6 +1281,8 @@ struct common_init_result::impl {
// note: the order in which model, context, etc. are declared matters because their destructors will be called bottom-to-top
common_threadpools threadpools;
llama_model_ptr model;
llama_context_ptr context;
@@ -1329,6 +1384,10 @@ common_init_result::common_init_result(common_params & params, bool model_only)
}
pimpl->context.reset(lctx);
set_process_priority(params.cpuparams.priority);
pimpl->threadpools.init(lctx, params);
}
llama_model * common_init_result::model() {
@@ -1677,6 +1736,10 @@ struct llama_context_params common_context_params_to_llama(const common_params &
return cparams;
}
//
// Threadpool utils
//
struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params) {
struct ggml_threadpool_params tpp;
@@ -1693,6 +1756,56 @@ struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const commo
return tpp;
}
common_threadpools::~common_threadpools() {
if (!free_fn) {
return;
}
free_fn(threadpool);
free_fn(threadpool_batch);
}
void common_threadpools::init(llama_context * ctx, const common_params & params) {
GGML_ASSERT(!threadpool);
GGML_ASSERT(!threadpool_batch);
COM_INF("llama threadpool init, n_threads = %d\n", (int) params.cpuparams.n_threads);
auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
if (!cpu_dev) {
COM_WRN("%s", "no CPU backend found\n");
return;
}
auto * reg = ggml_backend_dev_backend_reg(cpu_dev);
auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");
free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");
struct ggml_threadpool_params tpp_batch =
ggml_threadpool_params_from_cpu_params(params.cpuparams_batch);
struct ggml_threadpool_params tpp =
ggml_threadpool_params_from_cpu_params(params.cpuparams);
if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) {
threadpool_batch = ggml_threadpool_new_fn(&tpp_batch);
if (!threadpool_batch) {
COM_WRN("batch threadpool create failed : n_threads %d\n", tpp_batch.n_threads);
return;
}
// start the non-batch threadpool in the paused state
tpp.paused = true;
}
threadpool = ggml_threadpool_new_fn(&tpp);
if (!threadpool) {
COM_WRN("threadpool create failed : n_threads %d\n", tpp.n_threads);
free_fn(threadpool_batch);
threadpool_batch = nullptr;
return;
}
llama_attach_threadpool(ctx, threadpool, threadpool_batch);
}
//
// Batch utils
//