Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-20 01:31:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/release.yml
#	.github/workflows/ui-build-self-hosted.yml
#	.github/workflows/ui-build.yml
#	.github/workflows/ui-publish.yml
#	.github/workflows/ui-self-hosted.yml
#	.github/workflows/ui.yml
#	.gitignore
#	README.md
#	docs/ops.md
#	docs/ops/Vulkan.csv
#	ggml/CMakeLists.txt
#	scripts/sync-ggml.last
#	scripts/sync_vendor.py
#	scripts/ui-assets.cmake
#	tests/test-jinja.cpp
#	tests/test-llama-archs.cpp
This commit is contained in:
Concedo
2026-06-14 11:26:16 +08:00
parent ef5a724ee4 c2ba3e47a2
commit ea0351c71a
117 changed files with 11026 additions and 2390 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+7
View File
@@ -2244,6 +2244,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.image_max_tokens = value;
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_IMAGE_MAX_TOKENS"));
add_opt(common_arg(
{"--mtmd-batch-max-tokens"}, "N",
string_format("maximum number of image tokens per batch when encoding images (default: %d)", params.mtmd_batch_max_tokens),
[](common_params & params, int value) {
params.mtmd_batch_max_tokens = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS"));
if (llama_supports_rpc()) {
add_opt(common_arg(
{"--rpc"}, "SERVERS",
common/common.h
+1
View File
@@ -576,6 +576,7 @@ struct common_params {
std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"
int image_min_tokens = -1;
int image_max_tokens = -1;
int mtmd_batch_max_tokens = 1024;
// finetune
struct lr_opt lr;
common/fit.cpp
+29 -6
View File
@@ -26,7 +26,7 @@ class common_params_fit_exception : public std::runtime_error {
using std::runtime_error::runtime_error;
};
std::vector<llama_device_memory_data> common_get_device_memory_data(
static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
const char * path_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
@@ -150,6 +150,29 @@ std::vector<llama_device_memory_data> common_get_device_memory_data(
return ret;
}
common_device_memory_data_vec common_get_device_memory_data(
const char * path_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
uint32_t & hp_ngl,
uint32_t & hp_n_ctx_train,
uint32_t & hp_n_expert,
ggml_log_level log_level) {
std::vector<llama_device_memory_data> impl = common_get_device_memory_data_impl(
path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level);
common_device_memory_data_vec ret(impl.size());
for (size_t i = 0; i < impl.size(); i++) {
ret[i].total = impl[i].total;
ret[i].free = impl[i].free;
ret[i].model = impl[i].mb.model;
ret[i].context = impl[i].mb.context;
ret[i].compute = impl[i].mb.compute;
}
return ret;
}
static void common_params_fit_impl(
const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,
@@ -169,7 +192,7 @@ static void common_params_fit_impl(
// step 1: get data for default parameters and check whether any changes are necessary in the first place
LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
const dmds_t dmds_full = common_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
const dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
const size_t nd = devs.size(); // number of devices
std::vector<int64_t> margins; // this function uses int64_t rather than size_t for memory sizes to more conveniently handle deficits
@@ -304,7 +327,7 @@ static void common_params_fit_impl(
int64_t sum_projected_used_min_ctx = 0;
cparams->n_ctx = n_ctx_min;
const dmds_t dmds_min_ctx = common_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
const dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
if (nd == 0) {
sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total();
} else {
@@ -482,7 +505,7 @@ static void common_params_fit_impl(
llama_model_params mparams_copy = *mparams;
set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);
const dmds_t dmd_nl = common_get_device_memory_data(
const dmds_t dmd_nl = common_get_device_memory_data_impl(
path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
LOG_TRC("%s: memory for test allocation by device:\n", func_name);
@@ -510,7 +533,7 @@ static void common_params_fit_impl(
mparams->tensor_buft_overrides = tensor_buft_overrides;
LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
const dmds_t dmds_cpu_moe = common_get_device_memory_data(
const dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(
path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
for (size_t id = 0; id < nd; id++) {
@@ -940,7 +963,7 @@ void common_fit_print(
uint32_t hp_nct = 0; // hparams.n_ctx_train
uint32_t hp_nex = 0; // hparams.n_expert
auto dmd = common_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
GGML_ASSERT(dmd.size() == devs.size() + 1);
for (size_t id = 0; id < devs.size(); id++) {
common/fit.h
+32 -24
View File
@@ -1,9 +1,7 @@
#pragma once
#include "ggml.h"
#include "ggml-backend.h"
#include "llama.h"
#include "../src/llama-ext.h"
#include <vector>
@@ -18,31 +16,41 @@ enum common_params_fit_status {
// - this function is NOT thread safe because it modifies the global llama logger state
// - only parameters that have the same value as in llama_default_model_params are modified
// with the exception of the context size which is modified if and only if equal to 0
enum common_params_fit_status common_fit_params(
const char * path_model,
struct llama_model_params * mparams,
struct llama_context_params * cparams,
float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
struct llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
size_t * margins, // margins of memory to leave per device in bytes
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
enum ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
common_params_fit_status common_fit_params(
const char * path_model,
llama_model_params * mparams,
llama_context_params * cparams,
float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
size_t * margins, // margins of memory to leave per device in bytes
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
// print estimated memory to stdout
void common_fit_print(
const char * path_model,
struct llama_model_params * mparams,
struct llama_context_params * cparams);
const char * path_model,
llama_model_params * mparams,
llama_context_params * cparams);
void common_memory_breakdown_print(const struct llama_context * ctx);
void common_memory_breakdown_print(const llama_context * ctx);
struct common_device_memory_data {
int64_t total;
int64_t free;
size_t model;
size_t context;
size_t compute;
};
using common_device_memory_data_vec = std::vector<common_device_memory_data>;
// Load a model + context with no_alloc and return the per-device memory breakdown.
std::vector<llama_device_memory_data> common_get_device_memory_data(
const char * path_model,
const struct llama_model_params * mparams,
const struct llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
uint32_t & hp_ngl,
uint32_t & hp_n_ctx_train,
uint32_t & hp_n_expert,
enum ggml_log_level log_level);
common_device_memory_data_vec common_get_device_memory_data(
const char * path_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
uint32_t & hp_ngl,
uint32_t & hp_n_ctx_train,
uint32_t & hp_n_expert,
ggml_log_level log_level);
common/jinja/runtime.cpp
+2 -2
View File
@@ -761,9 +761,9 @@ value member_expression::execute_impl(context & ctx) {
if (is_stmt<slice_expression>(this->property)) {
auto s = cast_stmt<slice_expression>(this->property);
value start_val = s->start_expr ? s->start_expr->execute(ctx) : mk_val<value_int>(0);
value stop_val = s->stop_expr ? s->stop_expr->execute(ctx) : mk_val<value_int>(arr_size);
value step_val = s->step_expr ? s->step_expr->execute(ctx) : mk_val<value_int>(1);
value start_val = s->start_expr ? s->start_expr->execute(ctx) : (step_val->as_int() < 0 ? mk_val<value_int>(arr_size - 1) : mk_val<value_int>(0));
value stop_val = s->stop_expr ? s->stop_expr->execute(ctx) : (step_val->as_int() < 0 ? mk_val<value_int>(-1) : mk_val<value_int>(arr_size));
// translate to function call: obj.slice(start, stop, step)
JJ_DEBUG("Member expression is a slice: start %s, stop %s, step %s",
common/jinja/value.cpp
+26 -7
View File
@@ -90,14 +90,14 @@ static T slice(const T & array, int64_t start, int64_t stop, int64_t step = 1) {
stop_val = std::min(stop_val, len);
}
} else {
start_val = len - 1;
start_val = start;
if (start_val < 0) {
start_val = std::max(len + start_val, (int64_t)-1);
start_val = std::max(len + start_val, (int64_t)0);
} else {
start_val = std::min(start_val, len - 1);
}
stop_val = -1;
stop_val = stop;
if (stop_val < -1) {
stop_val = std::max(len + stop_val, (int64_t)-1);
} else {
@@ -673,6 +673,9 @@ const func_builtins & value_string_t::get_builtins() const {
std::string str = val_input->as_string().str();
// FIXME: Support non-specified delimiter (split on consecutive (no leading or trailing) whitespace)
std::string delim = (args.count() > 1) ? args.get_pos(1)->as_string().str() : " ";
if (delim.empty()) {
throw raised_exception("empty separator");
}
int64_t maxsplit = (args.count() > 2) ? args.get_pos(2)->as_int() : -1;
auto result = mk_val<value_array>();
size_t pos = 0;
@@ -697,6 +700,9 @@ const func_builtins & value_string_t::get_builtins() const {
std::string str = val_input->as_string().str();
// FIXME: Support non-specified delimiter (split on consecutive (no leading or trailing) whitespace)
std::string delim = (args.count() > 1) ? args.get_pos(1)->as_string().str() : " ";
if (delim.empty()) {
throw raised_exception("empty separator");
}
int64_t maxsplit = (args.count() > 2) ? args.get_pos(2)->as_int() : -1;
auto result = mk_val<value_array>();
size_t pos = 0;
@@ -722,10 +728,23 @@ const func_builtins & value_string_t::get_builtins() const {
if (count > 0) {
throw not_implemented_exception("String replace with count argument not implemented");
}
size_t pos = 0;
while ((pos = str.find(old_str, pos)) != std::string::npos) {
str.replace(pos, old_str.length(), new_str);
pos += new_str.length();
if (old_str != new_str) {
size_t pos = 0;
if (old_str.empty()) {
std::string new_res;
new_res.reserve(str.length() + new_str.length() * (str.length() + 1));
new_res += new_str;
for (const char c : str) {
new_res.push_back(c);
new_res += new_str;
}
str = new_res;
} else {
while ((pos = str.find(old_str, pos)) != std::string::npos) {
str.replace(pos, old_str.length(), new_str);
pos += new_str.length();
}
}
}
auto res = mk_val<value_string>(str);
res->val_str.mark_input_based_on(args.get_pos(0)->val_str);