Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-20 01:31:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/build-apple.yml
#	.github/workflows/build-self-hosted.yml
#	.github/workflows/release.yml
#	SECURITY.md
#	build-xcframework.sh
#	ci/run.sh
#	docs/development/HOWTO-add-model.md
#	examples/model-conversion/scripts/causal/convert-model.sh
#	examples/model-conversion/scripts/embedding/convert-model.sh
#	scripts/sync_vendor.py
#	scripts/ui-assets.cmake
#	tests/test-arg-parser.cpp
#	tests/test-backend-sampler.cpp
#	tests/test-grammar-parser.cpp
#	tests/test-llama-archs.cpp
#	tests/test-sampling.cpp
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/mtmd/CMakeLists.txt
#	tools/mtmd/mtmd.h
#	tools/mtmd/tests/test-deepseek-ocr.py
#	tools/server/README.md
#	tools/tts/CMakeLists.txt
#	tools/tts/convert_pt_to_hf.py
This commit is contained in:
Concedo
2026-08-07 20:46:56 +08:00
parent 56368c0fd4 6a32c29a74
commit 8a16f96307
125 changed files with 7633 additions and 2869 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-arch.cpp
+2
View File
@@ -144,6 +144,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_TALKIE, "talkie" },
{ LLM_ARCH_MELLUM, "mellum" },
{ LLM_ARCH_NANBEIGE, "nanbeige" },
{ LLM_ARCH_QWEN3TTS, "qwen3tts" },
{ LLM_ARCH_UNKNOWN, "(unknown)" },
};
@@ -1026,6 +1027,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_MINIMAX_M3:
case LLM_ARCH_MISTRAL4:
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_QWEN3TTS:
return false;
default:
return true;
src/llama-arch.h
+1
View File
@@ -149,6 +149,7 @@ enum llm_arch {
LLM_ARCH_MINIMAX_M3,
LLM_ARCH_DFLASH,
LLM_ARCH_NANBEIGE,
LLM_ARCH_QWEN3TTS,
LLM_ARCH_UNKNOWN,
};
src/llama-ext.h
+6
View File
@@ -124,3 +124,9 @@ LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx);
LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model);
// returns the number of extracted layers from target model
LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model);
// retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab)
// returns total number of elements or 0 on error
// if out is nullptr, returns the number of tokens without writing to out
// caller must allocate enough memory for out before calling
LLAMA_API uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out);
src/llama-grammar.cpp
+4 -2
View File
@@ -673,10 +673,12 @@ const char * llama_grammar_parser::parse_sequence(
} else {
throw std::runtime_error(std::string("expecting ',' at ") + pos);
}
bool has_max = max_times != UINT64_MAX;
if (min_times > MAX_REPETITION_THRESHOLD || (has_max && max_times > MAX_REPETITION_THRESHOLD)) {
if (min_times > MAX_REPETITION_THRESHOLD) {
throw std::runtime_error(std::string("number of repetitions exceeds sane defaults, please reduce the number of repetitions"));
}
if (max_times != UINT64_MAX && max_times > MAX_REPETITION_THRESHOLD) {
max_times = UINT64_MAX;
}
handle_repetitions(min_times, max_times);
} else {
break;
src/llama-model-loader.cpp
+16 -2
View File
@@ -1250,7 +1250,13 @@ struct ggml_tensor * llama_model_loader::create_tensor(
for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {
t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;
GGML_ASSERT(t_meta.ne[dim] >= 1);
t_meta.nb[dim] = dim == 0 ? ggml_type_size(type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];
if (dim == 0) {
t_meta.nb[dim] = ggml_type_size(type);
} else if (dim == 1) {
t_meta.nb[dim] = ggml_row_size(type, t_meta.ne[dim-1]);
} else {
t_meta.nb[dim] = t_meta.nb[dim-1]*t_meta.ne[dim-1];
}
GGML_ASSERT(t_meta.nb[dim] >= 1);
}
ggml_set_name(&t_meta, tn.str().c_str());
@@ -1273,10 +1279,18 @@ struct ggml_tensor * llama_model_loader::create_tensor(
if (flags & TENSOR_ALLOW_RESHAPE) {
for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {
t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;
t_meta.nb[dim] = dim == 0 ? ggml_type_size(t_meta.type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];
if (dim == 0) {
t_meta.nb[dim] = ggml_type_size(t_meta.type);
} else if (dim == 1) {
t_meta.nb[dim] = ggml_row_size(t_meta.type, t_meta.ne[dim-1]);
} else {
t_meta.nb[dim] = t_meta.ne[dim-1]*t_meta.nb[dim-1];
}
}
}
GGML_ASSERT(ggml_nbytes(&t_meta) == ggml_nbytes(cur));
ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta);
if (buft == nullptr) {
return nullptr;
src/llama-model.cpp
+53
View File
@@ -254,6 +254,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_qwen3vl(params);
case LLM_ARCH_QWEN3VLMOE:
return new llama_model_qwen3vlmoe(params);
case LLM_ARCH_QWEN3TTS:
return new llama_model_qwen3tts(params);
case LLM_ARCH_PHI2:
return new llama_model_phi2(params);
case LLM_ARCH_PHI3:
@@ -2835,6 +2837,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_QWEN3VLMOE:
case LLM_ARCH_QWEN35:
case LLM_ARCH_QWEN35MOE:
case LLM_ARCH_QWEN3TTS:
return LLAMA_ROPE_TYPE_IMROPE;
case LLM_ARCH_GLM4:
@@ -3029,6 +3032,21 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid,
int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_,
int flags) {
const int64_t n_embd_qkv = n_embd_q_ + n_embd_k_ + n_embd_v_;
if (flags & TENSOR_SKIP) {
const int skip = TENSOR_NOT_REQUIRED | TENSOR_SKIP;
create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, skip | TENSOR_SKIP_IF_VIRTUAL);
create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, skip | TENSOR_SKIP_IF_VIRTUAL);
create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", bid), {n_embd_, n_embd_q_}, skip);
create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", bid), {n_embd_, n_embd_k_}, skip);
create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", bid), {n_embd_, n_embd_v_}, skip);
create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", bid), {n_embd_q_}, skip);
create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", bid), {n_embd_k_}, skip);
create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", bid), {n_embd_v_}, skip);
return;
}
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
if (layer.wqkv) {
layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
@@ -3050,3 +3068,38 @@ const int32_t * llama_model_target_layer_ids(const struct llama_model * model) {
uint32_t llama_model_target_layer_ids_n(const struct llama_model * model) {
return (uint32_t) model->target_layer_ids.size();
}
uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out) {
if (model->vocab.n_tokens() == 0 || model->tok_embd == nullptr) {
return 0;
}
const ggml_tensor * tensor = model->tok_embd;
const size_t nelements = ggml_nelements(tensor);
GGML_ASSERT(nelements <= UINT32_MAX); // for the return type
if (out == nullptr) {
return (uint32_t) nelements;
}
if (tensor->type == GGML_TYPE_F32) {
ggml_backend_tensor_get(tensor, out, 0, nelements * sizeof(float));
return (uint32_t) nelements;
}
std::vector<uint8_t> buf(ggml_nbytes(tensor));
ggml_backend_tensor_get(tensor, buf.data(), 0, buf.size());
const ggml_type_traits * traits = ggml_get_type_traits(tensor->type);
if (tensor->type == GGML_TYPE_F16) {
ggml_fp16_to_fp32_row((const ggml_fp16_t *) buf.data(), out, nelements);
} else if (tensor->type == GGML_TYPE_BF16) {
ggml_bf16_to_fp32_row((const ggml_bf16_t *) buf.data(), out, nelements);
} else if (ggml_is_quantized(tensor->type) && traits->to_float != nullptr) {
traits->to_float(buf.data(), out, nelements);
} else {
GGML_ABORT("unsupported tensor type for dequantization: %s", ggml_type_name(tensor->type));
}
return (uint32_t) nelements;
}
src/llama-sampler.cpp
+8 -12
View File
@@ -3078,8 +3078,6 @@ struct llama_sampler * llama_sampler_init_top_n_sigma(float n) {
// DRY
struct llama_sampler_dry {
int32_t total_context_size;
const float dry_multiplier;
const float dry_base;
const int32_t dry_allowed_length;
@@ -3155,8 +3153,7 @@ static void llama_sampler_dry_apply(struct llama_sampler * smpl, llama_token_dat
return;
}
int32_t effective_dry_penalty_last_n = (ctx->dry_penalty_last_n == -1) ? ctx->total_context_size : std::max(ctx->dry_penalty_last_n, 0);
int last_n_repeat = std::min(std::min((int)ctx->last_tokens.size(), effective_dry_penalty_last_n), ctx->total_context_size);
int last_n_repeat = std::min((int) ctx->last_tokens.size(), ctx->dry_penalty_last_n);
if (last_n_repeat <= ctx->dry_allowed_length) {
return;
@@ -3369,7 +3366,7 @@ static struct llama_sampler * llama_sampler_dry_clone(const struct llama_sampler
llama_vocab dummy_vocab;
// dummy vocab is passed because it is only needed for raw sequence breaker processing, which we have already done and will simply be copying
auto * result = llama_sampler_init_dry(&dummy_vocab, ctx->total_context_size, ctx->dry_multiplier, ctx->dry_base, ctx->dry_allowed_length, ctx->dry_penalty_last_n, NULL, 0);
auto * result = llama_sampler_init_dry(&dummy_vocab, ctx->dry_multiplier, ctx->dry_base, ctx->dry_allowed_length, ctx->dry_penalty_last_n, NULL, 0);
// Copy the state, including the processed breakers
{
@@ -3400,8 +3397,8 @@ static struct llama_sampler_i llama_sampler_dry_i = {
/* .backend_set_input = */ nullptr,
};
struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab, int32_t n_ctx_train, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const char** seq_breakers, size_t num_breakers) {
int32_t effective_dry_penalty_last_n = (dry_penalty_last_n == -1) ? n_ctx_train : std::max(dry_penalty_last_n, 0);
struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const char** seq_breakers, size_t num_breakers) {
dry_penalty_last_n = std::max(dry_penalty_last_n, 0);
std::unordered_multimap<llama_token, std::vector<llama_token>> processed_breakers;
const int MAX_CHAR_LEN = 40;
const int MAX_SEQ_LEN = 20;
@@ -3438,23 +3435,22 @@ struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab,
return llama_sampler_init(
/* .iface = */ &llama_sampler_dry_i,
/* .ctx = */ new llama_sampler_dry {
/* .total_context_size = */ n_ctx_train,
/* .dry_multiplier = */ dry_multiplier,
/* .dry_base = */ dry_base,
/* .dry_allowed_length = */ dry_allowed_length,
/* .dry_penalty_last_n = */ dry_penalty_last_n,
/* .dry_processed_breakers = */ std::move(processed_breakers),
/* .dry_repeat_count = */ dry_enabled ? std::vector<int>(effective_dry_penalty_last_n, 0) : std::vector<int>{},
/* .dry_repeat_count = */ dry_enabled ? std::vector<int>(dry_penalty_last_n, 0) : std::vector<int>{},
/* .dry_max_token_repeat = */ {},
/* .last_tokens = */ dry_enabled ? ring_buffer<llama_token>(effective_dry_penalty_last_n) : ring_buffer<llama_token>(0),
/* .last_tokens = */ dry_enabled ? ring_buffer<llama_token>(dry_penalty_last_n) : ring_buffer<llama_token>(0),
}
);
}
// wrapper for test-sampling.cpp
struct llama_sampler * llama_sampler_init_dry_testing(int32_t context_size, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const std::vector<std::vector<llama_token>>& seq_breakers) {
struct llama_sampler * llama_sampler_init_dry_testing(float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const std::vector<std::vector<llama_token>>& seq_breakers) {
llama_vocab dummy_vocab;
auto * result = llama_sampler_init_dry(&dummy_vocab, context_size, dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, NULL, 0);
auto * result = llama_sampler_init_dry(&dummy_vocab, dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, NULL, 0);
auto * ctx = (llama_sampler_dry *) result->ctx;
// Process the token-based sequence breakers
src/llama-sampler.h
-1
View File
@@ -34,7 +34,6 @@ struct llama_sampler_chain {
};
struct llama_sampler * llama_sampler_init_dry_testing(
int32_t context_size,
float dry_multiplier,
float dry_base,
int32_t dry_allowed_length,
src/models/models.h
+5
View File
@@ -596,6 +596,11 @@ struct llama_model_qwen3vlmoe : public llama_model_base {
};
struct llama_model_qwen3tts : public llama_model_qwen3vl {
llama_model_qwen3tts(const struct llama_model_params & params) : llama_model_qwen3vl(params) {}
};
struct llama_model_phi2 : public llama_model_base {
llama_model_phi2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
src/models/qwen3tts.cpp
+3
View File
@@ -0,0 +1,3 @@
#include "models.h"
// llama_model_qwen3tts reuses llama_model_qwen3vl's hparams/tensors/graph logic
src/models/qwen3vl.cpp
+24 -1
View File
@@ -16,11 +16,16 @@ void llama_model_qwen3vl::load_arch_hparams(llama_model_loader & ml) {
void llama_model_qwen3vl::load_arch_tensors(llama_model_loader &) {
LLAMA_LOAD_LOCALS;
int64_t n_vocab_out = n_vocab;
if (arch == LLM_ARCH_QWEN3TTS) {
n_vocab_out = 3072;
}
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
// output
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab_out}, TENSOR_NOT_REQUIRED);
// if output is NULL, init from the input tok embed
if (output == NULL) {
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
@@ -166,6 +171,24 @@ llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_par
// lm_head
cur = build_lora_mm(model.output, cur, model.output_s);
int64_t n_vocab_in = model.tok_embd->ne[1];
int64_t n_vocab_out = model.output->ne[1];
if (n_vocab_in > n_vocab_out) {
// case: Qwen3TTS model with codec_head as output
GGML_ASSERT(model.output_norm);
int64_t pad = n_vocab_in - n_vocab_out;
// using this trick to get a scalar -inf tensor to pad the output
ggml_tensor * neg_inf = ggml_scale_bias(ctx0,
ggml_view_1d(ctx0, model.output_norm, 1, 0),
0.0f, -INFINITY);
neg_inf = ggml_repeat_4d(ctx0, neg_inf, pad, cur->ne[1], 1, 1);
cur = ggml_concat(ctx0, neg_inf, cur, 0); // [padded .. n_vocab_out, n_stream]
} else if (n_vocab_in < n_vocab_out) {
GGML_ABORT("invalid case");
}
cb(cur, "result_output", -1);
res->t_logits = cur;