Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 01:05:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/build-webgpu.yml
#	CMakeLists.txt
#	common/CMakeLists.txt
#	docs/development/HOWTO-add-model.md
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-sycl/CMakeLists.txt
#	tests/test-arg-parser.cpp
#	tests/test-jinja.cpp
#	tests/test-llama-archs.cpp
#	tests/test-save-load-state.cpp
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/llama-bench/llama-bench.cpp
#	tools/mtmd/CMakeLists.txt
#	tools/server/README.md
This commit is contained in:
Concedo
2026-07-27 22:28:38 +08:00
parent 90510f2b27 0324696b8e
commit fee0bf446c
74 changed files with 2516 additions and 361 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+50
View File
@@ -59,6 +59,7 @@
#include "models/llama4.cpp"
#include "models/llava.cpp"
#include "models/minicpmv.cpp"
#include "models/minimax-m3.cpp"
#include "models/paddleocr.cpp"
#include "models/pixtral.cpp"
#include "models/qwen2vl.cpp"
@@ -963,6 +964,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_mimovl>(ctx, img);
} break;
case PROJECTOR_TYPE_MINIMAX_M3:
{
builder = std::make_unique<clip_graph_minimax_m3>(ctx, img);
} break;
case PROJECTOR_TYPE_STEP3VL:
{
builder = std::make_unique<clip_graph_step3vl>(ctx, img);
@@ -1545,6 +1550,17 @@ struct clip_model_loader {
// LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__);
// }
} break;
case PROJECTOR_TYPE_MINIMAX_M3:
{
hparams.n_merge = 2; // spatial_merge_size
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
hparams.image_resize_pad = PAD_NONE;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
hparams.rope_theta = 10000.0f; // vision_config.rope_theta
// MiniMax-M3: max_pixels 451584 (=672^2) -> 576 merged tokens (image_seq_length)
hparams.set_limit_image_tokens(8, 576);
hparams.set_warmup_n_tokens(16*16);
} break;
case PROJECTOR_TYPE_MIMOVL:
{
hparams.n_merge = 2; // spatial_merge_size
@@ -2170,6 +2186,19 @@ struct clip_model_loader {
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false);
} break;
case PROJECTOR_TYPE_MINIMAX_M3:
{
// per-patch MLP: mm.1 -> gelu -> mm.2
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
// 2x2 merge MLP: mm.merge.fc1 -> gelu -> mm.merge.fc2
model.mm_merger_fc1_w = get_tensor(string_format(TN_MM_MERGER_FC1, "weight"));
model.mm_merger_fc1_b = get_tensor(string_format(TN_MM_MERGER_FC1, "bias"));
model.mm_merger_fc2_w = get_tensor(string_format(TN_MM_MERGER_FC2, "weight"));
model.mm_merger_fc2_b = get_tensor(string_format(TN_MM_MERGER_FC2, "bias"));
} break;
case PROJECTOR_TYPE_STEP3VL:
{
model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
@@ -3441,6 +3470,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_MIMOVL:
case PROJECTOR_TYPE_MINIMAX_M3:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_YOUTUVL:
{
@@ -3947,6 +3977,24 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
set_input_i32("positions", positions);
} break;
case PROJECTOR_TYPE_MINIMAX_M3:
{
const int n_merge = hparams.n_merge;
const int gh = image_size_height / patch_size;
const int gw = image_size_width / patch_size;
std::vector<int32_t> pos_h, pos_w;
pos_h.reserve(gh * gw);
pos_w.reserve(gh * gw);
for (int bh = 0; bh < gh / n_merge; bh++)
for (int bw = 0; bw < gw / n_merge; bw++)
for (int mh = 0; mh < n_merge; mh++)
for (int mw = 0; mw < n_merge; mw++) {
pos_h.push_back(bh * n_merge + mh);
pos_w.push_back(bw * n_merge + mw);
}
set_input_i32("minimax_pos_h", pos_h);
set_input_i32("minimax_pos_w", pos_w);
} break;
case PROJECTOR_TYPE_DOTS_OCR:
{
const int pw = image_size_width / patch_size;
@@ -4650,6 +4698,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_ffn_down_w->ne[1];
case PROJECTOR_TYPE_GLM_EDGE:
return ctx->model.mm_model_mlp_3_w->ne[1];
case PROJECTOR_TYPE_MINIMAX_M3:
return ctx->model.mm_merger_fc2_b->ne[0];
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_EXAONE4_5: