Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-20 01:31:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/ui-build.yml
#	.github/workflows/ui-publish.yml
#	.github/workflows/ui-self-hosted.yml
#	CMakeLists.txt
#	app/CMakeLists.txt
#	app/llama.cpp
#	common/arg.cpp
#	ggml/CMakeLists.txt
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp/htp-ops.h
#	ggml/src/ggml-hexagon/htp/main.c
#	ggml/src/ggml-hexagon/htp/unary-ops.c
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	scripts/snapdragon/ggml-hexagon-profile.py
#	scripts/sync-ggml.last
#	src/CMakeLists.txt
#	tests/test-llama-archs.cpp
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/mtmd/CMakeLists.txt
#	tools/mtmd/tests/test-deepseek-ocr.py
#	tools/server/README.md
This commit is contained in:
Concedo
2026-05-30 01:54:53 +08:00
parent baacd640c0 06d26dfdff
commit e541554d70
55 changed files with 2443 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+69 -5
View File
@@ -76,6 +76,7 @@
#include "models/siglip.cpp"
#include "models/whisper-enc.cpp"
#include "models/deepseekocr.cpp"
#include "models/deepseekocr2.cpp"
#include "models/mobilenetv5.cpp"
#include "models/youtuvl.cpp"
#include "models/yasa2.cpp"
@@ -306,6 +307,7 @@ clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) :
n_patches(n_patches_x * n_patches_y),
n_embd(hparams.n_embd),
n_head(hparams.n_head),
n_head_kv(hparams.n_head_kv),
d_head(n_embd / n_head),
n_layer(hparams.n_layer),
n_mmproj_embd(clip_n_mmproj_embd(ctx)),
@@ -461,9 +463,9 @@ ggml_tensor * clip_graph::build_vit(
}
}
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head_kv, n_pos);
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head_kv, n_pos);
if (norm_per_head) {
if (layer.q_norm) {
@@ -1012,6 +1014,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
{
builder = std::make_unique<clip_graph_deepseekocr>(ctx, img);
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
builder = std::make_unique<clip_graph_deepseekocr2>(ctx, img);
} break;
case PROJECTOR_TYPE_LFM2A:
{
builder = std::make_unique<clip_graph_conformer>(ctx, img);
@@ -1198,6 +1204,9 @@ struct clip_model_loader {
get_u32(string_format(KEY_PROJ_DIM, prefix), hparams.projection_dim);
get_f32(string_format(KEY_LAYER_NORM_EPS, prefix), hparams.eps);
// n_head_kv is optional (for GQA), default to n_head
hparams.n_head_kv = hparams.n_head;
if (is_vision) {
get_u32(KEY_IMAGE_SIZE, hparams.image_size);
get_u32(KEY_PATCH_SIZE, hparams.patch_size);
@@ -1598,6 +1607,7 @@ struct clip_model_loader {
hparams.set_warmup_n_tokens(28*28); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
hparams.patch_size = 16;
hparams.image_size = 1024;
@@ -1609,6 +1619,10 @@ struct clip_model_loader {
get_u32(KEY_SAM_N_HEAD, hparams.sam_n_head, true);
get_u32(KEY_SAM_N_EMBD, hparams.sam_n_embd, true);
get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true);
if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) {
// qwen2 encoder is GQA, requires KEY_N_HEAD_KV
get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
}
} break;
case PROJECTOR_TYPE_HUNYUANVL:
{
@@ -1640,6 +1654,9 @@ struct clip_model_loader {
hparams.audio_n_fft = 512;
hparams.audio_window_len = 320; // 20ms frame (NOT 25ms/400)
hparams.audio_hop_len = 160;
// due to a mistake in the original conversion code, rms_norm_eps is set to a wrong value
// since all gemma4a models use 1e-6, we just hardcode it here to avoid re-conversion
hparams.eps = 1e-6f;
} break;
case PROJECTOR_TYPE_GRANITE_SPEECH:
{
@@ -2460,6 +2477,7 @@ struct clip_model_loader {
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
model.pos_embed = get_tensor(string_format(TN_SAM_POS_EMBD, "weight"));
model.patch_embed_proj_w = get_tensor(string_format(TN_SAM_PATCH_EMBD, "weight"));
@@ -2490,10 +2508,12 @@ struct clip_model_loader {
model.neck_3_w = get_tensor(string_format(TN_SAM_NECK, 3, "weight"));
model.net_2 = get_tensor(string_format(TN_SAM_NET, 2, "weight"));
model.net_3 = get_tensor(string_format(TN_SAM_NET, 3, "weight"));
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false);
model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR);
model.mm_fc_w = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
model.mm_fc_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias"));
model.resample_query_768 = get_tensor(string_format(TN_RESMPL_QUERY, 768, "weight"), false);
model.resample_query_1024 = get_tensor(string_format(TN_RESMPL_QUERY, 1024, "weight"), false);
} break;
case PROJECTOR_TYPE_GEMMA4A:
{
@@ -3493,6 +3513,11 @@ void setup_init_vision_shim_kcpp(struct clip_ctx * ctx_v) {
img_end = "\n"; // prevent empty batch on llama-server
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
img_end = "\n"; // prevent empty batch on llama-server
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr2>(ctx_v);
} break;
case PROJECTOR_TYPE_HUNYUANVL:
{
// note: these use fullwidth ｜ (U+FF5C) and ▁ (U+2581) to match the tokenizer vocabulary
@@ -3789,7 +3814,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
case PROJECTOR_TYPE_DEEPSEEKOCR:
{
// SAM encoder applies two stride-2 convolutions (net_2 and net_3)
// which reduces spatial dimensions by 4x in each direction (16x total)
// that reduce spatial dimensions by 4x in each direction (16x total)
// E.g., 64x64 -> 16x16 patches
n_patches /= 16;
@@ -3805,6 +3830,15 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
int oh = (img->ny / patch_size) / merge;
n_patches = (ow + 1) * oh + 2;
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
// 1024 global view -> 256 query tokens + 1 view separator = 257;
// 768 local tile -> 144 query tokens, no separator.
n_patches /= 16;
if (img->add_viewsep) {
n_patches += 1; // view separator, appended only after the global view
}
} break;
case PROJECTOR_TYPE_LFM2A:
{
n_patches = ((((img->nx + 1) / 2) + 1) / 2 + 1) / 2;
@@ -4394,6 +4428,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
set_input_i32("pos_y", pos_y);
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
GGML_ASSERT(pos_w == pos_h);
@@ -4416,6 +4451,34 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
set_input_i32("rel_pos_indices_local", rel_pos_indices_local);
set_input_i32("rel_pos_indices_global", rel_pos_indices_global);
if (ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEKOCR2) {
// qwen2 encoder attention mask
// num_image_tokens = num_patches / 16
// 256 for 1024 global view
// 144 for 768 tile views
const int num_image_tokens = num_patches / 16;
const int seq_len = num_image_tokens * 2;
std::vector qwen2_mask(static_cast<size_t>(seq_len) * seq_len, 0.0f);
// attention mask layout
// +--------------+---------------+
// | all 0 | all -inf |
// +--------------+---------------+
// | all 0 | lower tri 0 |
// +--------------+---------------+
for (int i = 0; i < seq_len; i++) {
for (int j = 0; j < seq_len; j++) {
const bool zero = i < num_image_tokens ?
j < num_image_tokens :
j < num_image_tokens || j <= i;
qwen2_mask[static_cast<size_t>(i) * seq_len + j] = zero ? 0.0f : -1e9f;
}
}
set_input_f32("qwen2_attn_mask", qwen2_mask);
}
} break;
case PROJECTOR_TYPE_GEMMA3:
case PROJECTOR_TYPE_GEMMA3NV:
@@ -4965,6 +5028,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_COGVLM:
return ctx->model.mm_4h_to_h_w->ne[1];
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
return ctx->model.mm_fc_w->ne[1];
case PROJECTOR_TYPE_LFM2A:
return ctx->model.position_embeddings->ne[0];