Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 09:15:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/build-android.yml
#	.github/workflows/build-apple.yml
#	.github/workflows/build-cpu.yml
#	.github/workflows/build-cuda-ubuntu.yml
#	.github/workflows/build-cuda-windows.yml
#	.github/workflows/build-msys.yml
#	.github/workflows/build-opencl.yml
#	.github/workflows/build-openvino.yml
#	.github/workflows/build-riscv.yml
#	.github/workflows/build-sanitize.yml
#	.github/workflows/build-sycl.yml
#	.github/workflows/build-vulkan.yml
#	.github/workflows/build-wasm.yml
#	.github/workflows/build-webgpu.yml
#	.github/workflows/copilot-setup-steps.yml
#	.github/workflows/hip-quality-check.yml
#	.github/workflows/release.yml
#	.github/workflows/server.yml
#	CODEOWNERS
#	build-xcframework.sh
#	docs/android.md
#	docs/backend/SYCL.md
#	docs/build.md
#	ggml/CMakeLists.txt
#	ggml/src/ggml-cpu/CMakeLists.txt
#	ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp-opnode.h
#	ggml/src/ggml-hexagon/htp/cpy-ops.c
#	ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
#	ggml/src/ggml-hexagon/htp/get-rows-ops.c
#	ggml/src/ggml-hexagon/htp/htp-ctx.h
#	ggml/src/ggml-hexagon/htp/htp-ops.h
#	ggml/src/ggml-hexagon/htp/main.c
#	ggml/src/ggml-hexagon/htp/matmul-ops.c
#	ggml/src/ggml-hexagon/htp/matmul-ops.h
#	ggml/src/ggml-hexagon/htp/set-rows-ops.c
#	ggml/src/ggml-metal/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	tests/CMakeLists.txt
#	tests/test-backend-ops.cpp
#	tests/test-llama-archs.cpp
#	tests/test-save-load-state.cpp
#	tools/mtmd/CMakeLists.txt
This commit is contained in:
Concedo
2026-09-02 23:24:09 +08:00
parent 3bf2e14605 7798007a29
commit 2e04e6509d
74 changed files with 4125 additions and 611 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+120 -5
View File
@@ -81,6 +81,7 @@
#include "models/whisper-enc.cpp"
#include "models/deepseekocr.cpp"
#include "models/deepseekocr2.cpp"
#include "models/deepseek4v.cpp"
#include "models/mobilenetv5.cpp"
#include "models/youtuvl.cpp"
#include "models/yasa2.cpp"
@@ -1095,6 +1096,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_kimik25>(ctx, img);
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
builder = std::make_unique<clip_graph_deepseek4v>(ctx, img);
} break;
case PROJECTOR_TYPE_COGVLM:
{
builder = std::make_unique<clip_graph_cogvlm>(ctx, img);
@@ -1666,6 +1671,31 @@ struct clip_model_loader {
hparams.set_limit_image_tokens(2, 4096);
}
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_pad_color = {127, 127, 127};
hparams.rope_theta = 10000.0f;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
hparams.dsv4_max_n_token = 384;
hparams.dsv4_max_wh_ratio = 8;
const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
// handle min/max token counts from CLI
if (hparams.custom_image_min_tokens > 0) {
hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area;
}
if (hparams.custom_image_max_tokens > 0) {
// the cap is on the whole token block, keep some room for the resize solver
hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16);
}
hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area;
// a small custom max token count also lowers the min-pixel upscale threshold
hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels);
// avoid OOM on warmup
const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token));
hparams.set_warmup_n_tokens(warmup_side * warmup_side);
} break;
case PROJECTOR_TYPE_GEMMA3:
{
// default value (used by all model sizes in gemma 3 family)
@@ -2805,6 +2835,18 @@ struct clip_model_loader {
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
// sentinel token embeddings written into the output block
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
model.token_embd_img_start = get_tensor(TN_TOK_IMG_START);
model.token_embd_img_end = get_tensor(TN_TOK_IMG_END);
model.token_embd_img_pad = get_tensor(TN_TOK_IMG_PAD);
} break;
case PROJECTOR_TYPE_PIXTRAL:
{
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
@@ -3079,9 +3121,9 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
// code_predictor
model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight"));
model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias"));
// code_predictor, proj_in is absent when the talker and the predictor share the hidden size
model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight"), false);
model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias"), false);
model.gen_code_embd_w = get_tensor(string_format(TN_A_GEN_CODE_EMBD, "weight"));
model.gen_code_head_w = get_tensor(string_format(TN_A_GEN_CODE_HEAD, "weight"));
model.gen_code_out_embd_w = get_tensor(string_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
@@ -3252,8 +3294,8 @@ struct clip_model_loader {
model.mm_model_proj_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias"));
model.mm_pre_norm_w = get_tensor(string_format(TN_MM_PRE_NORM, "weight"));
model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
model.mm_img_begin = get_tensor(TN_TOK_IMG_BEGIN);
model.mm_img_end = get_tensor(TN_TOK_IMG_END);
model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN);
model.mm_img_end = get_tensor(TN_MM_IMG_END);
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false);
} break;
@@ -4241,6 +4283,13 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
int y_patch = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = x_patch * y_patch;
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
const int out_patch_size = params.patch_size * params.n_merge;
const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out;
} break;
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
@@ -5112,6 +5161,58 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
set_input_i32("pos_w", pos_data);
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
// set the 2D positions (mrope layout, only the first 2 channels are used)
int n_patches_per_row = image_size_width / patch_size;
std::vector<int32_t> positions(n_pos * 4, 0);
for (int i = 0; i < n_pos; i++) {
positions[i] = i / n_patches_per_row; // row
positions[n_pos + i] = i % n_patches_per_row; // col
}
set_input_i32("positions", positions);
// token block layout index (see clip_graph_deepseek4v::build)
// rows [0, n_grid) are the aligner output, the sentinels follow
const int n_merge = hparams.n_merge;
const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge;
const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge;
const int n_grid = n_llm_w * n_llm_h;
const int idx_start = n_grid;
const int idx_end = n_grid + 1;
const int idx_newline = n_grid + 2;
const int idx_pad = n_grid + 3;
const int lead_pad = imgs.entries[0].lead_pad;
const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad);
std::vector<int32_t> idx;
idx.reserve(bl.n_out);
for (int i = 0; i < lead_pad; i++) {
idx.push_back(idx_pad);
}
idx.push_back(idx_start);
// pairs of adjacent rows are interleaved column-wise ("N-layout")
// ref: build_image_block in inference/image_processor.py
for (int t = 0; t < bl.rows * bl.row_len; t++) {
const int g = t / (2 * bl.row_len);
const int rem = t % (2 * bl.row_len);
const int c = rem / 2; // column
const int r = 2 * g + rem % 2; // row
if (r >= n_llm_h) {
idx.push_back(idx_pad);
} else if (c == n_llm_w) {
idx.push_back(idx_newline);
} else {
idx.push_back(r * n_llm_w + c);
}
}
for (int i = 0; i < bl.pad_last; i++) {
idx.push_back(idx_pad);
}
idx.push_back(idx_end);
set_input_i32("layout_idx", idx);
} break;
case PROJECTOR_TYPE_GLM_EDGE:
{
// llava and other models
@@ -5853,6 +5954,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n");
LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens);
// TEMP debugging (parity validation), will be removed before merge
// when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data]
const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS");
if (dump_path && strcmp(dump_path, "1") != 0) {
FILE * f = fopen(dump_path, "wb");
if (f) {
const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd };
fwrite(hdr, sizeof(hdr), 1, f);
fwrite(emb_data.data(), sizeof(float), emb_data.size(), f);
fclose(f);
}
}
// Print first few values of first token
LOG_INF("Token 0 (first 16 values): ");
for (int i = 0; i < std::min((int64_t)16, n_embd); i++) {
@@ -5957,6 +6071,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_KIMIK25:
case PROJECTOR_TYPE_YASA2:
case PROJECTOR_TYPE_DEEPSEEK4V:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_HUNYUANVL:
return ctx->model.mm_model_proj->ne[1];