Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 09:15:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	docs/backend/SYCL.md
#	docs/backend/snapdragon/developer.md
#	examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp
#	examples/sycl/run-llama2.sh
#	examples/sycl/start-svr.sh
#	examples/sycl/test.sh
#	examples/sycl/win-run-llama2.bat
#	examples/sycl/win-start-svr.bat
#	examples/sycl/win-test.bat
#	ggml/CMakeLists.txt
#	ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c
#	ggml/src/ggml-et/ggml-et-ops.cpp
#	ggml/src/ggml-et/ggml-et-ops.h
#	ggml/src/ggml-sycl/ssm_scan.cpp
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl
#	scripts/bench-models.sh
#	scripts/snapdragon/adb/run-bench.sh
#	scripts/snapdragon/adb/run-cli.sh
#	scripts/snapdragon/adb/run-completion.sh
#	scripts/snapdragon/adb/run-mtmd.sh
#	scripts/snapdragon/windows/run-bench.ps1
#	scripts/snapdragon/windows/run-cli.ps1
#	scripts/snapdragon/windows/run-completion.ps1
#	scripts/snapdragon/windows/run-mtmd.ps1
#	scripts/sync-ggml.last
#	scripts/sync_vendor.py
#	tests/CMakeLists.txt
#	tests/test-backend-ops.cpp
#	tests/test-chat.cpp
#	tests/test-jinja.cpp
#	tests/test-llama-archs.cpp
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/llama-bench/README.md
#	tools/server/README.md
This commit is contained in:
Concedo
2026-08-15 22:50:04 +08:00
parent 531cac9cd3 adb55e5148
commit 82e15c08bf
60 changed files with 1781 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+46 -18
View File
@@ -1680,6 +1680,9 @@ struct clip_model_loader {
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
hparams.image_resize_pad = PAD_NONE;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
// n_merge is used as a divisor in clip_image_batch_encode
// (gh / n_merge); reject 0 to avoid int div-by-zero (DoS).
GGML_ASSERT(hparams.n_merge > 0);
hparams.rope_theta = 10000.0f; // vision_config.rope_theta
// MiniMax-M3: max_pixels 451584 (=672^2) -> 576 merged tokens (image_seq_length)
hparams.set_limit_image_tokens(8, 576);
@@ -1908,7 +1911,9 @@ struct clip_model_loader {
// unlimited-ocr shares the v1 projector but tiles up to 32
get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false);
get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false);
GGML_ASSERT(hparams.preproc_min_tiles <= hparams.preproc_max_tiles);
GGML_ASSERT(hparams.preproc_min_tiles >= 0
&& hparams.preproc_min_tiles <= hparams.preproc_max_tiles
&& hparams.preproc_max_tiles <= 256);
} break;
case PROJECTOR_TYPE_HUNYUANVL:
{
@@ -1973,6 +1978,9 @@ struct clip_model_loader {
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
get_u32(KEY_A_CHUNK_SIZE, hparams.audio_chunk_size);
// context_size is squared for the attn_dists/mask buffers; cap to prevent int32 overflow
// (legitimate values are small, e.g. 12-200; 8192^2 = 67M still fits int32)
GGML_ASSERT(hparams.audio_chunk_size > 0 && hparams.audio_chunk_size <= 8192);
get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size);
get_u32(KEY_A_MAX_POS_EMB, hparams.audio_max_pos_emb);
get_u32(KEY_A_PROJ_WINDOW_SIZE, hparams.audio_proj_window_size);
@@ -2012,8 +2020,9 @@ struct clip_model_loader {
// note: some models having hparams.image_size == 0, which means the image size is dynamic
throw std::runtime_error(string_format("%s: image_size (%d) cannot be negative\n", __func__, hparams.image_size));
}
if (hparams.image_size > 65536) {
throw std::runtime_error(string_format("%s: image_size (%d) is too large (max 65536)\n", __func__, hparams.image_size));
if (hparams.image_size > 8192) {
// cap prevents int32 overflow in n_patches = (image_size/patch_size)^2
throw std::runtime_error(string_format("%s: image_size (%d) is too large (max 8192)\n", __func__, hparams.image_size));
}
if (hparams.patch_size <= 0 || hparams.patch_size >= 65536) {
throw std::runtime_error(string_format("%s: patch_size (%d) must be positive and less than 65536\n", __func__, hparams.patch_size));
@@ -2024,9 +2033,12 @@ struct clip_model_loader {
if (hparams.image_max_pixels < hparams.image_min_pixels) {
throw std::runtime_error(string_format("%s: image_max_pixels (%d) is less than image_min_pixels (%d)\n", __func__, hparams.image_max_pixels, hparams.image_min_pixels));
}
if (hparams.n_merge < 0 || hparams.n_merge >= 65536) {
if (hparams.n_merge <= 0 || hparams.n_merge >= 65536) {
throw std::runtime_error(string_format("%s: n_merge (%d) must be greater than 0 and less than 65536\n", __func__, hparams.n_merge));
}
if (hparams.attn_window_size > 4096) {
throw std::runtime_error(string_format("%s: attn_window_size (%d) is too large (max 4096)\n", __func__, hparams.attn_window_size));
}
}
LOG_INF("%s: projector: %s\n", __func__, proj_type.c_str());
@@ -3824,6 +3836,9 @@ struct clip_model_loader {
}
return;
}
if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) {
throw std::runtime_error(string_format("%s: key '%s' is not an array\n", __func__, key.c_str()));
}
const auto type = gguf_get_arr_type(ctx_gguf.get(), i);
if (type != GGUF_TYPE_FLOAT32) {
throw std::runtime_error(string_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_FLOAT32)\n", __func__, key.c_str(), type, GGUF_TYPE_FLOAT32));
@@ -3858,6 +3873,9 @@ struct clip_model_loader {
}
return;
}
if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) {
throw std::runtime_error(string_format("%s: key '%s' is not an array\n", __func__, key.c_str()));
}
const auto type = gguf_get_arr_type(ctx_gguf.get(), i);
if (type != GGUF_TYPE_INT32) {
throw std::runtime_error(string_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_INT32)\n", __func__, key.c_str(), type, GGUF_TYPE_INT32));
@@ -4307,18 +4325,20 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_GRANITE4_VISION:
{
// Per-tile output token count: each projector block outputs
// query_side^2 tokens per window × n^2 windows.
// For 384×384 input: n = 24/8 = 3, query_side = 4 → 144.
// query_side^2 tokens per window x n^2 windows.
// For 384x384 input: n = 24/8 = 3, query_side = 4 -> 144.
const int window_side = ctx->model.hparams.downsample_window_side;
const int query_side = ctx->model.hparams.downsample_query_side;
const int side = img->nx() / params.patch_size;
const int n = side / window_side;
n_patches = (query_side * n) * (query_side * n);
if (img->add_newline) {
// For single-tile case: append 1 newline row.
// For multi-tile rowwise: handled by caller, but here we
// report the per-tile count including one trailing newline.
n_patches += 1;
const int out_side = query_side * n;
n_patches = out_side * out_side;
if (img->anyres.is_tiled()) {
// overview tile, then the unpadded tile grid with one newline per row
int off_x, off_y, w, h;
clip_anyres_unpad(img->anyres.grid_x * out_side, img->anyres.grid_y * out_side,
img->anyres.orig_nx, img->anyres.orig_ny, off_x, off_y, w, h);
n_patches += h * (w + 1);
}
} break;
default:
@@ -5498,13 +5518,13 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
const int context_size = ctx->model.hparams.audio_chunk_size;
const int max_pos_emb = ctx->model.hparams.audio_max_pos_emb;
std::vector<int32_t> dists(context_size * context_size);
std::vector<int32_t> dists((size_t) context_size * (size_t) context_size);
for (int i = 0; i < context_size; i++) {
for (int j = 0; j < context_size; j++) {
int d = i - j;
if (d < -context_size) d = -context_size;
if (d > context_size) d = context_size;
dists[i * context_size + j] = d + max_pos_emb;
dists[(size_t) i * (size_t) context_size + (size_t) j] = d + max_pos_emb;
}
}
set_input_i32("attn_dists", dists);
@@ -5513,13 +5533,13 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
const int remainder = n_frames % context_size;
if (remainder > 0) {
const int num_blocks = (n_frames + context_size - 1) / context_size;
std::vector<float> mask(context_size * context_size * num_blocks, 0.0f);
std::vector<float> mask((size_t) context_size * (size_t) context_size * (size_t) num_blocks, 0.0f);
const float neg_inf = -INFINITY;
const int last_block_offset = (num_blocks - 1) * context_size * context_size;
const size_t last_block_offset = (size_t) (num_blocks - 1) * (size_t) context_size * (size_t) context_size;
for (int q = 0; q < context_size; q++) {
for (int k = 0; k < context_size; k++) {
if (q >= remainder || k >= remainder) {
mask[last_block_offset + q * context_size + k] = neg_inf;
mask[last_block_offset + (size_t) q * (size_t) context_size + (size_t) k] = neg_inf;
}
}
}
@@ -5583,10 +5603,18 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
return idx;
};
// the same permutation is applied to every tile of the stacked image
auto upload = [&](const std::string & name, const std::vector<int32_t> & idx) {
ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str());
GGML_ASSERT(t);
ggml_backend_tensor_set(t, idx.data(), 0, idx.size() * sizeof(int32_t));
GGML_ASSERT(ggml_nelements(t) % (int64_t) idx.size() == 0);
const int n_rep = ggml_nelements(t) / idx.size();
std::vector<int32_t> buf;
buf.reserve(idx.size() * n_rep);
for (int i = 0; i < n_rep; ++i) {
buf.insert(buf.end(), idx.begin(), idx.end());
}
ggml_backend_tensor_set(t, buf.data(), 0, ggml_nbytes(t));
};
// Stage 1b only uses block 0's permutations; future stages