Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-17 16:25:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '5a69c974392020e514c3b2b2910bb92f847cb4c9' into concedo_experimental

Browse Source 
# Conflicts:
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-opencl/kernels/concat.cl
#	ggml/src/ggml-opencl/kernels/cpy.cl
#	ggml/src/ggml-opencl/kernels/get_rows.cl
#	ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl
#	tests/test-chat.cpp
#	tools/mtmd/CMakeLists.txt
#	tools/mtmd/clip.cpp
#	tools/mtmd/clip.h
This commit is contained in:
Concedo
2026-06-07 17:46:09 +08:00
parent 202aaef40b 5a69c97439
commit e40e5c660e
30 changed files with 1301 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+217 -56
View File
@@ -61,6 +61,7 @@
#include "models/gemma4uv.cpp"
#include "models/glm4v.cpp"
#include "models/granite-speech.cpp"
#include "models/granite4-vision.cpp"
#include "models/hunyuanvl.cpp"
#include "models/internvl.cpp"
#include "models/kimivl.cpp"
@@ -1061,6 +1062,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
{
builder = std::make_unique<clip_graph_yasa2>(ctx, img);
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
builder = std::make_unique<clip_graph_granite4_vision>(ctx, img);
} break;
default:
GGML_ABORT("missing cgraph builder");
}
@@ -1321,12 +1326,7 @@ struct clip_model_loader {
// to form the final visual features.
// NOTE: gguf conversions should standardize the values of the vision feature layer to
// be non-negative, since we use -1 to mark values as unset here.
std::vector<int> vision_feature_layer;
get_arr_int(KEY_FEATURE_LAYER, vision_feature_layer, false);
// convert std::vector to std::unordered_set
for (auto & layer : vision_feature_layer) {
hparams.vision_feature_layer.insert(layer);
}
get_arr_int(KEY_FEATURE_LAYER, hparams.vision_feature_layer, false);
// model-specific params
switch (model.proj_type) {
@@ -1719,6 +1719,23 @@ struct clip_model_loader {
hparams.image_pad_color = {127, 127, 127};
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
// SigLIP tower.
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
hparams.image_resize_pad = PAD_CEIL;
get_arr_int(KEY_FEATURE_LAYER, hparams.vision_feature_layer);
get_arr_int(KEY_PROJ_SPATIAL_OFFSETS, hparams.proj_spatial_offsets);
if (hparams.vision_feature_layer.size() != hparams.proj_spatial_offsets.size()) {
throw std::runtime_error(string_format("%s: vision_feature_layer.size() %d != proj_spatial_offsets.size() %d",
hparams.vision_feature_layer.size(), hparams.proj_spatial_offsets.size()));
}
get_u32(KEY_PROJ_SAMPLE_QUERY_SIDE, hparams.downsample_query_side);
get_u32(KEY_PROJ_SAMPLE_WINDOW_SIDE, hparams.downsample_window_side);
hparams.warmup_image_size = hparams.image_size;
} break;
default:
throw std::runtime_error(string_format("%s: unknown vision projector type %s\n", __func__, proj_type.c_str()));
}
@@ -2725,47 +2742,106 @@ struct clip_model_loader {
layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"));
}
model.qf_proj_query = get_tensor(TN_QF_PROJ_QUERY);
model.qf_proj_norm_w = get_tensor(string_format(TN_QF_PROJ_NORM, "weight"));
model.qf_proj_norm_b = get_tensor(string_format(TN_QF_PROJ_NORM, "bias"));
model.qf_proj_linear_w = get_tensor(string_format(TN_QF_PROJ_LINEAR, "weight"));
model.qf_proj_linear_b = get_tensor(string_format(TN_QF_PROJ_LINEAR, "bias"));
model.qf_proj_blocks.resize(1);
auto & qf = model.qf_proj_blocks[0];
qf.qf_proj_query = get_tensor(string_format(TN_QF_PROJ_QUERY, prefix));
qf.qf_proj_norm_w = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "weight"));
qf.qf_proj_norm_b = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "bias"));
qf.qf_proj_linear_w = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "weight"));
qf.qf_proj_linear_b = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "bias"));
const int n_proj_layers = 2;
model.qf_proj_layers.resize(n_proj_layers);
qf.qf_proj_layers.resize(n_proj_layers);
for (int il = 0; il < n_proj_layers; ++il) {
auto & pl = model.qf_proj_layers[il];
auto & pl = qf.qf_proj_layers[il];
pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, il, "weight"));
pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, il, "bias"));
pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, il, "weight"));
pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, il, "bias"));
pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, il, "weight"));
pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, il, "bias"));
pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, il, "weight"));
pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, il, "bias"));
pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, il, "weight"));
pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, il, "bias"));
pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "weight"));
pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "bias"));
pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "weight"));
pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "bias"));
pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "weight"));
pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "bias"));
pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "weight"));
pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "bias"));
pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "weight"));
pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "bias"));
pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, il, "weight"));
pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, il, "bias"));
pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, il, "weight"));
pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, il, "bias"));
pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, il, "weight"));
pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, il, "bias"));
pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, il, "weight"));
pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, il, "bias"));
pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, il, "weight"));
pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, il, "bias"));
pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "weight"));
pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "bias"));
pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "weight"));
pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "bias"));
pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "weight"));
pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "bias"));
pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "weight"));
pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "bias"));
pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "weight"));
pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "bias"));
pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, il, "weight"));
pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, il, "bias"));
pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, il, "weight"));
pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, il, "bias"));
pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, il, "weight"));
pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, il, "bias"));
pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, prefix, il, "weight"));
pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, prefix, il, "bias"));
pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "weight"));
pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "bias"));
pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "weight"));
pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "bias"));
}
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
// image_newline lives at the top-level.
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
// Load separate layerwise and spatial projector tensors
const auto projector_count = hparams.vision_feature_layer.size();
model.qf_proj_blocks.resize(projector_count);
for (size_t bid = 0; bid < projector_count; ++bid) {
auto & b = model.qf_proj_blocks[bid];
// non-layerwise tensors
b.qf_proj_img_pos = get_tensor(string_format(TN_MULTI_PROJ_IMG_POS, bid));
b.qf_proj_query = get_tensor(string_format(TN_MULTI_PROJ_QUERY, prefix, bid));
b.qf_proj_linear_w = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "weight"));
b.qf_proj_linear_b = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "bias"));
b.qf_proj_norm_w = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "weight"));
b.qf_proj_norm_b = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "bias"));
b.qf_proj_post_norm_w = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "weight"));
b.qf_proj_post_norm_b = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "bias"));
// laywerwise tensors
// NOTE: If any model uses multi-layer qformers, this will need to change
b.qf_proj_layers.resize(1);
auto & pl = b.qf_proj_layers[0];
pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "weight"));
pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "bias"));
pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "weight"));
pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "bias"));
pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "weight"));
pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "bias"));
pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "weight"));
pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "bias"));
pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "weight"));
pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "bias"));
pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "weight"));
pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "bias"));
pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "weight"));
pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "bias"));
pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "weight"));
pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "bias"));
pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "weight"));
pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "bias"));
pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "weight"));
pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "bias"));
pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, prefix, bid, "weight"));
pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, prefix, bid, "bias"));
pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "weight"));
pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "bias"));
pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "weight"));
pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "bias"));
}
} break;
default:
GGML_ASSERT(false && "unknown projector type");
}
@@ -3182,18 +3258,6 @@ void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny
memcpy(img->buf.data(), rgb_pixels, img->buf.size());
}
bool clip_image_load_from_file(const char * fname, clip_image_u8 * img) {
int nx, ny, nc;
auto * data = stbi_load(fname, &nx, &ny, &nc, 3);
if (!data) {
LOG_ERR("%s: failed to load image '%s'\n", __func__, fname);
return false;
}
clip_build_img_from_pixels(data, nx, ny, img);
stbi_image_free(data);
return true;
}
//note that the memory here must be subsequently freed!
uint8_t* make_new_letterbox_img(uint8_t* input_image, int nx, int ny, int nc, int target_width, int target_height) {
int new_image_size = (target_width * target_height * nc) + 512; //add some padding
@@ -3591,6 +3655,12 @@ void setup_init_vision_shim_kcpp(struct clip_ctx * ctx_v) {
img_end = "</vision>";
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
img_beg = "<image>";
img_end = "";
image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);
} break;
default:
throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj));
}
@@ -3621,10 +3691,6 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, str
return true;
}
ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx) {
return ctx->model.image_newline;
}
void clip_free(clip_ctx * ctx) {
if (ctx == nullptr) {
return;
@@ -3933,6 +3999,23 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
const int ds = ctx->model.hparams.audio_proj_downsample_rate;
n_patches = ((img->nx + ws - 1) / ws) * (ws / ds);
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
// Per-tile output token count: each projector block outputs
// query_side^2 tokens per window × n^2 windows.
// For 384×384 input: n = 24/8 = 3, query_side = 4 → 144.
const int window_side = ctx->model.hparams.downsample_window_side;
const int query_side = ctx->model.hparams.downsample_query_side;
const int side = img->nx / params.patch_size;
const int n = side / window_side;
n_patches = (query_side * n) * (query_side * n);
if (img->add_newline) {
// For single-tile case: append 1 newline row.
// For multi-tile rowwise: handled by caller, but here we
// report the per-tile count including one trailing newline.
n_patches += 1;
}
} break;
default:
GGML_ABORT("unsupported projector type");
}
@@ -4765,6 +4848,82 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
set_input_f32("attn_mask", mask);
}
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
// Granite Vision 4.1 uses precomputed permutation index
// tensors to express the _win / _unwin / spatial sampling
// reshapes as ggml_get_rows gathers. The names are set
// by g4v_gather() in models/granite4-vision.cpp.
const int patch_size = model.hparams.patch_size;
const int image_side = imgs.entries.front()->nx / patch_size;
const int window_side = hparams.downsample_window_side;
const int query_side = hparams.downsample_query_side;
const int n = image_side / window_side;
const int new_side = n * query_side;
// Builds the raster→window permutation indices for a
// (side, side) grid split into (n × n) windows of (win × win)
// tokens each. dst[w * win*win + p] = source raster index.
auto make_win_idx = [](int side, int win) {
const int nn = side / win;
std::vector<int32_t> idx(static_cast<size_t>(side) * side);
for (int wy = 0; wy < nn; ++wy) {
for (int wx = 0; wx < nn; ++wx) {
for (int iy = 0; iy < win; ++iy) {
for (int ix = 0; ix < win; ++ix) {
const int w = wy * nn + wx;
const int p = iy * win + ix;
const int y = wy * win + iy;
const int x = wx * win + ix;
idx[static_cast<size_t>(w) * (win*win) + p] = y * side + x;
}
}
}
}
return idx;
};
auto make_unwin_idx = [&](int side, int win) {
const std::vector<int32_t> fwd = make_win_idx(side, win);
std::vector<int32_t> inv(fwd.size());
for (size_t i = 0; i < fwd.size(); ++i) {
inv[fwd[i]] = static_cast<int32_t>(i);
}
return inv;
};
auto make_spatial_idx = [](int side, int offset) {
const int off_y = (offset >> 1) & 1;
const int off_x = offset & 1;
const int new_s = side / 2;
std::vector<int32_t> idx(static_cast<size_t>(new_s) * new_s);
for (int y = 0; y < new_s; ++y) {
for (int x = 0; x < new_s; ++x) {
idx[y * new_s + x] = (y * 2 + off_y) * side + (x * 2 + off_x);
}
}
return idx;
};
auto upload = [&](const std::string & name, const std::vector<int32_t> & idx) {
ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str());
GGML_ASSERT(t);
ggml_backend_tensor_set(t, idx.data(), 0, idx.size() * sizeof(int32_t));
};
// Stage 1b only uses block 0's permutations; future stages
// will upload all blocks.
for (size_t bid = 0; bid < hparams.vision_feature_layer.size(); ++bid) {
const std::string prefix = "g4v_blk" + std::to_string(bid) + "_";
upload(prefix + "win_idx", make_win_idx(image_side, window_side));
upload(prefix + "qwin_idx", make_win_idx(new_side, query_side));
upload(prefix + "unwin_idx", make_unwin_idx(new_side, query_side));
const auto spatial_offset = hparams.proj_spatial_offsets[bid];
if (spatial_offset >= 0) {
upload(prefix + "spatial_idx", make_spatial_idx(image_side,spatial_offset));
}
}
} break;
default:
GGML_ABORT("Unknown projector type");
}
@@ -5117,7 +5276,9 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_LFM2A:
return ctx->model.position_embeddings->ne[0];
case PROJECTOR_TYPE_GRANITE_SPEECH:
return ctx->model.qf_proj_linear_w->ne[1];
return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1];
case PROJECTOR_TYPE_GRANITE4_VISION:
return ctx->model.qf_proj_blocks.size() * ctx->model.hparams.projection_dim;
case PROJECTOR_TYPE_GLM4V:
return ctx->model.mm_ffn_down_w->ne[1];
default: