mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-19 17:25:07 +02:00
Merge commit '5a69c974392020e514c3b2b2910bb92f847cb4c9' into concedo_experimental
# Conflicts: # ggml/src/ggml-opencl/ggml-opencl.cpp # ggml/src/ggml-opencl/kernels/concat.cl # ggml/src/ggml-opencl/kernels/cpy.cl # ggml/src/ggml-opencl/kernels/get_rows.cl # ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl # tests/test-chat.cpp # tools/mtmd/CMakeLists.txt # tools/mtmd/clip.cpp # tools/mtmd/clip.h
This commit is contained in:
Concedo
+42
-31
@@ -35,20 +35,22 @@
|
||||
#define KEY_LAYER_NORM_EPS "clip.%s.attention.layer_norm_epsilon"
|
||||
|
||||
// vision-specific
|
||||
#define KEY_VISION_PROJ_TYPE "clip.vision.projector_type" // for models with mixed modalities
|
||||
#define KEY_IMAGE_SIZE "clip.vision.image_size"
|
||||
#define KEY_IMAGE_MIN_PIXELS "clip.vision.image_min_pixels"
|
||||
#define KEY_IMAGE_MAX_PIXELS "clip.vision.image_max_pixels"
|
||||
#define KEY_PREPROC_MIN_TILES "clip.vision.preproc_min_tiles"
|
||||
#define KEY_PREPROC_MAX_TILES "clip.vision.preproc_max_tiles"
|
||||
#define KEY_PREPROC_IMAGE_SIZE "clip.vision.preproc_image_size"
|
||||
#define KEY_PATCH_SIZE "clip.vision.patch_size"
|
||||
#define KEY_IMAGE_MEAN "clip.vision.image_mean"
|
||||
#define KEY_IMAGE_STD "clip.vision.image_std"
|
||||
#define KEY_FEATURE_LAYER "clip.vision.feature_layer"
|
||||
#define KEY_PROJ_SCALE_FACTOR "clip.vision.projector.scale_factor"
|
||||
#define KEY_SPATIAL_MERGE_SIZE "clip.vision.spatial_merge_size"
|
||||
#define KEY_IS_DEEPSTACK_LAYERS "clip.vision.is_deepstack_layers"
|
||||
#define KEY_VISION_PROJ_TYPE "clip.vision.projector_type" // for models with mixed modalities
|
||||
#define KEY_IMAGE_SIZE "clip.vision.image_size"
|
||||
#define KEY_IMAGE_MIN_PIXELS "clip.vision.image_min_pixels"
|
||||
#define KEY_IMAGE_MAX_PIXELS "clip.vision.image_max_pixels"
|
||||
#define KEY_PREPROC_MIN_TILES "clip.vision.preproc_min_tiles"
|
||||
#define KEY_PREPROC_MAX_TILES "clip.vision.preproc_max_tiles"
|
||||
#define KEY_PREPROC_IMAGE_SIZE "clip.vision.preproc_image_size"
|
||||
#define KEY_PATCH_SIZE "clip.vision.patch_size"
|
||||
#define KEY_IMAGE_MEAN "clip.vision.image_mean"
|
||||
#define KEY_IMAGE_STD "clip.vision.image_std"
|
||||
#define KEY_FEATURE_LAYER "clip.vision.feature_layer"
|
||||
#define KEY_PROJ_SCALE_FACTOR "clip.vision.projector.scale_factor"
|
||||
#define KEY_PROJ_SAMPLE_QUERY_SIDE "clip.vision.projector.query_side"
|
||||
#define KEY_PROJ_SAMPLE_WINDOW_SIDE "clip.vision.projector.window_side"
|
||||
#define KEY_PROJ_SPATIAL_OFFSETS "clip.vision.projector.spatial_offsets"
|
||||
#define KEY_SPATIAL_MERGE_SIZE "clip.vision.spatial_merge_size"
|
||||
|
||||
#define KEY_MM_PATCH_MERGE_TYPE "clip.vision.mm_patch_merge_type"
|
||||
#define KEY_IMAGE_GRID_PINPOINTS "clip.vision.image_grid_pinpoints"
|
||||
@@ -72,7 +74,6 @@
|
||||
#define KEY_A_PROJ_DOWNSAMPLE_RATE "clip.audio.projector.downsample_rate"
|
||||
#define KEY_A_PROJ_HEAD_COUNT "clip.audio.projector.head_count"
|
||||
|
||||
|
||||
//
|
||||
// tensor name constants
|
||||
//
|
||||
@@ -210,22 +211,28 @@
|
||||
#define TN_CTC_OUT_MID "a.enc_ctc_out_mid.%s"
|
||||
#define TN_ATTN_REL_POS_EMB "%s.blk.%d.attn_rel_pos_emb"
|
||||
// qformer projector
|
||||
#define TN_QF_PROJ_QUERY "a.proj_query"
|
||||
#define TN_QF_PROJ_NORM "a.proj_norm.%s"
|
||||
#define TN_QF_PROJ_LINEAR "a.proj_linear.%s"
|
||||
#define TN_QF_SELF_ATTN_Q "a.proj_blk.%d.self_attn_q.%s"
|
||||
#define TN_QF_SELF_ATTN_K "a.proj_blk.%d.self_attn_k.%s"
|
||||
#define TN_QF_SELF_ATTN_V "a.proj_blk.%d.self_attn_v.%s"
|
||||
#define TN_QF_SELF_ATTN_O "a.proj_blk.%d.self_attn_out.%s"
|
||||
#define TN_QF_SELF_ATTN_N "a.proj_blk.%d.self_attn_norm.%s"
|
||||
#define TN_QF_CROSS_ATTN_Q "a.proj_blk.%d.cross_attn_q.%s"
|
||||
#define TN_QF_CROSS_ATTN_K "a.proj_blk.%d.cross_attn_k.%s"
|
||||
#define TN_QF_CROSS_ATTN_V "a.proj_blk.%d.cross_attn_v.%s"
|
||||
#define TN_QF_CROSS_ATTN_O "a.proj_blk.%d.cross_attn_out.%s"
|
||||
#define TN_QF_CROSS_ATTN_N "a.proj_blk.%d.cross_attn_norm.%s"
|
||||
#define TN_QF_FFN_UP "a.proj_blk.%d.ffn_up.%s"
|
||||
#define TN_QF_FFN_DOWN "a.proj_blk.%d.ffn_down.%s"
|
||||
#define TN_QF_FFN_NORM "a.proj_blk.%d.ffn_norm.%s"
|
||||
#define TN_QF_PROJ_QUERY "%s.proj_query"
|
||||
#define TN_QF_PROJ_NORM "%s.proj_norm.%s"
|
||||
#define TN_QF_PROJ_LINEAR "%s.proj_linear.%s"
|
||||
#define TN_QF_SELF_ATTN_Q "%s.proj_blk.%d.self_attn_q.%s"
|
||||
#define TN_QF_SELF_ATTN_K "%s.proj_blk.%d.self_attn_k.%s"
|
||||
#define TN_QF_SELF_ATTN_V "%s.proj_blk.%d.self_attn_v.%s"
|
||||
#define TN_QF_SELF_ATTN_O "%s.proj_blk.%d.self_attn_out.%s"
|
||||
#define TN_QF_SELF_ATTN_N "%s.proj_blk.%d.self_attn_norm.%s"
|
||||
#define TN_QF_CROSS_ATTN_Q "%s.proj_blk.%d.cross_attn_q.%s"
|
||||
#define TN_QF_CROSS_ATTN_K "%s.proj_blk.%d.cross_attn_k.%s"
|
||||
#define TN_QF_CROSS_ATTN_V "%s.proj_blk.%d.cross_attn_v.%s"
|
||||
#define TN_QF_CROSS_ATTN_O "%s.proj_blk.%d.cross_attn_out.%s"
|
||||
#define TN_QF_CROSS_ATTN_N "%s.proj_blk.%d.cross_attn_norm.%s"
|
||||
#define TN_QF_FFN_UP "%s.proj_blk.%d.ffn_up.%s"
|
||||
#define TN_QF_FFN_DOWN "%s.proj_blk.%d.ffn_down.%s"
|
||||
#define TN_QF_FFN_NORM "%s.proj_blk.%d.ffn_norm.%s"
|
||||
// multi-projector qformer (bid => projector ID)
|
||||
#define TN_MULTI_PROJ_IMG_POS "v.proj_blk.%d.img_pos"
|
||||
#define TN_MULTI_PROJ_QUERY "%s.proj_blk.%d.query"
|
||||
#define TN_MULTI_PROJ_LINEAR "%s.proj_blk.%d.linear.%s"
|
||||
#define TN_MULTI_PROJ_NORM "%s.proj_blk.%d.norm.%s"
|
||||
#define TN_MULTI_PROJ_POST_NORM "%s.proj_blk.%d.post_norm.%s"
|
||||
|
||||
// gemma4 audio conformer
|
||||
#define TN_A_MM_INP_PROJ "mm.a.input_projection.%s"
|
||||
@@ -354,6 +361,7 @@ enum projector_type {
|
||||
PROJECTOR_TYPE_MINICPMV4_6,
|
||||
PROJECTOR_TYPE_GRANITE_SPEECH,
|
||||
PROJECTOR_TYPE_MIMOVL,
|
||||
PROJECTOR_TYPE_GRANITE4_VISION,
|
||||
PROJECTOR_TYPE_UNKNOWN,
|
||||
};
|
||||
|
||||
@@ -407,6 +415,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
|
||||
{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},
|
||||
{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},
|
||||
{ PROJECTOR_TYPE_MIMOVL, "mimovl"},
|
||||
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
|
||||
};
|
||||
|
||||
static projector_type clip_projector_type_from_string(const std::string & str) {
|
||||
@@ -438,6 +447,8 @@ struct clip_image_f32 {
|
||||
|
||||
// marks the global view in e.g., DeepSeek-OCR Models
|
||||
bool add_viewsep = false;
|
||||
// whether a learned newline token should be appended after the image (eg Granite4 Vision)
|
||||
bool add_newline = false;
|
||||
};
|
||||
|
||||
//
|
||||
|
||||
+27
-8
@@ -4,6 +4,7 @@
|
||||
#include "clip.h"
|
||||
#include "clip-impl.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <vector>
|
||||
#include <unordered_set>
|
||||
@@ -90,7 +91,7 @@ struct clip_hparams {
|
||||
|
||||
float eps = 1e-6;
|
||||
float rope_theta = 0.0;
|
||||
std::unordered_set<int32_t> vision_feature_layer;
|
||||
std::vector<int32_t> vision_feature_layer;
|
||||
int32_t attn_window_size = 0;
|
||||
int32_t n_wa_pattern = 0;
|
||||
std::unordered_set<int32_t> wa_layer_indexes; // explicit layer indexes that use full attention (for irregular patterns like YoutuVL)
|
||||
@@ -101,6 +102,11 @@ struct clip_hparams {
|
||||
int32_t sam_n_head = 0;
|
||||
int32_t sam_n_embd = 0;
|
||||
|
||||
// Granite4 Vision
|
||||
std::vector<int32_t> proj_spatial_offsets;
|
||||
int32_t downsample_query_side;
|
||||
int32_t downsample_window_side;
|
||||
|
||||
// audio
|
||||
int32_t n_mel_bins = 0; // whisper preprocessor
|
||||
int32_t proj_stack_factor = 0; // ultravox
|
||||
@@ -158,6 +164,10 @@ struct clip_hparams {
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool is_vision_feature_layer(int32_t layer) const {
|
||||
return std::find(vision_feature_layer.begin(), vision_feature_layer.end(), layer) != vision_feature_layer.end();
|
||||
}
|
||||
};
|
||||
|
||||
struct clip_layer {
|
||||
@@ -325,6 +335,20 @@ struct yasa2_stage {
|
||||
std::vector<yasa2_block> blocks;
|
||||
};
|
||||
|
||||
// QFormer projector block for models with 1 (or more) QFormer projectors
|
||||
// Granite Speech, Granite4 Vision
|
||||
struct qf_block {
|
||||
ggml_tensor * qf_proj_query = nullptr;
|
||||
ggml_tensor * qf_proj_norm_w = nullptr;
|
||||
ggml_tensor * qf_proj_norm_b = nullptr;
|
||||
ggml_tensor * qf_proj_linear_w = nullptr;
|
||||
ggml_tensor * qf_proj_linear_b = nullptr;
|
||||
ggml_tensor * qf_proj_post_norm_w = nullptr;
|
||||
ggml_tensor * qf_proj_post_norm_b = nullptr;
|
||||
ggml_tensor * qf_proj_img_pos = nullptr; // Vision only
|
||||
std::vector<clip_layer> qf_proj_layers;
|
||||
};
|
||||
|
||||
struct clip_model {
|
||||
clip_modality modality = CLIP_MODALITY_VISION;
|
||||
projector_type proj_type = PROJECTOR_TYPE_MLP;
|
||||
@@ -589,13 +613,8 @@ struct clip_model {
|
||||
ggml_tensor * ctc_out_b = nullptr;
|
||||
ggml_tensor * ctc_out_mid_w = nullptr;
|
||||
ggml_tensor * ctc_out_mid_b = nullptr;
|
||||
// qformer projector
|
||||
ggml_tensor * qf_proj_query = nullptr;
|
||||
ggml_tensor * qf_proj_norm_w = nullptr;
|
||||
ggml_tensor * qf_proj_norm_b = nullptr;
|
||||
ggml_tensor * qf_proj_linear_w = nullptr;
|
||||
ggml_tensor * qf_proj_linear_b = nullptr;
|
||||
std::vector<clip_layer> qf_proj_layers;
|
||||
// qformer projector(s)
|
||||
std::vector<qf_block> qf_proj_blocks;
|
||||
|
||||
bool audio_has_avgpool() const {
|
||||
return proj_type == PROJECTOR_TYPE_QWEN2A
|
||||
|
||||
+217
-56
@@ -61,6 +61,7 @@
|
||||
#include "models/gemma4uv.cpp"
|
||||
#include "models/glm4v.cpp"
|
||||
#include "models/granite-speech.cpp"
|
||||
#include "models/granite4-vision.cpp"
|
||||
#include "models/hunyuanvl.cpp"
|
||||
#include "models/internvl.cpp"
|
||||
#include "models/kimivl.cpp"
|
||||
@@ -1061,6 +1062,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
|
||||
{
|
||||
builder = std::make_unique<clip_graph_yasa2>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GRANITE4_VISION:
|
||||
{
|
||||
builder = std::make_unique<clip_graph_granite4_vision>(ctx, img);
|
||||
} break;
|
||||
default:
|
||||
GGML_ABORT("missing cgraph builder");
|
||||
}
|
||||
@@ -1321,12 +1326,7 @@ struct clip_model_loader {
|
||||
// to form the final visual features.
|
||||
// NOTE: gguf conversions should standardize the values of the vision feature layer to
|
||||
// be non-negative, since we use -1 to mark values as unset here.
|
||||
std::vector<int> vision_feature_layer;
|
||||
get_arr_int(KEY_FEATURE_LAYER, vision_feature_layer, false);
|
||||
// convert std::vector to std::unordered_set
|
||||
for (auto & layer : vision_feature_layer) {
|
||||
hparams.vision_feature_layer.insert(layer);
|
||||
}
|
||||
get_arr_int(KEY_FEATURE_LAYER, hparams.vision_feature_layer, false);
|
||||
|
||||
// model-specific params
|
||||
switch (model.proj_type) {
|
||||
@@ -1719,6 +1719,23 @@ struct clip_model_loader {
|
||||
hparams.image_pad_color = {127, 127, 127};
|
||||
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GRANITE4_VISION:
|
||||
{
|
||||
// SigLIP tower.
|
||||
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
|
||||
hparams.image_resize_pad = PAD_CEIL;
|
||||
|
||||
get_arr_int(KEY_FEATURE_LAYER, hparams.vision_feature_layer);
|
||||
get_arr_int(KEY_PROJ_SPATIAL_OFFSETS, hparams.proj_spatial_offsets);
|
||||
if (hparams.vision_feature_layer.size() != hparams.proj_spatial_offsets.size()) {
|
||||
throw std::runtime_error(string_format("%s: vision_feature_layer.size() %d != proj_spatial_offsets.size() %d",
|
||||
hparams.vision_feature_layer.size(), hparams.proj_spatial_offsets.size()));
|
||||
}
|
||||
|
||||
get_u32(KEY_PROJ_SAMPLE_QUERY_SIDE, hparams.downsample_query_side);
|
||||
get_u32(KEY_PROJ_SAMPLE_WINDOW_SIDE, hparams.downsample_window_side);
|
||||
hparams.warmup_image_size = hparams.image_size;
|
||||
} break;
|
||||
default:
|
||||
throw std::runtime_error(string_format("%s: unknown vision projector type %s\n", __func__, proj_type.c_str()));
|
||||
}
|
||||
@@ -2725,47 +2742,106 @@ struct clip_model_loader {
|
||||
layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"));
|
||||
}
|
||||
|
||||
model.qf_proj_query = get_tensor(TN_QF_PROJ_QUERY);
|
||||
model.qf_proj_norm_w = get_tensor(string_format(TN_QF_PROJ_NORM, "weight"));
|
||||
model.qf_proj_norm_b = get_tensor(string_format(TN_QF_PROJ_NORM, "bias"));
|
||||
model.qf_proj_linear_w = get_tensor(string_format(TN_QF_PROJ_LINEAR, "weight"));
|
||||
model.qf_proj_linear_b = get_tensor(string_format(TN_QF_PROJ_LINEAR, "bias"));
|
||||
model.qf_proj_blocks.resize(1);
|
||||
auto & qf = model.qf_proj_blocks[0];
|
||||
qf.qf_proj_query = get_tensor(string_format(TN_QF_PROJ_QUERY, prefix));
|
||||
qf.qf_proj_norm_w = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "weight"));
|
||||
qf.qf_proj_norm_b = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "bias"));
|
||||
qf.qf_proj_linear_w = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "weight"));
|
||||
qf.qf_proj_linear_b = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "bias"));
|
||||
|
||||
const int n_proj_layers = 2;
|
||||
model.qf_proj_layers.resize(n_proj_layers);
|
||||
qf.qf_proj_layers.resize(n_proj_layers);
|
||||
for (int il = 0; il < n_proj_layers; ++il) {
|
||||
auto & pl = model.qf_proj_layers[il];
|
||||
auto & pl = qf.qf_proj_layers[il];
|
||||
|
||||
pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, il, "weight"));
|
||||
pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, il, "bias"));
|
||||
pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, il, "weight"));
|
||||
pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, il, "bias"));
|
||||
pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, il, "weight"));
|
||||
pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, il, "bias"));
|
||||
pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, il, "weight"));
|
||||
pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, il, "bias"));
|
||||
pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, il, "weight"));
|
||||
pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, il, "bias"));
|
||||
pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "weight"));
|
||||
pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "bias"));
|
||||
pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "weight"));
|
||||
pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "bias"));
|
||||
pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "weight"));
|
||||
pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "bias"));
|
||||
pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "weight"));
|
||||
pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "bias"));
|
||||
pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "weight"));
|
||||
pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "bias"));
|
||||
|
||||
pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, il, "weight"));
|
||||
pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, il, "bias"));
|
||||
pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, il, "weight"));
|
||||
pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, il, "bias"));
|
||||
pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, il, "weight"));
|
||||
pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, il, "bias"));
|
||||
pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, il, "weight"));
|
||||
pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, il, "bias"));
|
||||
pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, il, "weight"));
|
||||
pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, il, "bias"));
|
||||
pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "weight"));
|
||||
pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "bias"));
|
||||
pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "weight"));
|
||||
pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "bias"));
|
||||
pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "weight"));
|
||||
pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "bias"));
|
||||
pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "weight"));
|
||||
pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "bias"));
|
||||
pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "weight"));
|
||||
pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "bias"));
|
||||
|
||||
pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, il, "weight"));
|
||||
pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, il, "bias"));
|
||||
pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, il, "weight"));
|
||||
pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, il, "bias"));
|
||||
pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, il, "weight"));
|
||||
pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, il, "bias"));
|
||||
pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, prefix, il, "weight"));
|
||||
pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, prefix, il, "bias"));
|
||||
pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "weight"));
|
||||
pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "bias"));
|
||||
pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "weight"));
|
||||
pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "bias"));
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GRANITE4_VISION:
|
||||
{
|
||||
// image_newline lives at the top-level.
|
||||
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
|
||||
|
||||
// Load separate layerwise and spatial projector tensors
|
||||
const auto projector_count = hparams.vision_feature_layer.size();
|
||||
model.qf_proj_blocks.resize(projector_count);
|
||||
for (size_t bid = 0; bid < projector_count; ++bid) {
|
||||
auto & b = model.qf_proj_blocks[bid];
|
||||
|
||||
// non-layerwise tensors
|
||||
b.qf_proj_img_pos = get_tensor(string_format(TN_MULTI_PROJ_IMG_POS, bid));
|
||||
b.qf_proj_query = get_tensor(string_format(TN_MULTI_PROJ_QUERY, prefix, bid));
|
||||
b.qf_proj_linear_w = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "weight"));
|
||||
b.qf_proj_linear_b = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "bias"));
|
||||
b.qf_proj_norm_w = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "weight"));
|
||||
b.qf_proj_norm_b = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "bias"));
|
||||
b.qf_proj_post_norm_w = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "weight"));
|
||||
b.qf_proj_post_norm_b = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "bias"));
|
||||
|
||||
// laywerwise tensors
|
||||
// NOTE: If any model uses multi-layer qformers, this will need to change
|
||||
b.qf_proj_layers.resize(1);
|
||||
auto & pl = b.qf_proj_layers[0];
|
||||
|
||||
pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "weight"));
|
||||
pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "bias"));
|
||||
pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "weight"));
|
||||
pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "bias"));
|
||||
pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "weight"));
|
||||
pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "bias"));
|
||||
pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "weight"));
|
||||
pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "bias"));
|
||||
pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "weight"));
|
||||
pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "bias"));
|
||||
|
||||
pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "weight"));
|
||||
pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "bias"));
|
||||
pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "weight"));
|
||||
pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "bias"));
|
||||
pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "weight"));
|
||||
pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "bias"));
|
||||
pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "weight"));
|
||||
pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "bias"));
|
||||
pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "weight"));
|
||||
pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "bias"));
|
||||
|
||||
pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, prefix, bid, "weight"));
|
||||
pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, prefix, bid, "bias"));
|
||||
pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "weight"));
|
||||
pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "bias"));
|
||||
pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "weight"));
|
||||
pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "bias"));
|
||||
}
|
||||
|
||||
} break;
|
||||
default:
|
||||
GGML_ASSERT(false && "unknown projector type");
|
||||
}
|
||||
@@ -3182,18 +3258,6 @@ void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny
|
||||
memcpy(img->buf.data(), rgb_pixels, img->buf.size());
|
||||
}
|
||||
|
||||
bool clip_image_load_from_file(const char * fname, clip_image_u8 * img) {
|
||||
int nx, ny, nc;
|
||||
auto * data = stbi_load(fname, &nx, &ny, &nc, 3);
|
||||
if (!data) {
|
||||
LOG_ERR("%s: failed to load image '%s'\n", __func__, fname);
|
||||
return false;
|
||||
}
|
||||
clip_build_img_from_pixels(data, nx, ny, img);
|
||||
stbi_image_free(data);
|
||||
return true;
|
||||
}
|
||||
|
||||
//note that the memory here must be subsequently freed!
|
||||
uint8_t* make_new_letterbox_img(uint8_t* input_image, int nx, int ny, int nc, int target_width, int target_height) {
|
||||
int new_image_size = (target_width * target_height * nc) + 512; //add some padding
|
||||
@@ -3591,6 +3655,12 @@ void setup_init_vision_shim_kcpp(struct clip_ctx * ctx_v) {
|
||||
img_end = "</vision>";
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GRANITE4_VISION:
|
||||
{
|
||||
img_beg = "<image>";
|
||||
img_end = "";
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);
|
||||
} break;
|
||||
default:
|
||||
throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj));
|
||||
}
|
||||
@@ -3621,10 +3691,6 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, str
|
||||
return true;
|
||||
}
|
||||
|
||||
ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx) {
|
||||
return ctx->model.image_newline;
|
||||
}
|
||||
|
||||
void clip_free(clip_ctx * ctx) {
|
||||
if (ctx == nullptr) {
|
||||
return;
|
||||
@@ -3933,6 +3999,23 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
|
||||
const int ds = ctx->model.hparams.audio_proj_downsample_rate;
|
||||
n_patches = ((img->nx + ws - 1) / ws) * (ws / ds);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GRANITE4_VISION:
|
||||
{
|
||||
// Per-tile output token count: each projector block outputs
|
||||
// query_side^2 tokens per window × n^2 windows.
|
||||
// For 384×384 input: n = 24/8 = 3, query_side = 4 → 144.
|
||||
const int window_side = ctx->model.hparams.downsample_window_side;
|
||||
const int query_side = ctx->model.hparams.downsample_query_side;
|
||||
const int side = img->nx / params.patch_size;
|
||||
const int n = side / window_side;
|
||||
n_patches = (query_side * n) * (query_side * n);
|
||||
if (img->add_newline) {
|
||||
// For single-tile case: append 1 newline row.
|
||||
// For multi-tile rowwise: handled by caller, but here we
|
||||
// report the per-tile count including one trailing newline.
|
||||
n_patches += 1;
|
||||
}
|
||||
} break;
|
||||
default:
|
||||
GGML_ABORT("unsupported projector type");
|
||||
}
|
||||
@@ -4765,6 +4848,82 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
|
||||
set_input_f32("attn_mask", mask);
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GRANITE4_VISION:
|
||||
{
|
||||
// Granite Vision 4.1 uses precomputed permutation index
|
||||
// tensors to express the _win / _unwin / spatial sampling
|
||||
// reshapes as ggml_get_rows gathers. The names are set
|
||||
// by g4v_gather() in models/granite4-vision.cpp.
|
||||
const int patch_size = model.hparams.patch_size;
|
||||
const int image_side = imgs.entries.front()->nx / patch_size;
|
||||
const int window_side = hparams.downsample_window_side;
|
||||
const int query_side = hparams.downsample_query_side;
|
||||
const int n = image_side / window_side;
|
||||
const int new_side = n * query_side;
|
||||
|
||||
// Builds the raster→window permutation indices for a
|
||||
// (side, side) grid split into (n × n) windows of (win × win)
|
||||
// tokens each. dst[w * win*win + p] = source raster index.
|
||||
auto make_win_idx = [](int side, int win) {
|
||||
const int nn = side / win;
|
||||
std::vector<int32_t> idx(static_cast<size_t>(side) * side);
|
||||
for (int wy = 0; wy < nn; ++wy) {
|
||||
for (int wx = 0; wx < nn; ++wx) {
|
||||
for (int iy = 0; iy < win; ++iy) {
|
||||
for (int ix = 0; ix < win; ++ix) {
|
||||
const int w = wy * nn + wx;
|
||||
const int p = iy * win + ix;
|
||||
const int y = wy * win + iy;
|
||||
const int x = wx * win + ix;
|
||||
idx[static_cast<size_t>(w) * (win*win) + p] = y * side + x;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return idx;
|
||||
};
|
||||
|
||||
auto make_unwin_idx = [&](int side, int win) {
|
||||
const std::vector<int32_t> fwd = make_win_idx(side, win);
|
||||
std::vector<int32_t> inv(fwd.size());
|
||||
for (size_t i = 0; i < fwd.size(); ++i) {
|
||||
inv[fwd[i]] = static_cast<int32_t>(i);
|
||||
}
|
||||
return inv;
|
||||
};
|
||||
|
||||
auto make_spatial_idx = [](int side, int offset) {
|
||||
const int off_y = (offset >> 1) & 1;
|
||||
const int off_x = offset & 1;
|
||||
const int new_s = side / 2;
|
||||
std::vector<int32_t> idx(static_cast<size_t>(new_s) * new_s);
|
||||
for (int y = 0; y < new_s; ++y) {
|
||||
for (int x = 0; x < new_s; ++x) {
|
||||
idx[y * new_s + x] = (y * 2 + off_y) * side + (x * 2 + off_x);
|
||||
}
|
||||
}
|
||||
return idx;
|
||||
};
|
||||
|
||||
auto upload = [&](const std::string & name, const std::vector<int32_t> & idx) {
|
||||
ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str());
|
||||
GGML_ASSERT(t);
|
||||
ggml_backend_tensor_set(t, idx.data(), 0, idx.size() * sizeof(int32_t));
|
||||
};
|
||||
|
||||
// Stage 1b only uses block 0's permutations; future stages
|
||||
// will upload all blocks.
|
||||
for (size_t bid = 0; bid < hparams.vision_feature_layer.size(); ++bid) {
|
||||
const std::string prefix = "g4v_blk" + std::to_string(bid) + "_";
|
||||
upload(prefix + "win_idx", make_win_idx(image_side, window_side));
|
||||
upload(prefix + "qwin_idx", make_win_idx(new_side, query_side));
|
||||
upload(prefix + "unwin_idx", make_unwin_idx(new_side, query_side));
|
||||
const auto spatial_offset = hparams.proj_spatial_offsets[bid];
|
||||
if (spatial_offset >= 0) {
|
||||
upload(prefix + "spatial_idx", make_spatial_idx(image_side,spatial_offset));
|
||||
}
|
||||
}
|
||||
} break;
|
||||
default:
|
||||
GGML_ABORT("Unknown projector type");
|
||||
}
|
||||
@@ -5117,7 +5276,9 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
|
||||
case PROJECTOR_TYPE_LFM2A:
|
||||
return ctx->model.position_embeddings->ne[0];
|
||||
case PROJECTOR_TYPE_GRANITE_SPEECH:
|
||||
return ctx->model.qf_proj_linear_w->ne[1];
|
||||
return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1];
|
||||
case PROJECTOR_TYPE_GRANITE4_VISION:
|
||||
return ctx->model.qf_proj_blocks.size() * ctx->model.hparams.projection_dim;
|
||||
case PROJECTOR_TYPE_GLM4V:
|
||||
return ctx->model.mm_ffn_down_w->ne[1];
|
||||
default:
|
||||
|
||||
@@ -100,16 +100,12 @@ struct clip_image_f32 * clip_image_f32_get_img(const struct clip_image_f32_batch
|
||||
*/
|
||||
void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny, struct clip_image_u8 * img);
|
||||
|
||||
bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img);
|
||||
|
||||
/** interpret bytes as an image file with length bytes_length, and use the result to populate img */
|
||||
bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img, const int maxdims);
|
||||
|
||||
/** preprocess img and store the result in res_imgs, pad_to_square may be overridden to false depending on model configuration */
|
||||
bool clip_image_preprocess(struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32_batch * res_imgs );
|
||||
|
||||
struct ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx);
|
||||
|
||||
bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec);
|
||||
bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec);
|
||||
|
||||
|
||||
@@ -199,8 +199,8 @@ ggml_cgraph * clip_graph_granite_speech::build() {
|
||||
|
||||
ggml_tensor * enc_windows = ggml_reshape_3d(ctx0, cur, n_embd, window_size, nblocks_proj);
|
||||
|
||||
ggml_tensor * queries = build_norm(model.qf_proj_query,
|
||||
model.qf_proj_norm_w, model.qf_proj_norm_b,
|
||||
ggml_tensor * queries = build_norm(model.qf_proj_blocks[0].qf_proj_query,
|
||||
model.qf_proj_blocks[0].qf_proj_norm_w, model.qf_proj_blocks[0].qf_proj_norm_b,
|
||||
NORM_TYPE_NORMAL, proj_eps, -1);
|
||||
{
|
||||
ggml_tensor * q_3d = ggml_reshape_3d(ctx0, queries, n_embd, num_queries, 1);
|
||||
@@ -209,8 +209,8 @@ ggml_cgraph * clip_graph_granite_speech::build() {
|
||||
queries = ggml_repeat(ctx0, q_3d, q_shape);
|
||||
}
|
||||
|
||||
for (int il = 0; il < (int)model.qf_proj_layers.size(); il++) {
|
||||
const auto & pl = model.qf_proj_layers[il];
|
||||
for (int il = 0; il < (int)model.qf_proj_blocks[0].qf_proj_layers.size(); il++) {
|
||||
const auto & pl = model.qf_proj_blocks[0].qf_proj_layers[il];
|
||||
|
||||
// self-attention
|
||||
{
|
||||
@@ -265,7 +265,7 @@ ggml_cgraph * clip_graph_granite_speech::build() {
|
||||
}
|
||||
|
||||
cur = ggml_reshape_2d(ctx0, queries, n_embd, num_queries * nblocks_proj);
|
||||
cur = ggml_add(ctx0, build_mm(model.qf_proj_linear_w, cur), model.qf_proj_linear_b);
|
||||
cur = ggml_add(ctx0, build_mm(model.qf_proj_blocks[0].qf_proj_linear_w, cur), model.qf_proj_blocks[0].qf_proj_linear_b);
|
||||
cb(cur, "projector_out", -1);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
#include "models.h"
|
||||
#include "../clip-impl.h"
|
||||
#include "../clip-model.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
/*
|
||||
* Granite Vision 4.1 clip graph
|
||||
*
|
||||
* Stage 1a: SigLIP vision tower (N layers, post-norm)
|
||||
* Stage 1b: WindowQFormer blocks (deepstack + spatial)
|
||||
* Stage 1c: Concatenate and pack outputs
|
||||
* Stage 1d: Append newline tokens if add_newline is set
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Member method implementations
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
ggml_tensor * clip_graph_granite4_vision::gather(
|
||||
ggml_tensor * src,
|
||||
const std::string & name,
|
||||
int idx_len) {
|
||||
ggml_tensor * idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, idx_len);
|
||||
ggml_set_name(idx, name.c_str());
|
||||
ggml_set_input(idx);
|
||||
return ggml_get_rows(ctx0, src, idx);
|
||||
}
|
||||
|
||||
ggml_tensor * clip_graph_granite4_vision::interp_down(
|
||||
ggml_tensor * src,
|
||||
int side,
|
||||
int new_side) {
|
||||
const int n_embd = src->ne[0];
|
||||
ggml_tensor * t = ggml_reshape_4d(ctx0, src, n_embd, side, side, 1);
|
||||
t = ggml_cont(ctx0, ggml_permute(ctx0, t, 2, 0, 1, 3));
|
||||
const int kernel = side / new_side;
|
||||
t = ggml_pool_2d(ctx0, t, GGML_OP_POOL_AVG, kernel, kernel, kernel, kernel, 0, 0);
|
||||
t = ggml_cont(ctx0, ggml_permute(ctx0, t, 1, 2, 0, 3));
|
||||
return ggml_reshape_2d(ctx0, t, n_embd, new_side * new_side);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// build_block - WindowQFormer block implementation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
ggml_tensor * clip_graph_granite4_vision::build_block(
|
||||
const qf_block & blk,
|
||||
ggml_tensor * h,
|
||||
int bid,
|
||||
int spatial_offset,
|
||||
int image_side,
|
||||
int window_side,
|
||||
int query_side,
|
||||
float qformer_eps) {
|
||||
|
||||
const int n_embd = h->ne[0];
|
||||
GGML_ASSERT(h->ne[1] == image_side * image_side);
|
||||
const int n = image_side / window_side;
|
||||
const int new_side = n * query_side;
|
||||
const int n_windows = n * n;
|
||||
const int enc_len = window_side * window_side;
|
||||
const int query_len = query_side * query_side;
|
||||
|
||||
auto cbx = [&](ggml_tensor * & t, const char * step) {
|
||||
const std::string name = "g4v_blk" + std::to_string(bid) + "_" + step;
|
||||
ggml_set_name(t, name.c_str());
|
||||
};
|
||||
|
||||
// 1. Top-level LN
|
||||
cbx(h, "inp");
|
||||
ggml_tensor * x = build_norm(h, blk.qf_proj_norm_w, blk.qf_proj_norm_b, NORM_TYPE_NORMAL, eps, bid);
|
||||
cbx(x, "norm");
|
||||
|
||||
// 2. enc = _win(x, image_side, window_side)
|
||||
ggml_tensor * enc;
|
||||
{
|
||||
ggml_tensor * enc_flat = gather(x,
|
||||
"g4v_blk" + std::to_string(bid) + "_win_idx",
|
||||
image_side * image_side);
|
||||
enc = ggml_reshape_3d(ctx0, enc_flat, n_embd, enc_len, n_windows);
|
||||
}
|
||||
cbx(enc, "enc");
|
||||
|
||||
// 3. downsampled = downsampler(x)
|
||||
ggml_tensor * d;
|
||||
(void) spatial_offset;
|
||||
if (spatial_offset >= 0) {
|
||||
d = gather(x,
|
||||
"g4v_blk" + std::to_string(bid) + "_spatial_idx",
|
||||
new_side * new_side);
|
||||
} else {
|
||||
d = interp_down(x, image_side, new_side);
|
||||
}
|
||||
cbx(d, "downsampled");
|
||||
|
||||
// 4. query_embeds = query + _win(d, new_side, query_side)
|
||||
ggml_tensor * q_in;
|
||||
{
|
||||
ggml_tensor * dw_flat = gather(d,
|
||||
"g4v_blk" + std::to_string(bid) + "_qwin_idx",
|
||||
new_side * new_side);
|
||||
ggml_tensor * dw = ggml_reshape_3d(ctx0, dw_flat, n_embd, query_len, n_windows);
|
||||
q_in = ggml_add(ctx0, dw, blk.qf_proj_query);
|
||||
}
|
||||
cbx(q_in, "query_embeds");
|
||||
|
||||
// 5. encoder_embeds = enc + image_positions → (C, enc_len, n_windows)
|
||||
ggml_tensor * e_in = ggml_add(ctx0, enc, blk.qf_proj_img_pos);
|
||||
cbx(e_in, "encoder_embeds");
|
||||
|
||||
// 6. Qformer forward.
|
||||
ggml_tensor * q = build_norm(q_in, blk.qf_proj_post_norm_w, blk.qf_proj_post_norm_b, NORM_TYPE_NORMAL, qformer_eps, bid);
|
||||
|
||||
// Helper for linear projections with window batching
|
||||
auto linear = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) -> ggml_tensor * {
|
||||
ggml_tensor * t = ggml_reshape_2d(ctx0, x, x->ne[0], x->ne[1] * x->ne[2]);
|
||||
t = build_mm(w, t);
|
||||
if (b) t = ggml_add(ctx0, t, b);
|
||||
return t;
|
||||
};
|
||||
|
||||
// Get the single QFormer layer
|
||||
GGML_ASSERT(blk.qf_proj_layers.size() == 1);
|
||||
const auto & pl = blk.qf_proj_layers[0];
|
||||
|
||||
// 6a. Self-attention
|
||||
ggml_tensor * sa_out;
|
||||
{
|
||||
const int d_h = 64;
|
||||
const int n_head = n_embd / d_h;
|
||||
const int nq = q->ne[1];
|
||||
const float scale = 1.0f / std::sqrt((float) d_h);
|
||||
|
||||
ggml_tensor * Q = linear(q, pl.q_w, pl.q_b);
|
||||
ggml_tensor * K = linear(q, pl.k_w, pl.k_b);
|
||||
ggml_tensor * V = linear(q, pl.v_w, pl.v_b);
|
||||
|
||||
Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_windows);
|
||||
K = ggml_reshape_4d(ctx0, K, d_h, n_head, nq, n_windows);
|
||||
V = ggml_reshape_4d(ctx0, V, d_h, n_head, nq, n_windows);
|
||||
|
||||
sa_out = build_attn(pl.o_w, pl.o_b, Q, K, V, nullptr, scale, bid);
|
||||
sa_out = ggml_reshape_3d(ctx0, sa_out, n_embd, nq, n_windows);
|
||||
|
||||
sa_out = ggml_add(ctx0, sa_out, q);
|
||||
sa_out = build_norm(sa_out, pl.ln_1_w, pl.ln_1_b,
|
||||
NORM_TYPE_NORMAL, qformer_eps, bid);
|
||||
}
|
||||
cbx(sa_out, "sa_out");
|
||||
|
||||
// 6b. Cross-attention
|
||||
ggml_tensor * ca_out;
|
||||
{
|
||||
const int d_h = 64;
|
||||
const int n_head = n_embd / d_h;
|
||||
const int nq = sa_out->ne[1];
|
||||
const int nkv = e_in->ne[1];
|
||||
const float scale = 1.0f / std::sqrt((float) d_h);
|
||||
|
||||
ggml_tensor * Q = linear(sa_out, pl.cross_attn_q_w, pl.cross_attn_q_b);
|
||||
ggml_tensor * K = linear(e_in, pl.cross_attn_k_w, pl.cross_attn_k_b);
|
||||
ggml_tensor * V = linear(e_in, pl.cross_attn_v_w, pl.cross_attn_v_b);
|
||||
|
||||
Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_windows);
|
||||
K = ggml_reshape_4d(ctx0, K, d_h, n_head, nkv, n_windows);
|
||||
V = ggml_reshape_4d(ctx0, V, d_h, n_head, nkv, n_windows);
|
||||
|
||||
ca_out = build_attn(pl.cross_attn_o_w, pl.cross_attn_o_b,
|
||||
Q, K, V, nullptr, scale, bid);
|
||||
ca_out = ggml_reshape_3d(ctx0, ca_out, n_embd, nq, n_windows);
|
||||
|
||||
ca_out = ggml_add(ctx0, ca_out, sa_out);
|
||||
ca_out = build_norm(ca_out, pl.cross_attn_norm_w, pl.cross_attn_norm_b,
|
||||
NORM_TYPE_NORMAL, qformer_eps, bid);
|
||||
}
|
||||
cbx(ca_out, "ca_out");
|
||||
|
||||
// 6c. FFN
|
||||
ggml_tensor * ffn;
|
||||
{
|
||||
ggml_tensor * t = ggml_reshape_2d(ctx0, ca_out, n_embd, query_len * n_windows);
|
||||
t = build_mm(pl.ff_up_w, t);
|
||||
if (pl.ff_up_b) t = ggml_add(ctx0, t, pl.ff_up_b);
|
||||
t = ggml_gelu_erf(ctx0, t);
|
||||
t = build_mm(pl.ff_down_w, t);
|
||||
if (pl.ff_down_b) t = ggml_add(ctx0, t, pl.ff_down_b);
|
||||
t = ggml_reshape_3d(ctx0, t, n_embd, query_len, n_windows);
|
||||
ffn = ggml_add(ctx0, t, ca_out);
|
||||
ffn = build_norm(ffn, pl.ln_2_w, pl.ln_2_b, NORM_TYPE_NORMAL, qformer_eps, bid);
|
||||
}
|
||||
cbx(ffn, "qformer_out");
|
||||
|
||||
// 7. _unwin back to raster
|
||||
ggml_tensor * unwinned;
|
||||
{
|
||||
ggml_tensor * flat = ggml_reshape_2d(ctx0, ffn, n_embd, query_len * n_windows);
|
||||
unwinned = gather(flat,
|
||||
"g4v_blk" + std::to_string(bid) + "_unwin_idx",
|
||||
new_side * new_side);
|
||||
}
|
||||
cbx(unwinned, "unwin");
|
||||
|
||||
// 8. out_linear
|
||||
ggml_tensor * out = build_mm(blk.qf_proj_linear_w, unwinned);
|
||||
if (blk.qf_proj_linear_b) out = ggml_add(ctx0, out, blk.qf_proj_linear_b);
|
||||
cbx(out, "out");
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// build() - top-level graph
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Build the K-tiled, base-scaled newline row tensor.
|
||||
// Shape: (n_mmproj_embd, 1)
|
||||
ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0) {
|
||||
const int K = (int) model.qf_proj_blocks.size();
|
||||
GGML_ASSERT(K > 0);
|
||||
GGML_ASSERT(n_mmproj_embd % K == 0);
|
||||
const int projection_dim = n_mmproj_embd / K;
|
||||
GGML_ASSERT(model.image_newline != nullptr);
|
||||
GGML_ASSERT(ggml_nelements(model.image_newline) == projection_dim);
|
||||
|
||||
// Build newline_row[k*projection_dim + d] = nl[d] * (k == 0 ? base : 1.0)
|
||||
ggml_tensor * nl = model.image_newline; // (projection_dim,)
|
||||
ggml_tensor * nl_first_2d = ggml_reshape_2d(ctx0, nl, projection_dim, 1);
|
||||
ggml_tensor * nl_row_2d;
|
||||
if (K == 1) {
|
||||
nl_row_2d = nl_first_2d;
|
||||
} else {
|
||||
ggml_tensor * nl_2d = ggml_reshape_2d(ctx0, nl, projection_dim, 1);
|
||||
ggml_tensor * rest_template = ggml_new_tensor_2d(
|
||||
ctx0, GGML_TYPE_F32, projection_dim, K - 1);
|
||||
ggml_tensor * nl_rest = ggml_repeat(ctx0, nl_2d, rest_template);
|
||||
nl_row_2d = ggml_concat(ctx0, nl_first_2d, nl_rest, 1); // (projection_dim, K)
|
||||
}
|
||||
nl_row_2d = ggml_cont(ctx0, nl_row_2d);
|
||||
return ggml_reshape_2d(ctx0, nl_row_2d, n_mmproj_embd, 1);
|
||||
}
|
||||
|
||||
// Append a single newline row at the end of the tile output.
|
||||
ggml_tensor * clip_graph_granite4_vision::append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output) {
|
||||
// For the single-tile case, append one newline row at the end.
|
||||
// For the multi-tile rowwise case, this will be called per-tile
|
||||
// (though currently only the single-tile path uses it).
|
||||
ggml_tensor * nl_row = build_newline_row(ctx0);
|
||||
return ggml_concat(ctx0, tile_output, nl_row, 1);
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_granite4_vision::build() {
|
||||
GGML_ASSERT(model.patch_embeddings_0 != nullptr);
|
||||
GGML_ASSERT(model.position_embeddings != nullptr);
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
GGML_ASSERT(!model.qf_proj_blocks.empty());
|
||||
|
||||
// --- Stage 1a: SigLIP encoder producing intermediate hidden states ---
|
||||
ggml_tensor * inp = build_inp();
|
||||
inp = ggml_add(ctx0, inp, model.position_embeddings);
|
||||
cb(inp, "pos_embed", -1);
|
||||
|
||||
ggml_tensor * inpL = inp;
|
||||
std::vector<ggml_tensor *> layer_outs(n_layer, nullptr);
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
const auto & layer = model.layers[il];
|
||||
ggml_tensor * cur = inpL;
|
||||
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
|
||||
// Self-attention
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, cur);
|
||||
if (layer.q_b) Qcur = ggml_add(ctx0, Qcur, layer.q_b);
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, cur);
|
||||
if (layer.k_b) Kcur = ggml_add(ctx0, Kcur, layer.k_b);
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, cur);
|
||||
if (layer.v_b) Vcur = ggml_add(ctx0, Vcur, layer.v_b);
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_patches);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_patches);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_patches);
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b,
|
||||
Qcur, Kcur, Vcur, nullptr, kq_scale, il);
|
||||
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
inpL = cur;
|
||||
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
cur = ggml_add(ctx0, inpL, cur);
|
||||
cb(cur, "layer_out", il);
|
||||
layer_outs[il] = cur;
|
||||
inpL = cur;
|
||||
}
|
||||
|
||||
// --- Stage 1b/1c: WindowQFormer blocks ---
|
||||
const int projector_count = hparams.vision_feature_layer.size();
|
||||
const float qformer_eps = 1e-12f;
|
||||
|
||||
ggml_tensor * mmproj = nullptr;
|
||||
for (int bid = 0; bid < projector_count; ++bid) {
|
||||
const auto & blk = model.qf_proj_blocks[bid];
|
||||
|
||||
int vlayer = hparams.vision_feature_layer[bid];
|
||||
GGML_ASSERT(vlayer >= 0 && vlayer < n_layer);
|
||||
ggml_tensor * h = layer_outs[vlayer];
|
||||
|
||||
ggml_tensor * stream = build_block(
|
||||
blk, h, bid,
|
||||
hparams.proj_spatial_offsets[bid],
|
||||
n_patches_x,
|
||||
hparams.downsample_window_side,
|
||||
hparams.downsample_query_side,
|
||||
qformer_eps);
|
||||
cb(stream, (std::string("proj_") + std::to_string(bid) + std::string("_v_out")).c_str(), vlayer);
|
||||
mmproj = mmproj ? ggml_concat(ctx0, mmproj, stream, 0) : stream;
|
||||
}
|
||||
|
||||
// --- Stage 1d: Append newline tokens if add_newline is set ---
|
||||
if (add_newline) {
|
||||
mmproj = append_rowwise_newlines(ctx0, mmproj);
|
||||
ggml_set_name(mmproj, "g4v_mmproj_out_nl");
|
||||
} else {
|
||||
ggml_set_name(mmproj, "g4v_mmproj_out");
|
||||
}
|
||||
ggml_build_forward_expand(gf, mmproj);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -51,7 +51,6 @@ ggml_cgraph * clip_graph_llava::build() {
|
||||
}
|
||||
|
||||
std::vector<ggml_tensor *> embedding_stack;
|
||||
const auto & vision_feature_layer = hparams.vision_feature_layer;
|
||||
|
||||
// loop over layers
|
||||
for (int il = 0; il < max_feature_layer; il++) {
|
||||
@@ -60,7 +59,7 @@ ggml_cgraph * clip_graph_llava::build() {
|
||||
|
||||
// If this is an embedding feature layer, save the output.
|
||||
// NOTE: 0 index here refers to the input to the encoder.
|
||||
if (vision_feature_layer.find(il) != vision_feature_layer.end()) {
|
||||
if (hparams.is_vision_feature_layer(il)) {
|
||||
embedding_stack.push_back(cur);
|
||||
}
|
||||
|
||||
@@ -135,7 +134,7 @@ ggml_cgraph * clip_graph_llava::build() {
|
||||
// process vision feature layers (used by granite)
|
||||
{
|
||||
// final layer is a vision feature layer
|
||||
if (vision_feature_layer.find(max_feature_layer) != vision_feature_layer.end()) {
|
||||
if (hparams.is_vision_feature_layer(max_feature_layer)) {
|
||||
embedding_stack.push_back(inpL);
|
||||
}
|
||||
|
||||
|
||||
@@ -211,3 +211,26 @@ struct clip_graph_exaone4_5 : clip_graph {
|
||||
clip_graph_exaone4_5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_granite4_vision : clip_graph {
|
||||
clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img)
|
||||
: clip_graph(ctx, img),
|
||||
add_newline(img.add_newline) {}
|
||||
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
private:
|
||||
// The graph is per-tile since only batch-size 1 is supported in clip. As
|
||||
// such, this value is set at construct time based on the tile that will be
|
||||
// encoded, then used during build to determine how to handle newlines.
|
||||
const bool add_newline;
|
||||
|
||||
ggml_tensor * gather(ggml_tensor * src, const std::string & name, int idx_len);
|
||||
ggml_tensor * interp_down(ggml_tensor * src, int side, int new_side);
|
||||
ggml_tensor * build_block(const qf_block & blk, ggml_tensor * h, int bid,
|
||||
int spatial_offset, int image_side, int window_side,
|
||||
int query_side, float qformer_eps);
|
||||
|
||||
ggml_tensor * build_newline_row(ggml_context * ctx0);
|
||||
ggml_tensor * append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output);
|
||||
};
|
||||
|
||||
+26
-3
@@ -513,6 +513,12 @@ struct mtmd_context {
|
||||
img_end = "</vision>";
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GRANITE4_VISION:
|
||||
{
|
||||
img_beg = "<image>";
|
||||
img_end = "";
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);
|
||||
} break;
|
||||
default:
|
||||
throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj));
|
||||
}
|
||||
@@ -808,6 +814,21 @@ struct mtmd_tokenizer {
|
||||
return 2;
|
||||
}
|
||||
|
||||
// Annotate llava-next style tiles so clip_n_output_tokens accounts
|
||||
// for per-tile newline injection.
|
||||
if (ctx->proj_type_v() == PROJECTOR_TYPE_GRANITE4_VISION) {
|
||||
if (batch_f32.entries.size() == 1) {
|
||||
// Single-tile (overview only): append one newline row.
|
||||
batch_f32.entries[0]->add_newline = true;
|
||||
} else {
|
||||
// Multi-tile: overview gets no newline, grid tiles get one.
|
||||
batch_f32.entries[0]->add_newline = false;
|
||||
for (size_t i = 1; i < batch_f32.entries.size(); ++i) {
|
||||
batch_f32.entries[i]->add_newline = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handle llava-uhd style preprocessing
|
||||
const bool has_tiling_grid = batch_f32.grid_x > 0 && batch_f32.grid_y > 0;
|
||||
if (
|
||||
@@ -872,9 +893,10 @@ struct mtmd_tokenizer {
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
size_t n_tokens = 0;
|
||||
for (const auto & entry : batch_f32.entries) {
|
||||
n_tokens += clip_n_output_tokens(ctx->ctx_v, entry.get());
|
||||
for (const auto & e : batch_f32.entries) {
|
||||
n_tokens += clip_n_output_tokens(ctx->ctx_v, e.get());
|
||||
}
|
||||
|
||||
mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens);
|
||||
@@ -1111,7 +1133,8 @@ int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens)
|
||||
|| proj_type == PROJECTOR_TYPE_MINICPMV
|
||||
|| proj_type == PROJECTOR_TYPE_GLM_EDGE
|
||||
|| proj_type == PROJECTOR_TYPE_INTERNVL
|
||||
|| proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) {
|
||||
|| proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2
|
||||
|| proj_type == PROJECTOR_TYPE_GRANITE4_VISION) {
|
||||
// TODO @ngxson : llava does not support batched encoding ; this should be fixed inside clip_image_batch_encode()
|
||||
const auto & entries = image_tokens->batch_f32.entries;
|
||||
// entries may have different token counts
|
||||
|
||||
Reference in New Issue
Block a user