diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 616dcac8c..ffa70be23 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -3672,6 +3672,11 @@ generation_outputs gpttype_generate(const generation_inputs inputs) img_start = "<|begin_of_image|>"; img_end = "<|end_of_image|>\n\n"; } + else if(ptype==PROJECTOR_TYPE_GEMMA4V) + { + img_start = "<|image>"; + img_end = "\n\n"; + } } TokenizeString(img_start, lv.chunk_start_seq, file_format, false); TokenizeString(img_end, lv.chunk_end_seq, file_format, false); diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 52ef5e79d..0a5e1d5c6 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -4041,6 +4041,10 @@ bool clip_is_llava(const struct clip_ctx * ctx) { return ctx->model.hparams.has_llava_projector; } +bool clip_is_gemma4(const struct clip_ctx * ctx) { //for kcpp use + return ctx->proj_type() == PROJECTOR_TYPE_GEMMA4V; +} + bool clip_has_vision_encoder(const struct clip_ctx * ctx) { return ctx->model.modality == CLIP_MODALITY_VISION; } diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index 84cdbebc0..7440152a3 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -112,6 +112,7 @@ bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct int clip_is_minicpmv(const struct clip_ctx * ctx); bool clip_is_glm(const struct clip_ctx * ctx); +bool clip_is_gemma4(const struct clip_ctx * ctx); bool clip_is_mrope(const struct clip_ctx * ctx); bool clip_is_llava(const struct clip_ctx * ctx); void set_clip_uses_gpu(bool usegpu); diff --git a/tools/mtmd/llava.cpp b/tools/mtmd/llava.cpp index e05f295f3..e30cb2465 100644 --- a/tools/mtmd/llava.cpp +++ b/tools/mtmd/llava.cpp @@ -78,7 +78,17 @@ bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, co max_ny = std::max(max_ny,b); } image_embd = (float *)malloc(clip_embd_nbytes_by_img(ctx_clip, max_nx, max_ny)); - } else { + } + else if(clip_is_gemma4(ctx_clip)) //gemma4 vision + { + const size_t n_imgs = clip_image_f32_batch_n_images(preprocessed_img.get()); + clip_image_f32 * img_res = clip_image_f32_get_img(preprocessed_img.get(), 0); + int n_img_pos = clip_n_output_tokens(ctx_clip, img_res); + int nbytes = n_img_pos * clip_n_mmproj_embd(ctx_clip) * sizeof(float); + image_embd = (float *)malloc(nbytes); + } + else + { image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip)*num_max_patches); // TODO: base on gridsize/llava model } if (!image_embd) {