mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-19 17:25:07 +02:00
fixed gemma4v image crashing on encode, however images are not yet working correctly
This commit is contained in:
@@ -4041,6 +4041,10 @@ bool clip_is_llava(const struct clip_ctx * ctx) {
|
||||
return ctx->model.hparams.has_llava_projector;
|
||||
}
|
||||
|
||||
bool clip_is_gemma4(const struct clip_ctx * ctx) { //for kcpp use
|
||||
return ctx->proj_type() == PROJECTOR_TYPE_GEMMA4V;
|
||||
}
|
||||
|
||||
bool clip_has_vision_encoder(const struct clip_ctx * ctx) {
|
||||
return ctx->model.modality == CLIP_MODALITY_VISION;
|
||||
}
|
||||
|
||||
@@ -112,6 +112,7 @@ bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct
|
||||
|
||||
int clip_is_minicpmv(const struct clip_ctx * ctx);
|
||||
bool clip_is_glm(const struct clip_ctx * ctx);
|
||||
bool clip_is_gemma4(const struct clip_ctx * ctx);
|
||||
bool clip_is_mrope(const struct clip_ctx * ctx);
|
||||
bool clip_is_llava(const struct clip_ctx * ctx);
|
||||
void set_clip_uses_gpu(bool usegpu);
|
||||
|
||||
+11
-1
@@ -78,7 +78,17 @@ bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, co
|
||||
max_ny = std::max(max_ny,b);
|
||||
}
|
||||
image_embd = (float *)malloc(clip_embd_nbytes_by_img(ctx_clip, max_nx, max_ny));
|
||||
} else {
|
||||
}
|
||||
else if(clip_is_gemma4(ctx_clip)) //gemma4 vision
|
||||
{
|
||||
const size_t n_imgs = clip_image_f32_batch_n_images(preprocessed_img.get());
|
||||
clip_image_f32 * img_res = clip_image_f32_get_img(preprocessed_img.get(), 0);
|
||||
int n_img_pos = clip_n_output_tokens(ctx_clip, img_res);
|
||||
int nbytes = n_img_pos * clip_n_mmproj_embd(ctx_clip) * sizeof(float);
|
||||
image_embd = (float *)malloc(nbytes);
|
||||
}
|
||||
else
|
||||
{
|
||||
image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip)*num_max_patches); // TODO: base on gridsize/llava model
|
||||
}
|
||||
if (!image_embd) {
|
||||
|
||||
Reference in New Issue
Block a user