Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-18 16:55:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

clip quantize skip problematic layer

Browse Source
This commit is contained in:
Concedo
2024-12-19 16:25:48 +08:00
parent ee486bad3e
commit fbf1345a66
1 changed files with 10 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/llava/clip.cpp
+10
View File
@@ -2722,6 +2722,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
return true;
}
static bool avoid_problematic_indivisible = true;
bool clip_model_quantize(const char * fname_inp, const char * fname_out, const int itype) {
ggml_type type = GGML_TYPE_Q4_1;
@@ -2782,6 +2783,15 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i
// quantize only 2D tensors
quantize &= (ggml_n_dims(cur) == 2);
//kcpp fix: do not quantize certain tensors if they are indivisible!
if(avoid_problematic_indivisible)
{
if(name=="v.position_embd.weight")
{
quantize = false;
}
}
if (quantize) {
new_type = type;
if (new_type >= GGML_TYPE_Q2_K && name.find("embd") != std::string::npos) {