Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 01:05:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix build, remove clip quantize (+1 squashed commits)

Browse Source 
Squashed commits:

[09ffa906b] fix build, remove clip quantize
This commit is contained in:
Concedo
2026-06-10 00:36:43 +08:00
parent cd49b19016
commit 6bae15da71
6 changed files with 3 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
-203
View File
@@ -4539,209 +4539,6 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
return true;
}
static bool avoid_problematic_indivisible = true;
bool clip_model_quantize(const char * fname_inp, const char * fname_out, const int itype) {
assert(itype < GGML_TYPE_COUNT);
ggml_type type = static_cast<ggml_type>(itype);
auto ccparams = clip_context_params {
/* use_gpu */ true,
/* flash_attn_type */ CLIP_FLASH_ATTN_TYPE_DISABLED,
/* image_min_tokens */ -1,
/* image_max_tokens */ -1,
};
clip_ctx * ctx_clip = new clip_ctx(ccparams);
clip_model_loader loader(fname_inp);
try {
loader.load_hparams(ctx_clip->model, CLIP_MODALITY_VISION);
loader.load_tensors(*ctx_clip);
// create a fake batch
const auto & hparams = ctx_clip->model.hparams;
clip_image_f32_batch batch;
clip_image_f32_ptr img(clip_image_f32_init());
if (ctx_clip->model.modality == CLIP_MODALITY_VISION) {
img->nx = hparams.warmup_image_size;
img->ny = hparams.warmup_image_size;
LOG_INF("%s: warmup with image size = %d x %d\n", __func__, img->nx, img->ny);
} else {
img->nx = hparams.warmup_audio_size;
img->ny = hparams.n_mel_bins;
LOG_INF("%s: warmup with audio size = %d\n", __func__, img->nx);
}
batch.entries.push_back(std::move(img));
loader.reserve_compute_meta(*ctx_clip,batch);
} catch (const std::exception & e) {
printf("%s: failed to load model '%s': %s\n", __func__, fname_inp, e.what());
if (ctx_clip) {
delete ctx_clip;
}
return false;
}
const auto & ctx_src = loader.ctx_gguf.get();
const auto & ctx_data = ctx_clip->ctx_data.get();
auto * ctx_out = gguf_init_empty();
gguf_set_kv(ctx_out, ctx_src);
gguf_set_val_u32(ctx_out, "general.quantization_version", GGML_QNT_VERSION);
gguf_set_val_u32(ctx_out, "general.file_type", itype);
auto fout = std::ofstream(fname_out, std::ios::binary);
const int n_tensors = gguf_get_n_tensors(ctx_src);
for (int i = 0; i < n_tensors; ++i) {
const char * name = gguf_get_tensor_name(ctx_src, i);
ggml_tensor * cur = ggml_get_tensor(ctx_data, name);
if(cur==nullptr)
{
printf("\nSkipped missing %s tensor!\n",name);
continue;
}
gguf_add_tensor(ctx_out, cur);
}
const size_t meta_size = gguf_get_meta_size(ctx_out);
for (size_t i = 0; i < meta_size; ++i) {
fout.put(0);
}
// regexes of tensor names to be quantized
const std::vector<std::string> k_names = {
".*weight",
};
std::vector<uint8_t> work(512);
std::vector<float> conv_buf(512);
size_t total_size_org = 0;
size_t total_size_new = 0;
for (int i = 0; i < n_tensors; ++i) {
const std::string name = gguf_get_tensor_name(ctx_src, i);
ggml_tensor * cur = ggml_get_tensor(ctx_data, name.c_str());
if(cur==nullptr)
{
printf("\nSkipped missing %s tensor!\n",name.c_str());
continue;
}
enum ggml_type new_type;
void * new_data;
size_t new_size;
bool quantize = false;
for (const auto & s : k_names) {
if (std::regex_match(name, std::regex(s))) {
quantize = true;
break;
}
}
// quantize only 2D tensors and bigger than block size
quantize &= (ggml_n_dims(cur) == 2) && cur->ne[0] > ggml_blck_size(type);
//kcpp fix: do not quantize certain tensors if they are indivisible!
if(avoid_problematic_indivisible)
{
if(name=="v.position_embd.weight")
{
quantize = false;
}
for(int d=0;d<ggml_n_dims(cur);++d)
{
const int64_t blck_size = ggml_blck_size(type);
if(d==0 && cur->ne[d] % blck_size != 0)
{
printf("\nSkipping %s because %zu is not divisible by %zu\n",name.c_str(),(size_t)cur->ne[d],(size_t)blck_size);
quantize = false;
break;
}
}
// //temp fix for gemma3
// if(name.find("ffn_up.weight") != std::string::npos)
// {
// quantize = false;
// }
}
if (quantize) {
new_type = type;
if (new_type >= GGML_TYPE_Q2_K && name.find("embd") != std::string::npos) {
new_type = GGML_TYPE_Q8_0; // ggml_get_rows needs non K type
// LOG_ERR("%s: quantizing %s to %s\n", __func__, name.c_str(), ggml_type_name(new_type));
}
const size_t n_elms = ggml_nelements(cur);
float * f32_data;
switch (cur->type) {
case GGML_TYPE_F32:
f32_data = (float *)cur->data;
break;
case GGML_TYPE_F16:
if (conv_buf.size() < n_elms) {
conv_buf.resize(n_elms);
}
for (size_t j = 0; j < n_elms; ++j) {
conv_buf[j] = ggml_fp16_to_fp32(((ggml_fp16_t *)cur->data)[j]);
}
f32_data = (float *)conv_buf.data();
break;
default:
LOG_ERR("%s: Please use an input file in f32 or f16\n", __func__);
gguf_free(ctx_out);
return false;
}
if (work.size() < n_elms * 4) {
work.resize(n_elms * 4);
}
new_data = work.data();
new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, n_elms/cur->ne[0], cur->ne[0], nullptr);
} else {
new_type = cur->type;
new_data = cur->data;
new_size = ggml_nbytes(cur);
}
const size_t orig_size = ggml_nbytes(cur);
total_size_org += orig_size;
total_size_new += new_size;
gguf_set_tensor_type(ctx_out, name.c_str(), new_type);
GGML_ASSERT(gguf_get_tensor_size(ctx_out, gguf_find_tensor(ctx_out, name.c_str())) == new_size);
gguf_set_tensor_data(ctx_out, name.c_str(), new_data);
fout.write((const char *)new_data, new_size);
size_t pad = GGML_PAD(new_size, gguf_get_alignment(ctx_out)) - new_size;
for (size_t j = 0; j < pad; ++j) {
fout.put(0);
}
LOG_INF("%s: n_dims = %d | quantize=%d | size = %f MB -> %f MB\n", name.c_str(), ggml_n_dims(cur), quantize,
orig_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);
}
// go back to beginning of file and write the updated metadata
fout.seekp(0, std::ios::beg);
std::vector<uint8_t> meta(meta_size);
gguf_get_meta_data(ctx_out, meta.data());
fout.write((const char *)meta.data(), meta_size);
fout.close();
clip_free(ctx_clip);
gguf_free(ctx_out);
{
LOG_INF("%s: original size = %8.2f MB\n", __func__, total_size_org / 1024.0 / 1024.0);
LOG_INF("%s: quantized size = %8.2f MB\n", __func__, total_size_new / 1024.0 / 1024.0);
}
return true;
}
int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
switch (ctx->model.proj_type) {
case PROJECTOR_TYPE_LDP: