mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-19 09:15:18 +02:00
fixed a bug in vision with mrope, mrope is refactored to match upstream, should be more accurate now
This commit is contained in:
Concedo
+15
-11
@@ -1851,22 +1851,25 @@ static void load_grammar(const std::string & gammarstr)
|
||||
}
|
||||
}
|
||||
|
||||
static bool kcpp_eval_image(llama_context * ctx_llama, float * img_embd, int num_img_tokens, int n_batch, int * n_past) {
|
||||
int n_embd = llama_model_n_embd_inp(llama_get_model(ctx_llama));
|
||||
static bool kcpp_eval_image(llama_context * ctx_llama, const media_chunk & mediachunk, int n_batch, int * n_past, bool is2d) {
|
||||
float * img_embd = mediachunk.clp_img_embd;
|
||||
int num_img_tokens = mediachunk.clp_image_tokens;
|
||||
int img_nx = mediachunk.nx;
|
||||
int img_ny = mediachunk.ny;
|
||||
int n_embd_mmproj = llama_model_n_embd_inp(llama_get_model(ctx_llama));
|
||||
const int image_n_past = *n_past;
|
||||
|
||||
kcpp_embd_batch media_batch = kcpp_embd_batch(img_embd, num_img_tokens, image_n_past, use_mrope, is2d, img_nx, img_ny);
|
||||
|
||||
for (int i = 0; i < num_img_tokens; i += n_batch) {
|
||||
int n_eval = num_img_tokens - i;
|
||||
if (n_eval > n_batch) {
|
||||
n_eval = n_batch;
|
||||
}
|
||||
float * embd = img_embd+i*n_embd;
|
||||
kcpp_embd_batch media_batch = kcpp_embd_batch(embd, n_eval, *n_past, use_mrope);
|
||||
if (llama_decode(ctx_llama, media_batch.batch)) {
|
||||
const int n_eval = std::min(n_batch, num_img_tokens - i);
|
||||
llama_batch batch_embd_view = media_batch.get_view(i, n_eval, n_embd_mmproj);
|
||||
if (llama_decode(ctx_llama, batch_embd_view)) {
|
||||
fprintf(stderr, "\n%s : failed to eval image\n", __func__);
|
||||
return false;
|
||||
}
|
||||
*n_past += n_eval;
|
||||
}
|
||||
*n_past += num_img_tokens;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -4700,7 +4703,8 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
|
||||
{
|
||||
printf("\rProcessing Media Embedding %d (%d tokens)",(i+1), chunk.clp_image_tokens);
|
||||
}
|
||||
bool err = kcpp_eval_image(llama_ctx_v4,chunk.clp_img_embd,chunk.clp_image_tokens,kcpp_data->n_batch,&n_past);
|
||||
bool is2d = (media_objects[i].is_audio?false:true);
|
||||
bool err = kcpp_eval_image(llama_ctx_v4,chunk,kcpp_data->n_batch,&n_past,is2d);
|
||||
llavatokensevaled += chunk.clp_image_tokens;
|
||||
if(!err)
|
||||
{
|
||||
|
||||
+146
-72
@@ -477,92 +477,166 @@ int32_t kcpp_quick_sample(float * logits, const int n_logits, const std::vector<
|
||||
return logits_id[idx].second;
|
||||
}
|
||||
|
||||
kcpp_embd_batch::kcpp_embd_batch(float * embd, int32_t n_tokens, int32_t npast, bool use_mrope)
|
||||
{
|
||||
int32_t seq_id = 0;
|
||||
pos.resize(n_tokens * (use_mrope?4:1));
|
||||
std::fill(pos.begin(), pos.end(), 0);
|
||||
n_seq_id.resize(n_tokens);
|
||||
seq_ids.resize(n_tokens + 1);
|
||||
logits.resize(n_tokens);
|
||||
seq_id_0.resize(1);
|
||||
seq_id_0[0] = seq_id;
|
||||
seq_ids [n_tokens] = nullptr;
|
||||
batch = {
|
||||
/*n_tokens =*/ n_tokens,
|
||||
/*tokens =*/ nullptr,
|
||||
/*embd =*/ embd,
|
||||
/*pos =*/ pos.data(),
|
||||
/*n_seq_id =*/ n_seq_id.data(),
|
||||
/*seq_id =*/ seq_ids.data(),
|
||||
/*logits =*/ logits.data(),
|
||||
};
|
||||
void kcpp_embd_batch::init_kcpp_batch(int32_t n_tokens,
|
||||
int32_t npast,
|
||||
bool use_mrope,
|
||||
bool return_all_logits,
|
||||
bool mrope_is_image,
|
||||
int img_nx,
|
||||
int img_ny) {
|
||||
const int n_pos_per_embd = use_mrope ? 4 : 1;
|
||||
const llama_seq_id seq_id = 0;
|
||||
|
||||
if(!use_mrope)
|
||||
{
|
||||
for (int i = 0; i < n_tokens; i++) {
|
||||
batch.pos [i] = npast + i;
|
||||
batch.n_seq_id[i] = 1;
|
||||
batch.seq_id [i] = seq_id_0.data();
|
||||
batch.logits [i] = false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (int i = 0; i < n_tokens; i++) {
|
||||
batch.n_seq_id[i] = 1;
|
||||
batch.seq_id [i] = seq_id_0.data();
|
||||
batch.logits [i] = false;
|
||||
}
|
||||
for (int j = 0; j < batch.n_tokens * 3; j++) {
|
||||
batch.pos[j] = npast + (j % batch.n_tokens);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (use_mrope && mrope_is_image) {
|
||||
GGML_ASSERT(img_nx > 0 && img_ny > 0);
|
||||
GGML_ASSERT(img_nx * img_ny == n_tokens);
|
||||
}
|
||||
|
||||
kcpp_embd_batch::kcpp_embd_batch(std::vector<llama_token> & tokens, int32_t npast, bool use_mrope, bool return_all_logits)
|
||||
{
|
||||
int32_t seq_id = 0;
|
||||
int32_t n_tokens = tokens.size();
|
||||
pos.resize(n_tokens * (use_mrope?4:1));
|
||||
pos.resize(n_tokens * n_pos_per_embd);
|
||||
std::fill(pos.begin(), pos.end(), 0);
|
||||
|
||||
n_seq_id.resize(n_tokens);
|
||||
seq_ids.resize(n_tokens + 1);
|
||||
logits.resize(n_tokens);
|
||||
seq_id_0.resize(1);
|
||||
seq_id_0[0] = seq_id;
|
||||
|
||||
seq_id_0[0] = seq_id;
|
||||
seq_ids[n_tokens] = nullptr;
|
||||
|
||||
batch.pos = pos.data();
|
||||
batch.n_seq_id = n_seq_id.data();
|
||||
batch.seq_id = seq_ids.data();
|
||||
batch.logits = logits.data();
|
||||
|
||||
for (int i = 0; i < n_tokens; ++i) {
|
||||
n_seq_id[i] = 1;
|
||||
seq_ids[i] = seq_id_0.data();
|
||||
logits[i] = return_all_logits;
|
||||
}
|
||||
|
||||
// ---- position encoding ----
|
||||
if (!use_mrope) {
|
||||
for (int i = 0; i < n_tokens; ++i) {
|
||||
pos[i] = npast + i;
|
||||
}
|
||||
} else if (!mrope_is_image) {
|
||||
// 1D M-RoPE (audio / embedding stream)
|
||||
for (int i = 0; i < n_tokens; ++i) {
|
||||
pos[i + 0 * n_tokens] = npast + i;
|
||||
pos[i + 1 * n_tokens] = npast + i;
|
||||
pos[i + 2 * n_tokens] = npast + i;
|
||||
pos[i + 3 * n_tokens] = 0;
|
||||
}
|
||||
} else {
|
||||
// 2D image M-RoPE
|
||||
int idx = 0;
|
||||
for (int y = 0; y < img_ny; ++y) {
|
||||
for (int x = 0; x < img_nx; ++x) {
|
||||
pos[idx + 0 * n_tokens] = npast;
|
||||
pos[idx + 1 * n_tokens] = npast + y;
|
||||
pos[idx + 2 * n_tokens] = npast + x;
|
||||
pos[idx + 3 * n_tokens] = 0;
|
||||
++idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Always request logits for last token
|
||||
logits[n_tokens - 1] = true;
|
||||
}
|
||||
|
||||
//for embeddings
|
||||
kcpp_embd_batch::kcpp_embd_batch(float * embd,
|
||||
int32_t n_tokens,
|
||||
int32_t npast,
|
||||
bool use_mrope,
|
||||
bool mrope_is_image,
|
||||
int img_nx,
|
||||
int img_ny) {
|
||||
batch = {
|
||||
/*n_tokens =*/ n_tokens,
|
||||
/*tokens =*/ tokens.data(),
|
||||
/*embd =*/ nullptr,
|
||||
/*pos =*/ pos.data(),
|
||||
/*n_seq_id =*/ n_seq_id.data(),
|
||||
/*seq_id =*/ seq_ids.data(),
|
||||
/*logits =*/ logits.data(),
|
||||
/* n_tokens = */ n_tokens,
|
||||
/* tokens = */ nullptr,
|
||||
/* embd = */ embd,
|
||||
/* pos = */ nullptr,
|
||||
/* n_seq_id = */ nullptr,
|
||||
/* seq_id = */ nullptr,
|
||||
/* logits = */ nullptr,
|
||||
};
|
||||
|
||||
if(!use_mrope)
|
||||
{
|
||||
for (int i = 0; i < n_tokens; i++) {
|
||||
batch.pos [i] = npast + i;
|
||||
batch.n_seq_id[i] = 1;
|
||||
batch.seq_id [i] = seq_id_0.data();
|
||||
batch.logits [i] = (return_all_logits?true:false);
|
||||
init_kcpp_batch(n_tokens, npast, use_mrope,
|
||||
/*return_all_logits=*/false, mrope_is_image, img_nx, img_ny);
|
||||
}
|
||||
|
||||
// for tokens
|
||||
kcpp_embd_batch::kcpp_embd_batch(std::vector<llama_token> & tokens,
|
||||
int32_t npast,
|
||||
bool use_mrope,
|
||||
bool return_all_logits,
|
||||
bool mrope_is_image,
|
||||
int img_nx,
|
||||
int img_ny) {
|
||||
batch = {
|
||||
/* n_tokens = */ (int32_t) tokens.size(),
|
||||
/* tokens = */ tokens.data(),
|
||||
/* embd = */ nullptr,
|
||||
/* pos = */ nullptr,
|
||||
/* n_seq_id = */ nullptr,
|
||||
/* seq_id = */ nullptr,
|
||||
/* logits = */ nullptr,
|
||||
};
|
||||
|
||||
init_kcpp_batch(batch.n_tokens, npast, use_mrope, return_all_logits, mrope_is_image, img_nx, img_ny);
|
||||
}
|
||||
|
||||
llama_batch kcpp_embd_batch::get_view(int offset, int n_tokens, int n_embd_mmproj) {
|
||||
GGML_ASSERT(offset >= 0);
|
||||
GGML_ASSERT(n_tokens > 0);
|
||||
GGML_ASSERT(offset + n_tokens <= batch.n_tokens);
|
||||
|
||||
const int total_tokens = batch.n_tokens;
|
||||
llama_pos * pos_ptr = nullptr;
|
||||
|
||||
// Detect M-RoPE vs normal RoPE
|
||||
const bool is_mrope = (pos.size() > (size_t)total_tokens);
|
||||
|
||||
pos_view.clear();
|
||||
|
||||
if (is_mrope) {
|
||||
const int n_pos_per_embd = pos.size() / total_tokens;
|
||||
GGML_ASSERT(n_pos_per_embd == 4);
|
||||
|
||||
// Layout:
|
||||
// src: [dim0_all_tokens][dim1_all_tokens][dim2_all_tokens][dim3_all_tokens]
|
||||
// dst: same layout, but only [offset : offset + n_tokens]
|
||||
pos_view.reserve(n_tokens * n_pos_per_embd);
|
||||
|
||||
for (int dim = 0; dim < n_pos_per_embd; ++dim) {
|
||||
const llama_pos * src =
|
||||
pos.data() + dim * total_tokens + offset;
|
||||
|
||||
pos_view.insert(
|
||||
pos_view.end(),
|
||||
src,
|
||||
src + n_tokens
|
||||
);
|
||||
}
|
||||
|
||||
pos_ptr = pos_view.data();
|
||||
}
|
||||
else
|
||||
{
|
||||
for (int i = 0; i < n_tokens; i++) {
|
||||
batch.n_seq_id[i] = 1;
|
||||
batch.seq_id [i] = seq_id_0.data();
|
||||
batch.logits [i] = (return_all_logits?true:false);
|
||||
}
|
||||
for (int j = 0; j < batch.n_tokens * 3; j++) {
|
||||
batch.pos[j] = npast + (j % batch.n_tokens);
|
||||
}
|
||||
else {
|
||||
// Normal RoPE: contiguous slice
|
||||
pos_ptr = pos.data() + offset;
|
||||
}
|
||||
batch.logits[n_tokens - 1] = true;
|
||||
|
||||
return {
|
||||
/* n_tokens = */ n_tokens,
|
||||
/* tokens = */ nullptr,
|
||||
/* embd = */ batch.embd ? batch.embd + offset*n_embd_mmproj : nullptr,
|
||||
/* pos = */ pos_ptr,
|
||||
/* n_seq_id = */ batch.n_seq_id + offset,
|
||||
/* seq_id = */ batch.seq_id + offset,
|
||||
/* logits = */ batch.logits + offset,
|
||||
};
|
||||
}
|
||||
|
||||
std::vector<std::string> split_string(const std::string& input, const std::string& separator) {
|
||||
|
||||
+44
-9
@@ -69,13 +69,48 @@ int32_t kcpp_quick_sample(float * logits, const int n_logits, const std::vector<
|
||||
std::vector<std::string> split_string(const std::string& input, const std::string& separator);
|
||||
bool kcpp_decode_audio_from_buf(const unsigned char * buf_in, size_t len, int target_sampler_rate, std::vector<float> & pcmf32_mono);
|
||||
|
||||
struct kcpp_embd_batch { //duplcated from llava_embd_batch
|
||||
std::vector<int32_t> pos;
|
||||
std::vector<int32_t> n_seq_id;
|
||||
std::vector<int32_t> seq_id_0;
|
||||
std::vector<int32_t *> seq_ids;
|
||||
std::vector<int8_t> logits;
|
||||
//duplcated and modified from llava_embd_batch
|
||||
struct kcpp_embd_batch {
|
||||
std::vector<llama_pos> pos;
|
||||
std::vector<llama_pos> pos_view;
|
||||
std::vector<int32_t> n_seq_id;
|
||||
std::vector<llama_seq_id> seq_id_0;
|
||||
std::vector<llama_seq_id*> seq_ids;
|
||||
std::vector<int8_t> logits;
|
||||
llama_batch batch;
|
||||
kcpp_embd_batch(float * embd, int32_t n_tokens, int32_t npast, bool use_mrope);
|
||||
kcpp_embd_batch(std::vector<llama_token> & tokens, int32_t npast, bool use_mrope, bool return_all_logits);
|
||||
};
|
||||
|
||||
llama_batch get_view(int offset, int n_tokens, int n_embd_mmproj);
|
||||
|
||||
// Embedding constructor
|
||||
kcpp_embd_batch(
|
||||
float * embd,
|
||||
int32_t n_tokens,
|
||||
int32_t npast,
|
||||
bool use_mrope,
|
||||
bool mrope_is_image = false,
|
||||
int img_nx = 0,
|
||||
int img_ny = 0
|
||||
);
|
||||
|
||||
// Token constructor
|
||||
kcpp_embd_batch(
|
||||
std::vector<llama_token> & tokens,
|
||||
int32_t npast,
|
||||
bool use_mrope,
|
||||
bool return_all_logits,
|
||||
bool mrope_is_image = false,
|
||||
int img_nx = 0,
|
||||
int img_ny = 0
|
||||
);
|
||||
|
||||
private:
|
||||
void init_kcpp_batch(
|
||||
int32_t n_tokens,
|
||||
int32_t npast,
|
||||
bool use_mrope,
|
||||
bool return_all_logits,
|
||||
bool mrope_is_image,
|
||||
int img_nx,
|
||||
int img_ny
|
||||
);
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user