From fb31059f9c3ce42b31303538c4d1fce98363e88e Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Fri, 19 Dec 2025 01:23:52 +0800 Subject: [PATCH] fixed a bug in vision with mrope, mrope is refactored to match upstream, should be more accurate now --- gpttype_adapter.cpp | 26 +++--- otherarch/utils.cpp | 218 +++++++++++++++++++++++++++++--------------- otherarch/utils.h | 53 +++++++++-- 3 files changed, 205 insertions(+), 92 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 53bfba794..7cfe83a85 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -1851,22 +1851,25 @@ static void load_grammar(const std::string & gammarstr) } } -static bool kcpp_eval_image(llama_context * ctx_llama, float * img_embd, int num_img_tokens, int n_batch, int * n_past) { - int n_embd = llama_model_n_embd_inp(llama_get_model(ctx_llama)); +static bool kcpp_eval_image(llama_context * ctx_llama, const media_chunk & mediachunk, int n_batch, int * n_past, bool is2d) { + float * img_embd = mediachunk.clp_img_embd; + int num_img_tokens = mediachunk.clp_image_tokens; + int img_nx = mediachunk.nx; + int img_ny = mediachunk.ny; + int n_embd_mmproj = llama_model_n_embd_inp(llama_get_model(ctx_llama)); + const int image_n_past = *n_past; + + kcpp_embd_batch media_batch = kcpp_embd_batch(img_embd, num_img_tokens, image_n_past, use_mrope, is2d, img_nx, img_ny); for (int i = 0; i < num_img_tokens; i += n_batch) { - int n_eval = num_img_tokens - i; - if (n_eval > n_batch) { - n_eval = n_batch; - } - float * embd = img_embd+i*n_embd; - kcpp_embd_batch media_batch = kcpp_embd_batch(embd, n_eval, *n_past, use_mrope); - if (llama_decode(ctx_llama, media_batch.batch)) { + const int n_eval = std::min(n_batch, num_img_tokens - i); + llama_batch batch_embd_view = media_batch.get_view(i, n_eval, n_embd_mmproj); + if (llama_decode(ctx_llama, batch_embd_view)) { fprintf(stderr, "\n%s : failed to eval image\n", __func__); return false; } - *n_past += n_eval; } + *n_past += num_img_tokens; return true; } @@ -4700,7 +4703,8 @@ generation_outputs gpttype_generate(const generation_inputs inputs) { printf("\rProcessing Media Embedding %d (%d tokens)",(i+1), chunk.clp_image_tokens); } - bool err = kcpp_eval_image(llama_ctx_v4,chunk.clp_img_embd,chunk.clp_image_tokens,kcpp_data->n_batch,&n_past); + bool is2d = (media_objects[i].is_audio?false:true); + bool err = kcpp_eval_image(llama_ctx_v4,chunk,kcpp_data->n_batch,&n_past,is2d); llavatokensevaled += chunk.clp_image_tokens; if(!err) { diff --git a/otherarch/utils.cpp b/otherarch/utils.cpp index bf55b09a7..60df611f0 100644 --- a/otherarch/utils.cpp +++ b/otherarch/utils.cpp @@ -477,92 +477,166 @@ int32_t kcpp_quick_sample(float * logits, const int n_logits, const std::vector< return logits_id[idx].second; } -kcpp_embd_batch::kcpp_embd_batch(float * embd, int32_t n_tokens, int32_t npast, bool use_mrope) -{ - int32_t seq_id = 0; - pos.resize(n_tokens * (use_mrope?4:1)); - std::fill(pos.begin(), pos.end(), 0); - n_seq_id.resize(n_tokens); - seq_ids.resize(n_tokens + 1); - logits.resize(n_tokens); - seq_id_0.resize(1); - seq_id_0[0] = seq_id; - seq_ids [n_tokens] = nullptr; - batch = { - /*n_tokens =*/ n_tokens, - /*tokens =*/ nullptr, - /*embd =*/ embd, - /*pos =*/ pos.data(), - /*n_seq_id =*/ n_seq_id.data(), - /*seq_id =*/ seq_ids.data(), - /*logits =*/ logits.data(), - }; +void kcpp_embd_batch::init_kcpp_batch(int32_t n_tokens, + int32_t npast, + bool use_mrope, + bool return_all_logits, + bool mrope_is_image, + int img_nx, + int img_ny) { + const int n_pos_per_embd = use_mrope ? 4 : 1; + const llama_seq_id seq_id = 0; - if(!use_mrope) - { - for (int i = 0; i < n_tokens; i++) { - batch.pos [i] = npast + i; - batch.n_seq_id[i] = 1; - batch.seq_id [i] = seq_id_0.data(); - batch.logits [i] = false; - } - } - else - { - for (int i = 0; i < n_tokens; i++) { - batch.n_seq_id[i] = 1; - batch.seq_id [i] = seq_id_0.data(); - batch.logits [i] = false; - } - for (int j = 0; j < batch.n_tokens * 3; j++) { - batch.pos[j] = npast + (j % batch.n_tokens); - } - } -} + if (use_mrope && mrope_is_image) { + GGML_ASSERT(img_nx > 0 && img_ny > 0); + GGML_ASSERT(img_nx * img_ny == n_tokens); + } -kcpp_embd_batch::kcpp_embd_batch(std::vector & tokens, int32_t npast, bool use_mrope, bool return_all_logits) -{ - int32_t seq_id = 0; - int32_t n_tokens = tokens.size(); - pos.resize(n_tokens * (use_mrope?4:1)); + pos.resize(n_tokens * n_pos_per_embd); std::fill(pos.begin(), pos.end(), 0); + n_seq_id.resize(n_tokens); seq_ids.resize(n_tokens + 1); logits.resize(n_tokens); seq_id_0.resize(1); - seq_id_0[0] = seq_id; + + seq_id_0[0] = seq_id; seq_ids[n_tokens] = nullptr; + + batch.pos = pos.data(); + batch.n_seq_id = n_seq_id.data(); + batch.seq_id = seq_ids.data(); + batch.logits = logits.data(); + + for (int i = 0; i < n_tokens; ++i) { + n_seq_id[i] = 1; + seq_ids[i] = seq_id_0.data(); + logits[i] = return_all_logits; + } + + // ---- position encoding ---- + if (!use_mrope) { + for (int i = 0; i < n_tokens; ++i) { + pos[i] = npast + i; + } + } else if (!mrope_is_image) { + // 1D M-RoPE (audio / embedding stream) + for (int i = 0; i < n_tokens; ++i) { + pos[i + 0 * n_tokens] = npast + i; + pos[i + 1 * n_tokens] = npast + i; + pos[i + 2 * n_tokens] = npast + i; + pos[i + 3 * n_tokens] = 0; + } + } else { + // 2D image M-RoPE + int idx = 0; + for (int y = 0; y < img_ny; ++y) { + for (int x = 0; x < img_nx; ++x) { + pos[idx + 0 * n_tokens] = npast; + pos[idx + 1 * n_tokens] = npast + y; + pos[idx + 2 * n_tokens] = npast + x; + pos[idx + 3 * n_tokens] = 0; + ++idx; + } + } + } + + // Always request logits for last token + logits[n_tokens - 1] = true; +} + +//for embeddings +kcpp_embd_batch::kcpp_embd_batch(float * embd, + int32_t n_tokens, + int32_t npast, + bool use_mrope, + bool mrope_is_image, + int img_nx, + int img_ny) { batch = { - /*n_tokens =*/ n_tokens, - /*tokens =*/ tokens.data(), - /*embd =*/ nullptr, - /*pos =*/ pos.data(), - /*n_seq_id =*/ n_seq_id.data(), - /*seq_id =*/ seq_ids.data(), - /*logits =*/ logits.data(), + /* n_tokens = */ n_tokens, + /* tokens = */ nullptr, + /* embd = */ embd, + /* pos = */ nullptr, + /* n_seq_id = */ nullptr, + /* seq_id = */ nullptr, + /* logits = */ nullptr, }; - if(!use_mrope) - { - for (int i = 0; i < n_tokens; i++) { - batch.pos [i] = npast + i; - batch.n_seq_id[i] = 1; - batch.seq_id [i] = seq_id_0.data(); - batch.logits [i] = (return_all_logits?true:false); + init_kcpp_batch(n_tokens, npast, use_mrope, + /*return_all_logits=*/false, mrope_is_image, img_nx, img_ny); +} + +// for tokens +kcpp_embd_batch::kcpp_embd_batch(std::vector & tokens, + int32_t npast, + bool use_mrope, + bool return_all_logits, + bool mrope_is_image, + int img_nx, + int img_ny) { + batch = { + /* n_tokens = */ (int32_t) tokens.size(), + /* tokens = */ tokens.data(), + /* embd = */ nullptr, + /* pos = */ nullptr, + /* n_seq_id = */ nullptr, + /* seq_id = */ nullptr, + /* logits = */ nullptr, + }; + + init_kcpp_batch(batch.n_tokens, npast, use_mrope, return_all_logits, mrope_is_image, img_nx, img_ny); +} + +llama_batch kcpp_embd_batch::get_view(int offset, int n_tokens, int n_embd_mmproj) { + GGML_ASSERT(offset >= 0); + GGML_ASSERT(n_tokens > 0); + GGML_ASSERT(offset + n_tokens <= batch.n_tokens); + + const int total_tokens = batch.n_tokens; + llama_pos * pos_ptr = nullptr; + + // Detect M-RoPE vs normal RoPE + const bool is_mrope = (pos.size() > (size_t)total_tokens); + + pos_view.clear(); + + if (is_mrope) { + const int n_pos_per_embd = pos.size() / total_tokens; + GGML_ASSERT(n_pos_per_embd == 4); + + // Layout: + // src: [dim0_all_tokens][dim1_all_tokens][dim2_all_tokens][dim3_all_tokens] + // dst: same layout, but only [offset : offset + n_tokens] + pos_view.reserve(n_tokens * n_pos_per_embd); + + for (int dim = 0; dim < n_pos_per_embd; ++dim) { + const llama_pos * src = + pos.data() + dim * total_tokens + offset; + + pos_view.insert( + pos_view.end(), + src, + src + n_tokens + ); } + + pos_ptr = pos_view.data(); } - else - { - for (int i = 0; i < n_tokens; i++) { - batch.n_seq_id[i] = 1; - batch.seq_id [i] = seq_id_0.data(); - batch.logits [i] = (return_all_logits?true:false); - } - for (int j = 0; j < batch.n_tokens * 3; j++) { - batch.pos[j] = npast + (j % batch.n_tokens); - } + else { + // Normal RoPE: contiguous slice + pos_ptr = pos.data() + offset; } - batch.logits[n_tokens - 1] = true; + + return { + /* n_tokens = */ n_tokens, + /* tokens = */ nullptr, + /* embd = */ batch.embd ? batch.embd + offset*n_embd_mmproj : nullptr, + /* pos = */ pos_ptr, + /* n_seq_id = */ batch.n_seq_id + offset, + /* seq_id = */ batch.seq_id + offset, + /* logits = */ batch.logits + offset, + }; } std::vector split_string(const std::string& input, const std::string& separator) { diff --git a/otherarch/utils.h b/otherarch/utils.h index b4c17dc0d..bf3b0ab83 100644 --- a/otherarch/utils.h +++ b/otherarch/utils.h @@ -69,13 +69,48 @@ int32_t kcpp_quick_sample(float * logits, const int n_logits, const std::vector< std::vector split_string(const std::string& input, const std::string& separator); bool kcpp_decode_audio_from_buf(const unsigned char * buf_in, size_t len, int target_sampler_rate, std::vector & pcmf32_mono); -struct kcpp_embd_batch { //duplcated from llava_embd_batch - std::vector pos; - std::vector n_seq_id; - std::vector seq_id_0; - std::vector seq_ids; - std::vector logits; +//duplcated and modified from llava_embd_batch +struct kcpp_embd_batch { + std::vector pos; + std::vector pos_view; + std::vector n_seq_id; + std::vector seq_id_0; + std::vector seq_ids; + std::vector logits; llama_batch batch; - kcpp_embd_batch(float * embd, int32_t n_tokens, int32_t npast, bool use_mrope); - kcpp_embd_batch(std::vector & tokens, int32_t npast, bool use_mrope, bool return_all_logits); -}; \ No newline at end of file + + llama_batch get_view(int offset, int n_tokens, int n_embd_mmproj); + + // Embedding constructor + kcpp_embd_batch( + float * embd, + int32_t n_tokens, + int32_t npast, + bool use_mrope, + bool mrope_is_image = false, + int img_nx = 0, + int img_ny = 0 + ); + + // Token constructor + kcpp_embd_batch( + std::vector & tokens, + int32_t npast, + bool use_mrope, + bool return_all_logits, + bool mrope_is_image = false, + int img_nx = 0, + int img_ny = 0 + ); + +private: + void init_kcpp_batch( + int32_t n_tokens, + int32_t npast, + bool use_mrope, + bool return_all_logits, + bool mrope_is_image, + int img_nx, + int img_ny + ); +};