#pragma once // shared internal utilities for the mtmd-helper-*.cpp translation units // (mtmd-helper.cpp, mtmd-helper-gen.cpp) // NOT part of the public mtmd-helper.h API #include "ggml.h" #include "llama.h" #include "mtmd.h" #include #include #include #include // // logging // struct mtmd_helper_logger { ggml_log_callback default_callback = [](ggml_log_level level, const char * text, void * user_data) { (void) level; (void) user_data; fputs(text, stderr); fflush(stderr); }; ggml_log_callback log_callback = default_callback; void * log_callback_user_data; void log_v(enum ggml_log_level level, const char * format, va_list args) { if (format == NULL) { return; } va_list args_copy; va_copy(args_copy, args); char buffer[128]; int len = vsnprintf(buffer, 128, format, args); if (len < 128) { log_callback(level, buffer, log_callback_user_data); } else { char * buffer2 = (char *) calloc(len + 1, sizeof(char)); vsnprintf(buffer2, len + 1, format, args_copy); buffer2[len] = 0; log_callback(level, buffer2, log_callback_user_data); free(buffer2); } va_end(args_copy); } void log(enum ggml_log_level level, const char * format, ...) { va_list args; va_start(args, format); log_v(level, format, args); va_end(args); } }; // inline, so all TUs including this header share one instance inline mtmd_helper_logger g_logger; #define LOG_DBG(...) g_logger.log(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) #define LOG_INF(...) g_logger.log(GGML_LOG_LEVEL_INFO, __VA_ARGS__) #define LOG_WRN(...) g_logger.log(GGML_LOG_LEVEL_WARN, __VA_ARGS__) #define LOG_ERR(...) g_logger.log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__) // // embd batch // // helper struct to make working with embd batch easier // note: this will be removed after llama_batch_ext refactoring struct decode_embd_batch { int n_pos_per_embd; int n_mmproj_embd; std::vector pos; std::vector pos_view; // used by mrope std::vector n_seq_id; std::vector seq_id_0; std::vector seq_ids; std::vector logits; llama_batch batch; decode_embd_batch(float * embd, int32_t n_tokens, int n_pos_per_embd, int n_mmproj_embd) : n_pos_per_embd(n_pos_per_embd), n_mmproj_embd(n_mmproj_embd) { GGML_ASSERT(n_tokens > 0 && n_pos_per_embd > 0 && n_mmproj_embd > 0); pos .resize((size_t) n_tokens * (size_t) n_pos_per_embd); n_seq_id.resize(n_tokens); seq_ids .resize(n_tokens + 1); logits .resize(n_tokens); seq_id_0.resize(1); seq_ids [n_tokens] = nullptr; batch = { /*n_tokens =*/ n_tokens, /*tokens =*/ nullptr, /*embd =*/ embd, /*pos =*/ pos.data(), /*n_seq_id =*/ n_seq_id.data(), /*seq_id =*/ seq_ids.data(), /*logits =*/ logits.data(), }; } void set_position_normal(llama_pos pos_0, llama_seq_id seq_id) { seq_id_0[0] = seq_id; for (int i = 0; i < batch.n_tokens; i++) { batch.pos [i] = pos_0 + i; batch.n_seq_id[i] = 1; batch.seq_id [i] = seq_id_0.data(); batch.logits [i] = false; } } // M-RoPE for image void set_position_mrope_2d(const std::vector & rel_pos, llama_seq_id seq_id) { GGML_ASSERT(n_pos_per_embd == 4); GGML_ASSERT(!rel_pos.empty() && (int32_t)rel_pos.size() == batch.n_tokens); seq_id_0[0] = seq_id; for (int32_t i = 0; i < batch.n_tokens; i++) { const size_t idx = (size_t) i; const size_t n_tokens = (size_t) batch.n_tokens; pos[idx ] = rel_pos[i].t; pos[idx + n_tokens ] = rel_pos[i].y; pos[idx + n_tokens * 2 ] = rel_pos[i].x; pos[idx + n_tokens * 3 ] = rel_pos[i].z; } for (int i = 0; i < batch.n_tokens; i++) { batch.n_seq_id[i] = 1; batch.seq_id [i] = seq_id_0.data(); batch.logits [i] = false; } } // M-RoPE for audio void set_position_mrope_1d(llama_pos pos_0, llama_seq_id seq_id) { GGML_ASSERT(n_pos_per_embd == 4); seq_id_0[0] = seq_id; for (int i = 0; i < batch.n_tokens; i++) { const size_t idx = (size_t) i; const size_t n_tokens = (size_t) batch.n_tokens; pos[idx ] = pos_0 + i; pos[idx + n_tokens ] = pos_0 + i; pos[idx + n_tokens * 2 ] = pos_0 + i; pos[idx + n_tokens * 3 ] = pos_0 + i; } for (int i = 0; i < batch.n_tokens; i++) { batch.n_seq_id[i] = 1; batch.seq_id [i] = seq_id_0.data(); batch.logits [i] = false; } } llama_batch get_view(int offset, int n_tokens) { GGML_ASSERT(offset >= 0 && n_tokens > 0 && offset + n_tokens <= batch.n_tokens); llama_pos * pos_ptr; pos_view.clear(); pos_view.reserve((size_t) n_tokens * (size_t) n_pos_per_embd); if (n_pos_per_embd > 1) { // mrope // for example, with layout of src: 1234...1234...1234...1234... // offset 2 will give us dst: 34...34...34...34... for (int i = 0; i < n_pos_per_embd; i++) { // assume n_tokens is less than or equal to batch.n_tokens // batch.n_tokens is number of **total** tokens // n_tokens is number of viewed token size_t src_idx = (size_t) i * (size_t) batch.n_tokens + (size_t) offset; pos_view.insert(pos_view.end(), pos.data() + src_idx, pos.data() + src_idx + n_tokens); } pos_ptr = pos_view.data(); } else { // normal pos_ptr = pos.data() + offset; } return { /*n_tokens =*/ n_tokens, /*tokens =*/ nullptr, /*embd =*/ batch.embd + offset * n_mmproj_embd, /*pos =*/ pos_ptr, /*n_seq_id =*/ batch.n_seq_id + offset, /*seq_id =*/ batch.seq_id + offset, /*logits =*/ batch.logits + offset, }; } };