diff --git a/src/llama-ext.h b/src/llama-ext.h index e1d8034b75..35d6e58adf 100644 --- a/src/llama-ext.h +++ b/src/llama-ext.h @@ -125,7 +125,8 @@ LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_mode // returns the number of extracted layers from target model LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model); -// retrieves the whole token embedding matrix in F32 format (n_embd x n_vocab) -// returns total number of elements (usually n_embd * n_vocab) or 0 on error +// retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab) +// returns total number of elements or 0 on error // if out is nullptr, returns the number of tokens without writing to out +// caller must allocate enough memory for out before calling LLAMA_API uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out); diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp index 8615b14448..5596620f07 100644 --- a/src/models/qwen3vl.cpp +++ b/src/models/qwen3vl.cpp @@ -186,7 +186,7 @@ llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_par cur = ggml_concat(ctx0, neg_inf, cur, 0); // [padded .. n_vocab_out, n_stream] } else if (n_vocab_in < n_vocab_out) { - GGML_ASSERT("invalid case"); + GGML_ABORT("invalid case"); } cb(cur, "result_output", -1); diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 65d98855f5..50644bf1e5 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -4745,6 +4745,22 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { const int64_t n_frames = (int64_t) params->codes->size() / n_codes; GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w); + // bound each code against its codebook's vocab before it becomes + // a ggml_get_rows index into the codebook tensor + const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1]; + const int64_t vocab_rest = model.c2w.quant_rest_cb_w->ne[1]; + for (int64_t f = 0; f < n_frames; f++) { + for (int64_t g = 0; g < n_codes; g++) { + const int32_t c = (*params->codes)[f * n_codes + g]; + const int64_t vocab = (g == 0) ? vocab_first : vocab_rest; + if (c < 0 || (int64_t) c >= vocab) { + LOG_ERR("%s: code out of range (frame %lld, group %lld, code %d, vocab %lld)\n", + __func__, (long long) f, (long long) g, c, (long long) vocab); + return false; + } + } + } + std::vector codes(n_frames_w * n_codes, 0); for (int64_t f = 0; f < n_frames; f++) { for (int64_t g = 0; g < n_codes; g++) { @@ -4768,6 +4784,12 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { offset += nb; } } else { + // code0 indexes gen_code_out_embd_w via ggml_get_rows; bound it + const int64_t vocab0 = model.gen_code_out_embd_w->ne[1]; + if (params->code0 < 0 || (int64_t) params->code0 >= vocab0) { + LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) vocab0); + return false; + } std::vector code0 = { params->code0 }; set_input_i32("inp_code0", code0); diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index d79e606ac9..2811d24df7 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -821,7 +821,7 @@ bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * // reflect pad by (n_fft - hop) / 2 = 384, matching center=False STFT framing const int pad = (hparams.audio_n_fft - hparams.audio_hop_len) / 2; - if ((int) n_samples < pad + 1) { + if (n_samples < (size_t) pad + 1) { return false; } diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index e98e9dfeeb..3b7762c214 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -49,7 +49,11 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri return LLAMA_TOKEN_NULL; } -static void write_wav16(std::vector & buf, const std::vector & pcm, int32_t rate) { +static bool write_wav16(std::vector & buf, const std::vector & pcm, int32_t rate) { + // RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header + if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) { + return false; + } const uint32_t data_sz = (uint32_t) (pcm.size() * 2); const uint32_t riff_sz = 36 + data_sz; const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2; @@ -68,6 +72,7 @@ static void write_wav16(std::vector & buf, const std::vector & pcm, int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, v)) * 32767.0f); put(&s, 2); } + return true; } class mtmd_gen_audio_pipeline { @@ -272,7 +277,10 @@ public: } out_buf.clear(); - write_wav16(out_buf, audio_pcm, info.sample_rate); + if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { + LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); + return 1; + } *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; @@ -307,7 +315,10 @@ private: return false; } tok_embd.resize(n_tok_embd); - llama_model_get_tok_embd(model, tok_embd.data()); + if (llama_model_get_tok_embd(model, tok_embd.data()) != n_tok_embd) { + LOG_ERR("mtmd_helper_gen_audio: token embedding copy failed\n"); + return false; + } specials_ok = true; return true; } diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 07c26efdf2..361af6dfb5 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -386,6 +386,15 @@ struct mtmd_context { "hint: you may be using wrong mmproj\n", n_embd_text, n_embd_clip)); } + if (ctx_gen_a) { + int n_embd_gen = clip_n_mmproj_embd(ctx_gen_a); + if (n_embd_text > 0 && n_embd_text != n_embd_gen) { + throw std::runtime_error(string_format( + "mismatch between text model (n_embd = %d) and gen-audio mmproj (n_embd = %d)\n" + "hint: you may be using wrong mmproj\n", + n_embd_text, n_embd_gen)); + } + } if (ctx_v) { init_vision(); }