From b4e10fc595a126930b68e3d6d52060f9094c3571 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Fri, 14 Aug 2026 17:34:27 +0200 Subject: [PATCH] impl mtmd cpp --- conversion/dots3.py | 1 + tools/mtmd/CMakeLists.txt | 1 + tools/mtmd/clip-graph.h | 6 ++ tools/mtmd/clip-impl.h | 11 ++- tools/mtmd/clip-model.h | 8 ++ tools/mtmd/clip.cpp | 129 ++++++++++++++++++++++++++++++-- tools/mtmd/models/dots3note.cpp | 61 +++++++++++++++ tools/mtmd/models/models.h | 5 ++ tools/mtmd/mtmd-audio.cpp | 94 +++++++++++++++++++++++ tools/mtmd/mtmd-audio.h | 9 +++ tools/mtmd/mtmd.cpp | 8 ++ 11 files changed, 325 insertions(+), 8 deletions(-) create mode 100644 tools/mtmd/models/dots3note.cpp diff --git a/conversion/dots3.py b/conversion/dots3.py index cd0686db2e..7811bb37e7 100644 --- a/conversion/dots3.py +++ b/conversion/dots3.py @@ -226,6 +226,7 @@ class Dots3NoteMmprojModel(MmprojModel): self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.DOTS3NOTE_A) self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["whisper_config"]["num_mel_bins"]) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-6) # Dots3NoteAudioRMSNorm default @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 769a44e0b7..b97e8e898f 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -29,6 +29,7 @@ add_library(mtmd models/models.h models/cogvlm.cpp models/conformer.cpp + models/dots3note.cpp models/dotsocr.cpp models/exaone4_5.cpp models/gemma4a.cpp diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.h index e12140ba00..2cf1b683a7 100644 --- a/tools/mtmd/clip-graph.h +++ b/tools/mtmd/clip-graph.h @@ -120,6 +120,12 @@ struct clip_graph { ffn_op_type type_op, int il) const; + ggml_tensor * build_moe_ffn( + ggml_tensor * cur, + const clip_layer & layer, + ffn_op_type type_op, + int il) const; + ggml_tensor * build_attn( ggml_tensor * wo, ggml_tensor * wo_b, diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index b2c8b40213..e8b3914773 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -74,6 +74,7 @@ #define KEY_SAM_N_HEAD "clip.vision.sam.head_count" #define KEY_SAM_N_BLOCK "clip.vision.sam.block_count" #define KEY_SAM_N_EMBD "clip.vision.sam.embedding_length" +#define KEY_VISION_N_EXPERT_USED "clip.vision.expert_used_count" // audio-specific #define KEY_AUDIO_PROJ_TYPE "clip.audio.projector_type" // for models with mixed modalities #define KEY_A_NUM_MEL_BINS "clip.audio.num_mel_bins" @@ -118,7 +119,11 @@ #define TN_FFN_DOWN "%s.blk.%d.ffn_down.%s" #define TN_FFN_GATE "%s.blk.%d.ffn_gate.%s" #define TN_FFN_UP "%s.blk.%d.ffn_up.%s" -#define TN_FFN_GATE "%s.blk.%d.ffn_gate.%s" +#define TN_FFN_GATE_INP "%s.blk.%d.ffn_gate_inp.%s" // MoE router (dots3note) +#define TN_FFN_GATE_EXPS "%s.blk.%d.ffn_gate_exps.%s" +#define TN_FFN_UP_EXPS "%s.blk.%d.ffn_up_exps.%s" +#define TN_FFN_DOWN_EXPS "%s.blk.%d.ffn_down_exps.%s" +#define TN_FFN_EXP_PROBS_B "%s.blk.%d.exp_probs_b.%s" #define TN_LN_1 "%s.blk.%d.ln1.%s" // layer norm #define TN_LN_2 "%s.blk.%d.ln2.%s" // layer norm #define TN_LS_1 "%s.blk.%d.ls1.%s" // layer scale @@ -470,6 +475,8 @@ enum projector_type { PROJECTOR_TYPE_COGVLM, PROJECTOR_TYPE_JANUS_PRO, PROJECTOR_TYPE_DOTS_OCR, + PROJECTOR_TYPE_DOTS3NOTE_V, + PROJECTOR_TYPE_DOTS3NOTE_A, PROJECTOR_TYPE_DEEPSEEKOCR, PROJECTOR_TYPE_DEEPSEEKOCR2, PROJECTOR_TYPE_LFM2A, @@ -532,6 +539,8 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_COGVLM, "cogvlm"}, { PROJECTOR_TYPE_JANUS_PRO, "janus_pro"}, { PROJECTOR_TYPE_DOTS_OCR, "dots_ocr"}, + { PROJECTOR_TYPE_DOTS3NOTE_V, "dots3note_v"}, + { PROJECTOR_TYPE_DOTS3NOTE_A, "dots3note_a"}, { PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"}, { PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"}, { PROJECTOR_TYPE_LFM2A, "lfm2a"}, diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index ad25c008e7..fcdabd633f 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -93,6 +93,7 @@ struct clip_hparams { float eps = 1e-6; float rope_theta = 0.0; + int32_t n_expert_used = 0; std::vector feature_layers; int32_t attn_window_size = 0; int32_t n_wa_pattern = 0; @@ -259,6 +260,13 @@ struct clip_layer { ggml_tensor * ff_down_w = nullptr; ggml_tensor * ff_down_b = nullptr; + // MoE FFN (dots3note vision pyramid blocks) + ggml_tensor * ff_gate_inp_w = nullptr; + ggml_tensor * ff_gate_exps_w = nullptr; + ggml_tensor * ff_up_exps_w = nullptr; + ggml_tensor * ff_down_exps_w = nullptr; + ggml_tensor * ff_exp_probs_b = nullptr; + // layernorm 2 (or pre-FFN norm) ggml_tensor * ln_2_w = nullptr; ggml_tensor * ln_2_b = nullptr; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 2fb2b5041d..18db6c6d09 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -515,11 +515,13 @@ ggml_tensor * clip_graph::build_vit( cb(cur, "ffn_inp_normed", il); // ffn - cur = build_ffn(cur, - layer.ff_up_w, layer.ff_up_b, - layer.ff_gate_w, layer.ff_gate_b, - layer.ff_down_w, layer.ff_down_b, - ffn_t, il); + cur = layer.ff_gate_exps_w + ? build_moe_ffn(cur, layer, ffn_t, il) + : build_ffn(cur, + layer.ff_up_w, layer.ff_up_b, + layer.ff_gate_w, layer.ff_gate_b, + layer.ff_down_w, layer.ff_down_b, + ffn_t, il); cb(cur, "ffn_out", il); @@ -700,6 +702,50 @@ ggml_tensor * clip_graph::build_ffn( return cur; } +// MoE FFN with sigmoid router and normalized top-k weights (dots3note vision) +// the router runs in fp32; exp_probs_b only affects expert selection, not the weights +ggml_tensor * clip_graph::build_moe_ffn(ggml_tensor * cur, const clip_layer & layer, ffn_op_type type_op, int il) const { + const int64_t n_tokens = cur->ne[1]; + const int64_t n_expert = layer.ff_gate_exps_w->ne[2]; + const int64_t n_expert_used = std::min((int64_t) hparams.n_expert_used, n_expert); + GGML_ASSERT(n_expert_used > 0); + GGML_ASSERT(type_op == FFN_SILU); + + ggml_tensor * probs = ggml_sigmoid(ctx0, build_mm(layer.ff_gate_inp_w, cur)); // [n_expert, n_tokens] + cb(probs, "ffn_moe_probs", il); + + ggml_tensor * sel = layer.ff_exp_probs_b + ? ggml_add(ctx0, probs, layer.ff_exp_probs_b) + : probs; + ggml_tensor * selected = ggml_top_k(ctx0, sel, n_expert_used); // [n_expert_used, n_tokens] + + ggml_tensor * weights = ggml_get_rows(ctx0, + ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected); + weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens); + weights = ggml_div(ctx0, weights, ggml_sum_rows(ctx0, weights)); + weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens); + cb(weights, "ffn_moe_weights", il); + + cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, n_tokens); + ggml_tensor * gate = ggml_mul_mat_id(ctx0, layer.ff_gate_exps_w, cur, selected); // [n_ff, n_expert_used, n_tokens] + ggml_tensor * up = ggml_mul_mat_id(ctx0, layer.ff_up_exps_w, cur, selected); + cur = ggml_mul(ctx0, ggml_silu(ctx0, gate), up); + cur = ggml_mul_mat_id(ctx0, layer.ff_down_exps_w, cur, selected); // [n_embd, n_expert_used, n_tokens] + cur = ggml_mul(ctx0, cur, weights); + + // sum over the selected experts + ggml_tensor * out = nullptr; + for (int64_t i = 0; i < n_expert_used; i++) { + ggml_tensor * v = ggml_view_2d(ctx0, cur, cur->ne[0], n_tokens, cur->nb[2], i * cur->nb[1]); + out = out ? ggml_add(ctx0, out, v) : v; + } + if (n_expert_used == 1) { + out = ggml_cont(ctx0, out); + } + cb(out, "ffn_moe_out", il); + return out; +} + ggml_tensor * clip_graph::build_attn( ggml_tensor * wo, ggml_tensor * wo_b, @@ -934,9 +980,14 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const builder = std::make_unique(ctx, img); } break; case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: // same ViT + merger; pyramid MoE is handled by build_vit { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: { @@ -1511,6 +1562,25 @@ struct clip_model_loader { get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup } break; + case PROJECTOR_TYPE_DOTS3NOTE_V: + { + hparams.rope_theta = 10000.0f; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + get_u32(KEY_VISION_N_EXPERT_USED, hparams.n_expert_used); + hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + hparams.rope_theta = 10000.0f; + hparams.audio_chunk_len = 60; // in seconds + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 400; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + } break; case PROJECTOR_TYPE_KIMIVL: { hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; @@ -2179,12 +2249,20 @@ struct clip_model_loader { layer.ln_1_b = get_tensor(string_format(TN_LN_1, prefix, il, "bias"), false); layer.ln_2_b = get_tensor(string_format(TN_LN_2, prefix, il, "bias"), false); + // MoE ffn (dots3note vision pyramid blocks); replaces the dense ffn when present + layer.ff_gate_inp_w = get_tensor(string_format(TN_FFN_GATE_INP, prefix, il, "weight"), false); + layer.ff_gate_exps_w = get_tensor(string_format(TN_FFN_GATE_EXPS, prefix, il, "weight"), false); + layer.ff_up_exps_w = get_tensor(string_format(TN_FFN_UP_EXPS, prefix, il, "weight"), false); + layer.ff_down_exps_w = get_tensor(string_format(TN_FFN_DOWN_EXPS, prefix, il, "weight"), false); + layer.ff_exp_probs_b = get_tensor(string_format(TN_FFN_EXP_PROBS_B, prefix, il, "weight"), false); + const bool is_moe = layer.ff_gate_exps_w != nullptr; + // ffn - layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, prefix, il, "weight")); + layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, prefix, il, "weight"), !is_moe); layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, prefix, il, "bias"), false); layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, prefix, il, "weight"), false); layer.ff_gate_b = get_tensor(string_format(TN_FFN_GATE, prefix, il, "bias"), false); - layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight")); + layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight"), !is_moe); layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "bias"), false); // mimovl per-head attention sink bias @@ -2666,6 +2744,7 @@ struct clip_model_loader { model.mm_patch_merger_w = get_tensor(string_format(TN_MM_PATCH_MERGER, "weight"), false); } break; case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: { model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); @@ -2676,6 +2755,23 @@ struct clip_model_loader { // post_trunk_norm: applied after all ViT blocks, before the merger model.post_ln_w = get_tensor(string_format(TN_MM_POST_NORM, "weight")); } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + model.conv2d_1_w = get_tensor(string_format(TN_CONV2D, 1, "weight")); + model.conv2d_1_b = get_tensor(string_format(TN_CONV2D, 1, "bias")); + model.conv2d_2_w = get_tensor(string_format(TN_CONV2D, 2, "weight")); + model.conv2d_2_b = get_tensor(string_format(TN_CONV2D, 2, "bias")); + model.conv2d_3_w = get_tensor(string_format(TN_CONV2D, 3, "weight")); + model.conv2d_3_b = get_tensor(string_format(TN_CONV2D, 3, "bias")); + model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight")); // no bias + // adapter: LayerNorm -> Linear -> GELU -> Linear + model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight")); + model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias")); + model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias")); + model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight")); + model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias")); + } break; case PROJECTOR_TYPE_ULTRAVOX: { model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); @@ -4058,12 +4154,18 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { } break; case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: { // dynamic size int n_merge = ctx->model.hparams.n_merge; int stride = n_merge * n_merge; n_patches = CLIP_ALIGN(n_patches, stride) / stride; } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + // 3x stride-2 conv2d over mel frames + n_patches = (img->nx() + 7) / 8; + } break; case PROJECTOR_TYPE_PIXTRAL: case PROJECTOR_TYPE_LIGHTONOCR: { @@ -4708,6 +4810,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { set_input_i32("minimax_pos_w", pos_w); } break; case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: { const int pw = image_size_width / patch_size; const int ph = image_size_height / patch_size; @@ -5198,6 +5301,16 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } set_input_i32("pos_w", pos_data); } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + GGML_ASSERT(imgs.entries.size() == 1); + const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = i; + } + set_input_i32("positions", positions); + } break; case PROJECTOR_TYPE_GEMMA4A: { GGML_ASSERT(imgs.entries.size() == 1); @@ -5686,6 +5799,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_PIXTRAL: case PROJECTOR_TYPE_LIGHTONOCR: case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: + case PROJECTOR_TYPE_DOTS3NOTE_A: return ctx->model.mm_2_w->ne[1]; case PROJECTOR_TYPE_MLP_NORM: return ctx->model.mm_3_b->ne[0]; diff --git a/tools/mtmd/models/dots3note.cpp b/tools/mtmd/models/dots3note.cpp new file mode 100644 index 0000000000..93c14fc796 --- /dev/null +++ b/tools/mtmd/models/dots3note.cpp @@ -0,0 +1,61 @@ +#include "models.h" + +ggml_cgraph * clip_graph_dots3note_a::build() { + // inp_raw: [n_frames, n_mel, 1], one 60s chunk, mel frames not padded + // the reference impl zero-masks conv inputs beyond the valid length at each stage; + // running on exactly the valid frames with the convs' zero padding is equivalent + ggml_tensor * inp = build_inp_raw(1); + GGML_ASSERT(inp->type == GGML_TYPE_F32); + + // 3x conv2d (k=3, s=2, p=1) + gelu + { + auto conv_block = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) { + x = ggml_conv_2d(ctx0, w, x, 2, 2, 1, 1, 1, 1); + x = ggml_add(ctx0, x, ggml_reshape_4d(ctx0, b, 1, 1, x->ne[2], 1)); + return ggml_gelu_erf(ctx0, x); + }; + + inp = conv_block(inp, model.conv2d_1_w, model.conv2d_1_b); + inp = conv_block(inp, model.conv2d_2_w, model.conv2d_2_b); + inp = conv_block(inp, model.conv2d_3_w, model.conv2d_3_b); + // inp: [OW=n_frames/8, OH=n_mel/8, OC=480, 1] + cb(inp, "after_conv_stem", -1); + } + + // [OW, OH, OC, 1] -> [OH*OC, OW], feature index f + OH*c (matches the reference permute+reshape) + inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3)); + inp = ggml_reshape_2d(ctx0, inp, inp->ne[0] * inp->ne[1], inp->ne[2]); + + // project to d_model (no bias) + inp = ggml_mul_mat(ctx0, model.conv_out_w, inp); + cb(inp, "after_conv_out", -1); + + const int64_t n_pos = inp->ne[1]; + + ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos); + ggml_set_name(positions, "positions"); + ggml_set_input(positions); + + // partial rotary: first half of each head, NEOX style + auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + return ggml_rope_ext(ctx0, cur, positions, nullptr, d_head/2, + GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + }; + + ggml_tensor * cur = build_vit(inp, n_pos, + NORM_TYPE_RMS, hparams.ffn_op, + nullptr, add_pos); + cb(cur, "after_transformer", -1); + + // adapter: LayerNorm -> Linear -> GELU -> Linear + cur = build_norm(cur, model.mm_norm_pre_w, model.mm_norm_pre_b, NORM_TYPE_NORMAL, 1e-5, -1); + cur = build_ffn(cur, + model.mm_1_w, model.mm_1_b, + nullptr, nullptr, + model.mm_2_w, model.mm_2_b, + FFN_GELU_ERF, -1); + cb(cur, "projected", -1); + + ggml_build_forward_expand(gf, cur); + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index ed8c1ea518..f8606fadbb 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -119,6 +119,11 @@ struct clip_graph_dotsocr : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_dots3note_a : clip_graph { + clip_graph_dots3note_a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; +}; + struct clip_graph_cogvlm : clip_graph { clip_graph_cogvlm(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index 98a8c11ee9..ce08f9e931 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -723,6 +723,100 @@ bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * sa return true; } +// +// mtmd_audio_preprocessor_dots3note +// +// Matches Dots3NoteFeatureExtractor: the waveform is split into 60s chunks and each chunk gets +// its own whisper-style log-mel (center=True, log10 + (max-8)/4). Only sample_length//hop frames +// per chunk are valid; the reference masks everything beyond them, so we emit exactly that many. +// + +void mtmd_audio_preprocessor_dots3note::initialize() { + cache.fill_sin_cos_table(hparams.audio_n_fft); + cache.fill_hann_window(hparams.audio_window_len, true); + cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate); +} + +bool mtmd_audio_preprocessor_dots3note::preprocess(const float * samples, + size_t n_samples, + std::vector & output) { + if (n_samples == 0) { + return false; + } + + GGML_ASSERT(!cache.sin_vals.empty()); + GGML_ASSERT(!cache.cos_vals.empty()); + GGML_ASSERT(!cache.filters.data.empty()); + + const int pad = hparams.audio_n_fft / 2; // center=True padding + const int hop = hparams.audio_hop_len; + const size_t chunk_samples = (size_t) hparams.audio_chunk_len * hparams.audio_sample_rate; + + for (size_t start = 0; start < n_samples; start += chunk_samples) { + const size_t n_chunk = std::min(chunk_samples, n_samples - start); + const float * chunk = samples + start; + + const int64_t n_valid = n_chunk / hop; + if (n_valid == 0) { + continue; // sub-hop tail, contributes no frames + } + + // reflect-pad the start; the reference zero-pads partial chunks to 60s before the STFT, + // so a partial chunk sees zeros past its end while a full chunk reflects its own tail + std::vector padded(n_chunk + 2 * pad, 0.0f); + for (int i = 0; i < pad; i++) { + int src = pad - i; + padded[i] = (src < (int) n_chunk) ? chunk[src] : 0.0f; + } + std::copy(chunk, chunk + n_chunk, padded.begin() + pad); + if (n_chunk == chunk_samples) { + for (int i = 0; i < pad; i++) { + int src = (int) n_chunk - 2 - i; + padded[n_chunk + pad + i] = (src >= 0) ? chunk[src] : 0.0f; + } + } + + filter_params params; + params.n_mel = hparams.n_mel_bins; + params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + params.hann_window_size = hparams.audio_window_len; + params.hop_length = hop; + params.sample_rate = hparams.audio_sample_rate; + params.no_padding = true; // padding already applied above + params.use_natural_log = false; + + mtmd_audio_mel mel_full; + if (!log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, mel_full)) { + return false; + } + GGML_ASSERT(mel_full.n_len >= n_valid); + + // per-chunk whisper-style normalization, then keep only the valid frames + mtmd_audio_mel out; + out.n_mel = mel_full.n_mel; + out.n_len = n_valid; + out.n_len_org = n_valid; + out.data.resize((size_t) out.n_mel * (size_t) out.n_len); + + double mmax = -1e20; + for (int64_t m = 0; m < out.n_mel; m++) { + for (int64_t t = 0; t < n_valid; t++) { + mmax = std::max(mmax, (double) mel_full.data[(size_t) m * mel_full.n_len + t]); + } + } + mmax -= 8.0; + for (int64_t m = 0; m < out.n_mel; m++) { + for (int64_t t = 0; t < n_valid; t++) { + const double v = std::max((double) mel_full.data[(size_t) m * mel_full.n_len + t], mmax); + out.data[(size_t) m * n_valid + t] = (float) ((v + 4.0) / 4.0); + } + } + + output.push_back(std::move(out)); + } + return !output.empty(); +} + // // mtmd_audio_preprocessor_mimo_audio // diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index 44ad098ae6..0f47d45022 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -111,6 +111,15 @@ struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor { mtmd_audio_cache cache; }; +struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor { + mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + void initialize() override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + + private: + mtmd_audio_cache cache; +}; + struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 4b9c45d626..7594b481bb 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -819,6 +819,7 @@ struct mtmd_context { image_preproc = std::make_unique(ctx_v); } break; case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: { // <|img|> ... (image embeddings) ... <|endofimg|> img_beg = "<|img|>"; @@ -970,6 +971,13 @@ struct mtmd_context { aud_end = ""; audio_preproc = std::make_unique(ctx_a); } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + // <|audio_comp_start|> ... (embeddings) ... <|audio_comp_end|> + aud_beg = "<|audio_comp_start|>"; + aud_end = "<|audio_comp_end|>"; + audio_preproc = std::make_unique(ctx_a); + } break; case PROJECTOR_TYPE_MIMO_AUDIO: { aud_beg = "<|mimo_audio_start|>";