Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 01:04:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into xsn/common_json

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-22 15:55:07 +02:00
parent ff0b1959a4 9fee29e943
commit aeae9626e7
98 changed files with 4523 additions and 1235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/CMakeLists.txt
+1
View File
@@ -30,6 +30,7 @@ add_library(mtmd
models/models.h
models/cogvlm.cpp
models/conformer.cpp
models/dots3note.cpp
models/dotsocr.cpp
models/exaone4_5.cpp
models/gemma4a.cpp
tools/mtmd/clip-graph.h
+6
View File
@@ -120,6 +120,12 @@ struct clip_graph {
ffn_op_type type_op,
int il) const;
ggml_tensor * build_moe_ffn(
ggml_tensor * cur,
const clip_layer & layer,
ffn_op_type type_op,
int il) const;
ggml_tensor * build_attn(
ggml_tensor * wo,
ggml_tensor * wo_b,
tools/mtmd/clip-impl.h
+10 -1
View File
@@ -75,6 +75,7 @@
#define KEY_SAM_N_HEAD "clip.vision.sam.head_count"
#define KEY_SAM_N_BLOCK "clip.vision.sam.block_count"
#define KEY_SAM_N_EMBD "clip.vision.sam.embedding_length"
#define KEY_VISION_N_EXPERT_USED "clip.vision.expert_used_count"
// audio-specific
#define KEY_AUDIO_PROJ_TYPE "clip.audio.projector_type" // for models with mixed modalities
#define KEY_A_NUM_MEL_BINS "clip.audio.num_mel_bins"
@@ -119,7 +120,11 @@
#define TN_FFN_DOWN "%s.blk.%d.ffn_down.%s"
#define TN_FFN_GATE "%s.blk.%d.ffn_gate.%s"
#define TN_FFN_UP "%s.blk.%d.ffn_up.%s"
#define TN_FFN_GATE "%s.blk.%d.ffn_gate.%s"
#define TN_FFN_GATE_INP "%s.blk.%d.ffn_gate_inp.%s" // MoE router (dots3note)
#define TN_FFN_GATE_EXPS "%s.blk.%d.ffn_gate_exps.%s"
#define TN_FFN_UP_EXPS "%s.blk.%d.ffn_up_exps.%s"
#define TN_FFN_DOWN_EXPS "%s.blk.%d.ffn_down_exps.%s"
#define TN_FFN_EXP_PROBS_B "%s.blk.%d.exp_probs_b.%s"
#define TN_LN_1 "%s.blk.%d.ln1.%s" // layer norm
#define TN_LN_2 "%s.blk.%d.ln2.%s" // layer norm
#define TN_LS_1 "%s.blk.%d.ls1.%s" // layer scale
@@ -471,6 +476,8 @@ enum projector_type {
PROJECTOR_TYPE_COGVLM,
PROJECTOR_TYPE_JANUS_PRO,
PROJECTOR_TYPE_DOTS_OCR,
PROJECTOR_TYPE_DOTS3NOTE_V,
PROJECTOR_TYPE_DOTS3NOTE_A,
PROJECTOR_TYPE_DEEPSEEKOCR,
PROJECTOR_TYPE_DEEPSEEKOCR2,
PROJECTOR_TYPE_LFM2A,
@@ -533,6 +540,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_COGVLM, "cogvlm"},
{ PROJECTOR_TYPE_JANUS_PRO, "janus_pro"},
{ PROJECTOR_TYPE_DOTS_OCR, "dots_ocr"},
{ PROJECTOR_TYPE_DOTS3NOTE_V, "dots3note_v"},
{ PROJECTOR_TYPE_DOTS3NOTE_A, "dots3note_a"},
{ PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"},
{ PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"},
{ PROJECTOR_TYPE_LFM2A, "lfm2a"},
tools/mtmd/clip-model.h
+8
View File
@@ -93,6 +93,7 @@ struct clip_hparams {
float eps = 1e-6;
float rope_theta = 0.0;
int32_t n_expert_used = 0;
std::vector<int32_t> feature_layers;
int32_t attn_window_size = 0;
int32_t n_wa_pattern = 0;
@@ -259,6 +260,13 @@ struct clip_layer {
ggml_tensor * ff_down_w = nullptr;
ggml_tensor * ff_down_b = nullptr;
// MoE FFN (dots3note vision pyramid blocks)
ggml_tensor * ff_gate_inp_w = nullptr;
ggml_tensor * ff_gate_exps_w = nullptr;
ggml_tensor * ff_up_exps_w = nullptr;
ggml_tensor * ff_down_exps_w = nullptr;
ggml_tensor * ff_exp_probs_b = nullptr;
// layernorm 2 (or pre-FFN norm)
ggml_tensor * ln_2_w = nullptr;
ggml_tensor * ln_2_b = nullptr;
tools/mtmd/clip.cpp
+122 -7
View File
@@ -514,11 +514,13 @@ ggml_tensor * clip_graph::build_vit(
cb(cur, "ffn_inp_normed", il);
// ffn
cur = build_ffn(cur,
layer.ff_up_w, layer.ff_up_b,
layer.ff_gate_w, layer.ff_gate_b,
layer.ff_down_w, layer.ff_down_b,
ffn_t, il);
cur = layer.ff_gate_exps_w
? build_moe_ffn(cur, layer, ffn_t, il)
: build_ffn(cur,
layer.ff_up_w, layer.ff_up_b,
layer.ff_gate_w, layer.ff_gate_b,
layer.ff_down_w, layer.ff_down_b,
ffn_t, il);
cb(cur, "ffn_out", il);
@@ -699,6 +701,50 @@ ggml_tensor * clip_graph::build_ffn(
return cur;
}
// MoE FFN with sigmoid router and normalized top-k weights (dots3note vision)
// the router runs in fp32; exp_probs_b only affects expert selection, not the weights
ggml_tensor * clip_graph::build_moe_ffn(ggml_tensor * cur, const clip_layer & layer, ffn_op_type type_op, int il) const {
const int64_t n_tokens = cur->ne[1];
const int64_t n_expert = layer.ff_gate_exps_w->ne[2];
const int64_t n_expert_used = std::min((int64_t) hparams.n_expert_used, n_expert);
GGML_ASSERT(n_expert_used > 0);
GGML_ASSERT(type_op == FFN_SILU);
ggml_tensor * probs = ggml_sigmoid(ctx0, build_mm(layer.ff_gate_inp_w, cur)); // [n_expert, n_tokens]
cb(probs, "ffn_moe_probs", il);
ggml_tensor * sel = layer.ff_exp_probs_b
? ggml_add(ctx0, probs, layer.ff_exp_probs_b)
: probs;
ggml_tensor * selected = ggml_top_k(ctx0, sel, n_expert_used); // [n_expert_used, n_tokens]
ggml_tensor * weights = ggml_get_rows(ctx0,
ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected);
weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens);
weights = ggml_div(ctx0, weights, ggml_sum_rows(ctx0, weights));
weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens);
cb(weights, "ffn_moe_weights", il);
cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, n_tokens);
ggml_tensor * gate = ggml_mul_mat_id(ctx0, layer.ff_gate_exps_w, cur, selected); // [n_ff, n_expert_used, n_tokens]
ggml_tensor * up = ggml_mul_mat_id(ctx0, layer.ff_up_exps_w, cur, selected);
cur = ggml_mul(ctx0, ggml_silu(ctx0, gate), up);
cur = ggml_mul_mat_id(ctx0, layer.ff_down_exps_w, cur, selected); // [n_embd, n_expert_used, n_tokens]
cur = ggml_mul(ctx0, cur, weights);
// sum over the selected experts
ggml_tensor * out = nullptr;
for (int64_t i = 0; i < n_expert_used; i++) {
ggml_tensor * v = ggml_view_2d(ctx0, cur, cur->ne[0], n_tokens, cur->nb[2], i * cur->nb[1]);
out = out ? ggml_add(ctx0, out, v) : v;
}
if (n_expert_used == 1) {
out = ggml_cont(ctx0, out);
}
cb(out, "ffn_moe_out", il);
return out;
}
ggml_tensor * clip_graph::build_attn(
ggml_tensor * wo,
ggml_tensor * wo_b,
@@ -933,9 +979,14 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
builder = std::make_unique<clip_graph_pixtral>(ctx, img);
} break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V: // same ViT + merger; pyramid MoE is handled by build_vit
{
builder = std::make_unique<clip_graph_dotsocr>(ctx, img);
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
builder = std::make_unique<clip_graph_dots3note_a>(ctx, img);
} break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
{
@@ -1510,6 +1561,25 @@ struct clip_model_loader {
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
hparams.rope_theta = 10000.0f;
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
get_u32(KEY_VISION_N_EXPERT_USED, hparams.n_expert_used);
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
hparams.rope_theta = 10000.0f;
hparams.audio_chunk_len = 60; // in seconds
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 400;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
} break;
case PROJECTOR_TYPE_KIMIVL:
{
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
@@ -2190,12 +2260,20 @@ struct clip_model_loader {
layer.ln_1_b = get_tensor(string_format(TN_LN_1, prefix, il, "bias"), false);
layer.ln_2_b = get_tensor(string_format(TN_LN_2, prefix, il, "bias"), false);
// MoE ffn (dots3note vision pyramid blocks); replaces the dense ffn when present
layer.ff_gate_inp_w = get_tensor(string_format(TN_FFN_GATE_INP, prefix, il, "weight"), false);
layer.ff_gate_exps_w = get_tensor(string_format(TN_FFN_GATE_EXPS, prefix, il, "weight"), false);
layer.ff_up_exps_w = get_tensor(string_format(TN_FFN_UP_EXPS, prefix, il, "weight"), false);
layer.ff_down_exps_w = get_tensor(string_format(TN_FFN_DOWN_EXPS, prefix, il, "weight"), false);
layer.ff_exp_probs_b = get_tensor(string_format(TN_FFN_EXP_PROBS_B, prefix, il, "weight"), false);
const bool is_moe = layer.ff_gate_exps_w != nullptr;
// ffn
layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, prefix, il, "weight"));
layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, prefix, il, "weight"), !is_moe);
layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, prefix, il, "bias"), false);
layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, prefix, il, "weight"), false);
layer.ff_gate_b = get_tensor(string_format(TN_FFN_GATE, prefix, il, "bias"), false);
layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight"));
layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight"), !is_moe);
layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "bias"), false);
// mimovl per-head attention sink bias
@@ -2677,6 +2755,7 @@ struct clip_model_loader {
model.mm_patch_merger_w = get_tensor(string_format(TN_MM_PATCH_MERGER, "weight"), false);
} break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
@@ -2687,6 +2766,23 @@ struct clip_model_loader {
// post_trunk_norm: applied after all ViT blocks, before the merger
model.post_ln_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
model.conv2d_1_w = get_tensor(string_format(TN_CONV2D, 1, "weight"));
model.conv2d_1_b = get_tensor(string_format(TN_CONV2D, 1, "bias"));
model.conv2d_2_w = get_tensor(string_format(TN_CONV2D, 2, "weight"));
model.conv2d_2_b = get_tensor(string_format(TN_CONV2D, 2, "bias"));
model.conv2d_3_w = get_tensor(string_format(TN_CONV2D, 3, "weight"));
model.conv2d_3_b = get_tensor(string_format(TN_CONV2D, 3, "bias"));
model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight")); // no bias
// adapter: LayerNorm -> Linear -> GELU -> Linear
model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"));
model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias"));
model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight"));
model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias"));
} break;
case PROJECTOR_TYPE_ULTRAVOX:
{
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
@@ -4075,12 +4171,18 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
} break;
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
// dynamic size
int n_merge = ctx->model.hparams.n_merge;
int stride = n_merge * n_merge;
n_patches = CLIP_ALIGN(n_patches, stride) / stride;
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
// 3x stride-2 conv2d over mel frames
n_patches = (img->nx() + 7) / 8;
} break;
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_LIGHTONOCR:
{
@@ -4727,6 +4829,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
set_input_i32("minimax_pos_w", pos_w);
} break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
const int pw = image_size_width / patch_size;
const int ph = image_size_height / patch_size;
@@ -5217,6 +5320,16 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
set_input_i32("pos_w", pos_data);
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
GGML_ASSERT(imgs.entries.size() == 1);
const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d
std::vector<int32_t> positions(n_pos);
for (int i = 0; i < n_pos; i++) {
positions[i] = i;
}
set_input_i32("positions", positions);
} break;
case PROJECTOR_TYPE_GEMMA4A:
{
GGML_ASSERT(imgs.entries.size() == 1);
@@ -5713,6 +5826,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_LIGHTONOCR:
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
case PROJECTOR_TYPE_DOTS3NOTE_A:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_MLP_NORM:
return ctx->model.mm_3_b->ne[0];
tools/mtmd/models/dots3note.cpp
+61
View File
@@ -0,0 +1,61 @@
#include "models.h"
ggml_cgraph * clip_graph_dots3note_a::build() {
// inp_raw: [n_frames, n_mel, 1], one 60s chunk, mel frames not padded
// the reference impl zero-masks conv inputs beyond the valid length at each stage;
// running on exactly the valid frames with the convs' zero padding is equivalent
ggml_tensor * inp = build_inp_raw(1);
GGML_ASSERT(inp->type == GGML_TYPE_F32);
// 3x conv2d (k=3, s=2, p=1) + gelu
{
auto conv_block = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) {
x = ggml_conv_2d(ctx0, w, x, 2, 2, 1, 1, 1, 1);
x = ggml_add(ctx0, x, ggml_reshape_4d(ctx0, b, 1, 1, x->ne[2], 1));
return ggml_gelu_erf(ctx0, x);
};
inp = conv_block(inp, model.conv2d_1_w, model.conv2d_1_b);
inp = conv_block(inp, model.conv2d_2_w, model.conv2d_2_b);
inp = conv_block(inp, model.conv2d_3_w, model.conv2d_3_b);
// inp: [OW=n_frames/8, OH=n_mel/8, OC=480, 1]
cb(inp, "after_conv_stem", -1);
}
// [OW, OH, OC, 1] -> [OH*OC, OW], feature index f + OH*c (matches the reference permute+reshape)
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
inp = ggml_reshape_2d(ctx0, inp, inp->ne[0] * inp->ne[1], inp->ne[2]);
// project to d_model (no bias)
inp = ggml_mul_mat(ctx0, model.conv_out_w, inp);
cb(inp, "after_conv_out", -1);
const int64_t n_pos = inp->ne[1];
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
ggml_set_name(positions, "positions");
ggml_set_input(positions);
// partial rotary: first half of each head, NEOX style
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
return ggml_rope_ext(ctx0, cur, positions, nullptr, d_head/2,
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
};
ggml_tensor * cur = build_vit(inp, n_pos,
NORM_TYPE_RMS, hparams.ffn_op,
nullptr, add_pos);
cb(cur, "after_transformer", -1);
// adapter: LayerNorm -> Linear -> GELU -> Linear
cur = build_norm(cur, model.mm_norm_pre_w, model.mm_norm_pre_b, NORM_TYPE_NORMAL, 1e-5, -1);
cur = build_ffn(cur,
model.mm_1_w, model.mm_1_b,
nullptr, nullptr,
model.mm_2_w, model.mm_2_b,
FFN_GELU_ERF, -1);
cb(cur, "projected", -1);
ggml_build_forward_expand(gf, cur);
return gf;
}
tools/mtmd/models/models.h
+5
View File
@@ -119,6 +119,11 @@ struct clip_graph_dotsocr : clip_graph {
ggml_cgraph * build() override;
};
struct clip_graph_dots3note_a : clip_graph {
clip_graph_dots3note_a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct clip_graph_cogvlm : clip_graph {
clip_graph_cogvlm(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
tools/mtmd/mtmd-audio.cpp
+94
View File
@@ -723,6 +723,100 @@ bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * sa
return true;
}
//
// mtmd_audio_preprocessor_dots3note
//
// Matches Dots3NoteFeatureExtractor: the waveform is split into 60s chunks and each chunk gets
// its own whisper-style log-mel (center=True, log10 + (max-8)/4). Only sample_length//hop frames
// per chunk are valid; the reference masks everything beyond them, so we emit exactly that many.
//
void mtmd_audio_preprocessor_dots3note::initialize() {
cache.fill_sin_cos_table(hparams.audio_n_fft);
cache.fill_hann_window(hparams.audio_window_len, true);
cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
}
bool mtmd_audio_preprocessor_dots3note::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
if (n_samples == 0) {
return false;
}
GGML_ASSERT(!cache.sin_vals.empty());
GGML_ASSERT(!cache.cos_vals.empty());
GGML_ASSERT(!cache.filters.data.empty());
const int pad = hparams.audio_n_fft / 2; // center=True padding
const int hop = hparams.audio_hop_len;
const size_t chunk_samples = (size_t) hparams.audio_chunk_len * hparams.audio_sample_rate;
for (size_t start = 0; start < n_samples; start += chunk_samples) {
const size_t n_chunk = std::min(chunk_samples, n_samples - start);
const float * chunk = samples + start;
const int64_t n_valid = n_chunk / hop;
if (n_valid == 0) {
continue; // sub-hop tail, contributes no frames
}
// reflect-pad the start; the reference zero-pads partial chunks to 60s before the STFT,
// so a partial chunk sees zeros past its end while a full chunk reflects its own tail
std::vector<float> padded(n_chunk + 2 * pad, 0.0f);
for (int i = 0; i < pad; i++) {
int src = pad - i;
padded[i] = (src < (int) n_chunk) ? chunk[src] : 0.0f;
}
std::copy(chunk, chunk + n_chunk, padded.begin() + pad);
if (n_chunk == chunk_samples) {
for (int i = 0; i < pad; i++) {
int src = (int) n_chunk - 2 - i;
padded[n_chunk + pad + i] = (src >= 0) ? chunk[src] : 0.0f;
}
}
filter_params params;
params.n_mel = hparams.n_mel_bins;
params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
params.hann_window_size = hparams.audio_window_len;
params.hop_length = hop;
params.sample_rate = hparams.audio_sample_rate;
params.no_padding = true; // padding already applied above
params.use_natural_log = false;
mtmd_audio_mel mel_full;
if (!log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, mel_full)) {
return false;
}
GGML_ASSERT(mel_full.n_len >= n_valid);
// per-chunk whisper-style normalization, then keep only the valid frames
mtmd_audio_mel out;
out.n_mel = mel_full.n_mel;
out.n_len = n_valid;
out.n_len_org = n_valid;
out.data.resize((size_t) out.n_mel * (size_t) out.n_len);
double mmax = -1e20;
for (int64_t m = 0; m < out.n_mel; m++) {
for (int64_t t = 0; t < n_valid; t++) {
mmax = std::max(mmax, (double) mel_full.data[(size_t) m * mel_full.n_len + t]);
}
}
mmax -= 8.0;
for (int64_t m = 0; m < out.n_mel; m++) {
for (int64_t t = 0; t < n_valid; t++) {
const double v = std::max((double) mel_full.data[(size_t) m * mel_full.n_len + t], mmax);
out.data[(size_t) m * n_valid + t] = (float) ((v + 4.0) / 4.0);
}
}
output.push_back(std::move(out));
}
return !output.empty();
}
//
// mtmd_audio_preprocessor_mimo_audio
//
tools/mtmd/mtmd-audio.h
+9
View File
@@ -111,6 +111,15 @@ struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
mtmd_audio_cache cache;
};
struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
private:
mtmd_audio_cache cache;
};
struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
tools/mtmd/mtmd-helper.cpp
+49
View File
@@ -358,6 +358,15 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int
} // namespace audio_helpers
static bool is_webp_file(const unsigned char * buf, size_t len) {
// WEBP ref: https://developers.google.com/speed/webp/docs/riff_container
return len >= 12 && memcmp(buf, "RIFF", 4) == 0 && memcmp(buf + 8, "WEBP", 4) == 0;
}
#ifdef MTMD_VIDEO
static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder);
#endif
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) {
// calculate the hash if needed
std::string id;
@@ -397,6 +406,19 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,
// otherwise, fallthrough to video decoding (if supported)
}
#ifdef MTMD_VIDEO
// stb_image does not support webp; decode it with ffmpeg as a single frame
if (!result && is_webp_file(buf, len)) {
result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder);
if (!result) {
LOG_ERR("%s: failed to decode webp buffer\n", __func__);
return {nullptr, nullptr};
}
mtmd_bitmap_set_id(result, id.empty() ? nullptr : id.c_str());
return {result, nullptr};
}
#endif
// last try: load as video
#ifdef MTMD_VIDEO
if (!result) {
@@ -820,6 +842,33 @@ static std::string video_resolve_bin(const char * bin_dir, const char * name) {
return result;
}
#ifdef MTMD_VIDEO
static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder) {
auto params = mtmd_helper_video_init_params_default();
mtmd_helper_video vctx;
vctx.mctx = mctx;
vctx.input_buf.assign(buf, buf + len);
vctx.ffmpeg_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg");
vctx.ffprobe_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe");
if (!vctx.probe(0.0f)) {
return nullptr;
}
if (placeholder) {
return mtmd_bitmap_init(vctx.info.width, vctx.info.height, nullptr);
}
// still image: the fps filter would output no frame, so disable it
vctx.fps_target = 0.0f;
if (!vctx.start_ffmpeg(0.0f)) {
return nullptr;
}
mtmd_bitmap * frame = vctx.read_next_frame();
if (frame) {
mtmd_bitmap_set_mergeable(frame, false);
}
return frame;
}
#endif
mtmd_helper_video * mtmd_helper_video_init(
mtmd_context * mctx,
const char * path,
tools/mtmd/mtmd-helper.h
+1
View File
@@ -45,6 +45,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm
// helper function to construct a mtmd_bitmap from a buffer containing a file
// supported formats:
// image: formats supported by stb_image: jpg, png, bmp, gif, etc.
// webp is decoded via ffmpeg, requires MTMD_VIDEO build with ffmpeg in PATH
// audio: formats supported by miniaudio: wav, mp3, flac
// note:
// - for now, video input is only supported via C++ helper functions
tools/mtmd/mtmd.cpp
+8
View File
@@ -825,6 +825,7 @@ struct mtmd_context {
image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v);
} break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
// <|img|> ... (image embeddings) ... <|endofimg|>
img_beg = "<|img|>";
@@ -976,6 +977,13 @@ struct mtmd_context {
aud_end = "<audio|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_gemma4ua>(ctx_a);
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
// <|audio_comp_start|> ... (embeddings) ... <|audio_comp_end|>
aud_beg = "<|audio_comp_start|>";
aud_end = "<|audio_comp_end|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_dots3note>(ctx_a);
} break;
case PROJECTOR_TYPE_MIMO_AUDIO:
{
aud_beg = "<|mimo_audio_start|>";
tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte
+5 -5
View File
@@ -15,7 +15,7 @@
NUMERIC_FIELDS,
POSITIVE_INTEGER_FIELDS,
SETTINGS_CHAT_SECTIONS,
SETTINGS_SECTION_TITLES
SETTINGS_SECTION_SLUGS
} from '$lib/constants';
import { ColorMode } from '$lib/enums/ui.enums';
import { RouterService } from '$lib/services/router.service';
@@ -46,7 +46,7 @@
let fetchInitiated = false;
$effect(() => {
if (serverStore.isRouterMode && currentSection.fields && !fetchInitiated) {
if (serverStore.isRouterMode && currentSection.fields?.length && !fetchInitiated) {
fetchInitiated = true;
void modelsStore
@@ -148,9 +148,9 @@
<h3 class="text-lg font-semibold">{currentSection.title}</h3>
</div>
{#if currentSection.title === SETTINGS_SECTION_TITLES.TOOLS}
{#if currentSection.slug === SETTINGS_SECTION_SLUGS.TOOLS}
<SettingsChatToolsTab />
{:else if currentSection.title === SETTINGS_SECTION_TITLES.IMPORT_EXPORT}
{:else if currentSection.slug === SETTINGS_SECTION_SLUGS.IMPORT_EXPORT}
<SettingsChatImportExportTab />
{:else if currentSection.fields}
<div class="space-y-6">
@@ -161,7 +161,7 @@
onThemeChange={handleThemeChange}
/>
{#if currentSection.title === SETTINGS_SECTION_TITLES.GENERAL}
{#if currentSection.slug === SETTINGS_SECTION_SLUGS.GENERAL}
<div class="flex justify-end">
<Button variant="outline" onclick={() => window.location.reload()}>
<RefreshCw class="h-3 w-3" />
tools/ui/src/lib/constants/index.ts
+1 -1
View File
@@ -48,7 +48,7 @@ export * from './pwa.constants';
export * from './routes.constants';
export * from './sandbox.constants';
export * from './settings-keys.constants';
export * from './settings-registry.constants';
export * from './settings.constants';
export * from './special-characters.constants';
export * from './stream.constants';
export * from './supported-file-types.constants';
tools/ui/src/lib/constants/routes.constants.ts
+2 -12
View File
@@ -10,18 +10,6 @@ export const URL_PARAMS = {
QUERY: 'q'
} as const;
/** Settings section slugs — used for routes and navigation. */
export const SETTINGS_SECTION_SLUGS = {
AGENTIC: 'agentic',
DEVELOPER: 'developer',
DISPLAY: 'display',
GENERAL: 'general',
IMPORT_EXPORT: 'import-export',
PENALTIES: 'penalties',
SAMPLING: 'sampling',
TOOLS: 'tools'
} as const;
export const ROUTES = {
/** Chat base — for dynamic chat URLs use RouterService. */
CHAT: '#/chat',
@@ -33,6 +21,8 @@ export const ROUTES = {
SEARCH: '#/search',
/** Settings base — for dynamic settings URLs use RouterService. */
SETTINGS: '#/settings',
/** Exit destination for the settings view (fallback when no referrer). */
SETTINGS_EXIT: '#/',
/** Root — start of the app. */
START: '#/'
} as const;
tools/ui/src/lib/constants/settings-registry.constants.ts → tools/ui/src/lib/constants/settings.constants.ts
+468 -537
View File
File diff suppressed because it is too large Load Diff
tools/ui/src/lib/services/index.ts
+10
View File
@@ -340,3 +340,13 @@ export { RouterService } from './router.service';
* @see migration.service.ts — full implementation (non-destructive)
*/
export { MigrationService } from './migration.service';
/**
* **SettingsService** - localStorage persistence layer for settings
*
* Stateless read/write of the settings config and user-override keys. Business
* logic (default merging, mobile defaults, theme migration) stays in the store.
*
* @see settingsStore in stores/settings/index.svelte.ts - reactive state + business logic
*/
export { SettingsService } from './settings.service';
tools/ui/src/lib/services/parameter-sync.service.ts
+14 -2
View File
@@ -6,11 +6,23 @@
* No reactive state; consumed by settingsStore.
*/
import { SETTINGS_KEYS, SYNCABLE_PARAMETERS } from '$lib/constants';
import { SETTINGS_KEYS, SETTINGS_REGISTRY } from '$lib/constants';
import { ParameterSource, SyncableParameterType } from '$lib/enums';
import type { ParameterInfo, ParameterRecord, ParameterValue } from '$lib/types';
import type { ParameterInfo, ParameterRecord, ParameterValue, SyncableParameter } from '$lib/types';
import { normalizeFloatingPoint } from '$lib/utils';
/** Mapping of UI setting keys to server parameter keys, derived from the registry. */
export const SYNCABLE_PARAMETERS: SyncableParameter[] = SETTINGS_REGISTRY.flatMap(
(section) => section.settings
)
.filter((s) => s.sync !== undefined)
.map((s) => ({
canSync: true,
key: s.key,
serverKey: s.sync!.serverKey,
type: s.sync!.paramType
}));
export class ParameterSyncService {
/**
* Check if a parameter can be synced from server.
tools/ui/src/lib/services/settings.service.ts
+76
View File
@@ -0,0 +1,76 @@
import { browser } from '$app/environment';
import { CONFIG_LOCALSTORAGE_KEY, USER_OVERRIDES_LOCALSTORAGE_KEY } from '$lib/constants';
/**
* SettingsService - localStorage persistence layer for settings
*
* Stateless read/write of the settings config and user-override keys. Business
* logic (default merging, mobile defaults, theme migration) stays in the store.
*
* **Architecture & Relationships:**
* - **settingsStore**: Primary consumer - loads config on init and persists on change
*
* @see settingsStore in stores/settings/index.svelte.ts - reactive state + business logic
*/
export class SettingsService {
/**
* Read the raw config and user overrides from localStorage.
* @returns Parsed values, or empty defaults when nothing is stored or parsing fails.
*/
static loadConfig(): {
config: Record<string, unknown>;
userOverrides: string[];
isFirstVisit: boolean;
} {
if (!browser) {
return { config: {}, isFirstVisit: false, userOverrides: [] };
}
try {
const storedConfigRaw = localStorage.getItem(CONFIG_LOCALSTORAGE_KEY);
const isFirstVisit = storedConfigRaw === null;
const config = JSON.parse(storedConfigRaw || '{}') as Record<string, unknown>;
const userOverrides = JSON.parse(
localStorage.getItem(USER_OVERRIDES_LOCALSTORAGE_KEY) || '[]'
) as string[];
return { config, isFirstVisit, userOverrides };
} catch (error) {
console.warn('Failed to parse config from localStorage, using defaults:', error);
return { config: {}, isFirstVisit: false, userOverrides: [] };
}
}
/**
* Migrate the legacy un-namespaced "theme" localStorage key.
* Returns the legacy theme value (and removes the key) when present, else null.
*/
static migrateLegacyTheme(): string | null {
if (!browser) return null;
const legacyTheme = localStorage.getItem('theme');
if (legacyTheme) {
localStorage.removeItem('theme');
return legacyTheme;
}
return null;
}
/**
* Persist the config and user overrides to localStorage.
*/
static saveConfig(config: Record<string, unknown>, userOverrides: string[]): void {
if (!browser) return;
try {
localStorage.setItem(CONFIG_LOCALSTORAGE_KEY, JSON.stringify(config));
localStorage.setItem(USER_OVERRIDES_LOCALSTORAGE_KEY, JSON.stringify(userOverrides));
} catch (error) {
console.error('Failed to save config to localStorage:', error);
}
}
}
tools/ui/src/lib/stores/settings/index.svelte.ts
+27 -49
View File
@@ -8,14 +8,10 @@
*/
import { browser } from '$app/environment';
import {
CONFIG_LOCALSTORAGE_KEY,
SETTING_CONFIG_DEFAULT,
SETTINGS_KEYS,
USER_OVERRIDES_LOCALSTORAGE_KEY
} from '$lib/constants';
import { SETTING_CONFIG_DEFAULT, SETTINGS_KEYS } from '$lib/constants';
import { ColorMode } from '$lib/enums';
import { ParameterSyncService } from '$lib/services/parameter-sync.service';
import { SettingsService } from '$lib/services/settings.service';
import { deviceStore } from '$lib/stores/device.svelte';
// direct imports between stores, not via the barrel, to avoid circular deps
import { serverStore } from '$lib/stores/server.svelte';
@@ -428,45 +424,37 @@ class SettingsStore {
}
/**
* Load configuration from localStorage
* Returns default values for missing keys to prevent breaking changes
* Load configuration from localStorage via the persistence service.
* Returns default values for missing keys to prevent breaking changes.
*/
private loadConfig() {
if (!browser) return;
try {
const storedConfigRaw = localStorage.getItem(CONFIG_LOCALSTORAGE_KEY);
const {
config: savedVal,
isFirstVisit,
userOverrides: savedOverrides
} = SettingsService.loadConfig();
// First visit: no stored config yet. Server ui_settings apply once in
// this state, then the user's config diverges freely.
this.isFirstVisit = storedConfigRaw === null;
// First visit: no stored config yet. Server ui_settings apply once in
// this state, then the user's config diverges freely.
this.isFirstVisit = isFirstVisit;
const savedVal = JSON.parse(storedConfigRaw || '{}');
// Merge with defaults to prevent breaking changes
this.config = {
...SETTING_CONFIG_DEFAULT,
...savedVal
};
// Merge with defaults to prevent breaking changes
this.config = {
...SETTING_CONFIG_DEFAULT,
...savedVal
};
// Default sendOnEnter to false on mobile when the user has no saved preference
if (!(SETTINGS_KEYS.SEND_ON_ENTER in savedVal)) {
if (deviceStore.isMobile) {
this.config[SETTINGS_KEYS.SEND_ON_ENTER] = false;
}
// Default sendOnEnter to false on mobile when the user has no saved preference
if (!(SETTINGS_KEYS.SEND_ON_ENTER in savedVal)) {
if (deviceStore.isMobile) {
this.config[SETTINGS_KEYS.SEND_ON_ENTER] = false;
}
// Load user overrides
const savedOverrides = JSON.parse(
localStorage.getItem(USER_OVERRIDES_LOCALSTORAGE_KEY) || '[]'
);
this.userOverrides = new Set(savedOverrides);
} catch (error) {
console.warn('Failed to parse config from localStorage, using defaults:', error);
this.config = { ...SETTING_CONFIG_DEFAULT };
this.userOverrides = new Set();
}
// Load user overrides
this.userOverrides = new Set(savedOverrides);
}
/**
@@ -478,32 +466,22 @@ class SettingsStore {
private migrateLegacyTheme() {
if (!browser) return;
const legacyTheme = localStorage.getItem('theme');
const legacyTheme = SettingsService.migrateLegacyTheme();
if (legacyTheme) {
this.config[SETTINGS_KEYS.THEME] = legacyTheme;
localStorage.removeItem('theme');
this.saveConfig();
setMode(legacyTheme as ColorMode);
}
}
/**
* Save the current configuration to localStorage
* Save the current configuration to localStorage via the persistence service.
*/
private saveConfig() {
if (!browser) return;
try {
localStorage.setItem(CONFIG_LOCALSTORAGE_KEY, JSON.stringify(this.config));
localStorage.setItem(
USER_OVERRIDES_LOCALSTORAGE_KEY,
JSON.stringify(Array.from(this.userOverrides))
);
} catch (error) {
console.error('Failed to save config to localStorage:', error);
}
SettingsService.saveConfig(this.config, Array.from(this.userOverrides));
}
}
tools/ui/src/lib/stores/settings/referrer.svelte.ts
+2 -2
View File
@@ -5,9 +5,9 @@
* there after a fallback exit. Standalone reactive value, no host.
*/
import { SETTINGS_FALLBACK_EXIT_ROUTE } from '$lib/constants';
import { ROUTES } from '$lib/constants';
let _url = $state<string>(SETTINGS_FALLBACK_EXIT_ROUTE);
let _url = $state<string>(ROUTES.SETTINGS_EXIT);
export const settingsReferrer = {
get url() {
tools/ui/src/lib/types/settings.d.ts
Vendored
+3 -1
View File
@@ -25,13 +25,15 @@ export interface SettingsEntry {
help: string;
defaultValue: SettingsConfigValue;
type: SettingsFieldType;
section?: string;
options?: Array<{ value: string; label: string; icon: Component }>;
/** Options rendered for RADIO fields. Each entry maps a `value` (the radio's selected value) to the underlying config `key` whose boolean state mirrors it. */
radioOptions?: Array<{ value: string; label: string; key: string; isExperimental?: boolean }>;
isExperimental?: boolean;
isPositiveInteger?: boolean;
/** When true, the field is rendered as a password input (e.g. API key). */
isPrivate?: boolean;
/** When false, the setting is stored/synced but has no standalone field; it is rendered by a sibling control or a dedicated page. */
standaloneField?: boolean;
placeholder?: string;
min?: number;
max?: number;
tools/ui/src/routes/settings/+layout.svelte
+2 -2
View File
@@ -4,7 +4,7 @@
import { goto } from '$app/navigation';
import { page } from '$app/state';
import { ActionIcon } from '$lib/components/app';
import { SETTINGS_FALLBACK_EXIT_ROUTE } from '$lib/constants';
import { ROUTES } from '$lib/constants';
let { children } = $props();
@@ -24,7 +24,7 @@
if (browser && window.history.length > 1 && !prevIsSettings) {
history.back();
} else {
goto(SETTINGS_FALLBACK_EXIT_ROUTE);
goto(ROUTES.SETTINGS_EXIT);
}
}
</script>