mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-19 09:15:18 +02:00
Merge commit '11b068d06605288ce7917534b46d52b47823dc13' into concedo_experimental
# Conflicts: # CONTRIBUTING.md # docs/backend/SYCL.md # docs/install.md # docs/speculative.md # ggml/src/ggml-hip/CMakeLists.txt # ggml/src/ggml-opencl/ggml-opencl.cpp # ggml/src/ggml-sycl/common.hpp # ggml/src/ggml-sycl/element_wise.cpp # ggml/src/ggml-sycl/fattn-onednn.cpp # ggml/src/ggml-sycl/ggml-sycl.cpp # ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp # ggml/src/ggml-webgpu/ggml-webgpu.cpp # ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl # ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl # tests/test-backend-ops.cpp # tests/test-chat.cpp # tests/test-llama-archs.cpp # tools/cli/README.md # tools/llama-bench/llama-bench.cpp # tools/mtmd/CMakeLists.txt # tools/server/README.md
This commit is contained in:
Concedo
@@ -13,6 +13,14 @@
|
||||
|
||||
struct build_vit_opts {
|
||||
ggml_tensor * attn_mask = nullptr;
|
||||
// TODO @ngxson : merge attn_mask and attn_mask_layers into one call
|
||||
std::vector<ggml_tensor *> attn_mask_layers; // one per layer
|
||||
|
||||
// hook at layer output embeddings
|
||||
std::function<void(ggml_tensor * cur, int il)> callback_layer_out = nullptr;
|
||||
|
||||
// whether to skip the automatic post-layernorm (model.post_ln_w) applied at the end
|
||||
bool skip_post_ln = false;
|
||||
};
|
||||
|
||||
struct clip_graph {
|
||||
|
||||
@@ -82,6 +82,13 @@
|
||||
#define KEY_A_PROJ_WINDOW_SIZE "clip.audio.projector.window_size"
|
||||
#define KEY_A_PROJ_DOWNSAMPLE_RATE "clip.audio.projector.downsample_rate"
|
||||
#define KEY_A_PROJ_HEAD_COUNT "clip.audio.projector.head_count"
|
||||
#define KEY_A_RVQ_NUM_QUANTIZERS "clip.audio.rvq.num_quantizers" // mimo-audio-tokenizer
|
||||
#define KEY_A_RVQ_CODEBOOK_SIZE "clip.audio.rvq.codebook_size" // mimo-audio-tokenizer: per-quantizer bin count
|
||||
#define KEY_A_WA_PATTERN_MODE "clip.audio.wa_pattern_mode" // mimo-audio-tokenizer, per-layer -1 (full) / 0 (windowed)
|
||||
#define KEY_A_ATTN_WINDOW_SIZE "clip.audio.window_size" // mimo-audio-tokenizer: sliding-window radius
|
||||
#define KEY_A_LOCAL_BLOCK_COUNT "clip.audio.local_block_count" // mimo-v2.5: input_local_transformer layer count
|
||||
#define KEY_A_LOCAL_GROUP_SIZE "clip.audio.local_group_size" // mimo-v2.5: input_local_transformer grouping size
|
||||
#define KEY_AUDIO_SUBSAMPLING_FACTOR "clip.audio.subsampling_factor"
|
||||
|
||||
//
|
||||
// tensor name constants
|
||||
@@ -175,6 +182,24 @@
|
||||
#define TN_MM_NORM_PRE "mm.a.norm_pre.%s"
|
||||
#define TN_MM_NORM_MID "mm.a.norm_mid.%s"
|
||||
|
||||
// mimo-audio-tokenizer
|
||||
#define TN_A_DOWNSAMPLE_CONV "a.downsample.conv.%s"
|
||||
#define TN_A_DOWNSAMPLE_NORM "a.downsample.norm.%s"
|
||||
#define TN_A_RVQ_CODEBOOK "a.rvq.codebook.%s"
|
||||
// mimo-v2.5: text-side RVQ code embedding ("text codebook")
|
||||
#define TN_MM_A_CODE_EMBD "mm.a.code_embd.%s"
|
||||
// mimo-v2.5: LLM-side connector (input_local_transformer)
|
||||
#define TN_MM_A_LOCAL_ATTN_Q "mm.a.local_blk.%d.attn_q.%s"
|
||||
#define TN_MM_A_LOCAL_ATTN_K "mm.a.local_blk.%d.attn_k.%s"
|
||||
#define TN_MM_A_LOCAL_ATTN_V "mm.a.local_blk.%d.attn_v.%s"
|
||||
#define TN_MM_A_LOCAL_ATTN_OUT "mm.a.local_blk.%d.attn_out.%s"
|
||||
#define TN_MM_A_LOCAL_FFN_GATE "mm.a.local_blk.%d.ffn_gate.%s"
|
||||
#define TN_MM_A_LOCAL_FFN_UP "mm.a.local_blk.%d.ffn_up.%s"
|
||||
#define TN_MM_A_LOCAL_FFN_DOWN "mm.a.local_blk.%d.ffn_down.%s"
|
||||
#define TN_MM_A_LOCAL_LN1 "mm.a.local_blk.%d.ln1.%s"
|
||||
#define TN_MM_A_LOCAL_LN2 "mm.a.local_blk.%d.ln2.%s"
|
||||
#define TN_MM_A_LOCAL_NORM "mm.a.local_norm.%s"
|
||||
|
||||
// cogvlm
|
||||
#define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s"
|
||||
#define TN_MM_H_TO_4H "mm.up.%s"
|
||||
@@ -314,6 +339,12 @@
|
||||
#define TN_YASA_STAGE_DOWN_CONV "v.stage.%d.down.conv.%s"
|
||||
#define TN_YASA_STAGE_BLK "v.stage.%d.blk.%d.%s.%s"
|
||||
|
||||
// parakeet
|
||||
#define TN_MEL_FILTERS "a.mel_filters"
|
||||
#define TN_WINDOW "a.window"
|
||||
#define TN_CONV_NORM_MEAN "%s.blk.%d.conv_norm_mean"
|
||||
#define TN_CONV_NORM_VAR "%s.blk.%d.conv_norm_var"
|
||||
|
||||
// align x to upper multiple of n
|
||||
#define CLIP_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n))
|
||||
|
||||
@@ -368,12 +399,14 @@ enum projector_type {
|
||||
PROJECTOR_TYPE_KIMIK25,
|
||||
PROJECTOR_TYPE_NEMOTRON_V2_VL,
|
||||
PROJECTOR_TYPE_HUNYUANVL,
|
||||
PROJECTOR_TYPE_PARAKEET,
|
||||
PROJECTOR_TYPE_EXAONE4_5,
|
||||
PROJECTOR_TYPE_MINICPMV4_6,
|
||||
PROJECTOR_TYPE_GRANITE_SPEECH,
|
||||
PROJECTOR_TYPE_MIMOVL,
|
||||
PROJECTOR_TYPE_MINIMAX_M3,
|
||||
PROJECTOR_TYPE_GRANITE4_VISION,
|
||||
PROJECTOR_TYPE_MIMO_AUDIO,
|
||||
PROJECTOR_TYPE_UNKNOWN,
|
||||
};
|
||||
|
||||
@@ -429,6 +462,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
|
||||
{ PROJECTOR_TYPE_MIMOVL, "mimovl"},
|
||||
{ PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"},
|
||||
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
|
||||
{ PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"},
|
||||
{ PROJECTOR_TYPE_PARAKEET, "parakeet"},
|
||||
};
|
||||
|
||||
static projector_type clip_projector_type_from_string(const std::string & str) {
|
||||
|
||||
+38
-8
@@ -110,6 +110,8 @@ struct clip_hparams {
|
||||
// audio
|
||||
int32_t n_mel_bins = 0; // whisper preprocessor
|
||||
int32_t proj_stack_factor = 0; // ultravox
|
||||
int32_t subsampling_factor = 0; // parakeet
|
||||
|
||||
int32_t audio_chunk_size = 0;
|
||||
int32_t audio_conv_kernel_size = 0;
|
||||
int32_t audio_max_pos_emb = 0;
|
||||
@@ -124,6 +126,18 @@ struct clip_hparams {
|
||||
int32_t audio_window_len = -1;
|
||||
int32_t audio_hop_len = -1;
|
||||
|
||||
// parakeet
|
||||
std::vector<float> mel_filters;
|
||||
std::vector<float> window;
|
||||
|
||||
// mimo-audio-tokenizer: residual vector quantizer
|
||||
int32_t rvq_num_quantizers = 0;
|
||||
std::vector<int32_t> rvq_codebook_size; // per-quantizer bin count (ragged, e.g. 1024/1024/256/128x17)
|
||||
|
||||
// mimo-v2.5: LLM-side connector (input_local_transformer)
|
||||
int32_t audio_local_n_layer = 0;
|
||||
int32_t audio_local_group_size = 0;
|
||||
|
||||
// legacy
|
||||
bool has_llava_projector = false;
|
||||
int minicpmv_version = 0;
|
||||
@@ -237,14 +251,16 @@ struct clip_layer {
|
||||
ggml_tensor * norm_conv_b = nullptr;
|
||||
ggml_tensor * linear_pos_w = nullptr;
|
||||
|
||||
ggml_tensor * conv_norm_w = nullptr;
|
||||
ggml_tensor * conv_norm_b = nullptr;
|
||||
ggml_tensor * conv_dw_w = nullptr;
|
||||
ggml_tensor * conv_dw_b = nullptr;
|
||||
ggml_tensor * conv_pw1_w = nullptr;
|
||||
ggml_tensor * conv_pw1_b = nullptr;
|
||||
ggml_tensor * conv_pw2_w = nullptr;
|
||||
ggml_tensor * conv_pw2_b = nullptr;
|
||||
ggml_tensor * conv_norm_w = nullptr;
|
||||
ggml_tensor * conv_norm_b = nullptr;
|
||||
ggml_tensor * conv_norm_mean = nullptr; // parakeet
|
||||
ggml_tensor * conv_norm_var = nullptr; // parakeet
|
||||
ggml_tensor * conv_dw_w = nullptr;
|
||||
ggml_tensor * conv_dw_b = nullptr;
|
||||
ggml_tensor * conv_pw1_w = nullptr;
|
||||
ggml_tensor * conv_pw1_b = nullptr;
|
||||
ggml_tensor * conv_pw2_w = nullptr;
|
||||
ggml_tensor * conv_pw2_b = nullptr;
|
||||
|
||||
// gemma4 audio conformer per-layer
|
||||
ggml_tensor * attn_pre_norm_w = nullptr;
|
||||
@@ -537,6 +553,20 @@ struct clip_model {
|
||||
ggml_tensor * mm_norm_pre_b = nullptr;
|
||||
ggml_tensor * mm_norm_mid_w = nullptr;
|
||||
|
||||
// mimo-audio-tokenizer: post-transformer downsample + RVQ codebook
|
||||
ggml_tensor * downsample_conv_w = nullptr; // no bias
|
||||
ggml_tensor * downsample_norm_w = nullptr;
|
||||
ggml_tensor * downsample_norm_b = nullptr;
|
||||
ggml_tensor * rvq_codebook = nullptr; // merged 3D [n_q, max_bins, dim]
|
||||
|
||||
// mimo-v2.5: text-side RVQ code embedding ("text codebook")
|
||||
ggml_tensor * mm_a_code_embd = nullptr; // merged 3D [n_channels, vocab, dim]
|
||||
|
||||
// mimo-v2.5: LLM-side connector (input_local_transformer, separate from the
|
||||
// audio_tokenizer's own encoder `layers`)
|
||||
std::vector<clip_layer> mm_a_local_layers;
|
||||
ggml_tensor * mm_a_local_norm_w = nullptr;
|
||||
|
||||
// qwen3a
|
||||
ggml_tensor * conv2d_1_w = nullptr;
|
||||
ggml_tensor * conv2d_1_b = nullptr;
|
||||
|
||||
+372
-9
@@ -61,11 +61,13 @@
|
||||
#include "models/minicpmv.cpp"
|
||||
#include "models/minimax-m3.cpp"
|
||||
#include "models/paddleocr.cpp"
|
||||
#include "models/parakeet.cpp"
|
||||
#include "models/pixtral.cpp"
|
||||
#include "models/qwen2vl.cpp"
|
||||
#include "models/qwen3vl.cpp"
|
||||
#include "models/mimovl.cpp"
|
||||
#include "models/qwen3a.cpp"
|
||||
#include "models/mimovl.cpp"
|
||||
#include "models/mimo-audio.cpp"
|
||||
#include "models/step3vl.cpp"
|
||||
#include "models/siglip.cpp"
|
||||
#include "models/whisper-enc.cpp"
|
||||
@@ -389,6 +391,11 @@ ggml_tensor * clip_graph::build_vit(
|
||||
auto & layer = model.layers[il];
|
||||
ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states
|
||||
|
||||
ggml_tensor * attn_mask = opts.attn_mask;
|
||||
if (opts.attn_mask_layers.size() > (size_t) il) {
|
||||
attn_mask = opts.attn_mask_layers[il];
|
||||
}
|
||||
|
||||
// layernorm1
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
|
||||
cb(cur, "layer_inp_normed", il);
|
||||
@@ -501,7 +508,7 @@ ggml_tensor * clip_graph::build_vit(
|
||||
|
||||
// build_attn returns a flat 2D [n_embd, n_pos*B]
|
||||
cur = build_attn(layer.o_w, layer.o_b,
|
||||
Qcur, Kcur, Vcur, opts.attn_mask, kq_scale, il);
|
||||
Qcur, Kcur, Vcur, attn_mask, kq_scale, il);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
|
||||
@@ -520,6 +527,10 @@ ggml_tensor * clip_graph::build_vit(
|
||||
|
||||
inpL = cur; // inpL = residual, cur = hidden_states
|
||||
|
||||
if (opts.callback_layer_out) {
|
||||
opts.callback_layer_out(cur, il);
|
||||
}
|
||||
|
||||
cb(cur, "ffn_inp", il);
|
||||
|
||||
// layernorm2 (pre-ffn norm)
|
||||
@@ -568,7 +579,7 @@ ggml_tensor * clip_graph::build_vit(
|
||||
}
|
||||
|
||||
// post-layernorm
|
||||
if (model.post_ln_w) {
|
||||
if (model.post_ln_w && !opts.skip_post_ln) {
|
||||
inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, -1);
|
||||
}
|
||||
|
||||
@@ -1061,6 +1072,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
|
||||
{
|
||||
builder = std::make_unique<clip_graph_qwen3a>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MIMO_AUDIO:
|
||||
{
|
||||
builder = std::make_unique<clip_graph_mimo_audio>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_YOUTUVL:
|
||||
{
|
||||
builder = std::make_unique<clip_graph_youtuvl>(ctx, img);
|
||||
@@ -1069,6 +1084,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
|
||||
{
|
||||
builder = std::make_unique<clip_graph_yasa2>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_PARAKEET:
|
||||
{
|
||||
builder = std::make_unique<clip_graph_parakeet>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GRANITE4_VISION:
|
||||
{
|
||||
builder = std::make_unique<clip_graph_granite4_vision>(ctx, img);
|
||||
@@ -1415,6 +1434,20 @@ struct clip_model_loader {
|
||||
{
|
||||
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_PARAKEET:
|
||||
{
|
||||
get_u32(KEY_AUDIO_SUBSAMPLING_FACTOR, hparams.subsampling_factor);
|
||||
GGML_ASSERT(hparams.subsampling_factor == 8 &&
|
||||
"subsampling_factor must match the conv strides in clip_graph_parakeet::build()");
|
||||
get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size);
|
||||
GGML_ASSERT(hparams.audio_conv_kernel_size > 0 && hparams.audio_conv_kernel_size % 2 == 1 &&
|
||||
"audio_conv_kernel_size must be a positive odd integer");
|
||||
hparams.audio_chunk_len = 0;
|
||||
hparams.audio_sample_rate = 16000;
|
||||
hparams.audio_n_fft = 512;
|
||||
hparams.audio_window_len = 400;
|
||||
hparams.audio_hop_len = 160;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_IDEFICS3:
|
||||
{
|
||||
// use default llava-uhd preprocessing params
|
||||
@@ -1652,6 +1685,45 @@ struct clip_model_loader {
|
||||
hparams.audio_window_len = 400;
|
||||
hparams.audio_hop_len = 160;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MIMO_AUDIO:
|
||||
{
|
||||
get_u32(KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers, false);
|
||||
get_arr_int(KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size, false);
|
||||
if (hparams.rvq_num_quantizers <= 0) {
|
||||
throw std::runtime_error(string_format("%s: mimo_audio: missing %s\n", __func__, KEY_A_RVQ_NUM_QUANTIZERS));
|
||||
}
|
||||
if ((int) hparams.rvq_codebook_size.size() != hparams.rvq_num_quantizers) {
|
||||
throw std::runtime_error(string_format(
|
||||
"%s: mimo_audio: %s length (%zu) must equal %s (%d)\n", __func__,
|
||||
KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size.size(),
|
||||
KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers));
|
||||
}
|
||||
hparams.ffn_op = FFN_GELU_ERF; // PyTorch F.gelu default (approximate="none")
|
||||
hparams.rope_theta = 10000.0f;
|
||||
|
||||
// audio preprocessing params (mel spectrogram)
|
||||
hparams.audio_sample_rate = 24000;
|
||||
hparams.audio_n_fft = 960;
|
||||
hparams.audio_window_len = 960;
|
||||
hparams.audio_hop_len = 240;
|
||||
|
||||
get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size);
|
||||
std::vector<int> wa_pattern;
|
||||
get_arr_int(KEY_A_WA_PATTERN_MODE, wa_pattern, true);
|
||||
if ((int) wa_pattern.size() != hparams.n_layer) {
|
||||
throw std::runtime_error(string_format(
|
||||
"%s: mimo_audio: %s length (%zu) must equal n_layer (%d)\n", __func__,
|
||||
KEY_A_WA_PATTERN_MODE, wa_pattern.size(), hparams.n_layer));
|
||||
}
|
||||
hparams.wa_pattern_mode.assign(wa_pattern.begin(), wa_pattern.end());
|
||||
|
||||
get_u32(KEY_A_LOCAL_BLOCK_COUNT, hparams.audio_local_n_layer);
|
||||
get_u32(KEY_A_LOCAL_GROUP_SIZE, hparams.audio_local_group_size);
|
||||
if (hparams.audio_local_group_size <= 0) {
|
||||
throw std::runtime_error(string_format(
|
||||
"%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE));
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_PADDLEOCR:
|
||||
{
|
||||
hparams.n_merge = 2;
|
||||
@@ -1923,16 +1995,46 @@ struct clip_model_loader {
|
||||
return cur;
|
||||
};
|
||||
|
||||
auto get_scalar = [&](const std::string & name, float default_val) {
|
||||
auto get_vector = [&](const std::string & name) {
|
||||
std::vector<float> result;
|
||||
auto it = tensor_offset.find(name);
|
||||
if (it == tensor_offset.end()) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const int64_t idx = gguf_find_tensor(ctx_gguf.get(), name.c_str());
|
||||
if (idx < 0) {
|
||||
throw std::runtime_error(string_format("%s: failed to find tensor %s\n", __func__, name.c_str()));
|
||||
}
|
||||
|
||||
if (const auto type = gguf_get_tensor_type(ctx_gguf.get(), idx); type != GGML_TYPE_F32) {
|
||||
throw std::runtime_error(string_format("%s: %s must be %s, was %s\n", __func__,
|
||||
name.c_str(), ggml_type_name(GGML_TYPE_F32), ggml_type_name(type)));
|
||||
}
|
||||
|
||||
const size_t n_bytes = gguf_get_tensor_size(ctx_gguf.get(), idx);
|
||||
if (n_bytes == 0) {
|
||||
throw std::runtime_error(string_format("%s: tensor %s is empty\n", __func__, name.c_str()));
|
||||
}
|
||||
|
||||
const size_t n_elems = n_bytes / sizeof(float);
|
||||
result.resize(n_elems);
|
||||
fin.seekg(it->second, std::ios::beg);
|
||||
fin.read(reinterpret_cast<char*>(result.data()), n_bytes);
|
||||
return result;
|
||||
};
|
||||
|
||||
auto get_scalar = [&](const std::string & name, float default_val) {
|
||||
auto v = get_vector(name);
|
||||
if (v.empty()) {
|
||||
return default_val;
|
||||
}
|
||||
size_t offset = it->second;
|
||||
fin.seekg(offset, std::ios::beg);
|
||||
float value;
|
||||
fin.read(reinterpret_cast<char*>(&value), sizeof(float));
|
||||
return value;
|
||||
if (v.size() != 1) {
|
||||
throw std::runtime_error(string_format("%s: expected scalar tensor '%s' but got %d elements\n",
|
||||
__func__, name.c_str(), (int) v.size()));
|
||||
}
|
||||
|
||||
return v[0];
|
||||
};
|
||||
|
||||
model.class_embedding = get_tensor(TN_CLASS_EMBD, false);
|
||||
@@ -2526,6 +2628,54 @@ struct clip_model_loader {
|
||||
model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
|
||||
model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias"));
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MIMO_AUDIO:
|
||||
{
|
||||
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
|
||||
model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
|
||||
model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
|
||||
model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
|
||||
model.downsample_conv_w = get_tensor(string_format(TN_A_DOWNSAMPLE_CONV, "weight"));
|
||||
model.downsample_norm_w = get_tensor(string_format(TN_A_DOWNSAMPLE_NORM, "weight"));
|
||||
model.downsample_norm_b = get_tensor(string_format(TN_A_DOWNSAMPLE_NORM, "bias"));
|
||||
model.rvq_codebook = get_tensor(string_format(TN_A_RVQ_CODEBOOK, "weight"), false);
|
||||
model.mm_a_code_embd = get_tensor(string_format(TN_MM_A_CODE_EMBD, "weight"), false);
|
||||
if (!model.rvq_codebook || !model.mm_a_code_embd) {
|
||||
throw std::runtime_error(string_format("%s: mimo_audio: missing %s or %s\n", __func__,
|
||||
TN_A_RVQ_CODEBOOK, TN_MM_A_CODE_EMBD));
|
||||
}
|
||||
// hparams.rvq_codebook_size comes from GGUF metadata and is independent of the
|
||||
// tensors' actual shapes - bound it so codebook/code_embd views built from it
|
||||
// (mimo-audio.cpp) can never read past either tensor's allocated bins/vocab.
|
||||
for (int32_t bins : hparams.rvq_codebook_size) {
|
||||
if (bins <= 0 || bins > model.rvq_codebook->ne[1] || bins > model.mm_a_code_embd->ne[1]) {
|
||||
throw std::runtime_error(string_format(
|
||||
"%s: mimo_audio: %s entry (%d) out of range for codebook/code_embd tensors\n",
|
||||
__func__, KEY_A_RVQ_CODEBOOK_SIZE, bins));
|
||||
}
|
||||
}
|
||||
|
||||
// LLM-side connector: input_local_transformer + projection
|
||||
model.mm_a_local_layers.resize(hparams.audio_local_n_layer);
|
||||
for (int il = 0; il < hparams.audio_local_n_layer; il++) {
|
||||
auto & layer = model.mm_a_local_layers[il];
|
||||
layer.q_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_Q, il, "weight"));
|
||||
layer.q_b = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_Q, il, "bias"));
|
||||
layer.k_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_K, il, "weight"));
|
||||
layer.k_b = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_K, il, "bias"));
|
||||
layer.v_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_V, il, "weight"));
|
||||
layer.v_b = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_V, il, "bias"));
|
||||
layer.o_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_OUT, il, "weight"));
|
||||
layer.ff_gate_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_GATE, il, "weight"));
|
||||
layer.ff_up_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_UP, il, "weight"));
|
||||
layer.ff_down_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_DOWN, il, "weight"));
|
||||
layer.ln_1_w = get_tensor(string_format(TN_MM_A_LOCAL_LN1, il, "weight"));
|
||||
layer.ln_2_w = get_tensor(string_format(TN_MM_A_LOCAL_LN2, il, "weight"));
|
||||
}
|
||||
model.mm_a_local_norm_w = get_tensor(string_format(TN_MM_A_LOCAL_NORM, "weight"));
|
||||
|
||||
model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
|
||||
model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VOXTRAL:
|
||||
{
|
||||
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
|
||||
@@ -2782,6 +2932,68 @@ struct clip_model_loader {
|
||||
layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"));
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_PARAKEET:
|
||||
{
|
||||
|
||||
hparams.mel_filters = get_vector(TN_MEL_FILTERS);
|
||||
hparams.window = get_vector(TN_WINDOW);
|
||||
|
||||
// Subsampling layers (conv1d)
|
||||
for (int i : {0, 2, 3, 5, 6}) {
|
||||
model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight"));
|
||||
model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias"));
|
||||
}
|
||||
model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight"));
|
||||
model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias"));
|
||||
|
||||
// Projection layers
|
||||
model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"), false);
|
||||
model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"), false);
|
||||
model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"), false);
|
||||
|
||||
// Encoder layers
|
||||
for (int il = 0; il < hparams.n_layer; ++il) {
|
||||
auto & layer = model.layers[il];
|
||||
|
||||
// Attention (from shared above)
|
||||
|
||||
// Relative position encoding
|
||||
layer.linear_pos_w = get_tensor(string_format(TN_LINEAR_POS, prefix, il, "weight"));
|
||||
layer.pos_bias_u = get_tensor(string_format(TN_POS_BIAS_U, prefix, il));
|
||||
layer.pos_bias_v = get_tensor(string_format(TN_POS_BIAS_V, prefix, il));
|
||||
|
||||
// Convolution module
|
||||
layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"));
|
||||
layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"), false);
|
||||
layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"));
|
||||
layer.conv_dw_b = get_tensor(string_format(TN_CONV_DW, prefix, il, "bias"), false);
|
||||
layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"));
|
||||
layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"));
|
||||
layer.conv_norm_mean = get_tensor(string_format(TN_CONV_NORM_MEAN, prefix, il));
|
||||
layer.conv_norm_var = get_tensor(string_format(TN_CONV_NORM_VAR, prefix, il));
|
||||
layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"));
|
||||
layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"), false);
|
||||
|
||||
// Feed-forward networks
|
||||
layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight"));
|
||||
layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias"));
|
||||
|
||||
layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
|
||||
layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias"));
|
||||
layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight"));
|
||||
layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"), false);
|
||||
layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
|
||||
layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"), false);
|
||||
|
||||
// Layer norms
|
||||
layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"));
|
||||
layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"));
|
||||
}
|
||||
|
||||
model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
|
||||
model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
|
||||
model.mm_model_mlp_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GRANITE_SPEECH:
|
||||
{
|
||||
model.inp_proj_w = get_tensor(string_format(TN_INP_PROJ, "weight"));
|
||||
@@ -3627,10 +3839,23 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
|
||||
}
|
||||
n_patches = n;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_PARAKEET:
|
||||
{
|
||||
n_patches = (img->nx() + (params.subsampling_factor - 1)) / params.subsampling_factor;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GEMMA4UA:
|
||||
{
|
||||
n_patches = img->nx(); // no downsampling: one token per raw waveform frame
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MIMO_AUDIO:
|
||||
{
|
||||
// conv1(s=1) + conv2(s=2) -> RVQ-encoder downsample conv(k=2,s=2)
|
||||
int n = img->nx();
|
||||
n = (n - 1) / 2 + 1; // conv1 + conv2
|
||||
n = (n - 2) / 2 + 1; // downsample conv
|
||||
const int group_size = params.audio_local_group_size;
|
||||
n_patches = (n + group_size - 1) / group_size;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GRANITE_SPEECH:
|
||||
{
|
||||
const int ws = ctx->model.hparams.audio_proj_window_size;
|
||||
@@ -4458,6 +4683,58 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
|
||||
set_input_f32("pos_emb", pos_emb);
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MIMO_AUDIO:
|
||||
{
|
||||
GGML_ASSERT(imgs.entries.size() == 1);
|
||||
const int n_frames = imgs.entries.front().nx();
|
||||
const int n_pos = (n_frames - 1) / 2 + 1; // matches conv1(s=1)+conv2(s=2) output length
|
||||
|
||||
std::vector<int32_t> positions(n_pos);
|
||||
for (int i = 0; i < n_pos; i++) {
|
||||
positions[i] = i;
|
||||
}
|
||||
set_input_i32("mimo_audio_positions", positions);
|
||||
|
||||
const int window = hparams.attn_window_size;
|
||||
GGML_ASSERT(window > 0);
|
||||
|
||||
const float neg_inf = std::numeric_limits<float>::lowest();
|
||||
std::vector<float> full_mask((size_t) n_pos * n_pos);
|
||||
std::vector<float> window_mask((size_t) n_pos * n_pos);
|
||||
for (int q = 0; q < n_pos; q++) {
|
||||
for (int k = 0; k < n_pos; k++) {
|
||||
const bool causal_ok = k <= q;
|
||||
full_mask[(size_t) q * n_pos + k] = causal_ok ? 0.0f : neg_inf;
|
||||
window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf;
|
||||
}
|
||||
}
|
||||
set_input_f32("mimo_audio_full_mask", full_mask);
|
||||
set_input_f32("mimo_audio_window_mask", window_mask);
|
||||
|
||||
// input_local_transformer: block-diagonal mask + in-group positions
|
||||
{
|
||||
const int n_pos_ds = (n_pos - 2) / 2 + 1; // matches downsample conv (k=2,s=2,p=0)
|
||||
const int group_size = hparams.audio_local_group_size;
|
||||
GGML_ASSERT(group_size > 0);
|
||||
const int n_groups = (n_pos_ds + group_size - 1) / group_size;
|
||||
const int n_padded = n_groups * group_size;
|
||||
|
||||
std::vector<int32_t> local_positions(n_padded);
|
||||
for (int i = 0; i < n_padded; i++) {
|
||||
local_positions[i] = i % group_size;
|
||||
}
|
||||
set_input_i32("mimo_audio_local_positions", local_positions);
|
||||
|
||||
std::vector<float> local_mask((size_t) n_padded * n_padded);
|
||||
for (int q = 0; q < n_padded; q++) {
|
||||
for (int k = 0; k < n_padded; k++) {
|
||||
const bool same_group = (q / group_size) == (k / group_size);
|
||||
local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf;
|
||||
}
|
||||
}
|
||||
set_input_f32("mimo_audio_local_mask", local_mask);
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_LFM2A:
|
||||
{
|
||||
GGML_ASSERT(imgs.entries.size() == 1);
|
||||
@@ -4479,6 +4756,88 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
|
||||
}
|
||||
set_input_f32("pos_emb", pos_emb);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_PARAKEET:
|
||||
{
|
||||
GGML_ASSERT(imgs.entries.size() == 1);
|
||||
struct ggml_tensor * attn_mask = ggml_graph_get_tensor(gf, "attn_mask");
|
||||
const int n_q = attn_mask->ne[1];
|
||||
const int n_k = attn_mask->ne[0];
|
||||
const int n_frames = imgs.entries.front().nx();
|
||||
const int n_tokens_real = (n_frames + hparams.subsampling_factor-1) / hparams.subsampling_factor;
|
||||
const float mask_value = -1e30f;
|
||||
|
||||
std::vector<float> mask_data(n_q * n_k);
|
||||
if (n_k == n_q) {
|
||||
// full attention: mask keys that are padding
|
||||
for (int q = 0; q < n_q; ++q) {
|
||||
for (int k = 0; k < n_k; ++k) {
|
||||
mask_data[q * n_k + k] = (k >= n_tokens_real) ? mask_value : 0.0f;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// local attention: mask keys outside the valid window
|
||||
const int att_left = n_k / 2;
|
||||
for (int q = 0; q < n_q; ++q) {
|
||||
for (int k = 0; k < n_k; ++k) {
|
||||
const int key = q - att_left + k;
|
||||
mask_data[q * n_k + k] = (key >= 0 && key < n_tokens_real) ? 0.0f : mask_value;
|
||||
}
|
||||
}
|
||||
}
|
||||
set_input_f32(attn_mask->name, mask_data);
|
||||
|
||||
// local attention skew mask: zeroes out the probs that were
|
||||
// computed for keys outside the valid sliding window.
|
||||
if (struct ggml_tensor * local_mask = ggml_graph_get_tensor(gf, "local_mask")) {
|
||||
const int lm_k = local_mask->ne[0];
|
||||
const int lm_q = local_mask->ne[1];
|
||||
const int window_size = lm_k - lm_q + 1;
|
||||
std::vector<float> lm_data(lm_q * lm_k);
|
||||
for (int q = 0; q < lm_q; ++q) {
|
||||
for (int k = 0; k < lm_k; ++k) {
|
||||
const int rel = k - q;
|
||||
lm_data[q * lm_k + k] = (rel >= 0 && rel < window_size) ? 1.0f : 0.0f;
|
||||
}
|
||||
}
|
||||
set_input_f32(local_mask->name, lm_data);
|
||||
}
|
||||
|
||||
// Generate rotation frequencies for relative positional encoding.
|
||||
{
|
||||
const int n_state = hparams.n_embd;
|
||||
const int d_half = n_state / 2;
|
||||
const float log_10000 = logf(10000.0f);
|
||||
std::vector<float> freqs(d_half);
|
||||
for (int k = 0; k < d_half; ++k) {
|
||||
freqs[k] = expf(-(float(k * 2) * log_10000 / float(n_state)));
|
||||
}
|
||||
set_input_f32("pos_freqs", freqs);
|
||||
}
|
||||
|
||||
// Generate relative positional distance values which scaled by
|
||||
// the frequency to produce the angles for sin/cos.
|
||||
{
|
||||
// window_size is only known after graph construction since it depends on
|
||||
// n_time from the conv output, so we read it back from the graph tensor.
|
||||
struct ggml_tensor * rel_pos = ggml_graph_get_tensor(gf, "rel_positions");
|
||||
const int window_size = rel_pos->ne[1];
|
||||
std::vector<float> pos(window_size);
|
||||
// local attention: window is fixed at [att_left, att_right]
|
||||
// full attention: window covers the full sequence, centered
|
||||
if (ggml_graph_get_tensor(gf, "local_mask")) {
|
||||
const int att_left = window_size / 2;
|
||||
for (int t = 0; t < window_size; ++t) {
|
||||
pos[t] = float(att_left - t);
|
||||
}
|
||||
} else {
|
||||
const int n_time = (window_size + 1) / 2;
|
||||
for (int t = 0; t < window_size; ++t) {
|
||||
pos[t] = float(n_time - 1 - t);
|
||||
}
|
||||
}
|
||||
set_input_f32(rel_pos->name, pos);
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GRANITE_SPEECH:
|
||||
{
|
||||
const int context_size = ctx->model.hparams.audio_chunk_size;
|
||||
@@ -4760,6 +5119,10 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
|
||||
return ctx->model.qf_proj_blocks.size() * ctx->model.hparams.projection_dim;
|
||||
case PROJECTOR_TYPE_GLM4V:
|
||||
return ctx->model.mm_ffn_down_w->ne[1];
|
||||
case PROJECTOR_TYPE_MIMO_AUDIO:
|
||||
return ctx->model.mm_2_w->ne[1];
|
||||
case PROJECTOR_TYPE_PARAKEET:
|
||||
return ctx->model.mm_1_w->ne[1];
|
||||
default:
|
||||
GGML_ABORT("Unknown projector type");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_mimo_audio::build() {
|
||||
ggml_tensor * inp = build_inp_raw(1); // [n_frames, n_mel, 1]
|
||||
|
||||
ggml_tensor * cur = ggml_conv_1d_ph(ctx0, model.conv1d_1_w, inp, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.conv1d_1_b);
|
||||
cur = ggml_gelu_erf(ctx0, cur);
|
||||
|
||||
cur = ggml_conv_1d_ph(ctx0, model.conv1d_2_w, cur, 2, 1);
|
||||
cur = ggml_add(ctx0, cur, model.conv1d_2_b);
|
||||
cur = ggml_gelu_erf(ctx0, cur);
|
||||
|
||||
ggml_tensor * inpL = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [n_embd, n_pos]
|
||||
const int64_t n_pos = inpL->ne[1];
|
||||
cb(inpL, "after_conv1d", -1);
|
||||
|
||||
GGML_ASSERT((int) hparams.wa_pattern_mode.size() == n_layer);
|
||||
|
||||
ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
|
||||
ggml_set_name(inp_pos, "mimo_audio_positions");
|
||||
ggml_set_input(inp_pos);
|
||||
|
||||
ggml_tensor * full_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
|
||||
ggml_set_name(full_mask, "mimo_audio_full_mask");
|
||||
ggml_set_input(full_mask);
|
||||
|
||||
ggml_tensor * window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
|
||||
ggml_set_name(window_mask, "mimo_audio_window_mask");
|
||||
ggml_set_input(window_mask);
|
||||
|
||||
build_vit_opts opts;
|
||||
opts.attn_mask_layers.resize(n_layer);
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
opts.attn_mask_layers[il] = hparams.wa_pattern_mode[il] == -1 ? full_mask : window_mask;
|
||||
}
|
||||
// the skip connection below must be added before the post-transformer norm,
|
||||
// so build_vit must not apply that norm itself
|
||||
opts.skip_post_ln = true;
|
||||
|
||||
// encoder_skip_layer_id=3 (1-indexed) -> capture output of layer index 2
|
||||
const int skip_capture_il = 2;
|
||||
GGML_ASSERT(n_layer > skip_capture_il);
|
||||
ggml_tensor * skip_hidden = nullptr;
|
||||
opts.callback_layer_out = [&](ggml_tensor * layer_cur, int il) {
|
||||
if (il == skip_capture_il) {
|
||||
skip_hidden = layer_cur;
|
||||
}
|
||||
};
|
||||
|
||||
auto add_pos = [&](ggml_tensor * x, const clip_layer &) {
|
||||
return ggml_rope_ext(ctx0, x, inp_pos, nullptr, d_head,
|
||||
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
};
|
||||
|
||||
inpL = build_vit(inpL, n_pos, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, add_pos, opts);
|
||||
inpL = ggml_reshape_2d(ctx0, inpL, n_embd, n_pos); // build_vit restores a (size-1) batch dim
|
||||
|
||||
GGML_ASSERT(skip_hidden != nullptr);
|
||||
inpL = ggml_add(ctx0, inpL, skip_hidden);
|
||||
|
||||
inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
cb(inpL, "after_transformer", -1);
|
||||
|
||||
// downsample: strided conv (no bias) + gelu + layernorm
|
||||
{
|
||||
ggml_tensor * ds = ggml_cont(ctx0, ggml_transpose(ctx0, inpL)); // [n_pos, n_embd]
|
||||
ds = ggml_conv_1d(ctx0, model.downsample_conv_w, ds, 2, 0, 1);
|
||||
ds = ggml_gelu_erf(ctx0, ds);
|
||||
ds = ggml_cont(ctx0, ggml_transpose(ctx0, ds)); // [n_embd, n_pos/2]
|
||||
ds = build_norm(ds, model.downsample_norm_w, model.downsample_norm_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
inpL = ds;
|
||||
}
|
||||
cb(inpL, "after_downsample", -1);
|
||||
|
||||
// RVQ quantize: codebook ne=[dim, max_bins, n_q]
|
||||
// quantize input vector to codes (type=I32)
|
||||
std::vector<ggml_tensor *> codes;
|
||||
{
|
||||
GGML_ASSERT(model.rvq_codebook != nullptr);
|
||||
const int64_t dim = model.rvq_codebook->ne[0];
|
||||
GGML_ASSERT(dim == inpL->ne[0]);
|
||||
GGML_ASSERT((int64_t) hparams.rvq_codebook_size.size() == model.rvq_codebook->ne[2]);
|
||||
|
||||
ggml_tensor * residual = inpL; // [dim, n_pos_ds]
|
||||
|
||||
for (size_t q = 0; q < hparams.rvq_codebook_size.size(); q++) {
|
||||
const int64_t bins = hparams.rvq_codebook_size[q];
|
||||
ggml_tensor * codebook_q = ggml_view_2d(ctx0, model.rvq_codebook, dim, bins,
|
||||
model.rvq_codebook->nb[1], q * model.rvq_codebook->nb[2]);
|
||||
codebook_q = ggml_cont(ctx0, codebook_q);
|
||||
|
||||
ggml_tensor * codebook_norm = ggml_sum_rows(ctx0, ggml_sqr(ctx0, codebook_q)); // [1, bins]
|
||||
codebook_norm = ggml_cont(ctx0, ggml_transpose(ctx0, codebook_norm)); // [bins, 1]
|
||||
|
||||
ggml_tensor * dot = ggml_mul_mat(ctx0, codebook_q, residual); // [bins, n_pos_ds]
|
||||
ggml_tensor * scores = ggml_sub(ctx0, ggml_scale(ctx0, dot, 2.0f), codebook_norm);
|
||||
|
||||
ggml_tensor * idx = ggml_argmax(ctx0, scores); // [n_pos_ds]
|
||||
codes.push_back(idx);
|
||||
|
||||
ggml_tensor * quant = ggml_get_rows(ctx0, codebook_q, idx); // [dim, n_pos_ds]
|
||||
residual = ggml_sub(ctx0, residual, quant);
|
||||
cb(idx, "rvq_code", (int) q);
|
||||
}
|
||||
}
|
||||
|
||||
// convert codes to LLM embeddings
|
||||
ggml_tensor * code_embd_sum = nullptr;
|
||||
{
|
||||
GGML_ASSERT(model.mm_a_code_embd != nullptr);
|
||||
const int64_t dim = model.mm_a_code_embd->ne[0];
|
||||
const int64_t vocab = model.mm_a_code_embd->ne[1];
|
||||
GGML_ASSERT((int64_t) codes.size() == model.mm_a_code_embd->ne[2]);
|
||||
GGML_ASSERT(dim == inpL->ne[0]);
|
||||
|
||||
for (size_t i = 0; i < codes.size(); i++) {
|
||||
ggml_tensor * table_i = ggml_view_2d(ctx0, model.mm_a_code_embd, dim, vocab,
|
||||
model.mm_a_code_embd->nb[1], i * model.mm_a_code_embd->nb[2]);
|
||||
table_i = ggml_cont(ctx0, table_i);
|
||||
|
||||
ggml_tensor * embd_i = ggml_get_rows(ctx0, table_i, codes[i]); // [dim, n_pos_ds]
|
||||
code_embd_sum = code_embd_sum ? ggml_add(ctx0, code_embd_sum, embd_i) : embd_i;
|
||||
}
|
||||
cb(code_embd_sum, "code_embd_sum", -1);
|
||||
}
|
||||
|
||||
// input_local_transformer
|
||||
// groups of `group_size` consecutive downsampled frames are processed together, attending only within their own group.
|
||||
// Implemented as a block-diagonal mask + in-group-repeating positions
|
||||
// (rather than a real batch dim) - same technique as the encoder's masks above, and as gemma4a's / deepseekocr2's chunked attention.
|
||||
|
||||
// note: hand-rolled here instead of build_vit() because this is a second, independent layer stack
|
||||
// (own layer array/count, RMSNorm instead of LN, SiLU FFN, own RoPE theta)
|
||||
|
||||
ggml_tensor * projected;
|
||||
{
|
||||
const int group_size = hparams.audio_local_group_size;
|
||||
GGML_ASSERT(group_size > 0);
|
||||
const int64_t n_pos_ds = code_embd_sum->ne[1];
|
||||
const int64_t n_groups = (n_pos_ds + group_size - 1) / group_size;
|
||||
const int64_t n_padded = n_groups * group_size;
|
||||
|
||||
ggml_tensor * cur_local = code_embd_sum;
|
||||
if (n_padded != n_pos_ds) {
|
||||
cur_local = ggml_pad(ctx0, cur_local, 0, (int) (n_padded - n_pos_ds), 0, 0);
|
||||
}
|
||||
|
||||
ggml_tensor * local_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_padded);
|
||||
ggml_set_name(local_pos, "mimo_audio_local_positions");
|
||||
ggml_set_input(local_pos);
|
||||
|
||||
ggml_tensor * local_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_padded, n_padded);
|
||||
ggml_set_name(local_mask, "mimo_audio_local_mask");
|
||||
ggml_set_input(local_mask);
|
||||
|
||||
const float local_rope_theta = 640000.0f; // audio_config.rope_theta (differs from the encoder's)
|
||||
auto apply_local_rope = [&](ggml_tensor * x) {
|
||||
return ggml_rope_ext(ctx0, x, local_pos, nullptr, d_head,
|
||||
GGML_ROPE_TYPE_NEOX, 0, local_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
};
|
||||
|
||||
for (int il = 0; il < hparams.audio_local_n_layer; il++) {
|
||||
auto & layer = model.mm_a_local_layers[il];
|
||||
|
||||
ggml_tensor * attn_in = build_norm(cur_local, layer.ln_1_w, nullptr, NORM_TYPE_RMS, eps, il);
|
||||
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, attn_in);
|
||||
if (layer.q_b) {
|
||||
Qcur = ggml_add(ctx0, Qcur, layer.q_b);
|
||||
}
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, attn_in);
|
||||
if (layer.k_b) {
|
||||
Kcur = ggml_add(ctx0, Kcur, layer.k_b);
|
||||
}
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, attn_in);
|
||||
if (layer.v_b) {
|
||||
Vcur = ggml_add(ctx0, Vcur, layer.v_b);
|
||||
}
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_padded);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_padded);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_padded);
|
||||
|
||||
Qcur = apply_local_rope(Qcur);
|
||||
Kcur = apply_local_rope(Kcur);
|
||||
|
||||
ggml_tensor * attn_out = build_attn(layer.o_w, nullptr, Qcur, Kcur, Vcur, local_mask, kq_scale, il);
|
||||
cur_local = ggml_add(ctx0, cur_local, attn_out);
|
||||
|
||||
ggml_tensor * ffn_in = build_norm(cur_local, layer.ln_2_w, nullptr, NORM_TYPE_RMS, eps, il);
|
||||
ggml_tensor * ffn_out = build_ffn(ffn_in,
|
||||
layer.ff_up_w, nullptr,
|
||||
layer.ff_gate_w, nullptr,
|
||||
layer.ff_down_w, nullptr,
|
||||
FFN_SILU, il);
|
||||
cur_local = ggml_add(ctx0, cur_local, ffn_out);
|
||||
}
|
||||
|
||||
cur_local = build_norm(cur_local, model.mm_a_local_norm_w, nullptr, NORM_TYPE_RMS, eps, -1);
|
||||
cb(cur_local, "after_local_transformer", -1);
|
||||
|
||||
// flatten each group of `group_size` frames into one (group_size*n_embd)-dim vector
|
||||
// (matching AudioProjection's flattened input)
|
||||
ggml_tensor * grouped = ggml_reshape_2d(ctx0, cur_local, n_embd * group_size, n_groups);
|
||||
|
||||
// AudioProjection: Linear (no bias) -> GELU -> Linear (no bias)
|
||||
projected = build_ffn(grouped,
|
||||
model.mm_1_w, nullptr,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, nullptr,
|
||||
FFN_GELU_ERF, -1);
|
||||
cb(projected, "after_projection", -1);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, projected);
|
||||
return gf;
|
||||
}
|
||||
@@ -210,6 +210,11 @@ struct clip_graph_qwen3a : clip_graph {
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_mimo_audio : clip_graph {
|
||||
clip_graph_mimo_audio(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_kimik25 : clip_graph {
|
||||
clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
@@ -217,6 +222,11 @@ struct clip_graph_kimik25 : clip_graph {
|
||||
ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode);
|
||||
};
|
||||
|
||||
struct clip_graph_parakeet : clip_graph {
|
||||
clip_graph_parakeet(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_exaone4_5 : clip_graph {
|
||||
clip_graph_exaone4_5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
#include "models.h"
|
||||
|
||||
static constexpr int PARAKEET_LOCAL_ATTN_THRESHOLD = 8192;
|
||||
static constexpr int PARAKEET_LOCAL_ATTN_WINDOW = 128;
|
||||
|
||||
// conv subsampling + conformer encoder
|
||||
ggml_cgraph * clip_graph_parakeet::build() {
|
||||
|
||||
// Conv subsampling
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
|
||||
|
||||
// [freq, time, channels, batch]
|
||||
ggml_tensor * cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[0], inp, 2, 2, 1, 1, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[0]);
|
||||
cb(cur, "pre_conv_0", -1);
|
||||
|
||||
cur = ggml_relu(ctx0, cur);
|
||||
cb(cur, "pre_conv_0_relu", -1);
|
||||
|
||||
// [freq, time, channels, batch]
|
||||
cur = ggml_conv_2d_dw_direct(ctx0, model.pre_encode_conv_X_w[2], cur, 2, 2, 1, 1, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[2]);
|
||||
cb(cur, "pre_conv_2", -1);
|
||||
|
||||
// [freq, time, channels, batch]
|
||||
cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[3], cur, 1, 1, 0, 0, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[3]);
|
||||
cb(cur, "pre_conv_3", -1);
|
||||
|
||||
cur = ggml_relu(ctx0, cur);
|
||||
cb(cur, "pre_conv_3_relu", -1);
|
||||
|
||||
// [freq, time, channels, batch]
|
||||
cur = ggml_conv_2d_dw_direct(ctx0, model.pre_encode_conv_X_w[5], cur, 2, 2, 1, 1, 1, 1);
|
||||
cb(cur, "pre_conv_5_direct", -1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[5]);
|
||||
cb(cur, "pre_conv_5", -1);
|
||||
|
||||
// [freq, time, channels, batch]
|
||||
cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[6], cur, 1, 1, 0, 0, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[6]);
|
||||
cb(cur, "pre_conv_6", -1);
|
||||
|
||||
cur = ggml_relu(ctx0, cur);
|
||||
cb(cur, "pre_conv_6_relu", -1);
|
||||
|
||||
// [freq, time, chan]
|
||||
cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
|
||||
// [freq, chan, time]
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
|
||||
const int n_freq = cur->ne[0];
|
||||
const int n_chan = cur->ne[1];
|
||||
const int n_frames = cur->ne[2];
|
||||
|
||||
// [freq, time, chan, batch] -> [(freq * chan), time]
|
||||
cur = ggml_reshape_2d(ctx0, cur, n_freq * n_chan, n_frames);
|
||||
|
||||
cur = build_mm(model.pre_encode_out_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_out_b);
|
||||
|
||||
ggml_set_name(cur, "pre_enc_out");
|
||||
|
||||
// Encoder
|
||||
|
||||
const auto & hparams = model.hparams;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_state = hparams.n_embd;
|
||||
const float fc_factor = 0.5f;
|
||||
|
||||
const int n_time = cur->ne[1];
|
||||
const bool local_attn = n_time > PARAKEET_LOCAL_ATTN_THRESHOLD;
|
||||
const int att_left = local_attn ? PARAKEET_LOCAL_ATTN_WINDOW : n_time - 1;
|
||||
const int att_right = local_attn ? PARAKEET_LOCAL_ATTN_WINDOW : n_time - 1;
|
||||
const int window_size = local_attn ? att_left + att_right + 1 : 2 * n_time - 1;
|
||||
const int d_half = n_state / 2;
|
||||
const int mask_dim = local_attn ? window_size : n_time;
|
||||
|
||||
// mask [key, n_time]
|
||||
struct ggml_tensor * attn_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, mask_dim, n_time);
|
||||
ggml_set_name(attn_mask, "attn_mask");
|
||||
ggml_set_input(attn_mask);
|
||||
|
||||
struct ggml_tensor * local_mask = nullptr;
|
||||
if (local_attn) {
|
||||
const int chunk = att_left + att_right;
|
||||
local_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, chunk + window_size - 1, chunk);
|
||||
ggml_set_name(local_mask, "local_mask");
|
||||
ggml_set_input(local_mask);
|
||||
}
|
||||
|
||||
struct ggml_tensor * pos_freqs = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, d_half);
|
||||
ggml_set_name(pos_freqs, "pos_freqs");
|
||||
ggml_set_input(pos_freqs);
|
||||
|
||||
struct ggml_tensor * rel_positions = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, window_size);
|
||||
ggml_set_name(rel_positions, "rel_positions");
|
||||
ggml_set_input(rel_positions);
|
||||
|
||||
struct ggml_tensor * freqs = ggml_repeat_4d(ctx0, pos_freqs, d_half, window_size, 1, 1);
|
||||
struct ggml_tensor * theta = ggml_mul(ctx0, freqs, rel_positions);
|
||||
|
||||
struct ggml_tensor * sin = ggml_reshape_3d(ctx0, ggml_sin(ctx0, theta), 1, d_half, window_size);
|
||||
struct ggml_tensor * cos = ggml_reshape_3d(ctx0, ggml_cos(ctx0, theta), 1, d_half, window_size);
|
||||
struct ggml_tensor * pos_emb = ggml_reshape_2d(ctx0, ggml_cont(ctx0, ggml_concat(ctx0, sin, cos, 0)), n_state, window_size);
|
||||
ggml_set_name(pos_emb, "pos_emb");
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
const auto & layer = model.layers[il];
|
||||
// FFN1
|
||||
{
|
||||
struct ggml_tensor * residual = cur;
|
||||
ggml_format_name(cur, "enc_%d_res", il);
|
||||
|
||||
// norm
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ff_norm_w), layer.ff_norm_b);
|
||||
ggml_format_name(cur, "enc_%d_ffn_norm_1", il);
|
||||
|
||||
cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, FFN_SILU, il);
|
||||
ggml_format_name(cur, "enc_%d_ffn_1", il);
|
||||
|
||||
cur = ggml_add(ctx0, residual, ggml_scale(ctx0, cur, fc_factor));
|
||||
ggml_format_name(cur, "enc_%d_res_ffn", il);
|
||||
}
|
||||
|
||||
// self attention block using relative positional encoding from model.position_embedding.
|
||||
{
|
||||
// [feat, time_frames, 1, 1]
|
||||
struct ggml_tensor * residual = cur;
|
||||
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ln_1_w), layer.ln_1_b);
|
||||
ggml_format_name(cur, "enc_%d_attn_norm", il);
|
||||
|
||||
const int n_head = hparams.n_head;
|
||||
const int d_head = n_state / n_head;
|
||||
|
||||
// [feat, time_frames, 1, 1]
|
||||
struct ggml_tensor * Q_cur = build_mm(layer.q_w, cur);
|
||||
struct ggml_tensor * K_cur = build_mm(layer.k_w, cur);
|
||||
struct ggml_tensor * V_cur = build_mm(layer.v_w, cur);
|
||||
|
||||
// [d_head, n_heads, n_time, 1]
|
||||
Q_cur = ggml_reshape_3d(ctx0, Q_cur, d_head, n_head, n_time);
|
||||
K_cur = ggml_reshape_3d(ctx0, K_cur, d_head, n_head, n_time);
|
||||
V_cur = ggml_reshape_3d(ctx0, V_cur, d_head, n_head, n_time);
|
||||
|
||||
// [n_state, window_size]
|
||||
struct ggml_tensor * pos = build_mm(layer.linear_pos_w, pos_emb);
|
||||
// [feat, head, window_size, 1]
|
||||
pos = ggml_reshape_3d(ctx0, pos, d_head, n_head, pos_emb->ne[1]);
|
||||
// [feat, window_size, head, 1]
|
||||
pos = ggml_cont(ctx0, ggml_permute(ctx0, pos, 0, 2, 1, 3));
|
||||
ggml_format_name(pos, "enc_%d_attn_pos", il);
|
||||
|
||||
if (local_attn) {
|
||||
const int chunk = att_left + att_right;
|
||||
const int n_group = (n_time + chunk - 1) / chunk;
|
||||
const int n_time_padded = n_group * chunk;
|
||||
const int n_kv_chunk = chunk + window_size - 1;
|
||||
const int n_kv_dense = n_kv_chunk * n_group;
|
||||
const bool need_padding = n_time_padded > n_time;
|
||||
|
||||
Q_cur = ggml_cont(ctx0, ggml_permute(ctx0, Q_cur, 0, 2, 1, 3));
|
||||
K_cur = ggml_cont(ctx0, ggml_permute(ctx0, K_cur, 0, 2, 1, 3));
|
||||
V_cur = ggml_cont(ctx0, ggml_permute(ctx0, V_cur, 0, 2, 1, 3));
|
||||
|
||||
// content bias
|
||||
struct ggml_tensor * bias_u = ggml_reshape_3d(ctx0, layer.pos_bias_u, d_head, 1, n_head);
|
||||
struct ggml_tensor * Q_u = ggml_add(ctx0, Q_cur, bias_u);
|
||||
|
||||
// position bias
|
||||
struct ggml_tensor * bias_v = ggml_reshape_3d(ctx0, layer.pos_bias_v, d_head, 1, n_head);
|
||||
struct ggml_tensor * Q_v = ggml_add(ctx0, Q_cur, bias_v);
|
||||
|
||||
// right pad the time dimension
|
||||
struct ggml_tensor * Q_u_padded = need_padding ?
|
||||
ggml_pad_ext(ctx0, Q_u, 0, 0, 0, n_time_padded - n_time, 0, 0, 0, 0) : Q_u;
|
||||
Q_u_padded = ggml_reshape_4d(ctx0, Q_u_padded, d_head, chunk, n_group, n_head);
|
||||
|
||||
// pad front and back for the first and last time frames
|
||||
struct ggml_tensor * K_padded = ggml_pad_ext(ctx0, K_cur, 0, 0, att_left, att_right, 0, 0, 0, 0);
|
||||
if (n_kv_dense > K_padded->ne[1]) {
|
||||
K_padded = ggml_pad_ext(ctx0, K_padded, 0, 0, 0, n_kv_dense - K_padded->ne[1], 0, 0, 0, 0);
|
||||
}
|
||||
|
||||
// sliding window view: each group spans n_kv_chunk keys but steps by chunk
|
||||
struct ggml_tensor * K_chunk = ggml_view_4d(ctx0, K_padded,
|
||||
d_head, n_kv_chunk, n_group, n_head,
|
||||
K_padded->nb[1],
|
||||
(size_t) chunk * K_padded->nb[1],
|
||||
K_padded->nb[2],
|
||||
0);
|
||||
K_chunk = ggml_cont(ctx0, K_chunk);
|
||||
|
||||
struct ggml_tensor * content_scores = ggml_mul_mat(ctx0, K_chunk, Q_u_padded);
|
||||
|
||||
// trim the dense output down to window_size scores per query
|
||||
content_scores = ggml_view_4d(ctx0, content_scores,
|
||||
window_size, chunk, n_group, n_head,
|
||||
(size_t) (chunk + window_size) * content_scores->nb[0],
|
||||
content_scores->nb[2],
|
||||
content_scores->nb[3],
|
||||
0);
|
||||
content_scores = ggml_cont(ctx0, content_scores);
|
||||
|
||||
// ungroup: [window_size, n_time_padded, n_head]
|
||||
content_scores = ggml_reshape_3d(ctx0, content_scores, window_size, n_time_padded, n_head);
|
||||
if (need_padding) {
|
||||
content_scores = ggml_view_3d(ctx0, content_scores,
|
||||
window_size, n_time, n_head,
|
||||
content_scores->nb[1],
|
||||
content_scores->nb[2],
|
||||
0);
|
||||
}
|
||||
|
||||
// Q_v: [d_head, time, head]
|
||||
Q_v = ggml_cont(ctx0, ggml_permute(ctx0, Q_v, 0, 2, 1, 3));
|
||||
struct ggml_tensor * rel_pos_scores = ggml_mul_mat(ctx0, pos, Q_v);
|
||||
|
||||
struct ggml_tensor * attn_scores = ggml_add(ctx0, content_scores, rel_pos_scores);
|
||||
attn_scores = ggml_soft_max_ext(ctx0, attn_scores, attn_mask, 1.0f / std::sqrt(d_head), 0.0f);
|
||||
ggml_format_name(attn_scores, "enc_%d_attn_probs", il);
|
||||
|
||||
// expand probs back to n_kv_chunk width for the V matmul
|
||||
struct ggml_tensor * probs_padded = need_padding ?
|
||||
ggml_pad_ext(ctx0, attn_scores, 0, 0, 0, n_time_padded - n_time, 0, 0, 0, 0) : attn_scores;
|
||||
|
||||
probs_padded = ggml_reshape_4d(ctx0, probs_padded, window_size, chunk, n_group, n_head);
|
||||
probs_padded = ggml_pad_ext(ctx0, probs_padded, 0, chunk, 0, 0, 0, 0, 0, 0);
|
||||
probs_padded = ggml_view_4d(ctx0, probs_padded,
|
||||
n_kv_chunk, chunk, n_group, n_head,
|
||||
(size_t) n_kv_chunk * probs_padded->nb[0],
|
||||
probs_padded->nb[2],
|
||||
probs_padded->nb[3],
|
||||
0);
|
||||
probs_padded = ggml_cont(ctx0, probs_padded);
|
||||
probs_padded = ggml_mul(ctx0, probs_padded, local_mask);
|
||||
|
||||
struct ggml_tensor * V_padded = ggml_pad_ext(ctx0, V_cur, 0, 0, att_left, att_right, 0, 0, 0, 0);
|
||||
if (n_kv_dense > V_padded->ne[1]) {
|
||||
V_padded = ggml_pad_ext(ctx0, V_padded, 0, 0, 0, n_kv_dense - V_padded->ne[1], 0, 0, 0, 0);
|
||||
}
|
||||
V_padded = ggml_cont(ctx0, ggml_transpose(ctx0, V_padded));
|
||||
|
||||
struct ggml_tensor * V_chunk = ggml_view_4d(ctx0, V_padded,
|
||||
n_kv_chunk, d_head, n_group, n_head,
|
||||
V_padded->nb[1],
|
||||
(size_t) chunk * V_padded->nb[0],
|
||||
V_padded->nb[2],
|
||||
0);
|
||||
V_chunk = ggml_cont(ctx0, V_chunk);
|
||||
|
||||
cur = ggml_mul_mat(ctx0, V_chunk, probs_padded);
|
||||
cur = ggml_reshape_3d(ctx0, cur, d_head, n_time_padded, n_head);
|
||||
if (need_padding) {
|
||||
cur = ggml_view_3d(ctx0, cur, d_head, n_time, n_head, cur->nb[1], cur->nb[2], 0);
|
||||
}
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));
|
||||
cur = ggml_reshape_2d(ctx0, cur, n_state, n_time);
|
||||
cur = build_mm(layer.o_w, cur);
|
||||
} else {
|
||||
// full attention
|
||||
struct ggml_tensor * Q_u = ggml_add(ctx0, Q_cur, layer.pos_bias_u);
|
||||
ggml_format_name(Q_u, "enc_%d_attn_q_u", il);
|
||||
|
||||
struct ggml_tensor * K_prep = ggml_permute(ctx0, K_cur, 0, 2, 1, 3);
|
||||
struct ggml_tensor * Q_prep = ggml_permute(ctx0, Q_u, 0, 2, 1, 3);
|
||||
struct ggml_tensor * content_scores = ggml_mul_mat(ctx0, K_prep, Q_prep);
|
||||
ggml_format_name(content_scores, "enc_%d_attn_content_scores", il);
|
||||
|
||||
struct ggml_tensor * Q_v = ggml_add(ctx0, Q_cur, layer.pos_bias_v);
|
||||
ggml_format_name(Q_v, "enc_%d_attn_q_v", il);
|
||||
|
||||
Q_v = ggml_permute(ctx0, Q_v, 0, 2, 1, 3);
|
||||
Q_v = ggml_cont(ctx0, Q_v);
|
||||
ggml_format_name(Q_v, "enc_%d_attn_q_v_perm", il);
|
||||
|
||||
struct ggml_tensor * rel_pos_scores = ggml_mul_mat(ctx0, pos, Q_v);
|
||||
ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos", il);
|
||||
|
||||
// Relative positional shift
|
||||
{
|
||||
const auto pos_window = rel_pos_scores->ne[0];
|
||||
const auto n_frame = rel_pos_scores->ne[1];
|
||||
const auto n_head = rel_pos_scores->ne[2];
|
||||
|
||||
rel_pos_scores = ggml_pad(ctx0, rel_pos_scores, 1, 0, 0, 0);
|
||||
rel_pos_scores = ggml_roll(ctx0, rel_pos_scores, 1, 0, 0, 0);
|
||||
|
||||
rel_pos_scores = ggml_reshape_3d(ctx0, rel_pos_scores, n_frame, pos_window + 1, n_head);
|
||||
rel_pos_scores = ggml_cont(ctx0, rel_pos_scores);
|
||||
ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos_reshaped", il);
|
||||
|
||||
int center = pos_window / 2;
|
||||
size_t offset = rel_pos_scores->nb[0] * (center+1);
|
||||
|
||||
rel_pos_scores = ggml_view_3d(ctx0, rel_pos_scores,
|
||||
n_frame, pos_window, n_head,
|
||||
(pos_window) * 4,
|
||||
rel_pos_scores->nb[2],
|
||||
offset);
|
||||
rel_pos_scores = ggml_cont(ctx0, rel_pos_scores);
|
||||
ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos_shifted", il);
|
||||
|
||||
rel_pos_scores = ggml_view_3d(ctx0, rel_pos_scores,
|
||||
content_scores->ne[0],
|
||||
content_scores->ne[1],
|
||||
rel_pos_scores->ne[2],
|
||||
rel_pos_scores->nb[1],
|
||||
rel_pos_scores->nb[2],
|
||||
0);
|
||||
rel_pos_scores = ggml_cont(ctx0, rel_pos_scores);
|
||||
ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos_shifted_view", il);
|
||||
}
|
||||
|
||||
struct ggml_tensor * attn_scores = ggml_add(ctx0, content_scores, rel_pos_scores);
|
||||
ggml_format_name(attn_scores, "enc_%d_attn_scores", il);
|
||||
attn_scores = ggml_scale(ctx0, attn_scores, 1.0f / std::sqrt(d_head));
|
||||
attn_scores = ggml_add(ctx0, attn_scores, attn_mask);
|
||||
ggml_format_name(attn_scores, "enc_%d_attn_scores_scaled", il);
|
||||
|
||||
struct ggml_tensor * probs = ggml_soft_max(ctx0, attn_scores);
|
||||
ggml_format_name(probs, "enc_%d_attn_probs", il);
|
||||
|
||||
V_cur = ggml_cont(ctx0, ggml_permute(ctx0, V_cur, 1, 2, 0, 3));
|
||||
ggml_format_name(V_cur, "enc_%d_attn_v_cur", il);
|
||||
cur = ggml_mul_mat(ctx0, probs, V_cur);
|
||||
ggml_format_name(cur, "enc_%d_attn_inp", il);
|
||||
|
||||
cur = ggml_permute(ctx0, cur, 2, 0, 1, 3);
|
||||
cur = ggml_cont_2d(ctx0, cur, n_state, n_time);
|
||||
cur = build_mm(layer.o_w, cur);
|
||||
}
|
||||
ggml_format_name(cur, "enc_%d_attn_out", il);
|
||||
|
||||
cur = ggml_add(ctx0, residual, cur);
|
||||
ggml_format_name(cur, "enc_%d_attn_res", il);
|
||||
}
|
||||
|
||||
// Convolution
|
||||
{
|
||||
struct ggml_tensor * residual = cur;
|
||||
ggml_format_name(cur, "enc_%d_residual_conv", il);
|
||||
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.norm_conv_w), layer.norm_conv_b);
|
||||
ggml_format_name(cur, "enc_%d_norm_conv", il);
|
||||
|
||||
// pointwise 1d convolution:
|
||||
cur = build_mm(layer.conv_pw1_w, cur);
|
||||
ggml_format_name(cur, "enc_%d_conv_pw1", il);
|
||||
|
||||
{
|
||||
int64_t d = cur->ne[0] / 2;
|
||||
struct ggml_tensor * signal = ggml_view_2d(ctx0, cur, d, cur->ne[1], cur->nb[1], 0);
|
||||
struct ggml_tensor * gate = ggml_view_2d(ctx0, cur, d, cur->ne[1], cur->nb[1], d * cur->nb[0]);
|
||||
|
||||
cur = ggml_mul(ctx0, signal, ggml_sigmoid(ctx0, gate));
|
||||
ggml_format_name(cur, "enc_%d_conv_glu", il);
|
||||
}
|
||||
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
|
||||
// use ggml_ssm_conv for f32 precision
|
||||
const int dw_pad = (hparams.audio_conv_kernel_size - 1) / 2;
|
||||
cur = ggml_pad(ctx0, cur, dw_pad, 0, 0, 0);
|
||||
cur = ggml_roll(ctx0, cur, dw_pad, 0, 0, 0);
|
||||
cur = ggml_pad(ctx0, cur, dw_pad, 0, 0, 0);
|
||||
ggml_format_name(cur, "enc_%d_conv_dw_pad", il);
|
||||
|
||||
cur = ggml_ssm_conv(ctx0, cur, layer.conv_dw_w);
|
||||
ggml_format_name(cur, "enc_%d_conv_1d_dw", il);
|
||||
|
||||
cur = ggml_sub(ctx0, cur, layer.conv_norm_mean);
|
||||
struct ggml_tensor * std = ggml_sqrt(ctx0, layer.conv_norm_var);
|
||||
cur = ggml_div(ctx0, cur, std);
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.conv_norm_w), layer.conv_norm_b);
|
||||
ggml_format_name(cur, "enc_%d_conv_bn", il);
|
||||
|
||||
cur = ggml_silu(ctx0, cur);
|
||||
ggml_format_name(cur, "enc_%d_conv_silu", il);
|
||||
|
||||
cur = build_mm(layer.conv_pw2_w, cur);
|
||||
ggml_format_name(cur, "enc_%d_conv_pw2", il);
|
||||
|
||||
cur = ggml_add(ctx0, residual, cur);
|
||||
ggml_format_name(cur, "enc_%d_conv_res", il);
|
||||
}
|
||||
|
||||
// FFN2
|
||||
{
|
||||
struct ggml_tensor * residual = cur;
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ff_norm_1_w), layer.ff_norm_1_b);
|
||||
ggml_format_name(cur, "enc_%d_ffn_norm_2", il);
|
||||
|
||||
cur = build_ffn(cur, layer.ff_up_1_w, nullptr, nullptr, nullptr, layer.ff_down_1_w, nullptr, FFN_SILU, il);
|
||||
cur = ggml_add(ctx0, residual, ggml_scale(ctx0, cur, 0.5));
|
||||
ggml_format_name(cur, "enc_%d_ffn_res", il);
|
||||
}
|
||||
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ln_2_w), layer.ln_2_b);
|
||||
}
|
||||
|
||||
cb(cur, "encoder_out", -1);
|
||||
|
||||
cur = ggml_rms_norm(ctx0, cur, 1e-6);
|
||||
cur = ggml_mul(ctx0, cur, model.mm_norm_pre_w);
|
||||
cb(cur, "sound_projection.norm", -1);
|
||||
|
||||
cur = build_ffn(cur, model.mm_0_w, model.mm_0_b, nullptr, nullptr, model.mm_1_w, model.mm_1_b, FFN_RELU_SQR, -1);
|
||||
cb(cur, "projected", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -725,6 +725,72 @@ bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * sa
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// mtmd_audio_preprocessor_mimo_audio
|
||||
//
|
||||
// Matches torchaudio.transforms.MelSpectrogram(power=1.0, center=True) followed by
|
||||
// log(clip(spec, min=1e-7)): HTK mel scale, no Slaney area norm, magnitude (not power)
|
||||
// spectrogram, natural log, reflect-padded by n_fft/2 on each side.
|
||||
//
|
||||
|
||||
void mtmd_audio_preprocessor_mimo_audio::initialize() {
|
||||
cache.fill_sin_cos_table(hparams.audio_n_fft);
|
||||
cache.fill_hann_window(hparams.audio_window_len, true);
|
||||
cache.fill_mel_filterbank_matrix(
|
||||
hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate,
|
||||
0.0f, hparams.audio_sample_rate / 2.0f,
|
||||
/*slaney_area_norm=*/ false,
|
||||
/*scale=*/ 1.0f,
|
||||
/*use_htk=*/ true
|
||||
);
|
||||
}
|
||||
|
||||
bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float * samples,
|
||||
size_t n_samples,
|
||||
std::vector<mtmd_audio_mel> & output) {
|
||||
if (n_samples == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
GGML_ASSERT(!cache.sin_vals.empty());
|
||||
GGML_ASSERT(!cache.cos_vals.empty());
|
||||
GGML_ASSERT(!cache.filters.data.empty());
|
||||
|
||||
const int pad = hparams.audio_n_fft / 2;
|
||||
|
||||
std::vector<float> padded(n_samples + 2 * pad, 0.0f);
|
||||
for (int i = 0; i < pad; i++) {
|
||||
int src = pad - i;
|
||||
padded[i] = (src < (int)n_samples) ? samples[src] : 0.0f;
|
||||
}
|
||||
std::copy(samples, samples + n_samples, padded.begin() + pad);
|
||||
for (int i = 0; i < pad; i++) {
|
||||
int src = (int)n_samples - 2 - i;
|
||||
padded[n_samples + pad + i] = (src >= 0) ? samples[src] : 0.0f;
|
||||
}
|
||||
|
||||
filter_params params;
|
||||
params.n_mel = hparams.n_mel_bins;
|
||||
params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
|
||||
params.hann_window_size = hparams.audio_window_len;
|
||||
params.hop_length = hparams.audio_hop_len;
|
||||
params.sample_rate = hparams.audio_sample_rate;
|
||||
params.no_padding = true; // reflect padding already applied above
|
||||
params.use_natural_log = true;
|
||||
params.use_magnitude = true;
|
||||
params.mel_floor = 1e-7f;
|
||||
params.norm_per_feature = false;
|
||||
|
||||
mtmd_audio_mel out;
|
||||
bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), 4, params, cache, out);
|
||||
if (!ok) {
|
||||
return false;
|
||||
}
|
||||
|
||||
output.push_back(std::move(out));
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// mtmd_audio_preprocessor_conformer
|
||||
//
|
||||
@@ -956,6 +1022,209 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s
|
||||
}
|
||||
|
||||
//
|
||||
// mtmd_audio_preprocessor_parakeet implementation
|
||||
//
|
||||
|
||||
void mtmd_audio_preprocessor_parakeet::worker_thread(
|
||||
int ith,
|
||||
const float * window_func,
|
||||
int window_size,
|
||||
const std::vector<float> & samples,
|
||||
int n_samples,
|
||||
int frame_size,
|
||||
int frame_step,
|
||||
int n_threads,
|
||||
int n_fft_bins,
|
||||
const mtmd_audio_cache & cache,
|
||||
mtmd_audio_mel & mel) {
|
||||
std::vector<float> fft_in(frame_size * 2, 0.0);
|
||||
std::vector<float> fft_out(frame_size * 2 * 2 * 2);
|
||||
|
||||
int n_fb = n_fft_bins;
|
||||
int i = ith;
|
||||
|
||||
GGML_ASSERT(n_fb == 1 + (frame_size / 2));
|
||||
|
||||
const double eps = 5.960464477539063e-08;
|
||||
|
||||
for (; i < std::min(n_samples / frame_step + 1, (int) mel.n_len); i += n_threads) {
|
||||
const int offset = i * frame_step;
|
||||
const int window_pad_left = (frame_size - window_size) / 2;
|
||||
|
||||
// Zero-pad left.
|
||||
std::fill(fft_in.begin(), fft_in.begin() + window_pad_left, 0.0f);
|
||||
|
||||
// Apply windowed samples in the center.
|
||||
const int n_to_process = std::min({window_size, n_samples - offset});
|
||||
for (int j = 0; j < n_to_process; j++) {
|
||||
fft_in[window_pad_left + j] = window_func[j] * samples[offset + window_pad_left + j];
|
||||
}
|
||||
|
||||
// Zero-pad right.
|
||||
std::fill(fft_in.begin() + window_pad_left + n_to_process, fft_in.begin() + frame_size, 0.0f);
|
||||
|
||||
// FFT.
|
||||
fft(cache, fft_in.data(), frame_size, fft_out.data());
|
||||
|
||||
// Calculate modulus^2 of complex numbers.
|
||||
for (int j = 0; j < n_fb; j++) {
|
||||
fft_out[j] = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]);
|
||||
}
|
||||
|
||||
// mel spectrogram.
|
||||
for (int j = 0; j < mel.n_mel; j++) {
|
||||
double sum = 0.0;
|
||||
int k = 0;
|
||||
for (k = 0; k < n_fb - 3; k += 4) {
|
||||
sum +=
|
||||
fft_out[k + 0] * cache.filters.data[j * n_fb + k + 0] +
|
||||
fft_out[k + 1] * cache.filters.data[j * n_fb + k + 1] +
|
||||
fft_out[k + 2] * cache.filters.data[j * n_fb + k + 2] +
|
||||
fft_out[k + 3] * cache.filters.data[j * n_fb + k + 3];
|
||||
}
|
||||
for (; k < n_fb; k++) {
|
||||
sum += fft_out[k] * cache.filters.data[j * n_fb + k];
|
||||
}
|
||||
mel.data[j * mel.n_len + i] = std::log(sum + eps);
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise fft_out are all zero.
|
||||
const double empty_sum = std::log(eps);
|
||||
for (; i < mel.n_len; i += n_threads) {
|
||||
for (int j = 0; j < mel.n_mel; j++) {
|
||||
mel.data[j * mel.n_len + i] = empty_sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void mtmd_audio_preprocessor_parakeet::initialize() {
|
||||
cache.fill_sin_cos_table(hparams.audio_n_fft);
|
||||
|
||||
const size_t n_fft = hparams.audio_n_fft / 2 + 1;
|
||||
GGML_ASSERT(hparams.mel_filters.size() == (size_t)hparams.n_mel_bins * n_fft);
|
||||
cache.filters.n_mel = hparams.n_mel_bins;
|
||||
cache.filters.n_fft = n_fft;
|
||||
cache.filters.data = hparams.mel_filters;
|
||||
|
||||
GGML_ASSERT(hparams.window.size() == (size_t)hparams.audio_window_len);
|
||||
GGML_ASSERT(hparams.window.size() <= (size_t) hparams.audio_n_fft);
|
||||
cache.hann_window = hparams.window;
|
||||
}
|
||||
|
||||
bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples,
|
||||
size_t n_samples_in,
|
||||
std::vector<mtmd_audio_mel> & output) {
|
||||
if (n_samples_in == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
filter_params params;
|
||||
params.n_mel = hparams.n_mel_bins;
|
||||
params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
|
||||
params.hann_window_size = hparams.audio_window_len;
|
||||
params.hop_length = hparams.audio_hop_len;
|
||||
params.sample_rate = hparams.audio_sample_rate;
|
||||
|
||||
GGML_ASSERT(!cache.sin_vals.empty());
|
||||
GGML_ASSERT(!cache.cos_vals.empty());
|
||||
GGML_ASSERT(!cache.filters.data.empty());
|
||||
|
||||
const float * window_func = cache.hann_window.data();
|
||||
const int window_size = params.hann_window_size;
|
||||
const int frame_size = (params.n_fft_bins - 1) * 2;
|
||||
const int frame_step = params.hop_length;
|
||||
|
||||
// Apply preemphasis filter (high-pass): x[i] = x[i] - 0.97 * x[i-1]
|
||||
std::vector<float> samples_preprocessed(samples, samples + n_samples_in);
|
||||
{
|
||||
const float preemph = 0.97f;
|
||||
for (int i = n_samples_in - 1; i > 0; i--) {
|
||||
samples_preprocessed[i] = samples_preprocessed[i] - preemph * samples_preprocessed[i - 1];
|
||||
}
|
||||
}
|
||||
|
||||
// Parakeet uses centered constant padding
|
||||
const size_t pad = (size_t)(frame_size / 2);
|
||||
std::vector<float> samples_padded(n_samples_in + 2 * pad, 0.0f);
|
||||
std::copy(samples_preprocessed.begin(), samples_preprocessed.end(), samples_padded.begin() + pad);
|
||||
|
||||
mtmd_audio_mel out_full;
|
||||
out_full.n_mel = params.n_mel;
|
||||
out_full.n_len = (samples_padded.size() - frame_size) / frame_step + 1;
|
||||
out_full.n_len_org = out_full.n_len;
|
||||
out_full.data.resize(out_full.n_mel * out_full.n_len);
|
||||
|
||||
const int n_threads = 4;
|
||||
std::vector<std::thread> workers(n_threads - 1);
|
||||
for (int iw = 0; iw < n_threads - 1; ++iw) {
|
||||
workers[iw] = std::thread(
|
||||
worker_thread, iw + 1,
|
||||
window_func,
|
||||
window_size,
|
||||
std::cref(samples_padded),
|
||||
samples_padded.size(),
|
||||
frame_size,
|
||||
frame_step,
|
||||
n_threads,
|
||||
params.n_fft_bins,
|
||||
std::cref(cache),
|
||||
std::ref(out_full)
|
||||
);
|
||||
}
|
||||
|
||||
worker_thread(0,
|
||||
window_func,
|
||||
window_size,
|
||||
samples_padded,
|
||||
samples_padded.size(),
|
||||
frame_size,
|
||||
frame_step,
|
||||
n_threads,
|
||||
params.n_fft_bins,
|
||||
cache,
|
||||
out_full);
|
||||
|
||||
for (int iw = 0; iw < n_threads - 1; ++iw) {
|
||||
workers[iw].join();
|
||||
}
|
||||
|
||||
// Per-feature normalization (only on valid frames)
|
||||
{
|
||||
const double eps = 1e-5;
|
||||
int valid_frames = n_samples_in / frame_step;
|
||||
|
||||
for (int j = 0; j < out_full.n_mel; j++) {
|
||||
double sum = 0.0;
|
||||
double sq_diff_sum = 0.0;
|
||||
|
||||
// Calculate Mean ONLY on valid audio frames
|
||||
for (int i = 0; i < valid_frames; i++) {
|
||||
sum += (double)out_full.data[j * out_full.n_len + i];
|
||||
}
|
||||
double mean = sum / valid_frames;
|
||||
|
||||
// Calculate Variance ONLY on valid audio frames
|
||||
for (int i = 0; i < valid_frames; i++) {
|
||||
double diff = (double)out_full.data[j * out_full.n_len + i] - mean;
|
||||
sq_diff_sum += diff * diff;
|
||||
}
|
||||
|
||||
double std_dev = std::sqrt(sq_diff_sum / (valid_frames - 1.0));
|
||||
double denominator = std_dev + eps;
|
||||
|
||||
// Apply to ALL frames (including the padded ones)
|
||||
for (int i = 0; i < out_full.n_len; i++) {
|
||||
out_full.data[j * out_full.n_len + i] = (float)((out_full.data[j * out_full.n_len + i] - mean) / denominator);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output.push_back(std::move(out_full));
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
// mtmd_audio_preprocessor_gemma4ua
|
||||
//
|
||||
|
||||
|
||||
@@ -111,6 +111,30 @@ struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
|
||||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
|
||||
|
||||
private:
|
||||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { }
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
|
||||
|
||||
private:
|
||||
mtmd_audio_cache cache;
|
||||
|
||||
static void worker_thread(int ith, const float * window_func, int window_size,
|
||||
const std::vector<float> & samples, int n_samples,
|
||||
int frame_size, int frame_step, int n_threads,
|
||||
int n_fft_bins,
|
||||
const mtmd_audio_cache & cache, mtmd_audio_mel & mel);
|
||||
};
|
||||
|
||||
//
|
||||
// streaming ISTFT - converts spectrogram frames back to audio one frame at a time
|
||||
//
|
||||
|
||||
@@ -724,12 +724,22 @@ struct mtmd_context {
|
||||
aud_end = "<audio|>";
|
||||
audio_preproc = std::make_unique<mtmd_audio_preprocessor_gemma4a>(ctx_a);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_PARAKEET:
|
||||
{
|
||||
audio_preproc = std::make_unique<mtmd_audio_preprocessor_parakeet>(ctx_a);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GEMMA4UA:
|
||||
{
|
||||
aud_beg = "<|audio>";
|
||||
aud_end = "<audio|>";
|
||||
audio_preproc = std::make_unique<mtmd_audio_preprocessor_gemma4ua>(ctx_a);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MIMO_AUDIO:
|
||||
{
|
||||
aud_beg = "<|mimo_audio_start|>";
|
||||
aud_end = "<|mimo_audio_end|>";
|
||||
audio_preproc = std::make_unique<mtmd_audio_preprocessor_mimo_audio>(ctx_a);
|
||||
} break;
|
||||
default:
|
||||
throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj));
|
||||
}
|
||||
|
||||
@@ -164,6 +164,8 @@ struct server_slot {
|
||||
llama_context * ctx_tgt = nullptr;
|
||||
llama_context * ctx_dft = nullptr;
|
||||
|
||||
common_memory mem;
|
||||
|
||||
// multimodal
|
||||
mtmd_context * mctx = nullptr;
|
||||
mtmd::batch_ptr mbatch = nullptr;
|
||||
@@ -253,10 +255,7 @@ struct server_slot {
|
||||
void prompt_clear() {
|
||||
SLT_TRC(*this, "clearing prompt with %zu tokens\n", prompt.tokens.size());
|
||||
|
||||
common_context_seq_rm(ctx_tgt, id, -1, -1);
|
||||
if (ctx_dft) {
|
||||
common_context_seq_rm(ctx_dft, id, -1, -1);
|
||||
}
|
||||
mem.seq_rm(id, -1, -1);
|
||||
|
||||
prompt.clear();
|
||||
}
|
||||
@@ -668,13 +667,8 @@ struct server_slot {
|
||||
void copy_state_to(server_slot & other) const {
|
||||
GGML_ASSERT(state == SLOT_STATE_DONE_PROMPT);
|
||||
|
||||
common_context_seq_rm(ctx_tgt, other.id, -1, -1);
|
||||
common_context_seq_cp(ctx_tgt, id, other.id, -1, -1);
|
||||
|
||||
if (ctx_dft) {
|
||||
common_context_seq_rm(ctx_dft, other.id, -1, -1);
|
||||
common_context_seq_cp(ctx_dft, id, other.id, -1, -1);
|
||||
}
|
||||
mem.seq_rm(other.id, -1, -1);
|
||||
mem.seq_cp(id, other.id, -1, -1);
|
||||
|
||||
other.n_decoded = n_decoded;
|
||||
other.n_remaining = n_remaining;
|
||||
@@ -1302,6 +1296,7 @@ private:
|
||||
slot.id = i;
|
||||
slot.ctx_tgt = ctx_tgt;
|
||||
slot.ctx_dft = ctx_dft;
|
||||
slot.mem.init(ctx_tgt, ctx_dft);
|
||||
slot.spec = spec.get();
|
||||
slot.n_ctx = n_ctx_slot;
|
||||
|
||||
@@ -1542,7 +1537,7 @@ private:
|
||||
|
||||
// find the slot that has at least n% prompt similarity
|
||||
if (slot_prompt_similarity != 0.0f) {
|
||||
float sim_best = 0;
|
||||
float f_sim_best = 0;
|
||||
|
||||
for (server_slot & slot : slots) {
|
||||
if (task.id_slot != -1 && slot.id != task.id_slot) {
|
||||
@@ -1551,6 +1546,7 @@ private:
|
||||
|
||||
// skip the slot if it is not available
|
||||
if (slot.is_processing()) {
|
||||
SLT_TRC(slot, " - skipping, is_processing = %d\n", slot.is_processing());
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1558,26 +1554,30 @@ private:
|
||||
|
||||
// skip the slot if it does not contains cached tokens
|
||||
if (tokens.empty()) {
|
||||
SLT_TRC(slot, "%s", " - skipping, slot is empty\n");
|
||||
continue;
|
||||
}
|
||||
|
||||
// fraction of the Longest Common Prefix length with respect to the input prompt length
|
||||
const float sim_cur = float(tokens.get_common_prefix(task.tokens)) / task.tokens.size();
|
||||
const size_t lcp_len = tokens.get_common_prefix(task.tokens);
|
||||
const float f_sim_cur = float(lcp_len) / task.tokens.size();
|
||||
|
||||
SLT_TRC(slot, " - checking sim = %.3f (%zu/%zu) > %.3f\n", f_sim_cur, lcp_len, task.tokens.size(), slot_prompt_similarity);
|
||||
|
||||
// select the current slot if the criteria match
|
||||
if (sim_cur > sim_best && sim_cur > slot_prompt_similarity) {
|
||||
sim_best = sim_cur;
|
||||
if (f_sim_cur > f_sim_best && f_sim_cur > slot_prompt_similarity) {
|
||||
f_sim_best = f_sim_cur;
|
||||
|
||||
ret = &slot;
|
||||
}
|
||||
}
|
||||
|
||||
if (ret != nullptr) {
|
||||
const float f_keep = (sim_best*task.tokens.size()) / ret->prompt.tokens.size();
|
||||
const float f_keep = (f_sim_best*task.tokens.size()) / ret->prompt.tokens.size();
|
||||
|
||||
if (task.id_slot == -1) {
|
||||
SLT_INF(*ret, "selected slot by LCP similarity, sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",
|
||||
sim_best, slot_prompt_similarity, f_keep);
|
||||
SLT_INF(*ret, "selected slot by LCP similarity, f_sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",
|
||||
f_sim_best, slot_prompt_similarity, f_keep);
|
||||
}
|
||||
|
||||
// if we are about to lose a large portion of the existing context - save it in the prompt cache
|
||||
@@ -2881,13 +2881,8 @@ private:
|
||||
|
||||
SLT_WRN(slot, "slot context shift, n_keep = %d, n_left = %d, n_discard = %d\n", n_keep, n_left, n_discard);
|
||||
|
||||
common_context_seq_rm (ctx_tgt, slot.id, n_keep , n_keep + n_discard);
|
||||
common_context_seq_add(ctx_tgt, slot.id, n_keep + n_discard, slot.prompt.n_tokens(), -n_discard);
|
||||
|
||||
if (ctx_dft) {
|
||||
common_context_seq_rm (ctx_dft, slot.id, n_keep , n_keep + n_discard);
|
||||
common_context_seq_add(ctx_dft, slot.id, n_keep + n_discard, slot.prompt.tokens.pos_next(), -n_discard);
|
||||
}
|
||||
slot.mem.seq_rm (slot.id, n_keep , n_keep + n_discard);
|
||||
slot.mem.seq_add(slot.id, n_keep + n_discard, slot.prompt.tokens.pos_next(), -n_discard);
|
||||
|
||||
// add generated tokens to cache
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/16818#discussion_r2473269481
|
||||
@@ -2998,7 +2993,9 @@ private:
|
||||
ckpt.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
|
||||
}
|
||||
|
||||
common_context_seq_rm(ctx_dft, slot.id, ckpt.pos_max + 1, -1);
|
||||
if (!llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, ckpt.pos_max + 1, -1)) {
|
||||
GGML_ABORT("failed to remove sequence %d\n", slot.id);
|
||||
}
|
||||
}
|
||||
|
||||
if (!draft.empty()) {
|
||||
@@ -3201,13 +3198,8 @@ private:
|
||||
|
||||
const int64_t kv_shift = (int64_t) head_p - (int64_t) head_c;
|
||||
|
||||
common_context_seq_rm (ctx_tgt, slot.id, head_p, head_c);
|
||||
common_context_seq_add(ctx_tgt, slot.id, head_c, head_c + n_match, kv_shift);
|
||||
|
||||
if (ctx_dft) {
|
||||
common_context_seq_rm (ctx_dft, slot.id, head_p, head_c);
|
||||
common_context_seq_add(ctx_dft, slot.id, head_c, head_c + n_match, kv_shift);
|
||||
}
|
||||
slot.mem.seq_rm (slot.id, head_p, head_c);
|
||||
slot.mem.seq_add(slot.id, head_c, head_c + n_match, kv_shift);
|
||||
|
||||
for (size_t i = 0; i < n_match; i++) {
|
||||
slot.prompt.tokens.set_token(head_p + i, slot.prompt.tokens[head_c + i]);
|
||||
@@ -3379,10 +3371,7 @@ private:
|
||||
|
||||
SLT_TRC(slot, "cached n_tokens = %d, memory_seq_rm [%d, end)\n", slot.prompt.n_tokens(), p0);
|
||||
|
||||
common_context_seq_rm(ctx_tgt, slot.id, p0, -1);
|
||||
if (ctx_dft) {
|
||||
common_context_seq_rm(ctx_dft, slot.id, p0, -1);
|
||||
}
|
||||
slot.mem.seq_rm(slot.id, p0, -1);
|
||||
|
||||
// If using an alora, there may be uncached tokens that come
|
||||
// before the invocation sequence. When this happens, the
|
||||
@@ -3837,18 +3826,14 @@ private:
|
||||
|
||||
SLT_DBG(slot, "restoring speculative checkpoint (pos_min = %d, pos_max = %d, size = %zu)\n", ckpt.pos_min, ckpt.pos_max, ckpt.size());
|
||||
|
||||
{
|
||||
ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
|
||||
|
||||
common_context_seq_rm(slot.ctx_tgt, slot.id, ckpt.pos_max + 1, -1);
|
||||
}
|
||||
ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
|
||||
|
||||
if (slot.ctx_dft) {
|
||||
ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
|
||||
|
||||
common_context_seq_rm(slot.ctx_dft, slot.id, ckpt.pos_max + 1, -1);
|
||||
}
|
||||
|
||||
slot.mem.seq_rm(slot.id, ckpt.pos_max + 1, -1);
|
||||
|
||||
slot.prompt.tokens.keep_first(ckpt.n_tokens);
|
||||
slot.smpl = std::move(smpl_save);
|
||||
|
||||
@@ -3889,10 +3874,7 @@ private:
|
||||
slot.sampled = ids.back(); // last accepted token
|
||||
SLT_DBG(slot, "add accepted tokens: sampled=%d, ids.size=%zu, n_draft=%zu\n", slot.sampled, ids.size(), n_draft);
|
||||
|
||||
common_context_seq_rm(slot.ctx_tgt, slot.id, slot.prompt.tokens.pos_next(), -1);
|
||||
if (slot.ctx_dft) {
|
||||
common_context_seq_rm(slot.ctx_dft, slot.id, slot.prompt.tokens.pos_next(), -1);
|
||||
}
|
||||
slot.mem.seq_rm(slot.id, slot.prompt.tokens.pos_next(), -1);
|
||||
|
||||
for (size_t i = 0; i < ids.size(); ++i) {
|
||||
completion_token_output result;
|
||||
|
||||
@@ -209,6 +209,7 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
|
||||
->set_hard_limits(0.0f, 1.0f)
|
||||
->set_desc("Minimum speculative decoding probability for draft tokens (0 = greedy)"));
|
||||
|
||||
|
||||
add((new field_str("speculative.type"))
|
||||
->set_desc("Speculative decoding method (for debugging and research purposes)")
|
||||
->set_handler([&](field_eval_context & ctx, const json & data) {
|
||||
|
||||
@@ -1742,9 +1742,9 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
|
||||
const int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
|
||||
|
||||
float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
|
||||
float sim_best = float(lcp_best) / tokens_new.size();
|
||||
float f_sim_best = float(lcp_best) / tokens_new.size();
|
||||
|
||||
SRV_TRC(" - looking for better prompt, base f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);
|
||||
SRV_TRC(" - looking for better prompt, base f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
|
||||
|
||||
auto it_best = states.end();
|
||||
|
||||
@@ -1753,23 +1753,25 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
|
||||
const int lcp_cur = it->prompt.tokens.get_common_prefix(tokens_new);
|
||||
|
||||
const float f_keep_cur = float(lcp_cur) / it->prompt.tokens.size();
|
||||
const float sim_cur = float(lcp_cur) / tokens_new.size();
|
||||
const float f_sim_cur = float(lcp_cur) / tokens_new.size();
|
||||
|
||||
SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur);
|
||||
|
||||
// don't trash large prompts
|
||||
if (f_keep_cur < 0.25f) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (f_keep_best < f_keep_cur && sim_best < sim_cur) {
|
||||
if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) {
|
||||
f_keep_best = f_keep_cur;
|
||||
sim_best = sim_cur;
|
||||
f_sim_best = f_sim_cur;
|
||||
|
||||
it_best = it;
|
||||
}
|
||||
}
|
||||
|
||||
if (it_best != states.end()) {
|
||||
SRV_TRC(" - found better prompt with f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);
|
||||
SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
|
||||
|
||||
{
|
||||
auto & data = it_best->data.main;
|
||||
|
||||
@@ -650,7 +650,7 @@ struct server_prompt_cache {
|
||||
|
||||
server_prompt_cache_state * alloc(const server_prompt & prompt, size_t state_size_main, size_t state_size_drft);
|
||||
|
||||
bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_main, llama_context * ctx_drft, int32_t id_slot);
|
||||
bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot);
|
||||
|
||||
void update();
|
||||
};
|
||||
|
||||
+3
-5
@@ -11,8 +11,7 @@
|
||||
classifyToolResult,
|
||||
formatJsonPretty,
|
||||
parseToolResultWithImages,
|
||||
type AgenticSection,
|
||||
type ToolResultLine
|
||||
type AgenticSection
|
||||
} from '$lib/utils';
|
||||
import { getBuiltinToolUi } from '$lib/constants/built-in-tools';
|
||||
import type { DatabaseMessageExtra } from '$lib/types';
|
||||
@@ -29,11 +28,10 @@
|
||||
let { section, open, isStreaming, attachments, onToggle }: Props = $props();
|
||||
|
||||
const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? '');
|
||||
|
||||
const parsedLines: ToolResultLine[] = $derived(
|
||||
const outputKind = $derived(classifyToolResult(section.toolResult));
|
||||
const parsedLines = $derived(
|
||||
section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : []
|
||||
);
|
||||
const outputKind = $derived(classifyToolResult(section.toolResult));
|
||||
</script>
|
||||
|
||||
<ToolCallBlock {section} {open} {isStreaming} meta={null} {title} {onToggle}>
|
||||
|
||||
-1
@@ -15,7 +15,6 @@
|
||||
let { section, open, isStreaming, onToggle }: Props = $props();
|
||||
|
||||
const editFileMeta = $derived(parseEditFileMeta(section));
|
||||
|
||||
const editDiffs = $derived(
|
||||
(editFileMeta?.edits ?? []).map((edit) => computeLineDiff(edit.oldText, edit.newText))
|
||||
);
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@
|
||||
const isStreamingCall = $derived(section.type === AgenticSectionType.TOOL_CALL_STREAMING);
|
||||
const showSpinner = $derived(isPending || (isStreamingCall && isStreaming));
|
||||
|
||||
const results: SearchResult[] = $derived(extractSearchResults(section.toolResult));
|
||||
const results = $derived(extractSearchResults(section.toolResult));
|
||||
const query = $derived(extractSearchQuery(section.toolArgs));
|
||||
|
||||
// Same icon-resolution chain as ChatMessageToolCallBlockDefault so
|
||||
|
||||
@@ -28,6 +28,18 @@ export const LATEX_MATH_AND_CODE_PATTERN =
|
||||
/** Regex to capture the content of a $$...\\\\...$$ block (display-formula with line-break) */
|
||||
export const LATEX_LINEBREAK_REGEXP = /\$\$([\s\S]*?\\\\[\s\S]*?)\$\$/;
|
||||
|
||||
/**
|
||||
* Matches the unescaped `\[...\]` display-math delimiter and surrounding
|
||||
* context so callers can insert line-breaks around the placeholder or convert
|
||||
* to inline when the formula has a non-empty trailing context (e.g. a table
|
||||
* cell that opens with `\[` and closes with content after `\]`).
|
||||
*
|
||||
* group 1: prefix before `\[`
|
||||
* group 2: formula body
|
||||
* group 3: trailing context after `\]`
|
||||
*/
|
||||
export const LATEX_DISPLAY_BLOCK_REGEXP = /([\S].*?)\\\[([\s\S]*?)\\\](.*)/g;
|
||||
|
||||
/**
|
||||
* Cheap gate for `preprocessLaTeX`. Every transformation it performs is triggered
|
||||
* by a `$` (inline/display math, currency escaping) or a backslash escape
|
||||
@@ -36,6 +48,76 @@ export const LATEX_LINEBREAK_REGEXP = /\$\$([\s\S]*?\\\\[\s\S]*?)\$\$/;
|
||||
*/
|
||||
export const LATEX_TRIGGER_REGEXP = /[$\\]/;
|
||||
|
||||
/** Inline LaTeX math delimiter (the dollar sign). */
|
||||
export const LATEX_INLINE_DELIMITER = '$';
|
||||
|
||||
/** Display LaTeX math delimiter (paired dollar signs). */
|
||||
export const LATEX_DISPLAY_DELIMITER = '$$';
|
||||
|
||||
/** Matches a single non-whitespace character. */
|
||||
export const LATEX_NON_WHITESPACE_REGEXP = /\S/;
|
||||
|
||||
/** Matches a character that may appear adjacent to `$`, indicating a non-TeX
|
||||
* context such as an identifier (`var$`, `$var`), currency ($5), or code. */
|
||||
export const LATEX_NEIGHBOR_CHAR_REGEXP = /[A-Za-z0-9_$-]/;
|
||||
|
||||
/** Matches a single digit (used to detect currency-like `$5`). */
|
||||
export const LATEX_DIGIT_REGEXP = /[0-9]/;
|
||||
|
||||
/** Matches the leading blockquote prefix (`> ` or `>`) on a markdown line. */
|
||||
export const LATEX_BLOCKQUOTE_PREFIX_REGEXP = /^(>\s*)/;
|
||||
|
||||
/** Matches the placeholder inserted by the protect/restore pipeline for a
|
||||
* protected LaTeX expression. Group 1 is the index into `latexExpressions`. */
|
||||
export const LATEX_PLACEHOLDER_REGEXP = /<<LATEX_(\d+)>>/g;
|
||||
|
||||
/** Matches the placeholder inserted by the protect/restore pipeline for a
|
||||
* protected code block. Group 1 is the index into `codeBlocks`. */
|
||||
export const CODE_BLOCK_PLACEHOLDER_REGEXP = /<<CODE_BLOCK_(\d+)>>/g;
|
||||
|
||||
/** Matches a `$` immediately followed by a digit, which is treated as a
|
||||
* currency amount (e.g. `$5`) and escaped to `\$5` so it isn't parsed as math. */
|
||||
export const LATEX_CURRENCY_DOLLAR_REGEXP = /\$(?=\d)/g;
|
||||
|
||||
/** Captures remaining `$$...$$`, `\[...\]`, `\(...\)` (only unescaped via
|
||||
* `(?<!\\)`) after the display-block pass has run. Group 1 holds the
|
||||
* matched formula. */
|
||||
export const LATEX_PROTECT_REGEXP =
|
||||
/(\$\$[\s\S]*?\$\$|(?<!\\)\\\[[\s\S]*?\\\]|(?<!\\)\\\(.*?\\\))/g;
|
||||
|
||||
/** Matches unescaped inline `\(...\)` (at least one char inside) used to
|
||||
* convert `\(` → `$` after the protect pass. */
|
||||
export const LATEX_INLINE_CONVERT_REGEXP = /(?<!\\)\\\((.+?)\\\)/g;
|
||||
|
||||
/** Matches unescaped display `\[...\]` used to convert `\[` → `$$`
|
||||
* after the protect pass. */
|
||||
export const LATEX_DISPLAY_CONVERT_REGEXP = /(?<!\\)\\\[([\s\S]*?)\\\]/g;
|
||||
|
||||
/** `\(` — opens an inline LaTeX math block. */
|
||||
export const LATEX_INLINE_OPEN = '\\(';
|
||||
|
||||
/** `\)` — closes an inline LaTeX math block. */
|
||||
export const LATEX_INLINE_CLOSE = '\\)';
|
||||
|
||||
/** `\[` — opens a display LaTeX math block. */
|
||||
export const LATEX_DISPLAY_OPEN = '\\[';
|
||||
|
||||
/** `\]` — closes a display LaTeX math block. */
|
||||
export const LATEX_DISPLAY_CLOSE = '\\]';
|
||||
|
||||
/** `\` — the LaTeX escape character. */
|
||||
export const LATEX_BACKSLASH = '\\';
|
||||
|
||||
/** `\$` — dollar sign escaped so it isn't parsed as math (used to disambiguate
|
||||
* currency amounts like `$5`). */
|
||||
export const LATEX_CURRENCY_ESCAPE = '\\$';
|
||||
|
||||
/** `\ce{` — mhchem chemistry command prefix. */
|
||||
export const LATEX_MHCHEM_CE = '\\ce{';
|
||||
|
||||
/** `\pu{` — mhchem physics-unit command prefix. */
|
||||
export const LATEX_MHCHEM_PU = '\\pu{';
|
||||
|
||||
/** map from mchem-regexp to replacement */
|
||||
export const MHCHEM_PATTERN_MAP: readonly [RegExp, string][] = [
|
||||
[/(\s)\$\\ce{/g, '$1$\\\\ce{'],
|
||||
|
||||
@@ -92,8 +92,17 @@ function deriveSingleTurnSections(
|
||||
|
||||
// 3. Persisted tool calls (from message.toolCalls field)
|
||||
const toolCalls = parseToolCalls(message.toolCalls);
|
||||
|
||||
// Index tool messages by toolCallId for O(1) lookup instead of O(n) find()
|
||||
const toolMsgById = new Map<string, DatabaseMessage>();
|
||||
for (const tm of toolMessages) {
|
||||
if (tm.toolCallId && !toolMsgById.has(tm.toolCallId)) {
|
||||
toolMsgById.set(tm.toolCallId, tm);
|
||||
}
|
||||
}
|
||||
|
||||
for (const tc of toolCalls) {
|
||||
const resultMsg = toolMessages.find((m) => m.toolCallId === tc.id);
|
||||
const resultMsg = tc.id ? toolMsgById.get(tc.id) : undefined;
|
||||
// Only show as pending/loading if we're actively streaming; otherwise it's just a tool call without result
|
||||
const type = resultMsg
|
||||
? AgenticSectionType.TOOL_CALL
|
||||
@@ -112,9 +121,10 @@ function deriveSingleTurnSections(
|
||||
}
|
||||
|
||||
// 4. Streaming tool calls (not yet persisted - currently being received)
|
||||
const persistedIds = new Set(toolCalls.map((t) => t.id).filter(Boolean));
|
||||
for (const tc of streamingToolCalls) {
|
||||
// Skip if already in persisted tool calls
|
||||
if (tc.id && toolCalls.find((t) => t.id === tc.id)) continue;
|
||||
if (tc.id && persistedIds.has(tc.id)) continue;
|
||||
sections.push({
|
||||
type: AgenticSectionType.TOOL_CALL_STREAMING,
|
||||
content: '',
|
||||
@@ -281,15 +291,31 @@ export function splitSearchSummaryList(
|
||||
return { lines };
|
||||
}
|
||||
|
||||
/** Bounded cache for parseToolResultWithImages results. */
|
||||
const TOOL_RESULT_LINES_CACHE_MAX_SIZE = 32;
|
||||
const toolResultLinesCache = new Map<string, ToolResultLine[]>();
|
||||
|
||||
/**
|
||||
* Parse tool result text into lines, matching image attachments by name.
|
||||
* Memoized: called per render during streaming on unchanged tool result
|
||||
* strings with unchanged extras.
|
||||
*/
|
||||
export function parseToolResultWithImages(
|
||||
toolResult: string,
|
||||
extras?: DatabaseMessageExtra[]
|
||||
): ToolResultLine[] {
|
||||
// Cache key includes image attachment names so we recompute when
|
||||
// attachments change, even if the count stays the same.
|
||||
const imageNames = (extras ?? [])
|
||||
.filter((e): e is DatabaseMessageExtraImageFile => e.type === AttachmentType.IMAGE)
|
||||
.map((e) => e.name)
|
||||
.join(NEWLINE);
|
||||
const cacheKey = `${imageNames}:${toolResult}`;
|
||||
const cached = toolResultLinesCache.get(cacheKey);
|
||||
if (cached !== undefined) return cached;
|
||||
|
||||
const lines = toolResult.split(NEWLINE);
|
||||
return lines.map((line) => {
|
||||
const result = lines.map((line) => {
|
||||
const match = line.match(ATTACHMENT_SAVED_REGEX);
|
||||
if (!match || !extras) return { text: line };
|
||||
|
||||
@@ -301,8 +327,19 @@ export function parseToolResultWithImages(
|
||||
|
||||
return { text: line, image };
|
||||
});
|
||||
|
||||
if (toolResultLinesCache.size >= TOOL_RESULT_LINES_CACHE_MAX_SIZE) {
|
||||
toolResultLinesCache.delete(toolResultLinesCache.keys().next().value!);
|
||||
}
|
||||
toolResultLinesCache.set(cacheKey, result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Bounded cache for classifyToolResult results. */
|
||||
const CLASSIFY_CACHE_MAX_SIZE = 32;
|
||||
const classifyCache = new Map<string, ToolResultKind>();
|
||||
|
||||
/**
|
||||
* Pick a renderer tier for a tool's result content.
|
||||
*
|
||||
@@ -312,25 +349,39 @@ export function parseToolResultWithImages(
|
||||
* through MarkdownContent for proper formatting.
|
||||
* text - everything else, rendered as plain text lines (with image
|
||||
* attachment resolution as a side effect).
|
||||
* Memoized: called per render during streaming on unchanged content.
|
||||
*/
|
||||
export function classifyToolResult(content: string | undefined): ToolResultKind {
|
||||
if (!content) return ToolResultKind.TEXT;
|
||||
|
||||
const cached = classifyCache.get(content);
|
||||
if (cached !== undefined) return cached;
|
||||
|
||||
const trimmed = content.trim();
|
||||
if (!trimmed) return ToolResultKind.TEXT;
|
||||
|
||||
let result: ToolResultKind = ToolResultKind.TEXT;
|
||||
|
||||
// Strongest signal: JSON object/array round-trips through JSON.parse.
|
||||
if (TOOL_RESULT_JSON_OPEN_REGEX.test(trimmed)) {
|
||||
try {
|
||||
JSON.parse(trimmed);
|
||||
return ToolResultKind.JSON;
|
||||
result = ToolResultKind.JSON;
|
||||
} catch (error) {
|
||||
console.error('[agentic] tool result looked like JSON but failed to parse:', error);
|
||||
}
|
||||
}
|
||||
|
||||
if (looksLikeMarkdown(trimmed)) return ToolResultKind.MARKDOWN;
|
||||
if (result === ToolResultKind.TEXT && looksLikeMarkdown(trimmed)) {
|
||||
result = ToolResultKind.MARKDOWN;
|
||||
}
|
||||
|
||||
return ToolResultKind.TEXT;
|
||||
if (classifyCache.size >= CLASSIFY_CACHE_MAX_SIZE) {
|
||||
classifyCache.delete(classifyCache.keys().next().value!);
|
||||
}
|
||||
classifyCache.set(content, result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -370,19 +421,35 @@ function looksLikeMarkdown(content: string): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Bounded cache for parsed tool-call JSON blobs. */
|
||||
const TOOL_CALLS_CACHE_MAX_SIZE = 64;
|
||||
const toolCallsParseCache = new Map<string, ApiChatCompletionToolCall[]>();
|
||||
|
||||
/**
|
||||
* Safely parse the toolCalls JSON string from a DatabaseMessage.
|
||||
* Memoized: the same JSON string is re-parsed on every render during
|
||||
* streaming, which is wasted CPU since tool calls don't change mid-stream.
|
||||
*/
|
||||
function parseToolCalls(toolCallsJson?: string): ApiChatCompletionToolCall[] {
|
||||
if (!toolCallsJson) return [];
|
||||
|
||||
const cached = toolCallsParseCache.get(toolCallsJson);
|
||||
if (cached) return cached;
|
||||
|
||||
let result: ApiChatCompletionToolCall[];
|
||||
try {
|
||||
const parsed = JSON.parse(toolCallsJson);
|
||||
|
||||
return Array.isArray(parsed) ? parsed : [];
|
||||
result = Array.isArray(parsed) ? parsed : [];
|
||||
} catch {
|
||||
return [];
|
||||
result = [];
|
||||
}
|
||||
|
||||
if (toolCallsParseCache.size >= TOOL_CALLS_CACHE_MAX_SIZE) {
|
||||
toolCallsParseCache.delete(toolCallsParseCache.keys().next().value!);
|
||||
}
|
||||
toolCallsParseCache.set(toolCallsJson, result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -34,6 +34,10 @@ function escapeCode(code: string): string {
|
||||
return code.replace(AMPERSAND_REGEX, '&').replace(LT_REGEX, '<').replace(GT_REGEX, '>');
|
||||
}
|
||||
|
||||
/** Bounded cache for highlightCode results. */
|
||||
const HIGHLIGHT_CACHE_MAX_SIZE = 64;
|
||||
const highlightCache = new Map<string, string>();
|
||||
|
||||
/**
|
||||
* Highlights code using highlight.js
|
||||
* @param code - The code to highlight
|
||||
@@ -47,23 +51,37 @@ function escapeCode(code: string): string {
|
||||
export function highlightCode(code: string, language: string, autoDetect = true): string {
|
||||
if (!code) return '';
|
||||
|
||||
// Cache key includes language and autoDetect flag since results differ.
|
||||
// During streaming, the same code string may be highlighted repeatedly
|
||||
// (e.g., when text after a code block changes but the code itself doesn't).
|
||||
const cacheKey = `${language}:${autoDetect}:${code}`;
|
||||
const cached = highlightCache.get(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
const trimmed = trimCodePadding(code);
|
||||
let result: string;
|
||||
|
||||
try {
|
||||
const lang = language.toLowerCase();
|
||||
const isSupported = hljs.getLanguage(lang);
|
||||
|
||||
if (isSupported) {
|
||||
return hljs.highlight(trimmed, { language: lang }).value;
|
||||
result = hljs.highlight(trimmed, { language: lang }).value;
|
||||
} else if (autoDetect) {
|
||||
return hljs.highlightAuto(trimmed).value;
|
||||
result = hljs.highlightAuto(trimmed).value;
|
||||
} else {
|
||||
return escapeCode(trimmed);
|
||||
result = escapeCode(trimmed);
|
||||
}
|
||||
} catch {
|
||||
// Fallback to escaped plain text
|
||||
return escapeCode(trimmed);
|
||||
result = escapeCode(trimmed);
|
||||
}
|
||||
|
||||
if (highlightCache.size >= HIGHLIGHT_CACHE_MAX_SIZE) {
|
||||
highlightCache.delete(highlightCache.keys().next().value!);
|
||||
}
|
||||
highlightCache.set(cacheKey, result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export { trimCodePadding };
|
||||
|
||||
@@ -1,9 +1,31 @@
|
||||
import {
|
||||
CODE_BLOCK_PLACEHOLDER_REGEXP,
|
||||
CODE_BLOCK_REGEXP,
|
||||
LATEX_BACKSLASH,
|
||||
LATEX_BLOCKQUOTE_PREFIX_REGEXP,
|
||||
LATEX_CURRENCY_DOLLAR_REGEXP,
|
||||
LATEX_CURRENCY_ESCAPE,
|
||||
LATEX_DIGIT_REGEXP,
|
||||
LATEX_DISPLAY_BLOCK_REGEXP,
|
||||
LATEX_DISPLAY_CLOSE,
|
||||
LATEX_DISPLAY_CONVERT_REGEXP,
|
||||
LATEX_DISPLAY_DELIMITER,
|
||||
LATEX_DISPLAY_OPEN,
|
||||
LATEX_INLINE_CLOSE,
|
||||
LATEX_INLINE_CONVERT_REGEXP,
|
||||
LATEX_INLINE_DELIMITER,
|
||||
LATEX_INLINE_OPEN,
|
||||
LATEX_MATH_AND_CODE_PATTERN,
|
||||
LATEX_MHCHEM_CE,
|
||||
LATEX_MHCHEM_PU,
|
||||
LATEX_LINEBREAK_REGEXP,
|
||||
LATEX_NEIGHBOR_CHAR_REGEXP,
|
||||
LATEX_NON_WHITESPACE_REGEXP,
|
||||
LATEX_PLACEHOLDER_REGEXP,
|
||||
LATEX_PROTECT_REGEXP,
|
||||
LATEX_TRIGGER_REGEXP,
|
||||
MHCHEM_PATTERN_MAP
|
||||
MHCHEM_PATTERN_MAP,
|
||||
NEWLINE
|
||||
} from '$lib/constants';
|
||||
|
||||
/**
|
||||
@@ -20,13 +42,13 @@ import {
|
||||
* @returns The processed string with LaTeX replaced by placeholders.
|
||||
*/
|
||||
export function maskInlineLaTeX(content: string, latexExpressions: string[]): string {
|
||||
if (!content.includes('$')) {
|
||||
if (!content.includes(LATEX_INLINE_DELIMITER)) {
|
||||
return content;
|
||||
}
|
||||
return content
|
||||
.split('\n')
|
||||
.split(NEWLINE)
|
||||
.map((line) => {
|
||||
if (line.indexOf('$') == -1) {
|
||||
if (line.indexOf(LATEX_INLINE_DELIMITER) == -1) {
|
||||
return line;
|
||||
}
|
||||
|
||||
@@ -34,7 +56,7 @@ export function maskInlineLaTeX(content: string, latexExpressions: string[]): st
|
||||
let currentPosition = 0;
|
||||
|
||||
while (currentPosition < line.length) {
|
||||
const openDollarIndex = line.indexOf('$', currentPosition);
|
||||
const openDollarIndex = line.indexOf(LATEX_INLINE_DELIMITER, currentPosition);
|
||||
|
||||
if (openDollarIndex == -1) {
|
||||
processedLine += line.slice(currentPosition);
|
||||
@@ -42,7 +64,7 @@ export function maskInlineLaTeX(content: string, latexExpressions: string[]): st
|
||||
}
|
||||
|
||||
// Is there a next $-sign?
|
||||
const closeDollarIndex = line.indexOf('$', openDollarIndex + 1);
|
||||
const closeDollarIndex = line.indexOf(LATEX_INLINE_DELIMITER, openDollarIndex + 1);
|
||||
|
||||
if (closeDollarIndex == -1) {
|
||||
processedLine += line.slice(currentPosition);
|
||||
@@ -62,14 +84,14 @@ export function maskInlineLaTeX(content: string, latexExpressions: string[]): st
|
||||
shouldSkipAsNonLatex = true;
|
||||
}
|
||||
|
||||
if (/[A-Za-z0-9_$-]/.test(charBeforeOpen)) {
|
||||
if (LATEX_NEIGHBOR_CHAR_REGEXP.test(charBeforeOpen)) {
|
||||
// Character, digit, $, _ or - before first '$', no TeX.
|
||||
shouldSkipAsNonLatex = true;
|
||||
}
|
||||
|
||||
if (
|
||||
/[0-9]/.test(charAfterOpen) &&
|
||||
(/[A-Za-z0-9_$-]/.test(charAfterClose) || ' ' == charBeforeClose)
|
||||
LATEX_DIGIT_REGEXP.test(charAfterOpen) &&
|
||||
(LATEX_NEIGHBOR_CHAR_REGEXP.test(charAfterClose) || ' ' == charBeforeClose)
|
||||
) {
|
||||
// First $ seems to belong to an amount.
|
||||
shouldSkipAsNonLatex = true;
|
||||
@@ -92,7 +114,7 @@ export function maskInlineLaTeX(content: string, latexExpressions: string[]): st
|
||||
|
||||
return processedLine;
|
||||
})
|
||||
.join('\n');
|
||||
.join(NEWLINE);
|
||||
}
|
||||
|
||||
function escapeBrackets(text: string): string {
|
||||
@@ -107,9 +129,9 @@ function escapeBrackets(text: string): string {
|
||||
if (codeBlock != null) {
|
||||
return codeBlock;
|
||||
} else if (squareBracket != null) {
|
||||
return `$$${squareBracket}$$`;
|
||||
return `${LATEX_DISPLAY_DELIMITER}${squareBracket}${LATEX_DISPLAY_DELIMITER}`;
|
||||
} else if (roundBracket != null) {
|
||||
return `$${roundBracket}$`;
|
||||
return `${LATEX_INLINE_DELIMITER}${roundBracket}${LATEX_INLINE_DELIMITER}`;
|
||||
}
|
||||
|
||||
return match;
|
||||
@@ -145,32 +167,49 @@ const doEscapeMhchem = false;
|
||||
* preprocessLaTeX("Price: $10. The equation is \\(x^2\\).")
|
||||
* // → "Price: $10. The equation is $x^2$."
|
||||
*/
|
||||
/** Bounded cache for preprocessLaTeX results. */
|
||||
const LATEX_CACHE_MAX_SIZE = 64;
|
||||
const latexCache = new Map<string, string>();
|
||||
|
||||
export function preprocessLaTeX(content: string): string {
|
||||
// See also:
|
||||
// https://github.com/danny-avila/LibreChat/blob/main/client/src/utils/latex.ts
|
||||
|
||||
// Memoize on the input string. During streaming the prefix before an
|
||||
// incomplete code block stays the same across multiple tokens, so the
|
||||
// full protect/restore pipeline would re-run unnecessarily.
|
||||
const cached = latexCache.get(content);
|
||||
if (cached !== undefined) return cached;
|
||||
|
||||
// Save original before the function mutates `content` through steps 0-8
|
||||
const originalContent = content;
|
||||
|
||||
// Every step below keys off a `$` or a backslash escape (\[ \] \( \) \ce{ \pu{).
|
||||
// With neither present the protect/restore passes round-trip the input
|
||||
// unchanged, so skip them: the step 2 scan is O(n^2) in line length and costs
|
||||
// ~90ms on a 26KB single-line message that contains no math at all. This
|
||||
// matters during streaming, where the whole message is reprocessed per frame.
|
||||
if (!LATEX_TRIGGER_REGEXP.test(content)) {
|
||||
if (latexCache.size >= LATEX_CACHE_MAX_SIZE) {
|
||||
latexCache.delete(latexCache.keys().next().value!);
|
||||
}
|
||||
latexCache.set(originalContent, content);
|
||||
return content;
|
||||
}
|
||||
|
||||
// Step 0: Temporarily remove blockquote markers (>) to process LaTeX correctly
|
||||
// Store the structure so we can restore it later
|
||||
const blockquoteMarkers: Map<number, string> = new Map();
|
||||
const lines = content.split('\n');
|
||||
const lines = content.split(NEWLINE);
|
||||
const processedLines = lines.map((line, index) => {
|
||||
const match = line.match(/^(>\s*)/);
|
||||
const match = line.match(LATEX_BLOCKQUOTE_PREFIX_REGEXP);
|
||||
if (match) {
|
||||
blockquoteMarkers.set(index, match[1]);
|
||||
return line.slice(match[1].length);
|
||||
}
|
||||
return line;
|
||||
});
|
||||
content = processedLines.join('\n');
|
||||
content = processedLines.join(NEWLINE);
|
||||
|
||||
// Step 1: Protect code blocks
|
||||
const codeBlocks: string[] = [];
|
||||
@@ -187,58 +226,52 @@ export function preprocessLaTeX(content: string): string {
|
||||
// Match \S...\[...\] and protect them and insert a line-break.
|
||||
// Guarded: with no `\[` present this pattern still probes every start offset,
|
||||
// expanding `.*?` to the end of each line before failing - O(n^2) for nothing.
|
||||
if (content.includes('\\[')) {
|
||||
content = content.replace(
|
||||
/([\S].*?)\\\[([\s\S]*?)\\\](.*)/g,
|
||||
(match, group1, group2, group3) => {
|
||||
// Check if there are characters following the formula (display-formula in a table-cell?)
|
||||
if (group1.endsWith('\\')) {
|
||||
return match; // Backslash before \[, do nothing.
|
||||
}
|
||||
const hasSuffix = /\S/.test(group3);
|
||||
let optBreak;
|
||||
|
||||
if (hasSuffix) {
|
||||
latexExpressions.push(`\\(${group2.trim()}\\)`); // Convert into inline.
|
||||
optBreak = '';
|
||||
} else {
|
||||
latexExpressions.push(`\\[${group2}\\]`);
|
||||
optBreak = '\n';
|
||||
}
|
||||
|
||||
return `${group1}${optBreak}<<LATEX_${latexExpressions.length - 1}>>${optBreak}${group3}`;
|
||||
if (content.includes(LATEX_DISPLAY_OPEN)) {
|
||||
content = content.replace(LATEX_DISPLAY_BLOCK_REGEXP, (match, group1, group2, group3) => {
|
||||
// Check if there are characters following the formula (display-formula in a table-cell?)
|
||||
if (group1.endsWith(LATEX_BACKSLASH)) {
|
||||
return match; // Backslash before \[, do nothing.
|
||||
}
|
||||
);
|
||||
const hasSuffix = LATEX_NON_WHITESPACE_REGEXP.test(group3);
|
||||
let optBreak;
|
||||
|
||||
if (hasSuffix) {
|
||||
latexExpressions.push(`${LATEX_INLINE_OPEN}${group2.trim()}${LATEX_INLINE_CLOSE}`); // Convert into inline.
|
||||
optBreak = '';
|
||||
} else {
|
||||
latexExpressions.push(`${LATEX_DISPLAY_OPEN}${group2}${LATEX_DISPLAY_CLOSE}`);
|
||||
optBreak = NEWLINE;
|
||||
}
|
||||
|
||||
return `${group1}${optBreak}<<LATEX_${latexExpressions.length - 1}>>${optBreak}${group3}`;
|
||||
});
|
||||
}
|
||||
|
||||
// Match \(...\), \[...\], $$...$$ and protect them
|
||||
content = content.replace(
|
||||
/(\$\$[\s\S]*?\$\$|(?<!\\)\\\[[\s\S]*?\\\]|(?<!\\)\\\(.*?\\\))/g,
|
||||
(match) => {
|
||||
latexExpressions.push(match);
|
||||
content = content.replace(LATEX_PROTECT_REGEXP, (match) => {
|
||||
latexExpressions.push(match);
|
||||
|
||||
return `<<LATEX_${latexExpressions.length - 1}>>`;
|
||||
}
|
||||
);
|
||||
return `<<LATEX_${latexExpressions.length - 1}>>`;
|
||||
});
|
||||
|
||||
// Protect inline $...$ but NOT if it looks like money (e.g., $10, $3.99)
|
||||
content = maskInlineLaTeX(content, latexExpressions);
|
||||
|
||||
// Step 3: Escape standalone $ before digits (currency like $5 → \$5)
|
||||
// (Now that inline math is protected, this will only escape dollars not already protected)
|
||||
content = content.replace(/\$(?=\d)/g, '\\$');
|
||||
content = content.replace(LATEX_CURRENCY_DOLLAR_REGEXP, LATEX_CURRENCY_ESCAPE);
|
||||
|
||||
// Step 4: Restore protected LaTeX expressions (they are valid)
|
||||
content = content.replace(/<<LATEX_(\d+)>>/g, (_, index) => {
|
||||
content = content.replace(LATEX_PLACEHOLDER_REGEXP, (_, index) => {
|
||||
let expr = latexExpressions[parseInt(index)];
|
||||
const match = expr.match(LATEX_LINEBREAK_REGEXP);
|
||||
if (match) {
|
||||
// Katex: The $$-delimiters should be in their own line
|
||||
// if there are \\-line-breaks.
|
||||
const formula = match[1];
|
||||
const prefix = formula.startsWith('\n') ? '' : '\n';
|
||||
const suffix = formula.endsWith('\n') ? '' : '\n';
|
||||
expr = '$$' + prefix + formula + suffix + '$$';
|
||||
const prefix = formula.startsWith(NEWLINE) ? '' : NEWLINE;
|
||||
const suffix = formula.endsWith(NEWLINE) ? '' : NEWLINE;
|
||||
expr = LATEX_DISPLAY_DELIMITER + prefix + formula + suffix + LATEX_DISPLAY_DELIMITER;
|
||||
}
|
||||
return expr;
|
||||
});
|
||||
@@ -247,7 +280,7 @@ export function preprocessLaTeX(content: string): string {
|
||||
// This must happen BEFORE restoring code blocks to avoid affecting code content
|
||||
content = escapeBrackets(content);
|
||||
|
||||
if (doEscapeMhchem && (content.includes('\\ce{') || content.includes('\\pu{'))) {
|
||||
if (doEscapeMhchem && (content.includes(LATEX_MHCHEM_CE) || content.includes(LATEX_MHCHEM_PU))) {
|
||||
content = escapeMhchem(content);
|
||||
}
|
||||
|
||||
@@ -257,31 +290,38 @@ export function preprocessLaTeX(content: string): string {
|
||||
// Using the look‑behind pattern `(?<!\\)` we skip matches
|
||||
// that are preceded by a backslash, e.g.
|
||||
// `Definitions\\(also called macros)` (title of chapter 20 in The TeXbook).
|
||||
.replace(/(?<!\\)\\\((.+?)\\\)/g, '$$$1$') // inline
|
||||
.replace(LATEX_INLINE_CONVERT_REGEXP, (_, formula: string) => {
|
||||
return `${LATEX_INLINE_DELIMITER}${formula}${LATEX_INLINE_DELIMITER}`;
|
||||
}) // inline
|
||||
.replace(
|
||||
// Using the look‑behind pattern `(?<!\\)` we skip matches
|
||||
// that are preceded by a backslash, e.g. `\\[4pt]`.
|
||||
/(?<!\\)\\\[([\s\S]*?)\\\]/g, // display, see also PR #16599
|
||||
(_, content: string) => {
|
||||
return `$$${content}$$`;
|
||||
LATEX_DISPLAY_CONVERT_REGEXP, // display, see also PR #16599
|
||||
(_, formula: string) => {
|
||||
return `${LATEX_DISPLAY_DELIMITER}${formula}${LATEX_DISPLAY_DELIMITER}`;
|
||||
}
|
||||
);
|
||||
|
||||
// Step 7: Restore code blocks
|
||||
// This happens AFTER all LaTeX conversions to preserve code content
|
||||
content = content.replace(/<<CODE_BLOCK_(\d+)>>/g, (_, index) => {
|
||||
content = content.replace(CODE_BLOCK_PLACEHOLDER_REGEXP, (_, index) => {
|
||||
return codeBlocks[parseInt(index)];
|
||||
});
|
||||
|
||||
// Step 8: Restore blockquote markers
|
||||
if (blockquoteMarkers.size > 0) {
|
||||
const finalLines = content.split('\n');
|
||||
const finalLines = content.split(NEWLINE);
|
||||
const restoredLines = finalLines.map((line, index) => {
|
||||
const marker = blockquoteMarkers.get(index);
|
||||
return marker ? marker + line : line;
|
||||
});
|
||||
content = restoredLines.join('\n');
|
||||
content = restoredLines.join(NEWLINE);
|
||||
}
|
||||
|
||||
if (latexCache.size >= LATEX_CACHE_MAX_SIZE) {
|
||||
latexCache.delete(latexCache.keys().next().value!);
|
||||
}
|
||||
latexCache.set(originalContent, content);
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
@@ -14,70 +14,94 @@ const JSON_ARRAY_CLOSE = ']';
|
||||
// comma when the model cut off mid-key.
|
||||
const TRAILING_JSON_PUNCTUATION_REGEX = /,?\s*$/;
|
||||
|
||||
/** Bounded cache for parsePartialJsonArgs results. */
|
||||
const PARTIAL_JSON_CACHE_MAX_SIZE = 32;
|
||||
const partialJsonCache = new Map<string, Record<string, unknown> | null>();
|
||||
|
||||
function cacheResult(input: string, result: Record<string, unknown> | null): void {
|
||||
if (partialJsonCache.size >= PARTIAL_JSON_CACHE_MAX_SIZE) {
|
||||
partialJsonCache.delete(partialJsonCache.keys().next().value!);
|
||||
}
|
||||
partialJsonCache.set(input, result);
|
||||
}
|
||||
|
||||
// Parse partial tool-arg JSON streamed token-by-token. Closes any
|
||||
// unterminated string and dangling open containers (in reverse order),
|
||||
// so parsers can still surface keys already received while the call
|
||||
// is still in flight.
|
||||
// is still in flight. Memoized: the char-by-char scanner runs on every
|
||||
// render during streaming even when toolArgs hasn't changed.
|
||||
export function parsePartialJsonArgs(toolArgsString: string): Record<string, unknown> | null {
|
||||
const cached = partialJsonCache.get(toolArgsString);
|
||||
if (cached !== undefined) return cached;
|
||||
|
||||
let result: Record<string, unknown> | null;
|
||||
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(toolArgsString);
|
||||
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
||||
return parsed as Record<string, unknown>;
|
||||
}
|
||||
return null;
|
||||
result =
|
||||
parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: null;
|
||||
} catch {
|
||||
let inString = false;
|
||||
let escape = false;
|
||||
const stack: ('{' | '[')[] = [];
|
||||
result = scanPartialJson(toolArgsString);
|
||||
}
|
||||
|
||||
for (let i = 0; i < toolArgsString.length; i++) {
|
||||
const ch = toolArgsString[i];
|
||||
if (escape) {
|
||||
escape = false;
|
||||
continue;
|
||||
}
|
||||
if (ch === JSON_BACKSLASH && inString) {
|
||||
escape = true;
|
||||
continue;
|
||||
}
|
||||
if (ch === JSON_QUOTE) {
|
||||
inString = !inString;
|
||||
continue;
|
||||
}
|
||||
if (inString) continue;
|
||||
if (ch === JSON_OBJECT_OPEN) stack.push(JSON_OBJECT_OPEN);
|
||||
else if (ch === JSON_OBJECT_CLOSE) {
|
||||
if (stack.length === 0 || stack[stack.length - 1] !== JSON_OBJECT_OPEN) return null;
|
||||
stack.pop();
|
||||
} else if (ch === JSON_ARRAY_OPEN) stack.push(JSON_ARRAY_OPEN);
|
||||
else if (ch === JSON_ARRAY_CLOSE) {
|
||||
if (stack.length === 0 || stack[stack.length - 1] !== JSON_ARRAY_OPEN) return null;
|
||||
stack.pop();
|
||||
}
|
||||
}
|
||||
cacheResult(toolArgsString, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
let completed = toolArgsString;
|
||||
/** Char-by-char scanner for unterminated partial JSON. */
|
||||
function scanPartialJson(toolArgsString: string): Record<string, unknown> | null {
|
||||
let inString = false;
|
||||
let escape = false;
|
||||
const stack: ('{' | '[')[] = [];
|
||||
|
||||
for (let i = 0; i < toolArgsString.length; i++) {
|
||||
const ch = toolArgsString[i];
|
||||
if (escape) {
|
||||
// Dangling escape at end of partial JSON: escape the trailing
|
||||
// backslash as a literal so we can close the string cleanly.
|
||||
completed += JSON_BACKSLASH;
|
||||
escape = false;
|
||||
continue;
|
||||
}
|
||||
if (inString) completed += JSON_QUOTE;
|
||||
if (!inString) completed = completed.replace(TRAILING_JSON_PUNCTUATION_REGEX, '');
|
||||
|
||||
// Close in reverse nesting order: innermost container first.
|
||||
for (let i = stack.length - 1; i >= 0; i--) {
|
||||
completed += stack[i] === JSON_OBJECT_OPEN ? JSON_OBJECT_CLOSE : JSON_ARRAY_CLOSE;
|
||||
if (ch === JSON_BACKSLASH && inString) {
|
||||
escape = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(completed);
|
||||
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
||||
return parsed as Record<string, unknown>;
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
if (ch === JSON_QUOTE) {
|
||||
inString = !inString;
|
||||
continue;
|
||||
}
|
||||
if (inString) continue;
|
||||
if (ch === JSON_OBJECT_OPEN) stack.push(JSON_OBJECT_OPEN);
|
||||
else if (ch === JSON_OBJECT_CLOSE) {
|
||||
if (stack.length === 0 || stack[stack.length - 1] !== JSON_OBJECT_OPEN) return null;
|
||||
stack.pop();
|
||||
} else if (ch === JSON_ARRAY_OPEN) stack.push(JSON_ARRAY_OPEN);
|
||||
else if (ch === JSON_ARRAY_CLOSE) {
|
||||
if (stack.length === 0 || stack[stack.length - 1] !== JSON_ARRAY_OPEN) return null;
|
||||
stack.pop();
|
||||
}
|
||||
}
|
||||
|
||||
let completed = toolArgsString;
|
||||
if (escape) {
|
||||
// Dangling escape at end of partial JSON: escape the trailing
|
||||
// backslash as a literal so we can close the string cleanly.
|
||||
completed += JSON_BACKSLASH;
|
||||
}
|
||||
if (inString) completed += JSON_QUOTE;
|
||||
if (!inString) completed = completed.replace(TRAILING_JSON_PUNCTUATION_REGEX, '');
|
||||
|
||||
// Close in reverse nesting order: innermost container first.
|
||||
for (let i = stack.length - 1; i >= 0; i--) {
|
||||
completed += stack[i] === JSON_OBJECT_OPEN ? JSON_OBJECT_CLOSE : JSON_ARRAY_CLOSE;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(completed);
|
||||
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,41 +155,71 @@ function parseChunk(chunk: string): SearchResult | null {
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Bounded cache for extractSearchResults results. */
|
||||
const SEARCH_RESULTS_CACHE_MAX_SIZE = 32;
|
||||
const searchResultsCache = new Map<string, SearchResult[]>();
|
||||
|
||||
/**
|
||||
* Extract a SearchResult[] from a tool-result string. Returns `[]` when
|
||||
* the input does not match the expected shape — useful for branching
|
||||
* between dedicated search-results rendering and the generic tool-call
|
||||
* block.
|
||||
* block. Memoized: called per render during streaming on unchanged
|
||||
* tool result strings.
|
||||
*/
|
||||
export function extractSearchResults(text: string | undefined | null): SearchResult[] {
|
||||
if (!text) return [];
|
||||
|
||||
const cached = searchResultsCache.get(text);
|
||||
if (cached) return cached;
|
||||
|
||||
const results: SearchResult[] = [];
|
||||
for (const chunk of splitChunks(text)) {
|
||||
const parsed = parseChunk(chunk);
|
||||
if (parsed) results.push(parsed);
|
||||
}
|
||||
|
||||
if (searchResultsCache.size >= SEARCH_RESULTS_CACHE_MAX_SIZE) {
|
||||
searchResultsCache.delete(searchResultsCache.keys().next().value!);
|
||||
}
|
||||
searchResultsCache.set(text, results);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/** Bounded cache for extractSearchQuery results. */
|
||||
const SEARCH_QUERY_CACHE_MAX_SIZE = 32;
|
||||
const searchQueryCache = new Map<string, string>();
|
||||
|
||||
/**
|
||||
* Best-effort extraction of the search query out of a tool call's JSON
|
||||
* argument blob. Currently looks for a `query` field (the convention
|
||||
* used by Exa and most web-search MCP servers); returns an empty string
|
||||
* if it cannot be located.
|
||||
* if it cannot be located. Memoized: called per render during streaming
|
||||
* on unchanged tool args strings.
|
||||
*/
|
||||
export function extractSearchQuery(toolArgs: string | undefined | null): string {
|
||||
if (!toolArgs) return '';
|
||||
|
||||
const cached = searchQueryCache.get(toolArgs);
|
||||
if (cached !== undefined) return cached;
|
||||
|
||||
let result = '';
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(toolArgs);
|
||||
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
||||
const candidate = (parsed as Record<string, unknown>)[SEARCH_TOOL_QUERY_FIELD];
|
||||
if (typeof candidate === 'string') return candidate.trim();
|
||||
if (typeof candidate === 'string') result = candidate.trim();
|
||||
}
|
||||
} catch {
|
||||
return '';
|
||||
result = '';
|
||||
}
|
||||
return '';
|
||||
|
||||
if (searchQueryCache.size >= SEARCH_QUERY_CACHE_MAX_SIZE) {
|
||||
searchQueryCache.delete(searchQueryCache.keys().next().value!);
|
||||
}
|
||||
searchQueryCache.set(toolArgs, result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
// Tests for the memoized parseToolCalls and O(1) tool message lookup in
|
||||
// deriveAgenticSections. These were added to prevent regressions where
|
||||
// streaming text tokens trigger redundant JSON.parse calls on unchanged
|
||||
// tool call data.
|
||||
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { deriveAgenticSections } from '$lib/utils/agentic';
|
||||
import type { ApiChatCompletionToolCall } from '$lib/types/api';
|
||||
import type { DatabaseMessage } from '$lib/types/database';
|
||||
import { MessageRole, AgenticSectionType } from '$lib/enums';
|
||||
|
||||
function makeMessage(overrides: Partial<DatabaseMessage>): DatabaseMessage {
|
||||
return {
|
||||
id: 'm1',
|
||||
convId: 'c1',
|
||||
type: 'text',
|
||||
timestamp: 0,
|
||||
role: MessageRole.ASSISTANT,
|
||||
content: '',
|
||||
parent: null,
|
||||
children: [],
|
||||
...overrides
|
||||
} as DatabaseMessage;
|
||||
}
|
||||
|
||||
describe('parseToolCalls memoization', () => {
|
||||
it('returns the same array reference for the same JSON string', () => {
|
||||
// parseToolCalls is not exported, but deriveAgenticSections uses it
|
||||
// internally. We verify memoization through behavior: calling
|
||||
// deriveAgenticSections twice with the same toolCalls should not
|
||||
// re-parse (which we verify by checking the returned sections
|
||||
// are equivalent).
|
||||
const toolCallsJson = JSON.stringify([
|
||||
{ id: 'call_1', type: 'function', function: { name: 'test', arguments: '{}' } }
|
||||
]);
|
||||
|
||||
const msg = makeMessage({ content: 'hello', toolCalls: toolCallsJson });
|
||||
const sections1 = deriveAgenticSections(msg, [], [], false);
|
||||
const sections2 = deriveAgenticSections(msg, [], [], false);
|
||||
|
||||
expect(sections1).toHaveLength(sections2.length);
|
||||
expect(sections1[0].type).toBe(sections2[0].type);
|
||||
});
|
||||
|
||||
it('does not re-parse JSON on cache hit', () => {
|
||||
const toolCallsJson = JSON.stringify([
|
||||
{ id: 'call_1', type: 'function', function: { name: 'test', arguments: '{}' } }
|
||||
]);
|
||||
|
||||
const msg = makeMessage({ content: 'hello', toolCalls: toolCallsJson });
|
||||
const spy = vi.spyOn(JSON, 'parse');
|
||||
|
||||
deriveAgenticSections(msg, [], [], false);
|
||||
const callsAfterFirst = spy.mock.calls.length;
|
||||
|
||||
deriveAgenticSections(msg, [], [], false);
|
||||
expect(spy.mock.calls.length).toBe(callsAfterFirst);
|
||||
|
||||
spy.mockRestore();
|
||||
});
|
||||
|
||||
it('handles empty/undefined toolCalls without error', () => {
|
||||
const msg = makeMessage({ content: 'hello' });
|
||||
const sections = deriveAgenticSections(msg, [], [], false);
|
||||
|
||||
expect(sections).toHaveLength(1);
|
||||
expect(sections[0].type).toBe(AgenticSectionType.TEXT);
|
||||
});
|
||||
|
||||
it('handles invalid JSON gracefully', () => {
|
||||
const msg = makeMessage({ content: 'hello', toolCalls: '{invalid' });
|
||||
const sections = deriveAgenticSections(msg, [], [], false);
|
||||
|
||||
// Should return just the text section, no tool call sections
|
||||
expect(sections).toHaveLength(1);
|
||||
expect(sections[0].type).toBe(AgenticSectionType.TEXT);
|
||||
});
|
||||
});
|
||||
|
||||
describe('deriveAgenticSections O(1) tool message lookup', () => {
|
||||
it('matches tool messages to tool calls by toolCallId', () => {
|
||||
const toolCallsJson = JSON.stringify([
|
||||
{ id: 'call_1', type: 'function', function: { name: 'test_1', arguments: '{}' } },
|
||||
{ id: 'call_2', type: 'function', function: { name: 'test_2', arguments: '{}' } }
|
||||
]);
|
||||
|
||||
const toolMessages = [
|
||||
makeMessage({ role: MessageRole.TOOL, toolCallId: 'call_1', content: 'result_1' }),
|
||||
makeMessage({ role: MessageRole.TOOL, toolCallId: 'call_2', content: 'result_2' })
|
||||
];
|
||||
|
||||
const msg = makeMessage({ content: 'hello', toolCalls: toolCallsJson });
|
||||
const sections = deriveAgenticSections(msg, toolMessages, [], false);
|
||||
|
||||
// Expect: TEXT + 2 TOOL_CALL sections
|
||||
const toolCallSections = sections.filter((s) => s.type === AgenticSectionType.TOOL_CALL);
|
||||
expect(toolCallSections).toHaveLength(2);
|
||||
expect(toolCallSections[0].toolResult).toBe('result_1');
|
||||
expect(toolCallSections[1].toolResult).toBe('result_2');
|
||||
});
|
||||
|
||||
it('handles missing tool messages (pending calls during streaming)', () => {
|
||||
const toolCallsJson = JSON.stringify([
|
||||
{ id: 'call_1', type: 'function', function: { name: 'test', arguments: '{}' } }
|
||||
]);
|
||||
|
||||
const msg = makeMessage({ content: '', toolCalls: toolCallsJson });
|
||||
const sections = deriveAgenticSections(msg, [], [], true);
|
||||
|
||||
const toolCallSection = sections.find((s) => s.type === AgenticSectionType.TOOL_CALL_PENDING);
|
||||
expect(toolCallSection).toBeDefined();
|
||||
expect(toolCallSection?.content).toBe('');
|
||||
});
|
||||
|
||||
it('scales with many tool calls (no O(n^2) blowup)', () => {
|
||||
const N = 100;
|
||||
const toolCalls = Array.from(
|
||||
{ length: N },
|
||||
(_, i): ApiChatCompletionToolCall => ({
|
||||
id: `call_${i}`,
|
||||
type: 'function',
|
||||
function: { name: `tool_${i}`, arguments: '{}' }
|
||||
})
|
||||
);
|
||||
const toolCallsJson = JSON.stringify(toolCalls);
|
||||
|
||||
const toolMessages = Array.from({ length: N }, (_, i) =>
|
||||
makeMessage({
|
||||
role: MessageRole.TOOL,
|
||||
toolCallId: `call_${i}`,
|
||||
content: `result_${i}`
|
||||
})
|
||||
);
|
||||
|
||||
const msg = makeMessage({ content: 'hello', toolCalls: toolCallsJson });
|
||||
|
||||
// If the lookup were still O(n^2), this would be noticeably slow
|
||||
const start = Date.now();
|
||||
const sections = deriveAgenticSections(msg, toolMessages, [], false);
|
||||
const elapsed = Date.now() - start;
|
||||
|
||||
const toolCallSections = sections.filter((s) => s.type === AgenticSectionType.TOOL_CALL);
|
||||
expect(toolCallSections).toHaveLength(N);
|
||||
expect(elapsed).toBeLessThan(100); // Should be fast with O(1) lookup
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user