mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-19 17:25:07 +02:00
Merge branch 'upstream' into concedo_experimental
# Conflicts: # CMakeLists.txt # docs/speculative.md # ggml/src/ggml-cuda/CMakeLists.txt # ggml/src/ggml-hexagon/ggml-hexagon.cpp # ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c # ggml/src/ggml-hexagon/htp/hmx-ops.h # ggml/src/ggml-hexagon/htp/main.c # ggml/src/ggml-hexagon/htp/matmul-ops.c # ggml/src/ggml-hexagon/htp/rope-ops.c # ggml/src/ggml-hexagon/htp/ssm-conv.c # ggml/src/ggml-opencl/ggml-opencl.cpp # scripts/snapdragon/adb/run-bench.sh # scripts/snapdragon/adb/run-cli.sh # scripts/snapdragon/adb/run-completion.sh # scripts/snapdragon/adb/run-mtmd.sh # scripts/snapdragon/windows/run-bench.ps1 # scripts/snapdragon/windows/run-cli.ps1 # scripts/snapdragon/windows/run-completion.ps1 # scripts/snapdragon/windows/run-mtmd.ps1 # src/llama-vocab.cpp # tests/test-backend-ops.cpp # tools/batched-bench/CMakeLists.txt # tools/batched-bench/batched-bench.cpp # tools/cli/CMakeLists.txt # tools/cli/README.md # tools/cli/cli.cpp # tools/completion/CMakeLists.txt # tools/completion/README.md # tools/llama-bench/CMakeLists.txt # tools/llama-bench/llama-bench.cpp # tools/mtmd/CMakeLists.txt # tools/mtmd/tests/test-deepseek-ocr.py # tools/mtmd/tests/tests-requirements.txt # tools/perplexity/CMakeLists.txt # tools/perplexity/perplexity.cpp # tools/quantize/CMakeLists.txt # tools/server/CMakeLists.txt # tools/server/README.md # ty.toml
This commit is contained in:
Concedo
@@ -0,0 +1,5 @@
|
||||
int llama_batched_bench(int argc, char ** argv);
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
return llama_batched_bench(argc, argv);
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
int llama_cli(int argc, char ** argv);
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
return llama_cli(argc, argv);
|
||||
}
|
||||
@@ -85,7 +85,10 @@ static void sigint_handler(int signo) {
|
||||
}
|
||||
#endif
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
// satisfies -Wmissing-declarations
|
||||
int llama_completion(int argc, char ** argv);
|
||||
|
||||
int llama_completion(int argc, char ** argv) {
|
||||
std::setlocale(LC_NUMERIC, "C");
|
||||
|
||||
common_params params;
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
int llama_completion(int argc, char ** argv);
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
return llama_completion(argc, argv);
|
||||
}
|
||||
@@ -1,6 +1,18 @@
|
||||
# llama-fit-params-impl: fit-params logic, reusable by app
|
||||
|
||||
set(TARGET llama-fit-params-impl)
|
||||
|
||||
add_library(${TARGET} STATIC fit-params.cpp)
|
||||
|
||||
target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
target_link_libraries(${TARGET} PUBLIC llama-common llama ${CMAKE_THREAD_LIBS_INIT})
|
||||
|
||||
# llama-fit-params executable
|
||||
|
||||
set(TARGET llama-fit-params)
|
||||
add_executable(${TARGET} fit-params.cpp)
|
||||
target_link_libraries(${TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT})
|
||||
|
||||
add_executable(${TARGET} main.cpp)
|
||||
target_link_libraries(${TARGET} PRIVATE llama-fit-params-impl)
|
||||
target_compile_features(${TARGET} PRIVATE cxx_std_17)
|
||||
|
||||
if(LLAMA_TOOLS_INSTALL)
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
// satisfies -Wmissing-declarations
|
||||
int llama_fit_params(int argc, char ** argv);
|
||||
|
||||
int llama_fit_params(int argc, char ** argv) {
|
||||
common_params params;
|
||||
|
||||
common_init();
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
int llama_fit_params(int argc, char ** argv);
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
return llama_fit_params(argc, argv);
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
int llama_bench(int argc, char ** argv);
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
return llama_bench(argc, argv);
|
||||
}
|
||||
@@ -170,7 +170,7 @@
|
||||
#define TN_TOK_BOI "v.boi"
|
||||
#define TN_TOK_EOI "v.eoi"
|
||||
|
||||
// hunyuanocr / hunyuanvl (shared GGUF tensor names)
|
||||
// hunyuanvl (shared GGUF tensor names)
|
||||
#define TN_MM_PRE_NORM "mm.pre_norm.%s"
|
||||
#define TN_TOK_IMG_BEGIN "mm.image_begin"
|
||||
#define TN_TOK_IMG_END "mm.image_end"
|
||||
@@ -343,7 +343,6 @@ enum projector_type {
|
||||
PROJECTOR_TYPE_YASA2,
|
||||
PROJECTOR_TYPE_KIMIK25,
|
||||
PROJECTOR_TYPE_NEMOTRON_V2_VL,
|
||||
PROJECTOR_TYPE_HUNYUANOCR,
|
||||
PROJECTOR_TYPE_HUNYUANVL,
|
||||
PROJECTOR_TYPE_MINICPMV4_6,
|
||||
PROJECTOR_TYPE_GRANITE_SPEECH,
|
||||
@@ -393,7 +392,6 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
|
||||
{ PROJECTOR_TYPE_YASA2, "yasa2"},
|
||||
{ PROJECTOR_TYPE_KIMIK25, "kimik25"},
|
||||
{ PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"},
|
||||
{ PROJECTOR_TYPE_HUNYUANOCR, "hunyuanocr"},
|
||||
{ PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"},
|
||||
{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},
|
||||
{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},
|
||||
|
||||
+14
-4
@@ -35,6 +35,16 @@ enum resize_algo {
|
||||
// RESIZE_ALGO_LANCZOS, // TODO
|
||||
};
|
||||
|
||||
// Padding style for img_tool::resize
|
||||
// PAD_NONE - no padding; direct resize to target dimensions
|
||||
// PAD_CEIL - aspect-preserving pad (default)
|
||||
// PAD_NEAREST - aspect-preserving pad with nearest-integer rounding (Pillow byte-parity)
|
||||
enum pad_style {
|
||||
PAD_NONE,
|
||||
PAD_CEIL,
|
||||
PAD_NEAREST,
|
||||
};
|
||||
|
||||
struct clip_hparams {
|
||||
int32_t image_size = 0;
|
||||
int32_t patch_size = 0;
|
||||
@@ -52,7 +62,7 @@ struct clip_hparams {
|
||||
int32_t image_min_pixels = -1;
|
||||
int32_t image_max_pixels = -1;
|
||||
resize_algo image_resize_algo = RESIZE_ALGO_BICUBIC;
|
||||
bool image_resize_pad = true; // if false, center-crop will be applied when resizing
|
||||
pad_style image_resize_pad = PAD_CEIL; // padding style when resizing
|
||||
std::array<uint8_t, 3> image_pad_color = {0, 0, 0};
|
||||
|
||||
// (preprocessor) for llava-uhd style models
|
||||
@@ -61,8 +71,8 @@ struct clip_hparams {
|
||||
int32_t preproc_max_tiles = 0;
|
||||
resize_algo image_resize_algo_rf = RESIZE_ALGO_BICUBIC;
|
||||
resize_algo image_resize_algo_ov = RESIZE_ALGO_BILINEAR;
|
||||
bool image_pad_rf = true; // if true, refined image will be padded (e.g. llava-1.6)
|
||||
bool image_pad_ov = false; // if true, overview image will be padded (e.g. llava-1.6)
|
||||
pad_style image_pad_rf = PAD_CEIL; // padding style for the refined image (e.g. llava-1.6)
|
||||
pad_style image_pad_ov = PAD_NONE; // padding style for the overview image (e.g. llava-1.6)
|
||||
std::array<uint8_t, 3> image_pad_color_rf = {0, 0, 0}; // padding color for refined image
|
||||
std::array<uint8_t, 3> image_pad_color_ov = {0, 0, 0}; // padding color for overview image
|
||||
|
||||
@@ -510,7 +520,7 @@ struct clip_model {
|
||||
ggml_tensor * mm_boi = nullptr;
|
||||
ggml_tensor * mm_eoi = nullptr;
|
||||
|
||||
// hunyuanocr perceiver
|
||||
// hunyuanvl perceiver
|
||||
ggml_tensor * mm_pre_norm_w = nullptr;
|
||||
ggml_tensor * mm_img_begin = nullptr;
|
||||
ggml_tensor * mm_img_end = nullptr;
|
||||
|
||||
+15
-30
@@ -58,7 +58,7 @@
|
||||
#include "models/gemma4v.cpp"
|
||||
#include "models/glm4v.cpp"
|
||||
#include "models/granite-speech.cpp"
|
||||
#include "models/hunyuanocr.cpp"
|
||||
#include "models/hunyuanvl.cpp"
|
||||
#include "models/internvl.cpp"
|
||||
#include "models/kimivl.cpp"
|
||||
#include "models/kimik25.cpp"
|
||||
@@ -996,10 +996,9 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
|
||||
{
|
||||
builder = std::make_unique<clip_graph_cogvlm>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_HUNYUANOCR:
|
||||
case PROJECTOR_TYPE_HUNYUANVL:
|
||||
{
|
||||
builder = std::make_unique<clip_graph_hunyuanocr>(ctx, img);
|
||||
builder = std::make_unique<clip_graph_hunyuanvl>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MLP:
|
||||
case PROJECTOR_TYPE_MLP_NORM:
|
||||
@@ -1316,12 +1315,12 @@ struct clip_model_loader {
|
||||
hparams.has_llava_projector = model.proj_type != PROJECTOR_TYPE_COGVLM;
|
||||
hparams.image_pad_color = {122, 116, 104};
|
||||
if (!hparams.image_res_candidates.empty()) {
|
||||
hparams.image_resize_pad = true;
|
||||
hparams.image_resize_pad = PAD_CEIL;
|
||||
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
|
||||
} else {
|
||||
// llava-1.6 default params
|
||||
hparams.image_pad_ov = false;
|
||||
hparams.image_pad_rf = true;
|
||||
hparams.image_pad_ov = PAD_NONE;
|
||||
hparams.image_pad_rf = PAD_CEIL;
|
||||
hparams.image_pad_color_rf = {122, 116, 104};
|
||||
hparams.image_resize_algo_rf = RESIZE_ALGO_BICUBIC;
|
||||
hparams.image_resize_algo_ov = RESIZE_ALGO_BILINEAR;
|
||||
@@ -1329,7 +1328,7 @@ struct clip_model_loader {
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GLM_EDGE:
|
||||
{
|
||||
hparams.image_resize_pad = true;
|
||||
hparams.image_resize_pad = PAD_CEIL;
|
||||
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MINICPMV:
|
||||
@@ -1529,7 +1528,7 @@ struct clip_model_loader {
|
||||
{
|
||||
hparams.n_merge = 2;
|
||||
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
|
||||
hparams.image_resize_pad = false;
|
||||
hparams.image_resize_pad = PAD_NONE;
|
||||
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
|
||||
get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true);
|
||||
std::vector<int> wa_layer_indexes_vec;
|
||||
@@ -1549,7 +1548,7 @@ struct clip_model_loader {
|
||||
|
||||
// reka model performs better when using resize_bicubic, which stretches
|
||||
// the image to fit fixed square size
|
||||
hparams.image_resize_pad = false;
|
||||
hparams.image_resize_pad = PAD_NONE;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GLM4V:
|
||||
{
|
||||
@@ -1604,31 +1603,23 @@ struct clip_model_loader {
|
||||
hparams.image_size = 1024;
|
||||
hparams.warmup_image_size = 1024;
|
||||
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
|
||||
hparams.image_pad_color[0] = hparams.image_mean[0];
|
||||
hparams.image_pad_color[1] = hparams.image_mean[1];
|
||||
hparams.image_pad_color[2] = hparams.image_mean[2];
|
||||
hparams.image_pad_color = {127, 127, 127};
|
||||
|
||||
get_u32(KEY_SAM_N_BLOCK, hparams.sam_n_layer, true);
|
||||
get_u32(KEY_SAM_N_HEAD, hparams.sam_n_head, true);
|
||||
get_u32(KEY_SAM_N_EMBD, hparams.sam_n_embd, true);
|
||||
get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_HUNYUANOCR:
|
||||
{
|
||||
hparams.n_merge = 2;
|
||||
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
|
||||
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
|
||||
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
|
||||
hparams.set_warmup_n_tokens(28*28);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_HUNYUANVL:
|
||||
{
|
||||
hparams.n_merge = 2;
|
||||
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
|
||||
hparams.image_resize_pad = false;
|
||||
hparams.image_resize_pad = PAD_NONE;
|
||||
hparams.ffn_op = FFN_GELU;
|
||||
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
|
||||
hparams.set_limit_image_tokens(256, 16384);
|
||||
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
|
||||
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false);
|
||||
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false);
|
||||
hparams.set_warmup_n_tokens(32*32);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_LFM2A:
|
||||
@@ -2438,7 +2429,6 @@ struct clip_model_loader {
|
||||
model.mm_boi = get_tensor(TN_TOK_BOI);
|
||||
model.mm_eoi = get_tensor(TN_TOK_EOI);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_HUNYUANOCR:
|
||||
case PROJECTOR_TYPE_HUNYUANVL:
|
||||
{
|
||||
// proj.0 -> mm.0 (conv1), proj.2 -> mm.2 (conv2), mlp -> mm.model.fc (linear)
|
||||
@@ -3294,7 +3284,7 @@ void setup_init_vision_shim_kcpp(struct clip_ctx * ctx_v) {
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MINICPMV:
|
||||
{
|
||||
int minicpmv_version = clip_is_minicpmv(ctx_v);
|
||||
int minicpmv_version = clip_get_hparams(ctx_v)->minicpmv_version;
|
||||
if (minicpmv_version == 2) {
|
||||
// minicpmv 2.5 format:
|
||||
// <image> (overview) </image><slice><image> (slice) </image><image> (slice) </image>\n ... </slice>
|
||||
@@ -3503,7 +3493,6 @@ void setup_init_vision_shim_kcpp(struct clip_ctx * ctx_v) {
|
||||
img_end = "\n"; // prevent empty batch on llama-server
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_HUNYUANOCR:
|
||||
case PROJECTOR_TYPE_HUNYUANVL:
|
||||
{
|
||||
// note: these use fullwidth ｜ (U+FF5C) and ▁ (U+2581) to match the tokenizer vocabulary
|
||||
@@ -3593,7 +3582,6 @@ int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 *
|
||||
case PROJECTOR_TYPE_MIMOVL:
|
||||
case PROJECTOR_TYPE_GLM4V:
|
||||
case PROJECTOR_TYPE_PADDLEOCR:
|
||||
case PROJECTOR_TYPE_HUNYUANOCR:
|
||||
case PROJECTOR_TYPE_HUNYUANVL:
|
||||
case PROJECTOR_TYPE_YOUTUVL:
|
||||
return (img->nx / params.patch_size) / 2;
|
||||
@@ -3810,7 +3798,6 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
|
||||
int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
|
||||
n_patches = h * (h + 1) + 1;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_HUNYUANOCR:
|
||||
case PROJECTOR_TYPE_HUNYUANVL:
|
||||
{
|
||||
int merge = ctx->model.hparams.n_merge;
|
||||
@@ -4446,7 +4433,6 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
|
||||
case PROJECTOR_TYPE_JANUS_PRO:
|
||||
case PROJECTOR_TYPE_PHI4:
|
||||
case PROJECTOR_TYPE_COGVLM:
|
||||
case PROJECTOR_TYPE_HUNYUANOCR:
|
||||
case PROJECTOR_TYPE_YASA2:
|
||||
{
|
||||
// do nothing
|
||||
@@ -4456,7 +4442,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
|
||||
// Compute the HunyuanVL 2D position embedding on CPU (with the
|
||||
// custom sf=(target+0.1)/n_grid bilinear sampling that the
|
||||
// reference implementation uses) and upload it to the graph
|
||||
// input declared in clip_graph_hunyuanocr::build().
|
||||
// input declared in clip_graph_hunyuanvl::build().
|
||||
GGML_ASSERT(model.position_embeddings != nullptr);
|
||||
ggml_tensor * src_t = model.position_embeddings;
|
||||
const int64_t n_embd = src_t->ne[0];
|
||||
@@ -4974,7 +4960,6 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
|
||||
case PROJECTOR_TYPE_KIMIK25:
|
||||
case PROJECTOR_TYPE_YASA2:
|
||||
return ctx->model.mm_2_w->ne[1];
|
||||
case PROJECTOR_TYPE_HUNYUANOCR:
|
||||
case PROJECTOR_TYPE_HUNYUANVL:
|
||||
return ctx->model.mm_model_proj->ne[1];
|
||||
case PROJECTOR_TYPE_COGVLM:
|
||||
|
||||
+159
-155
@@ -88,164 +88,168 @@ static ggml_tensor * get_rel_pos(ggml_context * ctx0,
|
||||
return cur; // [C, k_size, q_size]
|
||||
}
|
||||
|
||||
|
||||
ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
|
||||
// Building SAM
|
||||
const int n_embd = hparams.sam_n_embd;
|
||||
const int n_layer = hparams.sam_n_layer;
|
||||
const int n_heads = hparams.sam_n_head;
|
||||
const int d_heads = n_embd / n_heads;
|
||||
const int window = hparams.attn_window_size;
|
||||
|
||||
ggml_tensor * inpL;
|
||||
|
||||
inpL = ggml_conv_2d_sk_p0(ctx0, model.patch_embed_proj_w, inp_raw);
|
||||
inpL = ggml_add(ctx0, inpL, ggml_reshape_3d(ctx0, model.patch_embed_proj_b, 1, 1, n_embd));
|
||||
inpL = ggml_cont(ctx0, ggml_permute(ctx0, inpL, 1, 2, 0, 3));
|
||||
|
||||
ggml_tensor * rel_pos_indices_local;
|
||||
ggml_tensor * rel_pos_indices_global;
|
||||
|
||||
rel_pos_indices_local = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, window, window);
|
||||
rel_pos_indices_global = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, inpL->ne[1], inpL->ne[2]);
|
||||
ggml_set_name(rel_pos_indices_local, "rel_pos_indices_local");
|
||||
ggml_set_name(rel_pos_indices_global, "rel_pos_indices_global");
|
||||
ggml_set_input(rel_pos_indices_local);
|
||||
ggml_set_input(rel_pos_indices_global);
|
||||
|
||||
ggml_tensor * cur;
|
||||
const auto tgt_size = inpL->ne[1];
|
||||
const auto str_size = model.pos_embed->ne[1];
|
||||
|
||||
if (str_size != tgt_size) {
|
||||
ggml_tensor * old_pos_embed = nullptr;
|
||||
old_pos_embed = ggml_cont(ctx0, ggml_permute(ctx0, model.pos_embed, 2, 0, 1, 3));
|
||||
ggml_tensor * new_pos_embed =
|
||||
ggml_interpolate(ctx0, old_pos_embed, tgt_size, tgt_size, n_embd, 1, GGML_SCALE_MODE_BICUBIC);
|
||||
new_pos_embed = ggml_cont(ctx0, ggml_permute(ctx0, new_pos_embed, 1, 2, 0, 3));
|
||||
cur = ggml_add(ctx0, inpL, new_pos_embed);
|
||||
} else {
|
||||
cur = ggml_add(ctx0, inpL, model.pos_embed);
|
||||
}
|
||||
|
||||
// loop over layers
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
auto & layer = model.sam_layers[il];
|
||||
ggml_tensor * shortcut = cur;
|
||||
|
||||
// layernorm1
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
|
||||
const int64_t w0 = cur->ne[1];
|
||||
const int64_t h0 = cur->ne[2];
|
||||
|
||||
ggml_tensor * indices;
|
||||
|
||||
if (hparams.is_global_attn(il)) {
|
||||
indices = rel_pos_indices_global;
|
||||
} else {
|
||||
// local attention layer - apply window partition
|
||||
cur = window_partition(ctx0, cur, window);
|
||||
indices = rel_pos_indices_local;
|
||||
}
|
||||
|
||||
const int64_t W = cur->ne[1];
|
||||
const int64_t H = cur->ne[2];
|
||||
// self-attention
|
||||
{
|
||||
const int B = cur->ne[3];
|
||||
|
||||
cur = ggml_mul_mat(ctx0, layer.qkv_w, cur);
|
||||
cur = ggml_add(ctx0, cur, layer.qkv_b);
|
||||
cur = ggml_cont(ctx0, cur); // Ensure tensor is contiguous before reshape
|
||||
cur = ggml_reshape_4d(ctx0, cur, n_embd, 3, W * H, B);
|
||||
|
||||
ggml_tensor * Q;
|
||||
ggml_tensor * K;
|
||||
ggml_tensor * V;
|
||||
|
||||
Q = ggml_view_3d(ctx0, cur, n_embd, W * H, B, cur->nb[2], cur->nb[3], 0 * cur->nb[1]);
|
||||
Q = ggml_reshape_4d(ctx0, ggml_cont(ctx0, Q), d_heads, n_heads, W * H, B);
|
||||
|
||||
K = ggml_view_3d(ctx0, cur, n_embd, W * H, B, cur->nb[2], cur->nb[3], 1 * cur->nb[1]);
|
||||
K = ggml_reshape_4d(ctx0, ggml_cont(ctx0, K), d_heads, n_heads, W * H, B);
|
||||
|
||||
V = ggml_view_3d(ctx0, cur, n_embd, W * H, B, cur->nb[2], cur->nb[3], 2 * cur->nb[1]);
|
||||
V = ggml_reshape_4d(ctx0, ggml_cont(ctx0, V), d_heads, n_heads, W * H, B);
|
||||
|
||||
ggml_tensor * mask;
|
||||
ggml_tensor * rw;
|
||||
ggml_tensor * rh;
|
||||
ggml_tensor * qr;
|
||||
|
||||
rw = get_rel_pos(ctx0, layer.rel_pos_w, indices, W, W); // [W, W, C]
|
||||
rh = get_rel_pos(ctx0, layer.rel_pos_h, indices, H, H); // [H, H, C]
|
||||
qr = ggml_permute(ctx0, Q, 0, 2, 1, 3);
|
||||
qr = ggml_reshape_4d(ctx0, ggml_cont(ctx0, qr), d_heads, W, H, B * n_heads);
|
||||
|
||||
rw = ggml_mul_mat(ctx0, rw,
|
||||
ggml_cont(ctx0, ggml_permute(ctx0, qr, 0, 2, 1, 3))); // [B*n_heads, W, H, W]
|
||||
rw = ggml_cont(ctx0, ggml_permute(ctx0, rw, 0, 2, 1, 3)); // [B*n_heads, H, W, W]
|
||||
rw = ggml_reshape_4d(ctx0, rw, W, 1, W * H, n_heads * B);
|
||||
rw = ggml_repeat_4d(ctx0, rw, W, H, W * H, n_heads * B);
|
||||
rh = ggml_mul_mat(ctx0, rh, qr); // [B*n_heads, H, W, H]
|
||||
rh = ggml_reshape_4d(ctx0, rh, 1, H, W * H, n_heads * B);
|
||||
mask = ggml_add(ctx0, rw, rh); // [B*n_heads, H*W, H, W]
|
||||
mask = ggml_reshape_4d(ctx0, mask, W * H, W * H, n_heads, B);
|
||||
// casting mask to F16 only required when flash-attn is enabled
|
||||
if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
|
||||
mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);
|
||||
}
|
||||
|
||||
const float scale = 1.0f / sqrtf(static_cast<float>(d_heads));
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b, Q, K, V, mask, scale,
|
||||
il); // [B, H*W, n_embd]
|
||||
cur = ggml_reshape_4d(ctx0, ggml_cont(ctx0, cur), n_embd, W, H, B);
|
||||
}
|
||||
|
||||
if (hparams.is_global_attn(il) == false) {
|
||||
// local attention layer - reverse window partition
|
||||
cur = window_unpartition(ctx0, cur, w0, h0, window);
|
||||
}
|
||||
|
||||
// re-add the layer input, e.g., residual
|
||||
cur = ggml_add(ctx0, cur, shortcut);
|
||||
|
||||
ggml_tensor * inpFF = cur;
|
||||
|
||||
// layernorm2
|
||||
cur = build_norm(inpFF, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
|
||||
// ffn
|
||||
cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, nullptr, nullptr, layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
|
||||
// residual 2
|
||||
cur = ggml_add(ctx0, cur, inpFF);
|
||||
cb(cur, "sam_layer_out", il);
|
||||
}
|
||||
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
|
||||
|
||||
cur = ggml_conv_2d(ctx0, model.neck_0_w, cur, 1, 1, 0, 0, 1, 1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
|
||||
cur = build_norm(cur, model.neck_1_w, model.neck_1_b, NORM_TYPE_NORMAL, hparams.eps, -1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
|
||||
|
||||
cur = ggml_conv_2d(ctx0, model.neck_2_w, cur, 1, 1, 1, 1, 1, 1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
|
||||
cur = build_norm(cur, model.neck_3_w, model.neck_3_b, NORM_TYPE_NORMAL, hparams.eps, -1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
|
||||
|
||||
cur = ggml_conv_2d(ctx0, model.net_2, cur, 2, 2, 1, 1, 1, 1);
|
||||
cur = ggml_conv_2d(ctx0, model.net_3, cur, 2, 2, 1, 1, 1, 1);
|
||||
cb(cur, "sam_output", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_deepseekocr::build() {
|
||||
// patch embedding
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
|
||||
ggml_tensor * sam_out;
|
||||
// Building SAM
|
||||
{
|
||||
const int n_embd = hparams.sam_n_embd;
|
||||
const int n_layer = hparams.sam_n_layer;
|
||||
const int n_heads = hparams.sam_n_head;
|
||||
const int d_heads = n_embd / n_heads;
|
||||
const int window = hparams.attn_window_size;
|
||||
|
||||
ggml_tensor * inpL;
|
||||
|
||||
inpL = ggml_conv_2d_sk_p0(ctx0, model.patch_embed_proj_w, inp_raw);
|
||||
inpL = ggml_add(ctx0, inpL, ggml_reshape_3d(ctx0, model.patch_embed_proj_b, 1, 1, n_embd));
|
||||
inpL = ggml_cont(ctx0, ggml_permute(ctx0, inpL, 1, 2, 0, 3));
|
||||
|
||||
ggml_tensor * rel_pos_indices_local;
|
||||
ggml_tensor * rel_pos_indices_global;
|
||||
|
||||
rel_pos_indices_local = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, window, window);
|
||||
rel_pos_indices_global = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, inpL->ne[1], inpL->ne[2]);
|
||||
ggml_set_name(rel_pos_indices_local, "rel_pos_indices_local");
|
||||
ggml_set_name(rel_pos_indices_global, "rel_pos_indices_global");
|
||||
ggml_set_input(rel_pos_indices_local);
|
||||
ggml_set_input(rel_pos_indices_global);
|
||||
|
||||
ggml_tensor * cur;
|
||||
const auto tgt_size = inpL->ne[1];
|
||||
const auto str_size = model.pos_embed->ne[1];
|
||||
|
||||
if (str_size != tgt_size) {
|
||||
ggml_tensor * old_pos_embed = nullptr;
|
||||
old_pos_embed = ggml_cont(ctx0, ggml_permute(ctx0, model.pos_embed, 2, 0, 1, 3));
|
||||
ggml_tensor * new_pos_embed =
|
||||
ggml_interpolate(ctx0, old_pos_embed, tgt_size, tgt_size, n_embd, 1, GGML_SCALE_MODE_BICUBIC);
|
||||
new_pos_embed = ggml_cont(ctx0, ggml_permute(ctx0, new_pos_embed, 1, 2, 0, 3));
|
||||
cur = ggml_add(ctx0, inpL, new_pos_embed);
|
||||
} else {
|
||||
cur = ggml_add(ctx0, inpL, model.pos_embed);
|
||||
}
|
||||
|
||||
// loop over layers
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
auto & layer = model.sam_layers[il];
|
||||
ggml_tensor * shortcut = cur;
|
||||
|
||||
// layernorm1
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
|
||||
const int64_t w0 = cur->ne[1];
|
||||
const int64_t h0 = cur->ne[2];
|
||||
|
||||
ggml_tensor * indices;
|
||||
|
||||
if (hparams.is_global_attn(il)) {
|
||||
indices = rel_pos_indices_global;
|
||||
} else {
|
||||
// local attention layer - apply window partition
|
||||
cur = window_partition(ctx0, cur, window);
|
||||
indices = rel_pos_indices_local;
|
||||
}
|
||||
|
||||
const int64_t W = cur->ne[1];
|
||||
const int64_t H = cur->ne[2];
|
||||
// self-attention
|
||||
{
|
||||
const int B = cur->ne[3];
|
||||
|
||||
cur = ggml_mul_mat(ctx0, layer.qkv_w, cur);
|
||||
cur = ggml_add(ctx0, cur, layer.qkv_b);
|
||||
cur = ggml_cont(ctx0, cur); // Ensure tensor is contiguous before reshape
|
||||
cur = ggml_reshape_4d(ctx0, cur, n_embd, 3, W * H, B);
|
||||
|
||||
ggml_tensor * Q;
|
||||
ggml_tensor * K;
|
||||
ggml_tensor * V;
|
||||
|
||||
Q = ggml_view_3d(ctx0, cur, n_embd, W * H, B, cur->nb[2], cur->nb[3], 0 * cur->nb[1]);
|
||||
Q = ggml_reshape_4d(ctx0, ggml_cont(ctx0, Q), d_heads, n_heads, W * H, B);
|
||||
|
||||
K = ggml_view_3d(ctx0, cur, n_embd, W * H, B, cur->nb[2], cur->nb[3], 1 * cur->nb[1]);
|
||||
K = ggml_reshape_4d(ctx0, ggml_cont(ctx0, K), d_heads, n_heads, W * H, B);
|
||||
|
||||
V = ggml_view_3d(ctx0, cur, n_embd, W * H, B, cur->nb[2], cur->nb[3], 2 * cur->nb[1]);
|
||||
V = ggml_reshape_4d(ctx0, ggml_cont(ctx0, V), d_heads, n_heads, W * H, B);
|
||||
|
||||
ggml_tensor * mask;
|
||||
ggml_tensor * rw;
|
||||
ggml_tensor * rh;
|
||||
ggml_tensor * qr;
|
||||
|
||||
rw = get_rel_pos(ctx0, layer.rel_pos_w, indices, W, W); // [W, W, C]
|
||||
rh = get_rel_pos(ctx0, layer.rel_pos_h, indices, H, H); // [H, H, C]
|
||||
qr = ggml_permute(ctx0, Q, 0, 2, 1, 3);
|
||||
qr = ggml_reshape_4d(ctx0, ggml_cont(ctx0, qr), d_heads, W, H, B * n_heads);
|
||||
|
||||
rw = ggml_mul_mat(ctx0, rw,
|
||||
ggml_cont(ctx0, ggml_permute(ctx0, qr, 0, 2, 1, 3))); // [B*n_heads, W, H, W]
|
||||
rw = ggml_cont(ctx0, ggml_permute(ctx0, rw, 0, 2, 1, 3)); // [B*n_heads, H, W, W]
|
||||
rw = ggml_reshape_4d(ctx0, rw, W, 1, W * H, n_heads * B);
|
||||
rw = ggml_repeat_4d(ctx0, rw, W, H, W * H, n_heads * B);
|
||||
rh = ggml_mul_mat(ctx0, rh, qr); // [B*n_heads, H, W, H]
|
||||
rh = ggml_reshape_4d(ctx0, rh, 1, H, W * H, n_heads * B);
|
||||
mask = ggml_add(ctx0, rw, rh); // [B*n_heads, H*W, H, W]
|
||||
mask = ggml_reshape_4d(ctx0, mask, W * H, W * H, n_heads, B);
|
||||
mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);
|
||||
|
||||
const float scale = 1.0f / sqrtf(static_cast<float>(d_heads));
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b, Q, K, V, mask, scale,
|
||||
il); // [B, H*W, n_embd]
|
||||
cur = ggml_reshape_4d(ctx0, ggml_cont(ctx0, cur), n_embd, W, H, B);
|
||||
}
|
||||
|
||||
if (hparams.is_global_attn(il) == false) {
|
||||
// local attention layer - reverse window partition
|
||||
cur = window_unpartition(ctx0, cur, w0, h0, window);
|
||||
}
|
||||
|
||||
// re-add the layer input, e.g., residual
|
||||
cur = ggml_add(ctx0, cur, shortcut);
|
||||
|
||||
ggml_tensor * inpFF = cur;
|
||||
|
||||
// layernorm2
|
||||
cur = build_norm(inpFF, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
|
||||
// ffn
|
||||
cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, nullptr, nullptr, layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
|
||||
// residual 2
|
||||
cur = ggml_add(ctx0, cur, inpFF);
|
||||
cb(cur, "sam_layer_out", il);
|
||||
}
|
||||
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
|
||||
|
||||
cur = ggml_conv_2d(ctx0, model.neck_0_w, cur, 1, 1, 0, 0, 1, 1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
|
||||
cur = build_norm(cur, model.neck_1_w, model.neck_1_b, NORM_TYPE_NORMAL, hparams.eps, -1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
|
||||
|
||||
cur = ggml_conv_2d(ctx0, model.neck_2_w, cur, 1, 1, 1, 1, 1, 1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
|
||||
cur = build_norm(cur, model.neck_3_w, model.neck_3_b, NORM_TYPE_NORMAL, hparams.eps, -1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
|
||||
|
||||
cur = ggml_conv_2d(ctx0, model.net_2, cur, 2, 2, 1, 1, 1, 1);
|
||||
cur = ggml_conv_2d(ctx0, model.net_3, cur, 2, 2, 1, 1, 1, 1);
|
||||
cb(cur, "sam_output", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
sam_out = cur;
|
||||
}
|
||||
ggml_tensor * sam_out = build_sam(inp_raw);
|
||||
|
||||
ggml_tensor * clip_out;
|
||||
// Building DS-OCR CLIP
|
||||
|
||||
@@ -1,25 +1,15 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_hunyuanocr::build() {
|
||||
ggml_cgraph * clip_graph_hunyuanvl::build() {
|
||||
const int merge = hparams.n_merge;
|
||||
const int pw = n_patches_x;
|
||||
const int ph = n_patches_y;
|
||||
|
||||
// Position embedding interpolation.
|
||||
// HunyuanVL needs scale factors sf=(target+0.1)/n_grid, which the standard
|
||||
// ggml_interpolate cannot express. To avoid adding a new ggml op, the
|
||||
// resize is computed on CPU in clip_image_batch_encode and uploaded here
|
||||
// as a graph input (named "hunyuanvl_pos_embd").
|
||||
// HunyuanOCR uses the same square layout and the standard ratio-based
|
||||
// interpolation provided by resize_position_embeddings().
|
||||
ggml_tensor * pos_embd = nullptr;
|
||||
if (proj_type == PROJECTOR_TYPE_HUNYUANVL && model.position_embeddings) {
|
||||
pos_embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, ph * pw);
|
||||
ggml_set_name(pos_embd, "hunyuanvl_pos_embd");
|
||||
ggml_set_input(pos_embd);
|
||||
} else {
|
||||
pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BILINEAR);
|
||||
}
|
||||
// position embedding: declared as a graph input, filled on CPU
|
||||
// by clip_image_batch_encode (see PROJECTOR_TYPE_HUNYUANVL branch there).
|
||||
ggml_tensor * pos_embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, ph * pw);
|
||||
ggml_set_name(pos_embd, "hunyuanvl_pos_embd");
|
||||
ggml_set_input(pos_embd);
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
ggml_tensor * cur = build_vit(inp, n_patches, NORM_TYPE_NORMAL, hparams.ffn_op, pos_embd, nullptr);
|
||||
@@ -118,6 +118,7 @@ struct clip_graph_whisper_enc : clip_graph {
|
||||
struct clip_graph_deepseekocr : clip_graph {
|
||||
clip_graph_deepseekocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model
|
||||
};
|
||||
|
||||
struct clip_graph_conformer : clip_graph {
|
||||
@@ -141,8 +142,8 @@ struct clip_graph_glm4v : clip_graph {
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_hunyuanocr : clip_graph {
|
||||
clip_graph_hunyuanocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
struct clip_graph_hunyuanvl : clip_graph {
|
||||
clip_graph_hunyuanvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
|
||||
+56
-38
@@ -38,7 +38,7 @@ struct img_tool {
|
||||
clip_image_u8 & dst,
|
||||
const clip_image_size & target_resolution,
|
||||
resize_algo algo,
|
||||
bool add_padding = true, // TODO: define the behavior for add_padding = false
|
||||
pad_style padding = PAD_CEIL,
|
||||
std::array<uint8_t, 3> pad_color = {0, 0, 0}) {
|
||||
dst.nx = target_resolution.width;
|
||||
dst.ny = target_resolution.height;
|
||||
@@ -50,7 +50,7 @@ struct img_tool {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!add_padding) {
|
||||
if (padding == PAD_NONE) {
|
||||
// direct resize
|
||||
switch (algo) {
|
||||
case RESIZE_ALGO_BILINEAR:
|
||||
@@ -71,8 +71,15 @@ struct img_tool {
|
||||
float scale_w = static_cast<float>(target_resolution.width) / src.nx;
|
||||
float scale_h = static_cast<float>(target_resolution.height) / src.ny;
|
||||
float scale = std::min(scale_w, scale_h);
|
||||
int new_width = std::min(static_cast<int>(std::ceil(src.nx * scale)), target_resolution.width);
|
||||
int new_height = std::min(static_cast<int>(std::ceil(src.ny * scale)), target_resolution.height);
|
||||
|
||||
int new_width, new_height;
|
||||
if (padding == PAD_NEAREST) {
|
||||
new_width = std::min(static_cast<int>(std::round(src.nx * scale)), target_resolution.width);
|
||||
new_height = std::min(static_cast<int>(std::round(src.ny * scale)), target_resolution.height);
|
||||
} else {
|
||||
new_width = std::min(static_cast<int>(std::ceil(src.nx * scale)), target_resolution.width);
|
||||
new_height = std::min(static_cast<int>(std::ceil(src.ny * scale)), target_resolution.height);
|
||||
}
|
||||
|
||||
switch (algo) {
|
||||
case RESIZE_ALGO_BILINEAR:
|
||||
@@ -91,9 +98,14 @@ struct img_tool {
|
||||
// fill dst with pad_color
|
||||
fill(dst, pad_color);
|
||||
|
||||
int offset_x = (target_resolution.width - new_width) / 2;
|
||||
int offset_y = (target_resolution.height - new_height) / 2;
|
||||
|
||||
int offset_x, offset_y;
|
||||
if (padding == PAD_NEAREST) {
|
||||
offset_x = static_cast<int>(std::round((target_resolution.width - new_width) / 2.0f));
|
||||
offset_y = static_cast<int>(std::round((target_resolution.height - new_height) / 2.0f));
|
||||
} else {
|
||||
offset_x = (target_resolution.width - new_width) / 2;
|
||||
offset_y = (target_resolution.height - new_height) / 2;
|
||||
}
|
||||
composite(dst, resized_image, offset_x, offset_y);
|
||||
}
|
||||
}
|
||||
@@ -356,10 +368,10 @@ private:
|
||||
GGML_ASSERT(inSize > 0 && outSize > 0);
|
||||
double support, scale, filterscale;
|
||||
double center, ww, ss;
|
||||
int xx, x, ksize, xmin, xmax, xcnt;
|
||||
int xx, x, ksize, xmin, xmax;
|
||||
|
||||
// Calculate scaling factor: ratio of input range to output size
|
||||
filterscale = scale = (double)inSize / outSize;
|
||||
filterscale = scale = static_cast<double>(inSize) / outSize;
|
||||
// For upsampling (scale < 1), keep filterscale = 1 to maintain filter sharpness
|
||||
// For downsampling (scale > 1), widen filter to prevent aliasing
|
||||
if (filterscale < 1.0) {
|
||||
@@ -373,6 +385,7 @@ private:
|
||||
std::vector<double> pre_weights(outSize * ksize); // Temporary weights
|
||||
bounds.resize(outSize * 2);
|
||||
|
||||
|
||||
// For each output pixel, compute its filter coefficients
|
||||
for (xx = 0; xx < outSize; xx++) {
|
||||
// Calculate the center position in input space (pixel-center convention: +0.5)
|
||||
@@ -391,10 +404,10 @@ private:
|
||||
xmax = inSize;
|
||||
}
|
||||
|
||||
xcnt = xmax - xmin;
|
||||
xmax -= xmin;
|
||||
|
||||
// Compute filter weights for each contributing input pixel
|
||||
for (x = 0; x < xcnt; x++) {
|
||||
for (x = 0; x < xmax; x++) {
|
||||
// Distance from input pixel center to output pixel center in input space
|
||||
double w = bicubic_filter((x + xmin - center + 0.5) * ss);
|
||||
pre_weights[xx * ksize + x] = w;
|
||||
@@ -402,7 +415,7 @@ private:
|
||||
}
|
||||
|
||||
// Normalize weights to sum to 1.0 (preserves brightness)
|
||||
for (x = 0; x < xcnt; x++) {
|
||||
for (x = 0; x < xmax; x++) {
|
||||
if (ww != 0.0) {
|
||||
pre_weights[xx * ksize + x] /= ww;
|
||||
}
|
||||
@@ -415,18 +428,27 @@ private:
|
||||
|
||||
// Store input pixel range for this output pixel
|
||||
bounds[xx * 2 + 0] = xmin;
|
||||
bounds[xx * 2 + 1] = xcnt;
|
||||
bounds[xx * 2 + 1] = xmax;
|
||||
}
|
||||
|
||||
// Convert floating-point coefficients to fixed-point integers
|
||||
// Formula: int32 = round(float * 2^PRECISION_BITS)
|
||||
weights.resize(outSize * ksize);
|
||||
|
||||
const double fxp_scale = std::ldexp(1.0, PRECISION_BITS); // 1.0 * 2^PRECISION_BITS
|
||||
|
||||
for (int i = 0; i < outSize * ksize; i++) {
|
||||
double tmp_val = pre_weights[i] * fxp_scale;
|
||||
if (pre_weights[i] < 0) {
|
||||
weights[i] = static_cast<int32_t>(-0.5 + pre_weights[i] * (1 << PRECISION_BITS));
|
||||
tmp_val -= 0.5;
|
||||
} else {
|
||||
weights[i] = static_cast<int32_t>(0.5 + pre_weights[i] * (1 << PRECISION_BITS));
|
||||
tmp_val += 0.5;
|
||||
}
|
||||
tmp_val = std::round(tmp_val);
|
||||
tmp_val = std::clamp(tmp_val,
|
||||
static_cast<double>(std::numeric_limits<int32_t>::min()),
|
||||
static_cast<double>(std::numeric_limits<int32_t>::max()));
|
||||
weights[i] = static_cast<int32_t>(tmp_val);
|
||||
}
|
||||
|
||||
return ksize;
|
||||
@@ -1083,35 +1105,31 @@ bool mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img, cli
|
||||
//
|
||||
|
||||
bool mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) {
|
||||
const std::vector native_resolutions = {
|
||||
/*512 tiny , 640 small, */ 1024 /* base */, 1280 /* large */
|
||||
};
|
||||
// original image size
|
||||
const clip_image_size original_size{img.nx, img.ny};
|
||||
const int orig_w = original_size.width;
|
||||
const int orig_h = original_size.height;
|
||||
const int orig_area = orig_h * orig_w;
|
||||
static constexpr int native_resolutions[] = { 1024 /* base */, 1280 /* large */ };
|
||||
// TODO: support 512 (tiny) and 640 (small) once we have eval data for them
|
||||
|
||||
size_t mode_i = 0;
|
||||
int min_diff = orig_area;
|
||||
const int64_t orig_area = static_cast<int64_t>(img.nx) * img.ny;
|
||||
|
||||
for (size_t i = 0; i < native_resolutions.size(); i++) {
|
||||
int r = native_resolutions[i];
|
||||
if (std::abs(orig_area - r * r) < min_diff) {
|
||||
mode_i = i;
|
||||
min_diff = std::abs(orig_area - r * r);
|
||||
size_t mode_i = 0;
|
||||
int64_t min_diff = std::numeric_limits<int64_t>::max();
|
||||
for (size_t i = 0; i < std::size(native_resolutions); i++) {
|
||||
const int64_t r = native_resolutions[i];
|
||||
const int64_t diff = std::abs(orig_area - r * r);
|
||||
if (diff < min_diff) {
|
||||
mode_i = i;
|
||||
min_diff = diff;
|
||||
}
|
||||
}
|
||||
|
||||
/* Native Resolution (Base/Large) */
|
||||
const int image_size = native_resolutions[mode_i];
|
||||
|
||||
// scaled and padded image
|
||||
clip_image_u8_ptr scaled_img(clip_image_u8_init());
|
||||
img_tool::resize(img, *scaled_img, clip_image_size{image_size, image_size}, hparams.image_resize_algo);
|
||||
// Aspect-preserving fit-and-pad. Pillow bicubic + PAD_NEAREST for
|
||||
// byte-parity with the upstream deepseek-ai/DeepSeek-OCR HF preprocessor.
|
||||
clip_image_u8 padded;
|
||||
img_tool::resize(img, padded, {image_size, image_size}, RESIZE_ALGO_BICUBIC_PILLOW,
|
||||
PAD_NEAREST, hparams.image_pad_color);
|
||||
|
||||
clip_image_f32_ptr res(clip_image_f32_init());
|
||||
img_u8_to_f32(*scaled_img, *res, hparams.image_mean, hparams.image_std);
|
||||
img_u8_to_f32(padded, *res, hparams.image_mean, hparams.image_std);
|
||||
output.entries.push_back(std::move(res));
|
||||
|
||||
output.grid_x = 1;
|
||||
@@ -1246,7 +1264,7 @@ clip_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const clip_image_u8
|
||||
std::max(1, static_cast<int>(std::floor(resized.ny * scale))),
|
||||
};
|
||||
clip_image_u8 scaled;
|
||||
img_tool::resize(resized, scaled, new_size, RESIZE_ALGO_BILINEAR, false);
|
||||
img_tool::resize(resized, scaled, new_size, RESIZE_ALGO_BILINEAR, PAD_NONE);
|
||||
resized = std::move(scaled);
|
||||
}
|
||||
|
||||
@@ -1347,7 +1365,7 @@ bool mtmd_image_preprocessor_step3vl::preprocess(const clip_image_u8 & img, clip
|
||||
clip_image_u8 img_for_crop = prepared;
|
||||
if (instructions.refined_size.width != prepared.nx || instructions.refined_size.height != prepared.ny) {
|
||||
clip_image_u8 refined;
|
||||
img_tool::resize(prepared, refined, instructions.refined_size, RESIZE_ALGO_BILINEAR, false);
|
||||
img_tool::resize(prepared, refined, instructions.refined_size, RESIZE_ALGO_BILINEAR, PAD_NONE);
|
||||
img_for_crop = std::move(refined);
|
||||
}
|
||||
|
||||
|
||||
@@ -493,7 +493,6 @@ struct mtmd_context {
|
||||
img_end = "\n"; // prevent empty batch on llama-server
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_HUNYUANOCR:
|
||||
case PROJECTOR_TYPE_HUNYUANVL:
|
||||
{
|
||||
// note: these use fullwidth ｜ (U+FF5C) and ▁ (U+2581) to match the tokenizer vocabulary
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
|
||||
A Powdery Surface
|
||||
Is Closely Explored
|
||||
|
||||
By JOHN NOBLE WILFORD
|
||||
Special to The New York Times
|
||||
|
||||
HOUSTON, Monday, July 21—Men have landed and walked on the moon.
|
||||
|
||||
Two Americans, astronauts of Apollo 11, steered their fragile four-legged lunar module safely and smoothly to the historic landing yesterday at 4:17:40 P.M., Eastern daylight time.
|
||||
|
||||
Neil A. Armstrong, the 38-year-old civilian commander, radioed to earth and the mission control room here:
|
||||
|
||||
"Houston, Tranquility Base here. The Eagle has landed."
|
||||
|
||||
The first men to reach the moon—Mr. Armstrong and his co-pilot, Col. Edwin E. Aldrin Jr. of the Air Force—brought their ship to rest on a level, rock-strewn plain near the southwestern shore of the arid Sea of Tranquility.
|
||||
|
||||
About six and a half hours later, Mr. Armstrong opened the landing craft's hatch, stepped slowly down the ladder and declared as he planted the first human footprint on the lunar crust:
|
||||
|
||||
"That's one small step for man, one giant leap for mankind."
|
||||
|
||||
His first step on the moon came at 10:56:20 P.M., as a television camera outside the craft transmitted his every move to an awed and excited audience of hundreds of millions of people on earth.
|
||||
|
||||
Tentative Steps Test Soil
|
||||
@@ -0,0 +1,5 @@
|
||||
int llama_perplexity(int argc, char ** argv);
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
return llama_perplexity(argc, argv);
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
int llama_quantize(int argc, char ** argv);
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
return llama_quantize(argc, argv);
|
||||
}
|
||||
@@ -489,7 +489,10 @@ static bool parse_layer_prune(const char * data, std::vector<int> & prune_layers
|
||||
return true;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
// satisfies -Wmissing-declarations
|
||||
int llama_quantize(int argc, char ** argv);
|
||||
|
||||
int llama_quantize(int argc, char ** argv) {
|
||||
std::setlocale(LC_NUMERIC, "C");
|
||||
if (argc < 3) {
|
||||
usage(argv[0]);
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
int llama_server(int argc, char ** argv);
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
return llama_server(argc, argv);
|
||||
}
|
||||
@@ -71,7 +71,10 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t
|
||||
};
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
// satisfies -Wmissing-declarations
|
||||
int llama_server(int argc, char ** argv);
|
||||
|
||||
int llama_server(int argc, char ** argv) {
|
||||
std::setlocale(LC_NUMERIC, "C");
|
||||
|
||||
// own arguments required by this example
|
||||
|
||||
@@ -11,24 +11,28 @@
|
||||
|
||||
cd ../../
|
||||
|
||||
# Ensure node_modules are installed
|
||||
if [ ! -d "tools/ui/node_modules" ]; then
|
||||
echo "📦 Installing npm dependencies..."
|
||||
cd tools/ui && npm install && cd ../../
|
||||
fi
|
||||
|
||||
# Check and install git hooks if missing
|
||||
check_and_install_hooks() {
|
||||
local hooks_missing=false
|
||||
|
||||
# Check for required hooks
|
||||
if [ ! -f ".git/hooks/pre-commit" ] || [ ! -f ".git/hooks/pre-push" ] || [ ! -f ".git/hooks/post-push" ]; then
|
||||
if [ ! -f ".git/hooks/pre-commit" ] || [ ! -f ".git/hooks/pre-push" ]; then
|
||||
hooks_missing=true
|
||||
fi
|
||||
|
||||
if [ "$hooks_missing" = true ]; then
|
||||
echo "🔧 Git hooks missing, installing them..."
|
||||
cd tools/ui
|
||||
if bash scripts/install-git-hooks.sh; then
|
||||
if bash "$(dirname "$0")/git-hooks/install.sh"; then
|
||||
echo "✅ Git hooks installed successfully"
|
||||
else
|
||||
echo "⚠️ Failed to install git hooks, continuing anyway..."
|
||||
fi
|
||||
cd ../../
|
||||
else
|
||||
echo "✅ Git hooks already installed"
|
||||
fi
|
||||
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Install git hooks for llama-ui
|
||||
# Copies pre-commit and pre-push hooks into the repo's .git/hooks directory.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
|
||||
HOOKS_DIR="$REPO_ROOT/$(cd "$REPO_ROOT" && git rev-parse --git-path hooks)"
|
||||
|
||||
# Verify package.json exists
|
||||
if [ ! -f "$REPO_ROOT/tools/ui/package.json" ]; then
|
||||
echo "❌ package.json not found in tools/ui"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Installing git hooks for llama-ui..."
|
||||
|
||||
for hook in pre-commit pre-push; do
|
||||
src="$SCRIPT_DIR/${hook}.sh"
|
||||
dst="$HOOKS_DIR/$hook"
|
||||
|
||||
if cp "$src" "$dst" && chmod +x "$dst"; then
|
||||
echo " ✅ $hook"
|
||||
else
|
||||
echo " ❌ Failed to install $hook"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Pre-commit: format (staged) + type-check"
|
||||
echo "Pre-push: lint + test"
|
||||
echo ""
|
||||
echo "Hooks stash unstaged changes temporarily and restore them after."
|
||||
echo "Skip with: git commit --no-verify / git push --no-verify"
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Pre-commit hook for llama-ui
|
||||
# Runs: format (staged files only) + type-check
|
||||
# Stashes unstaged changes temporarily and restores them after.
|
||||
|
||||
# Only run when there are staged changes in tools/ui/
|
||||
if ! git diff --cached --name-only | grep -q "^tools/ui/"; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
REPO_ROOT=$(git rev-parse --show-toplevel)
|
||||
cd "$REPO_ROOT/tools/ui"
|
||||
|
||||
# Check that node_modules exists
|
||||
if [ ! -d "node_modules" ]; then
|
||||
echo "❌ node_modules not found. Run 'npm install' first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Stash unstaged changes in tools/ui/ so they don't interfere
|
||||
stash_name="pi-ui-precommit"
|
||||
git stash push --keep-index -u -m "$stash_name" -- tools/ui/ 2>/dev/null || true
|
||||
|
||||
echo "Running pre-commit checks for llama-ui..."
|
||||
|
||||
# Format only staged files
|
||||
staged_ui=$(git diff --cached --name-only -- tools/ui/)
|
||||
if [ -n "$staged_ui" ]; then
|
||||
echo "$staged_ui" | xargs npx --no-install prettier --write
|
||||
format_ok=$?
|
||||
# Re-stage formatted files
|
||||
git add tools/ui/
|
||||
else
|
||||
format_ok=0
|
||||
fi
|
||||
|
||||
# Type-check the clean tree
|
||||
npm run check
|
||||
check_ok=$?
|
||||
|
||||
# Restore stashed changes
|
||||
if git stash list | grep -q "$stash_name"; then
|
||||
git stash pop 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [ $format_ok -ne 0 ]; then
|
||||
echo "❌ Format failed"
|
||||
exit 1
|
||||
fi
|
||||
if [ $check_ok -ne 0 ]; then
|
||||
echo "❌ Type check failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Pre-commit checks passed"
|
||||
exit 0
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Pre-push hook for llama-ui
|
||||
# Runs: lint + test
|
||||
# Ignores unstaged changes (stashes them temporarily and restores after).
|
||||
|
||||
needs_check=false
|
||||
|
||||
# Read refs from stdin: local_ref local_sha remote_ref remote_sha
|
||||
while read local_ref local_sha remote_ref remote_sha; do
|
||||
# New branch or force-push — always check
|
||||
if [ "$local_sha" = "0000000000000000000000000000000000000000" ] || \
|
||||
[ "$remote_sha" = "0000000000000000000000000000000000000000" ]; then
|
||||
needs_check=true
|
||||
continue
|
||||
fi
|
||||
|
||||
# Check for changes in tools/ui/ between remote and local
|
||||
if git diff --name-only "$remote_sha...$local_sha" -- tools/ui/ | grep -q .; then
|
||||
needs_check=true
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$needs_check" = false ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
REPO_ROOT=$(git rev-parse --show-toplevel)
|
||||
cd "$REPO_ROOT/tools/ui"
|
||||
|
||||
# Check that node_modules exists
|
||||
if [ ! -d "node_modules" ]; then
|
||||
echo "❌ node_modules not found. Run 'npm install' first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Stash unstaged changes so they don't interfere with checks
|
||||
stash_name="pi-ui-prepush"
|
||||
git stash push -u -m "$stash_name" -- tools/ui/ 2>/dev/null || true
|
||||
|
||||
echo "Running pre-push checks for llama-ui..."
|
||||
|
||||
# Lint
|
||||
npm run lint
|
||||
lint_ok=$?
|
||||
|
||||
# Test
|
||||
npm test
|
||||
test_ok=$?
|
||||
|
||||
# Restore stashed changes
|
||||
if git stash list | grep -q "$stash_name"; then
|
||||
git stash pop 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [ $lint_ok -ne 0 ]; then
|
||||
echo "❌ Lint failed"
|
||||
exit 1
|
||||
fi
|
||||
if [ $test_ok -ne 0 ]; then
|
||||
echo "❌ Tests failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Pre-push checks passed"
|
||||
exit 0
|
||||
@@ -1,78 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Script to install pre-commit hook for llama-ui
|
||||
# Pre-commit: formats, checks, and builds the UI app
|
||||
|
||||
REPO_ROOT=$(git rev-parse --show-toplevel)
|
||||
PRE_COMMIT_HOOK="$REPO_ROOT/.git/hooks/pre-commit"
|
||||
|
||||
echo "Installing pre-commit hook for llama-ui..."
|
||||
|
||||
# Create the pre-commit hook
|
||||
cat > "$PRE_COMMIT_HOOK" << 'EOF'
|
||||
#!/bin/bash
|
||||
|
||||
# Check if there are any changes in the tools/ui directory
|
||||
if git diff --cached --name-only | grep -q "^tools/ui/"; then
|
||||
REPO_ROOT=$(git rev-parse --show-toplevel)
|
||||
cd "$REPO_ROOT/tools/ui"
|
||||
|
||||
# Check if package.json exists
|
||||
if [ ! -f "package.json" ]; then
|
||||
echo "Error: package.json not found in tools/ui"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Formatting and checking llama-ui code..."
|
||||
|
||||
# Run the format command
|
||||
npm run format
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: npm run format failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Run the lint command
|
||||
npm run lint
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: npm run lint failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Run the check command
|
||||
npm run check
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: npm run check failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ llama-ui code formatted and checked successfully"
|
||||
|
||||
# Build the llama-ui
|
||||
echo "Building llama-ui..."
|
||||
npm run build
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "❌ npm run build failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ llama-ui built successfully"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
EOF
|
||||
|
||||
# Make hook executable
|
||||
chmod +x "$PRE_COMMIT_HOOK"
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ Git hook installed successfully!"
|
||||
echo " Pre-commit: $PRE_COMMIT_HOOK"
|
||||
echo ""
|
||||
echo "The hook will automatically:"
|
||||
echo " • Format, lint and check llama-ui code before commits"
|
||||
echo " • Build llama-ui"
|
||||
else
|
||||
echo "❌ Failed to make hook executable"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1 @@
|
||||
export const MEGAPIXELS_TO_PIXELS = 1_000_000;
|
||||
@@ -18,6 +18,7 @@ export const SETTINGS_KEYS = {
|
||||
TITLE_GENERATION_USE_FIRST_LINE: 'titleGenerationUseFirstLine',
|
||||
TITLE_GENERATION_USE_LLM: 'titleGenerationUseLLM',
|
||||
TITLE_GENERATION_PROMPT: 'titleGenerationPrompt',
|
||||
MAX_IMAGE_RESOLUTION: 'maxImageMPixels',
|
||||
// Display
|
||||
SHOW_MESSAGE_STATS: 'showMessageStats',
|
||||
SHOW_THOUGHT_IN_PROGRESS: 'showThoughtInProgress',
|
||||
|
||||
@@ -193,6 +193,14 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
|
||||
defaultValue: TITLE_GENERATION.DEFAULT_PROMPT,
|
||||
type: SettingsFieldType.TEXTAREA,
|
||||
section: SETTINGS_SECTION_SLUGS.GENERAL
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.MAX_IMAGE_RESOLUTION,
|
||||
label: 'Maximum image resolution (megapixels)',
|
||||
help: 'Images larger than this will be resized before sending to server. Set to 0 to disable.',
|
||||
defaultValue: 0,
|
||||
type: SettingsFieldType.INPUT,
|
||||
section: SETTINGS_SECTION_SLUGS.GENERAL
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -55,3 +55,6 @@ export const VARIABLE_PREFIX_MODIFIER_REGEX = /:[\d]+$/;
|
||||
|
||||
/** Regex to strip one or more leading slashes */
|
||||
export const LEADING_SLASHES_REGEX = /^\/+/;
|
||||
|
||||
/** Regex to match base64-encoded image URIs (format: "data:image/[media type];base64,[data]")*/
|
||||
export const BASE64_IMAGE_URI_REGEX = /^data:(image\/[a-z0-9.\-+]+);base64/;
|
||||
|
||||
@@ -183,6 +183,10 @@ export enum MimeTypeAudio {
|
||||
MP3 = 'audio/mp3',
|
||||
MP4 = 'audio/mp4',
|
||||
WAV = 'audio/wav',
|
||||
WAVE = 'audio/wave',
|
||||
X_WAV = 'audio/x-wav',
|
||||
X_WAVE = 'audio/x-wave',
|
||||
X_PN_WAV = 'audio/x-pn-wav',
|
||||
WEBM = 'audio/webm',
|
||||
WEBM_OPUS = 'audio/webm;codecs=opus'
|
||||
}
|
||||
|
||||
@@ -5,12 +5,15 @@ import {
|
||||
ATTACHMENT_LABEL_PDF_FILE,
|
||||
ATTACHMENT_LABEL_MCP_PROMPT,
|
||||
ATTACHMENT_LABEL_MCP_RESOURCE,
|
||||
LEGACY_AGENTIC_REGEX
|
||||
LEGACY_AGENTIC_REGEX,
|
||||
SETTINGS_KEYS
|
||||
} from '$lib/constants';
|
||||
import {
|
||||
AttachmentType,
|
||||
ContentPartType,
|
||||
FileTypeAudio,
|
||||
MessageRole,
|
||||
MimeTypeAudio,
|
||||
ReasoningFormat,
|
||||
UrlProtocol
|
||||
} from '$lib/enums';
|
||||
@@ -19,8 +22,31 @@ import type {
|
||||
ApiChatMessageData,
|
||||
ApiChatCompletionToolCall
|
||||
} from '$lib/types/api';
|
||||
import type { DatabaseMessageExtraMcpPrompt, DatabaseMessageExtraMcpResource } from '$lib/types';
|
||||
import type {
|
||||
AudioInputFormat,
|
||||
DatabaseMessageExtraMcpPrompt,
|
||||
DatabaseMessageExtraMcpResource
|
||||
} from '$lib/types';
|
||||
import { modelsStore } from '$lib/stores/models.svelte';
|
||||
import { settingsStore } from '../stores/settings.svelte';
|
||||
import { capImageDataURLSize } from '../utils/cap-img-size';
|
||||
import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size';
|
||||
|
||||
function getAudioInputFormat(mimeType: string): AudioInputFormat {
|
||||
const normalizedMimeType = mimeType.trim().toLowerCase();
|
||||
|
||||
if (
|
||||
normalizedMimeType === MimeTypeAudio.WAV ||
|
||||
normalizedMimeType === MimeTypeAudio.WAVE ||
|
||||
normalizedMimeType === MimeTypeAudio.X_WAV ||
|
||||
normalizedMimeType === MimeTypeAudio.X_WAVE ||
|
||||
normalizedMimeType === MimeTypeAudio.X_PN_WAV
|
||||
) {
|
||||
return FileTypeAudio.WAV;
|
||||
}
|
||||
|
||||
return FileTypeAudio.MP3;
|
||||
}
|
||||
|
||||
export class ChatService {
|
||||
/**
|
||||
@@ -134,26 +160,28 @@ export class ChatService {
|
||||
continueFinalMessage
|
||||
} = options;
|
||||
|
||||
const normalizedMessages: ApiChatMessageData[] = messages
|
||||
.map((msg) => {
|
||||
if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
|
||||
const dbMsg = msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] };
|
||||
const normalizedMessages: ApiChatMessageData[] = (
|
||||
await Promise.all(
|
||||
messages.map((msg) => {
|
||||
if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
|
||||
const dbMsg = msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] };
|
||||
|
||||
return ChatService.convertDbMessageToApiChatMessageData(dbMsg);
|
||||
} else {
|
||||
return msg as ApiChatMessageData;
|
||||
}
|
||||
})
|
||||
.filter((msg) => {
|
||||
// Filter out empty system messages
|
||||
if (msg.role === MessageRole.SYSTEM) {
|
||||
const content = typeof msg.content === 'string' ? msg.content : '';
|
||||
return ChatService.convertDbMessageToApiChatMessageData(dbMsg);
|
||||
} else {
|
||||
return msg as ApiChatMessageData;
|
||||
}
|
||||
})
|
||||
)
|
||||
).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => {
|
||||
// Filter out empty system messages
|
||||
if (msg.role === MessageRole.SYSTEM) {
|
||||
const content = typeof msg.content === 'string' ? msg.content : '';
|
||||
|
||||
return content.trim().length > 0;
|
||||
}
|
||||
return content.trim().length > 0;
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
return true;
|
||||
});
|
||||
|
||||
// Filter out image attachments if the model doesn't support vision
|
||||
if (options.model && !modelsStore.modelSupportsVision(options.model)) {
|
||||
@@ -382,25 +410,27 @@ export class ChatService {
|
||||
excludeReasoning?: boolean,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
const normalizedMessages: ApiChatMessageData[] = messages
|
||||
.map((msg) => {
|
||||
if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
|
||||
return ChatService.convertDbMessageToApiChatMessageData(
|
||||
msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
|
||||
);
|
||||
}
|
||||
const normalizedMessages: ApiChatMessageData[] = (
|
||||
await Promise.all(
|
||||
messages.map((msg) => {
|
||||
if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
|
||||
return ChatService.convertDbMessageToApiChatMessageData(
|
||||
msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
|
||||
);
|
||||
}
|
||||
|
||||
return msg as ApiChatMessageData;
|
||||
})
|
||||
.filter((msg) => {
|
||||
if (msg.role === MessageRole.SYSTEM) {
|
||||
const content = typeof msg.content === 'string' ? msg.content : '';
|
||||
return msg as ApiChatMessageData;
|
||||
})
|
||||
)
|
||||
).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => {
|
||||
if (msg.role === MessageRole.SYSTEM) {
|
||||
const content = typeof msg.content === 'string' ? msg.content : '';
|
||||
|
||||
return content.trim().length > 0;
|
||||
}
|
||||
return content.trim().length > 0;
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
return true;
|
||||
});
|
||||
|
||||
const requestBody: Record<string, unknown> = {
|
||||
messages: normalizedMessages.map((msg: ApiChatMessageData) => {
|
||||
@@ -787,9 +817,9 @@ export class ChatService {
|
||||
* @returns {ApiChatMessageData} object formatted for the chat completion API
|
||||
* @static
|
||||
*/
|
||||
static convertDbMessageToApiChatMessageData(
|
||||
static async convertDbMessageToApiChatMessageData(
|
||||
message: DatabaseMessage & { extra?: DatabaseMessageExtra[] }
|
||||
): ApiChatMessageData {
|
||||
): Promise<ApiChatMessageData> {
|
||||
// Handle tool result messages (role: 'tool')
|
||||
if (message.role === MessageRole.TOOL && message.toolCallId) {
|
||||
return {
|
||||
@@ -867,9 +897,14 @@ export class ChatService {
|
||||
);
|
||||
|
||||
for (const image of imageFiles) {
|
||||
const maxImageResolution = settingsStore.getConfig(SETTINGS_KEYS.MAX_IMAGE_RESOLUTION);
|
||||
let base64Url = image.base64Url;
|
||||
if (maxImageResolution > 1 / MEGAPIXELS_TO_PIXELS) {
|
||||
base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution);
|
||||
}
|
||||
contentParts.push({
|
||||
type: ContentPartType.IMAGE_URL,
|
||||
image_url: { url: image.base64Url }
|
||||
image_url: { url: base64Url }
|
||||
});
|
||||
}
|
||||
|
||||
@@ -883,7 +918,7 @@ export class ChatService {
|
||||
type: ContentPartType.INPUT_AUDIO,
|
||||
input_audio: {
|
||||
data: audio.base64Data,
|
||||
format: audio.mimeType.includes('wav') ? 'wav' : 'mp3'
|
||||
format: getAudioInputFormat(audio.mimeType)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -416,21 +416,23 @@ class AgenticStore {
|
||||
|
||||
console.log(`[AgenticStore] Starting agentic flow with ${tools.length} tools`);
|
||||
|
||||
const normalizedMessages: ApiChatMessageData[] = messages
|
||||
.map((msg) => {
|
||||
if ('id' in msg && 'convId' in msg && 'timestamp' in msg)
|
||||
return ChatService.convertDbMessageToApiChatMessageData(
|
||||
msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
|
||||
);
|
||||
return msg as ApiChatMessageData;
|
||||
})
|
||||
.filter((msg) => {
|
||||
if (msg.role === MessageRole.SYSTEM) {
|
||||
const content = typeof msg.content === 'string' ? msg.content : '';
|
||||
return content.trim().length > 0;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
const normalizedMessages: ApiChatMessageData[] = (
|
||||
await Promise.all(
|
||||
messages.map((msg) => {
|
||||
if ('id' in msg && 'convId' in msg && 'timestamp' in msg)
|
||||
return ChatService.convertDbMessageToApiChatMessageData(
|
||||
msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
|
||||
);
|
||||
return msg as ApiChatMessageData;
|
||||
})
|
||||
)
|
||||
).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => {
|
||||
if (msg.role === MessageRole.SYSTEM) {
|
||||
const content = typeof msg.content === 'string' ? msg.content : '';
|
||||
return content.trim().length > 0;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
this.updateSession(conversationId, {
|
||||
isRunning: true,
|
||||
|
||||
Vendored
+4
-2
@@ -1,6 +1,8 @@
|
||||
import type { ContentPartType, ServerModelStatus, ServerRole } from '$lib/enums';
|
||||
import type { ContentPartType, FileTypeAudio, ServerModelStatus, ServerRole } from '$lib/enums';
|
||||
import type { ChatMessagePromptProgress, ChatRole } from './chat';
|
||||
|
||||
export type AudioInputFormat = FileTypeAudio.WAV | FileTypeAudio.MP3;
|
||||
|
||||
export interface ApiChatCompletionToolFunction {
|
||||
name: string;
|
||||
description?: string;
|
||||
@@ -20,7 +22,7 @@ export interface ApiChatMessageContentPart {
|
||||
};
|
||||
input_audio?: {
|
||||
data: string;
|
||||
format: 'wav' | 'mp3';
|
||||
format: AudioInputFormat;
|
||||
};
|
||||
input_video?: {
|
||||
data: string;
|
||||
|
||||
@@ -29,7 +29,8 @@ export type {
|
||||
ApiRouterModelsStatusResponse,
|
||||
ApiRouterModelsListResponse,
|
||||
ApiRouterModelsUnloadRequest,
|
||||
ApiRouterModelsUnloadResponse
|
||||
ApiRouterModelsUnloadResponse,
|
||||
AudioInputFormat
|
||||
} from './api';
|
||||
|
||||
// Chat types
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size';
|
||||
import { BASE64_IMAGE_URI_REGEX } from '$lib/constants/uri-template';
|
||||
import { MimeTypeImage } from '$lib/enums';
|
||||
|
||||
/**
|
||||
* Converts an Image base64 data URL to another Image data URL with capped dimensions to reduce file size.
|
||||
* @param base64UrlImage - The Image base64 data URL to convert
|
||||
* @param maxMegapixels - The maximum image size in megapixels for the output Image
|
||||
* @returns Promise resolving to Image data URL
|
||||
*/
|
||||
export function capImageDataURLSize(
|
||||
base64UrlImage: string,
|
||||
maxMegapixels: number
|
||||
): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
|
||||
const mimeMatch = base64UrlImage.match(BASE64_IMAGE_URI_REGEX);
|
||||
|
||||
if (!mimeMatch) {
|
||||
return reject(new Error('Invalid data URL format.'));
|
||||
}
|
||||
|
||||
const mimeType = mimeMatch[1] as MimeTypeImage;
|
||||
|
||||
if (!Object.values(MimeTypeImage).includes(mimeType)) {
|
||||
return reject(new Error(`Unsupported image MIME type: ${mimeType}`));
|
||||
}
|
||||
|
||||
const img = new Image();
|
||||
|
||||
img.onload = () => {
|
||||
try {
|
||||
const canvas = document.createElement('canvas');
|
||||
const ctx = canvas.getContext('2d');
|
||||
|
||||
if (!ctx) {
|
||||
throw new Error('Failed to get 2D canvas context.');
|
||||
}
|
||||
|
||||
const targetWidth = img.naturalWidth;
|
||||
const targetHeight = img.naturalHeight;
|
||||
const totalPixels = targetWidth * targetHeight;
|
||||
const maxPixels = Math.floor(maxMegapixels * MEGAPIXELS_TO_PIXELS);
|
||||
|
||||
if (maxPixels > 0 && totalPixels > maxPixels) {
|
||||
const scaleFactor = Math.sqrt(maxPixels / totalPixels);
|
||||
canvas.width = Math.floor(targetWidth * scaleFactor);
|
||||
canvas.height = Math.floor(targetHeight * scaleFactor);
|
||||
} else {
|
||||
return resolve(base64UrlImage);
|
||||
}
|
||||
|
||||
ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
|
||||
resolve(canvas.toDataURL(mimeType));
|
||||
} catch (err) {
|
||||
reject(err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
};
|
||||
|
||||
img.onerror = () => {
|
||||
reject(new Error('Failed to load image.'));
|
||||
};
|
||||
|
||||
img.src = base64UrlImage;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const errorMessage = `Error resizing image: ${message}`;
|
||||
console.error(errorMessage, error);
|
||||
reject(new Error(errorMessage));
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -18,8 +18,12 @@ import {
|
||||
MimeTypeText
|
||||
} from '$lib/enums';
|
||||
|
||||
function normalizeMimeType(mimeType: string): string {
|
||||
return mimeType.trim().toLowerCase();
|
||||
}
|
||||
|
||||
export function getFileTypeCategory(mimeType: string): FileTypeCategory | null {
|
||||
switch (mimeType) {
|
||||
switch (normalizeMimeType(mimeType)) {
|
||||
// Images
|
||||
case MimeTypeImage.JPEG:
|
||||
case MimeTypeImage.PNG:
|
||||
@@ -33,6 +37,10 @@ export function getFileTypeCategory(mimeType: string): FileTypeCategory | null {
|
||||
case MimeTypeAudio.MP3:
|
||||
case MimeTypeAudio.MP4:
|
||||
case MimeTypeAudio.WAV:
|
||||
case MimeTypeAudio.WAVE:
|
||||
case MimeTypeAudio.X_WAV:
|
||||
case MimeTypeAudio.X_WAVE:
|
||||
case MimeTypeAudio.X_PN_WAV:
|
||||
case MimeTypeAudio.WEBM:
|
||||
case MimeTypeAudio.WEBM_OPUS:
|
||||
return FileTypeCategory.AUDIO;
|
||||
|
||||
Reference in New Issue
Block a user