Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-20 01:31:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/build-openvino.yml
#	.github/workflows/server-sanitize.yml
#	AUTHORS
#	README.md
#	app/llama.cpp
#	ci/run.sh
#	common/build-info.h
#	docs/backend/SYCL.md
#	docs/build.md
#	docs/ops.md
#	ggml/CMakeLists.txt
#	ggml/src/ggml-cuda/CMakeLists.txt
#	ggml/src/ggml-musa/CMakeLists.txt
#	ggml/src/ggml-opencl/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-opencl/kernels/cvt.cl
#	ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl
#	ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl
#	ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl
#	ggml/src/ggml-opencl/kernels/rms_norm.cl
#	ggml/src/ggml-sycl/binbcast.cpp
#	ggml/src/ggml-sycl/binbcast.hpp
#	ggml/src/ggml-sycl/common.hpp
#	ggml/src/ggml-sycl/fattn.cpp
#	ggml/src/ggml-sycl/fusion.cpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-sycl/norm.cpp
#	ggml/src/ggml-sycl/norm.hpp
#	scripts/snapdragon/build.py
#	scripts/snapdragon/qdc/run_qdc_jobs.py
#	scripts/snapdragon/qdc/tests/linux/run_linux.sh
#	scripts/snapdragon/qdc/tests/run_backend_ops_posix.py
#	scripts/snapdragon/qdc/tests/run_bench_tests_posix.py
#	scripts/snapdragon/qdc/tests/utils.py
#	scripts/snapdragon/run.py
#	src/CMakeLists.txt
#	src/llama.cpp
#	tests/test-backend-ops.cpp
#	tests/test-json-schema-to-grammar.cpp
This commit is contained in:
Concedo
2026-09-04 16:59:17 +08:00
parent a3e6006530 4cbe8b070b
commit ccaed41be0
40 changed files with 614 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/mtmd-audio.cpp
+11 -11
View File
@@ -549,7 +549,7 @@ void mtmd_audio_preprocessor_whisper::initialize() {
bool mtmd_audio_preprocessor_whisper::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
std::vector<mtmd_audio_mel> & output) const {
if (n_samples == 0) {
// empty audio
return false;
@@ -637,7 +637,7 @@ void mtmd_audio_preprocessor_qwen3a::initialize() {
bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
std::vector<mtmd_audio_mel> & output) const {
if (n_samples == 0) {
return false;
}
@@ -739,7 +739,7 @@ void mtmd_audio_preprocessor_dots3note::initialize() {
bool mtmd_audio_preprocessor_dots3note::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
std::vector<mtmd_audio_mel> & output) const {
if (n_samples == 0) {
return false;
}
@@ -839,7 +839,7 @@ void mtmd_audio_preprocessor_mimo_audio::initialize() {
bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
std::vector<mtmd_audio_mel> & output) const {
if (n_samples == 0) {
return false;
}
@@ -898,7 +898,7 @@ void mtmd_audio_preprocessor_qwen3tts_spk::initialize() {
bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
std::vector<mtmd_audio_mel> & output) const {
if (n_samples == 0) {
return false;
}
@@ -955,7 +955,7 @@ void mtmd_audio_preprocessor_conformer::initialize() {
bool mtmd_audio_preprocessor_conformer::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
std::vector<mtmd_audio_mel> & output) const {
// empty audio
if (n_samples == 0) {
return false;
@@ -1003,7 +1003,7 @@ void mtmd_audio_preprocessor_granite_speech::initialize() {
bool mtmd_audio_preprocessor_granite_speech::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
std::vector<mtmd_audio_mel> & output) const {
if (n_samples == 0) {
return false;
}
@@ -1117,7 +1117,7 @@ void mtmd_audio_preprocessor_gemma4a::initialize() {
bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
std::vector<mtmd_audio_mel> & output) const {
if (n_samples == 0) {
return false;
}
@@ -1266,7 +1266,7 @@ void mtmd_audio_preprocessor_parakeet::initialize() {
bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples,
size_t n_samples_in,
std::vector<mtmd_audio_mel> & output) {
std::vector<mtmd_audio_mel> & output) const {
if (n_samples_in == 0) {
return false;
}
@@ -1386,7 +1386,7 @@ void mtmd_audio_preprocessor_gemma4ua::initialize() {
bool mtmd_audio_preprocessor_gemma4ua::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
std::vector<mtmd_audio_mel> & output) const {
if (n_samples == 0) {
return false;
}
@@ -1527,7 +1527,7 @@ std::vector<float> mtmd_audio_streaming_istft::flush() {
bool mtmd_audio_preprocessor_pockettts::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
std::vector<mtmd_audio_mel> & output) const {
// the encoder needs whole frames, see pad_for_conv1d() in the reference
const int64_t frame_size = (int64_t) hparams.mimi_downsample * 120;
if (n_samples == 0 || frame_size <= 0) {
tools/mtmd/mtmd-audio.h
+12 -12
View File
@@ -57,13 +57,13 @@ struct mtmd_audio_preprocessor {
virtual ~mtmd_audio_preprocessor() = default;
virtual void initialize() = 0; // NOT thread-safe
virtual bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) = 0;
virtual bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const = 0;
};
struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
private:
mtmd_audio_cache cache;
@@ -72,7 +72,7 @@ struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
private:
mtmd_audio_cache cache;
@@ -81,7 +81,7 @@ struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_granite_speech(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
private:
mtmd_audio_cache cache;
@@ -90,7 +90,7 @@ struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
private:
mtmd_audio_cache cache;
@@ -99,13 +99,13 @@ struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_gemma4ua : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_gemma4ua(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
};
struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_qwen3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
private:
mtmd_audio_cache cache;
@@ -114,7 +114,7 @@ struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
private:
mtmd_audio_cache cache;
@@ -123,7 +123,7 @@ struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
private:
mtmd_audio_cache cache;
@@ -132,7 +132,7 @@ struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
private:
mtmd_audio_cache cache;
@@ -142,13 +142,13 @@ struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_pockettts(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override {}
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
};
struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { }
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
private:
mtmd_audio_cache cache;
tools/mtmd/mtmd-helper.cpp
+9 -9
View File
@@ -370,11 +370,11 @@ static bool is_webp_file(const unsigned char * buf, size_t len) {
}
#ifdef MTMD_VIDEO
static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,
static mtmd_bitmap * decode_webp_with_ffmpeg(const mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,
const mtmd_helper_video_init_params & params);
#endif
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder,
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder,
mtmd_helper_init_opt opt) {
// calculate the hash if needed
std::string id;
@@ -460,7 +460,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,
return {nullptr, nullptr};
}
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder,
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(const mtmd_context * ctx, const char * fname, bool placeholder,
mtmd_helper_init_opt opt) {
#ifdef _WIN32
int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
@@ -505,7 +505,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx,
return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder, opt);
}
bool mtmd_helper_support_video(mtmd_context * ctx) {
bool mtmd_helper_support_video(const mtmd_context * ctx) {
#ifdef MTMD_VIDEO
return mtmd_support_vision(ctx);
#else
@@ -521,7 +521,7 @@ bool mtmd_helper_support_video(mtmd_context * ctx) {
#ifdef MTMD_VIDEO
struct mtmd_helper_video {
mtmd_context * mctx;
const mtmd_context * mctx;
std::string path;
std::vector<uint8_t> input_buf; // non-empty when initialized from buffer
std::string ffmpeg_bin;
@@ -887,7 +887,7 @@ static std::string video_resolve_bin(const char * bin_dir, const char * name) {
}
#ifdef MTMD_VIDEO
static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,
static mtmd_bitmap * decode_webp_with_ffmpeg(const mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,
const mtmd_helper_video_init_params & params) {
mtmd_helper_video vctx;
vctx.mctx = mctx;
@@ -914,7 +914,7 @@ static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned
#endif
mtmd_helper_video * mtmd_helper_video_init(
mtmd_context * mctx,
const mtmd_context * mctx,
const char * path,
mtmd_helper_video_init_params params) {
#ifdef MTMD_VIDEO
@@ -949,7 +949,7 @@ mtmd_helper_video * mtmd_helper_video_init(
}
mtmd_helper_video * mtmd_helper_video_init_from_buf(
mtmd_context * mctx,
const mtmd_context * mctx,
const unsigned char * buf, size_t len,
mtmd_helper_video_init_params params) {
#ifdef MTMD_VIDEO
@@ -1017,7 +1017,7 @@ int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx,
#endif
}
bool mtmd_helper_model_can_chat(llama_context * lctx, mtmd_context * mctx) {
bool mtmd_helper_model_can_chat(const llama_context * lctx, const mtmd_context * mctx) {
if (!mctx) {
return true;
}
tools/mtmd/mtmd-helper.h
+6 -6
View File
@@ -46,7 +46,7 @@ MTMD_API struct mtmd_helper_init_opt mtmd_helper_init_opt_default(void);
MTMD_API void mtmd_helper_log_set(ggml_log_callback log_callback, void * user_data);
// Returns true if this build includes video support (MTMD_VIDEO was ON at compile time).
MTMD_API bool mtmd_helper_support_video(mtmd_context * ctx);
MTMD_API bool mtmd_helper_support_video(const mtmd_context * ctx);
struct mtmd_helper_bitmap_wrapper {
mtmd_bitmap * bitmap;
@@ -58,7 +58,7 @@ struct mtmd_helper_bitmap_wrapper {
// returns nullptr on failure
// this function is thread-safe
MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(
mtmd_context * ctx,
const mtmd_context * ctx,
const char * fname,
bool placeholder,
struct mtmd_helper_init_opt opt);
@@ -75,7 +75,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(
// returns nullptr on failure
// this function is thread-safe
MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(
mtmd_context * ctx,
const mtmd_context * ctx,
const unsigned char * buf, size_t len,
bool placeholder,
struct mtmd_helper_init_opt opt);
@@ -153,7 +153,7 @@ struct mtmd_helper_video_info {
// returns NULL on failure (ffprobe not found, file unreadable, etc.)
MTMD_API mtmd_helper_video * mtmd_helper_video_init(
struct mtmd_context * mctx,
const struct mtmd_context * mctx,
const char * path,
struct mtmd_helper_video_init_params params);
@@ -162,7 +162,7 @@ MTMD_API mtmd_helper_video * mtmd_helper_video_init(
// Note: pipe input is not seekable, so seeking will use output-side seeking
// (ffmpeg decodes and discards frames up to the target position).
MTMD_API mtmd_helper_video * mtmd_helper_video_init_from_buf(
struct mtmd_context * mctx,
const struct mtmd_context * mctx,
const unsigned char * buf, size_t len,
struct mtmd_helper_video_init_params params);
MTMD_API void mtmd_helper_video_free(mtmd_helper_video * ctx);
@@ -177,7 +177,7 @@ MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx,
char ** out_text);
// return true if model can be used for chat
MTMD_API bool mtmd_helper_model_can_chat(struct llama_context * lctx, struct mtmd_context * mctx);
MTMD_API bool mtmd_helper_model_can_chat(const struct llama_context * lctx, const struct mtmd_context * mctx);
//
// Audio generation helpers
tools/mtmd/mtmd-image.cpp
+27 -27
View File
@@ -485,7 +485,7 @@ private:
// mtmd_image_preprocessor_llava_uhd
//
mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_image_u8 & img) const {
const clip_image_size original_size = img.get_size();
auto const inst = get_slice_instructions(original_size);
auto sliced = slice_image(img, inst);
@@ -499,7 +499,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_
return output;
}
mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_llava_uhd::get_slice_instructions(const clip_image_size & original_size) {
mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_llava_uhd::get_slice_instructions(const clip_image_size & original_size) const {
mtmd_image_preprocessor_llava_uhd::slice_instructions res;
// align slices by patch_size * n_merge so an integer number of merger output tokens fits per slice
const int n_merge = hparams.n_merge;
@@ -604,7 +604,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_ll
return res;
}
mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uhd::slice_image(const clip_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst) {
mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uhd::slice_image(const clip_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst) const {
slice_output output;
// resize to overview size
@@ -636,7 +636,7 @@ mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uh
return output;
}
clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale) {
clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale) const {
int width = original_size.width;
int height = original_size.height;
if ((width * height > scale_resolution * scale_resolution) || allow_upscale) {
@@ -650,7 +650,7 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_im
return res;
}
clip_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) {
clip_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) const {
float scale_width = static_cast<float>(target_max.width) / orig.width;
float scale_height = static_cast<float>(target_max.height) / orig.height;
float scale = std::min(scale_width, scale_height);
@@ -660,7 +660,7 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio(
};
}
clip_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions) {
clip_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions) const {
clip_image_size best_fit;
int min_wasted_area = std::numeric_limits<int>::max();
int max_effective_resolution = 0;
@@ -684,11 +684,11 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const
return best_fit;
}
int mtmd_image_preprocessor_llava_uhd::ensure_divide(int length, int patch_size) {
int mtmd_image_preprocessor_llava_uhd::ensure_divide(int length, int patch_size) const {
return std::max(static_cast<int>(std::round(static_cast<float>(length) / patch_size) * patch_size), patch_size);
}
clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale) {
clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale) const {
int width = original_size.width;
int height = original_size.height;
int grid_x = grid.width;
@@ -711,7 +711,7 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_im
return refine_size;
}
clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) {
clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const {
std::vector<int> candidate_split_grids_nums;
for (int i : {multiple - 1, multiple, multiple + 1}) {
if (i == 1 || i > max_slice_nums) {
@@ -747,7 +747,7 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_s
// mtmd_image_preprocessor_fixed_size
//
mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip_image_u8 & img) const {
clip_image_u8 resized_image;
int sz = hparams.image_size;
img_tool::resize(img, resized_image, {sz, sz},
@@ -763,7 +763,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip
// mtmd_image_preprocessor_dyn_size
//
mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const {
GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0);
clip_image_u8 resized_image;
const clip_image_size original_size = img.get_size();
@@ -790,7 +790,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i
// mtmd_image_preprocessor_longest_edge
//
mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img) const {
GGML_ASSERT(hparams.image_longest_edge > 0);
clip_image_u8 resized_image;
const clip_image_size original_size = img.get_size();
@@ -817,7 +817,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const cl
// mtmd_image_preprocessor_minicpmv
//
mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const clip_image_size & original_size) {
mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const clip_image_size & original_size) const {
if (hparams.n_merge == 2) {
const int slice_size = hparams.image_size;
const float ratio = (float)original_size.width * original_size.height / (slice_size * slice_size);
@@ -837,7 +837,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_mi
// mtmd_image_preprocessor_lfm2
//
mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image_u8 & img) const {
auto const inst = get_slice_instructions(img.get_size());
if (!inst.slices.empty()) {
return mtmd_image_preprocessor_llava_uhd::preprocess(img);
@@ -868,7 +868,7 @@ bool mtmd_image_preprocessor_lfm2::should_tile(
static_cast<double>(hparams.image_max_pixels) * max_pixels_tolerance;
}
mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) {
mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) const {
mtmd_image_preprocessor_llava_uhd::slice_instructions inst;
const int align_size = hparams.patch_size * hparams.n_merge;
inst.overview_size = img_tool::calc_size_preserved_ratio(
@@ -914,7 +914,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf
clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(
float aspect_ratio,
const std::vector<clip_image_size> & target_ratios,
int width, int height) {
int width, int height) const {
float best_ratio_diff = std::numeric_limits<float>::max();
clip_image_size best_ratio = {1, 1};
const float area = static_cast<float>(width * height);
@@ -935,7 +935,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(
return best_ratio;
}
std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios() {
std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios() const {
std::vector<clip_image_size> ratios;
for (int n = min_tiles; n <= max_tiles; n++) {
for (int w = 1; w <= n; w++) {
@@ -961,7 +961,7 @@ std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios() {
return ratios;
}
clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(int height, int width) {
clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(int height, int width) const {
const float aspect_ratio = static_cast<float>(width) / height;
const auto ratios = get_target_ratios();
return find_closest_aspect_ratio(aspect_ratio, ratios, width, height);
@@ -971,7 +971,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(int height, int wi
// mtmd_image_preprocessor_idefics3
//
mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_image_u8 & img) const {
// The refined size has two steps:
// 1. Resize w/ aspect-ratio preserving such that the longer side is
// the preprocessor longest size
@@ -1071,7 +1071,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
// mtmd_image_preprocessor_internvl
//
mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img) const {
GGML_ASSERT(!hparams.image_res_candidates.empty());
const clip_image_size original_size = img.get_size();
auto const inst = get_slice_instructions(original_size);
@@ -1206,7 +1206,7 @@ void mtmd_image_preprocessor_deepseek4v::safe_resize(int height, int width, int
}
// ref: load_image()
mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip_image_u8 & img) const {
mtmd_image_preproc_out out;
const int p = hparams.patch_size;
@@ -1244,7 +1244,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip
return out;
}
mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) const {
mtmd_image_preproc_out output;
int grid_w = 0;
int grid_h = 0;
@@ -1320,7 +1320,7 @@ void mtmd_image_preprocessor_step3vl::img_u8_resize_bilinear_to_f32(
int target_width,
int target_height,
const float mean[3],
const float std[3]) {
const float std[3]) const {
const auto src_size = src.get_size();
if (src_size.width == target_width && src_size.height == target_height) {
dst.from_u8(src);
@@ -1519,7 +1519,7 @@ mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step
return instructions;
}
mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_image_u8 & img) const {
clip_image_u8 prepared = prepare_image(img, hparams);
const auto instructions = build_slice_instructions(hparams, prepared.get_size());
@@ -1573,7 +1573,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_im
// mtmd_image_preprocessor_youtuvl
//
mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_image_u8 & img) const {
const int patch_size = hparams.patch_size; // typically 16
const int merge_size = hparams.n_merge; // typically 2
const int align_size = patch_size * merge_size; // 32
@@ -1622,7 +1622,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_im
return output;
}
mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img) const {
GGML_ASSERT(!hparams.image_res_candidates.empty());
const clip_image_size orig_size = img.get_size();
@@ -1717,7 +1717,7 @@ static clip_image_size muse_glimmer_grid_size(int img_w, int img_h, int patch_hw
return clip_image_size{ best_npw * patch_hw, best_nph * patch_hw };
}
mtmd_image_preproc_out mtmd_image_preprocessor_muse_glimmer::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out mtmd_image_preprocessor_muse_glimmer::preprocess(const clip_image_u8 & img) const {
const int patch_hw = hparams.patch_size * hparams.n_merge;
const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
GGML_ASSERT(patch_area > 0 && hparams.image_max_pixels > 0);
tools/mtmd/mtmd-image.h
+28 -28
View File
@@ -33,7 +33,7 @@ struct mtmd_image_preprocessor {
mtmd_image_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}
virtual ~mtmd_image_preprocessor() = default;
virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) = 0;
virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const = 0;
};
/**
@@ -59,7 +59,7 @@ struct mtmd_image_preprocessor {
*/
struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor {
mtmd_image_preprocessor_llava_uhd(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
struct slice_coordinates {
int x;
@@ -74,16 +74,16 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor {
std::vector<slice_coordinates> slices;
};
virtual slice_instructions get_slice_instructions(const clip_image_size & original_size);
virtual slice_instructions get_slice_instructions(const clip_image_size & original_size) const;
struct slice_output {
clip_image_u8 overview;
std::vector<clip_image_u8> slices;
};
slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst);
slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst) const;
protected:
clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false);
clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false) const;
/**
* Selects the best resolution from a list of possible resolutions based on the original size.
@@ -100,19 +100,19 @@ protected:
* @param possible_resolutions A list of possible resolutions
* @return The best fit resolution
*/
clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions);
clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions) const;
private:
clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max);
int ensure_divide(int length, int patch_size);
clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false);
clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio);
clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) const;
int ensure_divide(int length, int patch_size) const;
clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false) const;
clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const;
};
// downscale or upscale the input image to fixed size
struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor {
mtmd_image_preprocessor_fixed_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
};
// resize image to multiple of patch_size*n_merge, while preserving aspect ratio
@@ -120,19 +120,19 @@ struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor {
// this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc
struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {
mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
};
// similar to mtmd_image_preprocessor_dyn_size, but resize the image to have longest edge equal to hparams.image_longest_edge, while preserving aspect ratio
struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor {
mtmd_image_preprocessor_longest_edge(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
};
// ref: inference/image_processor.py in the HF repo (DeepSeek-V4-Flash-Vision)
struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor {
mtmd_image_preprocessor_deepseek4v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
private:
struct grid_info {
@@ -148,7 +148,7 @@ private:
// custom llava-uhd slicing logic for MiniCPM-V
struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd {
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
slice_instructions get_slice_instructions(const clip_image_size & original_size) override;
slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;
};
// custom llava-uhd slicing logic for LFM2
@@ -161,8 +161,8 @@ struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {
static constexpr int tile_size = 512;
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
slice_instructions get_slice_instructions(const clip_image_size & original_size) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;
static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size);
@@ -170,19 +170,19 @@ private:
clip_image_size find_closest_aspect_ratio(
float aspect_ratio,
const std::vector<clip_image_size> & target_ratios,
int width, int height);
std::vector<clip_image_size> get_target_ratios();
clip_image_size get_grid_layout(int height, int width);
int width, int height) const;
std::vector<clip_image_size> get_target_ratios() const;
clip_image_size get_grid_layout(int height, int width) const;
};
struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd {
mtmd_image_preprocessor_idefics3(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
};
struct mtmd_image_preprocessor_internvl : mtmd_image_preprocessor_llava_uhd {
mtmd_image_preprocessor_internvl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
};
// DeepSeek-OCR (v1/v2) global view + optional local tile grid
@@ -194,7 +194,7 @@ struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor {
tile_size(hparams.preproc_tile_size),
min_tiles(hparams.preproc_min_tiles),
max_tiles(hparams.preproc_max_tiles) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
private:
bool fuse_row; // v1 fuses a tile-row into one image; v2 keeps tiles separate
@@ -214,7 +214,7 @@ private:
// ref: https://huggingface.co/stepfun-ai/Step3-VL-10B/blob/main/processing_step3.py
struct mtmd_image_preprocessor_step3vl : mtmd_image_preprocessor_llava_uhd {
mtmd_image_preprocessor_step3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
static slice_instructions build_slice_instructions(const clip_hparams & params, const clip_image_size & prepared_size);
private:
@@ -230,7 +230,7 @@ private:
int target_width,
int target_height,
const float mean[3],
const float std[3]);
const float std[3]) const;
static int get_image_longest_edge(const clip_hparams & params);
static int determine_window_size(const clip_hparams & params, int longer, int shorter);
static int calc_crop_extent(int length, int window_size);
@@ -241,17 +241,17 @@ private:
struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor {
mtmd_image_preprocessor_youtuvl(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
};
// llava-next "anyres": stacks the overview and all tiles into one image, assembled by clip in a single graph
struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd {
mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
};
// pick the patch grid closest to the input aspect ratio under the per-image token cap, stretch-resize.
struct mtmd_image_preprocessor_muse_glimmer : mtmd_image_preprocessor {
mtmd_image_preprocessor_muse_glimmer(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
};
tools/mtmd/mtmd.cpp
+10 -10
View File
@@ -1117,7 +1117,7 @@ std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::
}
struct mtmd_tokenizer {
mtmd_context * ctx;
const mtmd_context * ctx;
std::string input_text; // note: can contain null bytes; do not use c_str()
bool add_special;
@@ -1140,9 +1140,9 @@ struct mtmd_tokenizer {
}
}
mtmd_tokenizer(mtmd_context * ctx,
mtmd_tokenizer(const mtmd_context * ctx,
const mtmd_input_text * text,
const mtmd_bitmap ** bmps,
const mtmd_bitmap * const * bmps,
size_t n_bitmaps) : ctx(ctx) {
add_special = text->add_special;
parse_special = text->parse_special;
@@ -1177,8 +1177,8 @@ struct mtmd_tokenizer {
expand_lazy_bitmaps();
}
mtmd_tokenizer(mtmd_context * ctx,
const mtmd_input_part ** input_parts,
mtmd_tokenizer(const mtmd_context * ctx,
const mtmd_input_part * const * input_parts,
size_t n_parts,
bool add_special) : ctx(ctx) {
this->add_special = add_special;
@@ -1733,10 +1733,10 @@ struct mtmd_tokenizer {
}
};
int32_t mtmd_tokenize(mtmd_context * ctx,
int32_t mtmd_tokenize(const mtmd_context * ctx,
mtmd_input_chunks * output,
const mtmd_input_text * text,
const mtmd_bitmap ** bitmaps,
const mtmd_bitmap * const * bitmaps,
size_t n_bitmaps) {
try {
mtmd_tokenizer tokenizer(ctx, text, bitmaps, n_bitmaps);
@@ -1747,9 +1747,9 @@ int32_t mtmd_tokenize(mtmd_context * ctx,
}
}
int32_t mtmd_tokenize_from_parts(mtmd_context * ctx,
int32_t mtmd_tokenize_from_parts(const mtmd_context * ctx,
mtmd_input_chunks * output,
const mtmd_input_part ** parts,
const mtmd_input_part * const * parts,
size_t n_parts,
bool add_special) {
for (size_t i = 0; i < n_parts; i++) {
@@ -2271,7 +2271,7 @@ void mtmd_bitmap_set_mergeable(mtmd_bitmap * bitmap, bool mergeable) {
bitmap->mergeable = mergeable;
}
mtmd_bitmap * mtmd_bitmap_init_lazy(mtmd_context * ctx,
mtmd_bitmap * mtmd_bitmap_init_lazy(const mtmd_context * ctx,
const char * id,
void * user_data,
mtmd_bitmap_lazy_callback callback) {
tools/mtmd/mtmd.h
+5 -5
View File
@@ -211,7 +211,7 @@ typedef int(* mtmd_bitmap_lazy_callback)(
mtmd_bitmap ** out_bitmap,
char ** out_text);
MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(mtmd_context * ctx,
MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(const mtmd_context * ctx,
const char * id, // usually set to file hash
void * user_data,
mtmd_bitmap_lazy_callback callback);
@@ -299,10 +299,10 @@ MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_im
// 0 on success
// 1 on number of bitmaps not matching the number of markers
// 2 on media preprocessing error
MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx,
MTMD_API int32_t mtmd_tokenize(const mtmd_context * ctx,
mtmd_input_chunks * output,
const mtmd_input_text * text,
const mtmd_bitmap ** bitmaps,
const mtmd_bitmap * const * bitmaps,
size_t n_bitmaps);
// same as mtmd_tokenize(), but takes an array of mtmd_input_part
@@ -311,9 +311,9 @@ MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx,
// - when you want to control parse_special for each text part
// note: per-part add_special will be ignored
// return 1 if a part has both text and bitmap set (or neither)
MTMD_API int32_t mtmd_tokenize_from_parts(mtmd_context * ctx,
MTMD_API int32_t mtmd_tokenize_from_parts(const mtmd_context * ctx,
mtmd_input_chunks * output,
const mtmd_input_part ** parts,
const mtmd_input_part * const * parts,
size_t n_parts,
bool add_special);
tools/server/tests/conftest.py
+16 -2
View File
@@ -1,7 +1,17 @@
import os
import pytest
from filelock import FileLock
from utils import *
@pytest.fixture(scope="session", autouse=True)
def configure_worker_port(request):
worker_id = getattr(request.config, "workerinput", {}).get("workerid", "master")
if worker_id != "master":
worker_num = int(worker_id[2:])
os.environ["PORT"] = str(8080 + worker_num * 10)
# ref: https://stackoverflow.com/questions/22627659/run-code-before-and-after-each-test-in-py-test
@pytest.fixture(autouse=True)
def stop_server_after_each_test():
@@ -16,6 +26,10 @@ def stop_server_after_each_test():
@pytest.fixture(scope="session", autouse=True)
def load_server_presets():
def load_server_presets(configure_worker_port, tmp_path_factory):
# this will be run once per test session, before any tests
ServerPreset.load_all()
# serialize model downloads across parallel workers.
root_tmp_dir = tmp_path_factory.getbasetemp().parent
with FileLock(str(root_tmp_dir / "load_all.lock")):
ServerPreset.load_all()
tools/server/tests/requirements.txt
+2
View File
@@ -1,5 +1,7 @@
aiohttp~=3.9.3
pytest~=8.3.3
pytest-xdist~=3.6
filelock~=3.16
numpy~=1.26.4
openai~=2.14.0
prometheus-client~=0.20.0
tools/server/tests/tests.sh
+5 -3
View File
@@ -6,13 +6,15 @@ cd $SCRIPT_DIR
set -eu
WORKERS="${PYTEST_WORKERS:-auto}"
if [ $# -lt 1 ]
then
if [[ "${SLOW_TESTS:-0}" == 1 ]]; then
pytest --durations=30 -v -x
pytest --durations=30 -v -x -n "${WORKERS}" --dist=worksteal
else
pytest --durations=30 -v -x -m "not slow"
pytest --durations=30 -v -x -n "${WORKERS}" --dist=worksteal -m "not slow"
fi
else
pytest --durations=30 "$@"
pytest --durations=30 -n "${WORKERS}" --dist=worksteal "$@"
fi
tools/server/tests/unit/test_compat_anthropic.py
-3
View File
@@ -21,7 +21,6 @@ def create_server():
global server
server = ServerPreset.tinyllama2()
server.model_alias = "tinyllama-2-anthropic"
server.server_port = 8082
server.n_slots = 1
server.n_ctx = 8192
server.n_batch = 2048
@@ -34,7 +33,6 @@ def vision_server():
server = ServerPreset.tinygemma3()
server.offline = False # Allow downloading the model
server.model_alias = "tinygemma3-anthropic"
server.server_port = 8083 # Different port to avoid conflicts
server.n_slots = 1
return server
@@ -1015,7 +1013,6 @@ def test_anthropic_thinking_with_reasoning_model(stream):
server.jinja = True
server.n_ctx = 8192
server.n_predict = 1024
server.server_port = 8084
server.start(timeout_seconds=600) # large model needs time to download
if stream:
tools/server/tests/unit/test_mcp_servers.py
-5
View File
@@ -37,7 +37,6 @@ def _start_server_with_mcp(mcp_json: str, **kwargs) -> ServerProcess:
srv = ServerPreset.router()
srv.server_tools = "all"
srv.no_ui = True
srv.server_port = 8085 # avoid conflict with load_all() which uses 8080
srv.mcp_servers_json = mcp_json
for k, v in kwargs.items():
setattr(srv, k, v)
@@ -183,7 +182,6 @@ def test_mcp_tools_not_listed_when_not_configured():
server = ServerPreset.router()
server.server_tools = "all"
server.no_ui = True
server.server_port = 8085
server.start()
try:
@@ -250,7 +248,6 @@ def test_mcp_tools_via_json_config_file():
server = ServerPreset.router()
server.server_tools = "all"
server.no_ui = True
server.server_port = 8085
server.mcp_servers_config = config_path
server.start()
@@ -468,7 +465,6 @@ def test_mcp_config_file_errors():
server = ServerPreset.router()
server.server_tools = "all"
server.no_ui = True
server.server_port = 8085
server.mcp_servers_json = "not valid json"
try:
server.start()
@@ -480,7 +476,6 @@ def test_mcp_config_file_errors():
server = ServerPreset.router()
server.server_tools = "all"
server.no_ui = True
server.server_port = 8085
server.mcp_servers_config = "/nonexistent/path.json"
try:
server.start()
tools/server/tests/unit/test_slot_save.py
+4 -4
View File
@@ -10,10 +10,10 @@ STATE_FILE_HEADER_SIZE = 12
server = ServerPreset.tinyllama2()
@pytest.fixture(autouse=True)
def create_server():
def create_server(tmp_path):
global server
server = ServerPreset.tinyllama2()
server.slot_save_path = "./tmp"
server.slot_save_path = str(tmp_path)
server.temperature = 0.0
@@ -94,7 +94,7 @@ def test_slot_restore_legacy_token_list():
assert res.body["n_saved"] == 84
# rewrite the token payload into a plain token list, as written by servers that predate the packed server_tokens format
path = os.path.join("tmp", "slot_legacy.bin")
path = os.path.join(server.slot_save_path, "slot_legacy.bin")
with open(path, "rb") as f:
data = bytearray(f.read())
@@ -462,7 +462,7 @@ def test_slot_save_restore_image_payload_larger_than_context(mmproj_server):
})
assert res.status_code == 200
path = os.path.join("tmp", "mm_slot_large_payload.bin")
path = os.path.join(server.slot_save_path, "mm_slot_large_payload.bin")
with open(path, "rb") as f:
data = bytearray(f.read())
payload_size = struct.unpack_from("=I", data, STATE_FILE_HEADER_SIZE - 4)[0]
tools/server/tests/unit/test_tool_call.py
-1
View File
@@ -21,7 +21,6 @@ def create_server():
global server
server = ServerPreset.tinyllama2()
server.model_alias = "tinyllama-2-tool-call"
server.server_port = 8081
server.n_slots = 1
server.n_ctx = 8192
server.n_batch = 2048
tools/server/tests/unit/test_tools_builtin.py
+6 -6
View File
@@ -64,11 +64,11 @@ def test_tools_builtin_read_file():
assert "def test_tools_builtin_read_file" in text
def test_tools_builtin_write_then_edit_file():
def test_tools_builtin_write_then_edit_file(tmp_path):
global server
server.start()
log_path = os.path.join(PROJECT_ROOT, "test.log")
log_path = str(tmp_path / "test.log")
try:
write_res = call_tool("write_file", {"path": log_path, "content": "line1\nline2\nline3\n"})
assert write_res["result"] == "file written successfully"
@@ -93,11 +93,11 @@ def test_tools_builtin_write_then_edit_file():
os.remove(log_path)
def test_tools_builtin_edit_file_rejects_non_unique_old_text():
def test_tools_builtin_edit_file_rejects_non_unique_old_text(tmp_path):
global server
server.start()
log_path = os.path.join(PROJECT_ROOT, "test.log")
log_path = str(tmp_path / "test.log")
try:
call_tool("write_file", {"path": log_path, "content": "dup\ndup\n"})
err = call_tool_expect_error("edit_file", {
@@ -275,11 +275,11 @@ def test_tools_builtin_docker_runtime_cleans_up_spawned_container():
assert leftover.returncode != 0, f"container {container_id} was not cleaned up after server exit"
def test_tools_builtin_edit_file_rejects_overlapping_edits():
def test_tools_builtin_edit_file_rejects_overlapping_edits(tmp_path):
global server
server.start()
log_path = os.path.join(PROJECT_ROOT, "test.log")
log_path = str(tmp_path / "test.log")
try:
call_tool("write_file", {"path": log_path, "content": "line1\nline2\n"})
err = call_tool_expect_error("edit_file", {
tools/server/tests/utils.py
+1
View File
@@ -294,6 +294,7 @@ class ServerProcess:
server_args.append("--backend_sampling")
if self.gcp_compat:
env["AIP_MODE"] = "PREDICTION"
env["AIP_HTTP_PORT"] = str(self.server_port)
args = [str(arg) for arg in [server_path, *server_args]]
print(f"tests: starting server with: {' '.join(args)}")