mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-19 17:25:07 +02:00
Merge commit 'd646c9d15500a702425e8a90c19d2400deecbd0c' into concedo_experimental
# Conflicts: # .github/actions/windows-setup-rocm/action.yml # .github/workflows/build-apple.yml # .github/workflows/release.yml # .github/workflows/server-self-hosted.yml # examples/training/README.md # ggml/src/ggml-hexagon/ggml-hexagon.cpp # ggml/src/ggml-hexagon/htp/hvx-arith.h # ggml/src/ggml-hexagon/htp/hvx-log.h # ggml/src/ggml-hexagon/htp/hvx-norm.h # ggml/src/ggml-hexagon/htp/hvx-scale.h # ggml/src/ggml-hexagon/htp/hvx-sqrt.h # ggml/src/ggml-hexagon/htp/unary-ops.c # ggml/src/ggml-hexagon/htp/unary-ops.h # ggml/src/ggml-sycl/mmvq.cpp # ggml/src/ggml-sycl/vecdotq.hpp # tests/test-backend-ops.cpp # tests/test-mtmd-c-api.c # tests/test-mtmd-impl.cpp
This commit is contained in:
Concedo
@@ -20,6 +20,7 @@ In short:
|
||||
A typical pipeline of the core libmtmd is as follows:
|
||||
- A bitmap (RGB image or PCM audio) is created
|
||||
- Bitmap and the text prompt is provided to `mtmd_tokenize()` that breaks the input into chunks
|
||||
- Alternatively, `mtmd_tokenize_from_parts()` takes a list of pre-split text/media parts instead of a marker-based prompt
|
||||
- The tokenizer function first expands a "lazy" bitmap if it finds one. Typically, this is used by video, so that one media token corresponds to one input bitmap
|
||||
- For models that support "fused" temporal frames like Qwen-VL, the tokenizer tries to merge pair of consecutive frames into one batch. Only bitmaps marked by `mtmd_bitmap_set_mergeable()` are merged
|
||||
- The preprocessor will then be called, which produces a list of chunks
|
||||
|
||||
+43
-15
@@ -109,16 +109,15 @@ struct mtmd_cli_context {
|
||||
mtmd_cli_context(common_params & params) : llama_init(common_init_from_params(params)) {
|
||||
model = llama_init->model();
|
||||
lctx = llama_init->context();
|
||||
if (!model || !lctx) {
|
||||
exit(1);
|
||||
}
|
||||
vocab = llama_model_get_vocab(model);
|
||||
smpl = common_sampler_init(model, params.sampling);
|
||||
n_threads = params.cpuparams.n_threads;
|
||||
batch = llama_batch_init(1, 0, 1); // batch for next token generation
|
||||
n_batch = params.n_batch;
|
||||
|
||||
if (!model || !lctx) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
init_vision_context(params);
|
||||
|
||||
if (!mtmd_helper_model_can_chat(lctx, ctx_vision.get())) {
|
||||
@@ -265,21 +264,50 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) {
|
||||
auto formatted_chat = chat_add_and_format(ctx, msg);
|
||||
LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.c_str());
|
||||
|
||||
mtmd_input_text text;
|
||||
text.text = formatted_chat.data();
|
||||
text.text_len = formatted_chat.size();
|
||||
text.add_special = add_bos;
|
||||
text.parse_special = true;
|
||||
|
||||
if (g_is_interrupted) return 0;
|
||||
|
||||
mtmd::input_chunks chunks(mtmd_input_chunks_init());
|
||||
// note: we replace the marker here instead of letting mtmd_tokenize() to do that
|
||||
// because we want to demonstrate how to use mtmd_tokenize_from_parts()
|
||||
|
||||
// split the formatted chat on the media marker to get text segments
|
||||
const std::string marker = mtmd_default_marker();
|
||||
std::vector<std::string> segments;
|
||||
size_t start = 0;
|
||||
size_t pos;
|
||||
while ((pos = formatted_chat.find(marker, start)) != std::string::npos) {
|
||||
segments.push_back(formatted_chat.substr(start, pos - start));
|
||||
start = pos + marker.size();
|
||||
}
|
||||
segments.push_back(formatted_chat.substr(start));
|
||||
|
||||
auto bitmaps_c_ptr = ctx.bitmaps.c_ptr();
|
||||
int32_t res = mtmd_tokenize(ctx.ctx_vision.get(),
|
||||
if (segments.size() - 1 != bitmaps_c_ptr.size()) {
|
||||
LOG_ERR("Number of media markers (%zu) does not match number of loaded media (%zu)\n",
|
||||
segments.size() - 1, bitmaps_c_ptr.size());
|
||||
return 1;
|
||||
}
|
||||
|
||||
// interleave text and media parts
|
||||
std::vector<mtmd_input_text> texts(segments.size());
|
||||
std::vector<mtmd_input_part> parts;
|
||||
for (size_t i = 0; i < segments.size(); i++) {
|
||||
texts[i] = {segments[i].data(), segments[i].size(), /* add_special */ false, /* parse_special */ true};
|
||||
parts.push_back({&texts[i], nullptr});
|
||||
if (i < bitmaps_c_ptr.size()) {
|
||||
parts.push_back({nullptr, bitmaps_c_ptr[i]});
|
||||
}
|
||||
}
|
||||
std::vector<const mtmd_input_part *> parts_ptr;
|
||||
for (const auto & p : parts) {
|
||||
parts_ptr.push_back(&p);
|
||||
}
|
||||
|
||||
mtmd::input_chunks chunks(mtmd_input_chunks_init());
|
||||
int32_t res = mtmd_tokenize_from_parts(ctx.ctx_vision.get(),
|
||||
chunks.ptr.get(), // output
|
||||
&text, // text
|
||||
bitmaps_c_ptr.data(),
|
||||
bitmaps_c_ptr.size());
|
||||
parts_ptr.data(),
|
||||
parts_ptr.size(),
|
||||
add_bos);
|
||||
if (res != 0) {
|
||||
LOG_ERR("Unable to tokenize prompt, res = %d\n", res);
|
||||
return 1;
|
||||
|
||||
@@ -980,6 +980,56 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
|
||||
//
|
||||
// CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737
|
||||
const clip_image_size original_size = img.get_size();
|
||||
|
||||
// old gguf files have no preprocessor longest size, custom token limits also need the generic size below
|
||||
if (hparams.image_longest_edge > 0 && hparams.image_min_pixels <= 0 && hparams.image_max_pixels <= 0) {
|
||||
const int tile_size = hparams.image_size;
|
||||
const int longest_edge = hparams.image_longest_edge;
|
||||
const double aspect_ratio = (double) original_size.width / original_size.height;
|
||||
|
||||
clip_image_size resized_size;
|
||||
if (original_size.width >= original_size.height) {
|
||||
resized_size.width = longest_edge;
|
||||
resized_size.height = (int) (longest_edge / aspect_ratio);
|
||||
resized_size.height += resized_size.height % 2;
|
||||
} else {
|
||||
resized_size.height = longest_edge;
|
||||
resized_size.width = (int) (longest_edge * aspect_ratio);
|
||||
resized_size.width += resized_size.width % 2;
|
||||
}
|
||||
|
||||
const int grid_x = (resized_size.width + tile_size - 1) / tile_size;
|
||||
const int grid_y = (resized_size.height + tile_size - 1) / tile_size;
|
||||
const clip_image_size refined_size = clip_image_size{grid_x * tile_size, grid_y * tile_size};
|
||||
|
||||
clip_image_u8 resized_img;
|
||||
img_tool::resize(img, resized_img, resized_size, hparams.image_resize_algo, PAD_NONE);
|
||||
|
||||
clip_image_u8 refined_img;
|
||||
img_tool::resize(resized_img, refined_img, refined_size, hparams.image_resize_algo, PAD_NONE);
|
||||
|
||||
clip_image_u8 overview;
|
||||
img_tool::resize(refined_img, overview, {tile_size, tile_size}, hparams.image_resize_algo, PAD_NONE);
|
||||
|
||||
std::vector<clip_image_u8> slices;
|
||||
for (int y = 0; y < grid_y; y++) {
|
||||
for (int x = 0; x < grid_x; x++) {
|
||||
clip_image_u8 slice;
|
||||
img_tool::crop(refined_img, slice, x * tile_size, y * tile_size, tile_size, tile_size);
|
||||
slices.push_back(std::move(slice));
|
||||
}
|
||||
}
|
||||
|
||||
LOG_DBG("%s: grid size: %d x %d (%d tiles) + overview\n", __func__, grid_x, grid_y, grid_x * grid_y);
|
||||
|
||||
mtmd_image_preproc_out output;
|
||||
output.append_overview(hparams, overview, true);
|
||||
output.append(hparams, slices, true);
|
||||
output.grid_x = grid_x;
|
||||
output.grid_y = grid_y;
|
||||
return output;
|
||||
}
|
||||
|
||||
const clip_image_size refined_size = img_tool::calc_size_preserved_ratio(
|
||||
original_size,
|
||||
{ hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge });
|
||||
|
||||
@@ -10,10 +10,12 @@
|
||||
#define MTMD_INTERNAL_HEADER
|
||||
|
||||
// bitmap is null for text parts
|
||||
struct mtmd_input_part {
|
||||
struct mtmd_internal_part {
|
||||
std::string text;
|
||||
const mtmd_bitmap * bitmap;
|
||||
// only used for text parts
|
||||
bool parse_special = false;
|
||||
};
|
||||
|
||||
// [QWEN_VIDEO] merged parts are erased from `parts`, so one group always maps to one part
|
||||
std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::vector<mtmd_input_part> & parts, int n_merge);
|
||||
std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::vector<mtmd_internal_part> & parts, int n_merge);
|
||||
|
||||
+50
-5
@@ -1097,7 +1097,7 @@ void mtmd_free(mtmd_context * ctx) {
|
||||
delete ctx;
|
||||
}
|
||||
|
||||
std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::vector<mtmd_input_part> & parts, int n_merge) {
|
||||
std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::vector<mtmd_internal_part> & parts, int n_merge) {
|
||||
std::vector<std::vector<const mtmd_bitmap *>> output;
|
||||
for (size_t i = 0; i < parts.size(); i++) {
|
||||
if (parts[i].bitmap == nullptr) {
|
||||
@@ -1124,7 +1124,7 @@ struct mtmd_tokenizer {
|
||||
bool parse_special;
|
||||
const llama_vocab * vocab;
|
||||
|
||||
using part = mtmd_input_part;
|
||||
using part = mtmd_internal_part;
|
||||
std::vector<part> parts;
|
||||
// these will be freed when mtmd_tokenizer finishes
|
||||
std::vector<mtmd::bitmap> bm_from_lazy; // TODO @ngxson : refactor, free bm_from_lazy progressively
|
||||
@@ -1160,7 +1160,7 @@ struct mtmd_tokenizer {
|
||||
}
|
||||
parts.push_back({"", bitmaps[i_bm++]});
|
||||
} else {
|
||||
parts.push_back({std::move(part), nullptr});
|
||||
parts.push_back({std::move(part), nullptr, parse_special});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1177,6 +1177,26 @@ struct mtmd_tokenizer {
|
||||
expand_lazy_bitmaps();
|
||||
}
|
||||
|
||||
mtmd_tokenizer(mtmd_context * ctx,
|
||||
const mtmd_input_part ** input_parts,
|
||||
size_t n_parts,
|
||||
bool add_special) : ctx(ctx) {
|
||||
this->add_special = add_special;
|
||||
parse_special = true; // only used for text returned by lazy bitmaps
|
||||
vocab = ctx->vocab;
|
||||
|
||||
for (size_t i = 0; i < n_parts; i++) {
|
||||
const mtmd_input_part * p = input_parts[i];
|
||||
if (p->text != nullptr) {
|
||||
parts.push_back({std::string(p->text->text, p->text->text_len), nullptr, p->text->parse_special});
|
||||
} else {
|
||||
parts.push_back({"", p->bitmap});
|
||||
}
|
||||
}
|
||||
|
||||
expand_lazy_bitmaps();
|
||||
}
|
||||
|
||||
void expand_lazy_bitmaps() {
|
||||
std::vector<part> expanded;
|
||||
expanded.reserve(parts.size());
|
||||
@@ -1201,7 +1221,7 @@ struct mtmd_tokenizer {
|
||||
LOG_DBG("%s: lazy callback returned bitmap with dimensions %d x %d\n", __func__, out_bm->nx, out_bm->ny);
|
||||
} else if (out_str) {
|
||||
auto & ptr = text_from_lazy.emplace_back(out_str); // remember to free it later
|
||||
expanded.push_back({ptr, nullptr});
|
||||
expanded.push_back({ptr, nullptr, parse_special});
|
||||
LOG_DBG("%s: lazy callback returned text: %s\n", __func__, out_str);
|
||||
}
|
||||
} else if (res == -1) {
|
||||
@@ -1245,7 +1265,7 @@ struct mtmd_tokenizer {
|
||||
return res;
|
||||
}
|
||||
} else {
|
||||
add_text(p.text, parse_special);
|
||||
add_text(p.text, p.parse_special);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1727,6 +1747,30 @@ int32_t mtmd_tokenize(mtmd_context * ctx,
|
||||
}
|
||||
}
|
||||
|
||||
int32_t mtmd_tokenize_from_parts(mtmd_context * ctx,
|
||||
mtmd_input_chunks * output,
|
||||
const mtmd_input_part ** parts,
|
||||
size_t n_parts,
|
||||
bool add_special) {
|
||||
for (size_t i = 0; i < n_parts; i++) {
|
||||
if ((parts[i]->text == nullptr) == (parts[i]->bitmap == nullptr)) {
|
||||
LOG_ERR("%s: part %zu must have either text or bitmap set, not both\n", __func__, i);
|
||||
return 1;
|
||||
}
|
||||
if (parts[i]->text != nullptr && parts[i]->text->text == nullptr) {
|
||||
LOG_ERR("%s: part %zu has null text pointer\n", __func__, i);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
try {
|
||||
mtmd_tokenizer tokenizer(ctx, parts, n_parts, add_special);
|
||||
return tokenizer.tokenize(output);
|
||||
} catch (const std::exception & e) {
|
||||
LOG_ERR("%s: error: %s\n", __func__, e.what());
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * image_tokens, std::vector<float> & out_embd) {
|
||||
clip_ctx * ctx_clip = ctx->ctx_v;
|
||||
if (!ctx_clip) {
|
||||
@@ -2132,6 +2176,7 @@ bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk
|
||||
case PROJECTOR_TYPE_GEMMA3:
|
||||
case PROJECTOR_TYPE_GEMMA4V:
|
||||
case PROJECTOR_TYPE_GEMMA4UV:
|
||||
case PROJECTOR_TYPE_DEEPSEEK4V:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
|
||||
+23
-4
@@ -73,6 +73,12 @@ struct mtmd_input_text {
|
||||
bool parse_special;
|
||||
};
|
||||
|
||||
struct mtmd_input_part {
|
||||
// only text or bitmap can be set, not both
|
||||
const struct mtmd_input_text * text;
|
||||
const struct mtmd_bitmap * bitmap;
|
||||
};
|
||||
|
||||
//
|
||||
// C API
|
||||
//
|
||||
@@ -83,6 +89,7 @@ typedef struct mtmd_image_tokens mtmd_image_tokens;
|
||||
typedef struct mtmd_input_chunk mtmd_input_chunk;
|
||||
typedef struct mtmd_input_chunks mtmd_input_chunks;
|
||||
typedef struct mtmd_input_text mtmd_input_text;
|
||||
typedef struct mtmd_input_part mtmd_input_part;
|
||||
typedef struct mtmd_batch mtmd_batch;
|
||||
|
||||
typedef bool (*mtmd_progress_callback)(float progress, void * user_data);
|
||||
@@ -276,10 +283,10 @@ struct mtmd_decoder_pos {
|
||||
// return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position)
|
||||
MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * image_tokens, llama_pos pos_0, size_t i);
|
||||
|
||||
// tokenize an input text prompt and a list of bitmaps (images/audio)
|
||||
// the prompt must have the input image marker (default: "<__media__>") in it
|
||||
// tokenize an input text prompt and a list of bitmaps (image/audio)
|
||||
// the prompt must have the input media marker (default: "<__media__>") in it
|
||||
// the default marker is defined by mtmd_default_marker()
|
||||
// the marker will be replaced with the image/audio chunk
|
||||
// the marker will be replaced with the media chunk
|
||||
// for example:
|
||||
// "here is an image: <__media__>\ndescribe it in detail."
|
||||
// this will gives 3 chunks:
|
||||
@@ -291,13 +298,25 @@ MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_im
|
||||
// return values:
|
||||
// 0 on success
|
||||
// 1 on number of bitmaps not matching the number of markers
|
||||
// 2 on image preprocessing error
|
||||
// 2 on media preprocessing error
|
||||
MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx,
|
||||
mtmd_input_chunks * output,
|
||||
const mtmd_input_text * text,
|
||||
const mtmd_bitmap ** bitmaps,
|
||||
size_t n_bitmaps);
|
||||
|
||||
// same as mtmd_tokenize(), but takes an array of mtmd_input_part
|
||||
// use cases:
|
||||
// - when you don't want to use media markers (they will be tokenized as normal text)
|
||||
// - when you want to control parse_special for each text part
|
||||
// note: per-part add_special will be ignored
|
||||
// return 1 if a part has both text and bitmap set (or neither)
|
||||
MTMD_API int32_t mtmd_tokenize_from_parts(mtmd_context * ctx,
|
||||
mtmd_input_chunks * output,
|
||||
const mtmd_input_part ** parts,
|
||||
size_t n_parts,
|
||||
bool add_special);
|
||||
|
||||
DEPRECATED(MTMD_API int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens),
|
||||
"use mtmd_encode_chunk() instead");
|
||||
|
||||
|
||||
@@ -1062,8 +1062,7 @@ json oaicompat_completion_params_parse(const json & body) {
|
||||
static void handle_media(
|
||||
std::vector<raw_buffer> & out_files,
|
||||
const std::string & url,
|
||||
const std::string & media_path,
|
||||
bool accept_base64_uri) {
|
||||
const std::string & media_path) {
|
||||
if (!media_path.empty()) {
|
||||
// should already be enforced by arg.cpp, but checking just in case
|
||||
GGML_ASSERT(media_path.back() == DIRECTORY_SEPARATOR);
|
||||
@@ -1104,15 +1103,17 @@ static void handle_media(
|
||||
data.assign((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
|
||||
out_files.push_back(data);
|
||||
|
||||
} else if (accept_base64_uri && string_starts_with(url, "data:")) {
|
||||
// try to decode base64 image
|
||||
} else if (string_starts_with(url, "data:")) {
|
||||
// try to decode base64 image, video, or audio
|
||||
std::vector<std::string> parts = string_split<std::string>(url, /*separator*/ ',');
|
||||
if (parts.size() != 2) {
|
||||
throw std::runtime_error("Invalid uri-encoded base64 value");
|
||||
} else if (!string_starts_with(parts[0], "data:image/")) {
|
||||
throw std::runtime_error("Invalid uri format: " + parts[0]);
|
||||
throw std::invalid_argument("Invalid uri-encoded base64 value");
|
||||
} else if (!string_starts_with(parts[0], "data:image/")
|
||||
&& !string_starts_with(parts[0], "data:video/")
|
||||
&& !string_starts_with(parts[0], "data:audio/")) {
|
||||
throw std::invalid_argument("Invalid uri format: " + parts[0]);
|
||||
} else if (!string_ends_with(parts[0], "base64")) {
|
||||
throw std::runtime_error("uri must be base64 encoded");
|
||||
throw std::invalid_argument("uri must be base64 encoded");
|
||||
} else {
|
||||
auto base64_data = parts[1];
|
||||
auto decoded_data = base64_decode(base64_data);
|
||||
@@ -1219,7 +1220,7 @@ json oaicompat_chat_params_parse(
|
||||
|
||||
json image_url = json_value(p, "image_url", json::object());
|
||||
std::string url = json_value(image_url, "url", std::string());
|
||||
handle_media(out_files, url, opt.media_path, true);
|
||||
handle_media(out_files, url, opt.media_path);
|
||||
|
||||
p["type"] = "media_marker";
|
||||
p["text"] = get_media_marker();
|
||||
@@ -1234,7 +1235,7 @@ json oaicompat_chat_params_parse(
|
||||
json input_audio = json_value(p, "input_audio", json::object());
|
||||
std::string url = json_value(input_audio, "data",
|
||||
json_value(input_audio, "url", std::string()));
|
||||
handle_media(out_files, url, opt.media_path, false);
|
||||
handle_media(out_files, url, opt.media_path);
|
||||
|
||||
p["type"] = "media_marker";
|
||||
p["text"] = get_media_marker();
|
||||
@@ -1248,7 +1249,7 @@ json oaicompat_chat_params_parse(
|
||||
json input_video = json_value(p, "input_video", json::object());
|
||||
std::string url = json_value(input_video, "data",
|
||||
json_value(input_video, "url", std::string()));
|
||||
handle_media(out_files, url, opt.media_path, false);
|
||||
handle_media(out_files, url, opt.media_path);
|
||||
|
||||
p["type"] = "media_marker";
|
||||
p["text"] = get_media_marker();
|
||||
|
||||
@@ -1493,11 +1493,22 @@ private:
|
||||
auto caps = common_chat_templates_get_caps(chat_params.tmpls.get());
|
||||
auto it = params_base.default_template_kwargs.find("preserve_reasoning");
|
||||
bool supported = caps.at("supports_preserve_reasoning");
|
||||
bool enabled = it != params_base.default_template_kwargs.end();
|
||||
bool specified = params_base.preserve_reasoning_specified;
|
||||
// note: the kwarg is enabled by default if not specified explicitly, so check the value
|
||||
bool enabled = it != params_base.default_template_kwargs.end() && it->second == "true";
|
||||
if (supported) {
|
||||
SRV_TRC("preserve_reasoning kwarg: %s\n",
|
||||
it == params_base.default_template_kwargs.end() ? "unset (template default)" : it->second.c_str());
|
||||
} else {
|
||||
SRV_TRC("%s", "preserve_reasoning kwarg: not supported by template\n");
|
||||
}
|
||||
if (supported && !specified) {
|
||||
SRV_WRN("%s", "chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve)\n");
|
||||
}
|
||||
if (supported && !enabled) {
|
||||
SRV_INF("%s", "chat template supports preserving reasoning, consider enabling it via --reasoning-preserve\n");
|
||||
}
|
||||
if (!supported && enabled) {
|
||||
if (!supported && specified && enabled) {
|
||||
SRV_WRN("%s", "chat template does NOT support preserving reasoning, --reasoning-preserve has no effect\n");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,6 +71,7 @@ def test_v1_models_supports_multimodal_capability():
|
||||
("What is this:\n", "malformed", False, None),
|
||||
("What is this:\n", "https://google.com/404", False, None), # non-existent image
|
||||
("What is this:\n", "https://ggml.ai", False, None), # non-image data
|
||||
("What is this:\n", "data:text/html;base64,aGVsbG8=", False, None), # unsupported data uri mime
|
||||
# TODO @ngxson : test with multiple images, no images and with audio
|
||||
]
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user