diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index 8b8e5f5f6..f719323c0 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -358,6 +358,15 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int } // namespace audio_helpers +static bool is_webp_file(const unsigned char * buf, size_t len) { + // WEBP ref: https://developers.google.com/speed/webp/docs/riff_container + return len >= 12 && memcmp(buf, "RIFF", 4) == 0 && memcmp(buf + 8, "WEBP", 4) == 0; +} + +#ifdef MTMD_VIDEO +static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder); +#endif + mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) { // calculate the hash if needed std::string id; @@ -397,6 +406,19 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, // otherwise, fallthrough to video decoding (if supported) } +#ifdef MTMD_VIDEO + // stb_image does not support webp; decode it with ffmpeg as a single frame + if (!result && is_webp_file(buf, len)) { + result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder); + if (!result) { + LOG_ERR("%s: failed to decode webp buffer\n", __func__); + return {nullptr, nullptr}; + } + mtmd_bitmap_set_id(result, id.empty() ? nullptr : id.c_str()); + return {result, nullptr}; + } +#endif + // last try: load as video #ifdef MTMD_VIDEO if (!result) { @@ -820,6 +842,33 @@ static std::string video_resolve_bin(const char * bin_dir, const char * name) { return result; } +#ifdef MTMD_VIDEO +static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder) { + auto params = mtmd_helper_video_init_params_default(); + mtmd_helper_video vctx; + vctx.mctx = mctx; + vctx.input_buf.assign(buf, buf + len); + vctx.ffmpeg_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg"); + vctx.ffprobe_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe"); + if (!vctx.probe(0.0f)) { + return nullptr; + } + if (placeholder) { + return mtmd_bitmap_init(vctx.info.width, vctx.info.height, nullptr); + } + // still image: the fps filter would output no frame, so disable it + vctx.fps_target = 0.0f; + if (!vctx.start_ffmpeg(0.0f)) { + return nullptr; + } + mtmd_bitmap * frame = vctx.read_next_frame(); + if (frame) { + mtmd_bitmap_set_mergeable(frame, false); + } + return frame; +} +#endif + mtmd_helper_video * mtmd_helper_video_init( mtmd_context * mctx, const char * path, diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 5c6b92419..58dfb1525 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -45,6 +45,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm // helper function to construct a mtmd_bitmap from a buffer containing a file // supported formats: // image: formats supported by stb_image: jpg, png, bmp, gif, etc. +// webp is decoded via ffmpeg, requires MTMD_VIDEO build with ffmpeg in PATH // audio: formats supported by miniaudio: wav, mp3, flac // note: // - for now, video input is only supported via C++ helper functions