diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e1fd45dd1..19f14e0d18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,10 +224,12 @@ add_subdirectory(src) # utils, programs, examples and tests # +# mtmd needs this even when common is not built +add_subdirectory(vendor/hash) + if (LLAMA_BUILD_COMMON) add_subdirectory(common) add_subdirectory(vendor/cpp-httplib) - add_subdirectory(vendor/hash) endif() if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION) diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 769a44e0b7..db758395ff 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -78,7 +78,7 @@ set_target_properties(mtmd PROPERTIES ) target_link_libraries (mtmd PUBLIC ggml llama) -target_link_libraries (mtmd PRIVATE Threads::Threads) +target_link_libraries (mtmd PRIVATE Threads::Threads vendor-hash) target_include_directories(mtmd PUBLIC .) target_include_directories(mtmd PRIVATE ../..) target_include_directories(mtmd PRIVATE ../../vendor) diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index d77c939664..bce8e38cc3 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -12,6 +12,8 @@ #include "mtmd-helper-common.h" #include "llama.h" +#include "hash.h" + #include #include #include @@ -356,25 +358,14 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int } // namespace audio_helpers -// Computes FNV-1a hash of the data -static std::string fnv_hash(const uint8_t * data, size_t len) { - const uint64_t fnv_prime = 0x100000001b3ULL; - uint64_t hash = 0xcbf29ce484222325ULL; - - for (size_t i = 0; i < len; ++i) { - hash ^= data[i]; - hash *= fnv_prime; - } - return std::to_string(hash); -} - mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) { // calculate the hash if needed std::string id; mtmd_bitmap * result = nullptr; if (!placeholder) { - id = fnv_hash(buf, len); + // use sha256 to prevent cache poisoning + id = hash_sha256_hex(buf, len); } if (audio_helpers::is_audio_file((const char *)buf, len)) { diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 832f7171ac..5c6b924197 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -49,7 +49,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm // note: // - for now, video input is only supported via C++ helper functions // - audio files will be auto-detected based on magic bytes -// - output bitmap will have FNV hash as the ID +// - output bitmap will have SHA-256 hash (hex string) as the ID // returns nullptr on failure // this function is thread-safe MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); diff --git a/tools/ui/embed.cpp b/tools/ui/embed.cpp index 914d51fa1d..b76c9047f2 100644 --- a/tools/ui/embed.cpp +++ b/tools/ui/embed.cpp @@ -259,6 +259,8 @@ int main(int argc, char ** argv) { } cpp += fmt("static const unsigned char asset_%d_data[] = {", i); append_bytes_hex(cpp, bytes); + + // note: this is a simple hash for cache busting, not a cryptographic hash; fnv is enough here const auto hash = fnv_hash(bytes.data(), bytes.size()); cpp += fmt("};\nstatic const std::size_t asset_%d_size = %zu;\n", diff --git a/vendor/hash/CMakeLists.txt b/vendor/hash/CMakeLists.txt index 122a6419f9..6788c12839 100644 --- a/vendor/hash/CMakeLists.txt +++ b/vendor/hash/CMakeLists.txt @@ -4,18 +4,27 @@ llama_add_compile_flags() set(CMAKE_POSITION_INDEPENDENT_CODE ON) -add_library(${TARGET} STATIC +set(VENDOR_SRCS xxhash/xxhash.c sha1/sha1.c sha256/sha256.c ) -# disable warnings in 3rd party code +add_library(${TARGET} STATIC + hash.cpp + hash.h + ${VENDOR_SRCS} +) + +target_compile_features(${TARGET} PRIVATE cxx_std_17) + +# disable warnings in 3rd party code, but keep them for hash.cpp if (CMAKE_C_COMPILER_ID STREQUAL "MSVC") - target_compile_options(${TARGET} PRIVATE /w) + set(NO_WARN_FLAG /w) else() - target_compile_options(${TARGET} PRIVATE -w) + set(NO_WARN_FLAG -w) endif() +set_source_files_properties(${VENDOR_SRCS} PROPERTIES COMPILE_OPTIONS ${NO_WARN_FLAG}) # sha256.c includes "rotate-bits/rotate-bits.h", so consumers get this dir too target_include_directories(${TARGET} PUBLIC .) diff --git a/vendor/hash/hash.cpp b/vendor/hash/hash.cpp new file mode 100644 index 0000000000..6493716bf7 --- /dev/null +++ b/vendor/hash/hash.cpp @@ -0,0 +1,23 @@ +#include "hash.h" + +extern "C" { +#include "sha256/sha256.h" +} + +static std::string to_hex(const unsigned char * digest, size_t len) { + static const char hex[] = "0123456789abcdef"; + + std::string out; + out.reserve(2*len); + for (size_t i = 0; i < len; ++i) { + out += hex[digest[i] >> 4]; + out += hex[digest[i] & 0xf]; + } + return out; +} + +std::string hash_sha256_hex(const void * data, size_t len) { + unsigned char digest[SHA256_DIGEST_SIZE]; + sha256_hash(digest, (const unsigned char *) data, len); + return to_hex(digest, SHA256_DIGEST_SIZE); +} diff --git a/vendor/hash/hash.h b/vendor/hash/hash.h new file mode 100644 index 0000000000..1298b4196b --- /dev/null +++ b/vendor/hash/hash.h @@ -0,0 +1,9 @@ +#pragma once + +// C++ wrapper for the vendored hash functions + +#include +#include + +// returns the SHA-256 digest as a lowercase hex string +std::string hash_sha256_hex(const void * data, size_t len);