diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index 58c4437604..4d183cbc9c 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -440,8 +440,8 @@ void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); } #if defined(_POSIX_MAPPED_FILES) || defined(_WIN32) // merge `ranges` and return their complement within [0, limit) -static std::vector> ranges_complement(std::vector> ranges, size_t limit) { - std::vector> res; +static llama_mmap::ranges ranges_complement(llama_mmap::ranges ranges, size_t limit) { + llama_mmap::ranges res; std::sort(ranges.begin(), ranges.end()); size_t pos = 0; @@ -465,7 +465,7 @@ struct llama_mmap::impl { #ifdef _POSIX_MAPPED_FILES std::vector> mapped_fragments; - impl(struct llama_file * file, size_t prefetch, bool numa, const std::vector> & lazy_ranges) { + impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) { size = file->size(); int fd = file->file_id(); int flags = MAP_SHARED; @@ -572,7 +572,7 @@ struct llama_mmap::impl { #elif defined(_WIN32) HANDLE hMapping = nullptr; - impl(struct llama_file * file, size_t prefetch, bool numa, const std::vector> & lazy_ranges) { + impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) { GGML_UNUSED(numa); size = file->size(); @@ -641,7 +641,7 @@ struct llama_mmap::impl { } } #else - impl(struct llama_file * file, size_t prefetch, bool numa, const std::vector> & lazy_ranges) { + impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) { GGML_UNUSED(file); GGML_UNUSED(prefetch); GGML_UNUSED(numa); @@ -663,7 +663,7 @@ struct llama_mmap::impl { }; llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa, - const std::vector> & lazy_ranges) : pimpl(std::make_unique(file, prefetch, numa, lazy_ranges)) {} + const ranges & lazy_ranges) : pimpl(std::make_unique(file, prefetch, numa, lazy_ranges)) {} llama_mmap::~llama_mmap() = default; size_t llama_mmap::size() const { return pimpl->size; } diff --git a/src/llama-mmap.h b/src/llama-mmap.h index bd67e7c002..cc28c8a73f 100644 --- a/src/llama-mmap.h +++ b/src/llama-mmap.h @@ -42,9 +42,12 @@ private: }; struct llama_mmap { + // list of [first, last) byte ranges within a file + using ranges = std::vector>; + llama_mmap(const llama_mmap &) = delete; llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false, - const std::vector> & lazy_ranges = {}); + const ranges & lazy_ranges = {}); ~llama_mmap(); size_t size() const; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 9db3140647..2f1a09a19a 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1288,6 +1288,9 @@ struct ggml_tensor * llama_model_loader::create_tensor( if (tensor_read_lazy == LLAMA_TENSOR_READ_LAZY_ON || ggml_nbytes(cur) > auto_lazy_min_size) { const auto & w = require_weight(tn.str().c_str()); lazy_tensor_ranges[w.idx].emplace_back(w.offs, w.offs + ggml_nbytes(cur)); + + LLAMA_LOG_INFO("%s: tensor %s (size = %zu MiB) lazy read enabled\n", + __func__, tn.str().c_str(), ggml_nbytes(cur)/1024/1024); } } @@ -1373,7 +1376,7 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps } const auto it_lazy = lazy_tensor_ranges.find(idx); - static const std::vector> no_lazy_ranges; + static const llama_mmap::ranges no_lazy_ranges; std::unique_ptr mapping = std::make_unique(file.get(), prefetch ? -1 : 0, is_numa, it_lazy != lazy_tensor_ranges.end() ? it_lazy->second : no_lazy_ranges); diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index 9898d7f621..ec5b692463 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -93,7 +93,7 @@ struct llama_model_loader { llama_mmaps mappings; // byte ranges of TENSOR_READ_LAZY tensors, per file index - std::map>> lazy_tensor_ranges; + std::map lazy_tensor_ranges; std::map weights_map; std::unordered_map kv_overrides;