Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 17:24:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

address comments

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-27 13:55:55 +02:00
parent 420c032e9f
commit 937fe54d11
4 changed files with 15 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-mmap.cpp
+6 -6
View File
@@ -440,8 +440,8 @@ void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }
#if defined(_POSIX_MAPPED_FILES) || defined(_WIN32)
// merge `ranges` and return their complement within [0, limit)
static std::vector<std::pair<size_t, size_t>> ranges_complement(std::vector<std::pair<size_t, size_t>> ranges, size_t limit) {
std::vector<std::pair<size_t, size_t>> res;
static llama_mmap::ranges ranges_complement(llama_mmap::ranges ranges, size_t limit) {
llama_mmap::ranges res;
std::sort(ranges.begin(), ranges.end());
size_t pos = 0;
@@ -465,7 +465,7 @@ struct llama_mmap::impl {
#ifdef _POSIX_MAPPED_FILES
std::vector<std::pair<size_t, size_t>> mapped_fragments;
impl(struct llama_file * file, size_t prefetch, bool numa, const std::vector<std::pair<size_t, size_t>> & lazy_ranges) {
impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {
size = file->size();
int fd = file->file_id();
int flags = MAP_SHARED;
@@ -572,7 +572,7 @@ struct llama_mmap::impl {
#elif defined(_WIN32)
HANDLE hMapping = nullptr;
impl(struct llama_file * file, size_t prefetch, bool numa, const std::vector<std::pair<size_t, size_t>> & lazy_ranges) {
impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {
GGML_UNUSED(numa);
size = file->size();
@@ -641,7 +641,7 @@ struct llama_mmap::impl {
}
}
#else
impl(struct llama_file * file, size_t prefetch, bool numa, const std::vector<std::pair<size_t, size_t>> & lazy_ranges) {
impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {
GGML_UNUSED(file);
GGML_UNUSED(prefetch);
GGML_UNUSED(numa);
@@ -663,7 +663,7 @@ struct llama_mmap::impl {
};
llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa,
const std::vector<std::pair<size_t, size_t>> & lazy_ranges) : pimpl(std::make_unique<impl>(file, prefetch, numa, lazy_ranges)) {}
const ranges & lazy_ranges) : pimpl(std::make_unique<impl>(file, prefetch, numa, lazy_ranges)) {}
llama_mmap::~llama_mmap() = default;
size_t llama_mmap::size() const { return pimpl->size; }
src/llama-mmap.h
+4 -1
View File
@@ -42,9 +42,12 @@ private:
};
struct llama_mmap {
// list of [first, last) byte ranges within a file
using ranges = std::vector<std::pair<size_t, size_t>>;
llama_mmap(const llama_mmap &) = delete;
llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false,
const std::vector<std::pair<size_t, size_t>> & lazy_ranges = {});
const ranges & lazy_ranges = {});
~llama_mmap();
size_t size() const;
src/llama-model-loader.cpp
+4 -1
View File
@@ -1288,6 +1288,9 @@ struct ggml_tensor * llama_model_loader::create_tensor(
if (tensor_read_lazy == LLAMA_TENSOR_READ_LAZY_ON || ggml_nbytes(cur) > auto_lazy_min_size) {
const auto & w = require_weight(tn.str().c_str());
lazy_tensor_ranges[w.idx].emplace_back(w.offs, w.offs + ggml_nbytes(cur));
LLAMA_LOG_INFO("%s: tensor %s (size = %zu MiB) lazy read enabled\n",
__func__, tn.str().c_str(), ggml_nbytes(cur)/1024/1024);
}
}
@@ -1373,7 +1376,7 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps
}
const auto it_lazy = lazy_tensor_ranges.find(idx);
static const std::vector<std::pair<size_t, size_t>> no_lazy_ranges;
static const llama_mmap::ranges no_lazy_ranges;
std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa,
it_lazy != lazy_tensor_ranges.end() ? it_lazy->second : no_lazy_ranges);
src/llama-model-loader.h
+1 -1
View File
@@ -93,7 +93,7 @@ struct llama_model_loader {
llama_mmaps mappings;
// byte ranges of TENSOR_READ_LAZY tensors, per file index
std::map<uint32_t, std::vector<std::pair<size_t, size_t>>> lazy_tensor_ranges;
std::map<uint32_t, llama_mmap::ranges> lazy_tensor_ranges;
std::map<std::string, llama_tensor_weight, weight_name_comparer> weights_map;
std::unordered_map<std::string, llama_model_kv_override> kv_overrides;