From 41b4b7073c8049b27edce3f7fe788e1ef8f50bac Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Thu, 27 Aug 2026 15:01:35 +0200 Subject: [PATCH] revert LLAMA_MMAP_RANDOM --- src/llama-mmap.cpp | 285 ------------------------------------- src/llama-mmap.h | 31 ---- src/llama-model-loader.cpp | 14 +- src/llama-model-loader.h | 4 - src/llama-model.cpp | 62 -------- src/llama-model.h | 13 -- src/models/models.h | 8 -- src/models/qwen4exp.cpp | 4 - 8 files changed, 2 insertions(+), 419 deletions(-) diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index e3dc783766..ed572da7fb 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -5,9 +5,7 @@ #include "ggml.h" #include -#include #include -#include #include #include #include @@ -440,78 +438,6 @@ void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); } // llama_mmap -llama_mmap_random_mode llama_mmap_random_mode_get() { - // read once: this is consulted per mapping and per gather - static const llama_mmap_random_mode mode = []() { - const char * env = getenv("LLAMA_MMAP_RANDOM"); - if (env == nullptr || strcmp(env, "0") == 0 || env[0] == '\0') { - return LLAMA_MMAP_RANDOM_OFF; - } - if (strcmp(env, "drop") == 0) { - return LLAMA_MMAP_RANDOM_DROP; - } - return LLAMA_MMAP_RANDOM_ON; - }(); - - return mode; -} - -bool llama_mmap_random_prefetch_enabled() { - return llama_mmap_random_mode_get() != LLAMA_MMAP_RANDOM_OFF; -} - -static size_t llama_mmap_page_size() { -#if defined(_WIN32) - SYSTEM_INFO si; - GetSystemInfo(&si); - return (size_t) si.dwPageSize; -#elif defined(_SC_PAGESIZE) - return (size_t) sysconf(_SC_PAGESIZE); -#else - return 4096; -#endif -} - -// the distinct pages the given rows fall on, as offsets into the mapping, merged into runs -// rows are smaller than a page and repeat, so this turns one hint per row into one hint per page -static std::vector> llama_mmap_row_pages( - size_t base_off, size_t stride, size_t row_size, size_t map_size, - const int32_t * rows, size_t n_rows, size_t page_size) { - std::vector pages; - pages.reserve(n_rows); - - for (size_t i = 0; i < n_rows; ++i) { - if (rows[i] < 0) { - continue; - } - const size_t first = base_off + (size_t) rows[i] * stride; - const size_t last = first + row_size; - // a corrupt or unexpected index must not turn into a hint outside the mapping - if (row_size == 0 || last > map_size || first < base_off) { - continue; - } - for (size_t p = first / page_size; p <= (last - 1) / page_size; ++p) { - pages.push_back(p); - } - } - - std::sort(pages.begin(), pages.end()); - pages.erase(std::unique(pages.begin(), pages.end()), pages.end()); - - std::vector> ranges; - for (size_t i = 0; i < pages.size(); ) { - size_t j = i + 1; - while (j < pages.size() && pages[j] == pages[j - 1] + 1) { - ++j; - } - const size_t off = pages[i] * page_size; - ranges.emplace_back(off, std::min((pages[j - 1] - pages[i] + 1) * page_size, map_size - off)); - i = j; - } - - return ranges; -} - struct llama_mmap::impl { #ifdef _POSIX_MAPPED_FILES std::vector> mapped_fragments; @@ -519,7 +445,6 @@ struct llama_mmap::impl { impl(struct llama_file * file, size_t prefetch, bool numa) { size = file->size(); int fd = file->file_id(); - fd_advise = fd; int flags = MAP_SHARED; if (numa) { prefetch = 0; } #ifdef __linux__ @@ -550,82 +475,6 @@ struct llama_mmap::impl { mapped_fragments.emplace_back(0, file->size()); } - // load streams the file once, so it asks for POSIX_FADV_SEQUENTIAL - // sparse gathers want the opposite: readahead buys nothing and costs page cache - // so flip the advice after load, over this range only - void advise_random_range(size_t offset, size_t len, bool drop) { - if (offset >= size || len == 0) { - return; - } - len = std::min(len, size - offset); - - // madvise needs an aligned start and rounds the length up, so round both out - // this can take in one page of the neighbour on each side - const size_t page = llama_mmap_page_size(); - const size_t first = offset & ~(page - 1); - const size_t last = std::min(size, (offset + len + page - 1) & ~(page - 1)); - -#if defined(__linux__) - if (drop) { - // on a shared file map this only tears down our page tables - if (madvise((char *) addr + first, last - first, MADV_DONTNEED)) { - LLAMA_LOG_WARN("warning: madvise(.., MADV_DONTNEED) failed: %s\n", strerror(errno)); - } - // this frees the page cache; it spares partial pages, so a neighbour on the end page keeps its own - if (fd_advise >= 0 && posix_fadvise(fd_advise, (off_t) offset, (off_t) len, POSIX_FADV_DONTNEED)) { - LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_DONTNEED) failed: %s\n", strerror(errno)); - } - } -#else - GGML_UNUSED(drop); -#endif - // no POSIX_FADV_RANDOM here: it marks the whole file, and its FMODE_RANDOM only affects read(), not faults - if (posix_madvise((char *) addr + first, last - first, POSIX_MADV_RANDOM)) { - LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n", strerror(errno)); - } - } - - void prefetch_except(const std::vector> & skip) { - const size_t page = llama_mmap_page_size(); - - size_t pos = 0; - for (const auto & [off, len] : skip) { - const size_t first = off & ~(page - 1); - if (first > pos) { - prefetch_range(pos, first - pos); - } - pos = std::max(pos, std::min(size, (off + len + page - 1) & ~(page - 1))); - } - if (pos < size) { - prefetch_range(pos, size - pos); - } - } - - void prefetch_range(size_t offset, size_t len) const { - if (posix_madvise((char *) addr + offset, len, POSIX_MADV_WILLNEED)) { - LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n", strerror(errno)); - } - } - - void prefetch_rows(const void * base, size_t stride, size_t row_size, - const int32_t * rows, size_t n_rows) const { -#if defined(_POSIX_MAPPED_FILES) - const size_t base_off = (const char *) base - (const char *) addr; - - for (const auto & [off, len] : llama_mmap_row_pages( - base_off, stride, row_size, size, rows, n_rows, llama_mmap_page_size())) { - // unchecked on purpose: thousands of hints per batch, and a failed one only costs the fault it would have saved - posix_madvise((char *) addr + off, len, POSIX_MADV_WILLNEED); - } -#else - GGML_UNUSED(base); - GGML_UNUSED(stride); - GGML_UNUSED(row_size); - GGML_UNUSED(rows); - GGML_UNUSED(n_rows); -#endif - } - static void align_range(size_t * first, size_t * last, size_t page_size) { size_t offset_in_page = *first & (page_size - 1); size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page; @@ -733,90 +582,6 @@ struct llama_mmap::impl { GGML_UNUSED(last); } - // Windows has no "read this range randomly" hint; not pulling the range in is what keeps the pages out - void advise_random_range(size_t offset, size_t len, bool drop) { - GGML_UNUSED(offset); - GGML_UNUSED(len); - GGML_UNUSED(drop); - } - - void prefetch_except(const std::vector> & skip) { -#if _WIN32_WINNT >= 0x602 - BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG); - HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll"); - - pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory"); - if (!pPrefetchVirtualMemory) { - return; - } - - const size_t page = llama_mmap_page_size(); - - std::vector entries; - size_t pos = 0; - for (const auto & [off, len] : skip) { - const size_t first = off & ~(page - 1); - if (first > pos) { - WIN32_MEMORY_RANGE_ENTRY e; - e.VirtualAddress = (char *) addr + pos; - e.NumberOfBytes = (SIZE_T) (first - pos); - entries.push_back(e); - } - pos = std::max(pos, std::min(size, (off + len + page - 1) & ~(page - 1))); - } - if (pos < size) { - WIN32_MEMORY_RANGE_ENTRY e; - e.VirtualAddress = (char *) addr + pos; - e.NumberOfBytes = (SIZE_T) (size - pos); - entries.push_back(e); - } - - if (!entries.empty() && !pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0)) { - LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n", - llama_format_win_err(GetLastError()).c_str()); - } -#else - GGML_UNUSED(skip); - LLAMA_LOG_DEBUG("skipping PrefetchVirtualMemory because _WIN32_WINNT < 0x602\n"); -#endif - } - - // PrefetchVirtualMemory takes every range in one call, so the reads go out together instead of one fault at a time - void prefetch_rows(const void * base, size_t stride, size_t row_size, - const int32_t * rows, size_t n_rows) const { -#if _WIN32_WINNT >= 0x602 - BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG); - HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll"); - - pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory"); - if (!pPrefetchVirtualMemory) { - return; - } - - const size_t base_off = (const char *) base - (const char *) addr; - - std::vector entries; - for (const auto & [off, len] : llama_mmap_row_pages( - base_off, stride, row_size, size, rows, n_rows, llama_mmap_page_size())) { - WIN32_MEMORY_RANGE_ENTRY e; - e.VirtualAddress = (char *) addr + off; - e.NumberOfBytes = (SIZE_T) len; - entries.push_back(e); - } - - if (!entries.empty()) { - // unchecked for the same reason as the POSIX branch: it is only a hint - pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0); - } -#else - GGML_UNUSED(base); - GGML_UNUSED(stride); - GGML_UNUSED(row_size); - GGML_UNUSED(rows); - GGML_UNUSED(n_rows); -#endif - } - ~impl() { if (hMapping) { if (addr) { @@ -846,45 +611,10 @@ struct llama_mmap::impl { throw std::runtime_error("mmap not supported"); } - - void advise_random_range(size_t offset, size_t len, bool drop) { - GGML_UNUSED(offset); - GGML_UNUSED(len); - GGML_UNUSED(drop); - - throw std::runtime_error("mmap not supported"); - } - - void prefetch_except(const std::vector> & skip) { - GGML_UNUSED(skip); - - throw std::runtime_error("mmap not supported"); - } - - void prefetch_rows(const void * base, size_t stride, size_t row_size, - const int32_t * rows, size_t n_rows) const { - GGML_UNUSED(base); - GGML_UNUSED(stride); - GGML_UNUSED(row_size); - GGML_UNUSED(rows); - GGML_UNUSED(n_rows); - - throw std::runtime_error("mmap not supported"); - } #endif - bool contains(const void * ptr, size_t len) const { - const char * p = (const char *) ptr; - const char * b = (const char *) addr; - - return p >= b && len <= size && (size_t) (p - b) <= size - len; - } - void * addr; size_t size; - - // the fd is kept only to re-advise the file; the mapping owns no reference to it - int fd_advise = -1; }; llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique(file, prefetch, numa)) {} @@ -895,21 +625,6 @@ void * llama_mmap::addr() const { return pimpl->addr; } void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); } -void llama_mmap::advise_random_range(size_t offset, size_t len, bool drop) { - pimpl->advise_random_range(offset, len, drop); -} - -void llama_mmap::prefetch_except(const std::vector> & skip) { - pimpl->prefetch_except(skip); -} - -bool llama_mmap::contains(const void * ptr, size_t len) const { return pimpl->contains(ptr, len); } - -void llama_mmap::prefetch_rows(const void * base, size_t stride, size_t row_size, - const int32_t * rows, size_t n_rows) const { - pimpl->prefetch_rows(base, stride, row_size, rows, n_rows); -} - #if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32) const bool llama_mmap::SUPPORTED = true; #else diff --git a/src/llama-mmap.h b/src/llama-mmap.h index c97a97a2d6..b7d5c61e95 100644 --- a/src/llama-mmap.h +++ b/src/llama-mmap.h @@ -3,7 +3,6 @@ #include #include #include -#include #include struct llama_file; @@ -51,22 +50,6 @@ struct llama_mmap { void unmap_fragment(size_t first, size_t last); - // opt-in, see llama_mmap_random_mode(). marks one byte range as randomly accessed - // only correct after load, since the loader streams the file sequentially - // offsets are into the file, which is also the mapping offset; the range is rounded out to whole pages - void advise_random_range(size_t offset, size_t len, bool drop); - - // eager pull-in for everything outside the given ranges, in place of the constructor's whole-file one - // used when part of the file must not be read ahead. ranges must be sorted - void prefetch_except(const std::vector> & skip); - - // true if [ptr, ptr + len) lies inside this mapping - bool contains(const void * ptr, size_t len) const; - - // ask the kernel to start reading the given rows, as one batch so the faults overlap - void prefetch_rows(const void * base, size_t stride, size_t row_size, - const int32_t * rows, size_t n_rows) const; - static const bool SUPPORTED; private: @@ -74,20 +57,6 @@ private: std::unique_ptr pimpl; }; -// how the model file mappings should be advised, from the LLAMA_MMAP_RANDOM environment variable. -// off unless the user asks: the random hints cost a lot of cold-prefill time on tensors that are read sequentially -enum llama_mmap_random_mode { - LLAMA_MMAP_RANDOM_OFF = 0, // upstream behaviour - LLAMA_MMAP_RANDOM_ON = 1, // advise the gather tables random after load, do not pull them in - LLAMA_MMAP_RANDOM_DROP = 2, // additionally drop what the load pulled in -}; - -llama_mmap_random_mode llama_mmap_random_mode_get(); - -// batched readahead for a sparse gather, not separately switchable -// MADV_RANDOM kills the kernel's readahead, so without this the gather faults once per row and runs 2.6x slower -bool llama_mmap_random_prefetch_enabled(); - struct llama_mlock { llama_mlock(); ~llama_mlock(); diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index ed18755439..5071556a1f 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1354,8 +1354,7 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps if (use_mmap) { mappings.reserve(files.size()); mmaps_used.reserve(files.size()); - for (size_t i = 0; i < files.size(); ++i) { - const auto & file = files[i]; + for (const auto & file : files) { bool is_numa = false; auto * dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); @@ -1367,16 +1366,7 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps } } - const auto no_prefetch = mmap_no_prefetch.find((uint16_t) i); - - // the eager pull-in would read a whole gather table to fill pages the gathers barely touch - // skip it for this file and ask for the rest, so streamed tensors keep their readahead - const bool split_prefetch = prefetch && !is_numa && no_prefetch != mmap_no_prefetch.end(); - - std::unique_ptr mapping = std::make_unique(file.get(), prefetch && !split_prefetch ? -1 : 0, is_numa); - if (split_prefetch) { - mapping->prefetch_except(no_prefetch->second); - } + std::unique_ptr mapping = std::make_unique(file.get(), prefetch ? -1 : 0, is_numa); mmaps_used.emplace_back(mapping->size(), 0); if (mlock_mmaps) { std::unique_ptr mlock_mmap(new llama_mlock()); diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index f7ff9cae1d..e9fe3592d4 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -88,10 +88,6 @@ struct llama_model_loader { llama_mmaps mappings; - // byte ranges, per source file, that init_mappings() must not pull in eagerly - // these are gather tables the model reads a few percent of. set under LLAMA_MMAP_RANDOM only, sorted by offset - std::map>> mmap_no_prefetch; - std::map weights_map; std::unordered_map kv_overrides; const llama_model_tensor_buft_override * tensor_buft_overrides; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 1e2b096b6f..e4f9f7bea4 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1148,17 +1148,6 @@ struct llama_model::impl { // model memory mapped files llama_mmaps mappings; - // gather tables that really came out of a mapping, resolved from gather_tables() during load - // empty unless the user opted in, which is all the feature costs when off - struct gather_range { - const ggml_tensor * tensor; - uint16_t idx; // source file, and so the mapping - size_t offs; // byte offset into that file - size_t len; - }; - - std::vector gather_ranges; - // objects representing data potentially being locked in memory llama_mlocks mlock_bufs; llama_mlocks mlock_mmaps; @@ -1689,21 +1678,6 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } } - // kept local until the mappings exist: gather_ranges must only hold ranges checked against a live mapping - std::vector nominated; - if (llama_mmap_random_mode_get() != LLAMA_MMAP_RANDOM_OFF) { - for (const ggml_tensor * t : gather_tables()) { - const auto * w = t ? ml.get_weight(ggml_get_name(t)) : nullptr; - if (w) { - nominated.push_back({ t, w->idx, w->offs, ggml_nbytes(w->tensor) }); - ml.mmap_no_prefetch[w->idx].emplace_back(w->offs, ggml_nbytes(w->tensor)); - } - } - for (auto & [_, ranges] : ml.mmap_no_prefetch) { - std::sort(ranges.begin(), ranges.end()); - } - } - ml.init_mappings(true, use_mlock ? &pimpl->mlock_mmaps : nullptr); pimpl->mappings.reserve(ml.mappings.size()); @@ -1837,47 +1811,11 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { for (auto & mapping : ml.mappings) { pimpl->mappings.emplace_back(std::move(mapping)); } - - // only safe once every tensor is read: the load is a sequential pass and wants its readahead - const llama_mmap_random_mode random_mode = llama_mmap_random_mode_get(); - - // a named tensor not served from its mapping was offloaded or copied, so nothing gathers from the file - // drop it rather than advise it - for (const auto & r : nominated) { - if (r.idx < pimpl->mappings.size() && pimpl->mappings[r.idx]->contains(r.tensor->data, r.len)) { - pimpl->gather_ranges.push_back(r); - } - } - - for (const auto & r : pimpl->gather_ranges) { - pimpl->mappings[r.idx]->advise_random_range(r.offs, r.len, random_mode == LLAMA_MMAP_RANDOM_DROP); - - LLAMA_LOG_INFO("%s: LLAMA_MMAP_RANDOM: %s advised for random access, %.2f MiB%s\n", - __func__, ggml_get_name(r.tensor), r.len / 1024.0 / 1024.0, - random_mode == LLAMA_MMAP_RANDOM_DROP ? ", dropped cached pages" : ""); - } } return true; } -void llama_model::prefetch_rows(const struct ggml_tensor * t, const int32_t * rows, size_t n_rows) const { - if (pimpl->gather_ranges.empty() || t == nullptr || t->data == nullptr || n_rows == 0) { - return; - } - if (!llama_mmap_random_prefetch_enabled()) { - return; - } - - // keyed off the tensor, not the mapping: the readahead must land where the advice did - for (const auto & r : pimpl->gather_ranges) { - if (r.tensor == t) { - pimpl->mappings[r.idx]->prefetch_rows(t->data, t->nb[1], ggml_row_size(t->type, t->ne[0]), rows, n_rows); - return; - } - } -} - ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags) { const buft_list_t * buft_list_layer = tn.bid == -1 ? nullptr : pimpl->dev_layer.at(tn.bid).buft_list; return ml.create_tensor( diff --git a/src/llama-model.h b/src/llama-model.h index 30d5e86d35..a2c25c6381 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -734,19 +734,6 @@ struct llama_model { const struct ggml_tensor * get_tensor(const char * name) const; - // ask the kernel to start reading the rows a gather is about to take from a host-mapped tensor - // the faults then overlap instead of paying one NVMe latency at a time - // - // does nothing unless gather_tables() named the tensor and it really is read from a mapping - // for anything else this is one empty-vector test - void prefetch_rows(const struct ggml_tensor * t, const int32_t * rows, size_t n_rows) const; - - // tensors that stay host-resident and are read by sparse row gathers, not streamed once - // under LLAMA_MMAP_RANDOM these get the random advice and the batched readahead of prefetch_rows() - // - // named by the model, not guessed from size: a big tensor read in full, like token_embd on CPU, still wants readahead - virtual std::vector gather_tables() const { return {}; } - float get_rope_freq_base (const llama_cparams & cparams, int il) const; float get_rope_freq_scale(const llama_cparams & cparams, int il) const; diff --git a/src/models/models.h b/src/models/models.h index 8059601a2f..ddc56bcac0 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2281,14 +2281,6 @@ struct llama_model_qwen4exp : public llama_model_base { void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; - // the PLE n-gram table is far too big to offload and is read by 16 tiny gathers per token - std::vector gather_tables() const override { - if (per_layer_tok_embd == nullptr) { - return {}; - } - return { per_layer_tok_embd }; - } - struct graph : public llm_build_delta_net_base { graph(const llama_model & model, const llm_graph_params & params); private: diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 4ba057c1e5..917a953453 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -974,10 +974,6 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) { } } - // the table is too big to offload, so it is gathered from the mapping: 16 faults per token, no two on the same page - // get_rows would take them one at a time; queued here they are in flight before the graph runs - pmodel.prefetch_rows(pmodel.per_layer_tok_embd, idx.data(), idx.size()); - ggml_backend_tensor_set(rows, idx.data(), 0, idx.size()*ggml_element_size(rows)); }