From 95da4ba86065dda8659981a508dfb52d8c3f1032 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Thu, 27 Aug 2026 04:20:48 +0000 Subject: [PATCH] llama: opt-in random-access mmap advice for host-resident gather tables qwen4exp keeps per_layer_token_embd on the host: 26.8 GiB at IQ4_NL, read by ggml_get_rows as 16 gathers of ~90-170 bytes per token, spread across 16 head regions ~20M rows apart. Measured over 4.75M gathers, no two consecutive gathers land on the same 4 KiB page, so the readahead the loader asks for buys nothing here and the whole table ends up cached to serve about 4% of itself. llama_mmap applies POSIX_FADV_SEQUENTIAL, MAP_POPULATE and a whole-file POSIX_MADV_WILLNEED unconditionally. Those are right for streaming the file once into buffers and wrong for whatever stays mapped afterwards. Under LLAMA_MMAP_RANDOM the eager pull-in is skipped and the mapping is advised random once every tensor has been read, so the load itself keeps its sequential readahead. That alone drops the table to 4.4% resident but serializes one NVMe latency per gather. The second half is what pays for it: the PLE input already computes every row index for the ubatch before the graph runs, so the pages those rows fall on are handed to the kernel in one batch and the reads overlap. POSIX_MADV_WILLNEED on POSIX, PrefetchVirtualMemory on Windows, which takes the discontiguous ranges in a single call. Off by default and off for every other model: the batched prefetch keys off "this mapping was advised random", which nothing sets unless the user opts in. -c 512 --chunks 60, cold, IQ1_S, mean of 3: default 35.3 s 26.82 GiB resident (100%) advice only 104.5 s 1.19 GiB resident (4.4%) advice + prefetch 34.2 s 1.19 GiB resident (4.4%) PPL 4.2346 +/- 0.07862 in all three. IQ1_S KLD is unchanged in every field, including Mean KLD 0.396070 +/- 0.001931 and Same top p 77.325%. --- src/llama-mmap.cpp | 230 ++++++++++++++++++++++++++++++++++++++++ src/llama-mmap.h | 30 ++++++ src/llama-model.cpp | 37 +++++++ src/llama-model.h | 8 ++ src/models/qwen4exp.cpp | 5 + 5 files changed, 310 insertions(+) diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index ed572da7fb..33167c7280 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -5,7 +5,9 @@ #include "ggml.h" #include +#include #include +#include #include #include #include @@ -438,6 +440,87 @@ void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); } // llama_mmap +llama_mmap_random_mode llama_mmap_random_mode_get() { + // read once: this is consulted per mapping and per gather + static const llama_mmap_random_mode mode = []() { + const char * env = getenv("LLAMA_MMAP_RANDOM"); + if (env == nullptr || strcmp(env, "0") == 0 || env[0] == '\0') { + return LLAMA_MMAP_RANDOM_OFF; + } + if (strcmp(env, "drop") == 0) { + return LLAMA_MMAP_RANDOM_DROP; + } + return LLAMA_MMAP_RANDOM_ON; + }(); + + return mode; +} + +bool llama_mmap_random_prefetch_enabled() { + // on with the feature, so the batched readahead that pays for the random hints cannot be + // left off by accident. separate only so the two halves can be measured apart. + static const bool enabled = []() { + const char * env = getenv("LLAMA_MMAP_RANDOM_PREFETCH"); + return env == nullptr ? true : strcmp(env, "0") != 0; + }(); + + return llama_mmap_random_mode_get() != LLAMA_MMAP_RANDOM_OFF && enabled; +} + +static size_t llama_mmap_page_size() { +#if defined(_WIN32) + SYSTEM_INFO si; + GetSystemInfo(&si); + return (size_t) si.dwPageSize; +#elif defined(_SC_PAGESIZE) + return (size_t) sysconf(_SC_PAGESIZE); +#else + return 4096; +#endif +} + +// the distinct pages the given rows fall on, as offsets into the mapping, merged into runs. +// a row is much smaller than a page and rows repeat within a batch, so this is what turns a +// hint per row into a hint per page. platform independent: the callers differ only in which +// syscall they hand the result to. +static std::vector> llama_mmap_row_pages( + size_t base_off, size_t stride, size_t row_size, size_t map_size, + const int32_t * rows, size_t n_rows, size_t page_size) { + std::vector pages; + pages.reserve(n_rows); + + for (size_t i = 0; i < n_rows; ++i) { + if (rows[i] < 0) { + continue; + } + const size_t first = base_off + (size_t) rows[i] * stride; + const size_t last = first + row_size; + // a corrupt or unexpected index must not turn into a hint outside the mapping + if (row_size == 0 || last > map_size || first < base_off) { + continue; + } + for (size_t p = first / page_size; p <= (last - 1) / page_size; ++p) { + pages.push_back(p); + } + } + + std::sort(pages.begin(), pages.end()); + pages.erase(std::unique(pages.begin(), pages.end()), pages.end()); + + std::vector> ranges; + for (size_t i = 0; i < pages.size(); ) { + size_t j = i + 1; + while (j < pages.size() && pages[j] == pages[j - 1] + 1) { + ++j; + } + const size_t off = pages[i] * page_size; + ranges.emplace_back(off, std::min((pages[j - 1] - pages[i] + 1) * page_size, map_size - off)); + i = j; + } + + return ranges; +} + struct llama_mmap::impl { #ifdef _POSIX_MAPPED_FILES std::vector> mapped_fragments; @@ -445,8 +528,13 @@ struct llama_mmap::impl { impl(struct llama_file * file, size_t prefetch, bool numa) { size = file->size(); int fd = file->file_id(); + fd_advise = fd; int flags = MAP_SHARED; if (numa) { prefetch = 0; } + // with the random hints the eager pull-in is pure waste: it reads the whole file to + // populate pages that the gathers will then hit at most a few percent of. the loader + // still gets sequential readahead for the tensors it actually copies out. + if (llama_mmap_random_mode_get() != LLAMA_MMAP_RANDOM_OFF) { prefetch = 0; } #ifdef __linux__ if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) { LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n", @@ -475,6 +563,54 @@ struct llama_mmap::impl { mapped_fragments.emplace_back(0, file->size()); } + // the load path asks for POSIX_FADV_SEQUENTIAL, which is right while the file is being + // streamed once into buffers and wrong for whatever stays host-resident afterwards: those + // tensors are read by sparse gathers, where readahead and fault-around buy nothing and cost + // page cache. flipping the advice only after load keeps both halves happy. + void advise_random(bool drop) { +#if defined(__linux__) + if (drop) { + // hand back what the load pulled in; the hot pages fault back on demand + if (madvise(addr, size, MADV_DONTNEED)) { + LLAMA_LOG_WARN("warning: madvise(.., MADV_DONTNEED) failed: %s\n", strerror(errno)); + } + if (fd_advise >= 0 && posix_fadvise(fd_advise, 0, 0, POSIX_FADV_DONTNEED)) { + LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_DONTNEED) failed: %s\n", strerror(errno)); + } + } + if (fd_advise >= 0 && posix_fadvise(fd_advise, 0, 0, POSIX_FADV_RANDOM)) { + LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_RANDOM) failed: %s\n", strerror(errno)); + } +#else + GGML_UNUSED(drop); +#endif +#if defined(_POSIX_MAPPED_FILES) + if (posix_madvise(addr, size, POSIX_MADV_RANDOM)) { + LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n", strerror(errno)); + } +#endif + } + + void prefetch_rows(const void * base, size_t stride, size_t row_size, + const int32_t * rows, size_t n_rows) const { +#if defined(_POSIX_MAPPED_FILES) + const size_t base_off = (const char *) base - (const char *) addr; + + for (const auto & [off, len] : llama_mmap_row_pages( + base_off, stride, row_size, size, rows, n_rows, llama_mmap_page_size())) { + // deliberately unchecked: this is a hint issued thousands of times per batch, and a + // failed hint only costs the fault it would have avoided + posix_madvise((char *) addr + off, len, POSIX_MADV_WILLNEED); + } +#else + GGML_UNUSED(base); + GGML_UNUSED(stride); + GGML_UNUSED(row_size); + GGML_UNUSED(rows); + GGML_UNUSED(n_rows); +#endif + } + static void align_range(size_t * first, size_t * last, size_t page_size) { size_t offset_in_page = *first & (page_size - 1); size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page; @@ -547,6 +683,9 @@ struct llama_mmap::impl { throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str())); } + // see the POSIX branch: opting in means the eager pull-in is waste + if (llama_mmap_random_mode_get() != LLAMA_MMAP_RANDOM_OFF) { prefetch = 0; } + addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0); DWORD error = GetLastError(); @@ -582,6 +721,50 @@ struct llama_mmap::impl { GGML_UNUSED(last); } + // Windows has no per-mapping "read this randomly" hint. skipping the eager + // PrefetchVirtualMemory in the constructor is what keeps the pages out; there is nothing + // further to say here, and nothing to drop back. + void advise_random(bool drop) { + GGML_UNUSED(drop); + } + + // PrefetchVirtualMemory takes the whole set of ranges in one call, which is exactly the + // batching this wants: the reads are issued together instead of one fault at a time. + void prefetch_rows(const void * base, size_t stride, size_t row_size, + const int32_t * rows, size_t n_rows) const { +#if _WIN32_WINNT >= 0x602 + BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG); + HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll"); + + pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory"); + if (!pPrefetchVirtualMemory) { + return; + } + + const size_t base_off = (const char *) base - (const char *) addr; + + std::vector entries; + for (const auto & [off, len] : llama_mmap_row_pages( + base_off, stride, row_size, size, rows, n_rows, llama_mmap_page_size())) { + WIN32_MEMORY_RANGE_ENTRY e; + e.VirtualAddress = (char *) addr + off; + e.NumberOfBytes = (SIZE_T) len; + entries.push_back(e); + } + + if (!entries.empty()) { + // unchecked for the same reason as the POSIX branch: it is only a hint + pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0); + } +#else + GGML_UNUSED(base); + GGML_UNUSED(stride); + GGML_UNUSED(row_size); + GGML_UNUSED(rows); + GGML_UNUSED(n_rows); +#endif + } + ~impl() { if (hMapping) { if (addr) { @@ -611,10 +794,38 @@ struct llama_mmap::impl { throw std::runtime_error("mmap not supported"); } + + void advise_random(bool drop) { + GGML_UNUSED(drop); + + throw std::runtime_error("mmap not supported"); + } + + void prefetch_rows(const void * base, size_t stride, size_t row_size, + const int32_t * rows, size_t n_rows) const { + GGML_UNUSED(base); + GGML_UNUSED(stride); + GGML_UNUSED(row_size); + GGML_UNUSED(rows); + GGML_UNUSED(n_rows); + + throw std::runtime_error("mmap not supported"); + } #endif + bool contains(const void * ptr, size_t len) const { + const char * p = (const char *) ptr; + const char * b = (const char *) addr; + + return p >= b && len <= size && (size_t) (p - b) <= size - len; + } + void * addr; size_t size; + + // the fd is kept only to re-advise the file; the mapping owns no reference to it + int fd_advise = -1; + bool random = false; }; llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique(file, prefetch, numa)) {} @@ -625,6 +836,25 @@ void * llama_mmap::addr() const { return pimpl->addr; } void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); } +void llama_mmap::advise_random(bool drop) { + pimpl->advise_random(drop); + // set here rather than per platform: the flag means "the caller opted this mapping in", and + // the batched prefetch is worth doing even where the advice itself is a no-op + pimpl->random = true; +} + +bool llama_mmap::is_random() const { return pimpl->random; } + +bool llama_mmap::contains(const void * ptr, size_t len) const { return pimpl->contains(ptr, len); } + +void llama_mmap::prefetch_rows(const void * base, size_t stride, size_t row_size, + const int32_t * rows, size_t n_rows) const { + if (!pimpl->random) { + return; + } + pimpl->prefetch_rows(base, stride, row_size, rows, n_rows); +} + #if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32) const bool llama_mmap::SUPPORTED = true; #else diff --git a/src/llama-mmap.h b/src/llama-mmap.h index b7d5c61e95..ab4ef60408 100644 --- a/src/llama-mmap.h +++ b/src/llama-mmap.h @@ -50,6 +50,22 @@ struct llama_mmap { void unmap_fragment(size_t first, size_t last); + // opt-in, see llama_mmap_random_mode(). marks the whole mapping as randomly accessed, which + // is only correct once loading is done: until then the loader streams the file sequentially. + void advise_random(bool drop); + + // true once advise_random() has been applied. everything that keys off "this mapping is read + // randomly" tests this, so nothing changes for a mapping the user did not opt in for. + bool is_random() const; + + // true if [ptr, ptr + len) lies inside this mapping + bool contains(const void * ptr, size_t len) const; + + // ask the kernel to start reading the given rows. issued as one batch so the faults overlap + // instead of serializing; a no-op unless the mapping is marked random. + void prefetch_rows(const void * base, size_t stride, size_t row_size, + const int32_t * rows, size_t n_rows) const; + static const bool SUPPORTED; private: @@ -57,6 +73,20 @@ private: std::unique_ptr pimpl; }; +// how the model file mappings should be advised, from the LLAMA_MMAP_RANDOM environment variable. +// off unless the user asks: the random hints cost a large cold-prefill slowdown on models whose +// host-resident tensors are read sequentially, so this cannot be a default. +enum llama_mmap_random_mode { + LLAMA_MMAP_RANDOM_OFF = 0, // upstream behaviour + LLAMA_MMAP_RANDOM_ON = 1, // skip MAP_POPULATE/WILLNEED, advise random after load + LLAMA_MMAP_RANDOM_DROP = 2, // additionally drop what the load pulled in +}; + +llama_mmap_random_mode llama_mmap_random_mode_get(); + +// whether batched readahead ahead of a sparse gather is enabled (LLAMA_MMAP_RANDOM_PREFETCH) +bool llama_mmap_random_prefetch_enabled(); + struct llama_mlock { llama_mlock(); ~llama_mlock(); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index f8b4a30c97..925e2aca61 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1149,6 +1149,10 @@ struct llama_model::impl { // model memory mapped files llama_mmaps mappings; + // set once loading is done, if any mapping was advised random. lets prefetch_rows() bail out + // without walking the mappings, which is the only cost the feature has when it is off. + bool mappings_random = false; + // objects representing data potentially being locked in memory llama_mlocks mlock_bufs; llama_mlocks mlock_mmaps; @@ -1812,11 +1816,44 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { for (auto & mapping : ml.mappings) { pimpl->mappings.emplace_back(std::move(mapping)); } + + // only now that every tensor has been read is it safe to say the file is read randomly: + // the load itself is a sequential pass and wants the readahead it has been getting. + const llama_mmap_random_mode random_mode = llama_mmap_random_mode_get(); + if (random_mode != LLAMA_MMAP_RANDOM_OFF) { + for (auto & mapping : pimpl->mappings) { + mapping->advise_random(random_mode == LLAMA_MMAP_RANDOM_DROP); + } + pimpl->mappings_random = !pimpl->mappings.empty(); + + LLAMA_LOG_INFO("%s: LLAMA_MMAP_RANDOM: advised %zu mapping(s) for random access%s\n", + __func__, pimpl->mappings.size(), + random_mode == LLAMA_MMAP_RANDOM_DROP ? ", dropped cached pages" : ""); + } } return true; } +void llama_model::prefetch_rows(const struct ggml_tensor * t, const int32_t * rows, size_t n_rows) const { + if (!pimpl->mappings_random || t == nullptr || t->data == nullptr || n_rows == 0) { + return; + } + if (!llama_mmap_random_prefetch_enabled()) { + return; + } + + // rows are addressed off the tensor, so the whole tensor has to sit in the mapping we find + const size_t nbytes = ggml_nbytes(t); + + for (const auto & mapping : pimpl->mappings) { + if (mapping->is_random() && mapping->contains(t->data, nbytes)) { + mapping->prefetch_rows(t->data, t->nb[1], ggml_row_size(t->type, t->ne[0]), rows, n_rows); + return; + } + } +} + ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags) { const buft_list_t * buft_list_layer = tn.bid == -1 ? nullptr : pimpl->dev_layer.at(tn.bid).buft_list; return ml.create_tensor( diff --git a/src/llama-model.h b/src/llama-model.h index a2c25c6381..cf7c75a476 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -734,6 +734,14 @@ struct llama_model { const struct ggml_tensor * get_tensor(const char * name) const; + // ask the kernel to start reading the rows a gather is about to take out of a host-mapped + // tensor, so the faults overlap instead of serializing one NVMe latency at a time. + // + // does nothing unless the tensor lives in a mapping that was advised random, which only + // happens under LLAMA_MMAP_RANDOM. off, and for anything not read out of a mapping (offloaded + // tensors, --load-mode none, non-POSIX hosts), this is one bool test. + void prefetch_rows(const struct ggml_tensor * t, const int32_t * rows, size_t n_rows) const; + float get_rope_freq_base (const llama_cparams & cparams, int il) const; float get_rope_freq_scale(const llama_cparams & cparams, int il) const; diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 533e9a300b..34282aa0c4 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -992,6 +992,11 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) { h.next_pos = pos + 1; } + // the table is far too big to offload, so it is gathered straight out of the mapping: one + // fault per row, 16 per token, no two of them on the same page. left to the get_rows those + // faults happen one at a time; queued here they are in flight before the graph even runs. + pmodel.prefetch_rows(pmodel.per_layer_tok_embd, idx.data(), idx.size()); + ggml_backend_tensor_set(rows, idx.data(), 0, idx.size()*ggml_element_size(rows)); }