llama: opt-in random-access mmap advice for host-resident gather tables

qwen4exp keeps per_layer_token_embd on the host: 26.8 GiB at IQ4_NL, read
by ggml_get_rows as 16 gathers of ~90-170 bytes per token, spread across
16 head regions ~20M rows apart. Measured over 4.75M gathers, no two
consecutive gathers land on the same 4 KiB page, so the readahead the
loader asks for buys nothing here and the whole table ends up cached to
serve about 4% of itself.

llama_mmap applies POSIX_FADV_SEQUENTIAL, MAP_POPULATE and a whole-file
POSIX_MADV_WILLNEED unconditionally. Those are right for streaming the
file once into buffers and wrong for whatever stays mapped afterwards.

Under LLAMA_MMAP_RANDOM the eager pull-in is skipped and the mapping is
advised random once every tensor has been read, so the load itself keeps
its sequential readahead. That alone drops the table to 4.4% resident but
serializes one NVMe latency per gather.

The second half is what pays for it: the PLE input already computes every
row index for the ubatch before the graph runs, so the pages those rows
fall on are handed to the kernel in one batch and the reads overlap.
POSIX_MADV_WILLNEED on POSIX, PrefetchVirtualMemory on Windows, which
takes the discontiguous ranges in a single call.

Off by default and off for every other model: the batched prefetch keys
off "this mapping was advised random", which nothing sets unless the user
opts in.

  -c 512 --chunks 60, cold, IQ1_S, mean of 3:

    default            35.3 s   26.82 GiB resident (100%)
    advice only       104.5 s    1.19 GiB resident (4.4%)
    advice + prefetch  34.2 s    1.19 GiB resident (4.4%)

  PPL 4.2346 +/- 0.07862 in all three. IQ1_S KLD is unchanged in every
  field, including Mean KLD 0.396070 +/- 0.001931 and Same top p 77.325%.

src/llama-mmap.cpp

@@ -5,7 +5,9 @@
 #include "ggml.h"
 
 #include <cstring>
+#include <cstdlib>
 #include <climits>
+#include <vector>
 #include <stdexcept>
 #include <cerrno>
 #include <algorithm>
@@ -438,6 +440,87 @@ void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }
 
 // llama_mmap
 
+llama_mmap_random_mode llama_mmap_random_mode_get() {
+    // read once: this is consulted per mapping and per gather
+    static const llama_mmap_random_mode mode = []() {
+        const char * env = getenv("LLAMA_MMAP_RANDOM");
+        if (env == nullptr || strcmp(env, "0") == 0 || env[0] == '\0') {
+            return LLAMA_MMAP_RANDOM_OFF;
+        }
+        if (strcmp(env, "drop") == 0) {
+            return LLAMA_MMAP_RANDOM_DROP;
+        }
+        return LLAMA_MMAP_RANDOM_ON;
+    }();
+
+    return mode;
+}
+
+bool llama_mmap_random_prefetch_enabled() {
+    // on with the feature, so the batched readahead that pays for the random hints cannot be
+    // left off by accident. separate only so the two halves can be measured apart.
+    static const bool enabled = []() {
+        const char * env = getenv("LLAMA_MMAP_RANDOM_PREFETCH");
+        return env == nullptr ? true : strcmp(env, "0") != 0;
+    }();
+
+    return llama_mmap_random_mode_get() != LLAMA_MMAP_RANDOM_OFF && enabled;
+}
+
+static size_t llama_mmap_page_size() {
+#if defined(_WIN32)
+    SYSTEM_INFO si;
+    GetSystemInfo(&si);
+    return (size_t) si.dwPageSize;
+#elif defined(_SC_PAGESIZE)
+    return (size_t) sysconf(_SC_PAGESIZE);
+#else
+    return 4096;
+#endif
+}
+
+// the distinct pages the given rows fall on, as offsets into the mapping, merged into runs.
+// a row is much smaller than a page and rows repeat within a batch, so this is what turns a
+// hint per row into a hint per page. platform independent: the callers differ only in which
+// syscall they hand the result to.
+static std::vector<std::pair<size_t, size_t>> llama_mmap_row_pages(
+        size_t base_off, size_t stride, size_t row_size, size_t map_size,
+        const int32_t * rows, size_t n_rows, size_t page_size) {
+    std::vector<size_t> pages;
+    pages.reserve(n_rows);
+
+    for (size_t i = 0; i < n_rows; ++i) {
+        if (rows[i] < 0) {
+            continue;
+        }
+        const size_t first = base_off + (size_t) rows[i] * stride;
+        const size_t last  = first + row_size;
+        // a corrupt or unexpected index must not turn into a hint outside the mapping
+        if (row_size == 0 || last > map_size || first < base_off) {
+            continue;
+        }
+        for (size_t p = first / page_size; p <= (last - 1) / page_size; ++p) {
+            pages.push_back(p);
+        }
+    }
+
+    std::sort(pages.begin(), pages.end());
+    pages.erase(std::unique(pages.begin(), pages.end()), pages.end());
+
+    std::vector<std::pair<size_t, size_t>> ranges;
+    for (size_t i = 0; i < pages.size(); ) {
+        size_t j = i + 1;
+        while (j < pages.size() && pages[j] == pages[j - 1] + 1) {
+            ++j;
+        }
+        const size_t off = pages[i] * page_size;
+        ranges.emplace_back(off, std::min((pages[j - 1] - pages[i] + 1) * page_size, map_size - off));
+        i = j;
+    }
+
+    return ranges;
+}
+
 struct llama_mmap::impl {
 #ifdef _POSIX_MAPPED_FILES
     std::vector<std::pair<size_t, size_t>> mapped_fragments;
@@ -445,8 +528,13 @@ struct llama_mmap::impl {
     impl(struct llama_file * file, size_t prefetch, bool numa) {
         size = file->size();
         int fd = file->file_id();
+        fd_advise = fd;
         int flags = MAP_SHARED;
         if (numa) { prefetch = 0; }
+        // with the random hints the eager pull-in is pure waste: it reads the whole file to
+        // populate pages that the gathers will then hit at most a few percent of. the loader
+        // still gets sequential readahead for the tensors it actually copies out.
+        if (llama_mmap_random_mode_get() != LLAMA_MMAP_RANDOM_OFF) { prefetch = 0; }
 #ifdef __linux__
         if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {
             LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",
@@ -475,6 +563,54 @@ struct llama_mmap::impl {
         mapped_fragments.emplace_back(0, file->size());
     }
 
+    // the load path asks for POSIX_FADV_SEQUENTIAL, which is right while the file is being
+    // streamed once into buffers and wrong for whatever stays host-resident afterwards: those
+    // tensors are read by sparse gathers, where readahead and fault-around buy nothing and cost
+    // page cache. flipping the advice only after load keeps both halves happy.
+    void advise_random(bool drop) {
+#if defined(__linux__)
+        if (drop) {
+            // hand back what the load pulled in; the hot pages fault back on demand
+            if (madvise(addr, size, MADV_DONTNEED)) {
+                LLAMA_LOG_WARN("warning: madvise(.., MADV_DONTNEED) failed: %s\n", strerror(errno));
+            }
+            if (fd_advise >= 0 && posix_fadvise(fd_advise, 0, 0, POSIX_FADV_DONTNEED)) {
+                LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_DONTNEED) failed: %s\n", strerror(errno));
+            }
+        }
+        if (fd_advise >= 0 && posix_fadvise(fd_advise, 0, 0, POSIX_FADV_RANDOM)) {
+            LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_RANDOM) failed: %s\n", strerror(errno));
+        }
+#else
+        GGML_UNUSED(drop);
+#endif
+#if defined(_POSIX_MAPPED_FILES)
+        if (posix_madvise(addr, size, POSIX_MADV_RANDOM)) {
+            LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n", strerror(errno));
+        }
+#endif
+    }
+
+    void prefetch_rows(const void * base, size_t stride, size_t row_size,
+                       const int32_t * rows, size_t n_rows) const {
+#if defined(_POSIX_MAPPED_FILES)
+        const size_t base_off = (const char *) base - (const char *) addr;
+
+        for (const auto & [off, len] : llama_mmap_row_pages(
+                    base_off, stride, row_size, size, rows, n_rows, llama_mmap_page_size())) {
+            // deliberately unchecked: this is a hint issued thousands of times per batch, and a
+            // failed hint only costs the fault it would have avoided
+            posix_madvise((char *) addr + off, len, POSIX_MADV_WILLNEED);
+        }
+#else
+        GGML_UNUSED(base);
+        GGML_UNUSED(stride);
+        GGML_UNUSED(row_size);
+        GGML_UNUSED(rows);
+        GGML_UNUSED(n_rows);
+#endif
+    }
+
     static void align_range(size_t * first, size_t * last, size_t page_size) {
         size_t offset_in_page = *first & (page_size - 1);
         size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;
@@ -547,6 +683,9 @@ struct llama_mmap::impl {
             throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));
         }
 
+        // see the POSIX branch: opting in means the eager pull-in is waste
+        if (llama_mmap_random_mode_get() != LLAMA_MMAP_RANDOM_OFF) { prefetch = 0; }
+
         addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
         DWORD error = GetLastError();
 
@@ -582,6 +721,50 @@ struct llama_mmap::impl {
         GGML_UNUSED(last);
     }
 
+    // Windows has no per-mapping "read this randomly" hint. skipping the eager
+    // PrefetchVirtualMemory in the constructor is what keeps the pages out; there is nothing
+    // further to say here, and nothing to drop back.
+    void advise_random(bool drop) {
+        GGML_UNUSED(drop);
+    }
+
+    // PrefetchVirtualMemory takes the whole set of ranges in one call, which is exactly the
+    // batching this wants: the reads are issued together instead of one fault at a time.
+    void prefetch_rows(const void * base, size_t stride, size_t row_size,
+                       const int32_t * rows, size_t n_rows) const {
+#if _WIN32_WINNT >= 0x602
+        BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
+        HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");
+
+        pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");
+        if (!pPrefetchVirtualMemory) {
+            return;
+        }
+
+        const size_t base_off = (const char *) base - (const char *) addr;
+
+        std::vector<WIN32_MEMORY_RANGE_ENTRY> entries;
+        for (const auto & [off, len] : llama_mmap_row_pages(
+                    base_off, stride, row_size, size, rows, n_rows, llama_mmap_page_size())) {
+            WIN32_MEMORY_RANGE_ENTRY e;
+            e.VirtualAddress = (char *) addr + off;
+            e.NumberOfBytes  = (SIZE_T) len;
+            entries.push_back(e);
+        }
+
+        if (!entries.empty()) {
+            // unchecked for the same reason as the POSIX branch: it is only a hint
+            pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0);
+        }
+#else
+        GGML_UNUSED(base);
+        GGML_UNUSED(stride);
+        GGML_UNUSED(row_size);
+        GGML_UNUSED(rows);
+        GGML_UNUSED(n_rows);
+#endif
+    }
+
     ~impl() {
         if (hMapping) {
             if (addr) {
@@ -611,10 +794,38 @@ struct llama_mmap::impl {
 
         throw std::runtime_error("mmap not supported");
     }
+
+    void advise_random(bool drop) {
+        GGML_UNUSED(drop);
+
+        throw std::runtime_error("mmap not supported");
+    }
+
+    void prefetch_rows(const void * base, size_t stride, size_t row_size,
+                       const int32_t * rows, size_t n_rows) const {
+        GGML_UNUSED(base);
+        GGML_UNUSED(stride);
+        GGML_UNUSED(row_size);
+        GGML_UNUSED(rows);
+        GGML_UNUSED(n_rows);
+
+        throw std::runtime_error("mmap not supported");
+    }
 #endif
 
+    bool contains(const void * ptr, size_t len) const {
+        const char * p = (const char *) ptr;
+        const char * b = (const char *) addr;
+
+        return p >= b && len <= size && (size_t) (p - b) <= size - len;
+    }
+
     void * addr;
     size_t size;
+
+    // the fd is kept only to re-advise the file; the mapping owns no reference to it
+    int  fd_advise = -1;
+    bool random    = false;
 };
 
 llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique<impl>(file, prefetch, numa)) {}
@@ -625,6 +836,25 @@ void * llama_mmap::addr() const { return pimpl->addr; }
 
 void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); }
 
+void llama_mmap::advise_random(bool drop) {
+    pimpl->advise_random(drop);
+    // set here rather than per platform: the flag means "the caller opted this mapping in", and
+    // the batched prefetch is worth doing even where the advice itself is a no-op
+    pimpl->random = true;
+}
+
+bool llama_mmap::is_random() const { return pimpl->random; }
+
+bool llama_mmap::contains(const void * ptr, size_t len) const { return pimpl->contains(ptr, len); }
+
+void llama_mmap::prefetch_rows(const void * base, size_t stride, size_t row_size,
+                               const int32_t * rows, size_t n_rows) const {
+    if (!pimpl->random) {
+        return;
+    }
+    pimpl->prefetch_rows(base, stride, row_size, rows, n_rows);
+}
+
 #if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)
 const bool llama_mmap::SUPPORTED  = true;
 #else

src/llama-mmap.h

@@ -50,6 +50,22 @@ struct llama_mmap {
 
     void unmap_fragment(size_t first, size_t last);
 
+    // opt-in, see llama_mmap_random_mode(). marks the whole mapping as randomly accessed, which
+    // is only correct once loading is done: until then the loader streams the file sequentially.
+    void advise_random(bool drop);
+
+    // true once advise_random() has been applied. everything that keys off "this mapping is read
+    // randomly" tests this, so nothing changes for a mapping the user did not opt in for.
+    bool is_random() const;
+
+    // true if [ptr, ptr + len) lies inside this mapping
+    bool contains(const void * ptr, size_t len) const;
+
+    // ask the kernel to start reading the given rows. issued as one batch so the faults overlap
+    // instead of serializing; a no-op unless the mapping is marked random.
+    void prefetch_rows(const void * base, size_t stride, size_t row_size,
+                       const int32_t * rows, size_t n_rows) const;
+
     static const bool SUPPORTED;
 
 private:
@@ -57,6 +73,20 @@ private:
     std::unique_ptr<impl> pimpl;
 };
 
+// how the model file mappings should be advised, from the LLAMA_MMAP_RANDOM environment variable.
+// off unless the user asks: the random hints cost a large cold-prefill slowdown on models whose
+// host-resident tensors are read sequentially, so this cannot be a default.
+enum llama_mmap_random_mode {
+    LLAMA_MMAP_RANDOM_OFF  = 0, // upstream behaviour
+    LLAMA_MMAP_RANDOM_ON   = 1, // skip MAP_POPULATE/WILLNEED, advise random after load
+    LLAMA_MMAP_RANDOM_DROP = 2, // additionally drop what the load pulled in
+};
+
+llama_mmap_random_mode llama_mmap_random_mode_get();
+
+// whether batched readahead ahead of a sparse gather is enabled (LLAMA_MMAP_RANDOM_PREFETCH)
+bool llama_mmap_random_prefetch_enabled();
+
 struct llama_mlock {
     llama_mlock();
     ~llama_mlock();

src/llama-model.cpp

@@ -1149,6 +1149,10 @@ struct llama_model::impl {
     // model memory mapped files
     llama_mmaps mappings;
 
+    // set once loading is done, if any mapping was advised random. lets prefetch_rows() bail out
+    // without walking the mappings, which is the only cost the feature has when it is off.
+    bool mappings_random = false;
+
     // objects representing data potentially being locked in memory
     llama_mlocks mlock_bufs;
     llama_mlocks mlock_mmaps;
@@ -1812,11 +1816,44 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
         for (auto & mapping : ml.mappings) {
             pimpl->mappings.emplace_back(std::move(mapping));
         }
+
+        // only now that every tensor has been read is it safe to say the file is read randomly:
+        // the load itself is a sequential pass and wants the readahead it has been getting.
+        const llama_mmap_random_mode random_mode = llama_mmap_random_mode_get();
+        if (random_mode != LLAMA_MMAP_RANDOM_OFF) {
+            for (auto & mapping : pimpl->mappings) {
+                mapping->advise_random(random_mode == LLAMA_MMAP_RANDOM_DROP);
+            }
+            pimpl->mappings_random = !pimpl->mappings.empty();
+
+            LLAMA_LOG_INFO("%s: LLAMA_MMAP_RANDOM: advised %zu mapping(s) for random access%s\n",
+                    __func__, pimpl->mappings.size(),
+                    random_mode == LLAMA_MMAP_RANDOM_DROP ? ", dropped cached pages" : "");
+        }
     }
 
     return true;
 }
 
+void llama_model::prefetch_rows(const struct ggml_tensor * t, const int32_t * rows, size_t n_rows) const {
+    if (!pimpl->mappings_random || t == nullptr || t->data == nullptr || n_rows == 0) {
+        return;
+    }
+    if (!llama_mmap_random_prefetch_enabled()) {
+        return;
+    }
+
+    // rows are addressed off the tensor, so the whole tensor has to sit in the mapping we find
+    const size_t nbytes = ggml_nbytes(t);
+
+    for (const auto & mapping : pimpl->mappings) {
+        if (mapping->is_random() && mapping->contains(t->data, nbytes)) {
+            mapping->prefetch_rows(t->data, t->nb[1], ggml_row_size(t->type, t->ne[0]), rows, n_rows);
+            return;
+        }
+    }
+}
+
 ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {
     const buft_list_t * buft_list_layer = tn.bid == -1 ? nullptr : pimpl->dev_layer.at(tn.bid).buft_list;
     return ml.create_tensor(

src/llama-model.h

@@ -734,6 +734,14 @@ struct llama_model {
 
     const struct ggml_tensor * get_tensor(const char * name) const;
 
+    // ask the kernel to start reading the rows a gather is about to take out of a host-mapped
+    // tensor, so the faults overlap instead of serializing one NVMe latency at a time.
+    //
+    // does nothing unless the tensor lives in a mapping that was advised random, which only
+    // happens under LLAMA_MMAP_RANDOM. off, and for anything not read out of a mapping (offloaded
+    // tensors, --load-mode none, non-POSIX hosts), this is one bool test.
+    void prefetch_rows(const struct ggml_tensor * t, const int32_t * rows, size_t n_rows) const;
+
     float get_rope_freq_base (const llama_cparams & cparams, int il) const;
     float get_rope_freq_scale(const llama_cparams & cparams, int il) const;
 

src/models/qwen4exp.cpp

@@ -992,6 +992,11 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) {
         h.next_pos = pos + 1;
     }
 
+    // the table is far too big to offload, so it is gathered straight out of the mapping: one
+    // fault per row, 16 per token, no two of them on the same page. left to the get_rows those
+    // faults happen one at a time; queued here they are in flight before the graph even runs.
+    pmodel.prefetch_rows(pmodel.per_layer_tok_embd, idx.data(), idx.size());
+
     ggml_backend_tensor_set(rows, idx.data(), 0, idx.size()*ggml_element_size(rows));
 }