Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 17:24:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

revert LLAMA_MMAP_RANDOM

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-27 15:01:35 +02:00
parent 243914706f
commit 41b4b7073c
8 changed files with 2 additions and 419 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-mmap.cpp
-285
View File
@@ -5,9 +5,7 @@
#include "ggml.h"
#include <cstring>
#include <cstdlib>
#include <climits>
#include <vector>
#include <stdexcept>
#include <cerrno>
#include <algorithm>
@@ -440,78 +438,6 @@ void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }
// llama_mmap
llama_mmap_random_mode llama_mmap_random_mode_get() {
// read once: this is consulted per mapping and per gather
static const llama_mmap_random_mode mode = []() {
const char * env = getenv("LLAMA_MMAP_RANDOM");
if (env == nullptr || strcmp(env, "0") == 0 || env[0] == '\0') {
return LLAMA_MMAP_RANDOM_OFF;
}
if (strcmp(env, "drop") == 0) {
return LLAMA_MMAP_RANDOM_DROP;
}
return LLAMA_MMAP_RANDOM_ON;
}();
return mode;
}
bool llama_mmap_random_prefetch_enabled() {
return llama_mmap_random_mode_get() != LLAMA_MMAP_RANDOM_OFF;
}
static size_t llama_mmap_page_size() {
#if defined(_WIN32)
SYSTEM_INFO si;
GetSystemInfo(&si);
return (size_t) si.dwPageSize;
#elif defined(_SC_PAGESIZE)
return (size_t) sysconf(_SC_PAGESIZE);
#else
return 4096;
#endif
}
// the distinct pages the given rows fall on, as offsets into the mapping, merged into runs
// rows are smaller than a page and repeat, so this turns one hint per row into one hint per page
static std::vector<std::pair<size_t, size_t>> llama_mmap_row_pages(
size_t base_off, size_t stride, size_t row_size, size_t map_size,
const int32_t * rows, size_t n_rows, size_t page_size) {
std::vector<size_t> pages;
pages.reserve(n_rows);
for (size_t i = 0; i < n_rows; ++i) {
if (rows[i] < 0) {
continue;
}
const size_t first = base_off + (size_t) rows[i] * stride;
const size_t last = first + row_size;
// a corrupt or unexpected index must not turn into a hint outside the mapping
if (row_size == 0 || last > map_size || first < base_off) {
continue;
}
for (size_t p = first / page_size; p <= (last - 1) / page_size; ++p) {
pages.push_back(p);
}
}
std::sort(pages.begin(), pages.end());
pages.erase(std::unique(pages.begin(), pages.end()), pages.end());
std::vector<std::pair<size_t, size_t>> ranges;
for (size_t i = 0; i < pages.size(); ) {
size_t j = i + 1;
while (j < pages.size() && pages[j] == pages[j - 1] + 1) {
++j;
}
const size_t off = pages[i] * page_size;
ranges.emplace_back(off, std::min((pages[j - 1] - pages[i] + 1) * page_size, map_size - off));
i = j;
}
return ranges;
}
struct llama_mmap::impl {
#ifdef _POSIX_MAPPED_FILES
std::vector<std::pair<size_t, size_t>> mapped_fragments;
@@ -519,7 +445,6 @@ struct llama_mmap::impl {
impl(struct llama_file * file, size_t prefetch, bool numa) {
size = file->size();
int fd = file->file_id();
fd_advise = fd;
int flags = MAP_SHARED;
if (numa) { prefetch = 0; }
#ifdef __linux__
@@ -550,82 +475,6 @@ struct llama_mmap::impl {
mapped_fragments.emplace_back(0, file->size());
}
// load streams the file once, so it asks for POSIX_FADV_SEQUENTIAL
// sparse gathers want the opposite: readahead buys nothing and costs page cache
// so flip the advice after load, over this range only
void advise_random_range(size_t offset, size_t len, bool drop) {
if (offset >= size || len == 0) {
return;
}
len = std::min(len, size - offset);
// madvise needs an aligned start and rounds the length up, so round both out
// this can take in one page of the neighbour on each side
const size_t page = llama_mmap_page_size();
const size_t first = offset & ~(page - 1);
const size_t last = std::min(size, (offset + len + page - 1) & ~(page - 1));
#if defined(__linux__)
if (drop) {
// on a shared file map this only tears down our page tables
if (madvise((char *) addr + first, last - first, MADV_DONTNEED)) {
LLAMA_LOG_WARN("warning: madvise(.., MADV_DONTNEED) failed: %s\n", strerror(errno));
}
// this frees the page cache; it spares partial pages, so a neighbour on the end page keeps its own
if (fd_advise >= 0 && posix_fadvise(fd_advise, (off_t) offset, (off_t) len, POSIX_FADV_DONTNEED)) {
LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_DONTNEED) failed: %s\n", strerror(errno));
}
}
#else
GGML_UNUSED(drop);
#endif
// no POSIX_FADV_RANDOM here: it marks the whole file, and its FMODE_RANDOM only affects read(), not faults
if (posix_madvise((char *) addr + first, last - first, POSIX_MADV_RANDOM)) {
LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n", strerror(errno));
}
}
void prefetch_except(const std::vector<std::pair<size_t, size_t>> & skip) {
const size_t page = llama_mmap_page_size();
size_t pos = 0;
for (const auto & [off, len] : skip) {
const size_t first = off & ~(page - 1);
if (first > pos) {
prefetch_range(pos, first - pos);
}
pos = std::max(pos, std::min(size, (off + len + page - 1) & ~(page - 1)));
}
if (pos < size) {
prefetch_range(pos, size - pos);
}
}
void prefetch_range(size_t offset, size_t len) const {
if (posix_madvise((char *) addr + offset, len, POSIX_MADV_WILLNEED)) {
LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n", strerror(errno));
}
}
void prefetch_rows(const void * base, size_t stride, size_t row_size,
const int32_t * rows, size_t n_rows) const {
#if defined(_POSIX_MAPPED_FILES)
const size_t base_off = (const char *) base - (const char *) addr;
for (const auto & [off, len] : llama_mmap_row_pages(
base_off, stride, row_size, size, rows, n_rows, llama_mmap_page_size())) {
// unchecked on purpose: thousands of hints per batch, and a failed one only costs the fault it would have saved
posix_madvise((char *) addr + off, len, POSIX_MADV_WILLNEED);
}
#else
GGML_UNUSED(base);
GGML_UNUSED(stride);
GGML_UNUSED(row_size);
GGML_UNUSED(rows);
GGML_UNUSED(n_rows);
#endif
}
static void align_range(size_t * first, size_t * last, size_t page_size) {
size_t offset_in_page = *first & (page_size - 1);
size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;
@@ -733,90 +582,6 @@ struct llama_mmap::impl {
GGML_UNUSED(last);
}
// Windows has no "read this range randomly" hint; not pulling the range in is what keeps the pages out
void advise_random_range(size_t offset, size_t len, bool drop) {
GGML_UNUSED(offset);
GGML_UNUSED(len);
GGML_UNUSED(drop);
}
void prefetch_except(const std::vector<std::pair<size_t, size_t>> & skip) {
#if _WIN32_WINNT >= 0x602
BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");
pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");
if (!pPrefetchVirtualMemory) {
return;
}
const size_t page = llama_mmap_page_size();
std::vector<WIN32_MEMORY_RANGE_ENTRY> entries;
size_t pos = 0;
for (const auto & [off, len] : skip) {
const size_t first = off & ~(page - 1);
if (first > pos) {
WIN32_MEMORY_RANGE_ENTRY e;
e.VirtualAddress = (char *) addr + pos;
e.NumberOfBytes = (SIZE_T) (first - pos);
entries.push_back(e);
}
pos = std::max(pos, std::min(size, (off + len + page - 1) & ~(page - 1)));
}
if (pos < size) {
WIN32_MEMORY_RANGE_ENTRY e;
e.VirtualAddress = (char *) addr + pos;
e.NumberOfBytes = (SIZE_T) (size - pos);
entries.push_back(e);
}
if (!entries.empty() && !pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0)) {
LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",
llama_format_win_err(GetLastError()).c_str());
}
#else
GGML_UNUSED(skip);
LLAMA_LOG_DEBUG("skipping PrefetchVirtualMemory because _WIN32_WINNT < 0x602\n");
#endif
}
// PrefetchVirtualMemory takes every range in one call, so the reads go out together instead of one fault at a time
void prefetch_rows(const void * base, size_t stride, size_t row_size,
const int32_t * rows, size_t n_rows) const {
#if _WIN32_WINNT >= 0x602
BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");
pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");
if (!pPrefetchVirtualMemory) {
return;
}
const size_t base_off = (const char *) base - (const char *) addr;
std::vector<WIN32_MEMORY_RANGE_ENTRY> entries;
for (const auto & [off, len] : llama_mmap_row_pages(
base_off, stride, row_size, size, rows, n_rows, llama_mmap_page_size())) {
WIN32_MEMORY_RANGE_ENTRY e;
e.VirtualAddress = (char *) addr + off;
e.NumberOfBytes = (SIZE_T) len;
entries.push_back(e);
}
if (!entries.empty()) {
// unchecked for the same reason as the POSIX branch: it is only a hint
pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0);
}
#else
GGML_UNUSED(base);
GGML_UNUSED(stride);
GGML_UNUSED(row_size);
GGML_UNUSED(rows);
GGML_UNUSED(n_rows);
#endif
}
~impl() {
if (hMapping) {
if (addr) {
@@ -846,45 +611,10 @@ struct llama_mmap::impl {
throw std::runtime_error("mmap not supported");
}
void advise_random_range(size_t offset, size_t len, bool drop) {
GGML_UNUSED(offset);
GGML_UNUSED(len);
GGML_UNUSED(drop);
throw std::runtime_error("mmap not supported");
}
void prefetch_except(const std::vector<std::pair<size_t, size_t>> & skip) {
GGML_UNUSED(skip);
throw std::runtime_error("mmap not supported");
}
void prefetch_rows(const void * base, size_t stride, size_t row_size,
const int32_t * rows, size_t n_rows) const {
GGML_UNUSED(base);
GGML_UNUSED(stride);
GGML_UNUSED(row_size);
GGML_UNUSED(rows);
GGML_UNUSED(n_rows);
throw std::runtime_error("mmap not supported");
}
#endif
bool contains(const void * ptr, size_t len) const {
const char * p = (const char *) ptr;
const char * b = (const char *) addr;
return p >= b && len <= size && (size_t) (p - b) <= size - len;
}
void * addr;
size_t size;
// the fd is kept only to re-advise the file; the mapping owns no reference to it
int fd_advise = -1;
};
llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique<impl>(file, prefetch, numa)) {}
@@ -895,21 +625,6 @@ void * llama_mmap::addr() const { return pimpl->addr; }
void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); }
void llama_mmap::advise_random_range(size_t offset, size_t len, bool drop) {
pimpl->advise_random_range(offset, len, drop);
}
void llama_mmap::prefetch_except(const std::vector<std::pair<size_t, size_t>> & skip) {
pimpl->prefetch_except(skip);
}
bool llama_mmap::contains(const void * ptr, size_t len) const { return pimpl->contains(ptr, len); }
void llama_mmap::prefetch_rows(const void * base, size_t stride, size_t row_size,
const int32_t * rows, size_t n_rows) const {
pimpl->prefetch_rows(base, stride, row_size, rows, n_rows);
}
#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)
const bool llama_mmap::SUPPORTED = true;
#else
src/llama-mmap.h
-31
View File
@@ -3,7 +3,6 @@
#include <cstdint>
#include <memory>
#include <vector>
#include <utility>
#include <cstdio>
struct llama_file;
@@ -51,22 +50,6 @@ struct llama_mmap {
void unmap_fragment(size_t first, size_t last);
// opt-in, see llama_mmap_random_mode(). marks one byte range as randomly accessed
// only correct after load, since the loader streams the file sequentially
// offsets are into the file, which is also the mapping offset; the range is rounded out to whole pages
void advise_random_range(size_t offset, size_t len, bool drop);
// eager pull-in for everything outside the given ranges, in place of the constructor's whole-file one
// used when part of the file must not be read ahead. ranges must be sorted
void prefetch_except(const std::vector<std::pair<size_t, size_t>> & skip);
// true if [ptr, ptr + len) lies inside this mapping
bool contains(const void * ptr, size_t len) const;
// ask the kernel to start reading the given rows, as one batch so the faults overlap
void prefetch_rows(const void * base, size_t stride, size_t row_size,
const int32_t * rows, size_t n_rows) const;
static const bool SUPPORTED;
private:
@@ -74,20 +57,6 @@ private:
std::unique_ptr<impl> pimpl;
};
// how the model file mappings should be advised, from the LLAMA_MMAP_RANDOM environment variable.
// off unless the user asks: the random hints cost a lot of cold-prefill time on tensors that are read sequentially
enum llama_mmap_random_mode {
LLAMA_MMAP_RANDOM_OFF = 0, // upstream behaviour
LLAMA_MMAP_RANDOM_ON = 1, // advise the gather tables random after load, do not pull them in
LLAMA_MMAP_RANDOM_DROP = 2, // additionally drop what the load pulled in
};
llama_mmap_random_mode llama_mmap_random_mode_get();
// batched readahead for a sparse gather, not separately switchable
// MADV_RANDOM kills the kernel's readahead, so without this the gather faults once per row and runs 2.6x slower
bool llama_mmap_random_prefetch_enabled();
struct llama_mlock {
llama_mlock();
~llama_mlock();
src/llama-model-loader.cpp
+2 -12
View File
@@ -1354,8 +1354,7 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps
if (use_mmap) {
mappings.reserve(files.size());
mmaps_used.reserve(files.size());
for (size_t i = 0; i < files.size(); ++i) {
const auto & file = files[i];
for (const auto & file : files) {
bool is_numa = false;
auto * dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
@@ -1367,16 +1366,7 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps
}
}
const auto no_prefetch = mmap_no_prefetch.find((uint16_t) i);
// the eager pull-in would read a whole gather table to fill pages the gathers barely touch
// skip it for this file and ask for the rest, so streamed tensors keep their readahead
const bool split_prefetch = prefetch && !is_numa && no_prefetch != mmap_no_prefetch.end();
std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch && !split_prefetch ? -1 : 0, is_numa);
if (split_prefetch) {
mapping->prefetch_except(no_prefetch->second);
}
std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa);
mmaps_used.emplace_back(mapping->size(), 0);
if (mlock_mmaps) {
std::unique_ptr<llama_mlock> mlock_mmap(new llama_mlock());
src/llama-model-loader.h
-4
View File
@@ -88,10 +88,6 @@ struct llama_model_loader {
llama_mmaps mappings;
// byte ranges, per source file, that init_mappings() must not pull in eagerly
// these are gather tables the model reads a few percent of. set under LLAMA_MMAP_RANDOM only, sorted by offset
std::map<uint16_t, std::vector<std::pair<size_t, size_t>>> mmap_no_prefetch;
std::map<std::string, llama_tensor_weight, weight_name_comparer> weights_map;
std::unordered_map<std::string, llama_model_kv_override> kv_overrides;
const llama_model_tensor_buft_override * tensor_buft_overrides;
src/llama-model.cpp
-62
View File
@@ -1148,17 +1148,6 @@ struct llama_model::impl {
// model memory mapped files
llama_mmaps mappings;
// gather tables that really came out of a mapping, resolved from gather_tables() during load
// empty unless the user opted in, which is all the feature costs when off
struct gather_range {
const ggml_tensor * tensor;
uint16_t idx; // source file, and so the mapping
size_t offs; // byte offset into that file
size_t len;
};
std::vector<gather_range> gather_ranges;
// objects representing data potentially being locked in memory
llama_mlocks mlock_bufs;
llama_mlocks mlock_mmaps;
@@ -1689,21 +1678,6 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
}
// kept local until the mappings exist: gather_ranges must only hold ranges checked against a live mapping
std::vector<impl::gather_range> nominated;
if (llama_mmap_random_mode_get() != LLAMA_MMAP_RANDOM_OFF) {
for (const ggml_tensor * t : gather_tables()) {
const auto * w = t ? ml.get_weight(ggml_get_name(t)) : nullptr;
if (w) {
nominated.push_back({ t, w->idx, w->offs, ggml_nbytes(w->tensor) });
ml.mmap_no_prefetch[w->idx].emplace_back(w->offs, ggml_nbytes(w->tensor));
}
}
for (auto & [_, ranges] : ml.mmap_no_prefetch) {
std::sort(ranges.begin(), ranges.end());
}
}
ml.init_mappings(true, use_mlock ? &pimpl->mlock_mmaps : nullptr);
pimpl->mappings.reserve(ml.mappings.size());
@@ -1837,47 +1811,11 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
for (auto & mapping : ml.mappings) {
pimpl->mappings.emplace_back(std::move(mapping));
}
// only safe once every tensor is read: the load is a sequential pass and wants its readahead
const llama_mmap_random_mode random_mode = llama_mmap_random_mode_get();
// a named tensor not served from its mapping was offloaded or copied, so nothing gathers from the file
// drop it rather than advise it
for (const auto & r : nominated) {
if (r.idx < pimpl->mappings.size() && pimpl->mappings[r.idx]->contains(r.tensor->data, r.len)) {
pimpl->gather_ranges.push_back(r);
}
}
for (const auto & r : pimpl->gather_ranges) {
pimpl->mappings[r.idx]->advise_random_range(r.offs, r.len, random_mode == LLAMA_MMAP_RANDOM_DROP);
LLAMA_LOG_INFO("%s: LLAMA_MMAP_RANDOM: %s advised for random access, %.2f MiB%s\n",
__func__, ggml_get_name(r.tensor), r.len / 1024.0 / 1024.0,
random_mode == LLAMA_MMAP_RANDOM_DROP ? ", dropped cached pages" : "");
}
}
return true;
}
void llama_model::prefetch_rows(const struct ggml_tensor * t, const int32_t * rows, size_t n_rows) const {
if (pimpl->gather_ranges.empty() || t == nullptr || t->data == nullptr || n_rows == 0) {
return;
}
if (!llama_mmap_random_prefetch_enabled()) {
return;
}
// keyed off the tensor, not the mapping: the readahead must land where the advice did
for (const auto & r : pimpl->gather_ranges) {
if (r.tensor == t) {
pimpl->mappings[r.idx]->prefetch_rows(t->data, t->nb[1], ggml_row_size(t->type, t->ne[0]), rows, n_rows);
return;
}
}
}
ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {
const buft_list_t * buft_list_layer = tn.bid == -1 ? nullptr : pimpl->dev_layer.at(tn.bid).buft_list;
return ml.create_tensor(
src/llama-model.h
-13
View File
@@ -734,19 +734,6 @@ struct llama_model {
const struct ggml_tensor * get_tensor(const char * name) const;
// ask the kernel to start reading the rows a gather is about to take from a host-mapped tensor
// the faults then overlap instead of paying one NVMe latency at a time
//
// does nothing unless gather_tables() named the tensor and it really is read from a mapping
// for anything else this is one empty-vector test
void prefetch_rows(const struct ggml_tensor * t, const int32_t * rows, size_t n_rows) const;
// tensors that stay host-resident and are read by sparse row gathers, not streamed once
// under LLAMA_MMAP_RANDOM these get the random advice and the batched readahead of prefetch_rows()
//
// named by the model, not guessed from size: a big tensor read in full, like token_embd on CPU, still wants readahead
virtual std::vector<const struct ggml_tensor *> gather_tables() const { return {}; }
float get_rope_freq_base (const llama_cparams & cparams, int il) const;
float get_rope_freq_scale(const llama_cparams & cparams, int il) const;
src/models/models.h
-8
View File
@@ -2281,14 +2281,6 @@ struct llama_model_qwen4exp : public llama_model_base {
void load_arch_hparams(llama_model_loader & ml) override;
void load_arch_tensors(llama_model_loader & ml) override;
// the PLE n-gram table is far too big to offload and is read by 16 tiny gathers per token
std::vector<const struct ggml_tensor *> gather_tables() const override {
if (per_layer_tok_embd == nullptr) {
return {};
}
return { per_layer_tok_embd };
}
struct graph : public llm_build_delta_net_base {
graph(const llama_model & model, const llm_graph_params & params);
private:
src/models/qwen4exp.cpp
-4
View File
@@ -974,10 +974,6 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) {
}
}
// the table is too big to offload, so it is gathered from the mapping: 16 faults per token, no two on the same page
// get_rows would take them one at a time; queued here they are in flight before the graph runs
pmodel.prefetch_rows(pmodel.per_layer_tok_embd, idx.data(), idx.size());
ggml_backend_tensor_set(rows, idx.data(), 0, idx.size()*ggml_element_size(rows));
}