mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-08-27 07:31:24 +02:00
Xuan-Son Nguyen
2fb989b9e7
* fit: also take into account n_streams * server: make the draft context follow the target context With a non-unified KV cache the target context now holds n_ctx_train tokens per sequence, while the draft context was still created with n_ctx = 0 and fell back to n_ctx_train / n_streams per sequence. A slot filled beyond that point makes the draft batch fail to decode, and the server answers 500 on the request. The draft context now takes its size from the target context, so both hold the same number of tokens per sequence. Contexts that share their cells with the target no longer need the kv_size override. The memory reserved for the draft model before fitting is measured at the largest context the target can take, since the draft context grows with the target and a fixed byte margin cannot express that. * fit: take an optional second model into account Illustrates the alternative discussed on the draft context fix. The memory of a draft or MTP context is currently handed to the fit as a fixed byte margin, which cannot express a memory that grows with the context the fit is still deciding on. common_fit_params now takes an optional second model that shares the devices of the main one. Its context follows the main context and its memory is measured again whenever that context changes, so the reduce path stays exact instead of conservative. A model that cannot be measured on its own, such as a shared cell MTP context, is skipped with a warning and the main model is fitted alone. This drops the reservation block in the server, which no longer has to probe the trained context size of the target to guess an upper bound. --------- Co-authored-by: Pascal <admin@serveurperso.com>
68 lines
3.3 KiB
C++
68 lines
3.3 KiB
C++
#pragma once
|
|
|
|
#include "ggml.h"
|
|
#include "llama.h"
|
|
|
|
#include <vector>
|
|
|
|
enum common_params_fit_status {
|
|
COMMON_PARAMS_FIT_STATUS_SUCCESS = 0, // found allocations that are projected to fit
|
|
COMMON_PARAMS_FIT_STATUS_FAILURE = 1, // could not find allocations that are projected to fit
|
|
COMMON_PARAMS_FIT_STATUS_ERROR = 2, // a hard error occurred, e.g. because no model could be found at the specified path
|
|
};
|
|
|
|
// a second model that shares the devices of the main model, e.g. a draft model
|
|
// - its context follows the context of the main model, so its memory is measured again whenever that context changes
|
|
// - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context
|
|
struct common_fit_extra_model {
|
|
const char * path_model;
|
|
llama_model_params * mparams;
|
|
llama_context_params * cparams;
|
|
bool shares_model;
|
|
};
|
|
|
|
// fits mparams and cparams to free device memory (assumes system memory is unlimited)
|
|
// - returns true if the parameters could be successfully modified to fit device memory
|
|
// - this function is NOT thread safe because it modifies the global llama logger state
|
|
// - only parameters that have the same value as in llama_default_model_params are modified
|
|
// with the exception of the context size which is modified if and only if equal to 0
|
|
common_params_fit_status common_fit_params(
|
|
const char * path_model,
|
|
llama_model_params * mparams,
|
|
llama_context_params * cparams,
|
|
float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
|
|
llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
|
|
size_t * margins, // margins of memory to leave per device in bytes
|
|
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
|
|
const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none
|
|
ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
|
|
|
|
// print estimated memory to stdout
|
|
void common_fit_print(
|
|
const char * path_model,
|
|
llama_model_params * mparams,
|
|
llama_context_params * cparams);
|
|
|
|
void common_memory_breakdown_print(const llama_context * ctx);
|
|
|
|
struct common_device_memory_data {
|
|
int64_t total;
|
|
int64_t free;
|
|
size_t model;
|
|
size_t context;
|
|
size_t compute;
|
|
};
|
|
|
|
using common_device_memory_data_vec = std::vector<common_device_memory_data>;
|
|
|
|
// Load a model + context with no_alloc and return the per-device memory breakdown.
|
|
common_device_memory_data_vec common_get_device_memory_data(
|
|
const char * path_model,
|
|
const llama_model_params * mparams,
|
|
const llama_context_params * cparams,
|
|
std::vector<ggml_backend_dev_t> & devs,
|
|
uint32_t & hp_ngl,
|
|
uint32_t & hp_n_ctx_train,
|
|
uint32_t & hp_n_expert,
|
|
ggml_log_level log_level);
|