mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-09-20 01:31:31 +02:00
feat: add EAGLE3 speculative decoding support
EAGLE3 is an encoder-decoder based speculative decoding method: - Extracts features from target model at specific layers - Uses feature fusion layer to compress target features - Generates draft tokens with single-layer decoder - Maps draft vocabulary to target vocabulary via d2t tensor Key changes: - Add LLM_ARCH_EAGLE3 architecture - Add EAGLE3 encoder/decoder graph (src/models/eagle3.cpp) - Add feature extraction from target model layers - Add g_embeddings handling for decoder input - Add GGML_TENSOR_FLAG_SYNC for GPU synchronization - Add --eagle3 flag for speculative-simple example - Add EAGLE3 model conversion in convert_hf_to_gguf.py
This commit is contained in:
@@ -363,6 +363,13 @@ extern "C" {
|
||||
bool kv_unified; // use a unified buffer across the input sequences when computing the attention
|
||||
// try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/14363
|
||||
|
||||
// EAGLE3 extraction configuration
|
||||
// When eagle3_model is set, layer extraction is automatically enabled
|
||||
const struct llama_model * eagle3_model; // EAGLE3 model to read extract_layers configuration from
|
||||
// If non-NULL, enables automatic feature extraction
|
||||
const struct llama_model * target_model; // reference to target model
|
||||
// only used to share embedding layer with eagle3 model
|
||||
};
|
||||
|
||||
// model quantization parameters
|
||||
@@ -846,6 +853,23 @@ extern "C" {
|
||||
llama_seq_id dest_seq_id,
|
||||
llama_state_seq_flags flags);
|
||||
|
||||
//
|
||||
// EAGLE3 draft model support
|
||||
//
|
||||
|
||||
// Get pointer to target model features extracted for EAGLE3 encoder
|
||||
// Returns NULL if no features are available
|
||||
// Format: [3*n_embd, n_tokens] - use model.hparams.n_embd and batch.n_tokens for dimensions
|
||||
LLAMA_API const float * llama_get_eagle3_target_features(struct llama_context * ctx);
|
||||
|
||||
// Set g_embeddings from EAGLE3 encoder output for decoder input
|
||||
// g_embd: pointer to encoder output embeddings
|
||||
LLAMA_API void llama_set_eagle3_g_embeddings(
|
||||
struct llama_context * ctx,
|
||||
const float * g_embd,
|
||||
int32_t n_embd,
|
||||
int32_t n_tokens);
|
||||
|
||||
//
|
||||
// Decoding
|
||||
//
|
||||
|
||||
Reference in New Issue
Block a user