llama.cpp/common/speculative.h

#pragma once

#include "llama.h"
#include "common.h"

struct common_speculative;

struct common_speculative_params {
    int n_draft = 16;  // max drafted tokens

    float p_min = 0.75f; // min probability required to accept a token in the draft
};

// comma separated list of all types
std::string common_speculative_type_name_str();

// convert string to type
enum common_speculative_type common_speculative_type_from_name(const std::string & name);

// convert type to string
std::string common_speculative_type_to_str(enum common_speculative_type type);

struct common_speculative * common_speculative_init(
        const struct common_params_speculative & params,
              struct llama_context             * ctx_tgt,
        const struct llama_context_params      & cparams_dft,
              struct llama_model               * model_dft);

void common_speculative_free(struct common_speculative * spec);

// sample up to n_draft tokens and add them to the batch using the draft model
llama_tokens common_speculative_gen_draft(
               struct common_speculative * spec,
        struct common_speculative_params   params,
                      const llama_tokens & prompt,
                             llama_token   id_last);

// informs the speculative decoder that n_accepted tokens were accepted by the target model
void common_speculative_accept(struct common_speculative * spec, uint16_t n_accepted);

// print statistics about the speculative decoding
void common_speculative_print_stats(const struct common_speculative * spec);