mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-20 01:31:42 +02:00
move kcpp params out
This commit is contained in:
Concedo
@@ -1,176 +0,0 @@
|
||||
int mainfn() {
|
||||
|
||||
kcpp_params = new gpt_params();
|
||||
int argc = 11;
|
||||
char* argv[11] = {
|
||||
"E:\\LLaMA\\llamacpp\\main.exe",
|
||||
"-ngl",
|
||||
"99",
|
||||
"-n",
|
||||
"32",
|
||||
"-m",
|
||||
"E:\\LLaMA\\models\\airoboros-mistral2.2-7b.Q4_K_S.gguf",
|
||||
"-c",
|
||||
"2128",
|
||||
"-p",
|
||||
"Niko the kobold stalked carefully down the alley,"
|
||||
};
|
||||
|
||||
if (!gpt_params_parse(argc, argv, *kcpp_params)) {
|
||||
return 1;
|
||||
}
|
||||
llama_sampling_params & sparams = kcpp_params->sparams;
|
||||
|
||||
|
||||
if (kcpp_params->seed == LLAMA_DEFAULT_SEED) {
|
||||
kcpp_params->seed = time(NULL);
|
||||
}
|
||||
|
||||
LOG_TEE("%s: seed = %u\n", __func__, kcpp_params->seed);
|
||||
|
||||
std::mt19937 rng(kcpp_params->seed);
|
||||
|
||||
LOG("%s: llama backend init\n", __func__);
|
||||
llama_backend_init(kcpp_params->numa);
|
||||
|
||||
llama_model * model;
|
||||
|
||||
// load the model and apply lora adapter, if any
|
||||
LOG("%s: load the model and apply lora adapter, if any\n", __func__);
|
||||
std::tie(model, llama_ctx_v4) = llama_init_from_gpt_params(*kcpp_params);
|
||||
llama_reset_timings(llama_ctx_v4);
|
||||
|
||||
if (model == NULL) {
|
||||
LOG_TEE("%s: error: unable to load model\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const int n_ctx = llama_n_ctx(llama_ctx_v4);
|
||||
const bool add_bos = true;
|
||||
std::vector<llama_token> embd_inp;
|
||||
|
||||
embd_inp = ::llama_tokenize(llama_ctx_v4, kcpp_params->prompt, add_bos, true);
|
||||
|
||||
// Should not run without any tokens
|
||||
if (embd_inp.empty()) {
|
||||
embd_inp.push_back(llama_token_bos(model));
|
||||
}
|
||||
|
||||
// number of tokens to keep when resetting context
|
||||
if (kcpp_params->n_keep < 0 || kcpp_params->n_keep > (int) embd_inp.size() || kcpp_params->instruct || kcpp_params->chatml) {
|
||||
kcpp_params->n_keep = (int)embd_inp.size();
|
||||
}
|
||||
|
||||
int n_past = 0;
|
||||
int n_remain = kcpp_params->n_predict;
|
||||
bool startedpred = false;
|
||||
int predamt = 0;
|
||||
int n_consumed = 0;
|
||||
|
||||
std::vector<int> input_tokens;
|
||||
std::vector<int> output_tokens;
|
||||
std::ostringstream output_ss;
|
||||
|
||||
|
||||
std::vector<llama_token> embd;
|
||||
struct llama_sampling_context * ctx_sampling = llama_sampling_init(sparams);
|
||||
|
||||
while (n_remain != 0) {
|
||||
// predict
|
||||
if (!embd.empty()) {
|
||||
int max_embd_size = n_ctx - 4;
|
||||
|
||||
// Ensure the input doesn't exceed the context size by truncating embd if necessary.
|
||||
if ((int) embd.size() > max_embd_size) {
|
||||
const int skipped_tokens = (int) embd.size() - max_embd_size;
|
||||
embd.resize(max_embd_size);
|
||||
}
|
||||
|
||||
{
|
||||
if (n_past + (int) embd.size() > n_ctx) {
|
||||
if (kcpp_params->n_predict == -2) {
|
||||
break;
|
||||
}
|
||||
const int n_left = n_past - kcpp_params->n_keep - 1;
|
||||
const int n_discard = n_left/2;
|
||||
llama_kv_cache_seq_rm (llama_ctx_v4, 0, kcpp_params->n_keep + 1 , kcpp_params->n_keep + n_discard + 1);
|
||||
llama_kv_cache_seq_add(llama_ctx_v4, 0, kcpp_params->n_keep + 1 + n_discard, n_past, -n_discard);
|
||||
|
||||
n_past -= n_discard;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
for (int i = 0; i < (int) embd.size(); i += kcpp_params->n_batch) {
|
||||
int n_eval = (int) embd.size() - i;
|
||||
if (n_eval > kcpp_params->n_batch) {
|
||||
n_eval = kcpp_params->n_batch;
|
||||
}
|
||||
|
||||
if (llama_decode(llama_ctx_v4, llama_batch_get_one(&embd[i], n_eval, n_past, 0))) {
|
||||
LOG_TEE("%s : failed to eval\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
n_past += n_eval;
|
||||
}
|
||||
}
|
||||
|
||||
embd.clear();
|
||||
|
||||
if ((int) embd_inp.size() <= n_consumed) {
|
||||
const llama_token id = llama_sampling_sample(ctx_sampling, llama_ctx_v4, nullptr);
|
||||
llama_sampling_accept(ctx_sampling, llama_ctx_v4, id, true);
|
||||
embd.push_back(id);
|
||||
// decrement remaining sampling budget
|
||||
--n_remain;
|
||||
if(!startedpred)
|
||||
{
|
||||
startedpred = true;
|
||||
timer_start();
|
||||
predamt += 1;
|
||||
}else
|
||||
{
|
||||
predamt += 1;
|
||||
}
|
||||
} else {
|
||||
while ((int) embd_inp.size() > n_consumed) {
|
||||
embd.push_back(embd_inp[n_consumed]);
|
||||
llama_sampling_accept(ctx_sampling, llama_ctx_v4, embd_inp[n_consumed], false);
|
||||
++n_consumed;
|
||||
|
||||
if ((int) embd.size() >= kcpp_params->n_batch) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// display text
|
||||
{
|
||||
for (auto id : embd) {
|
||||
const std::string token_str = llama_token_to_piece(llama_ctx_v4, id);
|
||||
printf("%s", token_str.c_str());
|
||||
if (embd.size() > 1) {
|
||||
input_tokens.push_back(id);
|
||||
} else {
|
||||
output_tokens.push_back(id);
|
||||
output_ss << token_str;
|
||||
}
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
}
|
||||
auto tt = timer_check();
|
||||
float pt1 = (tt*1000.0/(predamt));
|
||||
float ts1 = (1000.0/pt1);
|
||||
printf("\n\n Time:%.2fs (%.1fms/T = %.2fT/s) tokens: %d",tt,pt1,ts1,predamt);
|
||||
|
||||
llama_print_timings(llama_ctx_v4);
|
||||
|
||||
llama_free(llama_ctx_v4);
|
||||
llama_free_model(model);
|
||||
|
||||
llama_sampling_free(ctx_sampling);
|
||||
llama_backend_free();
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user