always prefilter to 5k logits

2026-09-18 16:55:14 +02:00 · 2024-08-12 22:27:06 +08:00
parent 6e0ebc90ad
commit 689a17d756
1 changed files with 3 additions and 0 deletions
@@ -839,6 +839,9 @@ int mirostat, float mirostat_tau, float mirostat_eta, float dry_multiplier, floa
        sample_grammar(file_format, n_vocab, &candidates_p, grammar);
    }

+    //prefilter to top 5k tokens for improved speed
+    llama_sample_top_k(nullptr, &candidates_p, 5000, 1);
+
    if (mirostat == 1 || mirostat == 2)
    {
        static float mirostat_mu = 2.0f * mirostat_tau;