mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-10 23:09:24 +02:00
improved reasoning budget
This commit is contained in:
Concedo
+57
-14
@@ -4147,7 +4147,7 @@ Current version indicated by LITEVER below.
|
||||
const alltalk_voices_endpoint = "/api/voices";
|
||||
const alltalk_rvc_voices_endpoint = "/api/rvcvoices";
|
||||
|
||||
const pollinations_img_endpoint = "https://gen.pollinations.ai/image";
|
||||
const pollinations_img_endpoint = "https://image.pollinations.ai/prompt";
|
||||
const pollinations_text_endpoint = "https://text.pollinations.ai/openai";
|
||||
const dummy_api_key = "kobo";
|
||||
|
||||
@@ -4361,7 +4361,6 @@ Current version indicated by LITEVER below.
|
||||
saved_palm_jailbreak:"", //gemini system prompt
|
||||
saved_palm_jailbreak2:"", //gemini postfix
|
||||
saved_openrouter_provider:"", //for overriding provider
|
||||
saved_oai_reasoning:"", //if set, reasoning level "none,minimal,low,medium,high", else default
|
||||
saved_oai_custommodel: "", //customized oai custom model
|
||||
saved_oai_role: 3, //0=user,1=assistant,2=system, 3=auto
|
||||
saved_a1111_url: default_a1111_base,
|
||||
@@ -4373,7 +4372,7 @@ Current version indicated by LITEVER below.
|
||||
prev_custom_endpoint_type: 0, //show a reconnect box to custom endpoint if needed. 0 is horde, otherwise its dropdown value+1
|
||||
prev_custom_endpoint_model: "", //we may not be able to match, but set it if we do
|
||||
prev_custom_endpoint_ischatcmpl: true,
|
||||
generate_images_mode: (localflag?0:1), //0=off, 1=horde, 2=a1111, 3=dalle, 4=comfy
|
||||
generate_images_mode: (localflag?0:1), //0=off, 1=horde, 2=a1111, 3=dalle, 4=comfy, 5=pollinations
|
||||
|
||||
autoscroll: true, //automatically scroll to bottom on render
|
||||
printer_view: false, //automatically scroll to bottom on render
|
||||
@@ -4505,6 +4504,7 @@ Current version indicated by LITEVER below.
|
||||
handle_mismatched_think: true,
|
||||
think_injected: 0, //0=normal, 1=force
|
||||
strip_thinking_mode: 1, //0=no stripping, 1=strip except recent, 2=strip all
|
||||
reasoning_effort: "", //if set, reasoning level "none,minimal,low,medium,high", else default empty string
|
||||
websearch_enabled: false,
|
||||
websearch_multipass: false,
|
||||
websearch_retain: false,
|
||||
@@ -9303,7 +9303,7 @@ Current version indicated by LITEVER below.
|
||||
nologo: true,
|
||||
private: true,
|
||||
referrer: "koboldai",
|
||||
key: localsettings.saved_dalle_key
|
||||
// key: localsettings.saved_dalle_key
|
||||
});
|
||||
|
||||
let gen_endpoint = `${pollinations_img_endpoint}/${encodeURIComponent(prompt)}?${pollinations_params.toString()}`;
|
||||
@@ -13693,7 +13693,7 @@ Current version indicated by LITEVER below.
|
||||
document.getElementById("oaidesc").classList.add("hidden");
|
||||
document.getElementById("pollinationsdesc").classList.add("hidden");
|
||||
document.getElementById("openrouterproviderbox").classList.add("hidden");
|
||||
document.getElementById("oaireasoningeffort").value = localsettings.saved_oai_reasoning;
|
||||
document.getElementById("oaireasoningeffort").value = localsettings.reasoning_effort;
|
||||
if(epchoice==2)
|
||||
{
|
||||
document.getElementById("oaidesc").classList.remove("hidden");
|
||||
@@ -14312,7 +14312,7 @@ Current version indicated by LITEVER below.
|
||||
localsettings.saved_openrouter_key = custom_oai_key;
|
||||
localsettings.saved_openrouter_provider = document.getElementById("openrouterproviders").value;
|
||||
}
|
||||
localsettings.saved_oai_reasoning = document.getElementById("oaireasoningeffort").value;
|
||||
localsettings.reasoning_effort = document.getElementById("oaireasoningeffort").value;
|
||||
localsettings.saved_oai_jailbreak = document.getElementById("jailbreakprompttext").value;
|
||||
if(localsettings.saved_oai_jailbreak=="")
|
||||
{
|
||||
@@ -15734,10 +15734,6 @@ Current version indicated by LITEVER below.
|
||||
mainmenu_untab(true);
|
||||
document.getElementById("settingscontainer").classList.remove("hidden");
|
||||
display_settings_tab(current_settings_tab_idx);
|
||||
|
||||
if(localsettings.generate_images_mode>=5){localsettings.generate_images_mode=0;} //todo: temp hack to remove pollinations images
|
||||
if(localsettings.tts_mode>=6){localsettings.tts_mode=0;} //todo: temp hack to remove pollinations tts
|
||||
|
||||
document.getElementById("max_context_length").value = document.getElementById("max_context_length_slide").value = localsettings.max_context_length;
|
||||
document.getElementById("max_length").value = document.getElementById("max_length_slide").value = localsettings.max_length;
|
||||
document.getElementById("temperature").value = document.getElementById("temperature_slide").value = localsettings.temperature;
|
||||
@@ -16817,6 +16813,7 @@ Current version indicated by LITEVER below.
|
||||
}
|
||||
localsettings.hide_think_stream = (document.getElementById("hide_think_stream").checked?true:false);
|
||||
localsettings.handle_mismatched_think = (document.getElementById("handle_mismatched_think").checked?true:false);
|
||||
localsettings.reasoning_effort = document.getElementById("reasoningeffort").value;
|
||||
|
||||
localsettings.tts_mode = document.getElementById("ttsselect").value;
|
||||
localsettings.xtts_voice = document.getElementById("xtts_voices").value;
|
||||
@@ -17107,6 +17104,7 @@ Current version indicated by LITEVER below.
|
||||
document.getElementById("generate_images_dalle_container").classList.add("hidden");
|
||||
document.getElementById("generate_images_local_model_container").classList.add("hidden");
|
||||
document.getElementById("generate_images_comfy_container").classList.add("hidden");
|
||||
document.getElementById("generate_images_pollinations_container").classList.add("hidden");
|
||||
if(document.getElementById("generate_images_mode").value==1){
|
||||
document.getElementById("generate_images_model_container").classList.remove("hidden");
|
||||
if(!image_models_fetched)
|
||||
@@ -17125,7 +17123,10 @@ Current version indicated by LITEVER below.
|
||||
}else if(document.getElementById("generate_images_mode").value==4){
|
||||
document.getElementById("generate_images_comfy_container").classList.remove("hidden");
|
||||
connect_to_comfyui(silent);
|
||||
}
|
||||
}else if(document.getElementById("generate_images_mode").value==5)
|
||||
{
|
||||
document.getElementById("generate_images_pollinations_container").classList.remove("hidden");
|
||||
}
|
||||
}
|
||||
|
||||
function get_theme_desc(themeid)
|
||||
@@ -20741,6 +20742,7 @@ Current version indicated by LITEVER below.
|
||||
submit_payload.params.banned_tokens = get_token_bans();
|
||||
submit_payload.params.render_special = localsettings.render_special_tags;
|
||||
submit_payload.params.logprobs = localsettings.request_logprobs;
|
||||
submit_payload.params.reasoning_effort = localsettings.reasoning_effort;
|
||||
let st = get_instruct_starttag(true);
|
||||
let et = get_instruct_endtag(true);
|
||||
if(st=="{{[INPUT]}}" || et=="{{[OUTPUT]}}")
|
||||
@@ -21055,10 +21057,20 @@ Current version indicated by LITEVER below.
|
||||
|
||||
if (document.getElementById("useoaichatcompl").checked)
|
||||
{
|
||||
let cannotreason = (targetep.toLowerCase().includes("text.pollinations.ai") || targetep.toLowerCase().includes("api.nvidia.com"));
|
||||
let cannotreason = (targetep.toLowerCase().includes("text.pollinations.ai"));
|
||||
if(!cannotreason)
|
||||
{
|
||||
let reasoneffort = document.getElementById("oaireasoningeffort").value;
|
||||
let reasoneffort = localsettings.reasoning_effort;
|
||||
if (targetep.toLowerCase().includes("api.nvidia.com")) {
|
||||
if (reasoneffort=="minimal")
|
||||
{
|
||||
reasoneffort = "low";
|
||||
}
|
||||
if (reasoneffort != "low" && reasoneffort != "medium" && reasoneffort != "high") //only accepts low, medium and high
|
||||
{
|
||||
reasoneffort = "";
|
||||
}
|
||||
}
|
||||
if(reasoneffort && reasoneffort!="")
|
||||
{
|
||||
oai_payload.reasoning_effort = reasoneffort;
|
||||
@@ -21254,7 +21266,7 @@ Current version indicated by LITEVER below.
|
||||
//pollinations always uses the exact same url for text gen regardless
|
||||
if(targetep.toLowerCase().includes("text.pollinations.ai"))
|
||||
{
|
||||
targetep = apply_proxy_url(pollinations_text_endpoint,true);
|
||||
targetep = pollinations_text_endpoint;
|
||||
oai_payload.private = true;
|
||||
oai_payload.referrer = "koboldai";
|
||||
oai_payload.seed = Math.floor(Math.random() * 99999999);
|
||||
@@ -22028,6 +22040,12 @@ Current version indicated by LITEVER below.
|
||||
genimg_payload.models = [desired_model];
|
||||
imgid = generate_comfy_image(genimg_payload, autoappend);
|
||||
}
|
||||
else if(localsettings.generate_images_mode==5) //pollinations
|
||||
{
|
||||
let desired_model = document.getElementById("generate_images_pollinations_model").value;
|
||||
genimg_payload.models = [desired_model];
|
||||
imgid = generate_pollinations_image(genimg_payload, autoappend);
|
||||
}
|
||||
|
||||
return imgid;
|
||||
}
|
||||
@@ -26774,6 +26792,7 @@ Current version indicated by LITEVER below.
|
||||
document.getElementById("stop_thinking_tag").value = localsettings.stop_thinking_tag;
|
||||
document.getElementById("hide_think_stream").checked = (localsettings.hide_think_stream?true:false);
|
||||
document.getElementById("handle_mismatched_think").checked = (localsettings.handle_mismatched_think?true:false);
|
||||
document.getElementById("reasoningeffort").value = localsettings.reasoning_effort;
|
||||
}
|
||||
|
||||
function populate_placeholder_tags()
|
||||
@@ -30092,6 +30111,7 @@ Current version indicated by LITEVER below.
|
||||
<option value="2">KCPP / Forge / A1111</option>
|
||||
<option value="3">OpenAI Compatible Img</option>
|
||||
<option value="4">ComfyUI</option>
|
||||
<option value="5">Pollinations.ai</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
@@ -30150,6 +30170,16 @@ Current version indicated by LITEVER below.
|
||||
</tr></table>
|
||||
</div>
|
||||
|
||||
<div id="generate_images_pollinations_container" class="settinglabel hidden settingsbox">
|
||||
<div class="settinglabel" style="display: flex; width: 100%">
|
||||
<div class="justifyleft">Model</div>
|
||||
<select title="Select Image Model" class="form-control push-right" id="generate_images_pollinations_model">
|
||||
<option value="turbo" selected>turbo</option>
|
||||
<option value="flux">flux</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<div class="settinglabel">
|
||||
<div class="justifyleft">Add positive prompt</div>
|
||||
@@ -30587,6 +30617,19 @@ Current version indicated by LITEVER below.
|
||||
</span> </div>
|
||||
<input title="Handle Mismatched Thinking" type="checkbox" id="handle_mismatched_think" class="push-right">
|
||||
</div>
|
||||
<div class="settinglabel">
|
||||
<div class="justifyleft">Reasoning Effort <span class="helpicon">?<span class="helptext">If set, restricts thinking length to this many tokens before forcing a reply.</span></span></div>
|
||||
<div class="push-right">
|
||||
<select title="Reasoning Effort" class="form-control" style="display:inline;" id="reasoningeffort">
|
||||
<option value="" selected>Reasoning: Default</option>
|
||||
<option value="none">Reasoning: None</option>
|
||||
<option value="minimal">Reasoning: Minimal</option>
|
||||
<option value="low">Reasoning: Low</option>
|
||||
<option value="medium">Reasoning: Medium</option>
|
||||
<option value="high">Reasoning: High</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="settinglabel">
|
||||
|
||||
@@ -142,7 +142,7 @@ struct generation_inputs
|
||||
const logit_bias * logit_biases = nullptr;
|
||||
const int banned_tokens_len = 0;
|
||||
const char ** banned_tokens = nullptr;
|
||||
const int reasoning_budget = 0;
|
||||
const int reasoning_budget = -1;
|
||||
};
|
||||
struct generation_outputs
|
||||
{
|
||||
|
||||
+75
-60
@@ -1754,9 +1754,63 @@ void sample_guidance(struct llama_context * ctx, struct llama_context * guidance
|
||||
}
|
||||
}
|
||||
|
||||
static int apply_reasoning_budget(int id, const std::vector<int> & start_think, const std::vector<int> & end_think, std::vector<int> & think_end_phrase_toks, int budget)
|
||||
{
|
||||
if(budget<0 || start_think.size()==0 || end_think.size()!=1 || think_end_phrase_toks.size()==0) //start_think can be 1-3 tokens long, end_think is always 1 token
|
||||
{
|
||||
return id;
|
||||
}
|
||||
|
||||
int end_think_index = -1;
|
||||
int start_think_index = -1;
|
||||
int ctx_size = (int)current_context_tokens.size();
|
||||
|
||||
for (int i = ctx_size - 1; i >= 0; --i) { // Search backwards for the latest end_think token
|
||||
if (end_think_index == -1 && current_context_tokens[i] == end_think[0]) {
|
||||
end_think_index = i;
|
||||
}
|
||||
if (start_think_index == -1) { // Search backwards for the latest start_think sequence
|
||||
int seq_len = (int) start_think.size();
|
||||
if (i - seq_len + 1 >= 0) {
|
||||
bool match = true;
|
||||
for (int j = 0; j < seq_len; ++j) {
|
||||
if (current_context_tokens[i - seq_len + 1 + j] != start_think[j]) {
|
||||
match = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (match) {
|
||||
start_think_index = i; // index of the last token of the start_think sequence
|
||||
}
|
||||
}
|
||||
}
|
||||
if (start_think_index != -1 && end_think_index != -1) { // Early exit once both are found
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (start_think_index == -1) { // If no start_think found, do nothing
|
||||
return id;
|
||||
}
|
||||
|
||||
if (end_think_index != -1 && end_think_index > start_think_index) { // If end_think comes after start_think, thinking is already closed
|
||||
return id;
|
||||
}
|
||||
|
||||
int tokens_since_start = ctx_size - 1 - start_think_index; // start_think is unclosed, check budget
|
||||
if (tokens_since_start >= budget) {
|
||||
int popped = think_end_phrase_toks[0]; // Force-close thinking by returning the end thinking phrase, pop front and return
|
||||
think_end_phrase_toks.erase(think_end_phrase_toks.begin()); // Elements shift left
|
||||
return popped;
|
||||
}
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
int SampleLogits(const float * logits, int n_ctx, int n_vocab, int rep_pen_range, float rep_pen, float rep_pen_slope, float presence_penalty, float top_k, float top_a, float top_p, float min_p, float typical_p, float tfs, float nsigma, float temp, std::mt19937 & rng,
|
||||
int mirostat, float mirostat_tau, float mirostat_eta, float dry_multiplier, float dry_base, int dry_allowed_length, int dry_penalty_last_n, float xtc_threshold, float xtc_probability,
|
||||
const std::vector<samplers> & sampler_order, llama_grammar * grammar, float dynatemp_range, float dynatemp_exponent, float smoothing_factor, float smoothing_curve, float adaptive_target)
|
||||
const std::vector<samplers> & sampler_order, llama_grammar * grammar, float dynatemp_range, float dynatemp_exponent, float smoothing_factor, float smoothing_curve, float adaptive_target,
|
||||
const std::vector<int> & think_start_seq, const std::vector<int> & think_end_seq, std::vector<int> & think_end_phrase_toks, int reasoning_budget)
|
||||
{
|
||||
// printf("SampleLogits called with: n_ctx=%d, n_vocab=%d, rep_pen_range=%d, rep_pen=%f, rep_pen_slope=%f, presence_penalty=%f, top_k=%f, top_a=%f, top_p=%f, min_p=%f, typical_p=%f, tfs=%f, nsigma=%f, temp=%f, mirostat=%d, mirostat_tau=%f, mirostat_eta=%f, dry_multiplier=%f, dry_base=%f, dry_allowed_length=%d, dry_penalty_last_n=%d, xtc_threshold=%f, xtc_probability=%f, sampler_order_size=%zu, dynatemp_range=%f, dynatemp_exponent=%f, smoothing_factor=%f\n",
|
||||
// n_ctx, n_vocab, rep_pen_range, rep_pen, rep_pen_slope, presence_penalty, top_k, top_a, top_p, min_p, typical_p, tfs, nsigma, temp, mirostat, mirostat_tau, mirostat_eta, dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, xtc_threshold, xtc_probability, sampler_order.size(), dynatemp_range, dynatemp_exponent, smoothing_factor);
|
||||
@@ -1776,6 +1830,19 @@ const std::vector<samplers> & sampler_order, llama_grammar * grammar, float dyna
|
||||
|
||||
llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
|
||||
|
||||
//apply reasoning budget
|
||||
int newid = apply_reasoning_budget(id, think_start_seq, think_end_seq, think_end_phrase_toks, kcpp_data->reasoning_budget);
|
||||
if (id != newid) {
|
||||
if(!is_quiet && debugmode!=-1)
|
||||
{
|
||||
printf("\n(Reasoning Budget of %d tokens exceeded! Finishing thinking...)\n", kcpp_data->reasoning_budget);
|
||||
}
|
||||
candidates[newid].logit += 99999;
|
||||
sample_top_k(&candidates_p, 1);
|
||||
id = sample_token(&candidates_p, rng);
|
||||
return id;
|
||||
}
|
||||
|
||||
//dry always first as logits cannot be resorted
|
||||
sample_dry(n_ctx, dry_penalty_last_n, dry_multiplier, dry_base, dry_allowed_length, dry_sequence_breakers, &candidates_p);
|
||||
|
||||
@@ -1870,57 +1937,6 @@ const std::vector<samplers> & sampler_order, llama_grammar * grammar, float dyna
|
||||
return id;
|
||||
}
|
||||
|
||||
static int apply_reasoning_budget(int id, const std::vector<int> & start_think, const std::vector<int> & end_think, int budget)
|
||||
{
|
||||
if(budget<=0 || start_think.size()==0 || end_think.size()!=1) //start_think can be 1-3 tokens long, end_think is always 1 token
|
||||
{
|
||||
return id;
|
||||
}
|
||||
|
||||
int end_think_index = -1;
|
||||
int start_think_index = -1;
|
||||
int ctx_size = (int)current_context_tokens.size();
|
||||
|
||||
for (int i = ctx_size - 1; i >= 0; --i) { // Search backwards for the latest end_think token
|
||||
if (end_think_index == -1 && current_context_tokens[i] == end_think[0]) {
|
||||
end_think_index = i;
|
||||
}
|
||||
if (start_think_index == -1) { // Search backwards for the latest start_think sequence
|
||||
int seq_len = (int) start_think.size();
|
||||
if (i - seq_len + 1 >= 0) {
|
||||
bool match = true;
|
||||
for (int j = 0; j < seq_len; ++j) {
|
||||
if (current_context_tokens[i - seq_len + 1 + j] != start_think[j]) {
|
||||
match = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (match) {
|
||||
start_think_index = i; // index of the last token of the start_think sequence
|
||||
}
|
||||
}
|
||||
}
|
||||
if (start_think_index != -1 && end_think_index != -1) { // Early exit once both are found
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (start_think_index == -1) { // If no start_think found, do nothing
|
||||
return id;
|
||||
}
|
||||
|
||||
if (end_think_index != -1 && end_think_index > start_think_index) { // If end_think comes after start_think, thinking is already closed
|
||||
return id;
|
||||
}
|
||||
|
||||
int tokens_since_start = ctx_size - 1 - start_think_index; // start_think is unclosed, check budget
|
||||
if (tokens_since_start >= budget) {
|
||||
return end_think[0]; // Force-close thinking by returning the end_think token
|
||||
}
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
static void grammar_accept_token(FileFormat file_format, int32_t n_vocab, struct llama_grammar * grammar, llama_token token)
|
||||
{
|
||||
const std::vector<llama_token> eog_tokens = GetEogIDs(file_format,n_vocab);
|
||||
@@ -3916,26 +3932,31 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
|
||||
//thinking budget handling
|
||||
std::vector<int> thinking_start_sequence;
|
||||
std::vector<int> thinking_end_sequence;
|
||||
std::vector<int> thinking_end_phrase_toksleft;
|
||||
std::string chat_template = "";
|
||||
if (file_format == FileFormat::GGUF_GENERIC) {
|
||||
chat_template = gpttype_get_chat_template();
|
||||
if (file_format_meta.model_architecture == llm_arch::LLM_ARCH_GEMMA4) {
|
||||
TokenizeString("<|channel>thought",thinking_start_sequence,file_format,false);
|
||||
TokenizeString("<channel|>",thinking_end_sequence,file_format,false);
|
||||
TokenizeString("\n(Reasoning Budget Exceeded)\n<channel|>",thinking_end_phrase_toksleft,file_format,false);
|
||||
//sanity check, start is 2 tokens and end is 1
|
||||
if(thinking_start_sequence.size()!=2 || thinking_end_sequence.size()!=1)
|
||||
{
|
||||
thinking_start_sequence.clear();
|
||||
thinking_end_sequence.clear();
|
||||
thinking_end_phrase_toksleft.clear();
|
||||
}
|
||||
} else {
|
||||
TokenizeString("<think>",thinking_start_sequence,file_format,false);
|
||||
TokenizeString("</think>",thinking_end_sequence,file_format,false);
|
||||
TokenizeString("\n(Reasoning Budget Exceeded)\n</think>",thinking_end_phrase_toksleft,file_format,false);
|
||||
//sanity check, start is 1 tokens and end is 1
|
||||
if(thinking_start_sequence.size()!=1 || thinking_end_sequence.size()!=1)
|
||||
{
|
||||
thinking_start_sequence.clear();
|
||||
thinking_end_sequence.clear();
|
||||
thinking_end_phrase_toksleft.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4914,20 +4935,14 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
|
||||
kcpp_data->mirostat, kcpp_data->mirostat_tau, kcpp_data->mirostat_eta,
|
||||
kcpp_data->dry_multiplier, kcpp_data->dry_base,
|
||||
kcpp_data->dry_allowed_length, kcpp_data->dry_penalty_last_n, kcpp_data->xtc_threshold, kcpp_data->xtc_probability,
|
||||
sampler_order, grammar, dynatemp_range, dynatemp_exponent, smoothing_factor, smoothing_curve, adaptive_target);
|
||||
sampler_order, grammar, dynatemp_range, dynatemp_exponent, smoothing_factor, smoothing_curve, adaptive_target,
|
||||
thinking_start_sequence, thinking_end_sequence, thinking_end_phrase_toksleft, kcpp_data->reasoning_budget);
|
||||
|
||||
if (adaptive_target > 0.0f) {
|
||||
float original_prob = original_candidates[id].p;
|
||||
adaptive_p_update_history(original_prob, adaptive_p_weighted_sum, adaptive_p_total_weight, adaptive_decay);
|
||||
}
|
||||
|
||||
//apply reasoning budget
|
||||
int newid = apply_reasoning_budget(id, thinking_start_sequence, thinking_end_sequence, kcpp_data->reasoning_budget);
|
||||
if (id != newid) {
|
||||
printf("\n(Reasoning Budget of %d tokens exceeded! Attempting to stop thinking, insert token %d!)\n", kcpp_data->reasoning_budget, newid);
|
||||
id = newid;
|
||||
}
|
||||
|
||||
if(draft_used)
|
||||
{
|
||||
int32_t draftedid = draft_results.draftids[logits_sampled];
|
||||
|
||||
+12
-1
@@ -2009,7 +2009,7 @@ def generate(genparams, stream_flag=False):
|
||||
ban_eos_token = genparams.get('ban_eos_token', False)
|
||||
stream_sse = stream_flag
|
||||
grammar = genparams.get('grammar', '')
|
||||
reasoning_budget = tryparseint(genparams.get('reasoning_budget', 0),0)
|
||||
|
||||
#translate grammar if its json
|
||||
try:
|
||||
grammarjson = json.loads(grammar)
|
||||
@@ -2075,6 +2075,17 @@ def generate(genparams, stream_flag=False):
|
||||
if max_length >= (max_context_length-min_remain_hardlimit):
|
||||
max_length = max_context_length-min_remain_hardlimit
|
||||
|
||||
reasoning_effort = genparams.get('reasoning_effort', '')
|
||||
reasoning_effort = reasoning_effort.strip().lower() if reasoning_effort else ''
|
||||
reasoning_budget = -1
|
||||
if reasoning_effort == "none":
|
||||
reasoning_budget = 0
|
||||
elif reasoning_effort == "minimal" or reasoning_effort == "low":
|
||||
reasoning_budget = tryparseint(0.25 * max_length,-1) # 25% of gen amount
|
||||
elif reasoning_effort == "medium":
|
||||
reasoning_budget = tryparseint(0.5 * max_length,-1) # 50% of gen amount
|
||||
else:
|
||||
pass #unrestricted
|
||||
|
||||
inputs.max_context_length = max_context_length # this will resize the context buffer if changed
|
||||
inputs.max_length = max_length
|
||||
|
||||
Reference in New Issue
Block a user