From 17c754a5fc91fd206ac747ea4128b2d4b1f22917 Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Sat, 18 Apr 2026 17:19:09 +0800 Subject: [PATCH] improved reasoning budget --- embd_res/klite.embd | 71 ++++++++++++++++++----- expose.h | 2 +- gpttype_adapter.cpp | 135 ++++++++++++++++++++++++-------------------- koboldcpp.py | 13 ++++- 4 files changed, 145 insertions(+), 76 deletions(-) diff --git a/embd_res/klite.embd b/embd_res/klite.embd index 03b407bad..ddceb613e 100644 --- a/embd_res/klite.embd +++ b/embd_res/klite.embd @@ -4147,7 +4147,7 @@ Current version indicated by LITEVER below. const alltalk_voices_endpoint = "/api/voices"; const alltalk_rvc_voices_endpoint = "/api/rvcvoices"; - const pollinations_img_endpoint = "https://gen.pollinations.ai/image"; + const pollinations_img_endpoint = "https://image.pollinations.ai/prompt"; const pollinations_text_endpoint = "https://text.pollinations.ai/openai"; const dummy_api_key = "kobo"; @@ -4361,7 +4361,6 @@ Current version indicated by LITEVER below. saved_palm_jailbreak:"", //gemini system prompt saved_palm_jailbreak2:"", //gemini postfix saved_openrouter_provider:"", //for overriding provider - saved_oai_reasoning:"", //if set, reasoning level "none,minimal,low,medium,high", else default saved_oai_custommodel: "", //customized oai custom model saved_oai_role: 3, //0=user,1=assistant,2=system, 3=auto saved_a1111_url: default_a1111_base, @@ -4373,7 +4372,7 @@ Current version indicated by LITEVER below. prev_custom_endpoint_type: 0, //show a reconnect box to custom endpoint if needed. 0 is horde, otherwise its dropdown value+1 prev_custom_endpoint_model: "", //we may not be able to match, but set it if we do prev_custom_endpoint_ischatcmpl: true, - generate_images_mode: (localflag?0:1), //0=off, 1=horde, 2=a1111, 3=dalle, 4=comfy + generate_images_mode: (localflag?0:1), //0=off, 1=horde, 2=a1111, 3=dalle, 4=comfy, 5=pollinations autoscroll: true, //automatically scroll to bottom on render printer_view: false, //automatically scroll to bottom on render @@ -4505,6 +4504,7 @@ Current version indicated by LITEVER below. handle_mismatched_think: true, think_injected: 0, //0=normal, 1=force strip_thinking_mode: 1, //0=no stripping, 1=strip except recent, 2=strip all + reasoning_effort: "", //if set, reasoning level "none,minimal,low,medium,high", else default empty string websearch_enabled: false, websearch_multipass: false, websearch_retain: false, @@ -9303,7 +9303,7 @@ Current version indicated by LITEVER below. nologo: true, private: true, referrer: "koboldai", - key: localsettings.saved_dalle_key + // key: localsettings.saved_dalle_key }); let gen_endpoint = `${pollinations_img_endpoint}/${encodeURIComponent(prompt)}?${pollinations_params.toString()}`; @@ -13693,7 +13693,7 @@ Current version indicated by LITEVER below. document.getElementById("oaidesc").classList.add("hidden"); document.getElementById("pollinationsdesc").classList.add("hidden"); document.getElementById("openrouterproviderbox").classList.add("hidden"); - document.getElementById("oaireasoningeffort").value = localsettings.saved_oai_reasoning; + document.getElementById("oaireasoningeffort").value = localsettings.reasoning_effort; if(epchoice==2) { document.getElementById("oaidesc").classList.remove("hidden"); @@ -14312,7 +14312,7 @@ Current version indicated by LITEVER below. localsettings.saved_openrouter_key = custom_oai_key; localsettings.saved_openrouter_provider = document.getElementById("openrouterproviders").value; } - localsettings.saved_oai_reasoning = document.getElementById("oaireasoningeffort").value; + localsettings.reasoning_effort = document.getElementById("oaireasoningeffort").value; localsettings.saved_oai_jailbreak = document.getElementById("jailbreakprompttext").value; if(localsettings.saved_oai_jailbreak=="") { @@ -15734,10 +15734,6 @@ Current version indicated by LITEVER below. mainmenu_untab(true); document.getElementById("settingscontainer").classList.remove("hidden"); display_settings_tab(current_settings_tab_idx); - - if(localsettings.generate_images_mode>=5){localsettings.generate_images_mode=0;} //todo: temp hack to remove pollinations images - if(localsettings.tts_mode>=6){localsettings.tts_mode=0;} //todo: temp hack to remove pollinations tts - document.getElementById("max_context_length").value = document.getElementById("max_context_length_slide").value = localsettings.max_context_length; document.getElementById("max_length").value = document.getElementById("max_length_slide").value = localsettings.max_length; document.getElementById("temperature").value = document.getElementById("temperature_slide").value = localsettings.temperature; @@ -16817,6 +16813,7 @@ Current version indicated by LITEVER below. } localsettings.hide_think_stream = (document.getElementById("hide_think_stream").checked?true:false); localsettings.handle_mismatched_think = (document.getElementById("handle_mismatched_think").checked?true:false); + localsettings.reasoning_effort = document.getElementById("reasoningeffort").value; localsettings.tts_mode = document.getElementById("ttsselect").value; localsettings.xtts_voice = document.getElementById("xtts_voices").value; @@ -17107,6 +17104,7 @@ Current version indicated by LITEVER below. document.getElementById("generate_images_dalle_container").classList.add("hidden"); document.getElementById("generate_images_local_model_container").classList.add("hidden"); document.getElementById("generate_images_comfy_container").classList.add("hidden"); + document.getElementById("generate_images_pollinations_container").classList.add("hidden"); if(document.getElementById("generate_images_mode").value==1){ document.getElementById("generate_images_model_container").classList.remove("hidden"); if(!image_models_fetched) @@ -17125,7 +17123,10 @@ Current version indicated by LITEVER below. }else if(document.getElementById("generate_images_mode").value==4){ document.getElementById("generate_images_comfy_container").classList.remove("hidden"); connect_to_comfyui(silent); - } + }else if(document.getElementById("generate_images_mode").value==5) + { + document.getElementById("generate_images_pollinations_container").classList.remove("hidden"); + } } function get_theme_desc(themeid) @@ -20741,6 +20742,7 @@ Current version indicated by LITEVER below. submit_payload.params.banned_tokens = get_token_bans(); submit_payload.params.render_special = localsettings.render_special_tags; submit_payload.params.logprobs = localsettings.request_logprobs; + submit_payload.params.reasoning_effort = localsettings.reasoning_effort; let st = get_instruct_starttag(true); let et = get_instruct_endtag(true); if(st=="{{[INPUT]}}" || et=="{{[OUTPUT]}}") @@ -21055,10 +21057,20 @@ Current version indicated by LITEVER below. if (document.getElementById("useoaichatcompl").checked) { - let cannotreason = (targetep.toLowerCase().includes("text.pollinations.ai") || targetep.toLowerCase().includes("api.nvidia.com")); + let cannotreason = (targetep.toLowerCase().includes("text.pollinations.ai")); if(!cannotreason) { - let reasoneffort = document.getElementById("oaireasoningeffort").value; + let reasoneffort = localsettings.reasoning_effort; + if (targetep.toLowerCase().includes("api.nvidia.com")) { + if (reasoneffort=="minimal") + { + reasoneffort = "low"; + } + if (reasoneffort != "low" && reasoneffort != "medium" && reasoneffort != "high") //only accepts low, medium and high + { + reasoneffort = ""; + } + } if(reasoneffort && reasoneffort!="") { oai_payload.reasoning_effort = reasoneffort; @@ -21254,7 +21266,7 @@ Current version indicated by LITEVER below. //pollinations always uses the exact same url for text gen regardless if(targetep.toLowerCase().includes("text.pollinations.ai")) { - targetep = apply_proxy_url(pollinations_text_endpoint,true); + targetep = pollinations_text_endpoint; oai_payload.private = true; oai_payload.referrer = "koboldai"; oai_payload.seed = Math.floor(Math.random() * 99999999); @@ -22028,6 +22040,12 @@ Current version indicated by LITEVER below. genimg_payload.models = [desired_model]; imgid = generate_comfy_image(genimg_payload, autoappend); } + else if(localsettings.generate_images_mode==5) //pollinations + { + let desired_model = document.getElementById("generate_images_pollinations_model").value; + genimg_payload.models = [desired_model]; + imgid = generate_pollinations_image(genimg_payload, autoappend); + } return imgid; } @@ -26774,6 +26792,7 @@ Current version indicated by LITEVER below. document.getElementById("stop_thinking_tag").value = localsettings.stop_thinking_tag; document.getElementById("hide_think_stream").checked = (localsettings.hide_think_stream?true:false); document.getElementById("handle_mismatched_think").checked = (localsettings.handle_mismatched_think?true:false); + document.getElementById("reasoningeffort").value = localsettings.reasoning_effort; } function populate_placeholder_tags() @@ -30092,6 +30111,7 @@ Current version indicated by LITEVER below. + @@ -30150,6 +30170,16 @@ Current version indicated by LITEVER below. + +
Add positive prompt
@@ -30587,6 +30617,19 @@ Current version indicated by LITEVER below.
+
+
Reasoning Effort ?If set, restricts thinking length to this many tokens before forcing a reply.
+
+ +
+
diff --git a/expose.h b/expose.h index 090daed2f..f7616e7c6 100644 --- a/expose.h +++ b/expose.h @@ -142,7 +142,7 @@ struct generation_inputs const logit_bias * logit_biases = nullptr; const int banned_tokens_len = 0; const char ** banned_tokens = nullptr; - const int reasoning_budget = 0; + const int reasoning_budget = -1; }; struct generation_outputs { diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 5b1390399..ba1adb130 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -1754,9 +1754,63 @@ void sample_guidance(struct llama_context * ctx, struct llama_context * guidance } } +static int apply_reasoning_budget(int id, const std::vector & start_think, const std::vector & end_think, std::vector & think_end_phrase_toks, int budget) +{ + if(budget<0 || start_think.size()==0 || end_think.size()!=1 || think_end_phrase_toks.size()==0) //start_think can be 1-3 tokens long, end_think is always 1 token + { + return id; + } + + int end_think_index = -1; + int start_think_index = -1; + int ctx_size = (int)current_context_tokens.size(); + + for (int i = ctx_size - 1; i >= 0; --i) { // Search backwards for the latest end_think token + if (end_think_index == -1 && current_context_tokens[i] == end_think[0]) { + end_think_index = i; + } + if (start_think_index == -1) { // Search backwards for the latest start_think sequence + int seq_len = (int) start_think.size(); + if (i - seq_len + 1 >= 0) { + bool match = true; + for (int j = 0; j < seq_len; ++j) { + if (current_context_tokens[i - seq_len + 1 + j] != start_think[j]) { + match = false; + break; + } + } + if (match) { + start_think_index = i; // index of the last token of the start_think sequence + } + } + } + if (start_think_index != -1 && end_think_index != -1) { // Early exit once both are found + break; + } + } + + if (start_think_index == -1) { // If no start_think found, do nothing + return id; + } + + if (end_think_index != -1 && end_think_index > start_think_index) { // If end_think comes after start_think, thinking is already closed + return id; + } + + int tokens_since_start = ctx_size - 1 - start_think_index; // start_think is unclosed, check budget + if (tokens_since_start >= budget) { + int popped = think_end_phrase_toks[0]; // Force-close thinking by returning the end thinking phrase, pop front and return + think_end_phrase_toks.erase(think_end_phrase_toks.begin()); // Elements shift left + return popped; + } + + return id; +} + int SampleLogits(const float * logits, int n_ctx, int n_vocab, int rep_pen_range, float rep_pen, float rep_pen_slope, float presence_penalty, float top_k, float top_a, float top_p, float min_p, float typical_p, float tfs, float nsigma, float temp, std::mt19937 & rng, int mirostat, float mirostat_tau, float mirostat_eta, float dry_multiplier, float dry_base, int dry_allowed_length, int dry_penalty_last_n, float xtc_threshold, float xtc_probability, -const std::vector & sampler_order, llama_grammar * grammar, float dynatemp_range, float dynatemp_exponent, float smoothing_factor, float smoothing_curve, float adaptive_target) +const std::vector & sampler_order, llama_grammar * grammar, float dynatemp_range, float dynatemp_exponent, float smoothing_factor, float smoothing_curve, float adaptive_target, +const std::vector & think_start_seq, const std::vector & think_end_seq, std::vector & think_end_phrase_toks, int reasoning_budget) { // printf("SampleLogits called with: n_ctx=%d, n_vocab=%d, rep_pen_range=%d, rep_pen=%f, rep_pen_slope=%f, presence_penalty=%f, top_k=%f, top_a=%f, top_p=%f, min_p=%f, typical_p=%f, tfs=%f, nsigma=%f, temp=%f, mirostat=%d, mirostat_tau=%f, mirostat_eta=%f, dry_multiplier=%f, dry_base=%f, dry_allowed_length=%d, dry_penalty_last_n=%d, xtc_threshold=%f, xtc_probability=%f, sampler_order_size=%zu, dynatemp_range=%f, dynatemp_exponent=%f, smoothing_factor=%f\n", // n_ctx, n_vocab, rep_pen_range, rep_pen, rep_pen_slope, presence_penalty, top_k, top_a, top_p, min_p, typical_p, tfs, nsigma, temp, mirostat, mirostat_tau, mirostat_eta, dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, xtc_threshold, xtc_probability, sampler_order.size(), dynatemp_range, dynatemp_exponent, smoothing_factor); @@ -1776,6 +1830,19 @@ const std::vector & sampler_order, llama_grammar * grammar, float dyna llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false }; + //apply reasoning budget + int newid = apply_reasoning_budget(id, think_start_seq, think_end_seq, think_end_phrase_toks, kcpp_data->reasoning_budget); + if (id != newid) { + if(!is_quiet && debugmode!=-1) + { + printf("\n(Reasoning Budget of %d tokens exceeded! Finishing thinking...)\n", kcpp_data->reasoning_budget); + } + candidates[newid].logit += 99999; + sample_top_k(&candidates_p, 1); + id = sample_token(&candidates_p, rng); + return id; + } + //dry always first as logits cannot be resorted sample_dry(n_ctx, dry_penalty_last_n, dry_multiplier, dry_base, dry_allowed_length, dry_sequence_breakers, &candidates_p); @@ -1870,57 +1937,6 @@ const std::vector & sampler_order, llama_grammar * grammar, float dyna return id; } -static int apply_reasoning_budget(int id, const std::vector & start_think, const std::vector & end_think, int budget) -{ - if(budget<=0 || start_think.size()==0 || end_think.size()!=1) //start_think can be 1-3 tokens long, end_think is always 1 token - { - return id; - } - - int end_think_index = -1; - int start_think_index = -1; - int ctx_size = (int)current_context_tokens.size(); - - for (int i = ctx_size - 1; i >= 0; --i) { // Search backwards for the latest end_think token - if (end_think_index == -1 && current_context_tokens[i] == end_think[0]) { - end_think_index = i; - } - if (start_think_index == -1) { // Search backwards for the latest start_think sequence - int seq_len = (int) start_think.size(); - if (i - seq_len + 1 >= 0) { - bool match = true; - for (int j = 0; j < seq_len; ++j) { - if (current_context_tokens[i - seq_len + 1 + j] != start_think[j]) { - match = false; - break; - } - } - if (match) { - start_think_index = i; // index of the last token of the start_think sequence - } - } - } - if (start_think_index != -1 && end_think_index != -1) { // Early exit once both are found - break; - } - } - - if (start_think_index == -1) { // If no start_think found, do nothing - return id; - } - - if (end_think_index != -1 && end_think_index > start_think_index) { // If end_think comes after start_think, thinking is already closed - return id; - } - - int tokens_since_start = ctx_size - 1 - start_think_index; // start_think is unclosed, check budget - if (tokens_since_start >= budget) { - return end_think[0]; // Force-close thinking by returning the end_think token - } - - return id; -} - static void grammar_accept_token(FileFormat file_format, int32_t n_vocab, struct llama_grammar * grammar, llama_token token) { const std::vector eog_tokens = GetEogIDs(file_format,n_vocab); @@ -3916,26 +3932,31 @@ generation_outputs gpttype_generate(const generation_inputs inputs) //thinking budget handling std::vector thinking_start_sequence; std::vector thinking_end_sequence; + std::vector thinking_end_phrase_toksleft; std::string chat_template = ""; if (file_format == FileFormat::GGUF_GENERIC) { chat_template = gpttype_get_chat_template(); if (file_format_meta.model_architecture == llm_arch::LLM_ARCH_GEMMA4) { TokenizeString("<|channel>thought",thinking_start_sequence,file_format,false); TokenizeString("",thinking_end_sequence,file_format,false); + TokenizeString("\n(Reasoning Budget Exceeded)\n",thinking_end_phrase_toksleft,file_format,false); //sanity check, start is 2 tokens and end is 1 if(thinking_start_sequence.size()!=2 || thinking_end_sequence.size()!=1) { thinking_start_sequence.clear(); thinking_end_sequence.clear(); + thinking_end_phrase_toksleft.clear(); } } else { TokenizeString("",thinking_start_sequence,file_format,false); TokenizeString("",thinking_end_sequence,file_format,false); + TokenizeString("\n(Reasoning Budget Exceeded)\n",thinking_end_phrase_toksleft,file_format,false); //sanity check, start is 1 tokens and end is 1 if(thinking_start_sequence.size()!=1 || thinking_end_sequence.size()!=1) { thinking_start_sequence.clear(); thinking_end_sequence.clear(); + thinking_end_phrase_toksleft.clear(); } } } @@ -4914,20 +4935,14 @@ generation_outputs gpttype_generate(const generation_inputs inputs) kcpp_data->mirostat, kcpp_data->mirostat_tau, kcpp_data->mirostat_eta, kcpp_data->dry_multiplier, kcpp_data->dry_base, kcpp_data->dry_allowed_length, kcpp_data->dry_penalty_last_n, kcpp_data->xtc_threshold, kcpp_data->xtc_probability, - sampler_order, grammar, dynatemp_range, dynatemp_exponent, smoothing_factor, smoothing_curve, adaptive_target); + sampler_order, grammar, dynatemp_range, dynatemp_exponent, smoothing_factor, smoothing_curve, adaptive_target, + thinking_start_sequence, thinking_end_sequence, thinking_end_phrase_toksleft, kcpp_data->reasoning_budget); if (adaptive_target > 0.0f) { float original_prob = original_candidates[id].p; adaptive_p_update_history(original_prob, adaptive_p_weighted_sum, adaptive_p_total_weight, adaptive_decay); } - //apply reasoning budget - int newid = apply_reasoning_budget(id, thinking_start_sequence, thinking_end_sequence, kcpp_data->reasoning_budget); - if (id != newid) { - printf("\n(Reasoning Budget of %d tokens exceeded! Attempting to stop thinking, insert token %d!)\n", kcpp_data->reasoning_budget, newid); - id = newid; - } - if(draft_used) { int32_t draftedid = draft_results.draftids[logits_sampled]; diff --git a/koboldcpp.py b/koboldcpp.py index 6fb20bdc1..6c64023e6 100755 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -2009,7 +2009,7 @@ def generate(genparams, stream_flag=False): ban_eos_token = genparams.get('ban_eos_token', False) stream_sse = stream_flag grammar = genparams.get('grammar', '') - reasoning_budget = tryparseint(genparams.get('reasoning_budget', 0),0) + #translate grammar if its json try: grammarjson = json.loads(grammar) @@ -2075,6 +2075,17 @@ def generate(genparams, stream_flag=False): if max_length >= (max_context_length-min_remain_hardlimit): max_length = max_context_length-min_remain_hardlimit + reasoning_effort = genparams.get('reasoning_effort', '') + reasoning_effort = reasoning_effort.strip().lower() if reasoning_effort else '' + reasoning_budget = -1 + if reasoning_effort == "none": + reasoning_budget = 0 + elif reasoning_effort == "minimal" or reasoning_effort == "low": + reasoning_budget = tryparseint(0.25 * max_length,-1) # 25% of gen amount + elif reasoning_effort == "medium": + reasoning_budget = tryparseint(0.5 * max_length,-1) # 50% of gen amount + else: + pass #unrestricted inputs.max_context_length = max_context_length # this will resize the context buffer if changed inputs.max_length = max_length