diff --git a/expose.h b/expose.h index 569993231..9a6163072 100644 --- a/expose.h +++ b/expose.h @@ -124,6 +124,7 @@ struct generation_inputs const float smoothing_factor = 0.0f; const float smoothing_curve = 1.0f; const float adaptive_target = -1.0f; + const float adaptive_decay = 0.9f; const float dry_multiplier = 0.0f; const float dry_base = 0.0f; const int dry_allowed_length = 0; diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 835742d18..03b99d7b9 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -1303,16 +1303,13 @@ llama_token_data_array * cur_p) //update EMA history AFTER sampling, update_adaptive_p_history(original_prob[idx]) } -inline void adaptive_p_update_history(float selected_token_prob, float & weighted_sum, float & total_weight) { +inline void adaptive_p_update_history(float selected_token_prob, float & weighted_sum, float & total_weight, float adaptive_decay) { // decay controls how quickly history influence fades (0.0 to 0.99) // lower values = faster adaptation, more reactive to recent tokens // higher values = slower adaptation, more stable over time - // effective history length ≈ 1/(1-decay) tokens - // example: decay=0.5 --> ~2 tokens; decay=0.9 --> ~10 tokens; decay=0.95 --> ~20 tokens // keep <= 0.99 to prevent unbounded accumulation - const float adaptive_p_decay = 0.90f; - weighted_sum = selected_token_prob + adaptive_p_decay * weighted_sum; - total_weight = 1.0f + adaptive_p_decay * total_weight; + weighted_sum = selected_token_prob + adaptive_decay * weighted_sum; + total_weight = 1.0f + adaptive_decay * total_weight; } @@ -3656,6 +3653,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) kcpp_data->smoothing_factor = inputs.smoothing_factor; kcpp_data->smoothing_curve = inputs.smoothing_curve; kcpp_data->adaptive_target = inputs.adaptive_target; + kcpp_data->adaptive_decay = inputs.adaptive_decay; // Parse dry sequence breakers / restart sequences kcpp_data->dry_sequence_breakers.clear(); @@ -4485,6 +4483,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) const float smoothing_factor = kcpp_data->smoothing_factor; const float smoothing_curve = kcpp_data->smoothing_curve; const float adaptive_target = kcpp_data->adaptive_target; + const float adaptive_decay = kcpp_data->adaptive_decay; if (!startedsampling) { @@ -4622,7 +4621,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) if (adaptive_target > 0.0f) { float original_prob = original_candidates[id].p; - adaptive_p_update_history(original_prob, adaptive_p_weighted_sum, adaptive_p_total_weight); + adaptive_p_update_history(original_prob, adaptive_p_weighted_sum, adaptive_p_total_weight, adaptive_decay); } if(draft_used) diff --git a/koboldcpp.py b/koboldcpp.py index 572f850a3..28e7f447c 100755 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -266,6 +266,7 @@ class generation_inputs(ctypes.Structure): ("smoothing_factor", ctypes.c_float), ("smoothing_curve", ctypes.c_float), ("adaptive_target", ctypes.c_float), + ("adaptive_decay", ctypes.c_float), ("dry_multiplier", ctypes.c_float), ("dry_base", ctypes.c_float), ("dry_allowed_length", ctypes.c_int), @@ -1605,8 +1606,9 @@ def generate(genparams, stream_flag=False): smoothing_factor = tryparsefloat(genparams.get('smoothing_factor', 0.0),0.0) smoothing_curve = tryparsefloat(genparams.get('smoothing_curve', 1.0),1.0) adaptive_target = tryparsefloat(genparams.get('adaptive_target', -1.0),-1.0) + adaptive_decay = tryparsefloat(genparams.get('adaptive_decay', 0.9),0.9) if adaptive_target>0 and min_p<=0 and top_p>=1.0: #adaptive p sampler requires a truncation sampler first, force a tiny min-p - min_p = 0.01 + min_p = 0.002 logit_biases = genparams.get('logit_bias', {}) render_special = genparams.get('render_special', False) banned_strings = genparams.get('banned_strings', []) # SillyTavern uses that name @@ -1671,6 +1673,7 @@ def generate(genparams, stream_flag=False): inputs.smoothing_factor = smoothing_factor inputs.smoothing_curve = smoothing_curve inputs.adaptive_target = adaptive_target + inputs.adaptive_decay = adaptive_decay inputs.grammar = grammar.encode("UTF-8") inputs.grammar_retain_state = grammar_retain_state inputs.allow_eos_token = not ban_eos_token diff --git a/otherarch/otherarch.h b/otherarch/otherarch.h index 7fd4658b2..014f6e4fa 100644 --- a/otherarch/otherarch.h +++ b/otherarch/otherarch.h @@ -51,6 +51,7 @@ struct kcpp_params { float dynatemp_range = 0.0f; // enables DynaTemp if neq 0. dynatemp_min = temperature - dt_range, dynatemp_max = temperature + dt_range float dynatemp_exponent = 1.0f; float adaptive_target = -1.0f; // 0.0 - 1.0, <=0.0 is disabled + float adaptive_decay = 0.9f; std::string model_filename = ""; // model path std::string prompt = "";