From 82cc19e0552f6c1ed80ab40902f75ced3558f7ee Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Mon, 6 Apr 2026 20:44:37 +0800 Subject: [PATCH] calculate some fields before autofit for more accurate estimate --- gpttype_adapter.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 19b9efa00..a41cc69da 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -2548,6 +2548,13 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in model_params.tensor_buft_overrides = tenos.data(); } + //set some ctx params early so autofit can use them. + llama_ctx_params.flash_attn_type = (kcpp_data->flash_attn?LLAMA_FLASH_ATTN_TYPE_ENABLED:LLAMA_FLASH_ATTN_TYPE_DISABLED); + llama_ctx_params.swa_full = kcpp_data->swa_full; + llama_ctx_params.type_k = (inputs.quant_k==2?GGML_TYPE_Q4_0:(inputs.quant_k==1?GGML_TYPE_Q8_0:(inputs.quant_k==3?GGML_TYPE_BF16:GGML_TYPE_F16))); + llama_ctx_params.type_v = (inputs.quant_v==2?GGML_TYPE_Q4_0:(inputs.quant_v==1?GGML_TYPE_Q8_0:(inputs.quant_v==3?GGML_TYPE_BF16:GGML_TYPE_F16))); + + //apply overrides from autofit float tensor_split_temp[128] = {0}; //temp buffer for autofit std::vector fit_params_target = std::vector(llama_max_devices(),1024*1024*1024); @@ -2666,11 +2673,6 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in llamamodel->vocab.set_eos_bos(0,0); } - llama_ctx_params.flash_attn_type = (kcpp_data->flash_attn?LLAMA_FLASH_ATTN_TYPE_ENABLED:LLAMA_FLASH_ATTN_TYPE_DISABLED); - llama_ctx_params.swa_full = kcpp_data->swa_full; - llama_ctx_params.type_k = (inputs.quant_k==2?GGML_TYPE_Q4_0:(inputs.quant_k==1?GGML_TYPE_Q8_0:(inputs.quant_k==3?GGML_TYPE_BF16:GGML_TYPE_F16))); - llama_ctx_params.type_v = (inputs.quant_v==2?GGML_TYPE_Q4_0:(inputs.quant_v==1?GGML_TYPE_Q8_0:(inputs.quant_v==3?GGML_TYPE_BF16:GGML_TYPE_F16))); - llama_ctx_v4 = llama_init_from_model(llamamodel, llama_ctx_params); if(load_guidance) {