Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 09:15:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.devops/vulkan.Dockerfile
#	.github/workflows/build.yml
#	ci/run.sh
#	examples/model-conversion/Makefile
#	examples/model-conversion/README.md
#	examples/model-conversion/scripts/causal/compare-logits.py
#	examples/model-conversion/scripts/embedding/run-converted-model.sh
#	examples/model-conversion/scripts/utils/common.py
#	examples/model-conversion/scripts/utils/semantic_check.py
#	ggml/src/ggml-cann/ggml-cann.cpp
#	ggml/src/ggml-cuda/CMakeLists.txt
#	ggml/src/ggml-opencl/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	scripts/pr2wt.sh
#	scripts/sync_vendor.py
#	tests/test-arg-parser.cpp
This commit is contained in:
Concedo
2026-01-09 01:23:10 +08:00
parent 956ab99934 15bff84bf5
commit 983baac46b
28 changed files with 2516 additions and 757 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpttype_adapter.cpp
+5 -1
View File
@@ -641,6 +641,7 @@ static void speculative_decoding_setup(std::string spec_model_filename, const ll
draft_model_params.use_mmap = base_model_params.use_mmap;
draft_model_params.use_mlock = base_model_params.use_mlock;
draft_model_params.use_direct_io = base_model_params.use_direct_io;
draft_model_params.n_gpu_layers = draft_gpulayers; //layers offload the speculative model.
draft_ctx_params.n_ctx = base_ctx_params.n_ctx;
draft_ctx_params.offload_kqv = base_ctx_params.offload_kqv;
@@ -2371,6 +2372,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
llama_ctx_params.kv_unified = true;
model_params.use_mmap = inputs.use_mmap;
model_params.use_mlock = inputs.use_mlock;
model_params.use_direct_io = false; //no direct io for now until stable
model_params.n_gpu_layers = inputs.gpulayers;
#if defined(GGML_USE_CLBLAST)
@@ -2549,6 +2551,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
//apply overrides from autofit
float tensor_split_temp[128] = {0}; //temp buffer for autofit
std::vector<size_t> fit_params_target = std::vector<size_t>(llama_max_devices(),1024*1024*1024);
if(inputs.autofit)
{
common_params temp_params;
@@ -2561,8 +2564,9 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
model_params.tensor_split = tensor_split_temp;
model_params.n_gpu_layers = -1; //must be this value to be considered default
printf("Autofit Reserve Space: %d MB\n",taxmb);
fit_params_target[0] = taxmb*1024*1024;
llama_params_fit(kcpp_data->model_filename.c_str(), &model_params, &llama_ctx_params,
tensor_split_temp, tenos.data(), taxmb*1024*1024, kcpp_data->n_ctx,
tensor_split_temp, tenos.data(), fit_params_target.data(), kcpp_data->n_ctx,
GGML_LOG_LEVEL_DEBUG);
printf("Autofit Result: ");
print_fitted_params(model_params,llama_ctx_params);