Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 09:15:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	AGENTS.md
#	CODEOWNERS
#	CONTRIBUTING.md
#	docs/backend/OPENCL.md
#	docs/development/HOWTO-add-model.md
#	examples/training/finetune.cpp
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp-drv.cpp
#	ggml/src/ggml-hexagon/htp/act-ops.c
#	ggml/src/ggml-hexagon/htp/dma-queue.c
#	ggml/src/ggml-hexagon/htp/dma-queue.h
#	ggml/src/ggml-hexagon/htp/flash-attn-ops.c
#	ggml/src/ggml-hexagon/htp/flash-attn-ops.h
#	ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
#	ggml/src/ggml-hexagon/htp/htp-ctx.h
#	ggml/src/ggml-hexagon/htp/htp-ops.h
#	ggml/src/ggml-hexagon/htp/htp-tensor.c
#	ggml/src/ggml-hexagon/htp/htp-tensor.h
#	ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h
#	ggml/src/ggml-hexagon/htp/hvx-reduce.h
#	ggml/src/ggml-hexagon/htp/main.c
#	ggml/src/ggml-hexagon/htp/matmul-ops.c
#	ggml/src/ggml-hexagon/htp/matmul-ops.h
#	ggml/src/ggml-hexagon/htp/unary-ops.c
#	ggml/src/ggml-hexagon/htp/unary-ops.h
#	ggml/src/ggml-opencl/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	scripts/compare-llama-bench.py
#	scripts/snapdragon/ggml-hexagon-profile.py
#	scripts/snapdragon/ggml-hexagon-trace.py
#	scripts/sync_vendor.py
#	tests/test-arg-parser.cpp
#	tests/test-chat.cpp
#	tests/test-model-load-cancel.cpp
#	tests/test-quantize-stats.cpp
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/llama-bench/llama-bench.cpp
#	tools/server/README.md
#	tools/ui/src/lib/constants/settings-registry.ts
This commit is contained in:
Concedo
2026-07-25 12:20:51 +08:00
parent eea06b0a89 ed7adbfefd
commit 49dbdaaab5
81 changed files with 40233 additions and 848 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpttype_adapter.cpp
+6 -14
View File
@@ -502,9 +502,7 @@ static size_t estimate_draft_autofit_tax_mb(
llama_model_params draft_model_params = llama_model_default_params();
llama_context_params draft_ctx_params = llama_context_default_params();
draft_model_params.use_mmap = base_model_params.use_mmap;
draft_model_params.use_mlock = base_model_params.use_mlock;
draft_model_params.use_direct_io = base_model_params.use_direct_io;
draft_model_params.load_mode = base_model_params.load_mode;
draft_model_params.n_gpu_layers = has_draft_model ? draft_gpulayers : 0;
draft_model_params.devices = base_model_params.devices;
draft_model_params.main_gpu = base_model_params.main_gpu;
@@ -554,8 +552,7 @@ static size_t estimate_draft_autofit_tax_mb(
{
llama_model_params draft_probe_params = draft_model_params;
draft_probe_params.no_alloc = true;
draft_probe_params.use_mmap = false;
draft_probe_params.use_mlock = false;
draft_probe_params.load_mode = LLAMA_LOAD_MODE_NONE;
llama_model * draft_probe = llama_model_load_from_file(spec_model_filename.c_str(), draft_probe_params);
if(draft_probe != nullptr)
@@ -591,8 +588,7 @@ static size_t estimate_draft_autofit_tax_mb(
{
llama_model_params ctx_other_model_params = base_model_params;
ctx_other_model_params.no_alloc = true;
ctx_other_model_params.use_mmap = false;
ctx_other_model_params.use_mlock = false;
ctx_other_model_params.load_mode = LLAMA_LOAD_MODE_NONE;
ctx_other_model = llama_model_load_from_file(main_model_filename.c_str(), ctx_other_model_params);
if(ctx_other_model != nullptr)
@@ -923,9 +919,7 @@ static void speculative_decoding_setup(std::string spec_model_filename, llama_co
llama_model_params draft_model_params = llama_model_default_params();
llama_context_params draft_ctx_params = llama_context_default_params();
draft_model_params.use_mmap = base_model_params.use_mmap;
draft_model_params.use_mlock = base_model_params.use_mlock;
draft_model_params.use_direct_io = base_model_params.use_direct_io;
draft_model_params.load_mode = base_model_params.load_mode;
draft_model_params.n_gpu_layers = draft_gpulayers; //layers offload the speculative model.
draft_model_params.devices = base_model_params.devices;
draft_ctx_params.n_ctx = base_ctx_params.n_ctx;
@@ -3235,11 +3229,9 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
// Match llama-server's target rollback slots for speculative verification.
llama_ctx_params.n_rs_seq = inputs.draft_amount;
}
model_params.use_mmap = inputs.use_mmap;
model_params.use_mlock = inputs.use_mlock;
model_params.use_direct_io = false; //no direct io for now until stable
model_params.load_mode = inputs.use_mlock ? LLAMA_LOAD_MODE_MLOCK : (inputs.use_mmap ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE);
model_params.n_gpu_layers = inputs.gpulayers;
kcpp_permit_any_repack = (model_params.use_mmap?false:true);
kcpp_permit_any_repack = (inputs.use_mmap?false:true);
//set device overrides if needed
std::vector<ggml_backend_dev_t> devices_override;