Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-10 14:59:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

double n_batch size when pipeline parallel is enabled, keep u_batch the same

Browse Source
This commit is contained in:
Concedo
2026-03-21 11:22:10 +08:00
parent 3113e3a643
commit 07327b6c10
2 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-backend.cpp
+1 -1
View File
@@ -670,7 +670,7 @@ static bool ggml_is_view_op(enum ggml_op op) {
#ifndef GGML_SCHED_MAX_COPIES
//kcpp can reduce this if you want to try make buffer sizes smaller on multigpu
#define GGML_SCHED_MAX_COPIES 4
#define GGML_SCHED_MAX_COPIES 2
#endif
struct ggml_backend_sched_split {
gpttype_adapter.cpp
+6 -2
View File
@@ -2149,16 +2149,20 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
kcpp_data->n_threads = inputs.threads;
kcpp_data->n_blasthreads = inputs.blasthreads;
bool isGguf = (file_format == FileFormat::GGUF_GENERIC);
kcpp_pipeline_parallelism = inputs.pipelineparallel;
kcpp_data->n_batch = GetBatchSize(inputs.batchsize, in_file_format);
kcpp_data->n_ubatch = kcpp_data->n_batch;
if(isGguf && kcpp_pipeline_parallelism)
{
//double the logical batch, while keeping the physical batch the same, pipeline parallel set GGML_SCHED_MAX_COPIES to 2
kcpp_data->n_batch *= 2;
}
kcpp_data->flash_attn = inputs.flash_attention;
kcpp_data->model_filename = inputs.model_filename;
kcpp_data->use_smartcontext = inputs.use_smartcontext;
kcpp_data->use_contextshift = inputs.use_contextshift;
kcpp_data->use_fastforward = inputs.use_fastforward;
kcpp_data->smartcache = inputs.smartcache;
kcpp_pipeline_parallelism = inputs.pipelineparallel;
kcpp_data->swa_full = !inputs.swa_support;
if (!kcpp_data->swa_full) {
if (inputs.use_contextshift) {