Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 17:25:07 +02:00
Code Issues Packages Projects Releases Wiki Activity

bug fixes for backend code

Browse Source
This commit is contained in:
Concedo
2026-08-22 17:28:48 +08:00
parent b9b3bbcec1
commit 2b78904b4a
6 changed files with 41 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+3 -3
View File
@@ -726,7 +726,7 @@ LLAMASERVER_CXXFLAGS := -I./tools/mtmd
#whisper objects
whispercpp_default.o: otherarch/whispercpp/whisper_adapter.cpp
whispercpp_default.o: otherarch/whispercpp/whisper_adapter.cpp otherarch/whispercpp/whisper.cpp kcpp_backend.h
$(CXX) $(CXXFLAGS) -c $< -o $@
#tts objects
@@ -740,7 +740,7 @@ music_default.o: otherarch/acestep/music_adapter.cpp
$(CXX) $(CXXFLAGS) -c $< -o $@
# idiotic "for easier compilation"
GPTTYPE_ADAPTER = gpttype_adapter.cpp model_adapter.h otherarch/otherarch.h include/llama.h otherarch/llama_v2.cpp otherarch/llama_v3.cpp otherarch/gptj_v1.cpp otherarch/gptj_v2.cpp otherarch/gptj_v3.cpp otherarch/gpt2_v1.cpp otherarch/gpt2_v2.cpp otherarch/gpt2_v3.cpp otherarch/rwkv_v2.cpp otherarch/rwkv_v3.cpp otherarch/neox_v2.cpp otherarch/neox_v3.cpp otherarch/mpt_v3.cpp
GPTTYPE_ADAPTER = gpttype_adapter.cpp kcpp_backend.h model_adapter.h otherarch/otherarch.h include/llama.h otherarch/llama_v2.cpp otherarch/llama_v3.cpp otherarch/gptj_v1.cpp otherarch/gptj_v2.cpp otherarch/gptj_v3.cpp otherarch/gpt2_v1.cpp otherarch/gpt2_v2.cpp otherarch/gpt2_v3.cpp otherarch/rwkv_v2.cpp otherarch/rwkv_v3.cpp otherarch/neox_v2.cpp otherarch/neox_v3.cpp otherarch/mpt_v3.cpp
gpttype_adapter_default.o: $(GPTTYPE_ADAPTER)
$(CXX) $(CXXFLAGS) -c $< -o $@
@@ -771,7 +771,7 @@ fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.
$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS)
sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp kcpp_backend.o build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp kcpp_backend.h kcpp_backend_default.o build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
gpttype_adapter.cpp
+6 -7
View File
@@ -525,11 +525,10 @@ std::string get_fitted_params_str(const llama_model_params & mparams, const llam
static bool has_tensor_split(const float* ratios, int num_ratios, ggml_backend_t backend = nullptr)
{
if(kcpp_backend_check(KCPP_BACKENDS_TENSOR_SPLIT, backend)) {
for (int i = 0; i < tensor_split_max; ++i) {
if (ratios[i] != 0.0f) {
return true;
}
const int ratios_count = std::min(num_ratios, tensor_split_max);
for (int i = 0; i < ratios_count; ++i) {
if (ratios[i] != 0.0f) {
return true;
}
}
return false;
@@ -2979,7 +2978,7 @@ static void connect_rpc_servers(const std::string & servers) {
mtmd_context_params init_mtmd_ctx_params(bool mmproj_cpu, bool dryrun)
{
if(kcpp_backend_check("mtl")) {
if(kcpp_backend_check(KCPP_BACKENDS_METAL)) {
if(file_format_meta.model_architecture == llm_arch::LLM_ARCH_QWEN2VL || file_format_meta.model_architecture == llm_arch::LLM_ARCH_GEMMA3)
{
mmproj_cpu = true;
@@ -2990,7 +2989,7 @@ mtmd_context_params init_mtmd_ctx_params(bool mmproj_cpu, bool dryrun)
}
}
llama_flash_attn_type mtmd_fa = (kcpp_data->flash_attn?LLAMA_FLASH_ATTN_TYPE_ENABLED:LLAMA_FLASH_ATTN_TYPE_DISABLED);
if(kcpp_backend_check("cuda")) {
if(kcpp_backend_check(KCPP_BACKENDS_USE_CUDA)) {
mtmd_fa = LLAMA_FLASH_ATTN_TYPE_DISABLED; //kcpp: disabled in 1.102.2 as some headsizes break on turing
}
if(mmproj_cpu)
kcpp_backend.cpp
+13 -6
View File
@@ -33,7 +33,7 @@ static bool has_any_prefix(const std::string& str, const std::string& prefixes,
size_t end = prefixes.find(delimiter);
while (start != std::string::npos) {
std::string prefix = prefixes.substr(start, end - start);
if (str.rfind(prefix, 0) == 0) {
if (!prefix.empty() && str.rfind(prefix, 0) == 0) {
return true;
}
if (end == std::string::npos) {
@@ -63,13 +63,16 @@ ggml_backend_dev_t kcpp_backend_get_device(int index)
return ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
}
} else {
if ((size_t) index >= ggml_backend_dev_count()) {
return nullptr;
}
return ggml_backend_dev_get((size_t)index);
}
}
// this is similar to sd_backend_is, except:
// - if no backend is provided, checks the first ggml device (should be equivalent to a compile-time check)
// - tests a |-separated list of device names
// - tests a |-separated list of backend/device name prefixes
int kcpp_backend_check(const char* name, ggml_backend_t backend)
{
std::string loname = to_lowercase(name);
@@ -88,7 +91,7 @@ int kcpp_backend_check(const char* name, ggml_backend_t backend)
const char * devname = "vulkan";
#elif defined(GGML_USE_METAL)
const char * devname = "mtl";
const char * devname = "metal";
#elif defined(GGML_USE_HIP)
const char * devname = "rocm";
@@ -107,19 +110,23 @@ int kcpp_backend_check(const char* name, ggml_backend_t backend)
#endif
if (devname != nullptr) {
return has_any_prefix(devname, name, '|');
return has_any_prefix(devname, loname, '|');
}
}
dev = get_ggml_main_device();
} else {
std::string lo_backend_name = to_lowercase(ggml_backend_name(backend));
if (has_any_prefix(lo_backend_name, loname, '|')) {
return true;
}
dev = ggml_backend_get_device(backend);
}
if (!dev) {
return false;
}
std::string lo_dev_name = to_lowercase(ggml_backend_dev_name(dev));
return has_any_prefix(lo_dev_name, name, '|');
return has_any_prefix(lo_dev_name, loname, '|');
}
bool kcpp_backend_metal_supports_family(ggml_backend_t backend, int family)
@@ -155,7 +162,7 @@ void kcpp_backend_cuda_ggmlv3_set_tensor_split(const float * tensor_split)
void kcpp_backend_cuda_ggmlv3_transform_tensor(void * data, struct ggml_v3_tensor * tensor)
{
#if defined(GGML_USE_CUDA)
ggml_v3_cuda_transform_tensor(tensor->data, tensor);
ggml_v3_cuda_transform_tensor(data, tensor);
#endif
}
kcpp_backend.h
+3
View File
@@ -7,6 +7,9 @@
// backends with GGML_USE_CUDA
#define KCPP_BACKENDS_USE_CUDA "cuda|rocm"
// Metal backend aliases
#define KCPP_BACKENDS_METAL "metal|mtl"
// backends that support tensor split
#define KCPP_BACKENDS_TENSOR_SPLIT "cuda|rocm|vulkan"
otherarch/llama_v3.cpp
+10 -12
View File
@@ -1255,7 +1255,7 @@ static void llama_v3_model_load_internal(
ctx_size +
mmapped_size - vram_weights; // weights in VRAM not in memory
if (llama_v3_use_allocator()){
if (llama_v3_use_scratch()){
mem_required +=
blasbatchmul*MEM_REQ_SCRATCH0_3(hparams.n_ctx).at(model.type) +
blasbatchmul*MEM_REQ_SCRATCH1_3().at(model.type) +
@@ -1815,15 +1815,15 @@ static bool llama_v3_eval_internal(
const int64_t n_embd = hparams.n_embd;
const int64_t n_vocab = hparams.n_vocab;
#ifdef LLAMA_V3_USE_ALLOCATOR
ggml_v3_allocr_reset(lctx.alloc);
#endif
if (llama_v3_use_allocator()) {
ggml_v3_allocr_reset(lctx.alloc);
}
ggml_v3_cgraph * gf = llama_v3_build_graph(lctx, tokens, embd, n_tokens, n_past);
#ifdef LLAMA_V3_USE_ALLOCATOR
ggml_v3_allocr_alloc_graph(lctx.alloc, gf);
#endif
if (llama_v3_use_allocator()) {
ggml_v3_allocr_alloc_graph(lctx.alloc, gf);
}
// LLAMA_V3_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_v3_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);
@@ -3377,8 +3377,7 @@ struct llama_v3_context * llama_v3_new_context_with_model(
ctx->embedding.resize(hparams.n_embd);
}
#ifdef LLAMA_V3_USE_ALLOCATOR
{
if (llama_v3_use_allocator()) {
static const size_t tensor_alignment = 32;
// the compute buffer is used to store the tensor and graph structs, while the allocator buffer is used for the tensor data
ctx->buf_compute.resize(ggml_v3_tensor_overhead()*GGML_V3_MAX_NODES + ggml_v3_graph_overhead());
@@ -3410,10 +3409,9 @@ struct llama_v3_context * llama_v3_new_context_with_model(
ctx->buf_alloc.resize(alloc_size);
ctx->alloc = ggml_v3_allocr_new(ctx->buf_alloc.addr, ctx->buf_alloc.size, tensor_alignment);
} else {
ctx->buf_compute.resize(blasbatchmul*MEM_REQ_EVAL_3().at(ctx->model.type) + ggml_v3_graph_overhead());
}
#else
ctx->buf_compute.resize(blasbatchmul*MEM_REQ_EVAL_3().at(ctx->model.type) + ggml_v3_graph_overhead());
#endif
if (llama_v3_use_scratch()) {
ctx->buf_scratch[0].resize(blasbatchmul*MEM_REQ_SCRATCH0_3(hparams.n_ctx).at(ctx->model.type));
otherarch/whispercpp/whisper.cpp
+6 -6
View File
@@ -1070,7 +1070,7 @@ static uint32_t whisper_kv_cache_get_padding(const struct whisper_context & wctx
return 1u;
}
if (kcpp_backend_check("mtl", wctx.backend)) {
if (kcpp_backend_check(KCPP_BACKENDS_METAL, wctx.backend)) {
return 32u;
}
@@ -1225,7 +1225,7 @@ static ggml_backend_t whisper_backend_init(const whisper_context_params & params
if (!backend_gpu) {
WHISPER_LOG_ERROR("%s: ggml_backend_dev_init() failed\n", __func__);
} else {
if (kcpp_backend_check("mtl", backend_gpu) && !kcpp_backend_metal_supports_family(backend_gpu, 7)) {
if (kcpp_backend_check(KCPP_BACKENDS_METAL, backend_gpu) && !kcpp_backend_metal_supports_family(backend_gpu, 7)) {
WHISPER_LOG_ERROR("%s: Metal GPU does not support family 7 - falling back to CPU\n", __func__);
ggml_backend_free(backend_gpu);
backend_gpu = NULL;
@@ -1861,7 +1861,7 @@ static struct ggml_cgraph * whisper_build_graph_encoder(
whisper_context & wctx,
whisper_state & wstate) {
auto ggml_mul_mat = kcpp_backend_check("mtl", wctx.backend)
auto ggml_mul_mat = kcpp_backend_check(KCPP_BACKENDS_METAL, wctx.backend)
? ggml_mul_mat_pad
: ggml_mul_mat_original;
@@ -2111,7 +2111,7 @@ static struct ggml_cgraph * whisper_build_graph_cross(
whisper_context & wctx,
whisper_state & wstate) {
auto ggml_mul_mat = kcpp_backend_check("mtl", wctx.backend)
auto ggml_mul_mat = kcpp_backend_check(KCPP_BACKENDS_METAL, wctx.backend)
? ggml_mul_mat_pad
: ggml_mul_mat_original;
@@ -2303,7 +2303,7 @@ static struct ggml_cgraph * whisper_build_graph_decoder(
bool save_alignment_heads_QKs,
bool worst_case) {
auto ggml_mul_mat = kcpp_backend_check("mtl", wctx.backend)
auto ggml_mul_mat = kcpp_backend_check(KCPP_BACKENDS_METAL, wctx.backend)
? ggml_mul_mat_pad
: ggml_mul_mat_original;
@@ -6565,7 +6565,7 @@ WHISPER_API int whisper_bench_ggml_mul_mat(int n_threads) {
WHISPER_API const char * whisper_bench_ggml_mul_mat_str(int n_threads) {
auto ggml_mul_mat = kcpp_backend_check("mtl")
auto ggml_mul_mat = kcpp_backend_check(KCPP_BACKENDS_METAL)
? ggml_mul_mat_pad
: ggml_mul_mat_original;