diff --git a/Makefile b/Makefile index 55b56b2d1..c44ee4557 100644 --- a/Makefile +++ b/Makefile @@ -726,7 +726,7 @@ LLAMASERVER_CXXFLAGS := -I./tools/mtmd #whisper objects -whispercpp_default.o: otherarch/whispercpp/whisper_adapter.cpp +whispercpp_default.o: otherarch/whispercpp/whisper_adapter.cpp otherarch/whispercpp/whisper.cpp kcpp_backend.h $(CXX) $(CXXFLAGS) -c $< -o $@ #tts objects @@ -740,7 +740,7 @@ music_default.o: otherarch/acestep/music_adapter.cpp $(CXX) $(CXXFLAGS) -c $< -o $@ # idiotic "for easier compilation" -GPTTYPE_ADAPTER = gpttype_adapter.cpp model_adapter.h otherarch/otherarch.h include/llama.h otherarch/llama_v2.cpp otherarch/llama_v3.cpp otherarch/gptj_v1.cpp otherarch/gptj_v2.cpp otherarch/gptj_v3.cpp otherarch/gpt2_v1.cpp otherarch/gpt2_v2.cpp otherarch/gpt2_v3.cpp otherarch/rwkv_v2.cpp otherarch/rwkv_v3.cpp otherarch/neox_v2.cpp otherarch/neox_v3.cpp otherarch/mpt_v3.cpp +GPTTYPE_ADAPTER = gpttype_adapter.cpp kcpp_backend.h model_adapter.h otherarch/otherarch.h include/llama.h otherarch/llama_v2.cpp otherarch/llama_v3.cpp otherarch/gptj_v1.cpp otherarch/gptj_v2.cpp otherarch/gptj_v3.cpp otherarch/gpt2_v1.cpp otherarch/gpt2_v2.cpp otherarch/gpt2_v3.cpp otherarch/rwkv_v2.cpp otherarch/rwkv_v3.cpp otherarch/neox_v2.cpp otherarch/neox_v3.cpp otherarch/mpt_v3.cpp gpttype_adapter_default.o: $(GPTTYPE_ADAPTER) $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -771,7 +771,7 @@ fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg. $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp kcpp_backend.o build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp kcpp_backend.h kcpp_backend_default.o build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 56bd03ef7..fdd85b42b 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -525,11 +525,10 @@ std::string get_fitted_params_str(const llama_model_params & mparams, const llam static bool has_tensor_split(const float* ratios, int num_ratios, ggml_backend_t backend = nullptr) { - if(kcpp_backend_check(KCPP_BACKENDS_TENSOR_SPLIT, backend)) { - for (int i = 0; i < tensor_split_max; ++i) { - if (ratios[i] != 0.0f) { - return true; - } + const int ratios_count = std::min(num_ratios, tensor_split_max); + for (int i = 0; i < ratios_count; ++i) { + if (ratios[i] != 0.0f) { + return true; } } return false; @@ -2979,7 +2978,7 @@ static void connect_rpc_servers(const std::string & servers) { mtmd_context_params init_mtmd_ctx_params(bool mmproj_cpu, bool dryrun) { - if(kcpp_backend_check("mtl")) { + if(kcpp_backend_check(KCPP_BACKENDS_METAL)) { if(file_format_meta.model_architecture == llm_arch::LLM_ARCH_QWEN2VL || file_format_meta.model_architecture == llm_arch::LLM_ARCH_GEMMA3) { mmproj_cpu = true; @@ -2990,7 +2989,7 @@ mtmd_context_params init_mtmd_ctx_params(bool mmproj_cpu, bool dryrun) } } llama_flash_attn_type mtmd_fa = (kcpp_data->flash_attn?LLAMA_FLASH_ATTN_TYPE_ENABLED:LLAMA_FLASH_ATTN_TYPE_DISABLED); - if(kcpp_backend_check("cuda")) { + if(kcpp_backend_check(KCPP_BACKENDS_USE_CUDA)) { mtmd_fa = LLAMA_FLASH_ATTN_TYPE_DISABLED; //kcpp: disabled in 1.102.2 as some headsizes break on turing } if(mmproj_cpu) diff --git a/kcpp_backend.cpp b/kcpp_backend.cpp index 58a737f40..30564cfab 100644 --- a/kcpp_backend.cpp +++ b/kcpp_backend.cpp @@ -33,7 +33,7 @@ static bool has_any_prefix(const std::string& str, const std::string& prefixes, size_t end = prefixes.find(delimiter); while (start != std::string::npos) { std::string prefix = prefixes.substr(start, end - start); - if (str.rfind(prefix, 0) == 0) { + if (!prefix.empty() && str.rfind(prefix, 0) == 0) { return true; } if (end == std::string::npos) { @@ -63,13 +63,16 @@ ggml_backend_dev_t kcpp_backend_get_device(int index) return ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); } } else { + if ((size_t) index >= ggml_backend_dev_count()) { + return nullptr; + } return ggml_backend_dev_get((size_t)index); } } // this is similar to sd_backend_is, except: // - if no backend is provided, checks the first ggml device (should be equivalent to a compile-time check) -// - tests a |-separated list of device names +// - tests a |-separated list of backend/device name prefixes int kcpp_backend_check(const char* name, ggml_backend_t backend) { std::string loname = to_lowercase(name); @@ -88,7 +91,7 @@ int kcpp_backend_check(const char* name, ggml_backend_t backend) const char * devname = "vulkan"; #elif defined(GGML_USE_METAL) - const char * devname = "mtl"; + const char * devname = "metal"; #elif defined(GGML_USE_HIP) const char * devname = "rocm"; @@ -107,19 +110,23 @@ int kcpp_backend_check(const char* name, ggml_backend_t backend) #endif if (devname != nullptr) { - return has_any_prefix(devname, name, '|'); + return has_any_prefix(devname, loname, '|'); } } dev = get_ggml_main_device(); } else { + std::string lo_backend_name = to_lowercase(ggml_backend_name(backend)); + if (has_any_prefix(lo_backend_name, loname, '|')) { + return true; + } dev = ggml_backend_get_device(backend); } if (!dev) { return false; } std::string lo_dev_name = to_lowercase(ggml_backend_dev_name(dev)); - return has_any_prefix(lo_dev_name, name, '|'); + return has_any_prefix(lo_dev_name, loname, '|'); } bool kcpp_backend_metal_supports_family(ggml_backend_t backend, int family) @@ -155,7 +162,7 @@ void kcpp_backend_cuda_ggmlv3_set_tensor_split(const float * tensor_split) void kcpp_backend_cuda_ggmlv3_transform_tensor(void * data, struct ggml_v3_tensor * tensor) { #if defined(GGML_USE_CUDA) - ggml_v3_cuda_transform_tensor(tensor->data, tensor); + ggml_v3_cuda_transform_tensor(data, tensor); #endif } diff --git a/kcpp_backend.h b/kcpp_backend.h index 2db63e0ca..ee9932033 100644 --- a/kcpp_backend.h +++ b/kcpp_backend.h @@ -7,6 +7,9 @@ // backends with GGML_USE_CUDA #define KCPP_BACKENDS_USE_CUDA "cuda|rocm" +// Metal backend aliases +#define KCPP_BACKENDS_METAL "metal|mtl" + // backends that support tensor split #define KCPP_BACKENDS_TENSOR_SPLIT "cuda|rocm|vulkan" diff --git a/otherarch/llama_v3.cpp b/otherarch/llama_v3.cpp index 7e5403215..cd4625b88 100644 --- a/otherarch/llama_v3.cpp +++ b/otherarch/llama_v3.cpp @@ -1255,7 +1255,7 @@ static void llama_v3_model_load_internal( ctx_size + mmapped_size - vram_weights; // weights in VRAM not in memory - if (llama_v3_use_allocator()){ + if (llama_v3_use_scratch()){ mem_required += blasbatchmul*MEM_REQ_SCRATCH0_3(hparams.n_ctx).at(model.type) + blasbatchmul*MEM_REQ_SCRATCH1_3().at(model.type) + @@ -1815,15 +1815,15 @@ static bool llama_v3_eval_internal( const int64_t n_embd = hparams.n_embd; const int64_t n_vocab = hparams.n_vocab; -#ifdef LLAMA_V3_USE_ALLOCATOR - ggml_v3_allocr_reset(lctx.alloc); -#endif + if (llama_v3_use_allocator()) { + ggml_v3_allocr_reset(lctx.alloc); + } ggml_v3_cgraph * gf = llama_v3_build_graph(lctx, tokens, embd, n_tokens, n_past); -#ifdef LLAMA_V3_USE_ALLOCATOR - ggml_v3_allocr_alloc_graph(lctx.alloc, gf); -#endif + if (llama_v3_use_allocator()) { + ggml_v3_allocr_alloc_graph(lctx.alloc, gf); + } // LLAMA_V3_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_v3_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs); @@ -3377,8 +3377,7 @@ struct llama_v3_context * llama_v3_new_context_with_model( ctx->embedding.resize(hparams.n_embd); } -#ifdef LLAMA_V3_USE_ALLOCATOR - { + if (llama_v3_use_allocator()) { static const size_t tensor_alignment = 32; // the compute buffer is used to store the tensor and graph structs, while the allocator buffer is used for the tensor data ctx->buf_compute.resize(ggml_v3_tensor_overhead()*GGML_V3_MAX_NODES + ggml_v3_graph_overhead()); @@ -3410,10 +3409,9 @@ struct llama_v3_context * llama_v3_new_context_with_model( ctx->buf_alloc.resize(alloc_size); ctx->alloc = ggml_v3_allocr_new(ctx->buf_alloc.addr, ctx->buf_alloc.size, tensor_alignment); + } else { + ctx->buf_compute.resize(blasbatchmul*MEM_REQ_EVAL_3().at(ctx->model.type) + ggml_v3_graph_overhead()); } -#else - ctx->buf_compute.resize(blasbatchmul*MEM_REQ_EVAL_3().at(ctx->model.type) + ggml_v3_graph_overhead()); -#endif if (llama_v3_use_scratch()) { ctx->buf_scratch[0].resize(blasbatchmul*MEM_REQ_SCRATCH0_3(hparams.n_ctx).at(ctx->model.type)); diff --git a/otherarch/whispercpp/whisper.cpp b/otherarch/whispercpp/whisper.cpp index 30e58e3f3..fb335542d 100644 --- a/otherarch/whispercpp/whisper.cpp +++ b/otherarch/whispercpp/whisper.cpp @@ -1070,7 +1070,7 @@ static uint32_t whisper_kv_cache_get_padding(const struct whisper_context & wctx return 1u; } - if (kcpp_backend_check("mtl", wctx.backend)) { + if (kcpp_backend_check(KCPP_BACKENDS_METAL, wctx.backend)) { return 32u; } @@ -1225,7 +1225,7 @@ static ggml_backend_t whisper_backend_init(const whisper_context_params & params if (!backend_gpu) { WHISPER_LOG_ERROR("%s: ggml_backend_dev_init() failed\n", __func__); } else { - if (kcpp_backend_check("mtl", backend_gpu) && !kcpp_backend_metal_supports_family(backend_gpu, 7)) { + if (kcpp_backend_check(KCPP_BACKENDS_METAL, backend_gpu) && !kcpp_backend_metal_supports_family(backend_gpu, 7)) { WHISPER_LOG_ERROR("%s: Metal GPU does not support family 7 - falling back to CPU\n", __func__); ggml_backend_free(backend_gpu); backend_gpu = NULL; @@ -1861,7 +1861,7 @@ static struct ggml_cgraph * whisper_build_graph_encoder( whisper_context & wctx, whisper_state & wstate) { - auto ggml_mul_mat = kcpp_backend_check("mtl", wctx.backend) + auto ggml_mul_mat = kcpp_backend_check(KCPP_BACKENDS_METAL, wctx.backend) ? ggml_mul_mat_pad : ggml_mul_mat_original; @@ -2111,7 +2111,7 @@ static struct ggml_cgraph * whisper_build_graph_cross( whisper_context & wctx, whisper_state & wstate) { - auto ggml_mul_mat = kcpp_backend_check("mtl", wctx.backend) + auto ggml_mul_mat = kcpp_backend_check(KCPP_BACKENDS_METAL, wctx.backend) ? ggml_mul_mat_pad : ggml_mul_mat_original; @@ -2303,7 +2303,7 @@ static struct ggml_cgraph * whisper_build_graph_decoder( bool save_alignment_heads_QKs, bool worst_case) { - auto ggml_mul_mat = kcpp_backend_check("mtl", wctx.backend) + auto ggml_mul_mat = kcpp_backend_check(KCPP_BACKENDS_METAL, wctx.backend) ? ggml_mul_mat_pad : ggml_mul_mat_original; @@ -6565,7 +6565,7 @@ WHISPER_API int whisper_bench_ggml_mul_mat(int n_threads) { WHISPER_API const char * whisper_bench_ggml_mul_mat_str(int n_threads) { - auto ggml_mul_mat = kcpp_backend_check("mtl") + auto ggml_mul_mat = kcpp_backend_check(KCPP_BACKENDS_METAL) ? ggml_mul_mat_pad : ggml_mul_mat_original;