Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-10 23:09:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

You'll never take us alive

Browse Source 
We swore that death will do us part
They'll call our crimes a work of art
This commit is contained in:
Concedo
2025-01-09 11:27:06 +08:00
parent dcfa1eca4e 53ff6b9b9f
commit e788b8289a
19 changed files with 1866 additions and 1704 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-model-loader.cpp
+8 -5
View File
@@ -22,7 +22,7 @@ const char * llama_file_version_name(llama_fver version) {
}
namespace GGUFMeta {
template <typename T, gguf_type gt_, T (*gfun)(const gguf_context *, const int)>
template <typename T, gguf_type gt_, T (*gfun)(const gguf_context *, const int64_t)>
struct GKV_Base_Type {
static constexpr gguf_type gt = gt_;
@@ -64,10 +64,11 @@ namespace GGUFMeta {
public:
static constexpr gguf_type gt = GGUF_TYPE_ARRAY;
static ArrayInfo getter(const gguf_context *ctx, const int k) {
const enum gguf_type arr_type = gguf_get_arr_type(ctx, k);
return ArrayInfo {
gguf_get_arr_type(ctx, k),
arr_type,
size_t(gguf_get_arr_n(ctx, k)),
gguf_get_arr_data(ctx, k),
arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx, k),
};
}
};
@@ -558,7 +559,7 @@ llama_model_loader::llama_model_loader(const std::string & fname, bool use_mmap,
const enum gguf_type type = gguf_get_kv_type(meta.get(), i);
const std::string type_name =
type == GGUF_TYPE_ARRAY
? format("%s[%s,%d]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(meta.get(), i)), gguf_get_arr_n(meta.get(), i))
? format("%s[%s,%zu]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(meta.get(), i)), gguf_get_arr_n(meta.get(), i))
: gguf_type_name(type);
std::string value = gguf_kv_to_str(meta.get(), i);
@@ -988,8 +989,10 @@ bool llama_model_loader::load_all_data(
bool shouldoffload = (layernum>=0 && clblast_offload_fallback_layers>layernum);
if(shouldoffload)
{
cur->backend = GGML_BACKEND_TYPE_GPU;
cur->clblast_offload_gpu = true;
ggml_cl_transform_tensor(cur->data, cur);
}else{
cur->clblast_offload_gpu = false;
}
#endif