Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 01:05:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Implemented basic GPU offloading for MPT, GPT-2, GPT-J and GPT-NeoX

Browse Source
This commit is contained in:
Concedo
2023-06-22 00:43:25 +08:00
parent b1f00fa9cc
commit 1b71752a9f
6 changed files with 99 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpttype_adapter.cpp
+2 -2
View File
@@ -671,7 +671,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
{
if(file_format==FileFormat::NEOX_6|| file_format==FileFormat::NEOX_7)
{
ModelLoadResult res = gpt_neox_model_load(params.model, neox_ctx_v3, vocab, file_format);
ModelLoadResult res = gpt_neox_model_load(params.model, neox_ctx_v3, vocab, file_format, inputs.gpulayers);
if(res==ModelLoadResult::FAIL)
{
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
@@ -733,7 +733,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
}
else if(file_format==FileFormat::MPT_1)
{
bool res = mpt_model_load(params.model, mpt_ctx_v3, vocab);
bool res = mpt_model_load(params.model, mpt_ctx_v3, vocab, inputs.gpulayers);
if(res==false)
{
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());