Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-06 13:01:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

auto fitting for draft models

Browse Source
This commit is contained in:
Concedo
2026-06-15 19:36:54 +08:00
parent 7b217c3d7c
commit be80f5dcbc
2 changed files with 191 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
koboldcpp.py
+2 -2
View File
@@ -1705,8 +1705,8 @@ def autoset_gpu_layers(ctxsize, sdquanted, bbs, musiclowvram): #shitty algo to d
calulated_gpu_overhead += max(350*1024*1024,modelfile_extracted_meta[4]*1.5)
if modelfile_extracted_meta[5] > 1024*1024*10: #mmproj tax (now internal to kcpp)
unsubmitted_overhead += max(350*1024*1024,modelfile_extracted_meta[5]*1.5)
if modelfile_extracted_meta[6] > 1024*1024*10: #draft model tax
calulated_gpu_overhead += (modelfile_extracted_meta[6] * 1.5)
if modelfile_extracted_meta[6] > 1024*1024*10: #draft model tax (now internal to kcpp)
unsubmitted_overhead += (modelfile_extracted_meta[6] * 1.6) + (150*1024*1024)
if modelfile_extracted_meta[7] > 1024*1024*10: #tts model tax
if modelfile_extracted_meta[7] < 1024*1024*1024: #less than 1gb probably means outetts, which needs more vram
calulated_gpu_overhead += max(600*1024*1024, modelfile_extracted_meta[7] * 3)