Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 01:05:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

wip integration of llava

Browse Source
This commit is contained in:
Concedo
2024-03-10 11:18:47 +08:00
parent ca19199bc8
commit c08d7e5042
9 changed files with 137 additions and 8003 deletions
Download Patch File Download Diff File Expand all files Collapse all files
koboldcpp.py
+11 -3
View File
@@ -19,6 +19,7 @@ stop_token_max = 16
ban_token_max = 16
tensor_split_max = 16
logit_bias_max = 16
images_max = 4
bias_min_value = -100.0
bias_max_value = 100.0
@@ -61,6 +62,7 @@ class generation_inputs(ctypes.Structure):
_fields_ = [("seed", ctypes.c_int),
("prompt", ctypes.c_char_p),
("memory", ctypes.c_char_p),
("images", ctypes.c_char_p * images_max),
("max_context_length", ctypes.c_int),
("max_length", ctypes.c_int),
("temperature", ctypes.c_float),
@@ -380,11 +382,16 @@ def load_model(model_filename):
ret = handle.load_model(inputs)
return ret
def generate(prompt, memory="", max_length=32, max_context_length=512, temperature=0.7, top_k=100, top_a=0.0, top_p=0.92, min_p=0.0, typical_p=1.0, tfs=1.0, rep_pen=1.0, rep_pen_range=128, presence_penalty=0.0, mirostat=0, mirostat_tau=5.0, mirostat_eta=0.1, sampler_order=[6,0,1,3,4,2,5], seed=-1, stop_sequence=[], use_default_badwordsids=False, stream_sse=False, grammar='', grammar_retain_state=False, genkey='', trimstop=False, quiet=False, dynatemp_range=0.0, dynatemp_exponent=1.0, smoothing_factor=0.0, logit_biases={}):
def generate(prompt, memory="", images=[], max_length=32, max_context_length=512, temperature=0.7, top_k=100, top_a=0.0, top_p=0.92, min_p=0.0, typical_p=1.0, tfs=1.0, rep_pen=1.0, rep_pen_range=128, presence_penalty=0.0, mirostat=0, mirostat_tau=5.0, mirostat_eta=0.1, sampler_order=[6,0,1,3,4,2,5], seed=-1, stop_sequence=[], use_default_badwordsids=False, stream_sse=False, grammar='', grammar_retain_state=False, genkey='', trimstop=False, quiet=False, dynatemp_range=0.0, dynatemp_exponent=1.0, smoothing_factor=0.0, logit_biases={}):
global maxctx, args, currentusergenkey, totalgens, pendingabortkey
inputs = generation_inputs()
inputs.prompt = prompt.encode("UTF-8")
inputs.memory = memory.encode("UTF-8")
for n in range(images_max):
if not images or n >= len(images):
inputs.images[n] = "".encode("UTF-8")
else:
inputs.images[n] = images[n].encode("UTF-8")
if max_length >= (max_context_length-1):
max_length = max_context_length-1
print("\nWarning: You are trying to generate with max_length near or exceeding max_context_length. Most of the context will be removed, and your outputs will not be very coherent.")
@@ -695,6 +702,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
return generate(
prompt=genparams.get('prompt', ""),
memory=genparams.get('memory', ""),
images=genparams.get('images', []),
max_context_length=genparams.get('max_context_length', maxctx),
max_length=genparams.get('max_length', 100),
temperature=genparams.get('temperature', 0.7),
@@ -1407,7 +1415,7 @@ def show_new_gui():
nocertifymode = ctk.IntVar(value=0)
lowvram_var = ctk.IntVar()
mmq_var = ctk.IntVar(value=1)
mmq_var = ctk.IntVar(value=0)
blas_threads_var = ctk.StringVar()
blas_size_var = ctk.IntVar()
version_var = ctk.StringVar(value="0")
@@ -2808,7 +2816,7 @@ def main(launch_args,start_server=True):
benchprompt = "11111111"
for i in range(0,10): #generate massive prompt
benchprompt += benchprompt
result = generate(benchprompt,memory="",max_length=benchlen,max_context_length=benchmaxctx,temperature=0.1,top_k=1,rep_pen=1,use_default_badwordsids=True)
result = generate(benchprompt,memory="",images=[],max_length=benchlen,max_context_length=benchmaxctx,temperature=0.1,top_k=1,rep_pen=1,use_default_badwordsids=True)
result = (result[:5] if len(result)>5 else "")
resultok = (result=="11111")
t_pp = float(handle.get_last_process_time())*float(benchmaxctx-benchlen)*0.001