Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-12 16:18:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

handle override jinja template

Browse Source
This commit is contained in:
Concedo
2026-04-19 00:30:28 +08:00
parent ff37b336a7
commit e5eab545f3
5 changed files with 50 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
koboldcpp.py
+29 -3
View File
@@ -157,6 +157,7 @@ embedded_kcpp_sdui_gz = None
embedded_lcpp_ui_gz = None
embedded_musicui = None
embedded_musicui_gz = None
preloaded_custom_jinja = ""
voicebank = {}
voicelist = ["kobo","cheery","sleepy","shouty","chatty"]
sslvalid = False
@@ -259,6 +260,7 @@ class load_model_inputs(ctypes.Structure):
("quant_k", ctypes.c_int),
("quant_v", ctypes.c_int),
("check_slowness", ctypes.c_bool),
("jinja_template", ctypes.c_char_p),
("highpriority", ctypes.c_bool),
("swa_support", ctypes.c_bool),
("swa_padding", ctypes.c_int),
@@ -1955,6 +1957,7 @@ def load_model(model_filename):
inputs.override_tensors = args.overridetensors.encode("UTF-8") if args.overridetensors else "".encode("UTF-8")
inputs.moecpu = (200 if args.moecpu > 200 else args.moecpu)
inputs.check_slowness = (not args.highpriority and os.name == 'nt' and 'Intel' in platform.processor())
inputs.jinja_template = preloaded_custom_jinja.encode("UTF-8")
inputs.highpriority = args.highpriority
inputs.swa_support = args.useswa
inputs.swa_padding = args.swapadding if args.useswa else 0
@@ -7287,6 +7290,7 @@ def show_gui():
customrope_base = ctk.StringVar(value="10000")
customrope_nativectx = ctk.StringVar(value=str(default_native_ctx))
chatcompletionsadapter_var = ctk.StringVar(value="AutoGuess")
jinjatemplate_var = ctk.StringVar()
jinja_var = ctk.IntVar(value=0)
jinja_tools_var = ctk.IntVar(value=0)
jinja_kwargs_var = ctk.StringVar()
@@ -8091,10 +8095,11 @@ def show_gui():
if fnam:
chatcompletionsadapter_var.set(fnam)
ctk.CTkButton(model_tab, 64, text="Pick Premade", command=pickpremadetemplate).grid(row=24, column=0, padx=(350), pady=2, stick="nw")
makefileentry(model_tab, "Jinja Template:", "Select a custom Jinja chat template", jinjatemplate_var, 30, width=280, filetypes=[("Jinja Template", "*.jinja")], singlerow=True, tooltiptxt="Select a custom Jinja chat template, will overwrite model jinja chat template")
mmproj_var.trace_add("write", gui_changed_modelfile)
draftmodel_var.trace_add("write", gui_changed_modelfile)
makefileentry(model_tab, "Download Dir:", "Select directory to store all model downloads", download_dir_var, 27, width=280, singlerow=True, dialog_type=2, tooltiptxt="Specify a directory to store any downloaded models.")
makefileentry(model_tab, "Download Dir:", "Select directory to store all model downloads", download_dir_var, 35, width=280, singlerow=True, dialog_type=2, tooltiptxt="Specify a directory to store any downloaded models.")
makecheckbox(model_tab, "Allow Launch Without Models", nomodel, 40, tooltiptxt="Allows running the WebUI with no model loaded.")
# Network Tab
@@ -8431,6 +8436,8 @@ def show_gui():
args.jinja_tools = (jinja_tools_var.get()==1)
if jinja_kwargs_var.get() != "":
args.jinja_kwargs = jinja_kwargs_var.get()
if jinjatemplate_var.get() != "":
args.jinjatemplate = jinjatemplate_var.get()
args.enableguidance = (enableguidance_var.get()==1)
args.overridekv = None if override_kv_var.get() == "" else override_kv_var.get()
args.overridetensors = None if override_tensors_var.get() == "" else override_tensors_var.get()
@@ -8719,6 +8726,7 @@ def show_gui():
if isinstance(jinja_kwargs, type({})):
jinja_kwargs = json.dumps(jinja_kwargs)
jinja_kwargs_var.set(jinja_kwargs)
jinjatemplate_var.set(mydict["jinjatemplate"] if ("jinjatemplate" in mydict and mydict["jinjatemplate"]) else "")
enableguidance_var.set(mydict["enableguidance"] if ("enableguidance" in mydict) else 0)
if "overridekv" in mydict and mydict["overridekv"]:
@@ -10177,7 +10185,7 @@ def disableSwappedFieldsInConfig(args, swapReqType):
def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
global embedded_kailite, embedded_kcpp_docs, embedded_kcpp_sdui, embedded_kailite_gz, embedded_kcpp_docs_gz, embedded_kcpp_sdui_gz, embedded_lcpp_ui_gz, embedded_musicui, embedded_musicui_gz, start_time, exitcounter, global_memory, using_gui_launcher
global libname, args, friendlymodelname, friendlysdmodelname, fullsdmodelpath, password, fullwhispermodelpath, ttsmodelpath, embeddingsmodelpath, musicdiffusionmodelpath, musicllmmodelpath, friendlyembeddingsmodelname, has_audio_support, has_vision_support, cached_chat_template
global libname, args, friendlymodelname, friendlysdmodelname, fullsdmodelpath, password, fullwhispermodelpath, ttsmodelpath, embeddingsmodelpath, musicdiffusionmodelpath, musicllmmodelpath, friendlyembeddingsmodelname, has_audio_support, has_vision_support, cached_chat_template, preloaded_custom_jinja
start_server = True
@@ -10408,6 +10416,23 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
dlfile = download_model_from_url(args.musicvae,[".gguf"],min_file_size=500000)
if dlfile:
args.musicvae = dlfile
if args.mcpfile and args.mcpfile!="":
dlfile = download_model_from_url(args.mcpfile,[".json"],min_file_size=64)
if dlfile:
args.mcpfile = dlfile
if args.jinjatemplate and args.jinjatemplate!="":
dlfile = download_model_from_url(args.jinjatemplate,[".jinja"],min_file_size=64)
if dlfile:
args.jinjatemplate = dlfile
if args.jinjatemplate and os.path.exists(args.jinjatemplate):
try:
print(f"Using custom Jinja template: {args.jinjatemplate}")
with open(args.jinjatemplate, 'r', encoding='utf-8', errors='ignore') as f:
preloaded_custom_jinja = f.read()
except Exception as e:
print(f"Error loading jinja templat: {e}")
preloaded_custom_jinja = ""
# sanitize and replace the default vanity name. remember me....
if args.model_param and args.model_param!="":
@@ -11224,7 +11249,8 @@ if __name__ == '__main__':
advparser.add_argument("--chatcompletionsadapter", metavar=('[filename]'), help="Select an optional ChatCompletions Adapter JSON file to force custom instruct tags.", default="AutoGuess")
advparser.add_argument("--jinja", help="Enables using jinja chat template formatting for chat completions endpoint. Other endpoints are unaffected. Tool calls are done without jinja.", action='store_true')
advparser.add_argument("--jinja_tools","--jinja-tools","--jinjatools", help="Enables using jinja chat template formatting for chat completions endpoint. Other endpoints are unaffected. Tool calls are done with jinja.", action='store_true')
advparser.add_argument("--jinja_kwargs","--jinja-kwargs","--jinjakwargs","--chat-template-kwargs", metavar=('{"parameter":"value",...}'), help="Set additiona fields for Jinja JSON template parser, must be a valid JSON object.", default="")
advparser.add_argument("--jinja_kwargs","--jinja-kwargs","--jinjakwargs","--chat-template-kwargs", metavar=('{"parameter":"value",...}'), help="Set additional fields for Jinja JSON template parser, must be a valid JSON object.", default="")
advparser.add_argument("--jinjatemplate","--chat-template-file", metavar=('[filename]'), help="Select a custom Jinja chat template, will overwrite model jinja chat template", default="")
advparser.add_argument("--noflashattention","--no-flash-attn","-nofa", help="Disables flash attention.", action='store_true')
advparser.add_argument("--lowvram","-nkvo","--no-kv-offload", help="If supported by the backend, do not offload KV to GPU (lowvram mode). Not recommended, will be slow.", action='store_true')
advparser.add_argument("--quantkv", help="Sets the KV cache data type quantization, options are f16/bf16/q8_0/q5_1/q4_0. Requires Flash Attention for full effect, otherwise only K cache is quantized.",metavar=('[quantization level f16/bf16/q8_0/q5_1/q4_0]'), type=str, choices=["f16","bf16","q8_0","q5_1","q4_0","0","1","2","3"], default="f16")