added a token counting endpoint, set mmq as default

2026-09-18 16:55:14 +02:00 · 2023-08-24 20:41:49 +08:00
parent 81a0ef342c
commit b95a4ccb22
5 changed files with 72 additions and 28 deletions
@@ -62,6 +62,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
 generation_outputs gpttype_generate(const generation_inputs inputs, generation_outputs &output);
 bool gpttype_generate_abort();
 const std::string & gpttype_get_pending_output();
+int gpttype_token_count(const std::string & input);

 void timer_start();
 double timer_check();