detokenize add special token ids

2026-09-19 17:25:07 +02:00 · 2026-05-17 17:04:47 +08:00
parent c1514e328b
commit de2c5f1cef
3 changed files with 20 additions and 7 deletions
@@ -158,6 +158,12 @@ struct token_count_outputs
    int count = 0;
    int * ids; //we'll just use shared memory for this one, bit of a hack
 };
+struct detokenize_inputs
+{
+    int count = 0;
+    int * ids; //we'll just use shared memory for this one, bit of a hack
+    bool special = false;
+};

 struct logprob_item {
    int option_count;