llama : add pre-tokenizer regexes for BLOOM and gpt3-finnish (#8850)

2026-09-20 01:31:42 +02:00 · 2024-08-15 10:17:12 +03:00
parent d5492f0525
commit 6bda7ce6c3
5 changed files with 19 additions and 1 deletions
@@ -410,6 +410,8 @@ struct llm_tokenizer_bpe {
                };
                break;
            case LLAMA_VOCAB_PRE_TYPE_PORO:
+            case LLAMA_VOCAB_PRE_TYPE_BLOOM:
+            case LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH:
                regex_exprs = {
                    " ?[^(\\s|.,!?…。，、।۔،)]+",
                };