Merge branch 'master' into cisc/test-tokenizers-remote

tests/CMakeLists.txt

@@ -92,7 +92,6 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-gpt-2             ARGS ${CMAKE
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-bpe         ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama-bpe.gguf)
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-spm         ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama-spm.gguf)
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-mpt               ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-mpt.gguf)
-llama_test(test-tokenizer-0 NAME test-tokenizer-0-nomic-bert-moe    ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-nomic-bert-moe.gguf)
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-phi-3             ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-phi-3.gguf)
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-qwen2             ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-qwen2.gguf)
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-refact            ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-refact.gguf)

tests/test-chat-parser.cpp

@@ -7,7 +7,6 @@
 //
 #include <exception>
 #include <iostream>
-#include <json.hpp>
 #include <string>
 
 #include "chat-parser.h"
@@ -15,8 +14,6 @@
 #include "log.h"
 #include "regex-partial.h"
 
-using json = nlohmann::ordered_json;
-
 template <class T>
 static void assert_equals(const T & expected, const T & actual) {
     if (expected != actual) {

tests/test-chat.cpp

@@ -5,16 +5,17 @@
 //
 //    cmake -B build && cmake --build build --parallel && ./build/bin/test-chat ../minja/build/tests/*.jinja 2>/dev/null
 //
-#include <fstream>
-#include <iostream>
-#include <json.hpp>
-#include <string>
-
 #include "chat.h"
 
 #include "../src/unicode.h"
 #include "../src/llama-grammar.h"
 
+#include <nlohmann/json.hpp>
+
+#include <fstream>
+#include <iostream>
+#include <string>
+
 using json = nlohmann::ordered_json;
 
 static std::ostream & operator<<(std::ostream & os, const common_chat_msg_diff & diff) {
@@ -1040,6 +1041,15 @@ static void test_template_output_parsers() {
                       "<tool_call>\n"
                       "{\"name\": \"python\", \"arguments\": {\"code\":\"# This is a program:\\nprint('hey')\"}}\n"
                       "</tool_call>");
+        assert_msg_equals(
+            simple_assist_msg("", /* reasoning_content= */ "<tool_call>nah uhg</tool_call>"),
+            common_chat_parse(
+                "<think><tool_call>nah uhg</tool_call>",
+                /* is_partial= */ false,
+                {
+                    /* .format = */ COMMON_CHAT_FORMAT_HERMES_2_PRO,
+                    /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
+                }));
     }
     {
         auto tmpls = read_templates("models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja");

tests/test-grammar-integration.cpp

@@ -7,6 +7,8 @@
 #include "../src/unicode.h"
 #include "../src/llama-grammar.h"
 
+#include <nlohmann/json.hpp>
+
 #include <cassert>
 #include <string>
 #include <vector>

tests/test-json-schema-to-grammar.cpp

@@ -6,6 +6,8 @@
 
 #include "../src/llama-grammar.h"
 
+#include <nlohmann/json.hpp>
+
 #include <cassert>
 #include <fstream>
 #include <sstream>