wip 2

common/chat.cpp

@@ -21,6 +21,7 @@
 #include <ctime>
 #include <exception>
 #include <functional>
+#include <iomanip>
 #include <map>
 
 #include <optional>

common/json-schema-to-grammar.cpp

@@ -947,7 +947,7 @@ public:
         }
         if ((schema_type.is_null() || schema_type == "object")
                 && (schema.contains("properties") ||
-                    (schema.contains("additionalProperties") && !schema["additionalProperties"].get<bool>()))) {
+                    (schema.contains("additionalProperties") && schema["additionalProperties"] != true))) {
             std::unordered_set<std::string> required;
             if (schema.contains("required") && schema["required"].is_array()) {
                 for (const auto & item : schema["required"]) {

common/json.cpp

@@ -98,6 +98,7 @@ common_json_value::common_json_value(const std::vector<T> & vals) : type(VAL_JSO
 #define COMMON_JSON_VEC(...) template common_json_value::common_json_value(const std::vector<__VA_ARGS__> &);
 
 COMMON_JSON_VEC(int)
+COMMON_JSON_VEC(unsigned char)
 COMMON_JSON_VEC(unsigned int)
 COMMON_JSON_VEC(long)
 COMMON_JSON_VEC(unsigned long)
@@ -106,6 +107,7 @@ COMMON_JSON_VEC(unsigned long long)
 COMMON_JSON_VEC(float)
 COMMON_JSON_VEC(double)
 COMMON_JSON_VEC(std::string)
+COMMON_JSON_VEC(std::vector<float>)
 COMMON_JSON_VEC(common_json)
 
 #undef COMMON_JSON_VEC
@@ -113,8 +115,10 @@ COMMON_JSON_VEC(common_json)
 common_json_value::common_json_value(std::initializer_list<common_json_item> items) :
     type(VAL_JSON), val_json(std::make_shared<common_json>(items)) {}
 
+// null, same as the backing library. operator[] turns it into an object,
+// push_back() into an array
 common_json::common_json() {
-    new (storage) ordered_json(ordered_json::object());
+    new (storage) ordered_json();
 }
 
 common_json::common_json(const common_json & other) {
@@ -125,7 +129,9 @@ common_json::common_json(common_json && other) noexcept {
     new (storage) ordered_json(std::move(as_json(&other)));
 }
 
-common_json::common_json(std::initializer_list<common_json_item> items) : common_json() {
+common_json::common_json(std::initializer_list<common_json_item> items) {
+    new (storage) ordered_json(ordered_json::object());
+
     for (const auto & item : items) {
         set(item);
     }
@@ -139,14 +145,8 @@ common_json::common_json(std::nullptr_t) {
     new (storage) ordered_json(nullptr);
 }
 
-common_json & common_json::operator=(const common_json & other) {
-    as_json(this) = as_json(&other);
-
-    return *this;
-}
-
-common_json & common_json::operator=(common_json && other) noexcept {
-    as_json(this) = std::move(as_json(&other));
+common_json & common_json::operator=(common_json other) noexcept {
+    as_json(this).swap(as_json(&other));
 
     return *this;
 }
@@ -186,7 +186,7 @@ common_json common_json::array(std::initializer_list<common_json_value> vals) {
 }
 
 common_json common_json::object() {
-    return common_json();
+    return common_json_from_raw(ordered_json::object());
 }
 
 common_json common_json::object(std::initializer_list<common_json_item> items) {
@@ -319,6 +319,11 @@ template <typename T> T common_json::get() const {
     return as_json(this).get<T>();
 }
 
+// the backing library cannot build a common_json, so this one is just a copy
+template <> common_json common_json::get<common_json>() const {
+    return *this;
+}
+
 // get<T>() is usable only for the types below
 
 #define COMMON_JSON_GET(...) template __VA_ARGS__ common_json::get<__VA_ARGS__>() const;
@@ -333,6 +338,7 @@ COMMON_JSON_GET(unsigned long long)
 COMMON_JSON_GET(float)
 COMMON_JSON_GET(double)
 COMMON_JSON_GET(std::string)
+COMMON_JSON_GET(std::vector<float>)
 COMMON_JSON_GET(std::vector<std::string>)
 COMMON_JSON_GET(std::set<std::string>)
 COMMON_JSON_GET(std::vector<int>)

common/json.h

@@ -12,6 +12,7 @@
 #include <memory>
 #include <stdexcept>
 #include <string>
+#include <string_view>
 #include <type_traits>
 #include <utility>
 #include <vector>
@@ -52,6 +53,9 @@ struct common_json_value {
     common_json_value(std::nullptr_t = nullptr) : type(VAL_NULL) {}
     common_json_value(bool val) : type(VAL_BOOL), val_bool(val) {}
     common_json_value(std::string val) : type(VAL_STRING), val_string(std::move(val)) {}
+    // a string_view does not convert to std::string on its own, and without this
+    // it would land on the common_json ctor below and recurse
+    common_json_value(std::string_view val) : type(VAL_STRING), val_string(val) {}
     common_json_value(const char * val);
     common_json_value(const common_json & val);
     // only for the types instantiated in json.cpp, the rest fails at link time
@@ -102,8 +106,9 @@ class common_json {
                                                   !std::is_same<typename std::decay<T>::type, common_json_value>::value, int>::type = 0>
     common_json(T && val) : common_json(common_json_value(std::forward<T>(val))) {}
 
-    common_json & operator=(const common_json & other);
-    common_json & operator=(common_json && other) noexcept;
+    // by value, same as the backing library: the right side is copied before the
+    // left side can invalidate it, e.g. msg["a"] = msg.at("b") where "a" is new
+    common_json & operator=(common_json other) noexcept;
 
     ~common_json();
 

tests/peg-parser/test-json-serialization.cpp

@@ -8,7 +8,7 @@ void test_json_serialization(testing &t) {
     auto json_serialized = original.to_json().dump();
 
     t.test("compare before/after", [&](testing &t) {
-        auto deserialized = common_peg_arena::from_json(nlohmann::json::parse(json_serialized));
+        auto deserialized = common_peg_arena::from_json(common_json::parse(json_serialized));
 
         // Test complex JSON
         std::string input = R"({"name": "test", "values": [1, 2, 3], "nested": {"a": true}})";
@@ -23,6 +23,6 @@ void test_json_serialization(testing &t) {
     });
 
     t.bench("deserialize", [&]() {
-        auto deserialized = common_peg_arena::from_json(nlohmann::json::parse(json_serialized));
+        auto deserialized = common_peg_arena::from_json(common_json::parse(json_serialized));
     }, 100);
 }

tests/peg-parser/tests.h

@@ -1,7 +1,7 @@
 #pragma once
 
 // Common includes for all test files
-#include <nlohmann/json.hpp>
+#include "json.h"
 #include <string>
 #include <vector>
 
@@ -11,9 +11,9 @@
 #include "simple-tokenize.h"
 
 struct bench_tool_call {
-    std::string            id;
-    std::string            name;
-    nlohmann::ordered_json args;
+    std::string id;
+    std::string name;
+    common_json args;
 };
 
 // Test function declarations

tests/test-chat-peg-parser.cpp

@@ -63,10 +63,10 @@ static json create_tools() {
                           { { "type", "string" }, { "description", "The city and state, e.g. San Francisco, CA" } } },
                         { "unit",
                           { { "type", "string" },
-                            { "enum", { "celsius", "fahrenheit" } },
+                            { "enum", json::array({ "celsius", "fahrenheit" }) },
                             { "description",
                               "The temperature unit to use. Infer this from the users location." } } } } },
-                    { "required", { "location", "unit" } },
+                    { "required", json::array({ "location", "unit" }) },
                 } },
           }                      }
     };
@@ -86,14 +86,14 @@ static json create_tools() {
                           { { "type", "string" }, { "description", "The city and state, e.g. San Francisco, CA" } } },
                         { "unit",
                           { { "type", "string" },
-                            { "enum", { "celsius", "fahrenheit" } },
+                            { "enum", json::array({ "celsius", "fahrenheit" }) },
                             { "description", "The temperature unit to use. Infer this from the users location." } } },
                         { "days",
                           { { "type", "integer" },
                             { "description", "Number of days to forecast (1-10)" },
                             { "minimum", 1 },
                             { "maximum", 10 } } } } },
-                    { "required", { "location", "unit" } },
+                    { "required", json::array({ "location", "unit" }) },
                 } },
           }                      }
     };
@@ -341,7 +341,7 @@ static void test_example_native(testing & t) {
                 { { "invoice_number", { { "type", "string" } } },
                   { "amount", { { "type", "number" } } },
                   { "due_date", { { "type", "string" } } } } },
-              { "required", { "invoice_number", "amount", "due_date" } } },
+              { "required", json::array({ "invoice_number", "amount", "due_date" }) } },
          /* .parallel_tool_calls =  */ false,
          /* .generation_prompt =    */ "<think>",
          /* .input =                */

tests/test-jinja.cpp

@@ -240,7 +240,7 @@ static void test_conditionals(testing & t) {
 
     test_template(t, "is undefined key falsy",
         "{{ 'yes' if not y['x'] else 'no' }}",
-        {{"y", {{}}}},
+        {{"y", json::array({nullptr})}},
         "yes"
     );
 
@@ -282,7 +282,7 @@ static void test_conditionals(testing & t) {
 
     test_template(t, "is non-empty object truthy",
         "{{ 'yes' if y else 'no' }}",
-        {{"y", {"x", false}}},
+        {{"y", json::array({"x", false})}},
         "yes"
     );
 

tests/test-model-resolution.cpp

@@ -55,7 +55,7 @@ static const char * COMMIT = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
 static void serve_repos(httplib::Server & server) {
     server.Get(R"(/api/models/(.+)/refs)", [](const httplib::Request & req, httplib::Response & res) {
         if (g_repos.count(req.matches[1])) {
-            res.set_content(common_json{{"branches", {{{"name", "main"}, {"targetCommit", COMMIT}}}}}.dump(),
+            res.set_content(common_json{{"branches", common_json::array({ common_json{{"name", "main"}, {"targetCommit", COMMIT}} })}}.dump(),
                             "application/json");
         } else {
             res.status = 404;