Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-03 11:35:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

big wip

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-21 22:22:14 +02:00
parent d1a9e5a8e6
commit ea4b4b2862
25 changed files with 265 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/peg-parser.cpp
+16 -13
View File
@@ -1,4 +1,6 @@
#include "peg-parser.h"
// the interface takes common_json, the parser internals stay on the library
#include "json-shim.h"
#include "common.h"
#include "json-schema-to-grammar.h"
@@ -1120,8 +1122,8 @@ common_peg_parser common_peg_parser_builder::chars(const std::string & classes,
return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max}));
}
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema, bool raw) {
return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared<nlohmann::ordered_json>(schema), raw}));
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw) {
return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared<common_json>(schema), raw}));
}
common_peg_parser common_peg_parser_builder::rule(const std::string & name, const common_peg_parser & p, bool trigger) {
@@ -1805,8 +1807,8 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo
}
}
static nlohmann::json serialize_parser_variant(const common_peg_parser_variant & variant) {
using json = nlohmann::json;
static nlohmann::ordered_json serialize_parser_variant(const common_peg_parser_variant & variant) {
using json = nlohmann::ordered_json;
return std::visit([](const auto & p) -> json {
using T = std::decay_t<decltype(p)>;
@@ -1860,7 +1862,7 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant &
{"type", "schema"},
{"child", p.child},
{"name", p.name},
{"schema", p.schema ? *p.schema : nullptr},
{"schema", p.schema ? common_json_raw<nlohmann::ordered_json>(*p.schema) : nlohmann::ordered_json(nullptr)},
{"raw", p.raw}
};
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
@@ -1888,19 +1890,19 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant &
}, variant);
}
nlohmann::json common_peg_arena::to_json() const {
auto parsers = nlohmann::json::array();
common_json common_peg_arena::to_json() const {
auto parsers = nlohmann::ordered_json::array();
for (const auto & parser : parsers_) {
parsers.push_back(serialize_parser_variant(parser));
}
return nlohmann::json{
return common_json_from_raw(nlohmann::ordered_json{
{"parsers", parsers},
{"rules", rules_},
{"root", root_}
};
});
}
static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json & j) {
static common_peg_parser_variant deserialize_parser_variant(const nlohmann::ordered_json & j) {
if (!j.contains("type") || !j["type"].is_string()) {
throw std::runtime_error("Parser variant JSON missing or invalid 'type' field");
}
@@ -2007,7 +2009,7 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json
parser.child = j["child"].get<common_peg_parser_id>();
parser.name = j["name"];
if (!j["schema"].is_null()) {
parser.schema = std::make_shared<nlohmann::ordered_json>(j["schema"]);
parser.schema = std::make_shared<common_json>(common_json_from_raw(j["schema"]));
}
parser.raw = j["raw"].get<bool>();
return parser;
@@ -2069,7 +2071,8 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json
throw std::runtime_error("Unknown parser type: " + type);
}
common_peg_arena common_peg_arena::from_json(const nlohmann::json & j) {
common_peg_arena common_peg_arena::from_json(const common_json & j_in) {
const nlohmann::ordered_json & j = common_json_raw<nlohmann::ordered_json>(j_in);
if (!j.contains("parsers") || !j["parsers"].is_array()) {
throw std::runtime_error("JSON missing or invalid 'parsers' array");
}
@@ -2109,7 +2112,7 @@ std::string common_peg_arena::save() const {
}
void common_peg_arena::load(const std::string & data) {
*this = from_json(nlohmann::json::parse(data));
*this = from_json(common_json::parse(data));
}
common_peg_arena build_peg_parser(const std::function<common_peg_parser(common_peg_parser_builder & builder)> & fn) {