From 63bea7a7017644cfaeeddc7da537508c16532d77 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Wed, 18 Sep 2024 23:06:00 +0200 Subject: [PATCH] fromYAML: rebase --- package.nix | 2 ++ src/libexpr/json-to-value-sax.hh | 17 ++++++++++++++++ src/libexpr/json-to-value.cc | 11 +++++++---- src/libexpr/meson.build | 13 +++++++++++++ src/libexpr/primops/fromYAML.cc | 25 +++++++++++++----------- src/libexpr/primops/meson.build | 1 + tests/unit/libexpr/meson.build | 1 + tests/unit/libexpr/yaml.cc | 33 ++++++++++++++++---------------- 8 files changed, 72 insertions(+), 31 deletions(-) create mode 100644 src/libexpr/json-to-value-sax.hh diff --git a/package.nix b/package.nix index 8ab184667..b8d86e73a 100644 --- a/package.nix +++ b/package.nix @@ -31,6 +31,7 @@ , openssl , pkg-config , rapidcheck +, rapidyaml , sqlite , toml11 , unixtools @@ -224,6 +225,7 @@ in { libgit2 libsodium openssl + rapidyaml sqlite toml11 xz diff --git a/src/libexpr/json-to-value-sax.hh b/src/libexpr/json-to-value-sax.hh new file mode 100644 index 000000000..c994a82fb --- /dev/null +++ b/src/libexpr/json-to-value-sax.hh @@ -0,0 +1,17 @@ +#pragma once +///@file + +#include +#include + +#include "json-to-value.hh" + +/** + * json_sax and unique_ptr require the inclusion of json.hpp, so this header shall not be included by other headers + **/ + +namespace nix { + +std::unique_ptr> makeJSONSaxParser(EvalState & s, Value & v); + +} diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 9ac56541a..9f552a218 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -1,10 +1,8 @@ -#include "json-to-value.hh" +#include "json-to-value-sax.hh" #include "value.hh" #include "eval.hh" #include -#include -#include using json = nlohmann::json; @@ -12,7 +10,7 @@ namespace nix { // for more information, refer to // https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp -class JSONSax : nlohmann::json_sax { +class JSONSax : public nlohmann::json_sax { class JSONState { protected: std::unique_ptr parent; @@ -80,6 +78,7 @@ class JSONSax : nlohmann::json_sax { public: JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {}; + virtual ~JSONSax() = default; bool null() override { @@ -177,4 +176,8 @@ void parseJSON(EvalState & state, const std::string_view & s_, Value & v) throw JSONParseError("Invalid JSON Value"); } +std::unique_ptr> makeJSONSaxParser(EvalState & state, Value & v) { + return { std::make_unique(state, v) }; +} + } diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 4d8a38b43..8eace9c6d 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -56,6 +56,18 @@ if bdw_gc.found() endif configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int()) +ryml = dependency( + 'ryml', + version : '>=0.7.1', + method : 'cmake', + include_type : 'system', + required : false, +) +if ryml.found() + deps_other += ryml + configdata.set('HAVE_RYML', 1) +endif + toml11 = dependency( 'toml11', version : '>=3.7.0', @@ -172,6 +184,7 @@ headers = [config_h] + files( 'gc-small-vector.hh', 'get-drvs.hh', 'json-to-value.hh', + 'json-to-value-sax.hh', # internal: 'lexer-helpers.hh', 'nixexpr.hh', 'parser-state.hh', diff --git a/src/libexpr/primops/fromYAML.cc b/src/libexpr/primops/fromYAML.cc index c4c469646..c9aa59fa4 100644 --- a/src/libexpr/primops/fromYAML.cc +++ b/src/libexpr/primops/fromYAML.cc @@ -19,15 +19,15 @@ struct NixContext { static void s_error [[ noreturn ]] (const char* msg, size_t len, ryml::Location, void *nixContext) { auto context = static_cast(nixContext); - if (nixContext) { - throw EvalError({ - .msg = hintfmt("while parsing the YAML string '%1%':\n\n%2%", + if (context) { + throw EvalError(context->state, ErrorInfo{ + .msg = fmt("while parsing the YAML string '%1%':\n\n%2%", context->yaml, std::string_view(msg, len)), - .errPos = context->state.positions[context->pos] + .pos = context->state.positions[context->pos] }); } else { - throw EvalError({ - .msg = hintfmt("failed assertion in rapidyaml library:\n\n%1%", + throw Error({ + .msg = fmt("failed assertion in rapidyaml library:\n\n%1%", std::string_view(msg, len)) }); } @@ -60,16 +60,17 @@ static void visitYAMLNode(NixContext & context, Value & v, ryml::ConstNodeRef t) v.mkAttrs(attrs); } else if (t.is_seq()) { - context.state.mkList(v, t.num_children()); + ListBuilder list(context.state, t.num_children()); size_t i = 0; for (ryml::ConstNodeRef child : t.children()) { - visitYAMLNode(context, *(v.listElems()[i++] = context.state.allocValue()), child); + visitYAMLNode(context, *(list[i++] = context.state.allocValue()), child); } + v.mkList(list); } else if (t.has_val()) { bool _bool; NixFloat _float; - NixInt _int; + NixInt::Inner _int; auto val = t.val(); auto valTag = ryml::TAG_NONE; bool isQuoted = t.is_val_quoted(); @@ -136,9 +137,11 @@ static RegisterPrimOp primop_fromYAML({ .pos = pos, .yaml = yaml }; - ryml::EventHandlerTree evth; - ryml::Callbacks callbacks(&context, nullptr, nullptr, s_error); + ryml::Callbacks callbacks; + callbacks.m_error = s_error; ryml::set_callbacks(callbacks); + callbacks.m_user_data = &context; + ryml::EventHandlerTree evth(callbacks); ryml::Parser parser(&evth); ryml::Tree tree = ryml::parse_in_arena(&parser, ryml::csubstr(yaml.begin(), yaml.size())); tree.resolve(); // resolve references diff --git a/src/libexpr/primops/meson.build b/src/libexpr/primops/meson.build index f910fe237..830125a5f 100644 --- a/src/libexpr/primops/meson.build +++ b/src/libexpr/primops/meson.build @@ -9,4 +9,5 @@ sources += files( 'fetchMercurial.cc', 'fetchTree.cc', 'fromTOML.cc', + 'fromYAML.cc', ) diff --git a/tests/unit/libexpr/meson.build b/tests/unit/libexpr/meson.build index 21c321334..dfdbc74fe 100644 --- a/tests/unit/libexpr/meson.build +++ b/tests/unit/libexpr/meson.build @@ -67,6 +67,7 @@ sources = files( 'value/context.cc', 'value/print.cc', 'value/value.cc', + 'yaml.cc' ) include_dirs = [include_directories('.')] diff --git a/tests/unit/libexpr/yaml.cc b/tests/unit/libexpr/yaml.cc index dde7b2401..99f8eb80c 100644 --- a/tests/unit/libexpr/yaml.cc +++ b/tests/unit/libexpr/yaml.cc @@ -1,10 +1,10 @@ #ifdef HAVE_RYML -#include "libexpr.hh" +#include "tests/libexpr.hh" #include "primops.hh" -// Ugly, however direct access to the SAX parser is required in order to parse multiple JSON objects from a stream -#include "json-to-value.cc" +// access to the json sax parser is required +#include "json-to-value-sax.hh" namespace nix { @@ -54,11 +54,11 @@ namespace nix { static bool parseJSON(EvalState & state, std::istream & s_, Value & v) { - JSONSax parser(state, v); - return nlohmann::json::sax_parse(s_, &parser, nlohmann::json::input_format_t::json, false); + auto parser = makeJSONSaxParser(state, v); + return nlohmann::json::sax_parse(s_, parser.get(), nlohmann::json::input_format_t::json, false); } - static Value parseJSONStream(EvalState & state, std::string_view json, PrimOpFun fromYAML) { + static Value parseJSONStream(EvalState & state, std::string_view json, std::function fromYAML) { std::stringstream ss; ss << json; std::list list; @@ -80,12 +80,12 @@ namespace nix { if (list.size() == 1) { root = *list.begin(); } else { - state.mkList(root, list.size()); - Value **elems = root.listElems(); + ListBuilder list_builder(state, list.size()); size_t i = 0; for (auto val : list) { - *(elems[i++] = state.allocValue()) = val; + *(list_builder[i++] = state.allocValue()) = val; } + root.mkList(list_builder); } return root; } @@ -94,13 +94,14 @@ namespace nix { protected: void execYAMLTest(std::string_view test) { - //const PrimOpFun fromYAML = state.getBuiltin("fromYAML").primOp->fun; - PrimOpFun fromYAML = nullptr; - for (const auto & primOp : *RegisterPrimOp::primOps) { - if (primOp.name == "__fromYAML") { - fromYAML = primOp.fun; + std::function fromYAML = [] () { + for (const auto & primOp : *RegisterPrimOp::primOps) { + if (primOp.name == "__fromYAML") { + return primOp.fun; + } } - } + return std::function(); + } (); EXPECT_FALSE(fromYAML == nullptr) << "The experimental feature \"fromYAML\" is not available"; Value testCases, testVal; Value *pTestVal = &testVal; @@ -112,7 +113,7 @@ namespace nix { for (auto testCase : testCases.listItems()) { bool fail = false; std::string_view yamlRaw; - for (auto attr = testCase->attrs->begin(); attr != testCase->attrs->end(); attr++) { + for (auto attr = testCase->attrs()->begin(); attr != testCase->attrs()->end(); attr++) { auto name = state.symbols[attr->name]; if (name == "json") { json = attr->value;