fromYAML: rebase

package.nix

@@ -31,6 +31,7 @@
 , openssl
 , pkg-config
 , rapidcheck
+, rapidyaml
 , sqlite
 , toml11
 , unixtools
@@ -224,6 +225,7 @@ in {
     libgit2
     libsodium
     openssl
+    rapidyaml
     sqlite
     toml11
     xz

src/libexpr/json-to-value-sax.hh

@@ -0,0 +1,17 @@
+#pragma once
+///@file
+
+#include <memory>
+#include <nlohmann/json.hpp>
+
+#include "json-to-value.hh"
+
+/**
+ * json_sax and unique_ptr require the inclusion of json.hpp, so this header shall not be included by other headers
+ **/
+
+namespace nix {
+
+std::unique_ptr<nlohmann::json_sax<nlohmann::json>> makeJSONSaxParser(EvalState & s, Value & v);
+
+}

src/libexpr/json-to-value.cc

@@ -1,10 +1,8 @@
-#include "json-to-value.hh"
+#include "json-to-value-sax.hh"
 #include "value.hh"
 #include "eval.hh"
 
 #include <limits>
-#include <variant>
-#include <nlohmann/json.hpp>
 
 using json = nlohmann::json;
 
@@ -12,7 +10,7 @@ namespace nix {
 
 // for more information, refer to
 // https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp
-class JSONSax : nlohmann::json_sax<json> {
+class JSONSax : public nlohmann::json_sax<json> {
     class JSONState {
     protected:
         std::unique_ptr<JSONState> parent;
@@ -80,6 +78,7 @@ class JSONSax : nlohmann::json_sax<json> {
 
 public:
     JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
+    virtual ~JSONSax() = default;
 
     bool null() override
     {
@@ -177,4 +176,8 @@ void parseJSON(EvalState & state, const std::string_view & s_, Value & v)
         throw JSONParseError("Invalid JSON Value");
 }
 
+std::unique_ptr<nlohmann::json_sax<json>> makeJSONSaxParser(EvalState & state, Value & v) {
+    return { std::make_unique<JSONSax>(state, v) };
+}
+
 }

src/libexpr/meson.build

@@ -56,6 +56,18 @@ if bdw_gc.found()
 endif
 configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int())
 
+ryml = dependency(
+  'ryml',
+  version : '>=0.7.1',
+  method : 'cmake',
+  include_type : 'system',
+  required : false,
+)
+if ryml.found()
+  deps_other += ryml
+  configdata.set('HAVE_RYML', 1)
+endif
+
 toml11 = dependency(
   'toml11',
   version : '>=3.7.0',
@@ -172,6 +184,7 @@ headers = [config_h] + files(
   'gc-small-vector.hh',
   'get-drvs.hh',
   'json-to-value.hh',
+  'json-to-value-sax.hh',
   # internal: 'lexer-helpers.hh',
   'nixexpr.hh',
   'parser-state.hh',

src/libexpr/primops/fromYAML.cc

@@ -19,15 +19,15 @@ struct NixContext {
 static void s_error [[ noreturn ]] (const char* msg, size_t len, ryml::Location, void *nixContext)
 {
     auto context = static_cast<const NixContext *>(nixContext);
-    if (nixContext) {
-        throw EvalError({
-            .msg = hintfmt("while parsing the YAML string '%1%':\n\n%2%",
+    if (context) {
+        throw EvalError(context->state, ErrorInfo{
+            .msg = fmt("while parsing the YAML string '%1%':\n\n%2%",
                 context->yaml, std::string_view(msg, len)),
-            .errPos = context->state.positions[context->pos]
+            .pos = context->state.positions[context->pos]
         });
     } else {
-        throw EvalError({
-            .msg = hintfmt("failed assertion in rapidyaml library:\n\n%1%",
+        throw Error({
+            .msg = fmt("failed assertion in rapidyaml library:\n\n%1%",
                 std::string_view(msg, len))
         });
     }
@@ -60,16 +60,17 @@ static void visitYAMLNode(NixContext & context, Value & v, ryml::ConstNodeRef t)
 
         v.mkAttrs(attrs);
     } else if (t.is_seq()) {
-        context.state.mkList(v, t.num_children());
+        ListBuilder list(context.state, t.num_children());
 
         size_t i = 0;
         for (ryml::ConstNodeRef child : t.children()) {
-            visitYAMLNode(context, *(v.listElems()[i++] = context.state.allocValue()), child);
+            visitYAMLNode(context, *(list[i++] = context.state.allocValue()), child);
         }
+        v.mkList(list);
     } else if (t.has_val()) {
         bool _bool;
         NixFloat _float;
-        NixInt _int;
+        NixInt::Inner _int;
         auto val = t.val();
         auto valTag = ryml::TAG_NONE;
         bool isQuoted = t.is_val_quoted();
@@ -136,9 +137,11 @@ static RegisterPrimOp primop_fromYAML({
             .pos = pos,
             .yaml = yaml
         };
-        ryml::EventHandlerTree evth;
-        ryml::Callbacks callbacks(&context, nullptr, nullptr, s_error);
+        ryml::Callbacks callbacks;
+        callbacks.m_error = s_error;
         ryml::set_callbacks(callbacks);
+        callbacks.m_user_data = &context;
+        ryml::EventHandlerTree evth(callbacks);
         ryml::Parser parser(&evth);
         ryml::Tree tree = ryml::parse_in_arena(&parser, ryml::csubstr(yaml.begin(), yaml.size()));
         tree.resolve(); // resolve references

src/libexpr/primops/meson.build

@@ -9,4 +9,5 @@ sources += files(
   'fetchMercurial.cc',
   'fetchTree.cc',
   'fromTOML.cc',
+  'fromYAML.cc',
 )

tests/unit/libexpr/meson.build

@@ -67,6 +67,7 @@ sources = files(
   'value/context.cc',
   'value/print.cc',
   'value/value.cc',
+  'yaml.cc'
 )
 
 include_dirs = [include_directories('.')]

tests/unit/libexpr/yaml.cc

@@ -1,10 +1,10 @@
 #ifdef HAVE_RYML
 
-#include "libexpr.hh"
+#include "tests/libexpr.hh"
 #include "primops.hh"
 
-// Ugly, however direct access to the SAX parser is required in order to parse multiple JSON objects from a stream
-#include "json-to-value.cc"
+// access to the json sax parser is required
+#include "json-to-value-sax.hh"
 
 
 namespace nix {
@@ -54,11 +54,11 @@ namespace nix {
 
     static bool parseJSON(EvalState & state, std::istream & s_, Value & v)
     {
-        JSONSax parser(state, v);
-        return nlohmann::json::sax_parse(s_, &parser, nlohmann::json::input_format_t::json, false);
+        auto parser = makeJSONSaxParser(state, v);
+        return nlohmann::json::sax_parse(s_, parser.get(), nlohmann::json::input_format_t::json, false);
     }
 
-    static Value parseJSONStream(EvalState & state, std::string_view json, PrimOpFun fromYAML) {
+    static Value parseJSONStream(EvalState & state, std::string_view json, std::function<PrimOpFun> fromYAML) {
         std::stringstream ss;
         ss << json;
         std::list<Value> list;
@@ -80,12 +80,12 @@ namespace nix {
         if (list.size() == 1) {
             root = *list.begin();
         } else {
-            state.mkList(root, list.size());
-            Value **elems = root.listElems();
+            ListBuilder list_builder(state, list.size());
             size_t i = 0;
             for (auto val : list) {
-                *(elems[i++] = state.allocValue()) = val;
+                *(list_builder[i++] = state.allocValue()) = val;
             }
+            root.mkList(list_builder);
         }
         return root;
     }
@@ -94,13 +94,14 @@ namespace nix {
         protected:
 
             void execYAMLTest(std::string_view test) {
-                //const PrimOpFun fromYAML = state.getBuiltin("fromYAML").primOp->fun;
-                PrimOpFun fromYAML = nullptr;
-                for (const auto & primOp : *RegisterPrimOp::primOps) {
-                    if (primOp.name == "__fromYAML") {
-                        fromYAML = primOp.fun;
+                std::function<PrimOpFun> fromYAML = [] () {
+                    for (const auto & primOp : *RegisterPrimOp::primOps) {
+                        if (primOp.name == "__fromYAML") {
+                            return primOp.fun;
+                        }
                     }
-                }
+                    return std::function<PrimOpFun>();
+                } ();
                 EXPECT_FALSE(fromYAML == nullptr) << "The experimental feature \"fromYAML\" is not available";
                 Value testCases, testVal;
                 Value *pTestVal = &testVal;
@@ -112,7 +113,7 @@ namespace nix {
                 for (auto testCase : testCases.listItems()) {
                     bool fail = false;
                     std::string_view yamlRaw;
-                    for (auto attr = testCase->attrs->begin(); attr != testCase->attrs->end(); attr++) {
+                    for (auto attr = testCase->attrs()->begin(); attr != testCase->attrs()->end(); attr++) {
                         auto name = state.symbols[attr->name];
                         if (name == "json") {
                             json = attr->value;