Merge 1194b5d2df into ab0f9f9089

git-utils: fix x86_64-w64-mingw32 build

Makefile.config.in

@@ -28,6 +28,7 @@ LOWDOWN_LIBS = @LOWDOWN_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
+RYML_LIBS = @RYML_LIBS@
 SHELL = @bash@
 SODIUM_LIBS = @SODIUM_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@

configure.ac

@@ -269,6 +269,29 @@ AS_CASE(["$readline_flavor"],
   [AC_MSG_ERROR([bad value "$readline_flavor" for --with-readline-flavor, must be one of: editline, readline])])
 PKG_CHECK_MODULES([EDITLINE], [$readline_flavor_pc], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"])
 
+# Look for rapidyaml.
+have_ryml=
+AC_ARG_ENABLE([ryml], AS_HELP_STRING([--disable-ryml], [Do not enable rapidyaml and disable builtins.fromYAML]), [], [have_ryml=1])
+AC_ARG_VAR([RYML_CPPFLAGS], [C/C++ preprocessor flags for RAPIDYAML])
+AC_ARG_VAR([RYML_LDFLAGS], [linker flags for RAPIDYAML])
+if test "x$have_ryml" != "x"; then
+    AC_LANG_PUSH([C++])
+    # append RYML_CPPFLAGS to CXXFLAGS because CPPFLAGS are not passed to the C++ compiler
+    CXXFLAGS="$RYML_CPPFLAGS $CXXFLAGS"
+    LDFLAGS="$RYML_LDFLAGS $RYML_LDFLAGS"
+    LIBS="-lryml $LIBS"
+    AC_CHECK_HEADERS([ryml.hpp], [true],
+      [AC_MSG_ERROR([Header of libryml is not found.])])
+    AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+        #include <ryml.hpp>
+      ]], [[ryml::Tree tree;]])],
+      [AC_DEFINE([HAVE_RYML], [1], [Use rapidyaml])
+       AC_SUBST(RYML_LIBS, [-lryml])],
+      [AC_MSG_ERROR([libryml is not found.])])
+    AC_LANG_POP([C++])
+fi
+AC_SUBST(HAVE_RYML, [$have_ryml])
+
 # Look for libsodium.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 

package.nix

@@ -31,6 +31,7 @@
 , openssl
 , pkg-config
 , rapidcheck
+, rapidyaml
 , sqlite
 , toml11
 , unixtools
@@ -224,6 +225,7 @@ in {
     libgit2
     libsodium
     openssl
+    rapidyaml
     sqlite
     toml11
     xz

packaging/dependencies.nix

@@ -164,6 +164,26 @@ scope: {
     '';
   });
 
+  rapidyaml = pkgs.rapidyaml.overrideAttrs(old: let
+    version = "0.7.2";
+    hash =  "sha256-vAYafhWo9xavM2j+mT3OGcX7ZSS25mieR/3b79BO+jA=";
+  in {
+    inherit version;
+
+    src = pkgs.fetchFromGitHub {
+      inherit hash;
+      owner = "biojppm";
+      repo = old.pname;
+      rev = "v${version}";
+      fetchSubmodules = true;
+    };
+
+    cmakeFlags = [
+      "-DRYML_WITH_TAB_TOKENS=ON"
+    ];
+  });
+
+
   # TODO change in Nixpkgs, Windows works fine. First commit of
   # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix.
   toml11 = pkgs.toml11.overrideAttrs (old: {

src/libexpr/json-to-value-sax.hh

@@ -0,0 +1,17 @@
+#pragma once
+///@file
+
+#include <memory>
+#include <nlohmann/json.hpp>
+
+#include "json-to-value.hh"
+
+/**
+ * json_sax and unique_ptr require the inclusion of json.hpp, so this header shall not be included by other headers
+ **/
+
+namespace nix {
+
+std::unique_ptr<nlohmann::json_sax<nlohmann::json>> makeJSONSaxParser(EvalState & s, Value & v);
+
+}

src/libexpr/json-to-value.cc

@@ -1,10 +1,8 @@
-#include "json-to-value.hh"
+#include "json-to-value-sax.hh"
 #include "value.hh"
 #include "eval.hh"
 
 #include <limits>
-#include <variant>
-#include <nlohmann/json.hpp>
 
 using json = nlohmann::json;
 
@@ -12,7 +10,7 @@ namespace nix {
 
 // for more information, refer to
 // https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp
-class JSONSax : nlohmann::json_sax<json> {
+class JSONSax : public nlohmann::json_sax<json> {
     class JSONState {
     protected:
         std::unique_ptr<JSONState> parent;
@@ -80,6 +78,7 @@ class JSONSax : nlohmann::json_sax<json> {
 
 public:
     JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
+    virtual ~JSONSax() = default;
 
     bool null() override
     {
@@ -177,4 +176,8 @@ void parseJSON(EvalState & state, const std::string_view & s_, Value & v)
         throw JSONParseError("Invalid JSON Value");
 }
 
+std::unique_ptr<nlohmann::json_sax<json>> makeJSONSaxParser(EvalState & state, Value & v) {
+    return { std::make_unique<JSONSax>(state, v) };
+}
+
 }

src/libexpr/local.mk

@@ -20,7 +20,7 @@ libexpr_CXXFLAGS += \
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS)
+libexpr_LDFLAGS += -lboost_context $(RYML_LIBS) $(THREAD_LDFLAGS)
 ifdef HOST_LINUX
   libexpr_LDFLAGS += -ldl
 endif

src/libexpr/meson.build

@@ -56,6 +56,18 @@ if bdw_gc.found()
 endif
 configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int())
 
+ryml = dependency(
+  'ryml',
+  version : '>=0.7.1',
+  method : 'cmake',
+  include_type : 'system',
+  required : false,
+)
+if ryml.found()
+  deps_other += ryml
+  configdata.set('HAVE_RYML', 1)
+endif
+
 toml11 = dependency(
   'toml11',
   version : '>=3.7.0',
@@ -172,6 +184,7 @@ headers = [config_h] + files(
   'gc-small-vector.hh',
   'get-drvs.hh',
   'json-to-value.hh',
+  'json-to-value-sax.hh',
   # internal: 'lexer-helpers.hh',
   'nixexpr.hh',
   'parser-state.hh',

src/libexpr/package.nix

@@ -16,6 +16,7 @@
 , boost
 , boehmgc
 , nlohmann_json
+, rapidyaml
 , toml11
 
 # Configuration Options
@@ -70,6 +71,7 @@ mkMesonDerivation (finalAttrs: {
   ];
 
   buildInputs = [
+    rapidyaml
     toml11
   ];
 

src/libexpr/primops/fromYAML.cc

@@ -0,0 +1,392 @@
+#ifdef HAVE_RYML
+
+#  include "primops.hh"
+#  include "eval-inline.hh"
+
+#  include <ryml.hpp>
+#  include <c4/format.hpp>
+#  include <c4/std/string.hpp>
+
+namespace {
+
+using namespace nix;
+
+/**
+ * Equality check of a compile time C-string *lhs* and another string *rhs*.
+ * Only call this function, if both strings have the same length.
+ */
+template<size_t N>
+inline bool isEqualSameLengthStr(const char * rhs, const char lhs[N + 1])
+{
+    bool result = true;
+    for (size_t i = 0; i < N; i++) {
+        result &= rhs[i] == lhs[i];
+    }
+    return result;
+}
+
+inline bool isNull(ryml::csubstr val)
+{
+    size_t len = val.size();
+    return len == 0 || (len == 1 && val[0] == '~')
+           || (len == 4 && (val[0] == 'n' || val[0] == 'N')
+               && (isEqualSameLengthStr<3>(&val[1], "ull") || isEqualSameLengthStr<4>(&val[0], "NULL")));
+}
+
+bool isInt_1_2(ryml::csubstr val)
+{
+    bool result = val.is_integer();
+    // ryml::from_chars accepts signed binary, octal and hexadecimal integers
+    // YAML 1.2 defines unsigned octal and hexadecimal integers (lower-case identifiers)
+    if (result && val.size() >= 3
+        && ((val.begins_with_any("+-") && val.sub(2, 1).begins_with_any("xXoObB"))
+            || val.sub(1, 1).begins_with_any("XObB"))) {
+        result = false;
+    }
+    return result;
+}
+
+/**
+ * Tries to parse a string into a floating point number according to the YAML 1.2 core schema, wrapping ryml::from_chars
+ */
+std::optional<NixFloat> parseFloat(std::optional<bool> isInt, ryml::csubstr val)
+{
+    std::optional<NixFloat> maybe_float;
+    NixFloat _float;
+    size_t len = val.size();
+    // first character has to match [0-9+-.]
+    if (isInt.value_or(false)) {
+        NixInt::Inner _int;
+        if (len == 2 && isEqualSameLengthStr<2>(&val[0], "-0")) {
+            // valid int, so that it would be parsed as 0.0 otherwise
+            maybe_float = -0.0;
+        } else if (ryml::from_chars(val.sub(val[0] == '+'), &_int)) {
+            maybe_float.emplace(_int);
+        }
+    } else if (len >= 1 && val[0] >= '+' && val[0] <= '9' && val[0] != ',' && val[0] != '/') {
+        size_t skip = val[0] == '+' || val[0] == '-';
+        if ((len == skip + 4) && val[skip + 0] == '.') {
+            auto sub = &val[skip + 1];
+            if (skip == 0
+                && (isEqualSameLengthStr<3>(sub, "nan")
+                    || (sub[0] == 'N' && (sub[1] == 'a' || sub[1] == 'A') && sub[2] == 'N'))) {
+                maybe_float = std::numeric_limits<NixFloat>::quiet_NaN();
+            } else if (
+                ((sub[0] == 'i' || sub[0] == 'I') && isEqualSameLengthStr<2>(sub + 1, "nf"))
+                || isEqualSameLengthStr<3>(sub, "INF")) {
+                NixFloat inf = std::numeric_limits<NixFloat>::infinity();
+                maybe_float = val[0] == '-' ? -inf : inf;
+            }
+        }
+        auto sub = &val[0] + 1;
+        if (len == skip + 3 && (isEqualSameLengthStr<3>(sub, "nan") || isEqualSameLengthStr<3>(sub, "inf"))) {
+            // ryml::from_chars converts "nan" and "inf"
+        } else if (
+            !maybe_float && ((!isInt && val.is_number()) || (isInt && val.is_real()))
+            && val.sub(1, std::min(size_t(2), len - 1)).first_of("xXoObB") == ryml::npos
+            && ryml::from_chars(val.sub(val[0] == '+'), &_float)) {
+            // isInt => !*isInt because of (isInt && *isInt) == false)
+            maybe_float = _float;
+        }
+    }
+    return maybe_float;
+}
+
+std::optional<bool> parseBool_1_2(ryml::csubstr val)
+{
+    std::optional<bool> _bool;
+    size_t len = val.size();
+    if (len == 4 && (val[0] == 't' || val[0] == 'T')) {
+        if (isEqualSameLengthStr<3>(&val[1], "rue") || isEqualSameLengthStr<4>(&val[0], "TRUE")) {
+            _bool = true;
+        }
+    } else if (len == 5 && (val[0] == 'f' || val[0] == 'F')) {
+        if (isEqualSameLengthStr<4>(&val[1], "alse") || isEqualSameLengthStr<5>(&val[0], "FALSE")) {
+            _bool = false;
+        }
+    }
+    return _bool;
+}
+
+std::optional<bool> parseBool_1_1(ryml::csubstr val)
+{
+    std::optional<bool> _bool;
+    switch (val.size()) {
+    case 1:
+        if (val[0] == 'n' || val[0] == 'N') {
+            _bool = false;
+        } else if (val[0] == 'y' || val[0] == 'Y') {
+            _bool = true;
+        }
+        break;
+    case 2:
+        // "no" or "on"
+        if (isEqualSameLengthStr<2>(&val[0], "no") || (val[0] == 'N' && (val[1] == 'o' || val[1] == 'O'))) {
+            _bool = false;
+        } else if (isEqualSameLengthStr<2>(&val[0], "on") || (val[0] == 'O' && (val[1] == 'n' || val[1] == 'N'))) {
+            _bool = true;
+        }
+        break;
+    case 3:
+        // "off" or "yes"
+        if (isEqualSameLengthStr<3>(&val[0], "off")
+            || (val[0] == 'O' && (isEqualSameLengthStr<2>(&val[1], "ff") || isEqualSameLengthStr<2>(&val[1], "FF")))) {
+            _bool = false;
+        } else if (
+            isEqualSameLengthStr<3>(&val[0], "yes")
+            || (val[0] == 'Y' && (isEqualSameLengthStr<2>(&val[1], "es") || isEqualSameLengthStr<2>(&val[1], "ES")))) {
+            _bool = true;
+        }
+        break;
+    case 4:
+    case 5:
+        _bool = parseBool_1_2(val);
+        break;
+    default:
+        break;
+    }
+    return _bool;
+}
+
+struct FromYAMLContext
+{
+    struct ParserOptions
+    {
+        bool useBoolYAML1_1 = false;
+
+        ParserOptions(FromYAMLContext &, const Bindings *);
+    };
+
+    EvalState & state;
+    const PosIdx pos;
+    const std::string_view yaml;
+    const ParserOptions options;
+
+    FromYAMLContext(EvalState &, PosIdx, std::string_view, const Bindings *);
+
+    inline std::optional<bool> parseBool(ryml::csubstr val) const
+    {
+        std::optional<bool> result;
+        if (options.useBoolYAML1_1) {
+            result = parseBool_1_1(val);
+        } else {
+            result = parseBool_1_2(val);
+        }
+        return result;
+    }
+
+    template<typename... Args>
+    void throwError [[noreturn]] (const char * c_fs, const Args &... args) const
+    {
+        std::string fs = "while parsing the YAML string ''%1%'':\n\n";
+        fs += c_fs;
+        throw EvalError(state, ErrorInfo{.msg = fmt(fs, yaml, args...), .pos = state.positions[pos]});
+    }
+
+    void visitYAMLNode(Value & v, ryml::ConstNodeRef t, bool isTopNode = false);
+};
+
+FromYAMLContext::FromYAMLContext(EvalState & state, PosIdx pos, std::string_view yaml, const Bindings * options)
+    : state(state)
+    , pos(pos)
+    , yaml(yaml)
+    , options(*this, options)
+{
+}
+
+FromYAMLContext::ParserOptions::ParserOptions(FromYAMLContext & context, const Bindings * options)
+{
+    auto symbol = context.state.symbols.create("useBoolYAML1_1");
+    const Attr * useBoolYAML1_1 = options->get(symbol);
+    if (useBoolYAML1_1) {
+        this->useBoolYAML1_1 =
+            context.state.forceBool(*useBoolYAML1_1->value, {}, "while evaluating the attribute \"useBoolYAML1_1\"");
+    }
+}
+
+void s_error [[noreturn]] (const char * msg, size_t len, ryml::Location, void * fromYAMLContext)
+{
+    auto context = static_cast<const FromYAMLContext *>(fromYAMLContext);
+    if (context) {
+        context->throwError("%2%", std::string_view(msg, len));
+    } else {
+        throw Error({.msg = fmt("failed assertion in rapidyaml library:\n\n%1%", std::string_view(msg, len))});
+    }
+}
+
+/**
+ * Parse YAML according to the YAML 1.2 core schema by default
+ * The behaviour can be modified by the FromYAMLOptions object in FromYAMLContext
+ */
+void FromYAMLContext::visitYAMLNode(Value & v, ryml::ConstNodeRef t, bool isTopNode)
+{
+    ryml::csubstr valTagStr;
+    auto valTag = ryml::TAG_NONE;
+    bool valTagCustom = t.has_val_tag();
+    bool valTagNonSpecificStr = false;
+    if (valTagCustom) {
+        valTagStr = t.val_tag();
+        if (!(valTagNonSpecificStr = valTagStr == "!")) {
+            valTag = ryml::to_tag(valTagStr);
+            valTagCustom = valTag == ryml::TAG_NONE;
+            if (valTagCustom) {
+                auto fs = "Error: Nix has no support for the unknown tag ''%2%'' in node ''%3%''";
+                throwError(fs, valTagStr, t);
+            }
+        }
+    }
+    if (t.is_map()) {
+        if (valTag != ryml::TAG_NONE && valTag != ryml::TAG_MAP) {
+            auto fs = "Error: Nix parsed ''%2%'' as map and only supported is the tag ''!!map'', but ''%3%'' was used";
+            throwError(fs, t, valTagStr);
+        }
+        auto attrs = state.buildBindings(t.num_children());
+
+        for (ryml::ConstNodeRef child : t.children()) {
+            auto key = child.key();
+            if (child.has_key_tag()) {
+                auto tag = ryml::to_tag(child.key_tag());
+                if (tag != ryml::TAG_NONE && tag != ryml::TAG_STR) {
+                    auto fs = "Error: Nix supports string keys only, but the key ''%2%'' has the tag ''%3%''";
+                    throwError(fs, child.key(), child.key_tag());
+                }
+            } else if (child.is_key_plain() && isNull(key)) {
+                auto fs = "Error: Nix supports string keys only, but the map ''%2%'' contains a null-key";
+                throwError(fs, t);
+            }
+            visitYAMLNode(attrs.alloc({key.begin(), key.size()}), child);
+        }
+
+        v.mkAttrs(attrs);
+        Symbol key;
+        // enforce uniqueness of keys
+        for (const auto & attr : *attrs.alreadySorted()) {
+            if (key == attr.name) {
+                auto fs = "Error: Non-unique key %2% after deserializing the map ''%3%''";
+                throwError(fs, state.symbols[key], t);
+            }
+            key = attr.name;
+        }
+    } else if (t.is_seq()) {
+        if (valTag != ryml::TAG_NONE && valTag != ryml::TAG_SEQ) {
+            auto fs =
+                "Error: Nix parsed ''%2%'' as sequence and only supported is the tag ''!!seq'', but ''%3%'' was used";
+            throwError(fs, t, valTagStr);
+        }
+        ListBuilder list(state, t.num_children());
+
+        bool isStream = t.is_stream();
+        size_t i = 0;
+        for (ryml::ConstNodeRef child : t.children()) {
+            // a stream of documents is handled as sequence, too
+            visitYAMLNode(*(list[i++] = state.allocValue()), child, isTopNode && isStream);
+        }
+        v.mkList(list);
+    } else if (t.has_val()) {
+        auto val = t.val();
+        bool isPlain = t.is_val_plain();
+        bool isEmpty = isPlain && val.empty();
+        if (isTopNode && isEmpty) {
+            throwError("Error: Empty document (plain empty scalars outside of collection)%2%", "");
+        }
+        if (valTagNonSpecificStr) {
+            valTag = ryml::TAG_STR;
+        }
+
+        auto scalarTypeCheck = [=](ryml::YamlTag_e tag) { return valTag == ryml::TAG_NONE ? isPlain : valTag == tag; };
+
+        // Caution: ryml::from_chars converts integers into booleans and also it might ignore trailing chars.
+        // Furthermore it doesn't accept a leading '+' character in integers
+        std::optional<bool> isInt;
+        std::optional<bool> _bool;
+        std::optional<NixFloat> _float;
+        NixInt::Inner _int;
+        bool trim = valTag == ryml::TAG_NULL || valTag == ryml::TAG_BOOL || valTag == ryml::TAG_INT
+                    || valTag == ryml::TAG_FLOAT;
+        auto vs = trim ? val.trim("\n\t ") : val;
+        if (scalarTypeCheck(ryml::TAG_NULL) && isNull(vs)) {
+            v.mkNull();
+        } else if (scalarTypeCheck(ryml::TAG_BOOL) && (_bool = parseBool(vs))) {
+            v.mkBool(*_bool);
+        } else if (
+            scalarTypeCheck(ryml::TAG_INT) && *(isInt = isInt_1_2(vs))
+            && ryml::from_chars(vs.sub(vs[0] == '+'), &_int)) {
+            v.mkInt(_int);
+        } else if (
+            ((valTag == ryml::TAG_FLOAT && (isInt = isInt_1_2(vs))) || (valTag == ryml::TAG_NONE && isPlain))
+            && (_float = parseFloat(isInt, vs))) {
+            // if the value is tagged with !!float, then isInt_1_2 evaluation is enforced because the int regex is not a
+            // subset of the float regex...
+            v.mkFloat(*_float);
+        } else if ((valTag == ryml::TAG_NONE && !valTagCustom) || valTag == ryml::TAG_STR) {
+            std::string_view value(val.begin(), val.size());
+            v.mkString(value);
+        } else {
+            throwError("Error: Value ''%2%'' with tag ''%3%'' is invalid", val, valTagStr);
+        }
+    } else {
+        auto val = t.has_val() ? t.val() : "";
+        auto fs = "BUG: Encountered unreachable code while parsing ''%2%'' with tag ''%3%''";
+        throwError(fs, val, valTagStr);
+    }
+}
+
+} /* namespace */
+
+namespace nix {
+
+static RegisterPrimOp primop_fromYAML(
+    {.name = "__fromYAML",
+     .args = {"e", "attrset"},
+     .doc = R"(
+       Convert a YAML 1.2 string *e* to a Nix value, if a conversion is possible.
+       The second argument is an attribute set with optional parameters for the parser.
+       For example,
+
+       ```nix
+       builtins.fromYAML ''{x: [1, 2, 3], y: !!str null, z: null}'' {}
+       ```
+
+       returns the value `{ x = [ 1 2 3 ]; y = "null"; z = null; }`.
+
+       Maps are converted to attribute sets, but only strings are supported as keys.
+
+       Scalars are converted to the type specified by their optional value tag. Parsing fails if a conversion is not possible.
+       Nix does not support all data types defined by the different YAML specs, e.g. Nix has no binary and timestamp data types.
+       Thus the types and tags defined by the YAML 1.2 core schema are used exclusively, i.e. untagged timestamps are parsed as strings.
+       Using any other tag fails.
+       A stream with multiple documents is mapped to a list except when the stream contains a single document.
+
+       Supported optional parameters in *attrset*:
+         - useBoolYAML1_1 :: bool ? false: When enabled booleans are parsed according to the YAML 1.1 spec, which matches more values than YAML 1.2.
+                                           This option improves compatibility because many applications and configs are still using YAML 1.1 features.
+     )",
+     .fun =
+         [](EvalState & state, const PosIdx pos, Value ** args, Value & val) {
+             auto yaml = state.forceStringNoCtx(
+                 *args[0], pos, "while evaluating the first argument passed to builtins.fromYAML");
+             state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.fromYAML");
+             auto options = args[1]->attrs();
+
+             FromYAMLContext context(state, pos, yaml, options);
+             ryml::Callbacks callbacks;
+             callbacks.m_error = s_error;
+             ryml::set_callbacks(callbacks);
+             callbacks.m_user_data = &context;
+             ryml::EventHandlerTree evth(callbacks);
+             ryml::Parser parser(&evth);
+             ryml::Tree tree = ryml::parse_in_arena(&parser, ryml::csubstr(yaml.begin(), yaml.size()));
+             tree.resolve(); // resolve references
+             tree.resolve_tags();
+
+             auto root = tree.crootref();
+             if (root.is_stream() && root.num_children() == 1 && root.child(0).is_doc()) {
+                 root = root.child(0);
+             }
+             context.visitYAMLNode(val, root, true);
+         },
+     .experimentalFeature = Xp::FromYaml});
+
+} /* namespace nix */
+
+#endif

src/libexpr/primops/meson.build

@@ -9,4 +9,5 @@ sources += files(
   'fetchMercurial.cc',
   'fetchTree.cc',
   'fromTOML.cc',
+  'fromYAML.cc',
 )

src/libfetchers/git-utils.cc

@@ -208,7 +208,7 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil
 static void initRepoAtomically(std::filesystem::path &path, bool bare) {
     if (pathExists(path.string())) return;
 
-    Path tmpDir = createTempDir(std::filesystem::path(path).parent_path());
+    Path tmpDir = createTempDir(os_string_to_string(PathViewNG { std::filesystem::path(path).parent_path() }));
     AutoDelete delTmpDir(tmpDir, true);
     Repository tmpRepo;
 

src/libutil/experimental-features.cc

@@ -24,7 +24,7 @@ struct ExperimentalFeatureDetails
  * feature, we either have no issue at all if few features are not added
  * at the end of the list, or a proper merge conflict if they are.
  */
-constexpr size_t numXpFeatures = 1 + static_cast<size_t>(Xp::PipeOperators);
+constexpr size_t numXpFeatures = 1 + static_cast<size_t>(Xp::FromYaml);
 
 constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails = {{
     {
@@ -302,6 +302,10 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         )",
         .trackingUrl = "https://github.com/NixOS/nix/milestone/55",
     },
+    {
+        .tag = Xp::FromYaml,
+        .name = "from-yaml",
+    },
 }};
 
 static_assert(

src/libutil/experimental-features.hh

@@ -37,6 +37,7 @@ enum struct ExperimentalFeature
     MountedSSHStore,
     VerifiedFetches,
     PipeOperators,
+    FromYaml,
 };
 
 /**

src/nix/eval.md

@@ -50,8 +50,9 @@ R""(
 
 # Description
 
-This command evaluates the given Nix expression and prints the
-result on standard output.
+This command evaluates the given Nix expression, and prints the result on standard output.
+
+It also evaluates any nested attribute values and list items.
 
 # Output format
 

src/nix/meson.build

@@ -212,18 +212,23 @@ nix_symlinks = [
   'nix-store',
 ]
 
+executable_suffix = ''
+if host_machine.system() == 'windows'
+  executable_suffix = '.exe'
+endif
+
 foreach linkname : nix_symlinks
   install_symlink(
-    linkname,
+    linkname + executable_suffix,
     # TODO(Qyriad): should these continue to be relative symlinks?
-    pointing_to : 'nix',
+    pointing_to : fs.name(this_exe),
     install_dir : get_option('bindir'),
     # The 'runtime' tag is what executables default to, which we want to emulate here.
     install_tag : 'runtime'
   )
   t = custom_target(
     command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'],
-    output: linkname,
+    output: linkname + executable_suffix,
     # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working)
     build_by_default: true
   )
@@ -233,15 +238,15 @@ endforeach
 
 install_symlink(
   'build-remote',
-  pointing_to : '..' / '..'/ get_option('bindir') / 'nix',
-  install_dir : get_option('libexecdir') / 'nix',
+  pointing_to : '..' / '..'/ get_option('bindir') / fs.name(this_exe),
+  install_dir : get_option('libexecdir') / fs.name(this_exe),
   # The 'runtime' tag is what executables default to, which we want to emulate here.
   install_tag : 'runtime'
 )
 
 custom_target(
   command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'],
-  output: 'build-remote',
+  output: 'build-remote' + executable_suffix,
   # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working)
   build_by_default: true
 )

tests/unit/libexpr/compose-yaml-test-suite.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+testclass="FromYAMLTest"
+testmethod="execYAMLTest"
+
+if [ -z "$1" ]; then
+	echo "Usage: $0 PathToYamlTestSuiteRepository"
+	echo
+	echo "This script processes test cases from the yaml-test-suite repository (https://github.com/yaml/yaml-test-suite) and converts them into gtest tests for 'builtins.fromYAML'."
+	exit 1
+fi
+
+echo "/* Generated by $(basename "$0") */"
+echo
+echo "#pragma once"
+echo
+
+for f in "$1"/src/*.yaml; do
+	testname="$(basename "${f}" .yaml)"
+	echo "static constexpr std::string_view T_${testname} = R\"RAW("
+	cat "${f}"
+	case "${testname}" in
+		"4ABK")
+			cat << EOL
+  json: |
+    {
+      "unquoted": "separate",
+      "http://foo.com": null,
+      "omitted value": null
+    }
+EOL
+			;;
+#		"SM9W")
+#			echo "  # not JSON compatible due to null key"
+#			echo "  fail: true"
+#			;;
+#		"UKK6")
+#			echo "  # empty document"
+#			echo "  fail: true"
+#			;;
+		*)
+			;;
+	esac
+	echo ")RAW\";"
+	echo
+done
+
+echo "namespace {"
+echo "using namespace nix;"
+echo
+for f in "$1"/src/*.yaml; do
+	testname="$(basename "${f}" .yaml)"
+	ignore="false"
+	skip="false"
+	throw_comment=""
+	# shellcheck disable=SC2221,SC2222
+	case "${testname}" in
+		"H7TQ"|"MUS6"|"ZYU8")
+			echo "/** This test is ignored because these tests are not required to fail and rapidyaml ignores the YAML version string."
+			ignore="true"
+			;;
+		"3HFZ"|"4EJS"|"5TRB"|"5U3A"|"7LBH"|"9C9N"|"9MQT"|"CVW2"|"CXX2"|"D49Q"|"DK4H"|"DK95"|"G5U8"|"JKF3"|"N782"|"QB6E"|"QLJ7"|"RXY3"|"S4GJ"|"S98Z"|"SY6V"|"VJP3"|"X4QW"|"Y79Y"|"YJV2"|"ZCZ6"|"ZL4Z"|"ZXT5"|"3HFZ"|"4EJS"|"5TRB"|"5U3A"|"7LBH"|"9C9N"|"9MQT"|"CVW2"|"CXX2"|"D49Q"|"DK4H"|"DK95"|"G5U8"|"JKF3"|"N782"|"QB6E"|"QLJ7"|"RXY3"|"S4GJ"|"S98Z"|"SY6V"|"VJP3"|"X4QW"|"Y79Y"|"YJV2"|"ZCZ6"|"ZL4Z"|"ZXT5")
+			skip="true"
+			;;
+		"565N")
+			throw_comment="nix has no binary data type"
+			;;
+		"5TYM"|"6CK3"|"6WLZ"|"7FWL"|"9WXW"|"C4HZ"|"CC74"|"CUP7"|"M5C3"|"P76L"|"UGM3"|"Z67P"|"Z9M4")
+			throw_comment="usage of unknown tags"
+			;;
+		"2XXW"|"J7PZ")
+			throw_comment="usage of optional tag like !!set and !!omap (not implemented)"
+			;;
+	esac
+	echo "TEST_F(${testclass}, T_${testname})"
+	echo "{"
+	if [ -n "${throw_comment}" ]; then
+		echo "    EXPECT_THROW(${testmethod}(T_${testname}), EvalError) << \"${throw_comment}\";"
+	else
+		if [ "${skip}" = "true" ]; then
+			echo "    GTEST_SKIP() << \"Reason: Invalid yaml is parsed successfully\";"
+		fi
+		echo "    ${testmethod}(T_${testname});"
+	fi
+	echo "}"
+	[[ "${ignore}" = "true" ]] && echo "*/"
+	echo
+done
+echo "} /* namespace */"

tests/unit/libexpr/meson.build

@@ -67,6 +67,7 @@ sources = files(
   'value/context.cc',
   'value/print.cc',
   'value/value.cc',
+  'yaml.cc'
 )
 
 include_dirs = [include_directories('.')]

tests/unit/libexpr/yaml.cc

@@ -0,0 +1,377 @@
+#ifdef HAVE_RYML
+
+#  include <cstring>
+#  include "tests/libexpr.hh"
+#  include "primops.hh"
+
+// access to the json sax parser is required
+#  include "json-to-value-sax.hh"
+
+namespace {
+using namespace nix;
+using FromYAMLFun = Value(EvalState &, Value, std::optional<Value>);
+
+/**
+ * replacement of non-ascii unicode characters, which indicate the presence of certain characters that would be
+ * otherwise hard to read
+ */
+std::string replaceUnicodePlaceholders(std::string_view str)
+{
+    constexpr std::string_view eop("\xe2\x88\x8e");
+    constexpr std::string_view filler{"\xe2\x80\x94"};
+    constexpr std::string_view space{"\xe2\x90\xa3"};
+    constexpr std::string_view newLine{"\xe2\x86\xb5"};
+    constexpr std::string_view tab("\xc2\xbb");
+    auto data = str.begin();
+    std::string::size_type last = 0;
+    const std::string::size_type size = str.size();
+    std::string ret;
+    ret.reserve(size);
+    for (std::string::size_type i = 0; i < size; i++) {
+        if ((str[i] & 0xc0) == 0xc0) {
+            char replaceWith = '\0';
+            std::string::size_type seqSize = 1;
+            std::string::size_type remSize = size - i;
+            if (remSize >= 3 && (filler.find(data + i, 0, 3) != eop.find(data + i, 0, 3))) {
+                seqSize = 3;
+            } else if (remSize >= 3 && space.find(data + i, 0, 3) != space.npos) {
+                replaceWith = ' ';
+                seqSize = 3;
+            } else if (remSize >= 3 && newLine.find(data + i, 0, 3) != newLine.npos) {
+                seqSize = 3;
+            } else if (remSize >= 2 && tab.find(data + i, 0, 2) != tab.npos) {
+                replaceWith = '\t';
+                seqSize = 2;
+            } else {
+                continue;
+            }
+            ret.append(str, last, i - last);
+            if (replaceWith != '\0') {
+                ret.append(&replaceWith, 1);
+            }
+            last = i + seqSize;
+            i += seqSize - 1;
+        }
+    }
+    ret.append(str.begin() + last, str.size() - last);
+    return ret;
+}
+
+bool parseJSON(EvalState & state, std::istream & s_, Value & v)
+{
+    auto parser = makeJSONSaxParser(state, v);
+    return nlohmann::json::sax_parse(s_, parser.get(), nlohmann::json::input_format_t::json, false);
+}
+
+Value parseJSONStream(EvalState & state, std::string_view json, std::function<FromYAMLFun> fromYAML)
+{
+    std::stringstream ss;
+    ss << json;
+    std::list<Value> list;
+    Value root, refJson;
+    std::streampos start = 0;
+    try {
+        while (ss.peek() != EOF && json.size() - ss.tellg() > 1) {
+            parseJSON(state, ss, refJson);
+            list.emplace_back(refJson);
+            // sanity check: builtins.fromJSON and builtins.fromYAML should return the same result when applied to a
+            // JSON string
+            root.mkString(std::string_view(json.begin() + start, ss.tellg() - start));
+            Value rymlJson = fromYAML(state, root, {});
+            EXPECT_EQ(printValue(state, refJson), printValue(state, rymlJson));
+            start = ss.tellg() + std::streampos(1);
+        }
+    } catch (const std::exception & e) {
+    }
+    if (list.size() == 1) {
+        root = *list.begin();
+    } else {
+        ListBuilder list_builder(state, list.size());
+        size_t i = 0;
+        for (auto val : list) {
+            *(list_builder[i++] = state.allocValue()) = val;
+        }
+        root.mkList(list_builder);
+    }
+    return root;
+}
+
+} /* namespace */
+
+namespace nix {
+// Testing the conversion from YAML
+
+class FromYAMLTest : public LibExprTest
+{
+protected:
+    static std::function<FromYAMLFun> getFromYAML()
+    {
+        static std::function<FromYAMLFun> fromYAML = []() {
+            for (const auto & primOp : *RegisterPrimOp::primOps) {
+                if (primOp.name == "__fromYAML") {
+                    auto primOpFun = primOp.fun;
+                    std::function<FromYAMLFun> function =
+                        [=](EvalState & state, Value yaml, std::optional<Value> options) {
+                            Value emptyOptions, result;
+                            auto bindings = state.buildBindings(0);
+                            emptyOptions.mkAttrs(bindings);
+                            Value * args[3] = {&yaml, options ? &*options : &emptyOptions, nullptr};
+                            primOpFun(state, noPos, args, result);
+                            return result;
+                        };
+                    return function;
+                }
+            }
+            ADD_FAILURE() << "The experimental feature \"fromYAML\" is not available";
+            return std::function<FromYAMLFun>();
+        }();
+        return fromYAML;
+    }
+
+    Value parseYAML(const char * str, std::optional<Value> options = {})
+    {
+        Value test;
+        test.mkString(str);
+        return getFromYAML()(state, test, options);
+    }
+
+    void execYAMLTest(std::string_view test)
+    {
+        auto fromYAML = getFromYAML();
+        Value testVal;
+        testVal.mkString(test);
+        Value testCases = fromYAML(state, testVal, {});
+        size_t ctr = 0;
+        std::string_view testName;
+        Value * json = nullptr;
+        for (auto testCase : testCases.listItems()) {
+            bool fail = false;
+            std::string_view yamlRaw;
+            for (auto attr = testCase->attrs()->begin(); attr != testCase->attrs()->end(); attr++) {
+                auto name = state.symbols[attr->name];
+                if (name == "json") {
+                    json = attr->value;
+                } else if (name == "yaml") {
+                    yamlRaw = attr->value->string_view();
+                } else if (name == "fail") {
+                    fail = attr->value->boolean();
+                } else if (name == "name") {
+                    testName = attr->value->string_view();
+                }
+            }
+            // extract expected result
+            Value jsonVal;
+            bool noJSON = !json || json->type() != nString;
+            if (!noJSON) {
+                std::string_view jsonStr = json->string_view();
+                // Test cases with "json: ''" are parsed as empty JSON and test cases with the value of the "json" node
+                // being a block scalar, have no JSON representation, if the block scalar contains the line "null"
+                // (indentation 0)
+                noJSON = jsonStr.empty()
+                         || (jsonStr != "null" && (jsonStr.starts_with("null") || jsonStr.ends_with("null")))
+                         || jsonStr.find("\nnull\n") != std::string_view::npos;
+                if (!noJSON) {
+                    jsonVal = parseJSONStream(state, jsonStr, fromYAML);
+                }
+            }
+            // extract the YAML to be parsed
+            std::string yamlStr = replaceUnicodePlaceholders(yamlRaw);
+            Value yaml, yamlVal;
+            yaml.mkString(yamlStr);
+            if (noJSON) {
+                EXPECT_THROW(yamlVal = fromYAML(state, yaml, {}), EvalError)
+                    << "Testcase #" << ctr
+                    << ": YAML has no JSON representation because of empty document or null key, parsed \""
+                    << printValue(state, yamlVal) << "\":\n"
+                    << yamlRaw;
+            } else if (!fail) {
+                yamlVal = fromYAML(state, yaml, {});
+                EXPECT_EQ(printValue(state, yamlVal), printValue(state, jsonVal))
+                    << "Testcase #" << ctr << ": Parsed YAML does not match expected JSON result:\n"
+                    << yamlRaw;
+            } else {
+                EXPECT_THROW(yamlVal = fromYAML(state, yaml, {}), EvalError)
+                    << "Testcase #" << ctr << " (" << testName << "): Parsing YAML has to throw an exception, but \""
+                    << printValue(state, yamlVal) << "\" was parsed:\n"
+                    << yamlRaw;
+            }
+            ctr++;
+        }
+    }
+};
+
+TEST_F(FromYAMLTest, NoContent)
+{
+    EXPECT_THROW(parseYAML(""), EvalError);
+}
+
+TEST_F(FromYAMLTest, Null)
+{
+    Value val = parseYAML("[ null, Null, NULL, ~, ]");
+    for (auto item : val.listItems()) {
+        EXPECT_EQ(item->type(), nNull);
+    }
+}
+
+TEST_F(FromYAMLTest, NaN)
+{
+    const char * nans[] = {".nan", ".NaN", ".NAN"};
+    for (auto str : nans) {
+        Value val = parseYAML(str);
+        ASSERT_EQ(val.type(), nFloat);
+        NixFloat _float = val.fpoint();
+        EXPECT_NE(_float, _float) << "'" << str << "' shall be parsed as NaN";
+    }
+    const char * strings[] = {"nan", "+nan", "-nan", "+.nan", "-.nan"};
+    for (auto str : strings) {
+        Value val = parseYAML(str);
+        ASSERT_EQ(val.type(), nString) << "'" << str << "' shall not be converted to a floating point type";
+        EXPECT_EQ(val.string_view(), std::string_view(str));
+    }
+}
+
+TEST_F(FromYAMLTest, Inf)
+{
+    NixFloat inf = std::numeric_limits<NixFloat>::infinity();
+    Value val = parseYAML("[ .INF, .Inf, .inf, +.INF, +.Inf, +.inf ]");
+    for (auto item : val.listItems()) {
+        ASSERT_EQ(item->type(), nFloat);
+        EXPECT_EQ(item->fpoint(), inf);
+    }
+    val = parseYAML("[ -.INF, -.Inf, -.inf ]");
+    for (auto item : val.listItems()) {
+        ASSERT_EQ(item->type(), nFloat);
+        EXPECT_EQ(item->fpoint(), -inf);
+    }
+    val = parseYAML("inf");
+    ASSERT_EQ(val.type(), nString) << "'inf' shall not be converted to a floating point type";
+    EXPECT_EQ(val.string_view(), "inf");
+}
+
+TEST_F(FromYAMLTest, Int)
+{
+    Value val = parseYAML("[ 1, +1, 0x1, 0o1 ]");
+    for (auto item : val.listItems()) {
+        ASSERT_EQ(item->type(), nInt);
+        EXPECT_EQ(item->integer(), NixInt(1));
+    }
+
+    const char * strings[] = {"+", "0b1", "0B1", "0O1", "0X1", "+0b1", "-0b1", "+0o1", "-0o1", "+0x1", "-0x1"};
+    for (auto str : strings) {
+        Value val = parseYAML(str);
+        ASSERT_EQ(val.type(), nString) << "'" << str << "' shall not be converted to an integer";
+        EXPECT_EQ(val.string_view(), str);
+    }
+}
+
+TEST_F(FromYAMLTest, Float)
+{
+    Value val = parseYAML("[ !!float 1, !!float 0x1, !!float 0o1, 1., +1., .1e1, +.1e1, 1.0, 10e-1, 10.e-1 ]");
+    for (auto item : val.listItems()) {
+        ASSERT_EQ(item->type(), nFloat);
+        EXPECT_EQ(item->fpoint(), 1.);
+    }
+    val = parseYAML("!!float -0");
+    ASSERT_EQ(val.type(), nFloat);
+    EXPECT_EQ(1. / val.fpoint(), 1. / -0.) << "\"!!float -0\" shall be parsed as -0.0";
+
+    const char * strings[] = {"0x1.", "0X1.", "0b1.", "0B1.", "0o1.", "0O1"};
+    for (auto str : strings) {
+        Value val = parseYAML(str);
+        ASSERT_EQ(val.type(), nString) << "'" << str << "' shall not be converted to a float";
+        EXPECT_EQ(val.string_view(), str);
+    }
+}
+
+TEST_F(FromYAMLTest, TrueYAML1_2)
+{
+    Value val = parseYAML("[ true, True, TRUE ]");
+    for (auto item : val.listItems()) {
+        ASSERT_EQ(item->type(), nBool);
+        EXPECT_TRUE(item->boolean());
+    }
+    const char * strings[] = {"y", "Y", "on", "On", "ON", "yes", "Yes", "YES"};
+    for (auto str : strings) {
+        Value val = parseYAML(str);
+        ASSERT_EQ(val.type(), nString) << "'" << str << "' shall not be converted to a boolean";
+        EXPECT_EQ(val.string_view(), std::string_view(str));
+    }
+}
+
+TEST_F(FromYAMLTest, TrueYAML1_1)
+{
+    Value options;
+    auto bindings = state.buildBindings(1);
+    bindings.alloc("useBoolYAML1_1").mkBool(true);
+    options.mkAttrs(bindings);
+
+    Value val = parseYAML("[ true, True, TRUE, y, Y, on, On, ON, yes, Yes, YES ]", options);
+    for (auto item : val.listItems()) {
+        ASSERT_EQ(item->type(), nBool);
+        EXPECT_TRUE(item->boolean());
+    }
+}
+
+TEST_F(FromYAMLTest, FalseYAML1_2)
+{
+    Value val = parseYAML("[ false, False, FALSE ]");
+    for (auto item : val.listItems()) {
+        ASSERT_EQ(item->type(), nBool);
+        EXPECT_FALSE(item->boolean());
+    }
+    const char * strings[] = {"n", "N", "no", "No", "NO", "off", "Off", "OFF"};
+    for (auto str : strings) {
+        Value val = parseYAML(str);
+        ASSERT_EQ(val.type(), nString) << "'" << str << "' shall not be converted to a boolean";
+        EXPECT_EQ(val.string_view(), std::string_view(str));
+    }
+}
+
+TEST_F(FromYAMLTest, FalseYAML1_1)
+{
+    Value options;
+    auto bindings = state.buildBindings(1);
+    bindings.alloc("useBoolYAML1_1").mkBool(true);
+    options.mkAttrs(bindings);
+
+    Value val = parseYAML("[ false, False, FALSE, n, N, no, No, NO, off, Off, OFF ]", options);
+    for (auto item : val.listItems()) {
+        ASSERT_EQ(item->type(), nBool);
+        EXPECT_FALSE(item->boolean());
+    }
+}
+
+TEST_F(FromYAMLTest, QuotedString)
+{
+    const char * strings[] = {
+        "\"null\"",
+        "\"~\"",
+        "\"\"",
+        "\".inf\"",
+        "\"+.inf\"",
+        "\"-.inf\"",
+        "\".nan\"",
+        "\"true\"",
+        "\"false\"",
+        "\"1\"",
+        "\"+1\"",
+        "\"-1\"",
+        "\"1.0\""};
+    for (auto str : strings) {
+        Value val = parseYAML(str);
+        ASSERT_EQ(val.type(), nString) << "'" << str << "' shall be parsed as string";
+        EXPECT_EQ(val.string_view(), std::string_view(&str[1], strlen(str) - 2));
+    }
+}
+
+TEST_F(FromYAMLTest, Map)
+{
+    EXPECT_THROW(parseYAML("{ \"2\": 2, 2: null }"), EvalError) << "non-unique keys";
+}
+
+} /* namespace nix */
+
+// include auto-generated header
+#  include "./yaml-test-suite.hh"
+
+#endif