NixOS/nix
NixOS/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2024-10-18 00:16:11 -04:00
Code Activity

fromYAML: format

Browse source
This commit is contained in:
Philipp Otterbein 2024-09-19 03:59:20 +02:00
parent 3e6b389234
commit e676131083
4 changed files with 2015 additions and 1658 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

120
src/libexpr/primops/fromYAML.cc
View file

@ -1,39 +1,37 @@
#ifdef HAVE_RYML
#include "primops.hh"
#include "eval-inline.hh"
#include <ryml.hpp>
#include <c4/format.hpp>
#include <c4/std/string.hpp>
# include "primops.hh"
# include "eval-inline.hh"
# include <ryml.hpp>
# include <c4/format.hpp>
# include <c4/std/string.hpp>
namespace nix {
struct NixContext {
struct NixContext
{
EvalState & state;
const PosIdx pos;
std::string_view yaml;
};
static void s_error [[ noreturn ]] (const char* msg, size_t len, ryml::Location, void *nixContext)
static void s_error [[noreturn]] (const char * msg, size_t len, ryml::Location, void * nixContext)
{
auto context = static_cast<const NixContext *>(nixContext);
if (context) {
throw EvalError(context->state, ErrorInfo{
.msg = fmt("while parsing the YAML string '%1%':\n\n%2%",
context->yaml, std::string_view(msg, len)),
.pos = context->state.positions[context->pos]
});
throw EvalError(
context->state,
ErrorInfo{
.msg = fmt("while parsing the YAML string '%1%':\n\n%2%", context->yaml, std::string_view(msg, len)),
.pos = context->state.positions[context->pos]});
} else {
throw Error({
.msg = fmt("failed assertion in rapidyaml library:\n\n%1%",
std::string_view(msg, len))
});
throw Error({.msg = fmt("failed assertion in rapidyaml library:\n\n%1%", std::string_view(msg, len))});
}
}
static void visitYAMLNode(NixContext & context, Value & v, ryml::ConstNodeRef t) {
static void visitYAMLNode(NixContext & context, Value & v, ryml::ConstNodeRef t)
{
bool fail = false;
if (t.is_map()) {
@ -44,7 +42,9 @@ static void visitYAMLNode(NixContext & context, Value & v, ryml::ConstNodeRef t)
auto tag = ryml::to_tag(child.key_tag());
if (tag != ryml::TAG_NONE && tag != ryml::TAG_STR) {
auto msg = ryml::formatrs<std::string>(
"Error: Nix supports string keys only, but the key '{}' has the tag '{}'", child.key(), child.key_tag());
"Error: Nix supports string keys only, but the key '{}' has the tag '{}'",
child.key(),
child.key_tag());
s_error(msg.data(), msg.size(), {}, &context);
}
} else if (child.key_is_null()) {
@ -80,7 +80,7 @@ static void visitYAMLNode(NixContext & context, Value & v, ryml::ConstNodeRef t)
valTag = tag == "!" && !isNull ? ryml::TAG_STR : ryml::to_tag(tag);
}
auto scalarTypeCheck = [=] (ryml::YamlTag_e tag) {
auto scalarTypeCheck = [=](ryml::YamlTag_e tag) {
return valTag == ryml::TAG_NONE ? !isQuoted : valTag == tag;
};
@ -111,53 +111,51 @@ static void visitYAMLNode(NixContext & context, Value & v, ryml::ConstNodeRef t)
}
}
static RegisterPrimOp primop_fromYAML({
.name = "__fromYAML",
.args = {"e"},
.doc = R"(
Convert a YAML 1.2 string to a Nix value, if a conversion is possible. For example,
static RegisterPrimOp primop_fromYAML(
{.name = "__fromYAML",
.args = {"e"},
.doc = R"(
Convert a YAML 1.2 string to a Nix value, if a conversion is possible. For example,
```nix
builtins.fromYAML ''{x: [1, 2, 3], y: !!str null, z: null}''
```
```nix
builtins.fromYAML ''{x: [1, 2, 3], y: !!str null, z: null}''
```
returns the value `{ x = [ 1 2 3 ]; y = "null"; z = null; }`.
returns the value `{ x = [ 1 2 3 ]; y = "null"; z = null; }`.
Maps are converted to attribute sets, but only strings are supported as keys.
Maps are converted to attribute sets, but only strings are supported as keys.
Scalars are converted to the type specified by their optional value tag. Parsing fails if a conversion is not possible.
Not all YAML types are supported by Nix, e.g. Nix has no binary and timestamp data types, so that parsing of YAML with any of these types fails.
Custom tags are ignored and a stream with multiple documents is mapped to a list except when the stream contains a single document.
)",
.fun = [] (EvalState & state, const PosIdx pos, Value * * args, Value & val) {
auto yaml = state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.fromYAML");
Scalars are converted to the type specified by their optional value tag. Parsing fails if a conversion is not possible.
Not all YAML types are supported by Nix, e.g. Nix has no binary and timestamp data types, so that parsing of YAML with any of these types fails.
Custom tags are ignored and a stream with multiple documents is mapped to a list except when the stream contains a single document.
)",
.fun =
[](EvalState & state, const PosIdx pos, Value ** args, Value & val) {
auto yaml =
state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.fromYAML");
NixContext context{
.state = state,
.pos = pos,
.yaml = yaml
};
ryml::Callbacks callbacks;
callbacks.m_error = s_error;
ryml::set_callbacks(callbacks);
callbacks.m_user_data = &context;
ryml::EventHandlerTree evth(callbacks);
ryml::Parser parser(&evth);
ryml::Tree tree = ryml::parse_in_arena(&parser, ryml::csubstr(yaml.begin(), yaml.size()));
tree.resolve(); // resolve references
tree.resolve_tags();
NixContext context{.state = state, .pos = pos, .yaml = yaml};
ryml::Callbacks callbacks;
callbacks.m_error = s_error;
ryml::set_callbacks(callbacks);
callbacks.m_user_data = &context;
ryml::EventHandlerTree evth(callbacks);
ryml::Parser parser(&evth);
ryml::Tree tree = ryml::parse_in_arena(&parser, ryml::csubstr(yaml.begin(), yaml.size()));
tree.resolve(); // resolve references
tree.resolve_tags();
auto root = tree.crootref();
if (!root.has_val() && !root.is_map() && !root.is_seq()) {
std::string msg = "YAML string has no content";
s_error(msg.data(), msg.size(), {}, &context);
}
if (root.is_stream() && root.num_children() == 1 && root.child(0).is_doc())
root = root.child(0);
visitYAMLNode(context, val, root);
},
.experimentalFeature = Xp::FromYaml
});
auto root = tree.crootref();
if (!root.has_val() && !root.is_map() && !root.is_seq()) {
std::string msg = "YAML string has no content";
s_error(msg.data(), msg.size(), {}, &context);
}
if (root.is_stream() && root.num_children() == 1 && root.child(0).is_doc()) {
root = root.child(0);
}
visitYAMLNode(context, val, root);
},
.experimentalFeature = Xp::FromYaml});
}

7
tests/unit/libexpr/compose-yaml-test-suite.sh
View file

@ -41,8 +41,6 @@ EOL
echo
done
echo
echo
echo "namespace nix {"
for f in "$1"/src/*.yaml; do
testname="$(basename "${f}" .yaml)"
@ -57,9 +55,10 @@ for f in "$1"/src/*.yaml; do
skip="true"
;;
esac
echo "TEST_F(${testclass}, T_${testname}) {"
echo "TEST_F(${testclass}, T_${testname})"
echo "{"
if [ "${testname}" = "565N" ]; then
echo " ASSERT_THROW(${testmethod}(T_${testname}),EvalError); // nix has no binary data type"
echo " ASSERT_THROW(${testmethod}(T_${testname}), EvalError); // nix has no binary data type"
else
if [ "${skip}" = "true" ]; then
echo " GTEST_SKIP() << \"Reason: Invalid yaml is parsed successfully\";"

3228
tests/unit/libexpr/yaml-test-suite.hh
View file

File diff suppressed because it is too large Load diff

318
tests/unit/libexpr/yaml.cc
View file

@ -1,174 +1,186 @@
#ifdef HAVE_RYML
#include "tests/libexpr.hh"
#include "primops.hh"
# include "tests/libexpr.hh"
# include "primops.hh"
// access to the json sax parser is required
#include "json-to-value-sax.hh"
# include "json-to-value-sax.hh"
namespace nix {
// Testing the conversion from YAML
/* replacement of non-ascii unicode characters, which indicate the presence of certain characters that would be otherwise hard to read */
static std::string replaceUnicodePlaceholders(std::string_view str) {
constexpr std::string_view eop("\xe2\x88\x8e");
constexpr std::string_view filler{"\xe2\x80\x94"};
constexpr std::string_view space{"\xe2\x90\xa3"};
constexpr std::string_view newLine{"\xe2\x86\xb5"};
constexpr std::string_view tab("\xc2\xbb");
auto data = str.begin();
std::string::size_type last = 0;
const std::string::size_type size = str.size();
std::string ret;
ret.reserve(size);
for (std::string::size_type i = 0; i < size; i++) {
if ((str[i] & 0xc0) == 0xc0) {
char replaceWith = '\0';
std::string::size_type seqSize = 1;
std::string::size_type remSize = size - i;
if (remSize >= 3 && (filler.find(data + i, 0, 3) != eop.find(data + i, 0, 3))) {
seqSize = 3;
} else if (remSize >= 3 && space.find(data + i, 0, 3) != space.npos) {
replaceWith = ' ';
seqSize = 3;
} else if (remSize >= 3 && newLine.find(data + i, 0, 3) != newLine.npos) {
seqSize = 3;
} else if (remSize >= 2 && tab.find(data + i, 0, 2) != tab.npos) {
replaceWith = '\t';
seqSize = 2;
} else {
continue;
}
ret.append(str, last, i - last);
if (replaceWith != '\0') {
ret.append(&replaceWith, 1);
}
last = i + seqSize;
i += seqSize - 1;
/* replacement of non-ascii unicode characters, which indicate the presence of certain characters that would be
* otherwise hard to read */
static std::string replaceUnicodePlaceholders(std::string_view str)
{
constexpr std::string_view eop("\xe2\x88\x8e");
constexpr std::string_view filler{"\xe2\x80\x94"};
constexpr std::string_view space{"\xe2\x90\xa3"};
constexpr std::string_view newLine{"\xe2\x86\xb5"};
constexpr std::string_view tab("\xc2\xbb");
auto data = str.begin();
std::string::size_type last = 0;
const std::string::size_type size = str.size();
std::string ret;
ret.reserve(size);
for (std::string::size_type i = 0; i < size; i++) {
if ((str[i] & 0xc0) == 0xc0) {
char replaceWith = '\0';
std::string::size_type seqSize = 1;
std::string::size_type remSize = size - i;
if (remSize >= 3 && (filler.find(data + i, 0, 3) != eop.find(data + i, 0, 3))) {
seqSize = 3;
} else if (remSize >= 3 && space.find(data + i, 0, 3) != space.npos) {
replaceWith = ' ';
seqSize = 3;
} else if (remSize >= 3 && newLine.find(data + i, 0, 3) != newLine.npos) {
seqSize = 3;
} else if (remSize >= 2 && tab.find(data + i, 0, 2) != tab.npos) {
replaceWith = '\t';
seqSize = 2;
} else {
continue;
}
ret.append(str, last, i - last);
if (replaceWith != '\0') {
ret.append(&replaceWith, 1);
}
last = i + seqSize;
i += seqSize - 1;
}
ret.append(str.begin() + last, str.size() - last);
return ret;
}
ret.append(str.begin() + last, str.size() - last);
return ret;
}
static bool parseJSON(EvalState & state, std::istream & s_, Value & v)
static bool parseJSON(EvalState & state, std::istream & s_, Value & v)
{
auto parser = makeJSONSaxParser(state, v);
return nlohmann::json::sax_parse(s_, parser.get(), nlohmann::json::input_format_t::json, false);
}
static Value parseJSONStream(EvalState & state, std::string_view json, std::function<PrimOpFun> fromYAML)
{
std::stringstream ss;
ss << json;
std::list<Value> list;
Value root, refJson;
Value * pRoot = &root, rymlJson;
std::streampos start = 0;
try {
while (ss.peek() != EOF && json.size() - ss.tellg() > 1) {
parseJSON(state, ss, refJson);
list.emplace_back(refJson);
// sanity check: builtins.fromJSON and builtins.fromYAML should return the same result when applied to a
// JSON string
root.mkString(std::string_view(json.begin() + start, ss.tellg() - start));
fromYAML(state, noPos, &pRoot, rymlJson);
EXPECT_EQ(printValue(state, refJson), printValue(state, rymlJson));
start = ss.tellg() + std::streampos(1);
}
} catch (const std::exception & e) {
}
if (list.size() == 1) {
root = *list.begin();
} else {
ListBuilder list_builder(state, list.size());
size_t i = 0;
for (auto val : list) {
*(list_builder[i++] = state.allocValue()) = val;
}
root.mkList(list_builder);
}
return root;
}
class FromYAMLTest : public LibExprTest
{
protected:
void execYAMLTest(std::string_view test)
{
auto parser = makeJSONSaxParser(state, v);
return nlohmann::json::sax_parse(s_, parser.get(), nlohmann::json::input_format_t::json, false);
}
static Value parseJSONStream(EvalState & state, std::string_view json, std::function<PrimOpFun> fromYAML) {
std::stringstream ss;
ss << json;
std::list<Value> list;
Value root, refJson;
Value *pRoot = &root, rymlJson;
std::streampos start = 0;
try {
while (ss.peek() != EOF && json.size() - ss.tellg() > 1) {
parseJSON(state, ss, refJson);
list.emplace_back(refJson);
// sanity check: builtins.fromJSON and builtins.fromYAML should return the same result when applied to a JSON string
root.mkString(std::string_view(json.begin() + start, ss.tellg() - start));
fromYAML(state, noPos, &pRoot, rymlJson);
EXPECT_EQ(printValue(state, refJson), printValue(state, rymlJson));
start = ss.tellg() + std::streampos(1);
}
} catch (const std::exception &e) {
}
if (list.size() == 1) {
root = *list.begin();
} else {
ListBuilder list_builder(state, list.size());
size_t i = 0;
for (auto val : list) {
*(list_builder[i++] = state.allocValue()) = val;
}
root.mkList(list_builder);
}
return root;
}
class FromYAMLTest : public LibExprTest {
protected:
void execYAMLTest(std::string_view test) {
std::function<PrimOpFun> fromYAML = [] () {
for (const auto & primOp : *RegisterPrimOp::primOps) {
if (primOp.name == "__fromYAML") {
return primOp.fun;
}
}
return std::function<PrimOpFun>();
} ();
EXPECT_FALSE(fromYAML == nullptr) << "The experimental feature \"fromYAML\" is not available";
Value testCases, testVal;
Value *pTestVal = &testVal;
testVal.mkString(test);
fromYAML(state, noPos, &pTestVal, testCases);
size_t ctr = 0;
std::string_view testName;
Value *json = nullptr;
for (auto testCase : testCases.listItems()) {
bool fail = false;
std::string_view yamlRaw;
for (auto attr = testCase->attrs()->begin(); attr != testCase->attrs()->end(); attr++) {
auto name = state.symbols[attr->name];
if (name == "json") {
json = attr->value;
} else if (name == "yaml") {
yamlRaw = state.forceStringNoCtx(*attr->value, noPos, "while interpreting the \"yaml\" field as string");
} else if (name == "fail") {
fail = state.forceBool(*attr->value, noPos, "while interpreting the \"fail\" field as bool");
} else if (name == "name") {
testName = state.forceStringNoCtx(*attr->value, noPos, "while interpreting the \"name\" field as string");
}
}
// extract expected result
Value jsonVal;
bool nullJSON = json && json->type() == nNull;
bool emptyJSON = !nullJSON;
if (json && !nullJSON) {
std::string_view jsonStr = state.forceStringNoCtx(*json, noPos, "while interpreting the \"json\" field as string");
emptyJSON = jsonStr.empty();
if (!emptyJSON) {
jsonVal = parseJSONStream(state, jsonStr, fromYAML);
jsonStr = printValue(state, jsonVal);
}
}
// extract the YAML to be parsed
std::string yamlStr = replaceUnicodePlaceholders(yamlRaw);
Value yaml, yamlVal;
Value *pYaml = &yaml;
yaml.mkString(yamlStr);
if (!fail) {
if (emptyJSON) {
EXPECT_THROW(
fromYAML(state, noPos, &pYaml, yamlVal),
EvalError) << "Testcase #" << ctr << ": Expected empty YAML, which should throw an exception, parsed \"" << printValue(state, yamlVal) << "\":\n" << yamlRaw;
} else {
fromYAML(state, noPos, &pYaml, yamlVal);
if (nullJSON) {
EXPECT_TRUE(yamlVal.type() == nNull) << "Testcase #" << ctr << ": Expected null YAML:\n" << yamlStr;
} else {
EXPECT_EQ(printValue(state, yamlVal), printValue(state, jsonVal)) << "Testcase #" << ctr << ": Parsed YAML does not match expected JSON result:\n" << yamlRaw;
}
}
} else {
EXPECT_THROW(
fromYAML(state, noPos, &pYaml, yamlVal),
EvalError) << "Testcase #" << ctr << " (" << testName << "): Parsing YAML has to throw an exception, but \"" << printValue(state, yamlVal) << "\" was parsed:\n" << yamlRaw;
}
ctr++;
std::function<PrimOpFun> fromYAML = []() {
for (const auto & primOp : *RegisterPrimOp::primOps) {
if (primOp.name == "__fromYAML") {
return primOp.fun;
}
}
};
return std::function<PrimOpFun>();
}();
EXPECT_FALSE(fromYAML == nullptr) << "The experimental feature \"fromYAML\" is not available";
Value testCases, testVal;
Value * pTestVal = &testVal;
testVal.mkString(test);
fromYAML(state, noPos, &pTestVal, testCases);
size_t ctr = 0;
std::string_view testName;
Value * json = nullptr;
for (auto testCase : testCases.listItems()) {
bool fail = false;
std::string_view yamlRaw;
for (auto attr = testCase->attrs()->begin(); attr != testCase->attrs()->end(); attr++) {
auto name = state.symbols[attr->name];
if (name == "json") {
json = attr->value;
} else if (name == "yaml") {
yamlRaw =
state.forceStringNoCtx(*attr->value, noPos, "while interpreting the \"yaml\" field as string");
} else if (name == "fail") {
fail = state.forceBool(*attr->value, noPos, "while interpreting the \"fail\" field as bool");
} else if (name == "name") {
testName =
state.forceStringNoCtx(*attr->value, noPos, "while interpreting the \"name\" field as string");
}
}
// extract expected result
Value jsonVal;
bool nullJSON = json && json->type() == nNull;
bool emptyJSON = !nullJSON;
if (json && !nullJSON) {
std::string_view jsonStr =
state.forceStringNoCtx(*json, noPos, "while interpreting the \"json\" field as string");
emptyJSON = jsonStr.empty();
if (!emptyJSON) {
jsonVal = parseJSONStream(state, jsonStr, fromYAML);
jsonStr = printValue(state, jsonVal);
}
}
// extract the YAML to be parsed
std::string yamlStr = replaceUnicodePlaceholders(yamlRaw);
Value yaml, yamlVal;
Value * pYaml = &yaml;
yaml.mkString(yamlStr);
if (!fail) {
if (emptyJSON) {
EXPECT_THROW(fromYAML(state, noPos, &pYaml, yamlVal), EvalError)
<< "Testcase #" << ctr << ": Expected empty YAML, which should throw an exception, parsed \""
<< printValue(state, yamlVal) << "\":\n"
<< yamlRaw;
} else {
fromYAML(state, noPos, &pYaml, yamlVal);
if (nullJSON) {
EXPECT_TRUE(yamlVal.type() == nNull) << "Testcase #" << ctr << ": Expected null YAML:\n"
<< yamlStr;
} else {
EXPECT_EQ(printValue(state, yamlVal), printValue(state, jsonVal))
<< "Testcase #" << ctr << ": Parsed YAML does not match expected JSON result:\n"
<< yamlRaw;
}
}
} else {
EXPECT_THROW(fromYAML(state, noPos, &pYaml, yamlVal), EvalError)
<< "Testcase #" << ctr << " (" << testName << "): Parsing YAML has to throw an exception, but \""
<< printValue(state, yamlVal) << "\" was parsed:\n"
<< yamlRaw;
}
ctr++;
}
}
};
} /* namespace nix */
// include auto-generated header
#include "./yaml-test-suite.hh"
# include "./yaml-test-suite.hh"
#endif