Render fetcher info in primop again

src/libexpr/eval.cc

@@ -500,7 +500,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
         v.mkApp(vPrimOp, vPrimOp);
         return addConstant(primOp.name, v, {
             .type = nThunk, // FIXME
-            .doc = primOp.doc,
+            .doc = primOp.doc.c_str(),
         });
     }
 
@@ -527,13 +527,14 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
     if (v.isPrimOp()) {
         auto v2 = &v;
-        if (auto * doc = v2->primOp()->doc)
+        auto & doc = v2->primOp()->doc;
+        if (doc != "")
             return Doc {
                 .pos = {},
                 .name = v2->primOp()->name,
                 .arity = v2->primOp()->arity,
                 .args = v2->primOp()->args,
-                .doc = doc,
+                .doc = doc.c_str(),
             };
     }
     if (v.isLambda()) {

src/libexpr/eval.hh

@@ -88,7 +88,7 @@ struct PrimOp
     /**
      * Optional free-form documentation about the primop.
      */
-    const char * doc = nullptr;
+    const std::string doc = "";
 
     /**
      * Add a trace item, `while calling the '<name>' builtin`

src/libexpr/local.mk

@@ -20,7 +20,7 @@ libexpr_CXXFLAGS += \
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS)
+libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS) $(CMARK_LIBS)
 ifdef HOST_LINUX
   libexpr_LDFLAGS += -ldl
 endif

src/libexpr/primops/fetchTree.cc

@@ -8,6 +8,7 @@
 #include "registry.hh"
 #include "tarball.hh"
 #include "url.hh"
+#include "cmark-cpp.hh"
 #include "value-to-json.hh"
 #include "fetch-to-store.hh"
 
@@ -210,7 +211,15 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args,
 static RegisterPrimOp primop_fetchTree({
     .name = "fetchTree",
     .args = {"input"},
-    .doc = R"(
+    .doc =  []() -> std::string {
+        using namespace cmark;
+
+        // Stores strings referenced by AST. Deallocate after rendering.
+        std::vector<std::string> textArena;
+
+        auto root = node_new(CMARK_NODE_DOCUMENT);
+
+        auto & before = textArena.emplace_back(stripIndentation(R"(
           Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
 
           - the resulting fixed-output [store path](@docroot@/store/store-path.md)
@@ -250,7 +259,45 @@ static RegisterPrimOp primop_fetchTree({
           <!-- TODO: It would be soooo much more predictable to work with (and
           document) if `fetchTree` was a curried call with the first parameter for
           `type` or an attribute like `builtins.fetchTree.git`! -->
+        )"));
+        parse_document(*root, before, CMARK_OPT_DEFAULT);
 
+        auto & schemes = node_append_child(*root, node_new(CMARK_NODE_LIST));
+
+        for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
+            auto & s = node_append_child(schemes, node_new(CMARK_NODE_ITEM));
+            {
+                auto & name_p = node_append_child(s, node_new(CMARK_NODE_PARAGRAPH));
+                auto & name = node_append_child(name_p, node_new(CMARK_NODE_TEXT));
+                node_set_literal(name, schemeName.data());
+            }
+            parse_document(s, scheme->schemeDescription(), CMARK_OPT_DEFAULT);
+
+            auto & attrs = node_append_child(s, node_new(CMARK_NODE_LIST));
+            for (const auto & [attrName, attribute] : scheme->allowedAttrs()) {
+                auto & a = node_append_child(attrs, node_new(CMARK_NODE_ITEM));
+                {
+                    auto & name_info = node_append_child(a, node_new(CMARK_NODE_PARAGRAPH));
+                    {
+                        auto & name = node_append_child(name_info, node_new(CMARK_NODE_CODE));
+                        auto & name_t = textArena.emplace_back(attrName);
+                        node_set_literal(name, name_t.c_str());
+                    }
+                    auto & info = node_append_child(name_info, node_new(CMARK_NODE_TEXT));
+                    auto & header = textArena.emplace_back(std::string { }
+                        + " (" + attribute.type
+                        + ", " + (attribute.required ? "required" : "optional")
+                        + ")");
+                    node_set_literal(info, header.c_str());
+                }
+                {
+                    auto & doc = textArena.emplace_back(stripIndentation(attribute.doc));
+                    parse_document(a, doc, CMARK_OPT_DEFAULT);
+                }
+            }
+        }
+
+        auto & after = textArena.emplace_back(stripIndentation(R"(
           The following input types are still subject to change:
 
           - `"path"`
@@ -295,7 +342,13 @@ static RegisterPrimOp primop_fetchTree({
           >   ```nix
           >   builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
           >   ```
-    )",
+        )"));
+        parse_document(*root, after, CMARK_OPT_DEFAULT);
+
+        auto p = render_commonmark(*root, CMARK_OPT_DEFAULT, 0);
+        assert(p);
+        return { &*p };
+    }(),
     .fun = prim_fetchTree,
     .experimentalFeature = Xp::FetchTree,
 });

src/libutil/cmark-cpp.hh

@@ -0,0 +1,85 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "util.hh"
+
+#include <cmark.h>
+
+namespace nix::cmark {
+
+using Node = struct cmark_node;
+using NodeType = cmark_node_type;
+using ListType = cmark_list_type;
+
+using Iter = struct cmark_iter;
+
+struct Deleter
+{
+    void operator () (Node * ptr) { cmark_node_free(ptr); }
+    void operator () (Iter * ptr) { cmark_iter_free(ptr); }
+};
+
+template <typename T>
+using UniquePtr = std::unique_ptr<Node, Deleter>;
+
+static inline void parse_document(Node & root, std::string_view s, int options)
+{
+    cmark_parser * parser = cmark_parser_new_with_mem_into_root(
+        options,
+        cmark_get_default_mem_allocator(),
+        &root);
+    cmark_parser_feed(parser, s.data(), s.size());
+    (void) cmark_parser_finish(parser);
+    cmark_parser_free(parser);
+}
+
+static inline UniquePtr<Node> parse_document(std::string_view s, int options)
+{
+    return UniquePtr<Node> {
+        cmark_parse_document(s.data(), s.size(), options)
+    };
+}
+
+static inline std::unique_ptr<char, FreeDeleter> render_commonmark(Node & root, int options, int width)
+{
+    return std::unique_ptr<char, FreeDeleter> {
+        cmark_render_commonmark(&root, options, width)
+    };
+}
+
+static inline std::unique_ptr<char, FreeDeleter> render_xml(Node & root, int options)
+{
+    return std::unique_ptr<char, FreeDeleter> {
+        cmark_render_xml(&root, options)
+    };
+}
+
+static inline UniquePtr<Node> node_new(NodeType type)
+{
+    return UniquePtr<Node> {
+        cmark_node_new(type)
+    };
+}
+
+/**
+ * The parent takes ownership
+ */
+static inline Node & node_append_child(Node & node, UniquePtr<Node> child)
+{
+    auto status = (bool) cmark_node_append_child(&node, &*child);
+    assert(status);
+    return *child.release();
+}
+
+static inline bool node_set_literal(Node & node, const char * content)
+{
+    return (bool) cmark_node_set_literal(&node, content);
+}
+
+static inline bool node_set_list_type(Node & node, ListType type)
+{
+    return (bool) cmark_node_set_list_type(&node, type);
+}
+
+}

src/libutil/util.hh

@@ -332,6 +332,16 @@ template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
 std::string showBytes(uint64_t bytes);
 
 
+/**
+ * For using `std::unique` with C functions.
+ */
+struct FreeDeleter
+{
+    template <typename T>
+    void operator()(T *p) const { std::free(p); }
+};
+
+
 /**
  * Provide an addition operator between strings and string_views
  * inexplicably omitted from the standard library.

src/nix/main.cc

@@ -458,7 +458,7 @@ void mainWrapped(int argc, char * * argv)
             auto b = nlohmann::json::object();
             if (!builtin.value->isPrimOp()) continue;
             auto primOp = builtin.value->primOp();
-            if (!primOp->doc) continue;
+            if (primOp->doc == "") continue;
             b["args"] = primOp->args;
             b["doc"] = trim(stripIndentation(primOp->doc));
             if (primOp->experimentalFeature)