Render fetcher info in primop again

src/libexpr/eval.cc

@@ -500,7 +500,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
         v.mkApp(vPrimOp, vPrimOp);
         return addConstant(primOp.name, v, {
             .type = nThunk, // FIXME
-            .doc = primOp.doc,
+            .doc = primOp.doc.c_str(),
         });
     }
 
@@ -527,13 +527,14 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
     if (v.isPrimOp()) {
         auto v2 = &v;
-        if (auto * doc = v2->primOp()->doc)
+        auto & doc = v2->primOp()->doc;
+        if (doc != "")
             return Doc {
                 .pos = {},
                 .name = v2->primOp()->name,
                 .arity = v2->primOp()->arity,
                 .args = v2->primOp()->args,
-                .doc = doc,
+                .doc = doc.c_str(),
             };
     }
     if (v.isLambda()) {

src/libexpr/eval.hh

@@ -88,7 +88,7 @@ struct PrimOp
     /**
      * Optional free-form documentation about the primop.
      */
-    const char * doc = nullptr;
+    const std::string doc = "";
 
     /**
      * Add a trace item, `while calling the '<name>' builtin`

src/libexpr/local.mk

@@ -20,7 +20,7 @@ libexpr_CXXFLAGS += \
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS)
+libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS) $(CMARK_LIBS)
 ifdef HOST_LINUX
   libexpr_LDFLAGS += -ldl
 endif

src/libexpr/primops/fetchTree.cc

@@ -8,6 +8,7 @@
 #include "registry.hh"
 #include "tarball.hh"
 #include "url.hh"
+#include "cmark-cpp.hh"
 #include "value-to-json.hh"
 #include "fetch-to-store.hh"
 
@@ -210,92 +211,144 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args,
 static RegisterPrimOp primop_fetchTree({
     .name = "fetchTree",
     .args = {"input"},
-    .doc = R"(
+    .doc =  []() -> std::string {
-      Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
+        using namespace cmark;
 
-      - the resulting fixed-output [store path](@docroot@/store/store-path.md)
+        // Stores strings referenced by AST. Deallocate after rendering.
-      - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
+        std::vector<std::string> textArena;
-      - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
 
-      *input* must be an attribute set with the following attributes:
+        auto root = node_new(CMARK_NODE_DOCUMENT);
 
-      - `type` (String, required)
+        auto & before = textArena.emplace_back(stripIndentation(R"(
+          Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
 
-        One of the [supported source types](#source-types).
+          - the resulting fixed-output [store path](@docroot@/store/store-path.md)
-        This determines other required and allowed input attributes.
+          - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
+          - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
 
-      - `narHash` (String, optional)
+          *input* must be an attribute set with the following attributes:
 
-        The `narHash` parameter can be used to substitute the source of the tree.
+          - `type` (String, required)
-        It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
-        If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
 
-      A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
+            One of the [supported source types](#source-types).
-      That is, `fetchTree` is idempotent.
+            This determines other required and allowed input attributes.
 
-      Downloads are cached in `$XDG_CACHE_HOME/nix`.
+          - `narHash` (String, optional)
-      The remote source will be fetched from the network if both are true:
-      - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
 
-        > **Note**
+            The `narHash` parameter can be used to substitute the source of the tree.
-        >
+            It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
-        > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
+            If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
 
-      - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
+          A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
+          That is, `fetchTree` is idempotent.
 
-      ## Source types
+          Downloads are cached in `$XDG_CACHE_HOME/nix`.
+          The remote source will be fetched from the network if both are true:
+          - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
 
-      The following source types and associated input attributes are supported.
+            > **Note**
+            >
+            > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
 
-      <!-- TODO: It would be soooo much more predictable to work with (and
+          - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
-      document) if `fetchTree` was a curried call with the first parameter for
-      `type` or an attribute like `builtins.fetchTree.git`! -->
 
-      The following input types are still subject to change:
+          ## Source types
 
-      - `"path"`
+          The following source types and associated input attributes are supported.
-      - `"github"`
-      - `"gitlab"`
-      - `"sourcehut"`
-      - `"mercurial"`
 
-     *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
+          <!-- TODO: It would be soooo much more predictable to work with (and
-     The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
+          document) if `fetchTree` was a curried call with the first parameter for
+          `type` or an attribute like `builtins.fetchTree.git`! -->
+        )"));
+        parse_document(*root, before, CMARK_OPT_DEFAULT);
 
-      > **Example**
+        auto & schemes = node_append_child(*root, node_new(CMARK_NODE_LIST));
-      >
-      > Fetch a GitHub repository using the attribute set representation:
-      >
-      > ```nix
-      > builtins.fetchTree {
-      >   type = "github";
-      >   owner = "NixOS";
-      >   repo = "nixpkgs";
-      >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
-      > }
-      > ```
-      >
-      > This evaluates to the following attribute set:
-      >
-      > ```nix
-      > {
-      >   lastModified = 1686503798;
-      >   lastModifiedDate = "20230611171638";
-      >   narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
-      >   outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
-      >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
-      >   shortRev = "ae2e6b3";
-      > }
-      > ```
 
-      > **Example**
+        for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
-      >
+            auto & s = node_append_child(schemes, node_new(CMARK_NODE_ITEM));
-      > Fetch the same GitHub repository using the URL-like syntax:
+            {
-      >
+                auto & name_p = node_append_child(s, node_new(CMARK_NODE_PARAGRAPH));
-      >   ```nix
+                auto & name = node_append_child(name_p, node_new(CMARK_NODE_TEXT));
-      >   builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
+                node_set_literal(name, schemeName.data());
-      >   ```
+            }
-    )",
+            parse_document(s, scheme->schemeDescription(), CMARK_OPT_DEFAULT);
+
+            auto & attrs = node_append_child(s, node_new(CMARK_NODE_LIST));
+            for (const auto & [attrName, attribute] : scheme->allowedAttrs()) {
+                auto & a = node_append_child(attrs, node_new(CMARK_NODE_ITEM));
+                {
+                    auto & name_info = node_append_child(a, node_new(CMARK_NODE_PARAGRAPH));
+                    {
+                        auto & name = node_append_child(name_info, node_new(CMARK_NODE_CODE));
+                        auto & name_t = textArena.emplace_back(attrName);
+                        node_set_literal(name, name_t.c_str());
+                    }
+                    auto & info = node_append_child(name_info, node_new(CMARK_NODE_TEXT));
+                    auto & header = textArena.emplace_back(std::string { }
+                        + " (" + attribute.type
+                        + ", " + (attribute.required ? "required" : "optional")
+                        + ")");
+                    node_set_literal(info, header.c_str());
+                }
+                {
+                    auto & doc = textArena.emplace_back(stripIndentation(attribute.doc));
+                    parse_document(a, doc, CMARK_OPT_DEFAULT);
+                }
+            }
+        }
+
+        auto & after = textArena.emplace_back(stripIndentation(R"(
+          The following input types are still subject to change:
+
+          - `"path"`
+          - `"github"`
+          - `"gitlab"`
+          - `"sourcehut"`
+          - `"mercurial"`
+
+         *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
+         The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
+
+          > **Example**
+          >
+          > Fetch a GitHub repository using the attribute set representation:
+          >
+          > ```nix
+          > builtins.fetchTree {
+          >   type = "github";
+          >   owner = "NixOS";
+          >   repo = "nixpkgs";
+          >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
+          > }
+          > ```
+          >
+          > This evaluates to the following attribute set:
+          >
+          > ```nix
+          > {
+          >   lastModified = 1686503798;
+          >   lastModifiedDate = "20230611171638";
+          >   narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
+          >   outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
+          >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
+          >   shortRev = "ae2e6b3";
+          > }
+          > ```
+
+          > **Example**
+          >
+          > Fetch the same GitHub repository using the URL-like syntax:
+          >
+          >   ```nix
+          >   builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
+          >   ```
+        )"));
+        parse_document(*root, after, CMARK_OPT_DEFAULT);
+
+        auto p = render_commonmark(*root, CMARK_OPT_DEFAULT, 0);
+        assert(p);
+        return { &*p };
+    }(),
     .fun = prim_fetchTree,
     .experimentalFeature = Xp::FetchTree,
 });

src/libutil/cmark-cpp.hh

@@ -0,0 +1,85 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "util.hh"
+
+#include <cmark.h>
+
+namespace nix::cmark {
+
+using Node = struct cmark_node;
+using NodeType = cmark_node_type;
+using ListType = cmark_list_type;
+
+using Iter = struct cmark_iter;
+
+struct Deleter
+{
+    void operator () (Node * ptr) { cmark_node_free(ptr); }
+    void operator () (Iter * ptr) { cmark_iter_free(ptr); }
+};
+
+template <typename T>
+using UniquePtr = std::unique_ptr<Node, Deleter>;
+
+static inline void parse_document(Node & root, std::string_view s, int options)
+{
+    cmark_parser * parser = cmark_parser_new_with_mem_into_root(
+        options,
+        cmark_get_default_mem_allocator(),
+        &root);
+    cmark_parser_feed(parser, s.data(), s.size());
+    (void) cmark_parser_finish(parser);
+    cmark_parser_free(parser);
+}
+
+static inline UniquePtr<Node> parse_document(std::string_view s, int options)
+{
+    return UniquePtr<Node> {
+        cmark_parse_document(s.data(), s.size(), options)
+    };
+}
+
+static inline std::unique_ptr<char, FreeDeleter> render_commonmark(Node & root, int options, int width)
+{
+    return std::unique_ptr<char, FreeDeleter> {
+        cmark_render_commonmark(&root, options, width)
+    };
+}
+
+static inline std::unique_ptr<char, FreeDeleter> render_xml(Node & root, int options)
+{
+    return std::unique_ptr<char, FreeDeleter> {
+        cmark_render_xml(&root, options)
+    };
+}
+
+static inline UniquePtr<Node> node_new(NodeType type)
+{
+    return UniquePtr<Node> {
+        cmark_node_new(type)
+    };
+}
+
+/**
+ * The parent takes ownership
+ */
+static inline Node & node_append_child(Node & node, UniquePtr<Node> child)
+{
+    auto status = (bool) cmark_node_append_child(&node, &*child);
+    assert(status);
+    return *child.release();
+}
+
+static inline bool node_set_literal(Node & node, const char * content)
+{
+    return (bool) cmark_node_set_literal(&node, content);
+}
+
+static inline bool node_set_list_type(Node & node, ListType type)
+{
+    return (bool) cmark_node_set_list_type(&node, type);
+}
+
+}

src/libutil/util.hh

@@ -332,6 +332,16 @@ template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
 std::string showBytes(uint64_t bytes);
 
 
+/**
+ * For using `std::unique` with C functions.
+ */
+struct FreeDeleter
+{
+    template <typename T>
+    void operator()(T *p) const { std::free(p); }
+};
+
+
 /**
  * Provide an addition operator between strings and string_views
  * inexplicably omitted from the standard library.

src/nix/main.cc

@@ -458,7 +458,7 @@ void mainWrapped(int argc, char * * argv)
             auto b = nlohmann::json::object();
             if (!builtin.value->isPrimOp()) continue;
             auto primOp = builtin.value->primOp();
-            if (!primOp->doc) continue;
+            if (primOp->doc == "") continue;
             b["args"] = primOp->args;
             b["doc"] = trim(stripIndentation(primOp->doc));
             if (primOp->experimentalFeature)