Merge f66a76134d into 694c378d14

Makefile.config.in

@@ -4,6 +4,7 @@ BOOST_LDFLAGS = @BOOST_LDFLAGS@
 BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
 CC = @CC@
 CFLAGS = @CFLAGS@
+CMARK_LIBS = @CMARK_LIBS@
 CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
 CXXLTO = @CXXLTO@

configure.ac

@@ -372,21 +372,24 @@ PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest])
 PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9])
 
 
+# Look for cmark library.
+PKG_CHECK_MODULES([CMARK], [libcmark], [CXXFLAGS="$CMARK_CFLAGS $CXXFLAGS"])
+
 # Look for lowdown library.
-AC_ARG_ENABLE([markdown], AS_HELP_STRING([--enable-markdown], [Enable Markdown rendering in the Nix binary (requires lowdown) [default=auto]]),
+AC_ARG_ENABLE([lowdown], AS_HELP_STRING([--enable-lowdown], [Enable Markdown rendering in the Nix binary (requires lowdown) [default=auto]]),
-  enable_markdown=$enableval, enable_markdown=auto)
+  enable_lowdown=$enableval, enable_lowdown=auto)
-AS_CASE(["$enable_markdown"],
+AS_CASE(["$enable_lowdown"],
   [yes | auto], [
     PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [
       CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"
       have_lowdown=1
       AC_DEFINE(HAVE_LOWDOWN, 1, [Whether lowdown is available and should be used for Markdown rendering.])
     ], [
-      AS_IF([test "x$enable_markdown" == "xyes"], [AC_MSG_ERROR([--enable-markdown was specified, but lowdown was not found.])])
+      AS_IF([test "x$enable_lowdown" == "xyes"], [AC_MSG_ERROR([--enable-lowdown was specified, but lowdown was not found.])])
     ])
   ],
   [no], [have_lowdown=],
-  [AC_MSG_ERROR([bad value "$enable_markdown" for --enable-markdown, must be one of: yes, no, auto])])
+  [AC_MSG_ERROR([bad value "$enable_lowdown" for --enable-lowdown, must be one of: yes, no, auto])])
 
 
 # Look for libgit2.

flake.lock

@@ -1,5 +1,21 @@
 {
   "nodes": {
+    "cmark": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1706295831,
+        "narHash": "sha256-nEI85W8w49ZVr17ycO+7aZvcgA3U2QphNZGrfQl2mSk=",
+        "owner": "commonmark",
+        "repo": "cmark",
+        "rev": "cd37711b8a08da67ba4e21a42614b86dd8def929",
+        "type": "github"
+      },
+      "original": {
+        "owner": "commonmark",
+        "repo": "cmark",
+        "type": "github"
+      }
+    },
     "flake-compat": {
       "flake": false,
       "locked": {
@@ -128,6 +144,7 @@
     },
     "root": {
       "inputs": {
+        "cmark": "cmark",
         "flake-compat": "flake-compat",
         "flake-parts": "flake-parts",
         "git-hooks-nix": "git-hooks-nix",

flake.nix

@@ -6,6 +6,8 @@
   inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446";
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
   inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; };
+  # Until https://github.com/commonmark/cmark/pull/524 is released
+  inputs.cmark = { url = "github:commonmark/cmark"; flake = false; };
 
   # dev tooling
   inputs.flake-parts.url = "github:hercules-ci/flake-parts";

package.nix

@@ -23,6 +23,7 @@
 , libseccomp
 , libsodium
 , man
+, cmark
 , lowdown
 , mdbook
 , mdbook-linkcheck
@@ -79,7 +80,7 @@
 , enableGC ? !stdenv.hostPlatform.isWindows
 
 # Whether to enable Markdown rendering in the Nix binary.
-, enableMarkdown ? !stdenv.hostPlatform.isWindows
+, enableLowdown ? !stdenv.hostPlatform.isWindows
 
 # Which interactive line editor library to use for Nix's repl.
 #
@@ -226,7 +227,8 @@ in {
     toml11
     xz
     ({ inherit readline editline; }.${readlineFlavor})
-  ] ++ lib.optionals enableMarkdown [
+    cmark
+  ] ++ lib.optionals enableLowdown [
     lowdown
   ] ++ lib.optionals buildUnitTests [
     gtest
@@ -253,7 +255,7 @@ in {
     (lib.enableFeature doInstallCheck "functional-tests")
     (lib.enableFeature enableManual "doc-gen")
     (lib.enableFeature enableGC "gc")
-    (lib.enableFeature enableMarkdown "markdown")
+    (lib.enableFeature enableLowdown "lowdown")
     (lib.enableFeature installUnitTests "install-unit-tests")
     (lib.withFeatureAs true "readline-flavor" readlineFlavor)
   ] ++ lib.optionals (!forDevShell) [

packaging/dependencies.nix

@@ -166,6 +166,11 @@ scope: {
       ];
   });
 
+  cmark = pkgs.cmark.overrideAttrs (_: {
+    src = inputs.cmark;
+    version = inputs.lastModifiedDate;
+  });
+
   busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override {
     useMusl = true;
     enableStatic = true;

packaging/hydra.nix

@@ -90,9 +90,9 @@ in
 
   # Toggles some settings for better coverage. Windows needs these
   # library combinations, and Debian build Nix with GNU readline too.
-  buildReadlineNoMarkdown = forAllSystems (system:
+  buildReadlineNoLowdown = forAllSystems (system:
     self.packages.${system}.nix.override {
-      enableMarkdown = false;
+      enableLowdown = false;
       readlineFlavor = "readline";
     }
   );

src/libexpr/eval.cc

@@ -500,7 +500,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
         v.mkApp(vPrimOp, vPrimOp);
         return addConstant(primOp.name, v, {
             .type = nThunk, // FIXME
-            .doc = primOp.doc,
+            .doc = primOp.doc.c_str(),
         });
     }
 
@@ -527,13 +527,14 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
     if (v.isPrimOp()) {
         auto v2 = &v;
-        if (auto * doc = v2->primOp()->doc)
+        auto & doc = v2->primOp()->doc;
+        if (doc != "")
             return Doc {
                 .pos = {},
                 .name = v2->primOp()->name,
                 .arity = v2->primOp()->arity,
                 .args = v2->primOp()->args,
-                .doc = doc,
+                .doc = doc.c_str(),
             };
     }
     if (v.isLambda()) {

src/libexpr/eval.hh

@@ -88,7 +88,7 @@ struct PrimOp
     /**
      * Optional free-form documentation about the primop.
      */
-    const char * doc = nullptr;
+    const std::string doc = "";
 
     /**
      * Add a trace item, `while calling the '<name>' builtin`

src/libexpr/local.mk

@@ -20,7 +20,7 @@ libexpr_CXXFLAGS += \
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS)
+libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS) $(CMARK_LIBS)
 ifdef HOST_LINUX
   libexpr_LDFLAGS += -ldl
 endif

src/libexpr/primops/fetchTree.cc

@@ -8,6 +8,7 @@
 #include "registry.hh"
 #include "tarball.hh"
 #include "url.hh"
+#include "cmark-cpp.hh"
 #include "value-to-json.hh"
 #include "fetch-to-store.hh"
 
@@ -210,223 +211,144 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args,
 static RegisterPrimOp primop_fetchTree({
     .name = "fetchTree",
     .args = {"input"},
-    .doc = R"(
+    .doc =  []() -> std::string {
-      Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
+        using namespace cmark;
 
-      - the resulting fixed-output [store path](@docroot@/store/store-path.md)
+        // Stores strings referenced by AST. Deallocate after rendering.
-      - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
+        std::vector<std::string> textArena;
-      - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
 
-      *input* must be an attribute set with the following attributes:
+        auto root = node_new(CMARK_NODE_DOCUMENT);
 
-      - `type` (String, required)
+        auto & before = textArena.emplace_back(stripIndentation(R"(
+          Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
 
-        One of the [supported source types](#source-types).
+          - the resulting fixed-output [store path](@docroot@/store/store-path.md)
-        This determines other required and allowed input attributes.
+          - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
+          - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
 
-      - `narHash` (String, optional)
+          *input* must be an attribute set with the following attributes:
 
-        The `narHash` parameter can be used to substitute the source of the tree.
+          - `type` (String, required)
-        It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
-        If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
 
-      A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
+            One of the [supported source types](#source-types).
-      That is, `fetchTree` is idempotent.
+            This determines other required and allowed input attributes.
 
-      Downloads are cached in `$XDG_CACHE_HOME/nix`.
+          - `narHash` (String, optional)
-      The remote source will be fetched from the network if both are true:
-      - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
 
-        > **Note**
+            The `narHash` parameter can be used to substitute the source of the tree.
-        >
+            It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
-        > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
+            If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
 
-      - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
+          A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
+          That is, `fetchTree` is idempotent.
 
-      ## Source types
+          Downloads are cached in `$XDG_CACHE_HOME/nix`.
+          The remote source will be fetched from the network if both are true:
+          - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
 
-      The following source types and associated input attributes are supported.
+            > **Note**
-
-      <!-- TODO: It would be soooo much more predictable to work with (and
-      document) if `fetchTree` was a curried call with the first parameter for
-      `type` or an attribute like `builtins.fetchTree.git`! -->
-
-      - `"file"`
-
-        Place a plain file into the Nix store.
-        This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
-
-        - `url` (String, required)
-
-          Supported protocols:
-
-          - `https`
-
-            > **Example**
             >
-            > ```nix
+            > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
-            > fetchTree {
-            >   type = "file";
-            >   url = "https://example.com/index.html";
-            > }
-            > ```
 
-          - `http`
+          - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
 
-            Insecure HTTP transfer for legacy sources.
+          ## Source types
 
-            > **Warning**
+          The following source types and associated input attributes are supported.
-            >
-            > HTTP performs no encryption or authentication.
-            > Use a `narHash` known in advance to ensure the output has expected contents.
 
-          - `file`
+          <!-- TODO: It would be soooo much more predictable to work with (and
+          document) if `fetchTree` was a curried call with the first parameter for
+          `type` or an attribute like `builtins.fetchTree.git`! -->
+        )"));
+        parse_document(*root, before, CMARK_OPT_DEFAULT);
 
-            A file on the local file system.
+        auto & schemes = node_append_child(*root, node_new(CMARK_NODE_LIST));
 
-            > **Example**
+        for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
-            >
+            auto & s = node_append_child(schemes, node_new(CMARK_NODE_ITEM));
-            > ```nix
+            {
-            > fetchTree {
+                auto & name_p = node_append_child(s, node_new(CMARK_NODE_PARAGRAPH));
-            >   type = "file";
+                auto & name = node_append_child(name_p, node_new(CMARK_NODE_TEXT));
-            >   url = "file:///home/eelco/nix/README.md";
+                node_set_literal(name, schemeName.data());
-            > }
+            }
-            > ```
+            parse_document(s, scheme->schemeDescription(), CMARK_OPT_DEFAULT);
 
-      - `"tarball"`
+            auto & attrs = node_append_child(s, node_new(CMARK_NODE_LIST));
+            for (const auto & [attrName, attribute] : scheme->allowedAttrs()) {
+                auto & a = node_append_child(attrs, node_new(CMARK_NODE_ITEM));
+                {
+                    auto & name_info = node_append_child(a, node_new(CMARK_NODE_PARAGRAPH));
+                    {
+                        auto & name = node_append_child(name_info, node_new(CMARK_NODE_CODE));
+                        auto & name_t = textArena.emplace_back(attrName);
+                        node_set_literal(name, name_t.c_str());
+                    }
+                    auto & info = node_append_child(name_info, node_new(CMARK_NODE_TEXT));
+                    auto & header = textArena.emplace_back(std::string { }
+                        + " (" + attribute.type
+                        + ", " + (attribute.required ? "required" : "optional")
+                        + ")");
+                    node_set_literal(info, header.c_str());
+                }
+                {
+                    auto & doc = textArena.emplace_back(stripIndentation(attribute.doc));
+                    parse_document(a, doc, CMARK_OPT_DEFAULT);
+                }
+            }
+        }
 
-        Download a tar archive and extract it into the Nix store.
+        auto & after = textArena.emplace_back(stripIndentation(R"(
-        This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
+          The following input types are still subject to change:
 
-        - `url` (String, required)
+          - `"path"`
+          - `"github"`
+          - `"gitlab"`
+          - `"sourcehut"`
+          - `"mercurial"`
 
-           > **Example**
+         *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
-           >
+         The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
-           > ```nix
-           > fetchTree {
-           >   type = "tarball";
-           >   url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
-           > }
-           > ```
-
-      - `"git"`
-
-        Fetch a Git tree and copy it to the Nix store.
-        This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
-
-        - `url` (String, required)
-
-          The URL formats supported are the same as for Git itself.
 
           > **Example**
           >
+          > Fetch a GitHub repository using the attribute set representation:
+          >
           > ```nix
-          > fetchTree {
+          > builtins.fetchTree {
-          >   type = "git";
+          >   type = "github";
-          >   url = "git@github.com:NixOS/nixpkgs.git";
+          >   owner = "NixOS";
+          >   repo = "nixpkgs";
+          >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
+          > }
+          > ```
+          >
+          > This evaluates to the following attribute set:
+          >
+          > ```nix
+          > {
+          >   lastModified = 1686503798;
+          >   lastModifiedDate = "20230611171638";
+          >   narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
+          >   outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
+          >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
+          >   shortRev = "ae2e6b3";
           > }
           > ```
 
-          > **Note**
+          > **Example**
           >
-          > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
+          > Fetch the same GitHub repository using the URL-like syntax:
+          >
+          >   ```nix
+          >   builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
+          >   ```
+        )"));
+        parse_document(*root, after, CMARK_OPT_DEFAULT);
 
-        - `ref` (String, optional)
+        auto p = render_commonmark(*root, CMARK_OPT_DEFAULT, 0);
-
+        assert(p);
-          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+        return { &*p };
-
+    }(),
-          A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
-
-          Default: `"HEAD"`
-
-        - `rev` (String, optional)
-
-          A Git revision; a commit hash.
-
-          Default: the tip of `ref`
-
-        - `shallow` (Bool, optional)
-
-          Make a shallow clone when fetching the Git tree.
-          When this is enabled, the options `ref` and `allRefs` have no effect anymore.
-
-          Default: `true`
-
-        - `submodules` (Bool, optional)
-
-          Also fetch submodules if available.
-
-          Default: `false`
-
-        - `allRefs` (Bool, optional)
-
-          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
-
-          Whether to fetch all references (eg. branches and tags) of the repository.
-          With this argument being true, it's possible to load a `rev` from *any* `ref`.
-          (Without setting this option, only `rev`s from the specified `ref` are supported).
-
-          Default: `false`
-
-        - `lastModified` (Integer, optional)
-
-          Unix timestamp of the fetched commit.
-
-          If set, pass through the value to the output attribute set.
-          Otherwise, generated from the fetched Git tree.
-
-        - `revCount` (Integer, optional)
-
-          Number of revisions in the history of the Git repository before the fetched commit.
-
-          If set, pass through the value to the output attribute set.
-          Otherwise, generated from the fetched Git tree.
-
-      The following input types are still subject to change:
-
-      - `"path"`
-      - `"github"`
-      - `"gitlab"`
-      - `"sourcehut"`
-      - `"mercurial"`
-
-     *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
-     The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
-
-      > **Example**
-      >
-      > Fetch a GitHub repository using the attribute set representation:
-      >
-      > ```nix
-      > builtins.fetchTree {
-      >   type = "github";
-      >   owner = "NixOS";
-      >   repo = "nixpkgs";
-      >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
-      > }
-      > ```
-      >
-      > This evaluates to the following attribute set:
-      >
-      > ```nix
-      > {
-      >   lastModified = 1686503798;
-      >   lastModifiedDate = "20230611171638";
-      >   narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
-      >   outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
-      >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
-      >   shortRev = "ae2e6b3";
-      > }
-      > ```
-
-      > **Example**
-      >
-      > Fetch the same GitHub repository using the URL-like syntax:
-      >
-      >   ```nix
-      >   builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
-      >   ```
-    )",
     .fun = prim_fetchTree,
     .experimentalFeature = Xp::FetchTree,
 });

src/libfetchers/fetchers.cc

@@ -8,8 +8,6 @@
 
 namespace nix::fetchers {
 
-using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
-
 std::unique_ptr<InputSchemeMap> inputSchemes = nullptr;
 
 void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
@@ -22,17 +20,9 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
     inputSchemes->insert_or_assign(schemeName, std::move(inputScheme));
 }
 
-nlohmann::json dumpRegisterInputSchemeInfo() {
+const InputSchemeMap & getAllInputSchemes()
-    using nlohmann::json;
+{
-
+    return *inputSchemes;
-    auto res = json::object();
-
-    for (auto & [name, scheme] : *inputSchemes) {
-        auto & r = res[name] = json::object();
-        r["allowedAttrs"] = scheme->allowedAttrs();
-    }
-
-    return res;
 }
 
 Input Input::fromURL(

src/libfetchers/fetchers.hh

@@ -179,14 +179,26 @@ struct InputScheme
      */
     virtual std::string_view schemeName() const = 0;
 
+    /**
+     * Longform description of this scheme, for documentation purposes.
+     */
+    virtual std::string schemeDescription() const = 0;
+
+    // TODO remove these defaults
+    struct AttributeInfo {
+        const char * type = "String";
+        bool required = true;
+        const char * doc = "";
+    };
+
     /**
      * Allowed attributes in an attribute set that is converted to an
-     * input.
+     * input, and documentation for each attribute.
      *
-     * `type` is not included from this set, because the `type` field is
+     * `type` is not included from this map, because the `type` field is
       parsed first to choose which scheme; `type` is always required.
      */
-    virtual StringSet allowedAttrs() const = 0;
+    virtual std::map<std::string, AttributeInfo> allowedAttrs() const = 0;
 
     virtual ParsedURL toURL(const Input & input) const;
 
@@ -244,7 +256,12 @@ struct InputScheme
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
 
-nlohmann::json dumpRegisterInputSchemeInfo();
+using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
+
+/**
+ * Use this for docs, not for finding a specific scheme
+ */
+const InputSchemeMap & getAllInputSchemes();
 
 struct PublicKey
 {

src/libfetchers/git.cc

@@ -201,26 +201,169 @@ struct GitInputScheme : InputScheme
         return "git";
     }
 
-    StringSet allowedAttrs() const override
+    std::string schemeDescription() const override
+    {
+        return stripIndentation(R"(
+          Fetch a Git tree and copy it to the Nix store.
+          This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
+        )");
+    }
+
+    std::map<std::string, AttributeInfo> allowedAttrs() const override
     {
         return {
-            "url",
+            {
-            "ref",
+                "url",
-            "rev",
+                {
-            "shallow",
+                    .type = "String",
-            "submodules",
+                    .required = true,
-            "exportIgnore",
+                    .doc = R"(
-            "lastModified",
+                      The URL formats supported are the same as for Git itself.
-            "revCount",
+
-            "narHash",
+                      > **Example**
-            "allRefs",
+                      >
-            "name",
+                      > ```nix
-            "dirtyRev",
+                      > fetchTree {
-            "dirtyShortRev",
+                      >   type = "git";
-            "verifyCommit",
+                      >   url = "git@github.com:NixOS/nixpkgs.git";
-            "keytype",
+                      > }
-            "publicKey",
+                      > ```
-            "publicKeys",
+
+                      > **Note**
+                      >
+                      > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
+                    )",
+                },
+            },
+            {
+                "ref",
+                {
+                    .type = "String",
+                    .required = false,
+                    .doc = R"(
+                      By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+                      A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
+
+                      Default: `"HEAD"`
+                    )",
+                },
+            },
+            {
+                "rev",
+                {
+                    .type = "String",
+                    .required = false,
+                    .doc = R"(
+                      A Git revision; a commit hash.
+
+                      Default: the tip of `ref`
+                    )",
+                },
+            },
+            {
+                "shallow",
+                {
+                    .type = "Bool",
+                    .required = false,
+                    .doc = R"(
+                      Make a shallow clone when fetching the Git tree.
+                      When this is enabled, the options `ref` and `allRefs` have no effect anymore.
+
+                      Default: `true`
+                    )",
+                },
+            },
+            {
+                "submodules",
+                {
+                    .type = "Bool",
+                    .required = false,
+                    .doc = R"(
+                      Also fetch submodules if available.
+
+                      Default: `false`
+                    )",
+                },
+            },
+            {
+                "lastModified",
+                {
+                    .type = "integer",
+                    .required = false,
+                    .doc = R"(
+                      Unix timestamp of the fetched commit.
+
+                      If set, pass through the value to the output attribute set.
+                      Otherwise, generated from the fetched Git tree.
+                    )",
+                },
+            },
+            {
+                "revCount",
+                {
+                    .type = "integer",
+                    .required = false,
+                    .doc = R"(
+                      Number of revisions in the history of the Git repository before the fetched commit.
+
+                      If set, pass through the value to the output attribute set.
+                      Otherwise, generated from the fetched Git tree.
+                    )",
+                },
+            },
+            {
+                "narHash",
+                {},
+            },
+            {
+                "allRefs",
+                {
+                    .type = "Bool",
+                    .required = false,
+                    .doc = R"(
+                      By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+                      Whether to fetch all references (eg. branches and tags) of the repository.
+                      With this argument being true, it's possible to load a `rev` from *any* `ref`.
+                      (Without setting this option, only `rev`s from the specified `ref` are supported).
+
+                      Default: `false`
+                    )",
+                },
+            },
+            {
+                "name",
+                {},
+            },
+            {
+                "dirtyRev",
+                {},
+            },
+            {
+                "dirtyShortRev",
+                {},
+            },
+            {
+                "exportIgnore",
+                {},
+            },
+            {
+                "verifyCommit",
+                {},
+            },
+            {
+                "keytype",
+                {},
+            },
+            {
+                "publicKey",
+                {},
+            },
+            {
+                "publicKeys",
+                {},
+            },
         };
     }
 

src/libfetchers/github.cc

@@ -107,17 +107,41 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
-    StringSet allowedAttrs() const override
+    std::map<std::string, AttributeInfo> allowedAttrs() const override
     {
         return {
-            "owner",
+            {
-            "repo",
+                "owner",
-            "ref",
+                {},
-            "rev",
+            },
-            "narHash",
+            {
-            "lastModified",
+                "repo",
-            "host",
+                {},
-            "treeHash",
+            },
+            {
+                "ref",
+                {},
+            },
+            {
+                "rev",
+                {},
+            },
+            {
+                "narHash",
+                {},
+            },
+            {
+                "lastModified",
+                {},
+            },
+            {
+                "host",
+                {},
+            },
+            {
+                "treeHash",
+                {},
+            },
         };
     }
 
@@ -330,6 +354,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
 {
     std::string_view schemeName() const override { return "github"; }
 
+    std::string schemeDescription() const override
+    {
+        // TODO
+        return "";
+    }
+
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // Github supports PAT/OAuth2 tokens and HTTP Basic
@@ -413,6 +443,12 @@ struct GitLabInputScheme : GitArchiveInputScheme
 {
     std::string_view schemeName() const override { return "gitlab"; }
 
+    std::string schemeDescription() const override
+    {
+        // TODO
+        return "";
+    }
+
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // Gitlab supports 4 kinds of authorization, two of which are
@@ -488,6 +524,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme
 {
     std::string_view schemeName() const override { return "sourcehut"; }
 
+    std::string schemeDescription() const override
+    {
+        // TODO
+        return "";
+    }
+
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // SourceHut supports both PAT and OAuth2. See

src/libfetchers/indirect.cc

@@ -57,13 +57,31 @@ struct IndirectInputScheme : InputScheme
         return "indirect";
     }
 
-    StringSet allowedAttrs() const override
+    std::string schemeDescription() const override
+    {
+        // TODO
+        return "";
+    }
+
+    std::map<std::string, AttributeInfo> allowedAttrs() const override
     {
         return {
-            "id",
+            {
-            "ref",
+                "id",
-            "rev",
+                {},
-            "narHash",
+            },
+            {
+                "ref",
+                {},
+            },
+            {
+                "rev",
+                {},
+            },
+            {
+                "narHash",
+                {},
+            },
         };
     }
 

src/libfetchers/mercurial.cc

@@ -78,15 +78,39 @@ struct MercurialInputScheme : InputScheme
         return "hg";
     }
 
-    StringSet allowedAttrs() const override
+    std::string schemeDescription() const override
+    {
+        // TODO
+        return "";
+    }
+
+    std::map<std::string, AttributeInfo> allowedAttrs() const override
     {
         return {
-            "url",
+            {
-            "ref",
+                "url",
-            "rev",
+                {},
-            "revCount",
+            },
-            "narHash",
+            {
-            "name",
+                "ref",
+                {},
+            },
+            {
+                "rev",
+                {},
+            },
+            {
+                "revCount",
+                {},
+            },
+            {
+                "narHash",
+                {},
+            },
+            {
+                "name",
+                {},
+            },
         };
     }
 

src/libfetchers/path.cc

@@ -40,19 +40,40 @@ struct PathInputScheme : InputScheme
         return "path";
     }
 
-    StringSet allowedAttrs() const override
+    std::string schemeDescription() const override
+    {
+        // TODO
+        return "";
+    }
+
+    std::map<std::string, AttributeInfo> allowedAttrs() const override
     {
         return {
-            "path",
+            {
+                "path",
+                {},
+            },
             /* Allow the user to pass in "fake" tree info
                attributes. This is useful for making a pinned tree work
                the same as the repository from which is exported (e.g.
                path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
              */
-            "rev",
+            {
-            "revCount",
+                "rev",
-            "lastModified",
+                {},
-            "narHash",
+            },
+            {
+                "revCount",
+                {},
+            },
+            {
+                "lastModified",
+                {},
+            },
+            {
+                "narHash",
+                {},
+            },
         };
     }
 

src/libfetchers/tarball.cc

@@ -278,17 +278,85 @@ struct CurlInputScheme : InputScheme
         return input;
     }
 
-    StringSet allowedAttrs() const override
+    std::map<std::string, AttributeInfo> allowedAttrs() const override
     {
         return {
-            "type",
+            {
-            "url",
+                "url",
-            "narHash",
+                {
-            "name",
+                    .type = "String",
-            "unpack",
+                    .required = true,
-            "rev",
+                    .doc = R"(
-            "revCount",
+                      Supported protocols:
-            "lastModified",
+
+                      - `https`
+
+                        > **Example**
+                        >
+                        > ```nix
+                        > fetchTree {
+                        >   type = "file";
+                        >   url = "https://example.com/index.html";
+                        > }
+                        > ```
+
+                      - `http`
+
+                        Insecure HTTP transfer for legacy sources.
+
+                        > **Warning**
+                        >
+                        > HTTP performs no encryption or authentication.
+                        > Use a `narHash` known in advance to ensure the output has expected contents.
+
+                      - `file`
+
+                        A file on the local file system.
+
+                        > **Example**
+                        >
+                        > ```nix
+                        > fetchTree {
+                        >   type = "file";
+                        >   url = "file:///home/eelco/nix/README.md";
+                        > }
+                        > ```
+
+                      > **Example**
+                      >
+                      > ```nix
+                      > fetchTree {
+                      >   type = "tarball";
+                      >   url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
+                      > }
+                      > ```
+                    )",
+                },
+            },
+            {
+                "narHash",
+                {},
+            },
+            {
+                "name",
+                {},
+            },
+            {
+                "unpack",
+                {},
+            },
+            {
+                "rev",
+                {},
+            },
+            {
+                "revCount",
+                {},
+            },
+            {
+                "lastModified",
+                {},
+            },
         };
     }
 
@@ -323,6 +391,14 @@ struct FileInputScheme : CurlInputScheme
 {
     std::string_view schemeName() const override { return "file"; }
 
+    std::string schemeDescription() const override
+    {
+        return stripIndentation(R"(
+          Place a plain file into the Nix store.
+          This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
+        )");
+    }
+
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
         auto parsedUrlScheme = parseUrlScheme(url.scheme);
@@ -357,6 +433,15 @@ struct TarballInputScheme : CurlInputScheme
 {
     std::string_view schemeName() const override { return "tarball"; }
 
+    std::string schemeDescription() const override
+    {
+        return stripIndentation(R"(
+          Download a tar archive and extract it into the Nix store.
+          This has the same underyling implementation as [`builtins.fetchTarball`](@doc
+    root@/language/builtins.md#builtins-fetchTarball)
+        )");
+    }
+
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
         auto parsedUrlScheme = parseUrlScheme(url.scheme);

src/libutil/cmark-cpp.hh

@@ -0,0 +1,85 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "util.hh"
+
+#include <cmark.h>
+
+namespace nix::cmark {
+
+using Node = struct cmark_node;
+using NodeType = cmark_node_type;
+using ListType = cmark_list_type;
+
+using Iter = struct cmark_iter;
+
+struct Deleter
+{
+    void operator () (Node * ptr) { cmark_node_free(ptr); }
+    void operator () (Iter * ptr) { cmark_iter_free(ptr); }
+};
+
+template <typename T>
+using UniquePtr = std::unique_ptr<Node, Deleter>;
+
+static inline void parse_document(Node & root, std::string_view s, int options)
+{
+    cmark_parser * parser = cmark_parser_new_with_mem_into_root(
+        options,
+        cmark_get_default_mem_allocator(),
+        &root);
+    cmark_parser_feed(parser, s.data(), s.size());
+    (void) cmark_parser_finish(parser);
+    cmark_parser_free(parser);
+}
+
+static inline UniquePtr<Node> parse_document(std::string_view s, int options)
+{
+    return UniquePtr<Node> {
+        cmark_parse_document(s.data(), s.size(), options)
+    };
+}
+
+static inline std::unique_ptr<char, FreeDeleter> render_commonmark(Node & root, int options, int width)
+{
+    return std::unique_ptr<char, FreeDeleter> {
+        cmark_render_commonmark(&root, options, width)
+    };
+}
+
+static inline std::unique_ptr<char, FreeDeleter> render_xml(Node & root, int options)
+{
+    return std::unique_ptr<char, FreeDeleter> {
+        cmark_render_xml(&root, options)
+    };
+}
+
+static inline UniquePtr<Node> node_new(NodeType type)
+{
+    return UniquePtr<Node> {
+        cmark_node_new(type)
+    };
+}
+
+/**
+ * The parent takes ownership
+ */
+static inline Node & node_append_child(Node & node, UniquePtr<Node> child)
+{
+    auto status = (bool) cmark_node_append_child(&node, &*child);
+    assert(status);
+    return *child.release();
+}
+
+static inline bool node_set_literal(Node & node, const char * content)
+{
+    return (bool) cmark_node_set_literal(&node, content);
+}
+
+static inline bool node_set_list_type(Node & node, ListType type)
+{
+    return (bool) cmark_node_set_list_type(&node, type);
+}
+
+}

src/libutil/util.hh

@@ -332,6 +332,16 @@ template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
 std::string showBytes(uint64_t bytes);
 
 
+/**
+ * For using `std::unique` with C functions.
+ */
+struct FreeDeleter
+{
+    template <typename T>
+    void operator()(T *p) const { std::free(p); }
+};
+
+
 /**
  * Provide an addition operator between strings and string_views
  * inexplicably omitted from the standard library.

src/nix/main.cc

@@ -217,21 +217,40 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
 
     std::string dumpCli()
     {
-        auto res = nlohmann::json::object();
+        using nlohmann::json;
+
+        auto res = json::object();
 
         res["args"] = toJSON();
 
-        auto stores = nlohmann::json::object();
+        {
-        for (auto & implem : *Implementations::registered) {
+            auto & stores = res["stores"] = json::object();
-            auto storeConfig = implem.getConfig();
+            for (const auto & implem : *Implementations::registered) {
-            auto storeName = storeConfig->name();
+                auto storeConfig = implem.getConfig();
-            auto & j = stores[storeName];
+                auto storeName = storeConfig->name();
-            j["doc"] = storeConfig->doc();
+                auto & j = stores[storeName];
-            j["settings"] = storeConfig->toJSON();
+                j["doc"] = storeConfig->doc();
-            j["experimentalFeature"] = storeConfig->experimentalFeature();
+                j["settings"] = storeConfig->toJSON();
+                j["experimentalFeature"] = storeConfig->experimentalFeature();
+            }
         }
-        res["stores"] = std::move(stores);
+
-        res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo();
+        {
+            auto & fetchers = res["fetchers"] = json::object();
+
+            for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
+                auto & s = fetchers[schemeName] = json::object();
+                s["description"] = scheme->schemeDescription();
+                auto & attrs = s["allowedAttrs"] = json::object();
+                for (auto & [fieldName, field] : scheme->allowedAttrs()) {
+                    auto & f = attrs[fieldName] = json::object();
+                    f["type"] = field.type;
+                    f["required"] = field.required;
+                    f["doc"] = stripIndentation(field.doc);
+                }
+            }
+
+        };
 
         return res.dump();
     }
@@ -439,7 +458,7 @@ void mainWrapped(int argc, char * * argv)
             auto b = nlohmann::json::object();
             if (!builtin.value->isPrimOp()) continue;
             auto primOp = builtin.value->primOp();
-            if (!primOp->doc) continue;
+            if (primOp->doc == "") continue;
             b["args"] = primOp->args;
             b["doc"] = trim(stripIndentation(primOp->doc));
             if (primOp->experimentalFeature)