Systematize builtins.fetchTree docs

And also render the docs nicely.

Add `cmark` as a public dependency to libnixutil

`cmark` is a more robust library for constructing Markdown than
`lowdown`. It in particular has the functionality we need to compute the
doc we need to compute. It is also more portable.

Unfortunately it doesn't have a terminal mode, so we cannot get rid of
lowdown yet.

src/libexpr/eval.cc

@@ -525,7 +525,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
             v,
             {
                 .type = nThunk, // FIXME
-                .doc = primOp.doc,
+                .doc = primOp.doc.c_str(),
             });
     }
 
@@ -565,13 +565,14 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
     if (v.isPrimOp()) {
         auto v2 = &v;
-        if (auto * doc = v2->primOp()->doc)
+        auto & doc = v2->primOp()->doc;
+        if (doc != "")
             return Doc{
                 .pos = {},
                 .name = v2->primOp()->name,
                 .arity = v2->primOp()->arity,
                 .args = v2->primOp()->args,
-                .doc = doc,
+                .doc = doc.c_str(),
             };
     }
     if (v.isLambda()) {

src/libexpr/include/nix/expr/eval.hh

@@ -108,7 +108,7 @@ struct PrimOp
     /**
      * Optional free-form documentation about the primop.
      */
-    const char * doc = nullptr;
+    const std::string doc = "";
 
     /**
      * Add a trace item, while calling the `<name>` builtin.

src/libexpr/primops/fetchTree.cc

@@ -8,6 +8,7 @@
 #include "nix/fetchers/registry.hh"
 #include "nix/fetchers/tarball.hh"
 #include "nix/util/url.hh"
+#include "nix/util/cmark-cpp.hh"
 #include "nix/expr/value-to-json.hh"
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/fetchers/input-cache.hh"
@@ -234,7 +235,15 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, V
 static RegisterPrimOp primop_fetchTree({
     .name = "fetchTree",
     .args = {"input"},
-    .doc = R"(
+    .doc =  []() -> std::string {
+        using namespace cmark;
+
+        // Stores strings referenced by AST. Deallocate after rendering.
+        std::vector<std::string> textArena;
+
+        auto root = node_new(CMARK_NODE_DOCUMENT);
+
+        auto & before = textArena.emplace_back(stripIndentation(R"(
           Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
 
           - the resulting fixed-output [store path](@docroot@/store/store-path.md)
@@ -258,7 +267,7 @@ static RegisterPrimOp primop_fetchTree({
           That is, `fetchTree` is idempotent.
 
           Downloads are cached in `$XDG_CACHE_HOME/nix`.
-      The remote source is fetched from the network if both are true:
+          The remote source will be fetched from the network if both are true:
           - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
 
             > **Note**
@@ -274,144 +283,45 @@ static RegisterPrimOp primop_fetchTree({
           <!-- TODO: It would be soooo much more predictable to work with (and
           document) if `fetchTree` was a curried call with the first parameter for
           `type` or an attribute like `builtins.fetchTree.git`! -->
+        )"));
+        parse_document(*root, before, CMARK_OPT_DEFAULT);
 
-      - `"file"`
+        auto & schemes = node_append_child(*root, node_new(CMARK_NODE_LIST));
 
-        Place a plain file into the Nix store.
+        for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
-        This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
+            auto & s = node_append_child(schemes, node_new(CMARK_NODE_ITEM));
+            {
+                auto & name_p = node_append_child(s, node_new(CMARK_NODE_PARAGRAPH));
+                auto & name = node_append_child(name_p, node_new(CMARK_NODE_TEXT));
+                node_set_literal(name, schemeName.data());
+            }
+            parse_document(s, scheme->schemeDescription(), CMARK_OPT_DEFAULT);
 
-        - `url` (String, required)
+            auto & attrs = node_append_child(s, node_new(CMARK_NODE_LIST));
-
+            for (const auto & [attrName, attribute] : scheme->allowedAttrs()) {
-          Supported protocols:
+                auto & a = node_append_child(attrs, node_new(CMARK_NODE_ITEM));
-
+                {
-          - `https`
+                    auto & name_info = node_append_child(a, node_new(CMARK_NODE_PARAGRAPH));
-
+                    {
-            > **Example**
+                        auto & name = node_append_child(name_info, node_new(CMARK_NODE_CODE));
-            >
+                        auto & name_t = textArena.emplace_back(attrName);
-            > ```nix
+                        node_set_literal(name, name_t.c_str());
-            > fetchTree {
+                    }
-            >   type = "file";
+                    auto & info = node_append_child(name_info, node_new(CMARK_NODE_TEXT));
-            >   url = "https://example.com/index.html";
+                    auto & header = textArena.emplace_back(std::string { }
-            > }
+                        + " (" + attribute.type
-            > ```
+                        + ", " + (attribute.required ? "required" : "optional")
-
+                        + ")");
-          - `http`
+                    node_set_literal(info, header.c_str());
-
+                }
-            Insecure HTTP transfer for legacy sources.
+                {
-
+                    auto & doc = textArena.emplace_back(stripIndentation(attribute.doc));
-            > **Warning**
+                    parse_document(a, doc, CMARK_OPT_DEFAULT);
-            >
+                }
-            > HTTP performs no encryption or authentication.
+            }
-            > Use a `narHash` known in advance to ensure the output has expected contents.
+        }
-
-          - `file`
-
-            A file on the local file system.
-
-            > **Example**
-            >
-            > ```nix
-            > fetchTree {
-            >   type = "file";
-            >   url = "file:///home/eelco/nix/README.md";
-            > }
-            > ```
-
-      - `"tarball"`
-
-        Download a tar archive and extract it into the Nix store.
-        This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
-
-        - `url` (String, required)
-
-           > **Example**
-           >
-           > ```nix
-           > fetchTree {
-           >   type = "tarball";
-           >   url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
-           > }
-           > ```
-
-      - `"git"`
-
-        Fetch a Git tree and copy it to the Nix store.
-        This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
-
-        - `url` (String, required)
-
-          The URL formats supported are the same as for Git itself.
-
-          > **Example**
-          >
-          > ```nix
-          > fetchTree {
-          >   type = "git";
-          >   url = "git@github.com:NixOS/nixpkgs.git";
-          > }
-          > ```
-
-          > **Note**
-          >
-          > If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
-
-        - `ref` (String, optional)
-
-          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
-
-          A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
-
-          Default: `"HEAD"`
-
-        - `rev` (String, optional)
-
-          A Git revision; a commit hash.
-
-          Default: the tip of `ref`
-
-        - `shallow` (Bool, optional)
-
-          Make a shallow clone when fetching the Git tree.
-          When this is enabled, the options `ref` and `allRefs` have no effect anymore.
-
-          Default: `true`
-
-        - `submodules` (Bool, optional)
-
-          Also fetch submodules if available.
-
-          Default: `false`
-
-        - `lfs` (Bool, optional)
-
-          Fetch any [Git LFS](https://git-lfs.com/) files.
-
-          Default: `false`
-
-        - `allRefs` (Bool, optional)
-
-          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
-
-          Whether to fetch all references (eg. branches and tags) of the repository.
-          With this argument being true, it's possible to load a `rev` from *any* `ref`.
-          (Without setting this option, only `rev`s from the specified `ref` are supported).
-
-          Default: `false`
-
-        - `lastModified` (Integer, optional)
-
-          Unix timestamp of the fetched commit.
-
-          If set, pass through the value to the output attribute set.
-          Otherwise, generated from the fetched Git tree.
-
-        - `revCount` (Integer, optional)
-
-          Number of revisions in the history of the Git repository before the fetched commit.
-
-          If set, pass through the value to the output attribute set.
-          Otherwise, generated from the fetched Git tree.
 
+        auto & after = textArena.emplace_back(stripIndentation(R"(
           The following input types are still subject to change:
 
           - `"path"`
@@ -456,7 +366,13 @@ static RegisterPrimOp primop_fetchTree({
           >   ```nix
           >   builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
           >   ```
-    )",
+        )"));
+        parse_document(*root, after, CMARK_OPT_DEFAULT);
+
+        auto p = render_commonmark(*root, CMARK_OPT_DEFAULT, 0);
+        assert(p);
+        return { &*p };
+    }(),
     .fun = prim_fetchTree,
     .experimentalFeature = Xp::FetchTree,
 });

src/libfetchers/fetchers.cc

@@ -26,18 +26,9 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
         throw Error("Input scheme with name %s already registered", schemeName);
 }
 
-nlohmann::json dumpRegisterInputSchemeInfo()
+const InputSchemeMap & getAllInputSchemes()
 {
-    using nlohmann::json;
+    return inputSchemes();
-
-    auto res = json::object();
-
-    for (auto & [name, scheme] : inputSchemes()) {
-        auto & r = res[name] = json::object();
-        r["allowedAttrs"] = scheme->allowedAttrs();
-    }
-
-    return res;
 }
 
 Input Input::fromURL(const Settings & settings, const std::string & url, bool requireTree)

src/libfetchers/git.cc

@@ -194,27 +194,181 @@ struct GitInputScheme : InputScheme
         return "git";
     }
 
-    StringSet allowedAttrs() const override
+    std::string schemeDescription() const override
+    {
+        return stripIndentation(R"(
+          Fetch a Git tree and copy it to the Nix store.
+          This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
+        )");
+    }
+
+    std::map<std::string, AttributeInfo> allowedAttrs() const override
     {
         return {
+            {
                 "url",
+                {
+                    .type = "String",
+                    .required = true,
+                    .doc = R"(
+                      The URL formats supported are the same as for Git itself.
+
+                      > **Example**
+                      >
+                      > ```nix
+                      > fetchTree {
+                      >   type = "git";
+                      >   url = "git@github.com:NixOS/nixpkgs.git";
+                      > }
+                      > ```
+
+                      > **Note**
+                      >
+                      > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
+                    )",
+                },
+            },
+            {
                 "ref",
+                {
+                    .type = "String",
+                    .required = false,
+                    .doc = R"(
+                      By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+                      A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
+
+                      Default: `"HEAD"`
+                    )",
+                },
+            },
+            {
                 "rev",
+                {
+                    .type = "String",
+                    .required = false,
+                    .doc = R"(
+                      A Git revision; a commit hash.
+
+                      Default: the tip of `ref`
+                    )",
+                },
+            },
+            {
                 "shallow",
+                {
+                    .type = "Bool",
+                    .required = false,
+                    .doc = R"(
+                      Make a shallow clone when fetching the Git tree.
+                      When this is enabled, the options `ref` and `allRefs` have no effect anymore.
+
+                      Default: `true`
+                    )",
+                },
+            },
+            {
                 "submodules",
+                {
+                    .type = "Bool",
+                    .required = false,
+                    .doc = R"(
+                      Also fetch submodules if available.
+
+                      Default: `false`
+                    )",
+                },
+            },
+            {
                 "lfs",
-            "exportIgnore",
+                {
+                    .type = "Bool",
+                    .required = false,
+                    .doc = R"(
+                      Fetch any [Git LFS](https://git-lfs.com/) files.
+
+                      Default: `false`
+                    )",
+                },
+            },
+            {
                 "lastModified",
+                {
+                    .type = "integer",
+                    .required = false,
+                    .doc = R"(
+                      Unix timestamp of the fetched commit.
+
+                      If set, pass through the value to the output attribute set.
+                      Otherwise, generated from the fetched Git tree.
+                    )",
+                },
+            },
+            {
                 "revCount",
+                {
+                    .type = "integer",
+                    .required = false,
+                    .doc = R"(
+                      Number of revisions in the history of the Git repository before the fetched commit.
+
+                      If set, pass through the value to the output attribute set.
+                      Otherwise, generated from the fetched Git tree.
+                    )",
+                },
+            },
+            {
                 "narHash",
+                {},
+            },
+            {
                 "allRefs",
+                {
+                    .type = "Bool",
+                    .required = false,
+                    .doc = R"(
+                      By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+                      Whether to fetch all references (eg. branches and tags) of the repository.
+                      With this argument being true, it's possible to load a `rev` from *any* `ref`.
+                      (Without setting this option, only `rev`s from the specified `ref` are supported).
+
+                      Default: `false`
+                    )",
+                },
+            },
+            {
                 "name",
+                {},
+            },
+            {
                 "dirtyRev",
+                {},
+            },
+            {
                 "dirtyShortRev",
+                {},
+            },
+            {
+                "exportIgnore",
+                {},
+            },
+            {
                 "verifyCommit",
+                {},
+            },
+            {
                 "keytype",
+                {},
+            },
+            {
                 "publicKey",
+                {},
+            },
+            {
                 "publicKeys",
+                {},
+            },
         };
     }
 

src/libfetchers/github.cc

@@ -110,17 +110,41 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
-    StringSet allowedAttrs() const override
+    std::map<std::string, AttributeInfo> allowedAttrs() const override
     {
         return {
+            {
                 "owner",
+                {},
+            },
+            {
                 "repo",
+                {},
+            },
+            {
                 "ref",
+                {},
+            },
+            {
                 "rev",
+                {},
+            },
+            {
                 "narHash",
+                {},
+            },
+            {
                 "lastModified",
+                {},
+            },
+            {
                 "host",
+                {},
+            },
+            {
                 "treeHash",
+                {},
+            },
         };
     }
 
@@ -361,6 +385,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
         return "github";
     }
 
+    std::string schemeDescription() const override
+    {
+        // TODO
+        return "";
+    }
+
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // Github supports PAT/OAuth2 tokens and HTTP Basic
@@ -442,6 +472,12 @@ struct GitLabInputScheme : GitArchiveInputScheme
         return "gitlab";
     }
 
+    std::string schemeDescription() const override
+    {
+        // TODO
+        return "";
+    }
+
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // Gitlab supports 4 kinds of authorization, two of which are
@@ -526,6 +562,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         return "sourcehut";
     }
 
+    std::string schemeDescription() const override
+    {
+        // TODO
+        return "";
+    }
+
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // SourceHut supports both PAT and OAuth2. See

src/libfetchers/include/nix/fetchers/fetchers.hh

@@ -210,14 +210,26 @@ struct InputScheme
      */
     virtual std::string_view schemeName() const = 0;
 
+    /**
+     * Longform description of this scheme, for documentation purposes.
+     */
+    virtual std::string schemeDescription() const = 0;
+
+    // TODO remove these defaults
+    struct AttributeInfo {
+        const char * type = "String";
+        bool required = true;
+        const char * doc = "";
+    };
+
     /**
      * Allowed attributes in an attribute set that is converted to an
-     * input.
+     * input, and documentation for each attribute.
      *
-     * `type` is not included from this set, because the `type` field is
+     * `type` is not included from this map, because the `type` field is
       parsed first to choose which scheme; `type` is always required.
      */
-    virtual StringSet allowedAttrs() const = 0;
+    virtual std::map<std::string, AttributeInfo> allowedAttrs() const = 0;
 
     virtual ParsedURL toURL(const Input & input) const;
 
@@ -269,7 +281,12 @@ struct InputScheme
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
 
-nlohmann::json dumpRegisterInputSchemeInfo();
+using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
+
+/**
+ * Use this for docs, not for finding a specific scheme
+ */
+const InputSchemeMap & getAllInputSchemes();
 
 struct PublicKey
 {

src/libfetchers/indirect.cc

@@ -60,13 +60,31 @@ struct IndirectInputScheme : InputScheme
         return "indirect";
     }
 
-    StringSet allowedAttrs() const override
+    std::string schemeDescription() const override
+    {
+        // TODO
+        return "";
+    }
+
+    std::map<std::string, AttributeInfo> allowedAttrs() const override
     {
         return {
+            {
                 "id",
+                {},
+            },
+            {
                 "ref",
+                {},
+            },
+            {
                 "rev",
+                {},
+            },
+            {
                 "narHash",
+                {},
+            },
         };
     }
 

src/libfetchers/mercurial.cc

@@ -68,15 +68,39 @@ struct MercurialInputScheme : InputScheme
         return "hg";
     }
 
-    StringSet allowedAttrs() const override
+    std::string schemeDescription() const override
+    {
+        // TODO
+        return "";
+    }
+
+    std::map<std::string, AttributeInfo> allowedAttrs() const override
     {
         return {
+            {
                 "url",
+                {},
+            },
+            {
                 "ref",
+                {},
+            },
+            {
                 "rev",
+                {},
+            },
+            {
                 "revCount",
+                {},
+            },
+            {
                 "narHash",
+                {},
+            },
+            {
                 "name",
+                {},
+            },
         };
     }
 

src/libfetchers/path.cc

@@ -40,19 +40,40 @@ struct PathInputScheme : InputScheme
         return "path";
     }
 
-    StringSet allowedAttrs() const override
+    std::string schemeDescription() const override
+    {
+        // TODO
+        return "";
+    }
+
+    std::map<std::string, AttributeInfo> allowedAttrs() const override
     {
         return {
+            {
                 "path",
+                {},
+            },
             /* Allow the user to pass in "fake" tree info
                attributes. This is useful for making a pinned tree work
                the same as the repository from which is exported (e.g.
                path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
              */
+            {
                 "rev",
+                {},
+            },
+            {
                 "revCount",
+                {},
+            },
+            {
                 "lastModified",
+                {},
+            },
+            {
                 "narHash",
+                {},
+            },
         };
     }
 

src/libfetchers/tarball.cc

@@ -278,7 +278,7 @@ struct CurlInputScheme : InputScheme
            HTTP request. Now that we've processed the Nix-specific
            attributes above, remove them so we don't also send them as
            part of the HTTP request. */
-        for (auto & param : allowedAttrs())
+        for (auto & [param, _] : allowedAttrs())
             url.query.erase(param);
 
         input.attrs.insert_or_assign("type", std::string{schemeName()});
@@ -286,17 +286,85 @@ struct CurlInputScheme : InputScheme
         return input;
     }
 
-    StringSet allowedAttrs() const override
+    std::map<std::string, AttributeInfo> allowedAttrs() const override
     {
         return {
-            "type",
+            {
                 "url",
+                {
+                    .type = "String",
+                    .required = true,
+                    .doc = R"(
+                      Supported protocols:
+
+                      - `https`
+
+                        > **Example**
+                        >
+                        > ```nix
+                        > fetchTree {
+                        >   type = "file";
+                        >   url = "https://example.com/index.html";
+                        > }
+                        > ```
+
+                      - `http`
+
+                        Insecure HTTP transfer for legacy sources.
+
+                        > **Warning**
+                        >
+                        > HTTP performs no encryption or authentication.
+                        > Use a `narHash` known in advance to ensure the output has expected contents.
+
+                      - `file`
+
+                        A file on the local file system.
+
+                        > **Example**
+                        >
+                        > ```nix
+                        > fetchTree {
+                        >   type = "file";
+                        >   url = "file:///home/eelco/nix/README.md";
+                        > }
+                        > ```
+
+                      > **Example**
+                      >
+                      > ```nix
+                      > fetchTree {
+                      >   type = "tarball";
+                      >   url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
+                      > }
+                      > ```
+                    )",
+                },
+            },
+            {
                 "narHash",
+                {},
+            },
+            {
                 "name",
+                {},
+            },
+            {
                 "unpack",
+                {},
+            },
+            {
                 "rev",
+                {},
+            },
+            {
                 "revCount",
+                {},
+            },
+            {
                 "lastModified",
+                {},
+            },
         };
     }
 
@@ -332,6 +400,14 @@ struct FileInputScheme : CurlInputScheme
         return "file";
     }
 
+    std::string schemeDescription() const override
+    {
+        return stripIndentation(R"(
+          Place a plain file into the Nix store.
+          This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
+        )");
+    }
+
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
         auto parsedUrlScheme = parseUrlScheme(url.scheme);
@@ -368,6 +444,15 @@ struct TarballInputScheme : CurlInputScheme
         return "tarball";
     }
 
+    std::string schemeDescription() const override
+    {
+        return stripIndentation(R"(
+          Download a tar archive and extract it into the Nix store.
+          This has the same underyling implementation as [`builtins.fetchTarball`](@doc
+    root@/language/builtins.md#builtins-fetchTarball)
+        )");
+    }
+
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
         auto parsedUrlScheme = parseUrlScheme(url.scheme);

src/libutil/include/nix/util/cmark-cpp.hh

@@ -0,0 +1,85 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "util.hh"
+
+#include <cmark.h>
+
+namespace nix::cmark {
+
+using Node = struct cmark_node;
+using NodeType = cmark_node_type;
+using ListType = cmark_list_type;
+
+using Iter = struct cmark_iter;
+
+struct Deleter
+{
+    void operator () (Node * ptr) { cmark_node_free(ptr); }
+    void operator () (Iter * ptr) { cmark_iter_free(ptr); }
+};
+
+template <typename T>
+using UniquePtr = std::unique_ptr<Node, Deleter>;
+
+static inline void parse_document(Node & root, std::string_view s, int options)
+{
+    cmark_parser * parser = cmark_parser_new_with_mem_into_root(
+        options,
+        cmark_get_default_mem_allocator(),
+        &root);
+    cmark_parser_feed(parser, s.data(), s.size());
+    (void) cmark_parser_finish(parser);
+    cmark_parser_free(parser);
+}
+
+static inline UniquePtr<Node> parse_document(std::string_view s, int options)
+{
+    return UniquePtr<Node> {
+        cmark_parse_document(s.data(), s.size(), options)
+    };
+}
+
+static inline std::unique_ptr<char, FreeDeleter> render_commonmark(Node & root, int options, int width)
+{
+    return std::unique_ptr<char, FreeDeleter> {
+        cmark_render_commonmark(&root, options, width)
+    };
+}
+
+static inline std::unique_ptr<char, FreeDeleter> render_xml(Node & root, int options)
+{
+    return std::unique_ptr<char, FreeDeleter> {
+        cmark_render_xml(&root, options)
+    };
+}
+
+static inline UniquePtr<Node> node_new(NodeType type)
+{
+    return UniquePtr<Node> {
+        cmark_node_new(type)
+    };
+}
+
+/**
+ * The parent takes ownership
+ */
+static inline Node & node_append_child(Node & node, UniquePtr<Node> child)
+{
+    auto status = (bool) cmark_node_append_child(&node, &*child);
+    assert(status);
+    return *child.release();
+}
+
+static inline bool node_set_literal(Node & node, const char * content)
+{
+    return (bool) cmark_node_set_literal(&node, content);
+}
+
+static inline bool node_set_list_type(Node & node, ListType type)
+{
+    return (bool) cmark_node_set_list_type(&node, type);
+}
+
+}

src/libutil/include/nix/util/meson.build

@@ -13,6 +13,7 @@ headers = files(
   'base-n.hh',
   'base-nix-32.hh',
   'callback.hh',
+  'cmark-cpp.hh',
   'canon-path.hh',
   'checked-arithmetic.hh',
   'chunked-vector.hh',

src/libutil/include/nix/util/util.hh

@@ -369,6 +369,16 @@ struct overloaded : Ts...
 template<class... Ts>
 overloaded(Ts...) -> overloaded<Ts...>;
 
+/**
+ * For using `std::unique` with C functions.
+ */
+struct FreeDeleter
+{
+    template <typename T>
+    void operator()(T *p) const { std::free(p); }
+};
+
+
 /**
  * Provide an addition operator between strings and string_views
  * inexplicably omitted from the standard library.

src/libutil/meson.build

@@ -110,6 +110,9 @@ deps_private += cpuid
 nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
 deps_public += nlohmann_json
 
+cmark = dependency('libcmark', required : true)
+deps_public += cmark
+
 cxx = meson.get_compiler('cpp')
 
 config_priv_h = configure_file(

src/libutil/package.nix

@@ -5,6 +5,7 @@
 
   boost,
   brotli,
+  cmark,
   libarchive,
   libblake3,
   libcpuid,
@@ -57,6 +58,7 @@ mkMesonLibrary (finalAttrs: {
 
   propagatedBuildInputs = [
     boost
+    cmark
     libarchive
     nlohmann_json
   ];

src/nix/main.cc

@@ -193,11 +193,14 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
 
     std::string dumpCli()
     {
-        auto res = nlohmann::json::object();
+        using nlohmann::json;
+
+        auto res = json::object();
 
         res["args"] = toJSON();
 
-        auto stores = nlohmann::json::object();
+        {
+            auto & stores = res["stores"] = json::object();
             for (auto & [storeName, implem] : Implementations::registered()) {
                 auto & j = stores[storeName];
                 j["doc"] = implem.doc;
@@ -205,8 +208,24 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
                 j["settings"] = implem.getConfig()->toJSON();
                 j["experimentalFeature"] = implem.experimentalFeature;
             }
-        res["stores"] = std::move(stores);
+        }
-        res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo();
+
+        {
+            auto & fetchers = res["fetchers"] = json::object();
+
+            for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
+                auto & s = fetchers[schemeName] = json::object();
+                s["description"] = scheme->schemeDescription();
+                auto & attrs = s["allowedAttrs"] = json::object();
+                for (auto & [fieldName, field] : scheme->allowedAttrs()) {
+                    auto & f = attrs[fieldName] = json::object();
+                    f["type"] = field.type;
+                    f["required"] = field.required;
+                    f["doc"] = stripIndentation(field.doc);
+                }
+            }
+
+        };
 
         return res.dump();
     }
@@ -437,7 +456,7 @@ void mainWrapped(int argc, char ** argv)
             if (!builtin.value->isPrimOp())
                 continue;
             auto primOp = builtin.value->primOp();
-            if (!primOp->doc)
+            if (primOp->doc == "")
                 continue;
             b["args"] = primOp->args;
             b["doc"] = trim(stripIndentation(primOp->doc));