nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-14 14:32:42 +01:00
Code Activity

Systematize builtins.fetchTree docs

Browse source 
And also render the docs nicely.

I would like to use a markdown AST for this, but to avoid new deps
(lowdown's AST doesn't suffice) I am just doing crude string
manipulations for now.
This commit is contained in:
John Ericson 2025-11-10 19:58:01 -05:00
parent b5cae2741b
commit 73f9e39349
13 changed files with 569 additions and 280 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/libexpr/eval.cc
View file

@ -525,7 +525,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
v,
{
.type = nThunk, // FIXME
.doc = primOp.doc,
.doc = primOp.doc.c_str(),
});
}
@ -565,13 +565,14 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
{
if (v.isPrimOp()) {
auto v2 = &v;
if (auto * doc = v2->primOp()->doc)
auto & doc = v2->primOp()->doc;
if (doc != "")
return Doc{
.pos = {},
.name = v2->primOp()->name,
.arity = v2->primOp()->arity,
.args = v2->primOp()->args,
.doc = doc,
.doc = doc.c_str(),
};
}
if (v.isLambda()) {

2
src/libexpr/include/nix/expr/eval.hh
View file

@ -108,7 +108,7 @@ struct PrimOp
/**
* Optional free-form documentation about the primop.
*/
const char * doc = nullptr;
const std::string doc = "";
/**
* Add a trace item, while calling the `<name>` builtin.

170
src/libexpr/primops/fetchTree.cc
View file

@ -234,7 +234,8 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, V
static RegisterPrimOp primop_fetchTree({
.name = "fetchTree",
.args = {"input"},
.doc = R"(
.doc = []() -> std::string {
std::string doc = stripIndentation(R"(
Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
- the resulting fixed-output [store path](@docroot@/store/store-path.md)
@ -274,144 +275,38 @@ static RegisterPrimOp primop_fetchTree({
<!-- TODO: It would be soooo much more predictable to work with (and
document) if `fetchTree` was a curried call with the first parameter for
`type` or an attribute like `builtins.fetchTree.git`! -->
)");
- `"file"`
auto indentString = [](std::string const & str, std::string const & indent) {
std::string result;
std::istringstream stream(str);
std::string line;
bool first = true;
while (std::getline(stream, line)) {
if (!first)
result += "\n";
result += indent + line;
first = false;
}
return result;
};
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
doc += "\n- `" + quoteString(schemeName, '"') + "`\n\n";
doc += indentString(scheme->schemeDescription(), " ");
if (!doc.empty() && doc.back() != '\n')
doc += "\n";
- `url` (String, required)
Supported protocols:
- `https`
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
- `http`
Insecure HTTP transfer for legacy sources.
> **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
- `file`
A file on the local file system.
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
- `"git"`
Fetch a Git tree and copy it to the Nix store.
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
- `allRefs` (Bool, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
- `lastModified` (Integer, optional)
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
- `lfs` (Bool, optional)
Fetch any [Git LFS](https://git-lfs.com/) files.
Default: `false`
- `ref` (String, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
- `rev` (String, optional)
A Git revision; a commit hash.
Default: the tip of `ref`
- `revCount` (Integer, optional)
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
- `shallow` (Bool, optional)
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
- `submodules` (Bool, optional)
Also fetch submodules if available.
Default: `false`
- `url` (String, required)
The URL formats supported are the same as for Git itself.
> **Example**
>
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> }
> ```
> **Note**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
- `"tarball"`
Download a tar archive and extract it into the Nix store.
This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
- `url` (String, required)
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
for (const auto & [attrName, attribute] : scheme->allowedAttrs()) {
doc += "\n - `" + attrName + "` (" + attribute.type + ", "
+ (attribute.required ? "required" : "optional") + ")\n\n";
doc += indentString(stripIndentation(attribute.doc), " ");
if (!doc.empty() && doc.back() != '\n')
doc += "\n";
}
}
doc += "\n" + stripIndentation(R"(
The following input types are still subject to change:
- `"path"`
@ -456,7 +351,10 @@ static RegisterPrimOp primop_fetchTree({
> ```nix
> builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
> ```
)",
)");
return doc;
}(),
.fun = prim_fetchTree,
.experimentalFeature = Xp::FetchTree,
});

13
src/libfetchers/fetchers.cc
View file

@ -26,18 +26,9 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
throw Error("Input scheme with name %s already registered", schemeName);
}
nlohmann::json dumpRegisterInputSchemeInfo()
const InputSchemeMap & getAllInputSchemes()
{
using nlohmann::json;
auto res = json::object();
for (auto & [name, scheme] : inputSchemes()) {
auto & r = res[name] = json::object();
r["allowedAttrs"] = scheme->allowedAttrs();
}
return res;
return inputSchemes();
}
Input Input::fromURL(const Settings & settings, const std::string & url, bool requireTree)

156
src/libfetchers/git.cc
View file

@ -194,27 +194,181 @@ struct GitInputScheme : InputScheme
return "git";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
return stripIndentation(R"(
Fetch a Git tree and copy it to the Nix store.
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
)");
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
{
"url",
{
.type = "String",
.required = true,
.doc = R"(
The URL formats supported are the same as for Git itself.
> **Example**
>
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> }
> ```
> **Note**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but uses the *current file contents* of the Git working directory.
)",
},
},
{
"ref",
{
.type = "String",
.required = false,
.doc = R"(
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
)",
},
},
{
"rev",
{
.type = "String",
.required = false,
.doc = R"(
A Git revision; a commit hash.
Default: the tip of `ref`
)",
},
},
{
"shallow",
{
.type = "Bool",
.required = false,
.doc = R"(
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
)",
},
},
{
"submodules",
{
.type = "Bool",
.required = false,
.doc = R"(
Also fetch submodules if available.
Default: `false`
)",
},
},
{
"lfs",
{
.type = "Bool",
.required = false,
.doc = R"(
Fetch any [Git LFS](https://git-lfs.com/) files.
Default: `false`
)",
},
},
{
"exportIgnore",
{},
},
{
"lastModified",
{
.type = "Integer",
.required = false,
.doc = R"(
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
)",
},
},
{
"revCount",
{
.type = "Integer",
.required = false,
.doc = R"(
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
)",
},
},
{
"narHash",
{},
},
{
"allRefs",
{
.type = "Bool",
.required = false,
.doc = R"(
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
)",
},
},
{
"name",
{},
},
{
"dirtyRev",
{},
},
{
"dirtyShortRev",
{},
},
{
"verifyCommit",
{},
},
{
"keytype",
{},
},
{
"publicKey",
{},
},
{
"publicKeys",
{},
},
};
}

44
src/libfetchers/github.cc
View file

@ -110,17 +110,41 @@ struct GitArchiveInputScheme : InputScheme
return input;
}
StringSet allowedAttrs() const override
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
{
"owner",
{},
},
{
"repo",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"narHash",
{},
},
{
"lastModified",
{},
},
{
"host",
{},
},
{
"treeHash",
{},
},
};
}
@ -361,6 +385,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
return "github";
}
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// Github supports PAT/OAuth2 tokens and HTTP Basic
@ -442,6 +472,12 @@ struct GitLabInputScheme : GitArchiveInputScheme
return "gitlab";
}
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// Gitlab supports 4 kinds of authorization, two of which are
@ -526,6 +562,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme
return "sourcehut";
}
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// SourceHut supports both PAT and OAuth2. See

26
src/libfetchers/include/nix/fetchers/fetchers.hh
View file

@ -210,14 +210,27 @@ struct InputScheme
*/
virtual std::string_view schemeName() const = 0;
/**
* Longform description of this scheme, for documentation purposes.
*/
virtual std::string schemeDescription() const = 0;
// TODO remove these defaults
struct AttributeInfo
{
const char * type = "String";
bool required = true;
const char * doc = "";
};
/**
* Allowed attributes in an attribute set that is converted to an
* input.
* input, and documentation for each attribute.
*
* `type` is not included from this set, because the `type` field is
* `type` is not included from this map, because the `type` field is
parsed first to choose which scheme; `type` is always required.
*/
virtual StringSet allowedAttrs() const = 0;
virtual std::map<std::string, AttributeInfo> allowedAttrs() const = 0;
virtual ParsedURL toURL(const Input & input) const;
@ -269,7 +282,12 @@ struct InputScheme
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
nlohmann::json dumpRegisterInputSchemeInfo();
using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
/**
* Use this for docs, not for finding a specific scheme
*/
const InputSchemeMap & getAllInputSchemes();
struct PublicKey
{

20
src/libfetchers/indirect.cc
View file

@ -60,13 +60,31 @@ struct IndirectInputScheme : InputScheme
return "indirect";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
// TODO
return "";
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
{
"id",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"narHash",
{},
},
};
}

26
src/libfetchers/mercurial.cc
View file

@ -68,15 +68,39 @@ struct MercurialInputScheme : InputScheme
return "hg";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
// TODO
return "";
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
{
"url",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"revCount",
{},
},
{
"narHash",
{},
},
{
"name",
{},
},
};
}

23
src/libfetchers/path.cc
View file

@ -40,19 +40,40 @@ struct PathInputScheme : InputScheme
return "path";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
// TODO
return "";
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
{
"path",
{},
},
/* Allow the user to pass in "fake" tree info
attributes. This is useful for making a pinned tree work
the same as the repository from which is exported (e.g.
path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
*/
{
"rev",
{},
},
{
"revCount",
{},
},
{
"lastModified",
{},
},
{
"narHash",
{},
},
};
}

98
src/libfetchers/tarball.cc
View file

@ -278,7 +278,7 @@ struct CurlInputScheme : InputScheme
HTTP request. Now that we've processed the Nix-specific
attributes above, remove them so we don't also send them as
part of the HTTP request. */
for (auto & param : allowedAttrs())
for (auto & [param, _] : allowedAttrs())
url.query.erase(param);
input.attrs.insert_or_assign("type", std::string{schemeName()});
@ -286,17 +286,76 @@ struct CurlInputScheme : InputScheme
return input;
}
StringSet allowedAttrs() const override
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"type",
{
"url",
{
.type = "String",
.required = true,
.doc = R"(
Supported protocols:
- `https`
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
- `http`
Insecure HTTP transfer for legacy sources.
> **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
- `file`
A file on the local file system.
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
)",
},
},
{
"narHash",
{},
},
{
"name",
{},
},
{
"unpack",
{},
},
{
"rev",
{},
},
{
"revCount",
{},
},
{
"lastModified",
{},
},
};
}
@ -332,6 +391,14 @@ struct FileInputScheme : CurlInputScheme
return "file";
}
std::string schemeDescription() const override
{
return stripIndentation(R"(
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
)");
}
bool isValidURL(const ParsedURL & url, bool requireTree) const override
{
auto parsedUrlScheme = parseUrlScheme(url.scheme);
@ -368,6 +435,31 @@ struct TarballInputScheme : CurlInputScheme
return "tarball";
}
std::string schemeDescription() const override
{
return stripIndentation(R"(
Download a tar archive and extract it into the Nix store.
This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
)");
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
auto attrs = CurlInputScheme::allowedAttrs();
// Override the "url" attribute to add tarball-specific example
attrs["url"].doc = R"(
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
)";
return attrs;
}
bool isValidURL(const ParsedURL & url, bool requireTree) const override
{
auto parsedUrlScheme = parseUrlScheme(url.scheme);

12
src/libutil/include/nix/util/util.hh
View file

@ -382,6 +382,18 @@ struct overloaded : Ts...
template<class... Ts>
overloaded(Ts...) -> overloaded<Ts...>;
/**
* For using `std::unique` with C functions.
*/
struct FreeDeleter
{
template<typename T>
void operator()(T * p) const
{
std::free(p);
}
};
/**
* Provide an addition operator between strings and string_views
* inexplicably omitted from the standard library.

28
src/nix/main.cc
View file

@ -193,11 +193,14 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
std::string dumpCli()
{
auto res = nlohmann::json::object();
using nlohmann::json;
auto res = json::object();
res["args"] = toJSON();
auto stores = nlohmann::json::object();
{
auto & stores = res["stores"] = json::object();
for (auto & [storeName, implem] : Implementations::registered()) {
auto & j = stores[storeName];
j["doc"] = implem.doc;
@ -205,8 +208,23 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
j["settings"] = implem.getConfig()->toJSON();
j["experimentalFeature"] = implem.experimentalFeature;
}
res["stores"] = std::move(stores);
res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo();
}
{
auto & fetchers = res["fetchers"] = json::object();
for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
auto & s = fetchers[schemeName] = json::object();
s["description"] = scheme->schemeDescription();
auto & attrs = s["allowedAttrs"] = json::object();
for (auto & [fieldName, field] : scheme->allowedAttrs()) {
auto & f = attrs[fieldName] = json::object();
f["type"] = field.type;
f["required"] = field.required;
f["doc"] = stripIndentation(field.doc);
}
}
};
return res.dump();
}
@ -437,7 +455,7 @@ void mainWrapped(int argc, char ** argv)
if (!builtin.value->isPrimOp())
continue;
auto primOp = builtin.value->primOp();
if (!primOp->doc)
if (primOp->doc == "")
continue;
b["args"] = primOp->args;
b["doc"] = trim(stripIndentation(primOp->doc));