nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-19 16:59:35 +01:00
Code Activity

Systematize builtins.fetchTree docs

Browse source 
And also render the docs nicely.

I would like to use a markdown AST for this, but to avoid new deps
(lowdown's AST doesn't suffice) I am just doing crude string
manipulations for now.
This commit is contained in:
John Ericson 2025-11-10 19:58:01 -05:00
parent 2cc0b1b404
commit bae1ca257a
12 changed files with 583 additions and 291 deletions
Download patch file Download diff file Expand all files Collapse all files

18
src/libexpr/eval.cc
View file

@ -517,15 +517,16 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
if (primOp.arity == 0) {
primOp.arity = 1;
auto vPrimOp = allocValue();
vPrimOp->mkPrimOp(new PrimOp(primOp));
vPrimOp->mkPrimOp(new PrimOp(std::move(primOp)));
Value v;
v.mkApp(vPrimOp, vPrimOp);
auto & primOp1 = *vPrimOp->primOp();
return addConstant(
primOp.name,
primOp1.name,
v,
{
.type = nThunk, // FIXME
.doc = primOp.doc,
.doc = primOp1.doc ? primOp1.doc->c_str() : nullptr,
});
}
@ -565,13 +566,14 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
{
if (v.isPrimOp()) {
auto v2 = &v;
if (auto * doc = v2->primOp()->doc)
auto & primOp = *v2->primOp();
if (primOp.doc)
return Doc{
.pos = {},
.name = v2->primOp()->name,
.arity = v2->primOp()->arity,
.args = v2->primOp()->args,
.doc = doc,
.name = primOp.name,
.arity = primOp.arity,
.args = primOp.args,
.doc = primOp.doc->c_str(),
};
}
if (v.isLambda()) {

2
src/libexpr/include/nix/expr/eval.hh
View file

@ -108,7 +108,7 @@ struct PrimOp
/**
* Optional free-form documentation about the primop.
*/
const char * doc = nullptr;
std::optional<std::string> doc;
/**
* Add a trace item, while calling the `<name>` builtin.

170
src/libexpr/primops/fetchTree.cc
View file

@ -235,7 +235,8 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, V
static RegisterPrimOp primop_fetchTree({
.name = "fetchTree",
.args = {"input"},
.doc = R"(
.doc = []() -> std::string {
std::string doc = stripIndentation(R"(
Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
- the resulting fixed-output [store path](@docroot@/store/store-path.md)
@ -275,144 +276,38 @@ static RegisterPrimOp primop_fetchTree({
<!-- TODO: It would be soooo much more predictable to work with (and
document) if `fetchTree` was a curried call with the first parameter for
`type` or an attribute like `builtins.fetchTree.git`! -->
)");
- `"file"`
auto indentString = [](std::string const & str, std::string const & indent) {
std::string result;
std::istringstream stream(str);
std::string line;
bool first = true;
while (std::getline(stream, line)) {
if (!first)
result += "\n";
result += indent + line;
first = false;
}
return result;
};
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
doc += "\n- `" + quoteString(schemeName, '"') + "`\n\n";
doc += indentString(scheme->schemeDescription(), " ");
if (!doc.empty() && doc.back() != '\n')
doc += "\n";
- `url` (String, required)
Supported protocols:
- `https`
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
- `http`
Insecure HTTP transfer for legacy sources.
> **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
- `file`
A file on the local file system.
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
- `"git"`
Fetch a Git tree and copy it to the Nix store.
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
- `allRefs` (Bool, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
- `lastModified` (Integer, optional)
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
- `lfs` (Bool, optional)
Fetch any [Git LFS](https://git-lfs.com/) files.
Default: `false`
- `ref` (String, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
- `rev` (String, optional)
A Git revision; a commit hash.
Default: the tip of `ref`
- `revCount` (Integer, optional)
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
- `shallow` (Bool, optional)
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
- `submodules` (Bool, optional)
Also fetch submodules if available.
Default: `false`
- `url` (String, required)
The URL formats supported are the same as for Git itself.
> **Example**
>
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> }
> ```
> **Note**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
- `"tarball"`
Download a tar archive and extract it into the Nix store.
This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
- `url` (String, required)
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
for (const auto & [attrName, attribute] : scheme->allowedAttrs()) {
doc += "\n - `" + attrName + "` (" + attribute.type + ", "
+ (attribute.required ? "required" : "optional") + ")\n\n";
doc += indentString(stripIndentation(attribute.doc), " ");
if (!doc.empty() && doc.back() != '\n')
doc += "\n";
}
}
doc += "\n" + stripIndentation(R"(
The following input types are still subject to change:
- `"path"`
@ -457,7 +352,10 @@ static RegisterPrimOp primop_fetchTree({
> ```nix
> builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
> ```
)",
)");
return doc;
}(),
.fun = prim_fetchTree,
.experimentalFeature = Xp::FetchTree,
});

13
src/libfetchers/fetchers.cc
View file

@ -26,18 +26,9 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
throw Error("Input scheme with name %s already registered", schemeName);
}
nlohmann::json dumpRegisterInputSchemeInfo()
const InputSchemeMap & getAllInputSchemes()
{
using nlohmann::json;
auto res = json::object();
for (auto & [name, scheme] : inputSchemes()) {
auto & r = res[name] = json::object();
r["allowedAttrs"] = scheme->allowedAttrs();
}
return res;
return inputSchemes();
}
Input Input::fromURL(const Settings & settings, const std::string & url, bool requireTree)

159
src/libfetchers/git.cc
View file

@ -194,28 +194,183 @@ struct GitInputScheme : InputScheme
return "git";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
return stripIndentation(R"(
Fetch a Git tree and copy it to the Nix store.
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
)");
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
static const std::map<std::string, AttributeInfo> attrs = {
{
return {
"url",
{
.type = "String",
.required = true,
.doc = R"(
The URL formats supported are the same as for Git itself.
> **Example**
>
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> }
> ```
> **Note**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but uses the *current file contents* of the Git working directory.
)",
},
},
{
"ref",
{
.type = "String",
.required = false,
.doc = R"(
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
)",
},
},
{
"rev",
{
.type = "String",
.required = false,
.doc = R"(
A Git revision; a commit hash.
Default: the tip of `ref`
)",
},
},
{
"shallow",
{
.type = "Bool",
.required = false,
.doc = R"(
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
)",
},
},
{
"submodules",
{
.type = "Bool",
.required = false,
.doc = R"(
Also fetch submodules if available.
Default: `false`
)",
},
},
{
"lfs",
{
.type = "Bool",
.required = false,
.doc = R"(
Fetch any [Git LFS](https://git-lfs.com/) files.
Default: `false`
)",
},
},
{
"exportIgnore",
{},
},
{
"lastModified",
{
.type = "Integer",
.required = false,
.doc = R"(
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
)",
},
},
{
"revCount",
{
.type = "Integer",
.required = false,
.doc = R"(
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
)",
},
},
{
"narHash",
{},
},
{
"allRefs",
{
.type = "Bool",
.required = false,
.doc = R"(
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
)",
},
},
{
"name",
{},
},
{
"dirtyRev",
{},
},
{
"dirtyShortRev",
{},
},
{
"verifyCommit",
{},
},
{
"keytype",
{},
},
{
"publicKey",
{},
},
{
"publicKeys",
{},
},
};
return attrs;
}
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override

47
src/libfetchers/github.cc
View file

@ -110,18 +110,43 @@ struct GitArchiveInputScheme : InputScheme
return input;
}
StringSet allowedAttrs() const override
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
static const std::map<std::string, AttributeInfo> attrs = {
{
return {
"owner",
{},
},
{
"repo",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"narHash",
{},
},
{
"lastModified",
{},
},
{
"host",
{},
},
{
"treeHash",
{},
},
};
return attrs;
}
std::optional<Input> inputFromAttrs(const fetchers::Settings & settings, const Attrs & attrs) const override
@ -361,6 +386,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
return "github";
}
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// Github supports PAT/OAuth2 tokens and HTTP Basic
@ -442,6 +473,12 @@ struct GitLabInputScheme : GitArchiveInputScheme
return "gitlab";
}
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// Gitlab supports 4 kinds of authorization, two of which are
@ -526,6 +563,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme
return "sourcehut";
}
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// SourceHut supports both PAT and OAuth2. See

26
src/libfetchers/include/nix/fetchers/fetchers.hh
View file

@ -203,14 +203,27 @@ struct InputScheme
*/
virtual std::string_view schemeName() const = 0;
/**
* Longform description of this scheme, for documentation purposes.
*/
virtual std::string schemeDescription() const = 0;
// TODO remove these defaults
struct AttributeInfo
{
const char * type = "String";
bool required = true;
const char * doc = "";
};
/**
* Allowed attributes in an attribute set that is converted to an
* input.
* input, and documentation for each attribute.
*
* `type` is not included from this set, because the `type` field is
* `type` is not included from this map, because the `type` field is
parsed first to choose which scheme; `type` is always required.
*/
virtual StringSet allowedAttrs() const = 0;
virtual const std::map<std::string, AttributeInfo> & allowedAttrs() const = 0;
virtual ParsedURL toURL(const Input & input) const;
@ -263,7 +276,12 @@ struct InputScheme
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
nlohmann::json dumpRegisterInputSchemeInfo();
using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
/**
* Use this for docs, not for finding a specific scheme
*/
const InputSchemeMap & getAllInputSchemes();
struct PublicKey
{

23
src/libfetchers/indirect.cc
View file

@ -60,14 +60,33 @@ struct IndirectInputScheme : InputScheme
return "indirect";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
// TODO
return "";
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
static const std::map<std::string, AttributeInfo> attrs = {
{
return {
"id",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"narHash",
{},
},
};
return attrs;
}
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override

29
src/libfetchers/mercurial.cc
View file

@ -68,16 +68,41 @@ struct MercurialInputScheme : InputScheme
return "hg";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
// TODO
return "";
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
static const std::map<std::string, AttributeInfo> attrs = {
{
return {
"url",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"revCount",
{},
},
{
"narHash",
{},
},
{
"name",
{},
},
};
return attrs;
}
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override

26
src/libfetchers/path.cc
View file

@ -40,20 +40,42 @@ struct PathInputScheme : InputScheme
return "path";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
// TODO
return "";
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
static const std::map<std::string, AttributeInfo> attrs = {
{
return {
"path",
{},
},
/* Allow the user to pass in "fake" tree info
attributes. This is useful for making a pinned tree work
the same as the repository from which is exported (e.g.
path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
*/
{
"rev",
{},
},
{
"revCount",
{},
},
{
"lastModified",
{},
},
{
"narHash",
{},
},
};
return attrs;
}
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override

109
src/libfetchers/tarball.cc
View file

@ -278,7 +278,7 @@ struct CurlInputScheme : InputScheme
HTTP request. Now that we've processed the Nix-specific
attributes above, remove them so we don't also send them as
part of the HTTP request. */
for (auto & param : allowedAttrs())
for (auto & [param, _] : allowedAttrs())
url.query.erase(param);
input.attrs.insert_or_assign("type", std::string{schemeName()});
@ -286,18 +286,83 @@ struct CurlInputScheme : InputScheme
return input;
}
StringSet allowedAttrs() const override
static const std::map<std::string, AttributeInfo> & allowedAttrsImpl()
{
static const std::map<std::string, AttributeInfo> attrs = {
{
return {
"type",
"url",
{
.type = "String",
.required = true,
.doc = R"(
Supported protocols:
- `https`
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
- `http`
Insecure HTTP transfer for legacy sources.
> **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
- `file`
A file on the local file system.
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
)",
},
},
{
"narHash",
{},
},
{
"name",
{},
},
{
"unpack",
{},
},
{
"rev",
{},
},
{
"revCount",
{},
},
{
"lastModified",
{},
},
};
return attrs;
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
return allowedAttrsImpl();
}
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@ -332,6 +397,14 @@ struct FileInputScheme : CurlInputScheme
return "file";
}
std::string schemeDescription() const override
{
return stripIndentation(R"(
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
)");
}
bool isValidURL(const ParsedURL & url, bool requireTree) const override
{
auto parsedUrlScheme = parseUrlScheme(url.scheme);
@ -369,6 +442,34 @@ struct TarballInputScheme : CurlInputScheme
return "tarball";
}
std::string schemeDescription() const override
{
return stripIndentation(R"(
Download a tar archive and extract it into the Nix store.
This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
)");
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
static const std::map<std::string, AttributeInfo> attrs = [] {
auto attrs = CurlInputScheme::allowedAttrsImpl();
// Override the "url" attribute to add tarball-specific example
attrs["url"].doc = R"(
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
)";
return attrs;
}();
return attrs;
}
bool isValidURL(const ParsedURL & url, bool requireTree) const override
{
auto parsedUrlScheme = parseUrlScheme(url.scheme);

28
src/nix/main.cc
View file

@ -193,11 +193,14 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
std::string dumpCli()
{
auto res = nlohmann::json::object();
using nlohmann::json;
auto res = json::object();
res["args"] = toJSON();
auto stores = nlohmann::json::object();
{
auto & stores = res["stores"] = json::object();
for (auto & [storeName, implem] : Implementations::registered()) {
auto & j = stores[storeName];
j["doc"] = implem.doc;
@ -205,8 +208,23 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
j["settings"] = implem.getConfig()->toJSON();
j["experimentalFeature"] = implem.experimentalFeature;
}
res["stores"] = std::move(stores);
res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo();
}
{
auto & fetchers = res["fetchers"] = json::object();
for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
auto & s = fetchers[schemeName] = json::object();
s["description"] = scheme->schemeDescription();
auto & attrs = s["allowedAttrs"] = json::object();
for (auto & [fieldName, field] : scheme->allowedAttrs()) {
auto & f = attrs[fieldName] = json::object();
f["type"] = field.type;
f["required"] = field.required;
f["doc"] = stripIndentation(field.doc);
}
}
};
return res.dump();
}
@ -440,7 +458,7 @@ void mainWrapped(int argc, char ** argv)
if (!primOp->doc)
continue;
b["args"] = primOp->args;
b["doc"] = trim(stripIndentation(primOp->doc));
b["doc"] = trim(stripIndentation(*primOp->doc));
if (primOp->experimentalFeature)
b["experimental-feature"] = primOp->experimentalFeature;
builtinsJson.emplace(state.symbols[builtin.name], std::move(b));