From 23f07fa974f37c4660066e0dcaf2d3c9c1ecabd5 Mon Sep 17 00:00:00 2001 From: Illia Bobyr Date: Mon, 13 Jan 2025 19:20:13 -0800 Subject: [PATCH 001/396] nix-profile.fish: Look for ca-bundle.crt in $NIX_PROFILES There seems to be no good reason for `nix-profile.fish` and `nix-profile-daemon.fish` to differ in how they look for the location of the `ca-bundle.crt` that might be installed by one of the packages. As `$NIX_PROFILES` points to user local paths, not checking there is strictly less useful, it seems? --- scripts/nix-profile.fish.in | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index 05d9a187d..59b3c695e 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -53,6 +53,13 @@ else if test -e "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" # fall back to cacert in set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" +else + # Fall back to what is in the nix profiles, favouring whatever is defined last. + for i in (string split ' ' $NIX_PROFILES) + if test -e "$i/etc/ssl/certs/ca-bundle.crt" + set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" + end + end end # Only use MANPATH if it is already set. In general `man` will just simply From 068cdfafb80d05884affc1bb3374b1c6b5754e85 Mon Sep 17 00:00:00 2001 From: Thomas Bereknyei Date: Wed, 26 Feb 2025 15:39:22 -0500 Subject: [PATCH 002/396] fix: update work meeting calendar link --- maintainers/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maintainers/README.md b/maintainers/README.md index fd9bbed8e..6553cd048 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -37,7 +37,7 @@ The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/n The team meets twice a week (times are denoted in the [Europe/Amsterdam](https://en.m.wikipedia.org/wiki/Time_in_the_Netherlands) time zone): -- Discussion meeting: [Wednesday 21:00-22:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=ZG5rZzNyajRjajducGV2NGY5aGkzYWIwdnJfMjAyNDA1MDhUMTkwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Discussion meeting: Wednesday 21:00-22:00 Europe/Amsterdam see [calendar](https://calendar.google.com/calendar/u/0/embed?src=b9o52fobqjak8oq8lfkhg3t0qg@group.calendar.google.com). 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min). @@ -46,7 +46,7 @@ The team meets twice a week (times are denoted in the [Europe/Amsterdam](https:/ - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: Mondays 14:00-16:00 Europe/Amsterdam see [calendar](https://calendar.google.com/calendar/u/0/embed?src=b9o52fobqjak8oq8lfkhg3t0qg@group.calendar.google.com). 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. From 2aa6e0f08499ff580ae78ba4b3ec1410a10a67f1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 10 Feb 2025 01:08:00 -0500 Subject: [PATCH 003/396] Expand manual on derivation outputs Note, this includes some text adapted from from Eelco's dissertation --- .gitignore | 2 +- doc/manual/source/SUMMARY.md.in | 5 +- doc/manual/source/glossary.md | 6 +- .../source/language/advanced-attributes.md | 263 +++++++++--------- doc/manual/source/language/derivations.md | 10 +- doc/manual/source/store/building.md | 2 +- .../store/{drv.md => derivation/index.md} | 41 +-- .../derivation/outputs/content-address.md | 192 +++++++++++++ .../source/store/derivation/outputs/index.md | 97 +++++++ .../store/derivation/outputs/input-address.md | 31 +++ .../store/store-object/content-address.md | 29 +- src/libexpr/primops.cc | 4 +- 12 files changed, 508 insertions(+), 174 deletions(-) rename doc/manual/source/store/{drv.md => derivation/index.md} (89%) create mode 100644 doc/manual/source/store/derivation/outputs/content-address.md create mode 100644 doc/manual/source/store/derivation/outputs/index.md create mode 100644 doc/manual/source/store/derivation/outputs/input-address.md diff --git a/.gitignore b/.gitignore index 337a7c154..9c4691240 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ /tests/functional/lang/*.err /tests/functional/lang/*.ast -outputs/ +/outputs *~ diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 0abe691cc..914d63a0e 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -22,7 +22,10 @@ - [Store Object](store/store-object.md) - [Content-Addressing Store Objects](store/store-object/content-address.md) - [Store Path](store/store-path.md) - - [Store Derivation and Deriving Path](store/drv.md) + - [Store Derivation and Deriving Path](store/derivation/index.md) + - [Derivation Outputs and Types of Derivations](store/derivation/outputs/index.md) + - [Content-addressing derivation outputs](store/derivation/outputs/content-address.md) + - [Input-addressing derivation outputs](store/derivation/outputs/input-address.md) - [Building](store/building.md) - [Store Types](store/types/index.md) {{#include ./store/types/SUMMARY.md}} diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index a19640705..db6d18f0e 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -22,7 +22,7 @@ - [store derivation]{#gloss-store-derivation} A single build task. - See [Store Derivation](@docroot@/store/drv.md#store-derivation) for details. + See [Store Derivation](@docroot@/store/derivation/index.md#store-derivation) for details. [store derivation]: #gloss-store-derivation @@ -30,7 +30,7 @@ A [store path] which uniquely identifies a [store derivation]. - See [Referencing Store Derivations](@docroot@/store/drv.md#derivation-path) for details. + See [Referencing Store Derivations](@docroot@/store/derivation/index.md#derivation-path) for details. Not to be confused with [deriving path]. @@ -252,7 +252,7 @@ Deriving paths are a way to refer to [store objects][store object] that might not yet be [realised][realise]. - See [Deriving Path](./store/drv.md#deriving-path) for details. + See [Deriving Path](./store/derivation/index.md#deriving-path) for details. Not to be confused with [derivation path]. diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index c384e956a..0722386c4 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -99,8 +99,8 @@ Derivations can declare some infrequently used optional attributes. to make it use the proxy server configuration specified by the user in the environment variables `http_proxy` and friends. - This attribute is only allowed in *fixed-output derivations* (see - below), where impurities such as these are okay since (the hash + This attribute is only allowed in [fixed-output derivations][fixed-output derivation], + where impurities such as these are okay since (the hash of) the output is known in advance. It is ignored for all other derivations. @@ -119,135 +119,6 @@ Derivations can declare some infrequently used optional attributes. [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) configuration setting. - - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ - These attributes declare that the derivation is a so-called *fixed-output derivation* (FOD), which means that a cryptographic hash of the output is already known in advance. - - As opposed to regular derivations, the [`builder`] executable of a fixed-output derivation has access to the network. - Nix computes a cryptographic hash of its output and compares that to the hash declared with these attributes. - If there is a mismatch, the derivation fails. - - The rationale for fixed-output derivations is derivations such as - those produced by the `fetchurl` function. This function downloads a - file from a given URL. To ensure that the downloaded file has not - been modified, the caller must also specify a cryptographic hash of - the file. For example, - - ```nix - fetchurl { - url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz"; - sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; - } - ``` - - It sometimes happens that the URL of the file changes, e.g., because - servers are reorganised or no longer available. We then must update - the call to `fetchurl`, e.g., - - ```nix - fetchurl { - url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; - sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; - } - ``` - - If a `fetchurl` derivation was treated like a normal derivation, the - output paths of the derivation and *all derivations depending on it* - would change. For instance, if we were to change the URL of the - Glibc source distribution in Nixpkgs (a package on which almost all - other packages depend) massive rebuilds would be needed. This is - unfortunate for a change which we know cannot have a real effect as - it propagates upwards through the dependency graph. - - For fixed-output derivations, on the other hand, the name of the - output path only depends on the `outputHash*` and `name` attributes, - while all other attributes are ignored for the purpose of computing - the output path. (The `name` attribute is included because it is - part of the path.) - - As an example, here is the (simplified) Nix expression for - `fetchurl`: - - ```nix - { stdenv, curl }: # The curl program is used for downloading. - - { url, sha256 }: - - stdenv.mkDerivation { - name = baseNameOf (toString url); - builder = ./builder.sh; - buildInputs = [ curl ]; - - # This is a fixed-output derivation; the output must be a regular - # file with SHA256 hash sha256. - outputHashMode = "flat"; - outputHashAlgo = "sha256"; - outputHash = sha256; - - inherit url; - } - ``` - - The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). - The "nix32" encoding is an adaptation of base-32 encoding. - The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings. - - The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. - It can currently be `"blake3", "sha1"`, `"sha256"`, `"sha512"`, or `null`. - `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format. - - The `outputHashMode` attribute determines how the hash is computed. - It must be one of the following values: - - - [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat) - - This is the default. - - - [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive) - - > **Compatibility** - > - > `"recursive"` is the traditional way of indicating this, - > and is supported since 2005 (virtually the entire history of Nix). - > `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), - > however support for it is only added in Nix version 2.21. - - - [`"text"`](@docroot@/store/store-object/content-address.md#method-text) - - > **Warning** - > - > The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature. - - - [`"git"`](@docroot@/store/store-object/content-address.md#method-git) - - > **Warning** - > - > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. - - - [`__contentAddressed`]{#adv-attr-__contentAddressed} - - > **Warning** - > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). - > - > To use this attribute, you must enable the - > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. - > For example, in [nix.conf](../command-ref/conf-file.md) you could add: - > - > ``` - > extra-experimental-features = ca-derivations - > ``` - - If this attribute is set to `true`, then the derivation - outputs will be stored in a content-addressed location rather than the - traditional input-addressed one. - - Setting this attribute also requires setting - [`outputHashMode`](#adv-attr-outputHashMode) - and - [`outputHashAlgo`](#adv-attr-outputHashAlgo) - like for *fixed-output derivations* (see above). - - It also implicitly requires that the machine to build the derivation must have the `ca-derivations` [system feature](@docroot@/command-ref/conf-file.md#conf-system-features). - - [`passAsFile`]{#adv-attr-passAsFile}\ A list of names of attributes that should be passed via files rather than environment variables. For example, if you have @@ -370,6 +241,134 @@ Derivations can declare some infrequently used optional attributes. ensures that the derivation can only be built on a machine with the `kvm` feature. -[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +## Setting the derivation type + +As discussed in [Derivation Outputs and Types of Derivations](@docroot@/store/derivation/outputs/index.md), there are multiples kinds of derivations / kinds of derivation outputs. +The choice of the following attributes determines which kind of derivation we are making. + +- [`__contentAddressed`] + +- [`outputHash`] + +- [`outputHashAlgo`] + +- [`outputHashMode`] + +The three types of derivations are chosen based on the following combinations of these attributes. +All other combinations are invalid. + +- [Input-addressing derivations](@docroot@/store/derivation/outputs/input-address.md) + + This is the default for `builtins.derivation`. + Nix only currently supports one kind of input-addressing, so no other information is needed. + + `__contentAddressed = false;` may also be included, but is not needed, and will trigger the experimental feature check. + +- [Fixed-output derivations][fixed-output derivation] + + All of [`outputHash`], [`outputHashAlgo`], and [`outputHashMode`]. + + + +- [(Floating) content-addressing derivations](@docroot@/store/derivation/outputs/content-address.md) + + Both [`outputHashAlgo`] and [`outputHashMode`], `__contentAddressed = true;`, and *not* `outputHash`. + + If an output hash was given, then the derivation output would be "fixed" not "floating". + +Here is more information on the `output*` attributes, and what values they may be set to: + + - [`outputHashMode`]{#adv-attr-outputHashMode} + + This specifies how the files of a content-addressing derivation output are digested to produce a content address. + + This works in conjunction with [`outputHashAlgo`](#adv-attr-outputHashAlgo). + Specifying one without the other is an error (unless [`outputHash` is also specified and includes its own hash algorithm as described below). + + The `outputHashMode` attribute determines how the hash is computed. + It must be one of the following values: + + - [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat) + + This is the default. + + - [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive) + + > **Compatibility** + > + > `"recursive"` is the traditional way of indicating this, + > and is supported since 2005 (virtually the entire history of Nix). + > `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + > however support for it is only added in Nix version 2.21. + + - [`"text"`](@docroot@/store/store-object/content-address.md#method-text) + + > **Warning** + > + > The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature. + + - [`"git"`](@docroot@/store/store-object/content-address.md#method-git) + + > **Warning** + > + > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + + See [content-addressing store objects](@docroot@/store/store-object/content-address.md) for more information about the process this flag controls. + + - [`outputHashAlgo`]{#adv-attr-outputHashAlgo} + + This specifies the hash alorithm used to digest the [file system object] data of a content-addressing derivation output. + + This works in conjunction with [`outputHashMode`](#adv-attr-outputHashAlgo). + Specifying one without the other is an error (unless [`outputHash` is also specified and includes its own hash algorithm as described below). + + The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. + It can currently be `"blake3"`, "sha1"`, `"sha256"`, `"sha512"`, or `null`. + + `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format, because in that case the choice of hash algorithm is determined by `outputHash`. + + - [`outputHash`]{#adv-attr-outputHashAlgo}; [`outputHash`]{#adv-attr-outputHashMode}\ + + This will specify the output hash of the single output of a [fixed-output derivation]. + + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). + The "nix32" encoding is an adaptation of base-32 encoding. + + > **Note** + > + > The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings. + > The [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings. + + - [`__contentAddressed`]{#adv-attr-__contentAddressed} + + > **Warning** + > + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). + > + > To use this attribute, you must enable the + > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. + > For example, in [nix.conf](../command-ref/conf-file.md) you could add: + > + > ``` + > extra-experimental-features = ca-derivations + > ``` + + This is a boolean with a default of `false`. + It determines whether the derivation is floating content-addressing. + +[`__contentAddressed`]: #adv-attr-__contentAddressed +[`outputHash`]: #adv-attr-outputHash +[`outputHashAlgo`]: #adv-attr-outputHashAlgo +[`outputHashMode`]: #adv-attr-outputHashMode + +[fixed-output derivation]: @docroot@/glossary.md#gloss-fixed-output-derivation +[file system object]: @docroot@/store/file-system-object.md +[store object]: @docroot@/store/store-object.md [xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations [xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/source/language/derivations.md b/doc/manual/source/language/derivations.md index 0f9284e98..43eec680b 100644 --- a/doc/manual/source/language/derivations.md +++ b/doc/manual/source/language/derivations.md @@ -1,7 +1,7 @@ # Derivations The most important built-in function is `derivation`, which is used to describe a single store-layer [store derivation]. -Consult the [store chapter](@docroot@/store/drv.md) for what a store derivation is; +Consult the [store chapter](@docroot@/store/derivation/index.md) for what a store derivation is; this section just concerns how to create one from the Nix language. This builtin function takes as input an attribute set, the attributes of which specify the inputs to the process. @@ -16,7 +16,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect - [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. - See [derivation outputs](@docroot@/store/drv.md#outputs) for what this is affects. + See [derivation outputs](@docroot@/store/derivation/index.md#outputs) for what this is affects. [store path]: @docroot@/store/store-path.md @@ -34,7 +34,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect - [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string)) - See [system](@docroot@/store/drv.md#system). + See [system](@docroot@/store/derivation/index.md#system). > **Example** > @@ -64,7 +64,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect - [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string)) - See [builder](@docroot@/store/drv.md#builder). + See [builder](@docroot@/store/derivation/index.md#builder). > **Example** > @@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect Default: `[ ]` - See [args](@docroot@/store/drv.md#args). + See [args](@docroot@/store/derivation/index.md#args). > **Example** > diff --git a/doc/manual/source/store/building.md b/doc/manual/source/store/building.md index 79808273e..feefa8e3f 100644 --- a/doc/manual/source/store/building.md +++ b/doc/manual/source/store/building.md @@ -10,7 +10,7 @@ ## Builder Execution -The [`builder`](./drv.md#builder) is executed as follows: +The [`builder`](./derivation/index.md#builder) is executed as follows: - A temporary directory is created under the directory specified by `TMPDIR` (default `/tmp`) where the build will take place. The diff --git a/doc/manual/source/store/drv.md b/doc/manual/source/store/derivation/index.md similarity index 89% rename from doc/manual/source/store/drv.md rename to doc/manual/source/store/derivation/index.md index 83ca80aaa..42cfa67f5 100644 --- a/doc/manual/source/store/drv.md +++ b/doc/manual/source/store/derivation/index.md @@ -9,15 +9,24 @@ This is where Nix distinguishes itself. ## Store Derivation {#store-derivation} -A derivation is a specification for running an executable on precisely defined input files to repeatably produce output files at uniquely determined file system paths. +A derivation is a specification for running an executable on precisely defined input to produce on more [store objects][store object]. +These store objects are known as the derivation's *outputs*. + +Derivations are *built*, in which case the process is spawned according to the spec, and when it exits, required to leave behind files which will (after post-processing) become the outputs of the derivation. +This process is described in detail in [Building](@docroot@/store/building.md). + + A derivation consists of: - A name - - A set of [*inputs*][inputs], a set of [deriving paths][deriving path] + - An [inputs specification][inputs], a set of [deriving paths][deriving path] - - A map of [*outputs*][outputs], from names to other data + - An [outputs specification][outputs], specifying which outputs should be produced, and various metadata about them. - The ["system" type][system] (e.g. `x86_64-linux`) where the executable is to run. @@ -26,8 +35,8 @@ A derivation consists of: [store derivation]: #store-derivation [inputs]: #inputs [input]: #inputs -[outputs]: #outputs -[output]: #outputs +[outputs]: ./outputs/index.md +[output]: ./outputs/index.md [process creation fields]: #process-creation-fields [builder]: #builder [args]: #args @@ -89,28 +98,6 @@ The [process creation fields] will presumably include many [store paths][store p But rather than somehow scanning all the other fields for inputs, Nix requires that all inputs be explicitly collected in the inputs field. It is instead the responsibility of the creator of a derivation (e.g. the evaluator) to ensure that every store object referenced in another field (e.g. referenced by store path) is included in this inputs field. -### Outputs {#outputs} - -The outputs are the derivations are the [store objects][store object] it is obligated to produce. - -Outputs are assigned names, and also consistent of other information based on the type of derivation. - -Output names can be any string which is also a valid [store path] name. -The store path of the output store object (also called an [output path] for short), has a name based on the derivation name and the output name. -In the general case, store paths have name `derivationName + "-" + outputName`. -However, an output named "out" has a store path with name is just the derivation name. -This is to allow derivations with a single output to avoid a superfluous `"-${outputName}"` in their single output's name when no disambiguation is needed. - -> **Example** -> -> A derivation is named `hello`, and has two outputs, `out`, and `dev` -> -> - The derivation's path will be: `/nix/store/-hello.drv`. -> -> - The store path of `out` will be: `/nix/store/-hello`. -> -> - The store path of `dev` will be: `/nix/store/-hello-dev`. - ### System {#system} The system type on which the [`builder`](#attr-builder) executable is meant to be run. diff --git a/doc/manual/source/store/derivation/outputs/content-address.md b/doc/manual/source/store/derivation/outputs/content-address.md new file mode 100644 index 000000000..21e940bc2 --- /dev/null +++ b/doc/manual/source/store/derivation/outputs/content-address.md @@ -0,0 +1,192 @@ +# Content-addressing derivation outputs + +The content-addressing of an output only depends on that store object itself, not any other information external (such has how it was made, when it was made, etc.). +As a consequence, a store object will be content-addressed the same way regardless of whether it was manually inserted into the store, outputted by some derivation, or outputted by a some other derivation. + +The output spec for a content-addressed output must contains the following field: + +- *method*: how the data of the store object is digested into a content address + +The possible choices of *method* are described in the [section on content-addressing store objects](@docroot@/store/store-object/content-address.md). +Given the method, the output's name (computed from the derivation name and output spec mapping as described above), and the data of the store object, the output's store path will be computed as described in that section. + +## Fixed-output content-addressing {#fixed} + +In this case the content-address of the *fixed* in advanced by the derivation itself. +In other words, when the derivation has finished [building](@docroot@/store/building.md), and the provisional output' content-address is computed as part of the process to turn it into a *bona fide* store object, the calculated content address must much that given in the derivation, or the build of that derivation will be deemed a failure. + +The output spec for an output with a fixed content addresses additionally contains: + +- *hash*, the hash expected from digesting the store object's file system objects. + This hash may be of a freely-chosen hash algorithm (that Nix supports) + +> **Design note** +> +> In principle, the output spec could also specify the references the store object should have, since the references and file system objects are equally parts of a content-addressed store object proper that contribute to its content-addressed. +> However, at this time, the references are not not done because all fixed content-addressed outputs are required to have no references (including no self-reference). +> +> Also in principle, rather than specifying the references and file system object data with separate hashes, a single hash that constraints both could be used. +> This could be done with the final store path's digest, or better yet, the hash that will become the store path's digest before it is truncated. +> +> These possible future extensions are included to elucidate the core property of fixed-output content addressing --- that all parts of the output must be cryptographically fixed with one or more hashes --- separate from the particulars of the currently-supported store object content-addressing schemes. + +### Design rationale + +What is the purpose of fixing an output's content address in advanced? +In abstract terms, the answer is carefully controlled impurity. +Unlike a regular derivation, the [builder] executable of a derivation that produced fixed outputs has access to the network. +The outputs' guaranteed content-addresses are supposed to mitigate the risk of the builder being given these capabilities; +regardless of what the builder does *during* the build, it cannot influence downstream builds in unanticipated ways because all information it passed downstream flows through the outputs whose content-addresses are fixed. + +[builder]: @docroot@/store/derivation/index.md#builder + +In concrete terms, the purpose of this feature is fetching fixed input data like source code from the network. +For example, consider a family of "fetch URL" derivations. +These derivations download files from given URL. +To ensure that the downloaded file has not been modified, each derivation must also specify a cryptographic hash of the file. +For example, + +```jsonc +{ + "outputs: { + "out": { + "method": "nar", + "hashAlgo": "sha256", + "hash: "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465", + }, + }, + "env": { + "url": "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz" + // ... + }, + // ... +} +``` + +It sometimes happens that the URL of the file changes, +e.g., because servers are reorganised or no longer available. +In these cases, we then must update the call to `fetchurl`, e.g., + +```diff + "env": { +- "url": "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz" ++ "url": "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz" + // ... + }, +``` + +If a `fetchurl` derivation's outputs were [input-addressed][input addressing], the output paths of the derivation and of *all derivations depending on it* would change. +For instance, if we were to change the URL of the Glibc source distribution in Nixpkgs (a package on which almost all other packages depend on Linux) massive rebuilds would be needed. +This is unfortunate for a change which we know cannot have a real effect as it propagates upwards through the dependency graph. + +For content-addressed outputs (fixed or floating), on the other hand, the outputs' store path only depends on the derivation's name, data, and the `method` of the outputs' specs. +The rest of the derivation is ignored for the purpose of computing the output path. + +> **History Note** +> +> Fixed content-addressing is especially important both today and historically as the *only* form of content-addressing that is stabilized. +> This is why the rationale above contrasts it with [input addressing]. + +## (Floating) Content-Addressing {#floating} + +> **Warning** +> This is part of an [experimental feature](@docroot@/development/experimental-features.md). +> +> To use this type of output addressing, you must enable the +> [`ca-derivations`][xp-feature-ca-derivations] experimental feature. +> For example, in [nix.conf](@docroot@/command-ref/conf-file.md) you could add: +> +> ``` +> extra-experimental-features = ca-derivations +> ``` + +With this experimemental feature enabled, derivation outputs can also be content-addressed *without* fixing in the output spec what the outputs' content address must be. + +### Purity + +Because the derivation output is not fixed (just like with [input addressing]), the [builder] is not given any impure capabilities [^purity]. + +> **Configuration note** +> +> Strictly speaking, the extent to which sandboxing and deprivilaging is possible varies with the environment Nix is running in. +> Nix's configuration settings indicate what level of sandboxing is required or enabled. +> Builds of derivations will fail if they request an absense of sandboxing which is not allowed. +> Builds of derivations will also fail if the level of sandboxing specified in the configure exceeds what is possible in teh given environment. +> +> (The "environment", in this case, consists of attributes such as the Operating System Nix runs atop, along with the operating-system-specific privilages that Nix has been granted. +> Because of how conventional operating systems like macos, Linux, etc. work, granting builders *fewer* privilages may ironically require that Nix be run with *more* privilages.) + +That said, derivations producing floating content-addressed outputs may declare their builders as impure (like the builders of derivations producing producing fixed outputs). +This is provisionally supported as part of the [`impure-derivations`][xp-feature-impure-derivations] experimental feature. + +### Compatibility negotiation + +Any derivation producing a floating content-addresssed output implicitly requires the `ca-derivations` [system feature](@docroot@/command-ref/conf-file.md#conf-system-features). +This prevents scheduling the building of the derivation on a machine without the experimental feature enabled. +Even once the experimental feature is stabilized, this is still useful in order to be allow using remote builder running odler versions of Nix, or alternative implementations that do not support floating content addressing. + +### Determinism + +In the earlier [discussion of how self-references are handled when content-addressing store objects](@docroot@/store/store-object/content-address.html#self-references), it was pointed out that methods of producing store objects ought to be deterministic regardless of the choice of provisional store path. +For store objects produced by manually inserting into the store to create a store object, the "method of production" is an informally concept --- formally, Nix has no idea where the store object came from, and content-addressing is crucial in order to ensure that the derivation is *intrinsically* tamper-proof. +But for store objects produced by derivation, the "method is quite formal" --- the whole point of derivations is to be a formal notion of building, after all. +In this case, we can elevate this informal property to a formal one. + +A *determinstic* content-addressing derivation should produce outputs with the same content addresses: + +1. Every time the builder is run + + This is because either the builder is completely sandboxed, or because all any remaining impurities that leak inside the build sandbox are ignored by the builder and do not influence its behavior. + +2. Regardless of the choice of any provisional outputs paths + + Provisional store paths must be chosen for any output that has a self-reference. + The choice of provisional store path can be thought of as an impurity, since it is an arbitrary choice. + + If provisional outputs paths are deterministically chosen, we are in the first branch of part (1). + The builder the data it produces based on it in arbitrary ways, but this gets us closer to to [input addressing]. + Deterministically choosing the provisional path may be considered "complete sandboxing" by removing an impurity, but this is unsatisfactory + + + + If provisional outputs paths are randomly chosen, we are in the second branch of part (1). + The builder *must* not let the random input affect the final outputs it produces, and multiple builds may be performed and the compared in order to ensure that this is in fact the case. + +### Floating versus Fixed + +While the destinction between content- and input-addressing is one of *mechanism*, the distinction between fixed and floating content addression is more one of *policy*. +A fixed output that passes its content address check is just like a floating output. +It is only in the potential for that check to fail that they are different. + +> **Design Note** +> +> In a future world where floating content-addressing is also stable, we in principle no longer need separate [fixed](#fixed) content-addressing. +> Instead, we could always use floating content-addressing, and separately assert the precise value content address of a given store object to be used as an input (of another derivation). +> A stand-alone assertion object of this sort is not yet implemented, but its possible creation is tracked in [Issue #11955](https://github.com/NixOS/nix/issues/11955). +> +> In the current version of Nix, fixed outputs which fail their hash check are still registered as valid store objects, just not registered as outputs of the derivation which produced them. +> This is an optimization that means if the wrong output hash is specified in a derivation, and then the derivation is recreated with the right output hash, derivation does not need to be rebuilt --- avoiding downloading potentially large amounts of data twice. +> This optimisation prefigures the design above: +> If the output hash assertion was removed outside the derivation itself, Nix could additionally not only register that outputted store object like today, but could also make note that derivation did in fact successfully download some data. +For example, for the "fetch URL" example above, making such a note is tantamount to recording what data is available at the time of download at the given URL. +> It would only be when Nix subsequently tries to build something with that (refining our example) downloaded source code that Nix would be forced to check the output hash assertion, preventing it from e.g. building compromised malware. +> +> Recapping, Nix would +> +> 1. successfully download data +> 2. insert that data into the store +> 3. associate (presumably with some sort of expiration policy) the downloaded data with the derivation that downloaded it +> +> But only use the downloaded store object in subsequent derivations that depended upon the assertion if the assertion passed. +> +> This possible future extension is included to illustrate this distinction: + +[input addressing]: ./input-address.md +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing +[xp-feature-impure-derivations]: @docroot@/development/experimental-features.md#xp-feature-impure-derivations diff --git a/doc/manual/source/store/derivation/outputs/index.md b/doc/manual/source/store/derivation/outputs/index.md new file mode 100644 index 000000000..15070a18f --- /dev/null +++ b/doc/manual/source/store/derivation/outputs/index.md @@ -0,0 +1,97 @@ +# Derivation Outputs and Types of Derivations + +As stated on the [main pages on derivations](../index.md#store-derivation), +a derivation produces [store objects], which are known as the *outputs* of the derivation. +Indeed, the entire point of derivations is to produce these outputs, and to reliably and reproducably produce these derivations each time the derivation is run. + +One of the parts of a derivation is its *outputs specification*, which specifies certain information about the outputs the derivation produces when run. +The outputs specification is a map, from names to specifications for individual outputs. + +## Output Names {#outputs} + +Output names can be any string which is also a valid [store path] name. +The name mapped to each output specification is not actually the name of the output. +In the general case, the output store object has name `derivationName + "-" + outputSpecName`, not any other metadata about it. +However, an output spec named "out" describes and output store object whose name is just the derivation name. + +> **Example** +> +> A derivation is named `hello`, and has two outputs, `out`, and `dev` +> +> - The derivation's path will be: `/nix/store/-hello.drv`. +> +> - The store path of `out` will be: `/nix/store/-hello`. +> +> - The store path of `dev` will be: `/nix/store/-hello-dev`. + +The outputs are the derivations are the [store objects][store object] it is obligated to produce. + +> **Note** +> +> The formal terminology here is somewhat at adds with everyday communication in the Nix community today. +> "output" in casual usage tends to refer to either to the actual output store object, or the notional output spec, depending on context. +> +> For example "hello's `dev` output" means the store object referred to by the store path `/nix/store/-hello-dev`. +> It is unusual to call this the "`hello-dev` output", even though `hello-dev` is the actual name of that store object. + +## Types of output addressing + +The main information contained in an output specification is how the derivation output is addressed. +In particular, the specification decides: + +- whether the output is [content-addressed](./content-address.md) or [input-addressed](./input-address.md) + +- if the content is content-addressed, how is it content addressed + +- if the content is content-addressed, [what is its content address](./content-address.md#fixed-content-addressing) (and thus what is its [store path]) + +## Types of derivations + +The sections on each type of derivation output addressing ended up discussing other attributes of the derivation besides its outputs, such as purity, scheduling, determinism, etc. +This is no concidence; for the type of a derivation is in fact one-for-one with the type of its outputs: + +- A derivation that produces *xyz-addressed* outputs is an *xyz-addressing* derivations. + +The rules for this are fairly concise: + +- All the outputs must be of the same type / use the same addressing + + - The derivation must have at least one output + + - Additionally, if the outputs are fixed content-addressed, there must be exactly one output, whose specification is mapped from the name `out`. + (The name `out` is special, according to the rules described above. + Having only one output and calling its specification `out` means the single output is effectively anonymous; the store path just has the derivation name.) + + (This is an arbitrary restriction that could be lifted.) + +- The output is either *fixed* or *floating*, indicating whether the its store path is known prior to building it. + + - With fixed content-addressing it is fixed. + + > A *fixed content-addressing* derivation is also called a *fixed-output derivation*, since that is the only currently-implemented form of fixed-output addressing + + - With floating content-addressing or input-addressing it is floating. + + > Thus, historically with Nix, with no experimental features enabled, *all* outputs are fixed. + +- The derivation may be *pure* or *impure*, indicating what read access to the outside world the [builder](../index.md#builder) has. + + - An input-addressing derivation *must* be pure. + + > If it is impure, we would have a large problem, because an input-addressed derivation always produces outputs with the same paths. + + + - A content-addressing derivation may be pure or impure + + - If it is impure, it may be be fixed (typical), or it may be floating if the additional [`impure-derivations`][xp-feature-impure-derivations] experimental feature is enabled. + + - If it is pure, it must be floating. + + - Pure, fixed content-addressing derivations are not suppported + + > There is no use for this forth combination. + > The sole purpose of an output's store path being fixed is to support the derivation being impure. + +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing +[xp-feature-impure-derivations]: @docroot@/development/experimental-features.md#xp-feature-impure-derivations diff --git a/doc/manual/source/store/derivation/outputs/input-address.md b/doc/manual/source/store/derivation/outputs/input-address.md new file mode 100644 index 000000000..54d9437d9 --- /dev/null +++ b/doc/manual/source/store/derivation/outputs/input-address.md @@ -0,0 +1,31 @@ +# Input-addressing derivation outputs + +[input addressing]: #input-addressing + +"Input addressing" means the address the store object by the *way it was made* rather than *what it is*. +That is to say, an input-addressed output's store path is a function not of the output itself, but the derivation that produced it. +Even if two store paths have the same contents, if they are produced in different ways, and one is input-addressed, then they will have different store paths, and thus guaranteed to not be the same store object. + + + +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing +[xp-feature-impure-derivations]: @docroot@/development/experimental-features.md#xp-feature-impure-derivations diff --git a/doc/manual/source/store/store-object/content-address.md b/doc/manual/source/store/store-object/content-address.md index 02dce2836..38a000d04 100644 --- a/doc/manual/source/store/store-object/content-address.md +++ b/doc/manual/source/store/store-object/content-address.md @@ -24,13 +24,17 @@ For the full specification of the algorithms involved, see the [specification of ### File System Objects -With all currently supported store object content addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object. +With all currently-supported store object content-addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object. ### References +#### References to other store object#### References to other store objectss + With all currently supported store object content addressing methods, other objects are referred to by their regular (string-encoded-) [store paths][Store Path]. +#### Self-references + Self-references however cannot be referred to by their path, because we are in the midst of describing how to compute that path! > The alternative would require finding as hash function fixed point, i.e. the solution to an equation in the form @@ -40,7 +44,28 @@ Self-references however cannot be referred to by their path, because we are in t > which is computationally infeasible. > As far as we know, this is equivalent to finding a hash collision. -Instead we just have a "has self reference" boolean, which will end up affecting the digest. +Instead we have a "has self reference" boolean, which end up affecting the digest: +In all currently-supported store object content-addressing methods, when hashing the file system object data, any occurence of store objects own store path in the digested data is replaced with a [sentinal value](https://en.wikipedia.org/wiki/Sentinel_value). +The hashes of these modified input streams are used instead. + +When validating the content-address of a store object after the fact, the above process works as written. +However, when first creating the store object we don't know the store object's store path, as explained just above. +We therefore, strictly speaking, do not know what value we will be replacing with the sentinental value in the inputs to hash functions. +What instead happens is that the provisional store object --- the data from which we wish to create a store object --- is paired with a provisional "scratch" store path (that presumably was choosen when the data was created). +That provisional store path is instead what is replaced with the sentinal value, rather than the final store object which we do not yet know. + +> **Design note** +> +> It is an informal property of content-addressed store objects that the choice of provisional store path should not matter. +> In other words, if a provisional store object is prepared in the same way except for the choice of provision store path, the provisional data need not be identical. +> But, after the sentinal value is substituted in place of each provisional store object's provision store path, the final so-normalized data *should* be identifical. +> +> If, conversely, the data after this normalization process is still different, we'll compute a different content-address. +> The method of preparing the provisional self-referenced data has *failed* to be deterministic in the sense of not *leaking* the choice of provisional store path --- a choice which is supposed to be arbitrary --- into the final store object. +> +> This property is informal because at this stage, we are just described store objects, which have no formal notion of their origin. +> Without such a formal notion, there is nothing to formally accuse of being insufficiently deterministic. +> Later in this chapter, when we cover [derivations](@docroot@/store/derivation/index.md), we will have a chance to make this a formal property, not of content-addressed store objects themselves, but of derivations that *produce* content-addressed store objects. ### Name and Store Directory diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7c9ce7104..8156d0320 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1595,7 +1595,7 @@ static RegisterPrimOp primop_placeholder({ .args = {"output"}, .doc = R"( Return at - [output placeholder string](@docroot@/store/drv.md#output-placeholder) + [output placeholder string](@docroot@/store/derivation/index.md#output-placeholder) for the specified *output* that will be substituted by the corresponding [output path](@docroot@/glossary.md#gloss-output-path) at build time. @@ -2139,7 +2139,7 @@ static RegisterPrimOp primop_outputOf({ .args = {"derivation-reference", "output-name"}, .doc = R"( Return the output path of a derivation, literally or using an - [input placeholder string](@docroot@/store/drv.md#input-placeholder) + [input placeholder string](@docroot@/store/derivation/index.md#input-placeholder) if needed. If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addresed), the output path will just be returned. From d6139a339b98c3a5675757d6df52c79124d953b6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 27 Feb 2025 17:48:28 +0100 Subject: [PATCH 004/396] packaging: Make hydraJobs.build.* complete --- packaging/hydra.nix | 102 +++++++++++++++++++++++++++++++++----------- 1 file changed, 78 insertions(+), 24 deletions(-) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 44cbd753c..74e245f26 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -29,32 +29,86 @@ let # Technically we could just return `pkgs.nixComponents`, but for Hydra it's # convention to transpose it, and to transpose it efficiently, we need to # enumerate them manually, so that we don't evaluate unnecessary package sets. - forAllPackages = lib.genAttrs [ - "nix-everything" - "nix-util" - "nix-util-c" - "nix-util-test-support" - "nix-util-tests" - "nix-store" - "nix-store-c" - "nix-store-test-support" - "nix-store-tests" - "nix-fetchers" - "nix-fetchers-tests" - "nix-expr" - "nix-expr-c" - "nix-expr-test-support" - "nix-expr-tests" - "nix-flake" - "nix-flake-tests" - "nix-main" - "nix-main-c" - "nix-cmd" - "nix-cli" - "nix-functional-tests" - ]; + # See listingIsComplete below. + forAllPackages = forAllPackages' { }; + forAllPackages' = + { + enableBindings ? false, + enableDocs ? false, # already have separate attrs for these + }: + lib.genAttrs ( + [ + "nix-everything" + "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-tests" + "nix-store" + "nix-store-c" + "nix-store-test-support" + "nix-store-tests" + "nix-fetchers" + "nix-fetchers-tests" + "nix-expr" + "nix-expr-c" + "nix-expr-test-support" + "nix-expr-tests" + "nix-flake" + "nix-flake-c" + "nix-flake-tests" + "nix-main" + "nix-main-c" + "nix-cmd" + "nix-cli" + "nix-functional-tests" + ] + ++ lib.optionals enableBindings [ + "nix-perl-bindings" + ] + ++ lib.optionals enableDocs [ + "nix-manual" + "nix-internal-api-docs" + "nix-external-api-docs" + ] + ); in { + /** + An internal check to make sure our package listing is complete. + */ + listingIsComplete = + let + arbitrarySystem = "x86_64-linux"; + listedPkgs = forAllPackages' { + enableBindings = true; + enableDocs = true; + } (_: null); + actualPkgs = lib.concatMapAttrs ( + k: v: if lib.strings.hasPrefix "nix-" k then { ${k} = null; } else { } + ) nixpkgsFor.${arbitrarySystem}.native.nixComponents; + diff = lib.concatStringsSep "\n" ( + lib.concatLists ( + lib.mapAttrsToList ( + k: _: + if (listedPkgs ? ${k}) && !(actualPkgs ? ${k}) then + [ "- ${k}: redundant?" ] + else if !(listedPkgs ? ${k}) && (actualPkgs ? ${k}) then + [ "- ${k}: missing?" ] + else + [ ] + ) (listedPkgs // actualPkgs) + ) + ); + in + if listedPkgs == actualPkgs then + { } + else + throw '' + Please update the components list in hydra.nix (or fix this check) + Differences: + ${diff} + ''; + # Binary package for various platforms. build = forAllPackages ( pkgName: forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName}) From 954e9101ba3d35da2d9319f7947fc5f2be6e37c9 Mon Sep 17 00:00:00 2001 From: Yannik Sander Date: Fri, 28 Feb 2025 13:14:16 +0100 Subject: [PATCH 005/396] Add host attribute of github/gitlab flakerefs to URL serialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `GitArchiveInputScheme::toUrl` currently drops the `host` attribute, creating invalid urls when locking `github:` or `gitlab:` urls pointing to alterative instances and serializing the input back to a url. ``` ❯ cat flake.nix { inputs.gnome-2048 = { url = "gitlab:GNOME/gnome-2048?host=gitlab.gnome.org"; flake = false; }; outputs = inputs: {}; } f1xb57354q79t_jpw5_h79cw0000gq/T/tmp.MOBbzbpT35 ❯ nix flake metadata warning: creating lock file '/private/var/folders/fb/f1xb57354q79t_jpw5_h79cw0000gq/T/tmp.MOBbzbpT35/flake.lock': • Added input 'gnome-2048': 'gitlab:GNOME/gnome-2048/70e0e430ca4bf590990433a3abdce6b631d50e6e?narHash=sha256-bya45ug2mDSU4SMn0fSBlZCuPl9y15B12ubKeb2A58s%3D' (2025-02-21) Resolved URL: path:/private/var/folders/fb/f1xb57354q79t_jpw5_h79cw0000gq/T/tmp.MOBbzbpT35 Locked URL: path:/private/var/folders/fb/f1xb57354q79t_jpw5_h79cw0000gq/T/tmp.MOBbzbpT35?lastModified=1740744684&narHash=sha256-nxUL/JiTYbZX2c1XiN/TC6aA1hf%2B1YXsUvhL7ASY2uE%3D Path: /nix/store/f4xczpwhdxs8gal1rika1c5bvhyd472l-source Last modified: 2025-02-28 13:11:24 Inputs: └───gnome-2048: gitlab:GNOME/gnome-2048/70e0e430ca4bf590990433a3abdce6b631d50e6e?narHash=sha256-bya45ug2mDSU4SMn0fSBlZCuPl9y15B12ubKeb2A58s%3D (2025-02-21 23:18:46) ``` Note the gnome-2048 input url missing the original host query. The Url after this commit: ``` [...] Inputs: └───gnome-2048: gitlab:GNOME/gnome-2048/70e0e430ca4bf590990433a3abdce6b631d50e6e?host=gitlab.gnome.org&narHash=sha256-bya45ug2mDSU4SMn0fSBlZCuPl9y15B12ubKeb2A58s%3D (2025-02-21 23:18:46) ``` --- src/libfetchers/github.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 9cddd8571..ea4dbe338 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -149,6 +149,9 @@ struct GitArchiveInputScheme : InputScheme }; if (auto narHash = input.getNarHash()) url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); + auto host = maybeGetStrAttr(input.attrs, "host"); + if (host) + url.query.insert_or_assign("host", *host); return url; } From 41085295ab3717b5ec8d348307dd4c9c1d378846 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 28 Feb 2025 17:40:32 +0100 Subject: [PATCH 006/396] packaging/everything.nix: Use a multi-output derivation This should fix a few packaging regressions. `dev` also includes a merged `includes/`, which may be helpful until inter-component includes are fixed properly. --- packaging/everything.nix | 200 ++++++++++++++++++++++++--------------- 1 file changed, 122 insertions(+), 78 deletions(-) diff --git a/packaging/everything.nix b/packaging/everything.nix index 0974a34df..c9ad26823 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -1,6 +1,7 @@ { lib, stdenv, + lndir, buildEnv, nix-util, @@ -38,7 +39,6 @@ nix-perl-bindings, testers, - runCommand, }: let @@ -119,92 +119,136 @@ let }; in -(buildEnv { - name = "nix-${nix-cli.version}"; - paths = [ - nix-cli - nix-manual.man +stdenv.mkDerivation (finalAttrs: { + pname = "nix"; + version = nix-cli.version; + + /** + This package uses a multi-output derivation, even though some outputs could + have been provided directly by the constituent component that provides it. + + This is because not all tooling handles packages composed of arbitrary + outputs yet. This includes nix itself, https://github.com/NixOS/nix/issues/6507. + + `devdoc` is also available, but not listed here, because this attribute is + not an output of the same derivation that provides `out`, `dev`, etc. + */ + outputs = [ + "out" + "dev" + "doc" + "man" ]; - meta.mainProgram = "nix"; -}).overrideAttrs - ( - finalAttrs: prevAttrs: { - doCheck = true; - doInstallCheck = true; + /** + Unpacking is handled in this package's constituent components + */ + dontUnpack = true; + /** + Building is handled in this package's constituent components + */ + dontBuild = true; - checkInputs = - [ - # Make sure the unit tests have passed - nix-util-tests.tests.run - nix-store-tests.tests.run - nix-expr-tests.tests.run - nix-fetchers-tests.tests.run - nix-flake-tests.tests.run + /** + `doCheck` controles whether tests are added as build gate for the combined package. + This includes both the unit tests and the functional tests, but not the + integration tests that run in CI (the flake's `hydraJobs` and some of the `checks`). + */ + doCheck = true; - # Make sure the functional tests have passed - nix-functional-tests + /** + `fixupPhase` currently doesn't understand that a symlink output isn't writable. - # dev bundle is ok - # (checkInputs must be empty paths??) - (runCommand "check-pkg-config" { checked = dev.tests.pkg-config; } "mkdir $out") - ] - ++ lib.optionals - (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) - [ - # Perl currently fails in static build - # TODO: Split out tests into a separate derivation? - nix-perl-bindings - ]; - passthru = prevAttrs.passthru // { - inherit (nix-cli) version; + We don't compile or link anything in this derivation, so fixups aren't needed. + */ + dontFixup = true; - /** - These are the libraries that are part of the Nix project. They are used - by the Nix CLI and other tools. + checkInputs = + [ + # Make sure the unit tests have passed + nix-util-tests.tests.run + nix-store-tests.tests.run + nix-expr-tests.tests.run + nix-fetchers-tests.tests.run + nix-flake-tests.tests.run - If you need to use these libraries in your project, we recommend to use - the `-c` C API libraries exclusively, if possible. + # Make sure the functional tests have passed + nix-functional-tests + ] + ++ lib.optionals + (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) + [ + # Perl currently fails in static build + # TODO: Split out tests into a separate derivation? + nix-perl-bindings + ]; - We also recommend that you build the complete package to ensure that the unit tests pass. - You could do this in CI, or by passing it in an unused environment variable. e.g in a `mkDerivation` call: + nativeBuildInputs = [ + lndir + ]; - ```nix - buildInputs = [ nix.libs.nix-util-c nix.libs.nix-store-c ]; - # Make sure the nix libs we use are ok - unusedInputsForTests = [ nix ]; - disallowedReferences = nix.all; - ``` - */ - inherit libs; + installPhase = + let + devPaths = lib.mapAttrsToList (_k: lib.getDev) finalAttrs.finalPackage.libs; + in + '' + mkdir -p $out $dev $doc $man - tests = prevAttrs.passthru.tests or { } // { - # TODO: create a proper fixpoint and: - # pkg-config = - # testers.hasPkgConfigModules { - # package = finalPackage; - # }; - }; + # Merged outputs + lndir ${nix-cli} $out + for lib in ${lib.escapeShellArgs devPaths}; do + lndir $lib $dev + done - /** - A derivation referencing the `dev` outputs of the Nix libraries. - */ - inherit dev; - inherit devdoc; - doc = nix-manual; - outputs = [ - "out" - "dev" - "devdoc" - "doc" - ]; - all = lib.attrValues ( - lib.genAttrs finalAttrs.passthru.outputs (outName: finalAttrs.finalPackage.${outName}) - ); + # Forwarded outputs + ln -s ${nix-manual} $doc + ln -s ${nix-manual.man} $man + ''; + + passthru = { + inherit (nix-cli) version; + + /** + These are the libraries that are part of the Nix project. They are used + by the Nix CLI and other tools. + + If you need to use these libraries in your project, we recommend to use + the `-c` C API libraries exclusively, if possible. + + We also recommend that you build the complete package to ensure that the unit tests pass. + You could do this in CI, or by passing it in an unused environment variable. e.g in a `mkDerivation` call: + + ```nix + buildInputs = [ nix.libs.nix-util-c nix.libs.nix-store-c ]; + # Make sure the nix libs we use are ok + unusedInputsForTests = [ nix ]; + disallowedReferences = nix.all; + ``` + */ + inherit libs; + + /** + Developer documentation for `nix`, in `share/doc/nix/{internal,external}-api/`. + + This is not a proper output; see `outputs` for context. + */ + inherit devdoc; + + /** + Extra tests that test this package, but do not run as part of the build. + See + */ + tests = { + pkg-config = testers.hasPkgConfigModules { + package = finalAttrs.finalPackage; }; - meta = prevAttrs.meta // { - description = "The Nix package manager"; - pkgConfigModules = dev.meta.pkgConfigModules; - }; - } - ) + }; + }; + + meta = { + mainProgram = "nix"; + description = "The Nix package manager"; + pkgConfigModules = dev.meta.pkgConfigModules; + }; + +}) From c99edc840ca3a25e82878a99c9f14021f7b270f3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Sat, 1 Mar 2025 18:44:48 +0000 Subject: [PATCH 007/396] libutil/file-system.hh: Fix typos --- src/libutil/file-system.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 204907339..a94603eaf 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -2,7 +2,7 @@ /** * @file * - * Utiltities for working with the file sytem and file paths. + * Utilities for working with the file system and file paths. */ #include "types.hh" From 529cbea3439e4d3dbe48c06871ed7f05ef8d5047 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 1 Mar 2025 22:54:57 +0100 Subject: [PATCH 008/396] .mergify.yml: Add backport 2.27-maintenance entry --- .mergify.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index 021157eb9..e134b0f46 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -117,3 +117,14 @@ pull_request_rules: labels: - automatic backport - merge-queue + + - name: backport patches to 2.27 + conditions: + - label=backport 2.27-maintenance + actions: + backport: + branches: + - "2.27-maintenance" + labels: + - automatic backport + - merge-queue From 0358007da337d5e82cfaeb93f98323d09a9a668e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 2 Mar 2025 00:06:00 +0100 Subject: [PATCH 009/396] remove fricklerhandwerk from CODEOWNERS stepping aside as a Nix maintainer: https://discourse.nixos.org/t/time-to-step-aside/61050 --- .github/CODEOWNERS | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a9ca74c17..763c5f27e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,16 +11,7 @@ .github/CODEOWNERS @edolstra # Documentation of built-in functions -src/libexpr/primops.cc @roberth @fricklerhandwerk - -# Documentation of settings -src/libexpr/eval-settings.hh @fricklerhandwerk -src/libstore/globals.hh @fricklerhandwerk - -# Documentation -doc/manual @fricklerhandwerk -maintainers/*.md @fricklerhandwerk -src/**/*.md @fricklerhandwerk +src/libexpr/primops.cc @roberth # Libstore layer /src/libstore @ericson2314 From f4f28cdd0e01a9bfb0901e8a53ead719265fa9b8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 27 Feb 2025 13:42:03 -0500 Subject: [PATCH 010/396] Revert "Revert "Revert "Adapt scheduler to work with dynamic derivations""" The bug reappeared after all, and the fix introduced a different bug. I just reverted on 2.27 first, in #12576, but upon further introspection and discussion with @roberth, with preparing for and travelling to Planet Nix I will not be able to fix it on `master` soon enough for a revert to not be warranted here in the meantime also. This reverts commit c98525235f5b8f1ed02fd1b3849b42e5f669d364. --- ...erivation-creation-and-realisation-goal.cc | 126 ------------------ ...erivation-creation-and-realisation-goal.hh | 88 ------------ src/libstore/build/derivation-goal.cc | 26 +++- src/libstore/build/derivation-goal.hh | 4 - src/libstore/build/entry-points.cc | 5 +- src/libstore/build/goal.cc | 2 +- src/libstore/build/goal.hh | 21 --- src/libstore/build/worker.cc | 92 +++---------- src/libstore/build/worker.hh | 24 ---- src/libstore/derived-path-map.cc | 4 - src/libstore/derived-path-map.hh | 7 +- src/libstore/meson.build | 2 - tests/functional/dyn-drv/build-built-drv.sh | 7 +- tests/functional/dyn-drv/dep-built-drv-2.sh | 2 +- tests/functional/dyn-drv/dep-built-drv.sh | 7 +- tests/functional/dyn-drv/failing-outer.sh | 2 + 16 files changed, 45 insertions(+), 374 deletions(-) delete mode 100644 src/libstore/build/derivation-creation-and-realisation-goal.cc delete mode 100644 src/libstore/build/derivation-creation-and-realisation-goal.hh diff --git a/src/libstore/build/derivation-creation-and-realisation-goal.cc b/src/libstore/build/derivation-creation-and-realisation-goal.cc deleted file mode 100644 index c33b7571f..000000000 --- a/src/libstore/build/derivation-creation-and-realisation-goal.cc +++ /dev/null @@ -1,126 +0,0 @@ -#include "derivation-creation-and-realisation-goal.hh" -#include "worker.hh" - -namespace nix { - -DerivationCreationAndRealisationGoal::DerivationCreationAndRealisationGoal( - ref drvReq, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built{.drvPath = drvReq, .outputs = wantedOutputs}) - , drvReq(drvReq) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - name = - fmt("outer obtaining drv from '%s' and then building outputs %s", - drvReq->to_string(worker.store), - std::visit( - overloaded{ - [&](const OutputsSpec::All) -> std::string { return "* (all of them)"; }, - [&](const OutputsSpec::Names os) { return concatStringsSep(", ", quoteStrings(os)); }, - }, - wantedOutputs.raw)); - trace("created outer"); - - worker.updateProgress(); -} - -DerivationCreationAndRealisationGoal::~DerivationCreationAndRealisationGoal() {} - -static StorePath pathPartOfReq(const SingleDerivedPath & req) -{ - return std::visit( - overloaded{ - [&](const SingleDerivedPath::Opaque & bo) { return bo.path; }, - [&](const SingleDerivedPath::Built & bfd) { return pathPartOfReq(*bfd.drvPath); }, - }, - req.raw()); -} - -std::string DerivationCreationAndRealisationGoal::key() -{ - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before "baboon". And - substitution goals and inner derivation goals always happen before - derivation goals (due to "b$"). */ - return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store); -} - -void DerivationCreationAndRealisationGoal::timedOut(Error && ex) {} - -void DerivationCreationAndRealisationGoal::addWantedOutputs(const OutputsSpec & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - auto newWanted = wantedOutputs.union_(outputs); - bool needRestart = !newWanted.isSubsetOf(wantedOutputs); - wantedOutputs = newWanted; - - if (!needRestart) - return; - - if (!optDrvPath) - // haven't started steps where the outputs matter yet - return; - worker.makeDerivationGoal(*optDrvPath, outputs, buildMode); -} - -Goal::Co DerivationCreationAndRealisationGoal::init() -{ - trace("outer init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (auto optDrvPath = [this]() -> std::optional { - if (buildMode != bmNormal) - return std::nullopt; - - auto drvPath = StorePath::dummy; - try { - drvPath = resolveDerivedPath(worker.store, *drvReq); - } catch (MissingRealisation &) { - return std::nullopt; - } - auto cond = worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath); - return cond ? std::optional{drvPath} : std::nullopt; - }()) { - trace( - fmt("already have drv '%s' for '%s', can go straight to building", - worker.store.printStorePath(*optDrvPath), - drvReq->to_string(worker.store))); - } else { - trace("need to obtain drv we want to build"); - addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq))); - co_await Suspend{}; - } - - trace("outer load and build derivation"); - - if (nrFailed != 0) { - co_return amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store))); - } - - StorePath drvPath = resolveDerivedPath(worker.store, *drvReq); - /* Build this step! */ - concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode); - { - auto g = upcast_goal(concreteDrvGoal); - /* We will finish with it ourselves, as if we were the derivational goal. */ - g->preserveException = true; - } - optDrvPath = std::move(drvPath); - addWaitee(upcast_goal(concreteDrvGoal)); - co_await Suspend{}; - - trace("outer build done"); - - buildResult = upcast_goal(concreteDrvGoal) - ->getBuildResult(DerivedPath::Built{ - .drvPath = drvReq, - .outputs = wantedOutputs, - }); - - auto g = upcast_goal(concreteDrvGoal); - co_return amDone(g->exitCode, g->ex); -} - -} diff --git a/src/libstore/build/derivation-creation-and-realisation-goal.hh b/src/libstore/build/derivation-creation-and-realisation-goal.hh deleted file mode 100644 index 40fe40053..000000000 --- a/src/libstore/build/derivation-creation-and-realisation-goal.hh +++ /dev/null @@ -1,88 +0,0 @@ -#pragma once - -#include "parsed-derivations.hh" -#include "store-api.hh" -#include "pathlocks.hh" -#include "goal.hh" - -namespace nix { - -struct DerivationGoal; - -/** - * This goal type is essentially the serial composition (like function - * composition) of a goal for getting a derivation, and then a - * `DerivationGoal` using the newly-obtained derivation. - * - * In the (currently experimental) general inductive case of derivations - * that are themselves build outputs, that first goal will be *another* - * `DerivationCreationAndRealisationGoal`. In the (much more common) base-case - * where the derivation has no provence and is just referred to by - * (content-addressed) store path, that first goal is a - * `SubstitutionGoal`. - * - * If we already have the derivation (e.g. if the evaluator has created - * the derivation locally and then instructured the store to build it), - * we can skip the first goal entirely as a small optimization. - */ -struct DerivationCreationAndRealisationGoal : public Goal -{ - /** - * How to obtain a store path of the derivation to build. - */ - ref drvReq; - - /** - * The path of the derivation, once obtained. - **/ - std::optional optDrvPath; - - /** - * The goal for the corresponding concrete derivation. - **/ - std::shared_ptr concreteDrvGoal; - - /** - * The specific outputs that we need to build. - */ - OutputsSpec wantedOutputs; - - /** - * The final output paths of the build. - * - * - For input-addressed derivations, always the precomputed paths - * - * - For content-addressed derivations, calcuated from whatever the - * hash ends up being. (Note that fixed outputs derivations that - * produce the "wrong" output still install that data under its - * true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - DerivationCreationAndRealisationGoal( - ref drvReq, - const OutputsSpec & wantedOutputs, - Worker & worker, - BuildMode buildMode = bmNormal); - virtual ~DerivationCreationAndRealisationGoal(); - - void timedOut(Error && ex) override; - - std::string key() override; - - /** - * Add wanted outputs to an already existing derivation goal. - */ - void addWantedOutputs(const OutputsSpec & outputs); - - Co init() override; - - JobCategory jobCategory() const override - { - return JobCategory::Administration; - }; -}; - -} diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 41762cde1..01da37df6 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -137,8 +137,21 @@ Goal::Co DerivationGoal::init() { trace("init"); if (useDerivation) { + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + + if (buildMode != bmNormal || !worker.evalStore.isValidPath(drvPath)) { + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); + co_await Suspend{}; + } + trace("loading derivation"); + if (nrFailed != 0) { + co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); + } + /* `drvPath' should already be a root, but let's be on the safe side: if the user forgot to make it a root, we wouldn't want things being garbage collected while we're busy. */ @@ -1540,24 +1553,23 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) if (!useDerivation || !drv) return; auto & fullDrv = *dynamic_cast(drv.get()); - std::optional info = tryGetConcreteDrvGoal(waitee); - if (!info) return; - const auto & [dg, drvReq] = *info; + auto * dg = dynamic_cast(&*waitee); + if (!dg) return; - auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get()); + auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath }); if (!nodeP) return; auto & outputs = nodeP->value; for (auto & outputName : outputs) { - auto buildResult = dg.get().getBuildResult(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(dg.get().drvPath), + auto buildResult = dg->getBuildResult(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { auto i = buildResult.builtOutputs.find(outputName); if (i != buildResult.builtOutputs.end()) inputDrvOutputs.insert_or_assign( - { dg.get().drvPath, outputName }, + { dg->drvPath, outputName }, i->second.outPath); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 3ff34509a..4622cb2b1 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -57,10 +57,6 @@ struct InitialOutput { /** * A goal for building some or all of the outputs of a derivation. - * - * The derivation must already be present, either in the store in a drv - * or in memory. If the derivation itself needs to be gotten first, a - * `DerivationCreationAndRealisationGoal` goal must be used instead. */ struct DerivationGoal : public Goal { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index a473daff9..3bf22320e 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,7 +1,6 @@ #include "worker.hh" #include "substitution-goal.hh" #ifndef _WIN32 // TODO Enable building on Windows -# include "derivation-creation-and-realisation-goal.hh" # include "derivation-goal.hh" #endif #include "local-store.hh" @@ -30,8 +29,8 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } if (i->exitCode != Goal::ecSuccess) { #ifndef _WIN32 // TODO Enable building on Windows - if (auto i2 = dynamic_cast(i.get())) - failed.insert(i2->drvReq->to_string(*this)); + if (auto i2 = dynamic_cast(i.get())) + failed.insert(printStorePath(i2->drvPath)); else #endif if (auto i2 = dynamic_cast(i.get())) diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index c381e5b58..9a16da145 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -175,7 +175,7 @@ Goal::Done Goal::amDone(ExitCode result, std::optional ex) exitCode = result; if (ex) { - if (!preserveException && !waiters.empty()) + if (!waiters.empty()) logError(ex->info()); else this->ex = std::move(*ex); diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 2db1098b7..1dd7ed525 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -50,16 +50,6 @@ enum struct JobCategory { * A substitution an arbitrary store object; it will use network resources. */ Substitution, - /** - * A goal that does no "real" work by itself, and just exists to depend on - * other goals which *do* do real work. These goals therefore are not - * limited. - * - * These goals cannot infinitely create themselves, so there is no risk of - * a "fork bomb" type situation (which would be a problem even though the - * goal do no real work) either. - */ - Administration, }; struct Goal : public std::enable_shared_from_this @@ -383,17 +373,6 @@ public: */ BuildResult getBuildResult(const DerivedPath &) const; - /** - * Hack to say that this goal should not log `ex`, but instead keep - * it around. Set by a waitee which sees itself as the designated - * continuation of this goal, responsible for reporting its - * successes or failures. - * - * @todo this is yet another not-nice hack in the goal system that - * we ought to get rid of. See #11927 - */ - bool preserveException = false; - /** * Exception containing an error message, if any. */ diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b765fc2a0..dbe86f43f 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -4,7 +4,6 @@ #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" #include "derivation-goal.hh" -#include "derivation-creation-and-realisation-goal.hh" #ifndef _WIN32 // TODO Enable building on Windows # include "local-derivation-goal.hh" # include "hook-instance.hh" @@ -44,24 +43,6 @@ Worker::~Worker() } -std::shared_ptr Worker::makeDerivationCreationAndRealisationGoal( - ref drvReq, - const OutputsSpec & wantedOutputs, - BuildMode buildMode) -{ - std::weak_ptr & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value; - std::shared_ptr goal = goal_weak.lock(); - if (!goal) { - goal = std::make_shared(drvReq, wantedOutputs, *this, buildMode); - goal_weak = goal; - wakeUp(goal); - } else { - goal->addWantedOutputs(wantedOutputs); - } - return goal; -} - - std::shared_ptr Worker::makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -139,7 +120,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - return makeDerivationCreationAndRealisationGoal(bfd.drvPath, bfd.outputs, buildMode); + if (auto bop = std::get_if(&*bfd.drvPath)) + return makeDerivationGoal(bop->path, bfd.outputs, buildMode); + else + throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -148,46 +132,24 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) } -template -static void cullMap(std::map & goalMap, F f) -{ - for (auto i = goalMap.begin(); i != goalMap.end();) - if (!f(i->second)) - i = goalMap.erase(i); - else ++i; -} - - template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - cullMap(goalMap, [&](const std::weak_ptr & gp) -> bool { - return gp.lock() != goal; - }); -} - -template -static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap); - -template -static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap) -{ - /* !!! inefficient */ - cullMap(goalMap, [&](DerivedPathMap>::ChildNode & node) -> bool { - if (node.value.lock() == goal) - node.value.reset(); - removeGoal(goal, node.childMap); - return !node.value.expired() || !node.childMap.empty(); - }); + for (auto i = goalMap.begin(); + i != goalMap.end(); ) + if (i->second.lock() == goal) { + auto j = i; ++j; + goalMap.erase(i); + i = j; + } + else ++i; } void Worker::removeGoal(GoalPtr goal) { - if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, outerDerivationGoals.map); - else if (auto drvGoal = std::dynamic_pointer_cast(goal)) + if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) @@ -253,9 +215,6 @@ void Worker::childStarted(GoalPtr goal, const std::set 0); nrLocalBuilds--; break; - case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; default: unreachable(); } @@ -334,9 +290,9 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { + if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Built { - .drvPath = goal->drvReq, + .drvPath = makeConstantStorePathRef(goal->drvPath), .outputs = goal->wantedOutputs, }); } else @@ -596,22 +552,4 @@ GoalPtr upcast_goal(std::shared_ptr subGoal) return subGoal; } -GoalPtr upcast_goal(std::shared_ptr subGoal) -{ - return subGoal; -} - -std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee) -{ - auto * odg = dynamic_cast(&*waitee); - if (!odg) return std::nullopt; - /* If we failed to obtain the concrete drv, we won't have created - the concrete derivation goal. */ - if (!odg->concreteDrvGoal) return std::nullopt; - return {{ - std::cref(*odg->concreteDrvGoal), - std::cref(*odg->drvReq), - }}; -} - } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index efd518f99..f5e617208 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "store-api.hh" -#include "derived-path-map.hh" #include "goal.hh" #include "realisation.hh" #include "muxable-pipe.hh" @@ -14,7 +13,6 @@ namespace nix { /* Forward definition. */ -struct DerivationCreationAndRealisationGoal; struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; @@ -33,25 +31,9 @@ class DrvOutputSubstitutionGoal; */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); -GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; -/** - * The current implementation of impure derivations has - * `DerivationGoal`s accumulate realisations from their waitees. - * Unfortunately, `DerivationGoal`s don't directly depend on other - * goals, but instead depend on `DerivationCreationAndRealisationGoal`s. - * - * We try not to share any of the details of any goal type with any - * other, for sake of modularity and quicker rebuilds. This means we - * cannot "just" downcast and fish out the field. So as an escape hatch, - * we have made the function, written in `worker.cc` where all the goal - * types are visible, and use it instead. - */ - -std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee); - /** * A mapping used to remember for each child process to what goal it * belongs, and comm channels for receiving log data and output @@ -121,9 +103,6 @@ private: * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ - - DerivedPathMap> outerDerivationGoals; - std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; @@ -217,9 +196,6 @@ public: * @ref DerivationGoal "derivation goal" */ private: - std::shared_ptr makeDerivationCreationAndRealisationGoal( - ref drvPath, - const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 0095a9d78..c97d52773 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -52,7 +52,6 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single // instantiations -#include "derivation-creation-and-realisation-goal.hh" namespace nix { template<> @@ -69,7 +68,4 @@ std::strong_ordering DerivedPathMap>::ChildNode::operator template struct DerivedPathMap>::ChildNode; template struct DerivedPathMap>; -template struct DerivedPathMap>; - - }; diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 61e0b5463..bd60fe887 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -21,11 +21,8 @@ namespace nix { * * @param V A type to instantiate for each output. It should probably * should be an "optional" type so not every interior node has to have a - * value. For example, the scheduler uses - * `DerivedPathMap>` to - * remember which goals correspond to which outputs. `* const Something` - * or `std::optional` would also be good choices for - * "optional" types. + * value. `* const Something` or `std::optional` would be + * good choices for "optional" types. */ template struct DerivedPathMap { diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 899ba33fe..496c5b10d 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -183,7 +183,6 @@ sources = files( 'binary-cache-store.cc', 'build-result.cc', 'build/derivation-goal.cc', - 'build/derivation-creation-and-realisation-goal.cc', 'build/drv-output-substitution-goal.cc', 'build/entry-points.cc', 'build/goal.cc', @@ -257,7 +256,6 @@ headers = [config_h] + files( 'binary-cache-store.hh', 'build-result.hh', 'build/derivation-goal.hh', - 'build/derivation-creation-and-realisation-goal.hh', 'build/drv-output-substitution-goal.hh', 'build/goal.hh', 'build/substitution-goal.hh', diff --git a/tests/functional/dyn-drv/build-built-drv.sh b/tests/functional/dyn-drv/build-built-drv.sh index fcb25a34b..647be9457 100644 --- a/tests/functional/dyn-drv/build-built-drv.sh +++ b/tests/functional/dyn-drv/build-built-drv.sh @@ -18,9 +18,4 @@ clearStore drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) -# Store layer needs bugfix -requireDaemonNewerThan "2.27pre20250205" - -out2=$(nix build "${drvDep}^out^out" --no-link) - -test $out1 == $out2 +expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" diff --git a/tests/functional/dyn-drv/dep-built-drv-2.sh b/tests/functional/dyn-drv/dep-built-drv-2.sh index 531af6bf7..3247720af 100644 --- a/tests/functional/dyn-drv/dep-built-drv-2.sh +++ b/tests/functional/dyn-drv/dep-built-drv-2.sh @@ -13,4 +13,4 @@ restartDaemon NIX_BIN_DIR="$(dirname "$(type -p nix)")" export NIX_BIN_DIR -nix build -L --file ./non-trivial.nix --no-link +expectStderr 1 nix build -L --file ./non-trivial.nix --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" diff --git a/tests/functional/dyn-drv/dep-built-drv.sh b/tests/functional/dyn-drv/dep-built-drv.sh index 9d470099a..4f6e9b080 100644 --- a/tests/functional/dyn-drv/dep-built-drv.sh +++ b/tests/functional/dyn-drv/dep-built-drv.sh @@ -4,11 +4,8 @@ source common.sh out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link) -# Store layer needs bugfix -requireDaemonNewerThan "2.27pre20250205" - clearStore -out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link) +expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" -diff -r $out1 $out2 +# diff -r $out1 $out2 diff --git a/tests/functional/dyn-drv/failing-outer.sh b/tests/functional/dyn-drv/failing-outer.sh index d888ea876..fbad70701 100644 --- a/tests/functional/dyn-drv/failing-outer.sh +++ b/tests/functional/dyn-drv/failing-outer.sh @@ -5,6 +5,8 @@ source common.sh # Store layer needs bugfix requireDaemonNewerThan "2.27pre20250205" +skipTest "dyn drv input scheduling had to be reverted for 2.27" + expected=100 if [[ -v NIX_DAEMON_PACKAGE ]]; then expected=1; fi # work around the daemon not returning a 100 status correctly From 8fdb50761d7997bf9462f23462044bc3005f1269 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 3 Mar 2025 10:31:23 -0500 Subject: [PATCH 011/396] `SingleDerivedPath` should be const in recursive data structures --- src/libstore/build/derivation-goal.cc | 4 ++-- src/libstore/derived-path.cc | 8 ++++---- src/libstore/derived-path.hh | 8 ++++---- src/nix/log.cc | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 01da37df6..b2bb9da75 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -333,9 +333,9 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() /* The inputs must be built before we can build this goal. */ inputDrvOutputs.clear(); if (useDerivation) { - std::function, const DerivedPathMap::ChildNode &)> addWaiteeDerivedPath; + std::function, const DerivedPathMap::ChildNode &)> addWaiteeDerivedPath; - addWaiteeDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + addWaiteeDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { if (!inputNode.value.empty()) addWaitee(worker.makeGoal( DerivedPath::Built { diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 1eef881de..3e3318965 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -170,7 +170,7 @@ void drvRequireExperiment( } SingleDerivedPath::Built SingleDerivedPath::Built::parse( - const StoreDirConfig & store, ref drv, + const StoreDirConfig & store, ref drv, OutputNameView output, const ExperimentalFeatureSettings & xpSettings) { @@ -182,7 +182,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse( } DerivedPath::Built DerivedPath::Built::parse( - const StoreDirConfig & store, ref drv, + const StoreDirConfig & store, ref drv, OutputNameView outputsS, const ExperimentalFeatureSettings & xpSettings) { @@ -201,7 +201,7 @@ static SingleDerivedPath parseWithSingle( return n == s.npos ? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s) : (SingleDerivedPath) SingleDerivedPath::Built::parse(store, - make_ref(parseWithSingle( + make_ref(parseWithSingle( store, s.substr(0, n), separator, @@ -234,7 +234,7 @@ static DerivedPath parseWith( return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) : (DerivedPath) DerivedPath::Built::parse(store, - make_ref(parseWithSingle( + make_ref(parseWithSingle( store, s.substr(0, n), separator, diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 4ba3fb37d..23f9c2b30 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -45,7 +45,7 @@ struct SingleDerivedPath; * path of the given output name. */ struct SingleDerivedPathBuilt { - ref drvPath; + ref drvPath; OutputName output; /** @@ -74,7 +74,7 @@ struct SingleDerivedPathBuilt { * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPathBuilt parse( - const StoreDirConfig & store, ref drvPath, + const StoreDirConfig & store, ref drvPath, OutputNameView outputs, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -172,7 +172,7 @@ static inline ref makeConstantStorePathRef(StorePath drvPath) * output name. */ struct DerivedPathBuilt { - ref drvPath; + ref drvPath; OutputsSpec outputs; /** @@ -201,7 +201,7 @@ struct DerivedPathBuilt { * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPathBuilt parse( - const StoreDirConfig & store, ref, + const StoreDirConfig & store, ref, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; diff --git a/src/nix/log.cc b/src/nix/log.cc index 2c35ed803..6e23188db 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -35,7 +35,7 @@ struct CmdLog : InstallableCommand // For compat with CLI today, TODO revisit auto oneUp = std::visit(overloaded { [&](const DerivedPath::Opaque & bo) { - return make_ref(bo); + return make_ref(bo); }, [&](const DerivedPath::Built & bfd) { return bfd.drvPath; From a58e0584f5e250ace8e31f53b63e323ebf35dee0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 24 Feb 2025 10:55:02 -0500 Subject: [PATCH 012/396] Rework derivation input resolution I refactored the way that input resolution works in `DerivationGoal`. To be honest, it is probably unclear to the reader whether this new way is better or worse. I suppose *intrinsic* motivation, I can say that - the more structured use of `inputGoal` (a local variable) is better than the shotgrun approach with `inputDrvOutputs` - A virtual `waiteeDone` was a hack, and now it's gone. However, the *real* motivation of this is not the above things, but that it is needed for my mammoth refactor fixing #11897 and #11928. It is nice that this step could come first, rather than making that refactor even bigger. --- src/libstore/build/derivation-goal.cc | 143 +++++++++++--------------- src/libstore/build/derivation-goal.hh | 9 -- src/libstore/build/goal.hh | 2 +- src/libstore/build/worker.cc | 5 + src/libstore/derivations.cc | 60 +++++------ src/libstore/derivations.hh | 2 +- 6 files changed, 91 insertions(+), 130 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b2bb9da75..a32a714e3 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -322,6 +322,18 @@ Goal::Co DerivationGoal::haveDerivation() } +/** + * Used for `inputGoals` local variable below + */ +struct value_comparison +{ + template + bool operator()(const ref & lhs, const ref & rhs) const { + return *lhs < *rhs; + } +}; + + /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ Goal::Co DerivationGoal::gaveUpOnSubstitution() @@ -330,19 +342,22 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() is no need to restart. */ needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed; - /* The inputs must be built before we can build this goal. */ - inputDrvOutputs.clear(); + std::map, GoalPtr, value_comparison> inputGoals; + if (useDerivation) { std::function, const DerivedPathMap::ChildNode &)> addWaiteeDerivedPath; addWaiteeDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { - if (!inputNode.value.empty()) - addWaitee(worker.makeGoal( + if (!inputNode.value.empty()) { + auto g = worker.makeGoal( DerivedPath::Built { .drvPath = inputDrv, .outputs = inputNode.value, }, - buildMode == bmRepair ? bmRepair : bmNormal)); + buildMode == bmRepair ? bmRepair : bmNormal); + inputGoals.insert_or_assign(inputDrv, g); + addWaitee(std::move(g)); + } for (const auto & [outputName, childNode] : inputNode.childMap) addWaiteeDerivedPath( make_ref(SingleDerivedPath::Built { inputDrv, outputName }), @@ -430,7 +445,12 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() [&](const DerivationType::Impure &) { return true; } - }, drvType.raw); + }, drvType.raw) + /* no inputs are outputs of dynamic derivations */ + || std::ranges::any_of( + fullDrv.inputDrvs.map.begin(), + fullDrv.inputDrvs.map.end(), + [](auto & pair) { return !pair.second.childMap.empty(); }); if (resolveDrv && !fullDrv.inputDrvs.map.empty()) { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -438,7 +458,19 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal. */ - std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs); + std::optional attempt = fullDrv.tryResolve(worker.store, + [&](ref drvPath, const std::string & outputName) -> std::optional { + auto mEntry = get(inputGoals, drvPath); + if (!mEntry) return std::nullopt; + + auto buildResult = (*mEntry)->getBuildResult(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}}); + if (!buildResult.success()) return std::nullopt; + + auto i = get(buildResult.builtOutputs, outputName); + if (!i) return std::nullopt; + + return i->outPath; + }); if (!attempt) { /* TODO (impure derivations-induced tech debt) (see below): The above attempt should have found it, but because we manage @@ -469,54 +501,31 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() co_return resolvedFinished(); } - std::function::ChildNode &)> accumInputPaths; + /* If we get this far, we know no dynamic drvs inputs */ - accumInputPaths = [&](const StorePath & depDrvPath, const DerivedPathMap::ChildNode & inputNode) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - auto getOutput = [&](const std::string & outputName) { - /* TODO (impure derivations-induced tech debt): - Tracking input derivation outputs statefully through the - goals is error prone and has led to bugs. - For a robust nix, we need to move towards the `else` branch, - which does not rely on goal state to match up with the - reality of the store, which is our real source of truth. - However, the impure derivations feature still relies on this - fragile way of doing things, because its builds do not have - a representation in the store, which is a usability problem - in itself. When implementing this logic entirely with lookups - make sure that they're cached. */ - if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) { - return *outPath; + for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map) { + for (auto & outputName : depNode.value) { + /* Don't need to worry about `inputGoals`, because + impure derivations are always resolved above. Can + just use DB. This case only happens in the (older) + input addressed and fixed output derivation cases. */ + auto outMap = [&]{ + for (auto * drvStore : { &worker.evalStore, &worker.store }) + if (drvStore->isValidPath(depDrvPath)) + return worker.store.queryDerivationOutputMap(depDrvPath, drvStore); + assert(false); + }(); + + auto outMapPath = outMap.find(outputName); + if (outMapPath == outMap.end()) { + throw Error( + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath)); } - else { - auto outMap = [&]{ - for (auto * drvStore : { &worker.evalStore, &worker.store }) - if (drvStore->isValidPath(depDrvPath)) - return worker.store.queryDerivationOutputMap(depDrvPath, drvStore); - assert(false); - }(); - auto outMapPath = outMap.find(outputName); - if (outMapPath == outMap.end()) { - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath)); - } - return outMapPath->second; - } - }; - - for (auto & outputName : inputNode.value) - worker.store.computeFSClosure(getOutput(outputName), inputPaths); - - for (auto & [outputName, childNode] : inputNode.childMap) - accumInputPaths(getOutput(outputName), childNode); - }; - - for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map) - accumInputPaths(depDrvPath, depNode); + worker.store.computeFSClosure(outMapPath->second, inputPaths); + } + } } /* Second, the input sources. */ @@ -1545,34 +1554,4 @@ Goal::Done DerivationGoal::done( return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } - -void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - Goal::waiteeDone(waitee, result); - - if (!useDerivation || !drv) return; - auto & fullDrv = *dynamic_cast(drv.get()); - - auto * dg = dynamic_cast(&*waitee); - if (!dg) return; - - auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath }); - if (!nodeP) return; - auto & outputs = nodeP->value; - - for (auto & outputName : outputs) { - auto buildResult = dg->getBuildResult(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(dg->drvPath), - .outputs = OutputsSpec::Names { outputName }, - }); - if (buildResult.success()) { - auto i = buildResult.builtOutputs.find(outputName); - if (i != buildResult.builtOutputs.end()) - inputDrvOutputs.insert_or_assign( - { dg->drvPath, outputName }, - i->second.outPath); - } - } -} - } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 4622cb2b1..7f594b92c 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -78,13 +78,6 @@ struct DerivationGoal : public Goal */ OutputsSpec wantedOutputs; - /** - * Mapping from input derivations + output names to actual store - * paths. This is filled in by waiteeDone() as each dependency - * finishes, before `trace("all inputs realised")` is reached. - */ - std::map, StorePath> inputDrvOutputs; - /** * See `needRestart`; just for that field. */ @@ -331,8 +324,6 @@ struct DerivationGoal : public Goal SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); - void waiteeDone(GoalPtr waitee, ExitCode result) override; - StorePathSet exportReferences(const StorePathSet & storePaths); JobCategory jobCategory() const override { diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 1dd7ed525..95572ef02 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -396,7 +396,7 @@ public: void addWaitee(GoalPtr waitee); - virtual void waiteeDone(GoalPtr waitee, ExitCode result); + void waiteeDone(GoalPtr waitee, ExitCode result); virtual void handleChildOutput(Descriptor fd, std::string_view data) { diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index dbe86f43f..dad460a1e 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -552,4 +552,9 @@ GoalPtr upcast_goal(std::shared_ptr subGoal) return subGoal; } +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ + return subGoal; +} + } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b54838a0a..c31c801c2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1052,49 +1052,36 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String std::optional Derivation::tryResolve(Store & store, Store * evalStore) const { - std::map, StorePath> inputDrvOutputs; - - std::function::ChildNode &)> accum; - accum = [&](auto & inputDrv, auto & node) { - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) { - if (outputPath) { - inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); - if (auto p = get(node.childMap, outputName)) - accum(*outputPath, *p); + return tryResolve( + store, + [&](ref drvPath, const std::string & outputName) -> std::optional { + try { + return resolveDerivedPath(store, SingleDerivedPath::Built{drvPath, outputName}, evalStore); + } catch (Error &) { + return std::nullopt; } - } - }; - - for (auto & [inputDrv, node] : inputDrvs.map) - accum(inputDrv, node); - - return tryResolve(store, inputDrvOutputs); + }); } static bool tryResolveInput( Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites, const DownstreamPlaceholder * placeholderOpt, - const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode, - const std::map, StorePath> & inputDrvOutputs) + ref drvPath, const DerivedPathMap::ChildNode & inputNode, + std::function(ref drvPath, const std::string & outputName)> queryResolutionChain) { - auto getOutput = [&](const std::string & outputName) { - auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName }); - if (!actualPathOpt) - warn("output %s of input %s missing, aborting the resolving", - outputName, - store.printStorePath(inputDrv) - ); - return actualPathOpt; - }; - auto getPlaceholder = [&](const std::string & outputName) { return placeholderOpt ? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName) - : DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName); + : [&]{ + auto * p = std::get_if(&drvPath->raw()); + // otherwise we should have had a placeholder to build-upon already + assert(p); + return DownstreamPlaceholder::unknownCaOutput(p->path, outputName); + }(); }; for (auto & outputName : inputNode.value) { - auto actualPathOpt = getOutput(outputName); + auto actualPathOpt = queryResolutionChain(drvPath, outputName); if (!actualPathOpt) return false; auto actualPath = *actualPathOpt; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { @@ -1106,13 +1093,12 @@ static bool tryResolveInput( } for (auto & [outputName, childNode] : inputNode.childMap) { - auto actualPathOpt = getOutput(outputName); - if (!actualPathOpt) return false; - auto actualPath = *actualPathOpt; auto nextPlaceholder = getPlaceholder(outputName); if (!tryResolveInput(store, inputSrcs, inputRewrites, - &nextPlaceholder, actualPath, childNode, - inputDrvOutputs)) + &nextPlaceholder, + make_ref(SingleDerivedPath::Built{drvPath, outputName}), + childNode, + queryResolutionChain)) return false; } return true; @@ -1120,7 +1106,7 @@ static bool tryResolveInput( std::optional Derivation::tryResolve( Store & store, - const std::map, StorePath> & inputDrvOutputs) const + std::function(ref drvPath, const std::string & outputName)> queryResolutionChain) const { BasicDerivation resolved { *this }; @@ -1129,7 +1115,7 @@ std::optional Derivation::tryResolve( for (auto & [inputDrv, inputNode] : inputDrvs.map) if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, - nullptr, inputDrv, inputNode, inputDrvOutputs)) + nullptr, make_ref(SingleDerivedPath::Opaque{inputDrv}), inputNode, queryResolutionChain)) return std::nullopt; rewriteDerivation(store, resolved, inputRewrites); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 5b2101ed5..7b62b81ba 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -369,7 +369,7 @@ struct Derivation : BasicDerivation */ std::optional tryResolve( Store & store, - const std::map, StorePath> & inputDrvOutputs) const; + std::function(ref drvPath, const std::string & outputName)> queryResolutionChain) const; /** * Check that the derivation is valid and does not present any From 1e00d14c29b4ec1fce709968cf3adb071681d4fa Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 3 Mar 2025 19:09:24 +0100 Subject: [PATCH 013/396] manual: Edit --- .../advanced-topics/distributed-builds.md | 4 ++- doc/manual/source/glossary.md | 20 ++++++++++++++- doc/manual/source/protocols/store-path.md | 2 +- doc/manual/source/store/building.md | 5 +++- doc/manual/source/store/derivation/index.md | 25 +++++++++++-------- .../derivation/outputs/content-address.md | 4 +-- .../store/derivation/outputs/input-address.md | 2 +- .../store/store-object/content-address.md | 20 +++++++-------- 8 files changed, 55 insertions(+), 27 deletions(-) diff --git a/doc/manual/source/advanced-topics/distributed-builds.md b/doc/manual/source/advanced-topics/distributed-builds.md index 66e371888..464b87d6e 100644 --- a/doc/manual/source/advanced-topics/distributed-builds.md +++ b/doc/manual/source/advanced-topics/distributed-builds.md @@ -20,7 +20,7 @@ For a local machine to forward a build to a remote machine, the remote machine m ## Testing -To test connecting to a remote Nix instance (in this case `mac`), run: +To test connecting to a remote [Nix instance] (in this case `mac`), run: ```console nix store info --store ssh://username@mac @@ -106,3 +106,5 @@ file included in `builders` via the syntax `@/path/to/file`. For example, causes the list of machines in `/etc/nix/machines` to be included. (This is the default.) + +[Nix instance]: @docroot@/glossary.md#gloss-nix-instance \ No newline at end of file diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index db6d18f0e..6a7501200 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -1,5 +1,13 @@ # Glossary +- [build system]{#gloss-build-system} + + Generic term for software that facilitates the building of software by automating the invocation of compilers, linkers, and other tools. + + Nix can be used as a generic build system. + It has no knowledge of any particular programming language or toolchain. + These details are specified in [derivation expressions](#gloss-derivation-expression). + - [content address]{#gloss-content-address} A @@ -19,6 +27,10 @@ Besides content addressing, the Nix store also uses [input addressing](#gloss-input-addressed-store-object). +- [content-addressed storage]{#gloss-content-addressed-store} + + The industry term for storage and retrieval systems using [content addressing](#gloss-content-address). A Nix store also has [input addressing](#gloss-input-addressed-store-object), and metadata. + - [store derivation]{#gloss-store-derivation} A single build task. @@ -88,6 +100,12 @@ [store]: #gloss-store +- [Nix instance]{#gloss-nix-instance} + + 1. An installation of Nix, which includes the presence of a [store], and the Nix package manager which operates on that store. + A local Nix installation and a [remote builder](@docroot@/advanced-topics/distributed-builds.md) are two examples of Nix instances. + 2. A running Nix process, such as the `nix` command. + - [binary cache]{#gloss-binary-cache} A *binary cache* is a Nix store which uses a different format: its @@ -220,7 +238,7 @@ directly or indirectly “reachable” from that store path; that is, it’s the closure of the path under the *references* relation. For a package, the closure of its derivation is equivalent to the - build-time dependencies, while the closure of its output path is + build-time dependencies, while the closure of its [output path] is equivalent to its runtime dependencies. For correct deployment it is necessary to deploy whole closures, since otherwise at runtime files could be missing. The command `nix-store --query --requisites ` prints out diff --git a/doc/manual/source/protocols/store-path.md b/doc/manual/source/protocols/store-path.md index 8ec6f8201..9abd83f4f 100644 --- a/doc/manual/source/protocols/store-path.md +++ b/doc/manual/source/protocols/store-path.md @@ -53,7 +53,7 @@ where method of content addressing store objects, if the hash algorithm is [SHA-256]. Just like in the "Text" case, we can have the store objects referenced by their paths. - Additionally, we can have an optional `:self` label to denote self reference. + Additionally, we can have an optional `:self` label to denote self-reference. - ```ebnf | "output:" id diff --git a/doc/manual/source/store/building.md b/doc/manual/source/store/building.md index feefa8e3f..dbfe6b5ca 100644 --- a/doc/manual/source/store/building.md +++ b/doc/manual/source/store/building.md @@ -54,7 +54,7 @@ The [`builder`](./derivation/index.md#builder) is executed as follows: it’s `out`.) - If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple Nix instances from performing the same + acquired to prevent multiple [Nix instances][Nix instance] from performing the same build at the same time. - A log of the combined standard output and error is written to @@ -95,3 +95,6 @@ If the builder exited successfully, the following steps happen in order to turn Nix also scans for references to other outputs' paths in the same way, because outputs are allowed to refer to each other. If the outputs' references to each other form a cycle, this is an error, because the references of store objects much be acyclic. + + +[Nix instance]: @docroot@/glossary.md#gloss-nix-instance diff --git a/doc/manual/source/store/derivation/index.md b/doc/manual/source/store/derivation/index.md index 42cfa67f5..911c28485 100644 --- a/doc/manual/source/store/derivation/index.md +++ b/doc/manual/source/store/derivation/index.md @@ -1,7 +1,7 @@ # Store Derivation and Deriving Path -Besides functioning as a [content addressed store] the Nix store layer works as a [build system]. -Other system (like Git or IPFS) also store and transfer immutable data, but they don't concern themselves with *how* that data was created. +Besides functioning as a [content-addressed store], the Nix store layer works as a [build system]. +Other systems (like Git or IPFS) also store and transfer immutable data, but they don't concern themselves with *how* that data was created. This is where Nix distinguishes itself. *Derivations* represent individual build steps, and *deriving paths* are needed to refer to the *outputs* of those build steps before they are built. @@ -42,6 +42,8 @@ A derivation consists of: [args]: #args [env]: #env [system]: #system +[content-addressed store]: @docroot@/glossary.md#gloss-content-addressed-store +[build system]: @docroot@/glossary.md#gloss-build-system ### Referencing derivations {#derivation-path} @@ -78,7 +80,7 @@ type DerivingPath = ConstantPath | OutputPath; ``` Deriving paths are necessary because, in general and particularly for [content-addressing derivations][content-addressing derivation], the [store path] of an [output] is not known in advance. -We can use an output deriving path to refer to such an out, instead of the store path which we do not yet know. +We can use an output deriving path to refer to such an output, instead of the store path which we do not yet know. [deriving path]: #deriving-path [validity]: @docroot@/glossary.md#gloss-validity @@ -89,25 +91,26 @@ A derivation is constructed from the parts documented in the following subsectio ### Inputs {#inputs} -The inputs are a set of [deriving paths][deriving path], refering to all store objects needed in order to perform this build step. +The inputs are a set of [deriving paths][deriving path], referring to all store objects needed in order to perform this build step. The [process creation fields] will presumably include many [store paths][store path]: - The path to the executable normally starts with a store path - The arguments and environment variables likely contain many other store paths. -But rather than somehow scanning all the other fields for inputs, Nix requires that all inputs be explicitly collected in the inputs field. It is instead the responsibility of the creator of a derivation (e.g. the evaluator) to ensure that every store object referenced in another field (e.g. referenced by store path) is included in this inputs field. +But rather than somehow scanning all the other fields for inputs, Nix requires that all inputs be explicitly collected in the inputs field. It is instead the responsibility of the creator of a derivation (e.g. the evaluator) to ensure that every store object referenced in another field (e.g. referenced by store path) is included in this inputs field. ### System {#system} The system type on which the [`builder`](#attr-builder) executable is meant to be run. -A necessary condition for Nix to schedule a given derivation on some Nix instance is for the "system" of that derivation to match that instance's [`system` configuration option]. +A necessary condition for Nix to schedule a given derivation on some [Nix instance] is for the "system" of that derivation to match that instance's [`system` configuration option] or [`extra-platforms` configuration option]. By putting the `system` in each derivation, Nix allows *heterogenous* build plans, where not all steps can be run on the same machine or same sort of machine. Nix can schedule builds such that it automatically builds on other platforms by [forwarding build requests](@docroot@/advanced-topics/distributed-builds.md) to other Nix instances. [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system +[`extra-platforms` configuration option]: @docroot@/command-ref/conf-file.md#conf-extra-platforms [content-addressing derivation]: @docroot@/glossary.md#gloss-content-addressing-derivation [realise]: @docroot@/glossary.md#gloss-realise @@ -240,14 +243,14 @@ That works because we've implicitly assumed that all derivations are created *st But what if derivations could also be created dynamically within Nix? In other words, what if derivations could be the outputs of other derivations? -:::{.note} -In the parlance of "Build Systems à la carte", we are generalizing the Nix store layer to be a "Monadic" instead of "Applicative" build system. -::: +> **Note** +> +> In the parlance of "Build Systems à la carte", we are generalizing the Nix store layer to be a "Monadic" instead of "Applicative" build system. How should we refer to such derivations? A deriving path works, the same as how we refer to other derivation outputs. But what about a dynamic derivations output? -(i.e. how do we refer to the output of an output of a derivation?) +(i.e. how do we refer to the output of a derivation, which is itself an output of a derivation?) For that we need to generalize the definition of deriving path, replacing the store path used to refer to the derivation with a nested deriving path: ```diff @@ -295,3 +298,5 @@ The result of this is that it is possible to have a chain of `^` at > |------------------------------------------------------------| |-----| > innermost constant store path (usual encoding) output name > ``` + +[Nix instance]: @docroot@/glossary.md#gloss-nix-instance diff --git a/doc/manual/source/store/derivation/outputs/content-address.md b/doc/manual/source/store/derivation/outputs/content-address.md index 21e940bc2..4539a5eba 100644 --- a/doc/manual/source/store/derivation/outputs/content-address.md +++ b/doc/manual/source/store/derivation/outputs/content-address.md @@ -12,7 +12,7 @@ Given the method, the output's name (computed from the derivation name and outpu ## Fixed-output content-addressing {#fixed} -In this case the content-address of the *fixed* in advanced by the derivation itself. +In this case the content address of the *fixed* in advanced by the derivation itself. In other words, when the derivation has finished [building](@docroot@/store/building.md), and the provisional output' content-address is computed as part of the process to turn it into a *bona fide* store object, the calculated content address must much that given in the derivation, or the build of that derivation will be deemed a failure. The output spec for an output with a fixed content addresses additionally contains: @@ -159,7 +159,7 @@ A *determinstic* content-addressing derivation should produce outputs with the s ### Floating versus Fixed -While the destinction between content- and input-addressing is one of *mechanism*, the distinction between fixed and floating content addression is more one of *policy*. +While the distinction between content- and input-addressing is one of *mechanism*, the distinction between fixed and floating content addressing is more one of *policy*. A fixed output that passes its content address check is just like a floating output. It is only in the potential for that check to fail that they are different. diff --git a/doc/manual/source/store/derivation/outputs/input-address.md b/doc/manual/source/store/derivation/outputs/input-address.md index 54d9437d9..e2e15a801 100644 --- a/doc/manual/source/store/derivation/outputs/input-address.md +++ b/doc/manual/source/store/derivation/outputs/input-address.md @@ -3,7 +3,7 @@ [input addressing]: #input-addressing "Input addressing" means the address the store object by the *way it was made* rather than *what it is*. -That is to say, an input-addressed output's store path is a function not of the output itself, but the derivation that produced it. +That is to say, an input-addressed output's store path is a function not of the output itself, but of the derivation that produced it. Even if two store paths have the same contents, if they are produced in different ways, and one is input-addressed, then they will have different store paths, and thus guaranteed to not be the same store object. See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. From 6cd2b4e169696410ed3959c13c2c041b51db9c21 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Apr 2025 18:40:27 +0200 Subject: [PATCH 206/396] Move alias support from NixArgs to MultiCommand This allows subcommands to declare aliases, e.g. `nix store ping` is now a proper alias of `nix store info`. --- doc/manual/meson.build | 1 - src/libutil/args.cc | 21 +++++++ src/libutil/include/nix/util/args.hh | 22 ++++++++ src/nix/main.cc | 83 +++++++++------------------- src/nix/store-info.cc | 15 +---- src/nix/store.cc | 6 +- 6 files changed, 75 insertions(+), 73 deletions(-) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 8796cee63..1d6687910 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -281,7 +281,6 @@ nix3_manpages = [ 'nix3-store', 'nix3-store-optimise', 'nix3-store-path-from-hash-part', - 'nix3-store-ping', 'nix3-store-prefetch-file', 'nix3-store-repair', 'nix3-store-sign', diff --git a/src/libutil/args.cc b/src/libutil/args.cc index b4177bf93..fb9d7163c 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -647,4 +647,25 @@ nlohmann::json MultiCommand::toJSON() return res; } +Strings::iterator MultiCommand::rewriteArgs(Strings & args, Strings::iterator pos) +{ + if (command) + return command->second->rewriteArgs(args, pos); + + if (aliasUsed || pos == args.end()) return pos; + auto arg = *pos; + auto i = aliases.find(arg); + if (i == aliases.end()) return pos; + auto & info = i->second; + if (info.status == AliasStatus::Deprecated) { + warn("'%s' is a deprecated alias for '%s'", + arg, concatStringsSep(" ", info.replacement)); + } + pos = args.erase(pos); + for (auto j = info.replacement.rbegin(); j != info.replacement.rend(); ++j) + pos = args.insert(pos, *j); + aliasUsed = true; + return pos; +} + } diff --git a/src/libutil/include/nix/util/args.hh b/src/libutil/include/nix/util/args.hh index 77c4fb5b6..463270374 100644 --- a/src/libutil/include/nix/util/args.hh +++ b/src/libutil/include/nix/util/args.hh @@ -393,8 +393,30 @@ public: nlohmann::json toJSON() override; + enum struct AliasStatus { + /** Aliases that don't go away */ + AcceptedShorthand, + /** Aliases that will go away */ + Deprecated, + }; + + /** An alias, except for the original syntax, which is in the map key. */ + struct AliasInfo { + AliasStatus status; + std::vector replacement; + }; + + /** + * A list of aliases (remapping a deprecated/shorthand subcommand + * to something else). + */ + std::map aliases; + + Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) override; + protected: std::string commandName = ""; + bool aliasUsed = false; }; Strings argvToStrings(int argc, char * * argv); diff --git a/src/nix/main.cc b/src/nix/main.cc index a2c9dcf68..f229ba2a4 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -50,19 +50,6 @@ void chrootHelper(int argc, char * * argv); namespace nix { -enum struct AliasStatus { - /** Aliases that don't go away */ - AcceptedShorthand, - /** Aliases that will go away */ - Deprecated, -}; - -/** An alias, except for the original syntax, which is in the map key. */ -struct AliasInfo { - AliasStatus status; - std::vector replacement; -}; - /* Check if we have a non-loopback/link-local network interface. */ static bool haveInternet() { @@ -153,54 +140,34 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs .handler = {[&]() { refresh = true; }}, .experimentalFeature = Xp::NixCommand, }); - } - std::map aliases = { - {"add-to-store", { AliasStatus::Deprecated, {"store", "add-path"}}}, - {"cat-nar", { AliasStatus::Deprecated, {"nar", "cat"}}}, - {"cat-store", { AliasStatus::Deprecated, {"store", "cat"}}}, - {"copy-sigs", { AliasStatus::Deprecated, {"store", "copy-sigs"}}}, - {"dev-shell", { AliasStatus::Deprecated, {"develop"}}}, - {"diff-closures", { AliasStatus::Deprecated, {"store", "diff-closures"}}}, - {"dump-path", { AliasStatus::Deprecated, {"store", "dump-path"}}}, - {"hash-file", { AliasStatus::Deprecated, {"hash", "file"}}}, - {"hash-path", { AliasStatus::Deprecated, {"hash", "path"}}}, - {"ls-nar", { AliasStatus::Deprecated, {"nar", "ls"}}}, - {"ls-store", { AliasStatus::Deprecated, {"store", "ls"}}}, - {"make-content-addressable", { AliasStatus::Deprecated, {"store", "make-content-addressed"}}}, - {"optimise-store", { AliasStatus::Deprecated, {"store", "optimise"}}}, - {"ping-store", { AliasStatus::Deprecated, {"store", "info"}}}, - {"sign-paths", { AliasStatus::Deprecated, {"store", "sign"}}}, - {"shell", { AliasStatus::AcceptedShorthand, {"env", "shell"}}}, - {"show-derivation", { AliasStatus::Deprecated, {"derivation", "show"}}}, - {"show-config", { AliasStatus::Deprecated, {"config", "show"}}}, - {"to-base16", { AliasStatus::Deprecated, {"hash", "to-base16"}}}, - {"to-base32", { AliasStatus::Deprecated, {"hash", "to-base32"}}}, - {"to-base64", { AliasStatus::Deprecated, {"hash", "to-base64"}}}, - {"verify", { AliasStatus::Deprecated, {"store", "verify"}}}, - {"doctor", { AliasStatus::Deprecated, {"config", "check"}}}, + aliases = { + {"add-to-store", { AliasStatus::Deprecated, {"store", "add-path"}}}, + {"cat-nar", { AliasStatus::Deprecated, {"nar", "cat"}}}, + {"cat-store", { AliasStatus::Deprecated, {"store", "cat"}}}, + {"copy-sigs", { AliasStatus::Deprecated, {"store", "copy-sigs"}}}, + {"dev-shell", { AliasStatus::Deprecated, {"develop"}}}, + {"diff-closures", { AliasStatus::Deprecated, {"store", "diff-closures"}}}, + {"dump-path", { AliasStatus::Deprecated, {"store", "dump-path"}}}, + {"hash-file", { AliasStatus::Deprecated, {"hash", "file"}}}, + {"hash-path", { AliasStatus::Deprecated, {"hash", "path"}}}, + {"ls-nar", { AliasStatus::Deprecated, {"nar", "ls"}}}, + {"ls-store", { AliasStatus::Deprecated, {"store", "ls"}}}, + {"make-content-addressable", { AliasStatus::Deprecated, {"store", "make-content-addressed"}}}, + {"optimise-store", { AliasStatus::Deprecated, {"store", "optimise"}}}, + {"ping-store", { AliasStatus::Deprecated, {"store", "info"}}}, + {"sign-paths", { AliasStatus::Deprecated, {"store", "sign"}}}, + {"shell", { AliasStatus::AcceptedShorthand, {"env", "shell"}}}, + {"show-derivation", { AliasStatus::Deprecated, {"derivation", "show"}}}, + {"show-config", { AliasStatus::Deprecated, {"config", "show"}}}, + {"to-base16", { AliasStatus::Deprecated, {"hash", "to-base16"}}}, + {"to-base32", { AliasStatus::Deprecated, {"hash", "to-base32"}}}, + {"to-base64", { AliasStatus::Deprecated, {"hash", "to-base64"}}}, + {"verify", { AliasStatus::Deprecated, {"store", "verify"}}}, + {"doctor", { AliasStatus::Deprecated, {"config", "check"}}}, + }; }; - bool aliasUsed = false; - - Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) override - { - if (aliasUsed || command || pos == args.end()) return pos; - auto arg = *pos; - auto i = aliases.find(arg); - if (i == aliases.end()) return pos; - auto & info = i->second; - if (info.status == AliasStatus::Deprecated) { - warn("'%s' is a deprecated alias for '%s'", - arg, concatStringsSep(" ", info.replacement)); - } - pos = args.erase(pos); - for (auto j = info.replacement.rbegin(); j != info.replacement.rend(); ++j) - pos = args.insert(pos, *j); - aliasUsed = true; - return pos; - } - std::string description() override { return "a tool for reproducible and declarative configuration management"; diff --git a/src/nix/store-info.cc b/src/nix/store-info.cc index 323dfeee4..c4c63ae3a 100644 --- a/src/nix/store-info.cc +++ b/src/nix/store-info.cc @@ -7,7 +7,7 @@ using namespace nix; -struct CmdPingStore : StoreCommand, MixJSON +struct CmdInfoStore : StoreCommand, MixJSON { std::string description() override { @@ -46,15 +46,4 @@ struct CmdPingStore : StoreCommand, MixJSON } }; -struct CmdInfoStore : CmdPingStore -{ - void run(nix::ref store) override - { - warn("'nix store ping' is a deprecated alias for 'nix store info'"); - CmdPingStore::run(store); - } -}; - - -static auto rCmdPingStore = registerCommand2({"store", "info"}); -static auto rCmdInfoStore = registerCommand2({"store", "ping"}); +static auto rCmdInfoStore = registerCommand2({"store", "info"}); diff --git a/src/nix/store.cc b/src/nix/store.cc index b40b6d068..80f9363ca 100644 --- a/src/nix/store.cc +++ b/src/nix/store.cc @@ -5,7 +5,11 @@ using namespace nix; struct CmdStore : NixMultiCommand { CmdStore() : NixMultiCommand("store", RegisterCommand::getCommandsFor({"store"})) - { } + { + aliases = { + {"ping", { AliasStatus::Deprecated, {"info"}}}, + }; + } std::string description() override { From a0d3003bf21233feccf9c7a7b0ac782547e14535 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 10 Apr 2025 22:50:28 +0200 Subject: [PATCH 207/396] glossary: refine the definition of "package" This change follows the definition from aptitude, but using precise notions from Nix: > package managers deal with packages: collections of files that are > bundled together and can be installed and removed as a group. > [...] > If a package A depends upon another package B, then B is required > for A to operate properly. > [...] > The job of a package manager is to present an interface which assists > the user in managing the collection of packages installed on his or her system. > > -- An interesting addition: > Packages are abstractions defining the granularity at which users can act > (add, remove, upgrade, etc.) on available software. > A distribution is a collection of packages maintained (hopefully) coherently. > > -- Package Upgrades in FOSS Distributions: Details and Challenges > (Roberto Di Cosmo, Stefano Zacchiroli; 2009) Notably these quotes and this change don't say anything about installation, or what it means for software to be available. In practice, this is handled downstream, e.g. in NixOS or Home Manager. Nix historically provides rudimentary facilities for package management such as `nix-env`, but I claim they are widely agreed upon being discouraged, with plenty of arguments provided in . Similarly, the specific structure of packages is determined downstream, since Nix is policy-free: > Nix is policy-free; it provides mechanisms to implement various deployment policies, but does not enforce a specific one. > > -- The Purely Functional Software Deployment Model (Eelco Dolstra; 2006) Specifically, Nix mechanisms do not define what a package is supposed to be: > It's worth noting that the Nix language is intended as a DSL for package and configuration management, but it has no notions of "packages" or "configurations". > > -- This is why we say, Nix *allows* denoting packages in a certain way, but doesn't enforce any particular way. --- doc/manual/redirects.js | 1 + doc/manual/source/glossary.md | 17 ++++++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 17fb66f28..961243848 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -374,6 +374,7 @@ const redirects = { }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", + "package-attribute-set": "#package", "gloss-chroot-store": "store/types/local-store.html", "gloss-content-addressed-derivation": "#gloss-content-addressing-derivation", }, diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index 6a7501200..f9263c75b 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -215,7 +215,7 @@ > **Example** > - > Building and deploying software using Nix entails writing Nix expressions as a high-level description of packages and compositions thereof. + > Building and deploying software using Nix entails writing Nix expressions to describe [packages][package] and compositions thereof. - [reference]{#gloss-reference} @@ -330,18 +330,17 @@ - [package]{#package} - 1. A software package; a collection of files and other data. + A software package; files that belong together for a particular purpose, and metadata. - 2. A [package attribute set]. + Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. -- [package attribute set]{#package-attribute-set} + The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#attribute-set) containing: + - attributes that refer to the files of a package, typically in the form of [derivation outputs](#output), + - attributes with metadata, such as information about how the package is supposed to be used. - An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as - - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), - - attributes that declare something about how the package is supposed to be installed or used, - - other metadata or arbitrary attributes. + The exact shape of these attribute sets is up to convention. - [package attribute set]: #package-attribute-set + [package]: #package - [string interpolation]{#gloss-string-interpolation} From bf65bc7eb7dd19db04742455d62cffee04153051 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 10 Apr 2025 23:50:55 +0200 Subject: [PATCH 208/396] Nix language reference: highlight characteristic features Nix shipping with Yet Another Programming Language is often questioned among beginners. This change highlights distinctive aspects of the Nix language to ease the learning curve and better orient readers around what really matters for using Nix. Since it's on topic, this change also polishes the wording on the motivation for string contexts. --- doc/manual/source/language/index.md | 9 ++++++++- doc/manual/source/language/string-context.md | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/doc/manual/source/language/index.md b/doc/manual/source/language/index.md index 2bfdbb8a0..5bb939e18 100644 --- a/doc/manual/source/language/index.md +++ b/doc/manual/source/language/index.md @@ -11,7 +11,14 @@ The language is: - *domain-specific* - It comes with [built-in functions](@docroot@/language/builtins.md) to integrate with the Nix store, which manages files and performs the derivations declared in the Nix language. + The Nix language is purpose-built for working with text files. + Its most characteristic features are: + + - [File system path primitives](@docroot@/language/types.md#type-path), for accessing source files + - [Indented strings](@docroot@/language/string-literals.md) and [string interpolation](@docroot@/language/string-interpolation.md), for creating file contents + - [Strings with contexts](@docroot@/language/string-context.md), for transparently linking files + + It comes with [built-in functions](@docroot@/language/builtins.md) to integrate with the [Nix store](@docroot@/store/index.md), which manages files and enables [realising](@docroot@/glossary.md#gloss-realise) derivations declared in the Nix language. - *declarative* diff --git a/doc/manual/source/language/string-context.md b/doc/manual/source/language/string-context.md index 6a3482cfd..46b418d99 100644 --- a/doc/manual/source/language/string-context.md +++ b/doc/manual/source/language/string-context.md @@ -13,8 +13,8 @@ The purpose of string contexts is to collect non-string values attached to strin [string concatenation](./operators.md#string-concatenation), [string interpolation](./string-interpolation.md), and similar operations. -The idea is that a user can combine together values to create a build instructions for derivations without manually keeping track of where they come from. -Then the Nix language implicitly does that bookkeeping to efficiently obtain the closure of derivation inputs. +The idea is that a user can reference other files when creating text files through Nix expressions, without manually keeping track of the exact paths. +Nix will ensure that the all referenced files are accessible – that all [store paths](@docroot@/glossary.md#gloss-store-path) are [valid](@docroot@/glossary.md#gloss-validitiy). > **Note** > From 9da01e69f96346d73c2d1c03adce109f3e57a9a4 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Wed, 19 Feb 2025 18:51:02 +0100 Subject: [PATCH 209/396] libstore S3: fix progress bar and make file transfers interruptible --- src/libstore/filetransfer.cc | 4 - src/libstore/s3-binary-cache-store.cc | 115 ++++++++++++++++++++++---- 2 files changed, 101 insertions(+), 18 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 475637d74..d00330601 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -789,10 +789,6 @@ struct curlFileTransfer : public FileTransfer S3Helper s3Helper(profile, region, scheme, endpoint); - Activity act(*logger, lvlTalkative, actFileTransfer, - fmt("downloading '%s'", request.uri), - {request.uri}, request.parentAct); - // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 87f5feb45..ca03c7cd8 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -160,7 +160,10 @@ ref S3Helper::makeConfig( S3Helper::FileTransferResult S3Helper::getObject( const std::string & bucketName, const std::string & key) { - debug("fetching 's3://%s/%s'...", bucketName, key); + std::string uri = "s3://" + bucketName + "/" + key; + Activity act(*logger, lvlTalkative, actFileTransfer, + fmt("downloading '%s'", uri), + Logger::Fields{uri}, getCurActivity()); auto request = Aws::S3::Model::GetObjectRequest() @@ -171,6 +174,26 @@ S3Helper::FileTransferResult S3Helper::getObject( return Aws::New("STRINGSTREAM"); }); + size_t bytesDone = 0; + size_t bytesExpected = 0; + request.SetDataReceivedEventHandler([&](const Aws::Http::HttpRequest * req, Aws::Http::HttpResponse * resp, long long l) { + if (!bytesExpected && resp->HasHeader("Content-Length")) { + if (auto length = string2Int(resp->GetHeader("Content-Length"))) { + bytesExpected = *length; + } + } + bytesDone += l; + act.progress(bytesDone, bytesExpected); + }); + + request.SetContinueRequestHandler([](const Aws::Http::HttpRequest*) { + try { + checkInterrupt(); + return true; + } catch(...) {} + return false; + }); + FileTransferResult res; auto now1 = std::chrono::steady_clock::now(); @@ -180,6 +203,8 @@ S3Helper::FileTransferResult S3Helper::getObject( auto result = checkAws(fmt("AWS error fetching '%s'", key), client->GetObject(request)); + act.progress(result.GetContentLength(), result.GetContentLength()); + res.data = decompress(result.GetContentEncoding(), dynamic_cast(result.GetBody()).str()); @@ -307,11 +332,35 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual std::shared_ptr transferManager; std::once_flag transferManagerCreated; + struct AsyncContext : public Aws::Client::AsyncCallerContext + { + mutable std::mutex mutex; + mutable std::condition_variable cv; + const Activity & act; + + void notify() const + { + cv.notify_one(); + } + + void wait() const + { + std::unique_lock lk(mutex); + cv.wait(lk); + } + + AsyncContext(const Activity & act) : act(act) {} + }; + void uploadFile(const std::string & path, std::shared_ptr> istream, const std::string & mimeType, const std::string & contentEncoding) { + std::string uri = "s3://" + bucketName + "/" + path; + Activity act(*logger, lvlTalkative, actFileTransfer, + fmt("uploading '%s'", uri), + Logger::Fields{uri}, getCurActivity()); istream->seekg(0, istream->end); auto size = istream->tellg(); istream->seekg(0, istream->beg); @@ -330,16 +379,25 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual transferConfig.bufferSize = bufferSize; transferConfig.uploadProgressCallback = - [](const TransferManager *transferManager, - const std::shared_ptr - &transferHandle) + [](const TransferManager * transferManager, + const std::shared_ptr & transferHandle) { - //FIXME: find a way to properly abort the multipart upload. - //checkInterrupt(); - debug("upload progress ('%s'): '%d' of '%d' bytes", - transferHandle->GetKey(), - transferHandle->GetBytesTransferred(), - transferHandle->GetBytesTotalSize()); + auto context = std::dynamic_pointer_cast(transferHandle->GetContext()); + size_t bytesDone = transferHandle->GetBytesTransferred(); + size_t bytesTotal = transferHandle->GetBytesTotalSize(); + try { + checkInterrupt(); + context->act.progress(bytesDone, bytesTotal); + } catch (...) { + context->notify(); + } + }; + transferConfig.transferStatusUpdatedCallback = + [](const TransferManager * transferManager, + const std::shared_ptr & transferHandle) + { + auto context = std::dynamic_pointer_cast(transferHandle->GetContext()); + context->notify(); }; transferManager = TransferManager::Create(transferConfig); @@ -353,29 +411,56 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual if (contentEncoding != "") throw Error("setting a content encoding is not supported with S3 multi-part uploads"); + auto context = std::make_shared(act); std::shared_ptr transferHandle = transferManager->UploadFile( istream, bucketName, path, mimeType, Aws::Map(), - nullptr /*, contentEncoding */); + context /*, contentEncoding */); - transferHandle->WaitUntilFinished(); + TransferStatus status = transferHandle->GetStatus(); + while (status == TransferStatus::IN_PROGRESS || status == TransferStatus::NOT_STARTED) { + try { + checkInterrupt(); + context->wait(); + } catch (...) { + transferHandle->Cancel(); + transferHandle->WaitUntilFinished(); + } + status = transferHandle->GetStatus(); + } + act.progress(transferHandle->GetBytesTransferred(), transferHandle->GetBytesTotalSize()); - if (transferHandle->GetStatus() == TransferStatus::FAILED) + if (status == TransferStatus::FAILED) throw Error("AWS error: failed to upload 's3://%s/%s': %s", bucketName, path, transferHandle->GetLastError().GetMessage()); - if (transferHandle->GetStatus() != TransferStatus::COMPLETED) + if (status != TransferStatus::COMPLETED) throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", bucketName, path); } else { + act.progress(0, size); auto request = Aws::S3::Model::PutObjectRequest() .WithBucket(bucketName) .WithKey(path); + size_t bytesSent = 0; + request.SetDataSentEventHandler([&](const Aws::Http::HttpRequest * req, long long l) { + bytesSent += l; + act.progress(bytesSent, size); + }); + + request.SetContinueRequestHandler([](const Aws::Http::HttpRequest*) { + try { + checkInterrupt(); + return true; + } catch(...) {} + return false; + }); + request.SetContentType(mimeType); if (contentEncoding != "") @@ -385,6 +470,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto result = checkAws(fmt("AWS error uploading '%s'", path), s3Helper.client->PutObject(request)); + + act.progress(size, size); } auto now2 = std::chrono::steady_clock::now(); From db297d3dda12306459341da01e9892b4df2d6d37 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Wed, 12 Mar 2025 00:50:20 +0100 Subject: [PATCH 210/396] libstore: same progress bar behavior for PUT and POST requests - no differentiation between uploads and downloads in CLI --- src/libstore/filetransfer.cc | 24 +++++-------------- .../include/nix/store/filetransfer.hh | 2 +- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index d00330601..a917188d9 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -93,7 +93,7 @@ struct curlFileTransfer : public FileTransfer : fileTransfer(fileTransfer) , request(request) , act(*logger, lvlTalkative, actFileTransfer, - request.post ? "" : fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), + fmt("%sing '%s'", request.verb(), request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) , finalSink([this](std::string_view data) { @@ -270,19 +270,11 @@ struct curlFileTransfer : public FileTransfer return getInterrupted(); } - int silentProgressCallback(curl_off_t dltotal, curl_off_t dlnow) - { - return getInterrupted(); - } - static int progressCallbackWrapper(void * userp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { - return ((TransferItem *) userp)->progressCallback(dltotal, dlnow); - } - - static int silentProgressCallbackWrapper(void * userp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) - { - return ((TransferItem *) userp)->silentProgressCallback(dltotal, dlnow); + auto & item = *static_cast(userp); + auto isUpload = bool(item.request.data); + return item.progressCallback(isUpload ? ultotal : dltotal, isUpload ? ulnow : dlnow); } static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) @@ -349,10 +341,7 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper); curl_easy_setopt(req, CURLOPT_HEADERDATA, this); - if (request.post) - curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, silentProgressCallbackWrapper); - else - curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, progressCallbackWrapper); + curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, progressCallbackWrapper); curl_easy_setopt(req, CURLOPT_XFERINFODATA, this); curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0); @@ -445,8 +434,7 @@ struct curlFileTransfer : public FileTransfer if (httpStatus == 304 && result.etag == "") result.etag = request.expectedETag; - if (!request.post) - act.progress(result.bodySize, result.bodySize); + act.progress(result.bodySize, result.bodySize); done = true; callback(std::move(result)); } diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index b0fe8fcce..f9b1f620f 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -77,7 +77,7 @@ struct FileTransferRequest FileTransferRequest(std::string_view uri) : uri(uri), parentAct(getCurActivity()) { } - std::string verb() + std::string verb() const { return data ? "upload" : "download"; } From 49f757c24ae10e6d32c19e27fd646fc21aca7679 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Fri, 11 Apr 2025 22:34:15 +0200 Subject: [PATCH 211/396] add isInterrupted() call and replace some checkInterrupt() occurrences --- src/libstore/s3-binary-cache-store.cc | 17 ++++------------- src/libutil/include/nix/util/signals.hh | 5 +++++ .../unix/include/nix/util/signals-impl.hh | 13 +++++++++---- .../windows/include/nix/util/signals-impl.hh | 7 ++++++- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ca03c7cd8..f9e583307 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -187,11 +187,7 @@ S3Helper::FileTransferResult S3Helper::getObject( }); request.SetContinueRequestHandler([](const Aws::Http::HttpRequest*) { - try { - checkInterrupt(); - return true; - } catch(...) {} - return false; + return !isInterrupted(); }); FileTransferResult res; @@ -420,10 +416,9 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual TransferStatus status = transferHandle->GetStatus(); while (status == TransferStatus::IN_PROGRESS || status == TransferStatus::NOT_STARTED) { - try { - checkInterrupt(); + if (!isInterrupted()) { context->wait(); - } catch (...) { + } else { transferHandle->Cancel(); transferHandle->WaitUntilFinished(); } @@ -454,11 +449,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual }); request.SetContinueRequestHandler([](const Aws::Http::HttpRequest*) { - try { - checkInterrupt(); - return true; - } catch(...) {} - return false; + return !isInterrupted(); }); request.SetContentType(mimeType); diff --git a/src/libutil/include/nix/util/signals.hh b/src/libutil/include/nix/util/signals.hh index 45130a90c..5a2ba8e75 100644 --- a/src/libutil/include/nix/util/signals.hh +++ b/src/libutil/include/nix/util/signals.hh @@ -26,6 +26,11 @@ static inline bool getInterrupted(); */ void setInterruptThrown(); +/** + * @note Does nothing on Windows + */ +static inline bool isInterrupted(); + /** * @note Does nothing on Windows */ diff --git a/src/libutil/unix/include/nix/util/signals-impl.hh b/src/libutil/unix/include/nix/util/signals-impl.hh index ffa967344..7397744b2 100644 --- a/src/libutil/unix/include/nix/util/signals-impl.hh +++ b/src/libutil/unix/include/nix/util/signals-impl.hh @@ -85,17 +85,22 @@ static inline bool getInterrupted() return unix::_isInterrupted; } +static inline bool isInterrupted() +{ + using namespace unix; + return _isInterrupted || (interruptCheck && interruptCheck()); +} + /** * Throw `Interrupted` exception if the process has been interrupted. * * Call this in long-running loops and between slow operations to terminate * them as needed. */ -void inline checkInterrupt() +inline void checkInterrupt() { - using namespace unix; - if (_isInterrupted || (interruptCheck && interruptCheck())) - _interrupted(); + if (isInterrupted()) + unix::_interrupted(); } /** diff --git a/src/libutil/windows/include/nix/util/signals-impl.hh b/src/libutil/windows/include/nix/util/signals-impl.hh index 043f39100..f716ffd1a 100644 --- a/src/libutil/windows/include/nix/util/signals-impl.hh +++ b/src/libutil/windows/include/nix/util/signals-impl.hh @@ -22,7 +22,12 @@ inline void setInterruptThrown() /* Do nothing for now */ } -void inline checkInterrupt() +static inline bool isInterrupted() +{ + /* Do nothing for now */ +} + +inline void checkInterrupt() { /* Do nothing for now */ } From 51073607eaef1fabd486609dbad29683d15e7b0f Mon Sep 17 00:00:00 2001 From: Dean De Leo Date: Thu, 13 Mar 2025 20:02:09 +0700 Subject: [PATCH 212/396] S3: opt-in the STSProfileCredentialsProvider The STSProfileCredentialsProviders allows to assume a specific IAM role when accessing an S3 repository. Sometimes this is needed to obtain the permissions to operate on the bucket. --- packaging/dependencies.nix | 1 + src/libstore/meson.build | 3 +++ src/libstore/s3-binary-cache-store.cc | 31 +++++++++++++++++++++------ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index ed05843c7..16d1f1376 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -38,6 +38,7 @@ scope: { aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { apis = [ + "identity-management" "s3" "transfer" ]; diff --git a/src/libstore/meson.build b/src/libstore/meson.build index ae50686b6..255f83f74 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -135,6 +135,9 @@ if aws_s3.found() '-L' + aws_s3.get_variable('libdir'), '-laws-cpp-sdk-transfer', '-laws-cpp-sdk-s3', + '-laws-cpp-sdk-identity-management', + '-laws-cpp-sdk-cognito-identity', + '-laws-cpp-sdk-sts', '-laws-cpp-sdk-core', '-laws-crt-cpp', ], diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f9e583307..26c21de44 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +72,29 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem #endif }; +/* Retrieve the credentials from the list of AWS default providers, with the addition of the STS creds provider. This + last can be used to acquire further permissions with a specific IAM role. + Roughly based on https://github.com/aws/aws-sdk-cpp/issues/150#issuecomment-538548438 +*/ +struct CustomAwsCredentialsProviderChain : public Aws::Auth::AWSCredentialsProviderChain +{ + CustomAwsCredentialsProviderChain(const std::string & profile) + { + if (profile.empty()) { + // Use all the default AWS providers, plus the possibility to acquire a IAM role directly via a profile. + Aws::Auth::DefaultAWSCredentialsProviderChain default_aws_chain; + for (auto provider : default_aws_chain.GetProviders()) + AddProvider(provider); + AddProvider(std::make_shared()); + } else { + // Override the profile name to retrieve from the AWS config and credentials. I believe this option + // comes from the ?profile querystring in nix.conf. + AddProvider(std::make_shared(profile.c_str())); + AddProvider(std::make_shared(profile)); + } + } +}; + static void initAWS() { static std::once_flag flag; @@ -102,13 +126,8 @@ S3Helper::S3Helper( const std::string & endpoint) : config(makeConfig(region, scheme, endpoint)) , client(make_ref( - profile == "" - ? std::dynamic_pointer_cast( - std::make_shared()) - : std::dynamic_pointer_cast( - std::make_shared(profile.c_str())), + std::make_shared(profile), *config, - // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 #if AWS_SDK_VERSION_MAJOR == 1 && AWS_SDK_VERSION_MINOR < 3 false, #else From b1b75e1d7a8bb8a1ec978e2a6ef340a85515953e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 12 Apr 2025 10:20:30 +0200 Subject: [PATCH 213/396] tests/functional: add test for alias commands --- tests/functional/store-info.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh index beecc2dd9..b1e0772b5 100755 --- a/tests/functional/store-info.sh +++ b/tests/functional/store-info.sh @@ -3,9 +3,11 @@ source common.sh STORE_INFO=$(nix store info 2>&1) +LEGACY_STORE_INFO=$(nix store ping 2>&1) # alias to nix store info STORE_INFO_JSON=$(nix store info --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" +echo "$LEGACY_STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then DAEMON_VERSION=$("$NIX_DAEMON_PACKAGE"/bin/nix daemon --version | cut -d' ' -f3) @@ -13,6 +15,7 @@ if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi + expect 127 NIX_REMOTE=unix:"$PWD"/store nix store info || \ fail "nix store info on a non-existent store should fail" From f64b8957c7fcedb5d819c6912a5236a1b5fe8433 Mon Sep 17 00:00:00 2001 From: Anthony Wang Date: Sat, 12 Apr 2025 19:17:27 -0400 Subject: [PATCH 214/396] Fix typo in string context docs --- doc/manual/source/language/string-context.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/language/string-context.md b/doc/manual/source/language/string-context.md index 46b418d99..081ab469d 100644 --- a/doc/manual/source/language/string-context.md +++ b/doc/manual/source/language/string-context.md @@ -115,7 +115,7 @@ It creates an [attribute set] representing the string context, which can be insp ## Clearing string contexts -[`buitins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. +[`builtins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. The returned string can be used in more ways, e.g. by operators that require the string context to be empty. The requirement to explicitly discard the string context in such use cases helps ensure that string context elements are not lost by mistake. The "unsafe" marker is only there to remind that Nix normally guarantees that dependencies are tracked, whereas the returned string has lost them. From f6df573a91f652070c79d4e101838418df949676 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 12 Apr 2025 19:19:58 -0400 Subject: [PATCH 215/396] flake.lock: Update Nixpkgs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes evaluation for Windows. There are unfortunately deps that still don't build, but this can be fixed next. Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/52faf482a3889b7619003c0daec593a1912fddc1?narHash=sha256-6hl6L/tRnwubHcA4pfUUtk542wn2Om%2BD4UnDhlDW9BE%3D' (2025-03-30) → 'github:NixOS/nixpkgs/f675531bc7e6657c10a18b565cfebd8aa9e24c14?narHash=sha256-gbl9hE39nQRpZaLjhWKmEu5ejtQsgI5TWYrIVVJn30U%3D' (2025-04-09) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 7e008fadc..5154c1685 100644 --- a/flake.lock +++ b/flake.lock @@ -63,11 +63,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1743315132, - "narHash": "sha256-6hl6L/tRnwubHcA4pfUUtk542wn2Om+D4UnDhlDW9BE=", + "lastModified": 1744232761, + "narHash": "sha256-gbl9hE39nQRpZaLjhWKmEu5ejtQsgI5TWYrIVVJn30U=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "52faf482a3889b7619003c0daec593a1912fddc1", + "rev": "f675531bc7e6657c10a18b565cfebd8aa9e24c14", "type": "github" }, "original": { From 56d37656ac35a7c6be772562796fd4614d4e6a3a Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sun, 13 Apr 2025 04:36:09 +0200 Subject: [PATCH 216/396] libexpr: fix UB in builtins.ceil and builtins.floor tighten and fix specification of both builtins --- src/libexpr-tests/primops.cc | 20 +++++++++++ src/libexpr/primops.cc | 68 +++++++++++++++++++++++++++++------- 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/src/libexpr-tests/primops.cc b/src/libexpr-tests/primops.cc index c3e50863d..7695a587a 100644 --- a/src/libexpr-tests/primops.cc +++ b/src/libexpr-tests/primops.cc @@ -56,11 +56,31 @@ namespace nix { TEST_F(PrimOpTest, ceil) { auto v = eval("builtins.ceil 1.9"); ASSERT_THAT(v, IsIntEq(2)); + auto intMin = eval("builtins.ceil (-4611686018427387904 - 4611686018427387904)"); + ASSERT_THAT(intMin, IsIntEq(std::numeric_limits::min())); + ASSERT_THROW(eval("builtins.ceil 1.0e200"), EvalError); + ASSERT_THROW(eval("builtins.ceil -1.0e200"), EvalError); + ASSERT_THROW(eval("builtins.ceil (1.0e200 * 1.0e200)"), EvalError); // inf + ASSERT_THROW(eval("builtins.ceil (-1.0e200 * 1.0e200)"), EvalError); // -inf + ASSERT_THROW(eval("builtins.ceil (1.0e200 * 1.0e200 - 1.0e200 * 1.0e200)"), EvalError); // nan + // bugs in previous Nix versions + ASSERT_THROW(eval("builtins.ceil (4611686018427387904 + 4611686018427387903)"), EvalError); + ASSERT_THROW(eval("builtins.ceil (-4611686018427387904 - 4611686018427387903)"), EvalError); } TEST_F(PrimOpTest, floor) { auto v = eval("builtins.floor 1.9"); ASSERT_THAT(v, IsIntEq(1)); + auto intMin = eval("builtins.ceil (-4611686018427387904 - 4611686018427387904)"); + ASSERT_THAT(intMin, IsIntEq(std::numeric_limits::min())); + ASSERT_THROW(eval("builtins.ceil 1.0e200"), EvalError); + ASSERT_THROW(eval("builtins.ceil -1.0e200"), EvalError); + ASSERT_THROW(eval("builtins.ceil (1.0e200 * 1.0e200)"), EvalError); // inf + ASSERT_THROW(eval("builtins.ceil (-1.0e200 * 1.0e200)"), EvalError); // -inf + ASSERT_THROW(eval("builtins.ceil (1.0e200 * 1.0e200 - 1.0e200 * 1.0e200)"), EvalError); // nan + // bugs in previous Nix versions + ASSERT_THROW(eval("builtins.ceil (4611686018427387904 + 4611686018427387903)"), EvalError); + ASSERT_THROW(eval("builtins.ceil (-4611686018427387904 - 4611686018427387903)"), EvalError); } TEST_F(PrimOpTest, tryEvalFailure) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5e331f84d..5b24849d2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -888,18 +888,40 @@ static void prim_ceil(EvalState & state, const PosIdx pos, Value * * args, Value { auto value = state.forceFloat(*args[0], args[0]->determinePos(pos), "while evaluating the first argument passed to builtins.ceil"); - v.mkInt(ceil(value)); + auto ceilValue = ceil(value); + bool isInt = args[0]->type() == nInt; + constexpr NixFloat int_min = std::numeric_limits::min(); // power of 2, so that no rounding occurs + if (ceilValue >= int_min && ceilValue < -int_min) { + v.mkInt(ceilValue); + } else if (isInt) { + // a NixInt, e.g. INT64_MAX, can be rounded to -int_min due to the cast to NixFloat + state.error("Due to a bug (see https://github.com/NixOS/nix/issues/12899) the NixInt argument %1% caused undefined behavior in previous Nix versions.\n\tFuture Nix versions might implement the correct behavior.", args[0]->integer().value).atPos(pos).debugThrow(); + } else { + state.error("NixFloat argument %1% is not in the range of NixInt", args[0]->fpoint()).atPos(pos).debugThrow(); + } + // `forceFloat` casts NixInt to NixFloat, but instead NixInt args shall be returned unmodified + if (isInt) { + auto arg = args[0]->integer(); + auto res = v.integer(); + if (arg != res) { + state.error("Due to a bug (see https://github.com/NixOS/nix/issues/12899) a loss of precision occured in previous Nix versions because the NixInt argument %1% was rounded to %2%.\n\tFuture Nix versions might implement the correct behavior.", arg, res).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_ceil({ .name = "__ceil", - .args = {"double"}, + .args = {"number"}, .doc = R"( - Converts an IEEE-754 double-precision floating-point number (*double*) to - the next higher integer. + Rounds and converts *number* to the next higher NixInt value if possible, i.e. `ceil *number* >= *number*` and + `ceil *number* - *number* < 1`. - If the datatype is neither an integer nor a "float", an evaluation error will be - thrown. + An evaluation error is thrown, if there exists no such NixInt value `ceil *number*`. + Due to bugs in previous Nix versions an evaluation error might be thrown, if the datatype of *number* is + a NixInt and if `*number* < -9007199254740992` or `*number* > 9007199254740992`. + + If the datatype of *number* is neither a NixInt (signed 64-bit integer) nor a NixFloat + (IEEE-754 double-precision floating-point number), an evaluation error will be thrown. )", .fun = prim_ceil, }); @@ -907,18 +929,40 @@ static RegisterPrimOp primop_ceil({ static void prim_floor(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto value = state.forceFloat(*args[0], args[0]->determinePos(pos), "while evaluating the first argument passed to builtins.floor"); - v.mkInt(floor(value)); + auto floorValue = floor(value); + bool isInt = args[0]->type() == nInt; + constexpr NixFloat int_min = std::numeric_limits::min(); // power of 2, so that no rounding occurs + if (floorValue >= int_min && floorValue < -int_min) { + v.mkInt(floorValue); + } else if (isInt) { + // a NixInt, e.g. INT64_MAX, can be rounded to -int_min due to the cast to NixFloat + state.error("Due to a bug (see https://github.com/NixOS/nix/issues/12899) the NixInt argument %1% caused undefined behavior in previous Nix versions.\n\tFuture Nix versions might implement the correct behavior.", args[0]->integer().value).atPos(pos).debugThrow(); + } else { + state.error("NixFloat argument %1% is not in the range of NixInt", args[0]->fpoint()).atPos(pos).debugThrow(); + } + // `forceFloat` casts NixInt to NixFloat, but instead NixInt args shall be returned unmodified + if (isInt) { + auto arg = args[0]->integer(); + auto res = v.integer(); + if (arg != res) { + state.error("Due to a bug (see https://github.com/NixOS/nix/issues/12899) a loss of precision occured in previous Nix versions because the NixInt argument %1% was rounded to %2%.\n\tFuture Nix versions might implement the correct behavior.", arg, res).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_floor({ .name = "__floor", - .args = {"double"}, + .args = {"number"}, .doc = R"( - Converts an IEEE-754 double-precision floating-point number (*double*) to - the next lower integer. + Rounds and converts *number* to the next lower NixInt value if possible, i.e. `floor *number* <= *number*` and + `*number* - floor *number* < 1`. - If the datatype is neither an integer nor a "float", an evaluation error will be - thrown. + An evaluation error is thrown, if there exists no such NixInt value `floor *number*`. + Due to bugs in previous Nix versions an evaluation error might be thrown, if the datatype of *number* is + a NixInt and if `*number* < -9007199254740992` or `*number* > 9007199254740992`. + + If the datatype of *number* is neither a NixInt (signed 64-bit integer) nor a NixFloat + (IEEE-754 double-precision floating-point number), an evaluation error will be thrown. )", .fun = prim_floor, }); From 0123640009c1340db7a36e2da1301ce7a4fc530d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 3 Feb 2025 10:28:39 -0500 Subject: [PATCH 217/396] `ParsedDerivation`: don't take `drvPath` It is just use for adding context to errors, but we have `addTrace` to do that. Let the callers do that instead. The callers doing so is a bit duplicated, yes, but this will get better once `DerivationOptions` is included in `Derivation`. --- src/libstore-tests/derivation-advanced-attrs.cc | 8 ++++---- src/libstore/build/derivation-goal.cc | 9 +++++++-- .../include/nix/store/parsed-derivations.hh | 3 +-- src/libstore/misc.cc | 10 ++++++++-- src/libstore/parsed-derivations.cc | 14 +++++++------- src/nix-build/nix-build.cc | 10 ++++++++-- 6 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 9a901ebff..b8cfa2498 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -81,7 +81,7 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults) auto drvPath = writeDerivation(*store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); + ParsedDerivation parsedDrv(got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_TRUE(!parsedDrv.hasStructuredAttrs()); @@ -116,7 +116,7 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes) auto drvPath = writeDerivation(*store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); + ParsedDerivation parsedDrv(got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); StringSet systemFeatures{"rainbow", "uid-range"}; @@ -157,7 +157,7 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr auto drvPath = writeDerivation(*store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); + ParsedDerivation parsedDrv(got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_TRUE(parsedDrv.hasStructuredAttrs()); @@ -191,7 +191,7 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr auto drvPath = writeDerivation(*store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); + ParsedDerivation parsedDrv(got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); StringSet systemFeatures{"rainbow", "uid-range"}; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 76456dac5..8e43ab2ae 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -180,8 +180,13 @@ Goal::Co DerivationGoal::haveDerivation() { trace("have derivation"); - parsedDrv = std::make_unique(drvPath, *drv); - drvOptions = std::make_unique(DerivationOptions::fromParsedDerivation(*parsedDrv)); + parsedDrv = std::make_unique(*drv); + try { + drvOptions = std::make_unique(DerivationOptions::fromParsedDerivation(*parsedDrv)); + } catch (Error & e) { + e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath)); + throw; + } if (!drv->type().hasKnownOutputPaths()) experimentalFeatureSettings.require(Xp::CaDerivations); diff --git a/src/libstore/include/nix/store/parsed-derivations.hh b/src/libstore/include/nix/store/parsed-derivations.hh index d65db6133..2b1ab97d5 100644 --- a/src/libstore/include/nix/store/parsed-derivations.hh +++ b/src/libstore/include/nix/store/parsed-derivations.hh @@ -12,7 +12,6 @@ struct DerivationOptions; class ParsedDerivation { - StorePath drvPath; BasicDerivation & drv; std::unique_ptr structuredAttrs; @@ -34,7 +33,7 @@ class ParsedDerivation public: - ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv); + ParsedDerivation(BasicDerivation & drv); ~ParsedDerivation(); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 0e2b62db5..b9e729092 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -222,8 +222,14 @@ void Store::queryMissing(const std::vector & targets, if (knownOutputPaths && invalid.empty()) return; auto drv = make_ref(derivationFromPath(drvPath)); - ParsedDerivation parsedDrv(StorePath(drvPath), *drv); - DerivationOptions drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); + ParsedDerivation parsedDrv(*drv); + DerivationOptions drvOptions; + try { + drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); + } catch (Error & e) { + e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath)); + throw; + } if (!knownOutputPaths && settings.useSubstitutes && drvOptions.substitutesAllowed()) { experimentalFeatureSettings.require(Xp::CaDerivations); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index cc7203c6b..66bf76cac 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -5,8 +5,8 @@ namespace nix { -ParsedDerivation::ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv) - : drvPath(drvPath), drv(drv) +ParsedDerivation::ParsedDerivation(BasicDerivation & drv) + : drv(drv) { /* Parse the __json attribute, if any. */ auto jsonAttr = drv.env.find("__json"); @@ -14,7 +14,7 @@ ParsedDerivation::ParsedDerivation(const StorePath & drvPath, BasicDerivation & try { structuredAttrs = std::make_unique(nlohmann::json::parse(jsonAttr->second)); } catch (std::exception & e) { - throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what()); + throw Error("cannot process __json attribute: %s", e.what()); } } } @@ -29,7 +29,7 @@ std::optional ParsedDerivation::getStringAttr(const std::string & n return {}; else { if (!i->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath.to_string()); + throw Error("attribute '%s' of must be a string", name); return i->get(); } } else { @@ -49,7 +49,7 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const return def; else { if (!i->is_boolean()) - throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath.to_string()); + throw Error("attribute '%s' must be a Boolean", name); return i->get(); } } else { @@ -69,11 +69,11 @@ std::optional ParsedDerivation::getStringsAttr(const std::string & name return {}; else { if (!i->is_array()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string()); + throw Error("attribute '%s' must be a list of strings", name); Strings res; for (auto j = i->begin(); j != i->end(); ++j) { if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string()); + throw Error("attribute '%s' must be a list of strings", name); res.push_back(j->get()); } return res; diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 45f891808..d90630438 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -544,8 +544,14 @@ static void main_nix_build(int argc, char * * argv) env["NIX_STORE"] = store->storeDir; env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); - ParsedDerivation parsedDrv(packageInfo.requireDrvPath(), drv); - DerivationOptions drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); + ParsedDerivation parsedDrv(drv); + DerivationOptions drvOptions; + try { + drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); + } catch (Error & e) { + e.addTrace({}, "while parsing derivation '%s'", store->printStorePath(packageInfo.requireDrvPath())); + throw; + } int fileNr = 0; From 7ea536fe84d2fad2ce2f291910f51c159531273e Mon Sep 17 00:00:00 2001 From: Picnoir Date: Mon, 14 Apr 2025 10:30:47 +0200 Subject: [PATCH 218/396] Narinfo sign: multiple signatures variant This is a small optimization used when we're signing a narinfo for multiple keys in one go. Using this sign variant, we only compute the NAR fingerprint once, then sign it with all the keys. --- src/libstore/binary-cache-store.cc | 4 +--- src/libstore/include/nix/store/path-info.hh | 1 + src/libstore/path-info.cc | 8 ++++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 744bccef0..bdc281044 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -279,9 +279,7 @@ ref BinaryCacheStore::addToStoreCommon( stats.narWriteCompressedBytes += fileSize; stats.narWriteCompressionTimeMs += duration; - for (auto &signer: signers) { - narInfo->sign(*this, *signer); - } + narInfo->sign(*this, signers); /* Atomically write the NAR info file.*/ writeNarInfo(narInfo); diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index 9bd493422..4691bfa95 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -144,6 +144,7 @@ struct ValidPathInfo : UnkeyedValidPathInfo { std::string fingerprint(const Store & store) const; void sign(const Store & store, const Signer & signer); + void sign(const Store & store, const std::vector> & signers); /** * @return The `ContentAddressWithReferences` that determines the diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index df20edb3b..5400a9da1 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -40,6 +40,14 @@ void ValidPathInfo::sign(const Store & store, const Signer & signer) sigs.insert(signer.signDetached(fingerprint(store))); } +void ValidPathInfo::sign(const Store & store, const std::vector> & signers) +{ + auto fingerprint = this->fingerprint(store); + for (auto & signer: signers) { + sigs.insert(signer->signDetached(fingerprint)); + } +} + std::optional ValidPathInfo::contentAddressWithReferences() const { if (! ca) From 4966217b6a7f479ad13fd2bfdac2f73457b7159e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Apr 2025 14:29:14 +0200 Subject: [PATCH 219/396] Move the InputCache to EvalState --- src/libcmd/repl.cc | 3 --- src/libexpr/eval.cc | 3 +++ src/libexpr/include/nix/expr/eval.hh | 7 ++++++- src/libfetchers/include/nix/fetchers/input-cache.hh | 2 +- src/libfetchers/input-cache.cc | 5 ++--- src/libflake-c/nix_api_flake.cc | 3 +-- src/libflake/flake/flake.cc | 6 +++--- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 3805942ce..c5a95268b 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -28,7 +28,6 @@ #include "nix/expr/print.hh" #include "nix/util/ref.hh" #include "nix/expr/value.hh" -#include "nix/fetchers/input-cache.hh" #include "nix/util/strings.hh" @@ -459,7 +458,6 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":l" || command == ":load") { state->resetFileCache(); - fetchers::InputCache::getCache()->clear(); loadFile(arg); } @@ -469,7 +467,6 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":r" || command == ":reload") { state->resetFileCache(); - fetchers::InputCache::getCache()->clear(); reloadFiles(); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9d60676f5..10a33c042 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -19,6 +19,7 @@ #include "nix/util/url.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/tarball.hh" +#include "nix/fetchers/input-cache.hh" #include "parser-tab.hh" @@ -310,6 +311,7 @@ EvalState::EvalState( )} , store(store) , buildStore(buildStore ? buildStore : store) + , inputCache(fetchers::InputCache::create()) , debugRepl(nullptr) , debugStop(false) , trylevel(0) @@ -1152,6 +1154,7 @@ void EvalState::resetFileCache() { fileEvalCache.clear(); fileParseCache.clear(); + inputCache->clear(); } diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 61da225fc..6a6959bd8 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -33,7 +33,10 @@ namespace nix { constexpr size_t maxPrimOpArity = 8; class Store; -namespace fetchers { struct Settings; } +namespace fetchers { +struct Settings; +struct InputCache; +} struct EvalSettings; class EvalState; class StorePath; @@ -300,6 +303,8 @@ public: RootValue vImportedDrvToDerivation = nullptr; + ref inputCache; + /** * Debugger */ diff --git a/src/libfetchers/include/nix/fetchers/input-cache.hh b/src/libfetchers/include/nix/fetchers/input-cache.hh index 6a7194741..a7ca34487 100644 --- a/src/libfetchers/include/nix/fetchers/input-cache.hh +++ b/src/libfetchers/include/nix/fetchers/input-cache.hh @@ -25,7 +25,7 @@ struct InputCache virtual void clear() = 0; - static ref getCache(); + static ref create(); }; } diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 6772d67c7..716143899 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -72,10 +72,9 @@ struct InputCacheImpl : InputCache } }; -ref InputCache::getCache() +ref InputCache::create() { - static auto cache = make_ref(); - return cache; + return make_ref(); } } diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc index 0cc9a6a10..f100bf146 100644 --- a/src/libflake-c/nix_api_flake.cc +++ b/src/libflake-c/nix_api_flake.cc @@ -7,7 +7,6 @@ #include "nix_api_fetchers.h" #include "nix/flake/flake.hh" -#include "nix/fetchers/input-cache.hh" nix_flake_settings * nix_flake_settings_new(nix_c_context * context) { @@ -178,7 +177,7 @@ nix_locked_flake * nix_flake_lock( { nix_clear_err(context); try { - nix::fetchers::InputCache::getCache()->clear(); + eval_state->state.resetFileCache(); auto lockedFlake = nix::make_ref(nix::flake::lockFlake( *flakeSettings->settings, eval_state->state, *flakeReference->flakeRef, *flags->lockFlags)); return new nix_locked_flake{lockedFlake}; diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index cee39defc..afeefdaec 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -341,7 +341,7 @@ static Flake getFlake( const InputAttrPath & lockRootAttrPath) { // Fetch a lazy tree first. - auto cachedInput = fetchers::InputCache::getCache()->getAccessor(state.store, originalRef.input, useRegistries); + auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries); auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), originalRef.subdir); auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), originalRef.subdir); @@ -356,7 +356,7 @@ static Flake getFlake( debug("refetching input '%s' due to self attribute", newLockedRef); // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'. newLockedRef.input.attrs.erase("narHash"); - auto cachedInput2 = fetchers::InputCache::getCache()->getAccessor(state.store, newLockedRef.input, useRegistries); + auto cachedInput2 = state.inputCache->getAccessor(state.store, newLockedRef.input, useRegistries); cachedInput.accessor = cachedInput2.accessor; lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); } @@ -717,7 +717,7 @@ LockedFlake lockFlake( if (auto resolvedPath = resolveRelativePath()) { return {*resolvedPath, *input.ref}; } else { - auto cachedInput = fetchers::InputCache::getCache()->getAccessor(state.store, input.ref->input, useRegistries); + auto cachedInput = state.inputCache->getAccessor(state.store, input.ref->input, useRegistries); auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); From c0ed07755a409660ca0a4aad40cfe3d1a0ad2162 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Apr 2025 15:18:29 +0200 Subject: [PATCH 220/396] Mention BLAKE3 in the Nix 2.27 release notes --- doc/manual/source/release-notes/rl-2.27.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/manual/source/release-notes/rl-2.27.md b/doc/manual/source/release-notes/rl-2.27.md index b4918029a..3643f7476 100644 --- a/doc/manual/source/release-notes/rl-2.27.md +++ b/doc/manual/source/release-notes/rl-2.27.md @@ -38,6 +38,15 @@ Curl created sockets without setting `FD_CLOEXEC`/`SOCK_CLOEXEC`. This could previously cause connections to remain open forever when using commands like `nix shell`. This change sets the `FD_CLOEXEC` flag using a `CURLOPT_SOCKOPTFUNCTION` callback. +- Add BLAKE3 hash algorithm [#12379](https://github.com/NixOS/nix/pull/12379) + + Nix now supports the BLAKE3 hash algorithm as an experimental feature (`blake3-hashes`): + + ```console + # nix hash file ./file --type blake3 --extra-experimental-features blake3-hashes + blake3-34P4p+iZXcbbyB1i4uoF7eWCGcZHjmaRn6Y7QdynLwU= + ``` + # Contributors This release was made possible by the following 21 contributors: From 7acc229c8fd5c41c460a5b7aa28debf168cbce3d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 14 Apr 2025 11:18:33 -0400 Subject: [PATCH 221/396] Use the same variable for content addressing in functional tests `CONTENT_ADDRESSED` -> `NIX_TESTS_CA_BY_DEFAULT` --- tests/functional/build-remote-content-addressed-floating.sh | 2 +- tests/functional/build-remote.sh | 2 +- tests/functional/ca/nix-shell.sh | 2 +- tests/functional/nix-shell.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/build-remote-content-addressed-floating.sh b/tests/functional/build-remote-content-addressed-floating.sh index 33d667f92..370915905 100755 --- a/tests/functional/build-remote-content-addressed-floating.sh +++ b/tests/functional/build-remote-content-addressed-floating.sh @@ -6,6 +6,6 @@ file=build-hook-ca-floating.nix enableFeatures "ca-derivations" -CONTENT_ADDRESSED=true +NIX_TESTS_CA_BY_DEFAULT=true source build-remote.sh diff --git a/tests/functional/build-remote.sh b/tests/functional/build-remote.sh index 3231341cb..62cc85888 100644 --- a/tests/functional/build-remote.sh +++ b/tests/functional/build-remote.sh @@ -13,7 +13,7 @@ unset NIX_STATE_DIR function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; } EXTRA_SYSTEM_FEATURES=() -if [[ -n "${CONTENT_ADDRESSED-}" ]]; then +if [[ -n "${NIX_TESTS_CA_BY_DEFAULT-}" ]]; then EXTRA_SYSTEM_FEATURES=("ca-derivations") fi diff --git a/tests/functional/ca/nix-shell.sh b/tests/functional/ca/nix-shell.sh index d1fbe54d1..7b30b2ac8 100755 --- a/tests/functional/ca/nix-shell.sh +++ b/tests/functional/ca/nix-shell.sh @@ -2,6 +2,6 @@ source common.sh -CONTENT_ADDRESSED=true +NIX_TESTS_CA_BY_DEFAULT=true cd .. source ./nix-shell.sh diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index b054b7f75..bc49333b5 100755 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -4,7 +4,7 @@ source common.sh clearStoreIfPossible -if [[ -n ${CONTENT_ADDRESSED:-} ]]; then +if [[ -n ${NIX_TESTS_CA_BY_DEFAULT:-} ]]; then shellDotNix="$PWD/ca-shell.nix" else shellDotNix="$PWD/shell.nix" From 307dbe991415404b12992d6bd73bd293f0b743e1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 14 Apr 2025 11:15:56 -0400 Subject: [PATCH 222/396] Test derivation options with content-addressing too Now, both the unit and functional tests relating to derivation options are tested both ways -- with input addressing and content-addressing derivations. --- .../advanced-attributes-defaults.drv | 1 - ...d-attributes-structured-attrs-defaults.drv | 1 - .../advanced-attributes-structured-attrs.drv | 1 - .../data/derivation/advanced-attributes.drv | 1 - .../ca/advanced-attributes-defaults.drv | 1 + .../ca/advanced-attributes-defaults.json | 25 ++ ...d-attributes-structured-attrs-defaults.drv | 1 + ...-attributes-structured-attrs-defaults.json | 26 ++ .../advanced-attributes-structured-attrs.drv | 1 + .../advanced-attributes-structured-attrs.json | 44 +++ .../derivation/ca/advanced-attributes.drv | 1 + .../derivation/ca/advanced-attributes.json | 50 +++ .../ia/advanced-attributes-defaults.drv | 1 + .../advanced-attributes-defaults.json | 0 ...d-attributes-structured-attrs-defaults.drv | 1 + ...-attributes-structured-attrs-defaults.json | 0 .../advanced-attributes-structured-attrs.drv | 1 + .../advanced-attributes-structured-attrs.json | 0 .../derivation/ia/advanced-attributes.drv | 1 + .../derivation/ia/advanced-attributes.json | 47 +++ .../derivation-advanced-attrs.cc | 333 ++++++++++++++---- src/libstore/derivations.cc | 2 +- .../ca/derivation-advanced-attributes.sh | 6 + tests/functional/ca/meson.build | 3 +- .../derivation-advanced-attributes.sh | 12 +- .../advanced-attributes-defaults.nix | 22 +- ...d-attributes-structured-attrs-defaults.nix | 22 +- .../advanced-attributes-structured-attrs.nix | 23 +- .../derivation/advanced-attributes.nix | 23 +- .../ca/advanced-attributes-defaults.drv | 1 + ...d-attributes-structured-attrs-defaults.drv | 1 + .../advanced-attributes-structured-attrs.drv | 1 + .../derivation/ca/advanced-attributes.drv | 1 + .../{ => ia}/advanced-attributes-defaults.drv | 0 ...d-attributes-structured-attrs-defaults.drv | 0 .../advanced-attributes-structured-attrs.drv | 0 .../{ => ia}/advanced-attributes.drv | 0 37 files changed, 560 insertions(+), 94 deletions(-) delete mode 120000 src/libstore-tests/data/derivation/advanced-attributes-defaults.drv delete mode 120000 src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.drv delete mode 120000 src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.drv delete mode 120000 src/libstore-tests/data/derivation/advanced-attributes.drv create mode 120000 src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.drv create mode 100644 src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json create mode 120000 src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.drv create mode 100644 src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json create mode 120000 src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.drv create mode 100644 src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json create mode 120000 src/libstore-tests/data/derivation/ca/advanced-attributes.drv create mode 100644 src/libstore-tests/data/derivation/ca/advanced-attributes.json create mode 120000 src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.drv rename src/libstore-tests/data/derivation/{ => ia}/advanced-attributes-defaults.json (100%) create mode 120000 src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.drv rename src/libstore-tests/data/derivation/{ => ia}/advanced-attributes-structured-attrs-defaults.json (100%) create mode 120000 src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.drv rename src/libstore-tests/data/derivation/{ => ia}/advanced-attributes-structured-attrs.json (100%) create mode 120000 src/libstore-tests/data/derivation/ia/advanced-attributes.drv create mode 100644 src/libstore-tests/data/derivation/ia/advanced-attributes.json create mode 100755 tests/functional/ca/derivation-advanced-attributes.sh create mode 100644 tests/functional/derivation/ca/advanced-attributes-defaults.drv create mode 100644 tests/functional/derivation/ca/advanced-attributes-structured-attrs-defaults.drv create mode 100644 tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv create mode 100644 tests/functional/derivation/ca/advanced-attributes.drv rename tests/functional/derivation/{ => ia}/advanced-attributes-defaults.drv (100%) rename tests/functional/derivation/{ => ia}/advanced-attributes-structured-attrs-defaults.drv (100%) rename tests/functional/derivation/{ => ia}/advanced-attributes-structured-attrs.drv (100%) rename tests/functional/derivation/{ => ia}/advanced-attributes.drv (100%) diff --git a/src/libstore-tests/data/derivation/advanced-attributes-defaults.drv b/src/libstore-tests/data/derivation/advanced-attributes-defaults.drv deleted file mode 120000 index f8f30ac32..000000000 --- a/src/libstore-tests/data/derivation/advanced-attributes-defaults.drv +++ /dev/null @@ -1 +0,0 @@ -../../../../tests/functional/derivation/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.drv b/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.drv deleted file mode 120000 index 837e9a0e4..000000000 --- a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.drv +++ /dev/null @@ -1 +0,0 @@ -../../../../tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.drv b/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.drv deleted file mode 120000 index e08bb5737..000000000 --- a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.drv +++ /dev/null @@ -1 +0,0 @@ -../../../../tests/functional/derivation/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/advanced-attributes.drv b/src/libstore-tests/data/derivation/advanced-attributes.drv deleted file mode 120000 index 1dc394a0a..000000000 --- a/src/libstore-tests/data/derivation/advanced-attributes.drv +++ /dev/null @@ -1 +0,0 @@ -../../../../tests/functional/derivation/advanced-attributes.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.drv b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.drv new file mode 120000 index 000000000..a9b4f7fa7 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ca/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json new file mode 100644 index 000000000..bc67236b5 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json @@ -0,0 +1,25 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "builder": "/bin/bash", + "name": "advanced-attributes-defaults", + "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9", + "outputHashAlgo": "sha256", + "outputHashMode": "recursive", + "system": "my-system" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-defaults", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "my-system" +} diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.drv b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.drv new file mode 120000 index 000000000..61da0470a --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ca/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json new file mode 100644 index 000000000..7d3c932b2 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json @@ -0,0 +1,26 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}", + "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", + "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs-defaults", + "outputs": { + "dev": { + "hashAlgo": "sha256", + "method": "nar" + }, + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "my-system" +} diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.drv b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.drv new file mode 120000 index 000000000..c396ee853 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json new file mode 100644 index 000000000..584fd2113 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -0,0 +1,44 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"],\"disallowedRequisites\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"],\"allowedRequisites\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "bin": "/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m", + "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", + "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" + }, + "inputDrvs": { + "/nix/store/spfzlnkwb1v8s62yvh8vj1apd1kwjr5f-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + }, + "/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs", + "outputs": { + "bin": { + "hashAlgo": "sha256", + "method": "nar" + }, + "dev": { + "hashAlgo": "sha256", + "method": "nar" + }, + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "my-system" +} diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.drv b/src/libstore-tests/data/derivation/ca/advanced-attributes.drv new file mode 120000 index 000000000..acba9064d --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ca/advanced-attributes.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.json b/src/libstore-tests/data/derivation/ca/advanced-attributes.json new file mode 100644 index 000000000..69d40b135 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.json @@ -0,0 +1,50 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__darwinAllowLocalNetworking": "1", + "__impureHostDeps": "/usr/bin/ditto", + "__noChroot": "1", + "__sandboxProfile": "sandcastle", + "allowSubstitutes": "", + "allowedReferences": "/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8", + "allowedRequisites": "/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8", + "builder": "/bin/bash", + "disallowedReferences": "/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99", + "disallowedRequisites": "/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99", + "impureEnvVars": "UNICORN", + "name": "advanced-attributes", + "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9", + "outputHashAlgo": "sha256", + "outputHashMode": "recursive", + "preferLocalBuild": "1", + "requiredSystemFeatures": "rainbow uid-range", + "system": "my-system" + }, + "inputDrvs": { + "/nix/store/spfzlnkwb1v8s62yvh8vj1apd1kwjr5f-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + }, + "/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "inputSrcs": [], + "name": "advanced-attributes", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "my-system" +} diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.drv b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.drv new file mode 120000 index 000000000..7f1aa367e --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ia/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json similarity index 100% rename from src/libstore-tests/data/derivation/advanced-attributes-defaults.json rename to src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.drv b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.drv new file mode 120000 index 000000000..77aa67353 --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ia/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json similarity index 100% rename from src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.json rename to src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.drv b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.drv new file mode 120000 index 000000000..a4e25feba --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json similarity index 100% rename from src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.json rename to src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.drv b/src/libstore-tests/data/derivation/ia/advanced-attributes.drv new file mode 120000 index 000000000..ecc2f5f38 --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ia/advanced-attributes.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.json b/src/libstore-tests/data/derivation/ia/advanced-attributes.json new file mode 100644 index 000000000..d51524e20 --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.json @@ -0,0 +1,47 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__darwinAllowLocalNetworking": "1", + "__impureHostDeps": "/usr/bin/ditto", + "__noChroot": "1", + "__sandboxProfile": "sandcastle", + "allowSubstitutes": "", + "allowedReferences": "/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo", + "allowedRequisites": "/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo", + "builder": "/bin/bash", + "disallowedReferences": "/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar", + "disallowedRequisites": "/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar", + "impureEnvVars": "UNICORN", + "name": "advanced-attributes", + "out": "/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes", + "preferLocalBuild": "1", + "requiredSystemFeatures": "rainbow uid-range", + "system": "my-system" + }, + "inputDrvs": { + "/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + }, + "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "inputSrcs": [], + "name": "advanced-attributes", + "outputs": { + "out": { + "path": "/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes" + } + }, + "system": "my-system" +} diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index b8cfa2498..48f117f1c 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -18,68 +18,93 @@ using nlohmann::json; class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest { - std::filesystem::path unitTestData = getUnitTestData() / "derivation"; +protected: + std::filesystem::path unitTestData = getUnitTestData() / "derivation" / "ia"; public: std::filesystem::path goldenMaster(std::string_view testStem) const override { return unitTestData / testStem; } + + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; }; -#define TEST_ATERM_JSON(STEM, NAME) \ - TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_json) \ - { \ - readTest(NAME ".json", [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - /* Use DRV file instead of C++ literal as source of truth. */ \ - auto aterm = readFile(goldenMaster(NAME ".drv")); \ - auto expected = parseDerivation(*store, std::move(aterm), NAME); \ - Derivation got = Derivation::fromJSON(*store, encoded); \ - EXPECT_EQ(got, expected); \ - }); \ - } \ - \ - TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_to_json) \ - { \ - writeTest( \ - NAME ".json", \ - [&]() -> json { \ - /* Use DRV file instead of C++ literal as source of truth. */ \ - auto aterm = readFile(goldenMaster(NAME ".drv")); \ - return parseDerivation(*store, std::move(aterm), NAME).toJSON(*store); \ - }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ - } \ - \ - TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_aterm) \ - { \ - readTest(NAME ".drv", [&](auto encoded) { \ - /* Use JSON file instead of C++ literal as source of truth. */ \ - auto json = json::parse(readFile(goldenMaster(NAME ".json"))); \ - auto expected = Derivation::fromJSON(*store, json); \ - auto got = parseDerivation(*store, std::move(encoded), NAME); \ - EXPECT_EQ(got.toJSON(*store), expected.toJSON(*store)); \ - EXPECT_EQ(got, expected); \ - }); \ - } \ - \ +class CaDerivationAdvancedAttrsTest : public DerivationAdvancedAttrsTest +{ + void SetUp() override + { + unitTestData = getUnitTestData() / "derivation" / "ca"; + mockXpSettings.set("experimental-features", "ca-derivations"); + } +}; + +template +class DerivationAdvancedAttrsBothTest : public Fixture +{}; + +using BothFixtures = ::testing::Types; + +TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures); + +#define TEST_ATERM_JSON(STEM, NAME) \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json) \ + { \ + this->readTest(NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ + auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \ + Derivation got = Derivation::fromJSON(*this->store, encoded, this->mockXpSettings); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \ + { \ + this->writeTest( \ + NAME ".json", \ + [&]() -> json { \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ + return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings) \ + .toJSON(*this->store); \ + }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } \ + \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \ + { \ + this->readTest(NAME ".drv", [&](auto encoded) { \ + /* Use JSON file instead of C++ literal as source of truth. */ \ + auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ + auto expected = Derivation::fromJSON(*this->store, json, this->mockXpSettings); \ + auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \ + EXPECT_EQ(got.toJSON(*this->store), expected.toJSON(*this->store)); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ /* No corresponding write test, because we need to read the drv to write the json file */ -TEST_ATERM_JSON(advancedAttributes_defaults, "advanced-attributes-defaults"); TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults"); -TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attributes-structured-attrs"); +TEST_ATERM_JSON(advancedAttributes_defaults, "advanced-attributes"); TEST_ATERM_JSON(advancedAttributes_structuredAttrs, "advanced-attributes-structured-attrs-defaults"); +TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attributes-structured-attrs"); #undef TEST_ATERM_JSON -TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults) +TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) { - readTest("advanced-attributes-defaults.drv", [&](auto encoded) { - auto got = parseDerivation(*store, std::move(encoded), "foo"); + this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*store, got, NoRepair, true); + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); ParsedDerivation parsedDrv(got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); @@ -101,25 +126,50 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults) EXPECT_EQ(checksForAllOutputs.disallowedReferences, StringSet{}); EXPECT_EQ(checksForAllOutputs.disallowedRequisites, StringSet{}); } - EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet()); - EXPECT_EQ(options.canBuildLocally(*store, got), false); - EXPECT_EQ(options.willBuildLocally(*store, got), false); + EXPECT_EQ(options.canBuildLocally(*this->store, got), false); + EXPECT_EQ(options.willBuildLocally(*this->store, got), false); EXPECT_EQ(options.substitutesAllowed(), true); EXPECT_EQ(options.useUidRange(got), false); }); }; -TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes) +TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_defaults) { - readTest("advanced-attributes.drv", [&](auto encoded) { - auto got = parseDerivation(*store, std::move(encoded), "foo"); + this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*store, got, NoRepair, true); + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); ParsedDerivation parsedDrv(got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); - StringSet systemFeatures{"rainbow", "uid-range"}; + EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); + }); +}; + +TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults) +{ + this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + + EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); + }); +}; + +TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes) +{ + this->readTest("advanced-attributes.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_TRUE(!parsedDrv.hasStructuredAttrs()); @@ -128,6 +178,23 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes) EXPECT_EQ(options.impureHostDeps, StringSet{"/usr/bin/ditto"}); EXPECT_EQ(options.impureEnvVars, StringSet{"UNICORN"}); EXPECT_EQ(options.allowLocalNetworking, true); + EXPECT_EQ(options.canBuildLocally(*this->store, got), false); + EXPECT_EQ(options.willBuildLocally(*this->store, got), false); + EXPECT_EQ(options.substitutesAllowed(), false); + EXPECT_EQ(options.useUidRange(got), true); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, advancedAttributes) +{ + this->readTest("advanced-attributes.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + { auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); ASSERT_TRUE(checksForAllOutputs_ != nullptr); @@ -142,20 +209,55 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes) EXPECT_EQ( checksForAllOutputs.disallowedRequisites, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); } + + StringSet systemFeatures{"rainbow", "uid-range"}; + EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); - EXPECT_EQ(options.canBuildLocally(*store, got), false); - EXPECT_EQ(options.willBuildLocally(*store, got), false); - EXPECT_EQ(options.substitutesAllowed(), false); - EXPECT_EQ(options.useUidRange(got), true); }); }; -TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs_defaults) +TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) { - readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { - auto got = parseDerivation(*store, std::move(encoded), "foo"); + this->readTest("advanced-attributes.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*store, got, NoRepair, true); + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + + { + auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); + ASSERT_TRUE(checksForAllOutputs_ != nullptr); + auto & checksForAllOutputs = *checksForAllOutputs_; + + EXPECT_EQ( + checksForAllOutputs.allowedReferences, + StringSet{"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"}); + EXPECT_EQ( + checksForAllOutputs.allowedRequisites, + StringSet{"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"}); + EXPECT_EQ( + checksForAllOutputs.disallowedReferences, + StringSet{"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"}); + EXPECT_EQ( + checksForAllOutputs.disallowedRequisites, + StringSet{"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"}); + } + + StringSet systemFeatures{"rainbow", "uid-range"}; + systemFeatures.insert("ca-derivations"); + + EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); + }); +}; + +TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_defaults) +{ + this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); ParsedDerivation parsedDrv(got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); @@ -176,25 +278,50 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr EXPECT_EQ(checksPerOutput.size(), 0u); } - EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet()); - EXPECT_EQ(options.canBuildLocally(*store, got), false); - EXPECT_EQ(options.willBuildLocally(*store, got), false); + EXPECT_EQ(options.canBuildLocally(*this->store, got), false); + EXPECT_EQ(options.willBuildLocally(*this->store, got), false); EXPECT_EQ(options.substitutesAllowed(), true); EXPECT_EQ(options.useUidRange(got), false); }); }; -TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs) +TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults) { - readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { - auto got = parseDerivation(*store, std::move(encoded), "foo"); + this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*store, got, NoRepair, true); + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); ParsedDerivation parsedDrv(got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); - StringSet systemFeatures{"rainbow", "uid-range"}; + EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); + }); +}; + +TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults) +{ + this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + + EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); + }); +}; + +TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) +{ + this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_TRUE(parsedDrv.hasStructuredAttrs()); @@ -204,6 +331,32 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr EXPECT_EQ(options.impureEnvVars, StringSet{"UNICORN"}); EXPECT_EQ(options.allowLocalNetworking, true); + { + auto output_ = get(std::get<1>(options.outputChecks), "dev"); + ASSERT_TRUE(output_); + auto & output = *output_; + + EXPECT_EQ(output.maxSize, 789); + EXPECT_EQ(output.maxClosureSize, 5909); + } + + EXPECT_EQ(options.canBuildLocally(*this->store, got), false); + EXPECT_EQ(options.willBuildLocally(*this->store, got), false); + EXPECT_EQ(options.substitutesAllowed(), false); + EXPECT_EQ(options.useUidRange(got), true); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) +{ + this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + { { auto output_ = get(std::get<1>(options.outputChecks), "out"); @@ -222,22 +375,50 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr EXPECT_EQ(output.disallowedReferences, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); EXPECT_EQ(output.disallowedRequisites, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); } + } + StringSet systemFeatures{"rainbow", "uid-range"}; + + EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); + }); +}; + +TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) +{ + this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + + { { - auto output_ = get(std::get<1>(options.outputChecks), "dev"); + auto output_ = get(std::get<1>(options.outputChecks), "out"); ASSERT_TRUE(output_); auto & output = *output_; - EXPECT_EQ(output.maxSize, 789); - EXPECT_EQ(output.maxClosureSize, 5909); + EXPECT_EQ(output.allowedReferences, StringSet{"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"}); + EXPECT_EQ(output.allowedRequisites, StringSet{"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"}); + } + + { + auto output_ = get(std::get<1>(options.outputChecks), "bin"); + ASSERT_TRUE(output_); + auto & output = *output_; + + EXPECT_EQ( + output.disallowedReferences, StringSet{"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"}); + EXPECT_EQ( + output.disallowedRequisites, StringSet{"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"}); } } + StringSet systemFeatures{"rainbow", "uid-range"}; + systemFeatures.insert("ca-derivations"); + EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); - EXPECT_EQ(options.canBuildLocally(*store, got), false); - EXPECT_EQ(options.willBuildLocally(*store, got), false); - EXPECT_EQ(options.substitutesAllowed(), false); - EXPECT_EQ(options.useUidRange(got), true); }); }; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a4e04e78a..42de5ee0c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1355,7 +1355,7 @@ Derivation Derivation::fromJSON( for (auto & [outputName, output] : outputs) { res.outputs.insert_or_assign( outputName, - DerivationOutput::fromJSON(store, res.name, outputName, output)); + DerivationOutput::fromJSON(store, res.name, outputName, output, xpSettings)); } } catch (Error & e) { e.addTrace({}, "while reading key 'outputs'"); diff --git a/tests/functional/ca/derivation-advanced-attributes.sh b/tests/functional/ca/derivation-advanced-attributes.sh new file mode 100755 index 000000000..b70463e5c --- /dev/null +++ b/tests/functional/ca/derivation-advanced-attributes.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +export NIX_TESTS_CA_BY_DEFAULT=1 + +cd .. +source derivation-advanced-attributes.sh diff --git a/tests/functional/ca/meson.build b/tests/functional/ca/meson.build index 7a7fcc5cf..a4611ca42 100644 --- a/tests/functional/ca/meson.build +++ b/tests/functional/ca/meson.build @@ -8,10 +8,11 @@ suites += { 'name': 'ca', 'deps': [], 'tests': [ + 'build-cache.sh', 'build-with-garbage-path.sh', 'build.sh', - 'build-cache.sh', 'concurrent-builds.sh', + 'derivation-advanced-attributes.sh', 'derivation-json.sh', 'duplicate-realisation-in-closure.sh', 'eval-store.sh', diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh index 6707b345c..a7530e11c 100755 --- a/tests/functional/derivation-advanced-attributes.sh +++ b/tests/functional/derivation-advanced-attributes.sh @@ -12,11 +12,19 @@ badExitCode=0 store="$TEST_ROOT/store" +if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + drvDir=ia + flags=(--arg contentAddress false) +else + drvDir=ca + flags=(--arg contentAddress true --extra-experimental-features ca-derivations) +fi + for nixFile in derivation/*.nix; do - drvPath=$(env -u NIX_STORE nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") + drvPath=$(env -u NIX_STORE nix-instantiate --store "$store" --pure-eval "${flags[@]}" --expr "$(< "$nixFile")") testName=$(basename "$nixFile" .nix) got="${store}${drvPath}" - expected="derivation/$testName.drv" + expected="derivation/${drvDir}/${testName}.drv" diffAndAcceptInner "$testName" "$got" "$expected" done diff --git a/tests/functional/derivation/advanced-attributes-defaults.nix b/tests/functional/derivation/advanced-attributes-defaults.nix index d466003b0..51f359cf0 100644 --- a/tests/functional/derivation/advanced-attributes-defaults.nix +++ b/tests/functional/derivation/advanced-attributes-defaults.nix @@ -1,6 +1,24 @@ -derivation { - name = "advanced-attributes-defaults"; +{ contentAddress }: + +let + caArgs = + if contentAddress then + { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } + else + { }; + + derivation' = args: derivation (caArgs // args); + system = "my-system"; + +in +derivation' { + inherit system; + name = "advanced-attributes-defaults"; builder = "/bin/bash"; args = [ "-c" diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix index 3c6ad4900..ec51f0e28 100644 --- a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix +++ b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix @@ -1,6 +1,24 @@ -derivation { - name = "advanced-attributes-structured-attrs-defaults"; +{ contentAddress }: + +let + caArgs = + if contentAddress then + { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } + else + { }; + + derivation' = args: derivation (caArgs // args); + system = "my-system"; + +in +derivation' { + inherit system; + name = "advanced-attributes-structured-attrs-defaults"; builder = "/bin/bash"; args = [ "-c" diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.nix b/tests/functional/derivation/advanced-attributes-structured-attrs.nix index 4c596be45..b789cdaa7 100644 --- a/tests/functional/derivation/advanced-attributes-structured-attrs.nix +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.nix @@ -1,6 +1,21 @@ +{ contentAddress }: + let + caArgs = + if contentAddress then + { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } + else + { }; + + derivation' = args: derivation (caArgs // args); + system = "my-system"; - foo = derivation { + + foo = derivation' { inherit system; name = "foo"; builder = "/bin/bash"; @@ -9,7 +24,8 @@ let "echo foo > $out" ]; }; - bar = derivation { + + bar = derivation' { inherit system; name = "bar"; builder = "/bin/bash"; @@ -18,8 +34,9 @@ let "echo bar > $out" ]; }; + in -derivation { +derivation' { inherit system; name = "advanced-attributes-structured-attrs"; builder = "/bin/bash"; diff --git a/tests/functional/derivation/advanced-attributes.nix b/tests/functional/derivation/advanced-attributes.nix index 7f365ce65..52786783f 100644 --- a/tests/functional/derivation/advanced-attributes.nix +++ b/tests/functional/derivation/advanced-attributes.nix @@ -1,6 +1,21 @@ +{ contentAddress }: + let + caArgs = + if contentAddress then + { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } + else + { }; + + derivation' = args: derivation (caArgs // args); + system = "my-system"; - foo = derivation { + + foo = derivation' { inherit system; name = "foo"; builder = "/bin/bash"; @@ -9,7 +24,8 @@ let "echo foo > $out" ]; }; - bar = derivation { + + bar = derivation' { inherit system; name = "bar"; builder = "/bin/bash"; @@ -18,8 +34,9 @@ let "echo bar > $out" ]; }; + in -derivation { +derivation' { inherit system; name = "advanced-attributes"; builder = "/bin/bash"; diff --git a/tests/functional/derivation/ca/advanced-attributes-defaults.drv b/tests/functional/derivation/ca/advanced-attributes-defaults.drv new file mode 100644 index 000000000..2c8160963 --- /dev/null +++ b/tests/functional/derivation/ca/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +Derive([("out","","r:sha256","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("builder","/bin/bash"),("name","advanced-attributes-defaults"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"),("outputHashAlgo","sha256"),("outputHashMode","recursive"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/ca/advanced-attributes-structured-attrs-defaults.drv b/tests/functional/derivation/ca/advanced-attributes-structured-attrs-defaults.drv new file mode 100644 index 000000000..bf56e05d6 --- /dev/null +++ b/tests/functional/derivation/ca/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +Derive([("dev","","r:sha256",""),("out","","r:sha256","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}"),("dev","/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9")]) \ No newline at end of file diff --git a/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv new file mode 100644 index 000000000..307beb53e --- /dev/null +++ b/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +Derive([("bin","","r:sha256",""),("dev","","r:sha256",""),("out","","r:sha256","")],[("/nix/store/spfzlnkwb1v8s62yvh8vj1apd1kwjr5f-foo.drv",["out"]),("/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"],\"disallowedRequisites\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"],\"allowedRequisites\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m"),("dev","/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9")]) \ No newline at end of file diff --git a/tests/functional/derivation/ca/advanced-attributes.drv b/tests/functional/derivation/ca/advanced-attributes.drv new file mode 100644 index 000000000..343f895ca --- /dev/null +++ b/tests/functional/derivation/ca/advanced-attributes.drv @@ -0,0 +1 @@ +Derive([("out","","r:sha256","")],[("/nix/store/spfzlnkwb1v8s62yvh8vj1apd1kwjr5f-foo.drv",["out"]),("/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"),("allowedRequisites","/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"),("builder","/bin/bash"),("disallowedReferences","/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"),("disallowedRequisites","/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"),("outputHashAlgo","sha256"),("outputHashMode","recursive"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-defaults.drv b/tests/functional/derivation/ia/advanced-attributes-defaults.drv similarity index 100% rename from tests/functional/derivation/advanced-attributes-defaults.drv rename to tests/functional/derivation/ia/advanced-attributes-defaults.drv diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/functional/derivation/ia/advanced-attributes-structured-attrs-defaults.drv similarity index 100% rename from tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv rename to tests/functional/derivation/ia/advanced-attributes-structured-attrs-defaults.drv diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv similarity index 100% rename from tests/functional/derivation/advanced-attributes-structured-attrs.drv rename to tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv diff --git a/tests/functional/derivation/advanced-attributes.drv b/tests/functional/derivation/ia/advanced-attributes.drv similarity index 100% rename from tests/functional/derivation/advanced-attributes.drv rename to tests/functional/derivation/ia/advanced-attributes.drv From d285b800337f295671f1c98fb1dd11f24b6b1965 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 3 Feb 2025 11:28:43 -0500 Subject: [PATCH 223/396] Move `exportReferencesGraph` to `DerivationOptions` Tests are updated accordingly. --- .../advanced-attributes-structured-attrs.json | 6 +- .../derivation/ca/advanced-attributes.json | 5 +- .../advanced-attributes-structured-attrs.json | 18 ++--- .../derivation/ia/advanced-attributes.json | 9 ++- .../derivation-advanced-attrs.cc | 72 +++++++++++++++++++ src/libstore/derivation-options.cc | 30 ++++++++ .../include/nix/store/derivation-options.hh | 21 ++++++ .../include/nix/store/parsed-derivations.hh | 3 +- .../include/nix/store/restricted-store.hh | 2 +- src/libstore/parsed-derivations.cc | 21 +++--- .../unix/build/local-derivation-goal.cc | 34 +++------ src/nix-build/nix-build.cc | 2 +- .../advanced-attributes-structured-attrs.nix | 2 + .../derivation/advanced-attributes.nix | 6 ++ .../advanced-attributes-structured-attrs.drv | 2 +- .../derivation/ca/advanced-attributes.drv | 2 +- .../advanced-attributes-structured-attrs.drv | 2 +- .../derivation/ia/advanced-attributes.drv | 2 +- 18 files changed, 183 insertions(+), 56 deletions(-) diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json index 584fd2113..ddc887633 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -5,7 +5,7 @@ ], "builder": "/bin/bash", "env": { - "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"],\"disallowedRequisites\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"],\"allowedRequisites\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"],\"refs2\":[\"/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"],\"disallowedRequisites\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"],\"allowedRequisites\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", "bin": "/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m", "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" @@ -24,7 +24,9 @@ ] } }, - "inputSrcs": [], + "inputSrcs": [ + "/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv" + ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.json b/src/libstore-tests/data/derivation/ca/advanced-attributes.json index 69d40b135..57a1141b5 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.json @@ -15,6 +15,7 @@ "builder": "/bin/bash", "disallowedReferences": "/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99", "disallowedRequisites": "/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99", + "exportReferencesGraph": "refs1 /08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8 refs2 /nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv", "impureEnvVars": "UNICORN", "name": "advanced-attributes", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9", @@ -38,7 +39,9 @@ ] } }, - "inputSrcs": [], + "inputSrcs": [ + "/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv" + ], "name": "advanced-attributes", "outputs": { "out": { diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json index 324428124..2ba83c21c 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json @@ -5,10 +5,10 @@ ], "builder": "/bin/bash", "env": { - "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", - "bin": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin", - "dev": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev", - "out": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"refs2\":[\"/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "bin": "/nix/store/spb9y9agq61rvq29fhkfw1ql00adjq7d-advanced-attributes-structured-attrs-bin", + "dev": "/nix/store/0v889x74f5d5swbjivcik2yw4gg2pm52-advanced-attributes-structured-attrs-dev", + "out": "/nix/store/vnzmd26f5hx5dp5xrfs2kshqvcpxhrhh-advanced-attributes-structured-attrs" }, "inputDrvs": { "/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv": { @@ -24,17 +24,19 @@ ] } }, - "inputSrcs": [], + "inputSrcs": [ + "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv" + ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { - "path": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin" + "path": "/nix/store/spb9y9agq61rvq29fhkfw1ql00adjq7d-advanced-attributes-structured-attrs-bin" }, "dev": { - "path": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev" + "path": "/nix/store/0v889x74f5d5swbjivcik2yw4gg2pm52-advanced-attributes-structured-attrs-dev" }, "out": { - "path": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" + "path": "/nix/store/vnzmd26f5hx5dp5xrfs2kshqvcpxhrhh-advanced-attributes-structured-attrs" } }, "system": "my-system" diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.json b/src/libstore-tests/data/derivation/ia/advanced-attributes.json index d51524e20..06ae314d7 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.json @@ -15,9 +15,10 @@ "builder": "/bin/bash", "disallowedReferences": "/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar", "disallowedRequisites": "/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar", + "exportReferencesGraph": "refs1 /nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo refs2 /nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv", "impureEnvVars": "UNICORN", "name": "advanced-attributes", - "out": "/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes", + "out": "/nix/store/jvm2xsx0lm29byzr59yzjw7c14fa9z5f-advanced-attributes", "preferLocalBuild": "1", "requiredSystemFeatures": "rainbow uid-range", "system": "my-system" @@ -36,11 +37,13 @@ ] } }, - "inputSrcs": [], + "inputSrcs": [ + "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv" + ], "name": "advanced-attributes", "outputs": { "out": { - "path": "/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes" + "path": "/nix/store/jvm2xsx0lm29byzr59yzjw7c14fa9z5f-advanced-attributes" } }, "system": "my-system" diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 48f117f1c..4408720e8 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -99,6 +99,8 @@ TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attribute #undef TEST_ATERM_JSON +using ExportReferencesMap = decltype(DerivationOptions::exportReferencesGraph); + TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) { this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { @@ -116,6 +118,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) EXPECT_EQ(options.impureHostDeps, StringSet{}); EXPECT_EQ(options.impureEnvVars, StringSet{}); EXPECT_EQ(options.allowLocalNetworking, false); + EXPECT_EQ(options.exportReferencesGraph, ExportReferencesMap{}); { auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); ASSERT_TRUE(checksForAllOutputs_ != nullptr); @@ -195,6 +198,23 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes) ParsedDerivation parsedDrv(got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + EXPECT_EQ( + options.exportReferencesGraph, + (ExportReferencesMap{ + { + "refs1", + { + "/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo", + }, + }, + { + "refs2", + { + "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv", + }, + }, + })); + { auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); ASSERT_TRUE(checksForAllOutputs_ != nullptr); @@ -226,6 +246,23 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) ParsedDerivation parsedDrv(got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + EXPECT_EQ( + options.exportReferencesGraph, + (ExportReferencesMap{ + { + "refs1", + { + "/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8", + }, + }, + { + "refs2", + { + "/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv", + }, + }, + })); + { auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); ASSERT_TRUE(checksForAllOutputs_ != nullptr); @@ -269,6 +306,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d EXPECT_EQ(options.impureHostDeps, StringSet{}); EXPECT_EQ(options.impureEnvVars, StringSet{}); EXPECT_EQ(options.allowLocalNetworking, false); + EXPECT_EQ(options.exportReferencesGraph, ExportReferencesMap{}); { auto * checksPerOutput_ = std::get_if<1>(&options.outputChecks); @@ -357,6 +395,23 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) ParsedDerivation parsedDrv(got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + EXPECT_EQ( + options.exportReferencesGraph, + (ExportReferencesMap{ + { + "refs1", + { + "/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo", + }, + }, + { + "refs2", + { + "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv", + }, + }, + })); + { { auto output_ = get(std::get<1>(options.outputChecks), "out"); @@ -393,6 +448,23 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) ParsedDerivation parsedDrv(got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + EXPECT_EQ( + options.exportReferencesGraph, + (ExportReferencesMap{ + { + "refs1", + { + "/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8", + }, + }, + { + "refs2", + { + "/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv", + }, + }, + })); + { { auto output_ = get(std::get<1>(options.outputChecks), "out"); diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 962222f6d..526e682f7 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -3,9 +3,11 @@ #include "nix/store/parsed-derivations.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" + #include #include #include +#include namespace nix { @@ -126,6 +128,34 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation } return res; }(), + .exportReferencesGraph = + [&] { + std::map ret; + + if (auto structuredAttrs = parsed.structuredAttrs.get()) { + auto e = optionalValueAt(*structuredAttrs, "exportReferencesGraph"); + if (!e || !e->is_object()) + return ret; + for (auto & [key, storePathsJson] : getObject(*e)) { + ret.insert_or_assign(key, storePathsJson); + } + } else { + auto s = getOr(parsed.drv.env, "exportReferencesGraph", ""); + Strings ss = tokenizeString(s); + if (ss.size() % 2 != 0) + throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); + for (Strings::iterator i = ss.begin(); i != ss.end();) { + auto fileName = std::move(*i++); + static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); + if (!std::regex_match(fileName, regex)) + throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); + + auto & storePathS = *i++; + ret.insert_or_assign(std::move(fileName), StringSet{storePathS}); + } + } + return ret; + }(), .additionalSandboxProfile = parsed.getStringAttr("__sandboxProfile").value_or(defaults.additionalSandboxProfile), .noChroot = parsed.getBoolAttr("__noChroot", defaults.noChroot), diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index 8f549b737..bc9ad3317 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -95,6 +95,27 @@ struct DerivationOptions */ StringSet passAsFile; + /** + * The `exportReferencesGraph' feature allows the references graph + * to be passed to a builder + * + * ### Legacy case + * + * Given a `name` `pathSet` key-value pair, the references graph of + * `pathSet` will be stored in a text file `name' in the temporary + * build directory. The text files have the format used by + * `nix-store + * --register-validity'. However, the `deriver` fields are left + * empty. + * + * ### "Structured attributes" case + * + * The same information will be put put in the final structured + * attributes give to the builder. The set of paths in the original JSON + * is replaced with a list of `PathInfo` in JSON format. + */ + std::map exportReferencesGraph; + /** * env: __sandboxProfile * diff --git a/src/libstore/include/nix/store/parsed-derivations.hh b/src/libstore/include/nix/store/parsed-derivations.hh index 2b1ab97d5..e23eab1bb 100644 --- a/src/libstore/include/nix/store/parsed-derivations.hh +++ b/src/libstore/include/nix/store/parsed-derivations.hh @@ -42,7 +42,8 @@ public: return static_cast(structuredAttrs); } - std::optional prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths); + std::optional + prepareStructuredAttrs(Store & store, const DerivationOptions & drvOptions, const StorePathSet & inputPaths); }; std::string writeStructuredAttrsShell(const nlohmann::json & json); diff --git a/src/libstore/include/nix/store/restricted-store.hh b/src/libstore/include/nix/store/restricted-store.hh index 84b455456..67c26c88b 100644 --- a/src/libstore/include/nix/store/restricted-store.hh +++ b/src/libstore/include/nix/store/restricted-store.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "local-store.hh" +#include "nix/store/local-store.hh" namespace nix { diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 66bf76cac..218a1cf35 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,4 +1,5 @@ #include "nix/store/parsed-derivations.hh" +#include "nix/store/derivation-options.hh" #include #include @@ -151,7 +152,10 @@ static nlohmann::json pathInfoToJSON( return jsonList; } -std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) +std::optional ParsedDerivation::prepareStructuredAttrs( + Store & store, + const DerivationOptions & drvOptions, + const StorePathSet & inputPaths) { if (!structuredAttrs) return std::nullopt; @@ -164,15 +168,12 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s json["outputs"] = outputs; /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(store.toStorePath(p.get()).first); - json[i.key()] = pathInfoToJSON(store, - store.exportReferences(storePaths, inputPaths)); - } + for (auto & [key, inputPaths] : drvOptions.exportReferencesGraph) { + StorePathSet storePaths; + for (auto & p : inputPaths) + storePaths.insert(store.toStorePath(p).first); + json[key] = pathInfoToJSON(store, + store.exportReferences(storePaths, storePaths)); } return json; diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 8adc001f0..965ece136 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -24,7 +24,6 @@ #include "nix/store/restricted-store.hh" #include "nix/store/config.hh" -#include #include #include @@ -973,32 +972,17 @@ void LocalDerivationGoal::startBuilder() /* Handle exportReferencesGraph(), if set. */ if (!parsedDrv->hasStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - auto s = getOr(drv->env, "exportReferencesGraph", ""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - auto fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - + for (auto & [fileName, ss] : drvOptions->exportReferencesGraph) { + StorePathSet storePathSet; + for (auto & storePathS : ss) { + if (!worker.store.isInStore(storePathS)) + throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); + storePathSet.insert(worker.store.toStorePath(storePathS).first); + } /* Write closure info to . */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - worker.store.exportReferences({storePath}, inputPaths), false, false)); + worker.store.exportReferences(storePathSet, inputPaths), false, false)); } } @@ -1592,7 +1576,7 @@ void LocalDerivationGoal::initEnv() void LocalDerivationGoal::writeStructuredAttrs() { - if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, inputPaths)) { + if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, *drvOptions, inputPaths)) { auto json = structAttrsJson.value(); nlohmann::json rewritten; for (auto & [i, v] : json["outputs"].get()) { diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d90630438..c2368dc5b 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -584,7 +584,7 @@ static void main_nix_build(int argc, char * * argv) for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) accumInputClosure(inputDrv, inputNode); - if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, inputs)) { + if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, drvOptions, inputs)) { auto json = structAttrs.value(); structuredAttrsRC = writeStructuredAttrsShell(json); diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.nix b/tests/functional/derivation/advanced-attributes-structured-attrs.nix index b789cdaa7..60bbee61c 100644 --- a/tests/functional/derivation/advanced-attributes-structured-attrs.nix +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.nix @@ -75,4 +75,6 @@ derivation' { ]; preferLocalBuild = true; allowSubstitutes = false; + exportReferencesGraph.refs1 = [ foo ]; + exportReferencesGraph.refs2 = [ bar.drvPath ]; } diff --git a/tests/functional/derivation/advanced-attributes.nix b/tests/functional/derivation/advanced-attributes.nix index 52786783f..09d5c33c2 100644 --- a/tests/functional/derivation/advanced-attributes.nix +++ b/tests/functional/derivation/advanced-attributes.nix @@ -59,4 +59,10 @@ derivation' { ]; preferLocalBuild = true; allowSubstitutes = false; + exportReferencesGraph = [ + "refs1" + foo + "refs2" + bar.drvPath + ]; } diff --git a/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv index 307beb53e..b22fbdb2d 100644 --- a/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv +++ b/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv @@ -1 +1 @@ -Derive([("bin","","r:sha256",""),("dev","","r:sha256",""),("out","","r:sha256","")],[("/nix/store/spfzlnkwb1v8s62yvh8vj1apd1kwjr5f-foo.drv",["out"]),("/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"],\"disallowedRequisites\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"],\"allowedRequisites\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m"),("dev","/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9")]) \ No newline at end of file +Derive([("bin","","r:sha256",""),("dev","","r:sha256",""),("out","","r:sha256","")],[("/nix/store/spfzlnkwb1v8s62yvh8vj1apd1kwjr5f-foo.drv",["out"]),("/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv",["out"])],["/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"],\"refs2\":[\"/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"],\"disallowedRequisites\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"],\"allowedRequisites\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m"),("dev","/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9")]) \ No newline at end of file diff --git a/tests/functional/derivation/ca/advanced-attributes.drv b/tests/functional/derivation/ca/advanced-attributes.drv index 343f895ca..93996c15c 100644 --- a/tests/functional/derivation/ca/advanced-attributes.drv +++ b/tests/functional/derivation/ca/advanced-attributes.drv @@ -1 +1 @@ -Derive([("out","","r:sha256","")],[("/nix/store/spfzlnkwb1v8s62yvh8vj1apd1kwjr5f-foo.drv",["out"]),("/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"),("allowedRequisites","/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"),("builder","/bin/bash"),("disallowedReferences","/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"),("disallowedRequisites","/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"),("outputHashAlgo","sha256"),("outputHashMode","recursive"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file +Derive([("out","","r:sha256","")],[("/nix/store/spfzlnkwb1v8s62yvh8vj1apd1kwjr5f-foo.drv",["out"]),("/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv",["out"])],["/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"),("allowedRequisites","/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"),("builder","/bin/bash"),("disallowedReferences","/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"),("disallowedRequisites","/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"),("exportReferencesGraph","refs1 /08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8 refs2 /nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"),("outputHashAlgo","sha256"),("outputHashMode","recursive"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv index e47a41ad5..b9359bbbd 100644 --- a/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv +++ b/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv @@ -1 +1 @@ -Derive([("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev"),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs")]) \ No newline at end of file +Derive([("bin","/nix/store/spb9y9agq61rvq29fhkfw1ql00adjq7d-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/0v889x74f5d5swbjivcik2yw4gg2pm52-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/vnzmd26f5hx5dp5xrfs2kshqvcpxhrhh-advanced-attributes-structured-attrs","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],["/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"refs2\":[\"/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/spb9y9agq61rvq29fhkfw1ql00adjq7d-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/0v889x74f5d5swbjivcik2yw4gg2pm52-advanced-attributes-structured-attrs-dev"),("out","/nix/store/vnzmd26f5hx5dp5xrfs2kshqvcpxhrhh-advanced-attributes-structured-attrs")]) \ No newline at end of file diff --git a/tests/functional/derivation/ia/advanced-attributes.drv b/tests/functional/derivation/ia/advanced-attributes.drv index ec3112ab2..8e69b8c68 100644 --- a/tests/functional/derivation/ia/advanced-attributes.drv +++ b/tests/functional/derivation/ia/advanced-attributes.drv @@ -1 +1 @@ -Derive([("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("allowedRequisites","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("builder","/bin/bash"),("disallowedReferences","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("disallowedRequisites","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file +Derive([("out","/nix/store/jvm2xsx0lm29byzr59yzjw7c14fa9z5f-advanced-attributes","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],["/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("allowedRequisites","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("builder","/bin/bash"),("disallowedReferences","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("disallowedRequisites","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("exportReferencesGraph","refs1 /nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo refs2 /nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/jvm2xsx0lm29byzr59yzjw7c14fa9z5f-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file From 1e31b60043b28b679b8ef770d261dcfe137a3044 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 3 Feb 2025 11:41:33 -0500 Subject: [PATCH 224/396] Limit `ParsedDerivation` just to the derivation's environment This moves us towards getting rid of `ParsedDerivation` and just having `DerivationOptions`. Co-Authored-By: HaeNoe --- .../derivation-advanced-attrs.cc | 24 +++++++------- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/derivation-options.cc | 4 +-- .../include/nix/store/parsed-derivations.hh | 11 ++++--- src/libstore/misc.cc | 2 +- src/libstore/parsed-derivations.cc | 31 ++++++++++--------- .../unix/build/local-derivation-goal.cc | 7 ++++- src/nix-build/nix-build.cc | 9 ++++-- 8 files changed, 52 insertions(+), 38 deletions(-) diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 4408720e8..0eefc2673 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -108,7 +108,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got); + ParsedDerivation parsedDrv(got.env); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_TRUE(!parsedDrv.hasStructuredAttrs()); @@ -143,7 +143,7 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_defaults) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got); + ParsedDerivation parsedDrv(got.env); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); @@ -157,7 +157,7 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got); + ParsedDerivation parsedDrv(got.env); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); @@ -171,7 +171,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got); + ParsedDerivation parsedDrv(got.env); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_TRUE(!parsedDrv.hasStructuredAttrs()); @@ -195,7 +195,7 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got); + ParsedDerivation parsedDrv(got.env); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_EQ( @@ -243,7 +243,7 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got); + ParsedDerivation parsedDrv(got.env); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_EQ( @@ -296,7 +296,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got); + ParsedDerivation parsedDrv(got.env); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_TRUE(parsedDrv.hasStructuredAttrs()); @@ -330,7 +330,7 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got); + ParsedDerivation parsedDrv(got.env); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); @@ -344,7 +344,7 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_default auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got); + ParsedDerivation parsedDrv(got.env); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); @@ -358,7 +358,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got); + ParsedDerivation parsedDrv(got.env); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_TRUE(parsedDrv.hasStructuredAttrs()); @@ -392,7 +392,7 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got); + ParsedDerivation parsedDrv(got.env); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_EQ( @@ -445,7 +445,7 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got); + ParsedDerivation parsedDrv(got.env); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_EQ( diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8e43ab2ae..c2232b657 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -180,7 +180,7 @@ Goal::Co DerivationGoal::haveDerivation() { trace("have derivation"); - parsedDrv = std::make_unique(*drv); + parsedDrv = std::make_unique(drv->env); try { drvOptions = std::make_unique(DerivationOptions::fromParsedDerivation(*parsedDrv)); } catch (Error & e) { diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 526e682f7..5d22f1f66 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -116,7 +116,7 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation .passAsFile = [&] { StringSet res; - if (auto * passAsFileString = get(parsed.drv.env, "passAsFile")) { + if (auto * passAsFileString = get(parsed.env, "passAsFile")) { if (parsed.hasStructuredAttrs()) { if (shouldWarn) { warn( @@ -140,7 +140,7 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation ret.insert_or_assign(key, storePathsJson); } } else { - auto s = getOr(parsed.drv.env, "exportReferencesGraph", ""); + auto s = getOr(parsed.env, "exportReferencesGraph", ""); Strings ss = tokenizeString(s); if (ss.size() % 2 != 0) throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); diff --git a/src/libstore/include/nix/store/parsed-derivations.hh b/src/libstore/include/nix/store/parsed-derivations.hh index e23eab1bb..30a294187 100644 --- a/src/libstore/include/nix/store/parsed-derivations.hh +++ b/src/libstore/include/nix/store/parsed-derivations.hh @@ -12,7 +12,7 @@ struct DerivationOptions; class ParsedDerivation { - BasicDerivation & drv; + const StringPairs & env; std::unique_ptr structuredAttrs; std::optional getStringAttr(const std::string & name) const; @@ -33,7 +33,7 @@ class ParsedDerivation public: - ParsedDerivation(BasicDerivation & drv); + ParsedDerivation(const StringPairs & env); ~ParsedDerivation(); @@ -42,8 +42,11 @@ public: return static_cast(structuredAttrs); } - std::optional - prepareStructuredAttrs(Store & store, const DerivationOptions & drvOptions, const StorePathSet & inputPaths); + std::optional prepareStructuredAttrs( + Store & store, + const DerivationOptions & drvOptions, + const StorePathSet & inputPaths, + const DerivationOutputs & outputs); }; std::string writeStructuredAttrsShell(const nlohmann::json & json); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index b9e729092..cfa82845d 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -222,7 +222,7 @@ void Store::queryMissing(const std::vector & targets, if (knownOutputPaths && invalid.empty()) return; auto drv = make_ref(derivationFromPath(drvPath)); - ParsedDerivation parsedDrv(*drv); + ParsedDerivation parsedDrv(drv->env); DerivationOptions drvOptions; try { drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 218a1cf35..9aa16ff2c 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -6,12 +6,12 @@ namespace nix { -ParsedDerivation::ParsedDerivation(BasicDerivation & drv) - : drv(drv) +ParsedDerivation::ParsedDerivation(const StringPairs & env) + : env(env) { /* Parse the __json attribute, if any. */ - auto jsonAttr = drv.env.find("__json"); - if (jsonAttr != drv.env.end()) { + auto jsonAttr = env.find("__json"); + if (jsonAttr != env.end()) { try { structuredAttrs = std::make_unique(nlohmann::json::parse(jsonAttr->second)); } catch (std::exception & e) { @@ -34,8 +34,8 @@ std::optional ParsedDerivation::getStringAttr(const std::string & n return i->get(); } } else { - auto i = drv.env.find(name); - if (i == drv.env.end()) + auto i = env.find(name); + if (i == env.end()) return {}; else return i->second; @@ -54,8 +54,8 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const return i->get(); } } else { - auto i = drv.env.find(name); - if (i == drv.env.end()) + auto i = env.find(name); + if (i == env.end()) return def; else return i->second == "1"; @@ -80,8 +80,8 @@ std::optional ParsedDerivation::getStringsAttr(const std::string & name return res; } } else { - auto i = drv.env.find(name); - if (i == drv.env.end()) + auto i = env.find(name); + if (i == env.end()) return {}; else return tokenizeString(i->second); @@ -155,17 +155,18 @@ static nlohmann::json pathInfoToJSON( std::optional ParsedDerivation::prepareStructuredAttrs( Store & store, const DerivationOptions & drvOptions, - const StorePathSet & inputPaths) + const StorePathSet & inputPaths, + const DerivationOutputs & outputs) { if (!structuredAttrs) return std::nullopt; auto json = *structuredAttrs; /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv.outputs) - outputs[i.first] = hashPlaceholder(i.first); - json["outputs"] = outputs; + nlohmann::json outputsJson; + for (auto & i : outputs) + outputsJson[i.first] = hashPlaceholder(i.first); + json["outputs"] = std::move(outputsJson); /* Handle exportReferencesGraph. */ for (auto & [key, inputPaths] : drvOptions.exportReferencesGraph) { diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 965ece136..18300904e 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1576,7 +1576,12 @@ void LocalDerivationGoal::initEnv() void LocalDerivationGoal::writeStructuredAttrs() { - if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, *drvOptions, inputPaths)) { + if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs( + worker.store, + *drvOptions, + inputPaths, + drv->outputs)) + { auto json = structAttrsJson.value(); nlohmann::json rewritten; for (auto & [i, v] : json["outputs"].get()) { diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index c2368dc5b..7ace4d4b4 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -544,7 +544,7 @@ static void main_nix_build(int argc, char * * argv) env["NIX_STORE"] = store->storeDir; env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); - ParsedDerivation parsedDrv(drv); + ParsedDerivation parsedDrv(drv.env); DerivationOptions drvOptions; try { drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); @@ -584,7 +584,12 @@ static void main_nix_build(int argc, char * * argv) for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) accumInputClosure(inputDrv, inputNode); - if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, drvOptions, inputs)) { + if (auto structAttrs = parsedDrv.prepareStructuredAttrs( + *store, + drvOptions, + inputs, + drv.outputs)) + { auto json = structAttrs.value(); structuredAttrsRC = writeStructuredAttrsShell(json); From d8be4f618f567a995a2c02d6703966a07e620910 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 3 Feb 2025 12:52:10 -0500 Subject: [PATCH 225/396] Scrap `ParsedDerivation` for parts Only a much smaller `StructuredAttrs` remains, the rest is is now moved to `DerivationOptions`. This gets us quite close to `std::optional` and `DerivationOptions` being included in `Derivation` as fields. --- .../derivation-advanced-attrs.cc | 56 +++---- src/libstore/build/derivation-goal.cc | 7 +- src/libstore/derivation-options.cc | 140 ++++++++++++++---- .../nix/store/build/derivation-goal.hh | 3 +- .../include/nix/store/derivation-options.hh | 9 +- .../include/nix/store/parsed-derivations.hh | 61 ++++---- src/libstore/misc.cc | 6 +- src/libstore/parsed-derivations.cc | 102 ++----------- .../unix/build/local-derivation-goal.cc | 13 +- src/nix-build/nix-build.cc | 36 ++--- 10 files changed, 213 insertions(+), 220 deletions(-) diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 0eefc2673..ce5804126 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -108,10 +108,10 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got.env); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); - EXPECT_TRUE(!parsedDrv.hasStructuredAttrs()); + EXPECT_TRUE(!parsedDrv); EXPECT_EQ(options.additionalSandboxProfile, ""); EXPECT_EQ(options.noChroot, false); @@ -143,8 +143,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_defaults) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got.env); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); }); @@ -157,8 +157,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got.env); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); }); @@ -171,10 +171,10 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got.env); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); - EXPECT_TRUE(!parsedDrv.hasStructuredAttrs()); + EXPECT_TRUE(!parsedDrv); EXPECT_EQ(options.additionalSandboxProfile, "sandcastle"); EXPECT_EQ(options.noChroot, true); @@ -195,8 +195,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got.env); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); EXPECT_EQ( options.exportReferencesGraph, @@ -243,8 +243,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got.env); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); EXPECT_EQ( options.exportReferencesGraph, @@ -296,10 +296,10 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got.env); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); - EXPECT_TRUE(parsedDrv.hasStructuredAttrs()); + EXPECT_TRUE(parsedDrv); EXPECT_EQ(options.additionalSandboxProfile, ""); EXPECT_EQ(options.noChroot, false); @@ -330,8 +330,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got.env); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); }); @@ -344,8 +344,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_default auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got.env); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); }); @@ -358,10 +358,10 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got.env); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); - EXPECT_TRUE(parsedDrv.hasStructuredAttrs()); + EXPECT_TRUE(parsedDrv); EXPECT_EQ(options.additionalSandboxProfile, "sandcastle"); EXPECT_EQ(options.noChroot, true); @@ -392,8 +392,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got.env); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); EXPECT_EQ( options.exportReferencesGraph, @@ -445,8 +445,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(got.env); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); EXPECT_EQ( options.exportReferencesGraph, diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index c2232b657..1a3d23c48 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -180,9 +180,12 @@ Goal::Co DerivationGoal::haveDerivation() { trace("have derivation"); - parsedDrv = std::make_unique(drv->env); + if (auto parsedOpt = StructuredAttrs::tryParse(drv->env)) { + parsedDrv = std::make_unique(*parsedOpt); + } try { - drvOptions = std::make_unique(DerivationOptions::fromParsedDerivation(*parsedDrv)); + drvOptions = std::make_unique( + DerivationOptions::fromStructuredAttrs(drv->env, parsedDrv.get())); } catch (Error & e) { e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath)); throw; diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 5d22f1f66..9872265e2 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -1,6 +1,8 @@ #include "nix/store/derivation-options.hh" #include "nix/util/json-utils.hh" #include "nix/store/parsed-derivations.hh" +#include "nix/store/derivations.hh" +#include "nix/store/store-api.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" @@ -11,38 +13,112 @@ namespace nix { +static std::optional +getStringAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name) +{ + if (parsed) { + auto i = parsed->structuredAttrs.find(name); + if (i == parsed->structuredAttrs.end()) + return {}; + else { + if (!i->is_string()) + throw Error("attribute '%s' of must be a string", name); + return i->get(); + } + } else { + auto i = env.find(name); + if (i == env.end()) + return {}; + else + return i->second; + } +} + +static bool getBoolAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name, bool def) +{ + if (parsed) { + auto i = parsed->structuredAttrs.find(name); + if (i == parsed->structuredAttrs.end()) + return def; + else { + if (!i->is_boolean()) + throw Error("attribute '%s' must be a Boolean", name); + return i->get(); + } + } else { + auto i = env.find(name); + if (i == env.end()) + return def; + else + return i->second == "1"; + } +} + +static std::optional +getStringsAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name) +{ + if (parsed) { + auto i = parsed->structuredAttrs.find(name); + if (i == parsed->structuredAttrs.end()) + return {}; + else { + if (!i->is_array()) + throw Error("attribute '%s' must be a list of strings", name); + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' must be a list of strings", name); + res.push_back(j->get()); + } + return res; + } + } else { + auto i = env.find(name); + if (i == env.end()) + return {}; + else + return tokenizeString(i->second); + } +} + +static std::optional +getStringSetAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name) +{ + auto ss = getStringsAttr(env, parsed, name); + return ss ? (std::optional{StringSet{ss->begin(), ss->end()}}) : (std::optional{}); +} + using OutputChecks = DerivationOptions::OutputChecks; using OutputChecksVariant = std::variant>; -DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation & parsed, bool shouldWarn) +DerivationOptions +DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn) { DerivationOptions defaults = {}; - auto structuredAttrs = parsed.structuredAttrs.get(); - - if (shouldWarn && structuredAttrs) { - if (get(*structuredAttrs, "allowedReferences")) { + if (shouldWarn && parsed) { + if (get(parsed->structuredAttrs, "allowedReferences")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead"); } - if (get(*structuredAttrs, "allowedRequisites")) { + if (get(parsed->structuredAttrs, "allowedRequisites")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead"); } - if (get(*structuredAttrs, "disallowedRequisites")) { + if (get(parsed->structuredAttrs, "disallowedRequisites")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead"); } - if (get(*structuredAttrs, "disallowedReferences")) { + if (get(parsed->structuredAttrs, "disallowedReferences")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead"); } - if (get(*structuredAttrs, "maxSize")) { + if (get(parsed->structuredAttrs, "maxSize")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead"); } - if (get(*structuredAttrs, "maxClosureSize")) { + if (get(parsed->structuredAttrs, "maxClosureSize")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead"); } @@ -50,9 +126,9 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation return { .outputChecks = [&]() -> OutputChecksVariant { - if (auto structuredAttrs = parsed.structuredAttrs.get()) { + if (parsed) { std::map res; - if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { + if (auto outputChecks = get(parsed->structuredAttrs, "outputChecks")) { for (auto & [outputName, output] : getObject(*outputChecks)) { OutputChecks checks; @@ -90,10 +166,10 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation return OutputChecks{ // legacy non-structured-attributes case .ignoreSelfRefs = true, - .allowedReferences = parsed.getStringSetAttr("allowedReferences"), - .disallowedReferences = parsed.getStringSetAttr("disallowedReferences").value_or(StringSet{}), - .allowedRequisites = parsed.getStringSetAttr("allowedRequisites"), - .disallowedRequisites = parsed.getStringSetAttr("disallowedRequisites").value_or(StringSet{}), + .allowedReferences = getStringSetAttr(env, parsed, "allowedReferences"), + .disallowedReferences = getStringSetAttr(env, parsed, "disallowedReferences").value_or(StringSet{}), + .allowedRequisites = getStringSetAttr(env, parsed, "allowedRequisites"), + .disallowedRequisites = getStringSetAttr(env, parsed, "disallowedRequisites").value_or(StringSet{}), }; } }(), @@ -101,8 +177,8 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation [&] { std::map res; - if (auto structuredAttrs = parsed.structuredAttrs.get()) { - if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { + if (parsed) { + if (auto udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) { for (auto & [outputName, output] : getObject(*udr)) { if (!output.is_boolean()) throw Error("attribute 'unsafeDiscardReferences.\"%s\"' must be a Boolean", outputName); @@ -116,8 +192,8 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation .passAsFile = [&] { StringSet res; - if (auto * passAsFileString = get(parsed.env, "passAsFile")) { - if (parsed.hasStructuredAttrs()) { + if (auto * passAsFileString = get(env, "passAsFile")) { + if (parsed) { if (shouldWarn) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'passAsFile'; because all JSON is always passed via file"); @@ -132,18 +208,18 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation [&] { std::map ret; - if (auto structuredAttrs = parsed.structuredAttrs.get()) { - auto e = optionalValueAt(*structuredAttrs, "exportReferencesGraph"); + if (parsed) { + auto e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph"); if (!e || !e->is_object()) return ret; for (auto & [key, storePathsJson] : getObject(*e)) { ret.insert_or_assign(key, storePathsJson); } } else { - auto s = getOr(parsed.env, "exportReferencesGraph", ""); + auto s = getOr(env, "exportReferencesGraph", ""); Strings ss = tokenizeString(s); if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); + throw Error("odd number of tokens in 'exportReferencesGraph': '%1%'", s); for (Strings::iterator i = ss.begin(); i != ss.end();) { auto fileName = std::move(*i++); static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); @@ -157,15 +233,15 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation return ret; }(), .additionalSandboxProfile = - parsed.getStringAttr("__sandboxProfile").value_or(defaults.additionalSandboxProfile), - .noChroot = parsed.getBoolAttr("__noChroot", defaults.noChroot), - .impureHostDeps = parsed.getStringSetAttr("__impureHostDeps").value_or(defaults.impureHostDeps), - .impureEnvVars = parsed.getStringSetAttr("impureEnvVars").value_or(defaults.impureEnvVars), - .allowLocalNetworking = parsed.getBoolAttr("__darwinAllowLocalNetworking", defaults.allowLocalNetworking), + getStringAttr(env, parsed, "__sandboxProfile").value_or(defaults.additionalSandboxProfile), + .noChroot = getBoolAttr(env, parsed, "__noChroot", defaults.noChroot), + .impureHostDeps = getStringSetAttr(env, parsed, "__impureHostDeps").value_or(defaults.impureHostDeps), + .impureEnvVars = getStringSetAttr(env, parsed, "impureEnvVars").value_or(defaults.impureEnvVars), + .allowLocalNetworking = getBoolAttr(env, parsed, "__darwinAllowLocalNetworking", defaults.allowLocalNetworking), .requiredSystemFeatures = - parsed.getStringSetAttr("requiredSystemFeatures").value_or(defaults.requiredSystemFeatures), - .preferLocalBuild = parsed.getBoolAttr("preferLocalBuild", defaults.preferLocalBuild), - .allowSubstitutes = parsed.getBoolAttr("allowSubstitutes", defaults.allowSubstitutes), + getStringSetAttr(env, parsed, "requiredSystemFeatures").value_or(defaults.requiredSystemFeatures), + .preferLocalBuild = getBoolAttr(env, parsed, "preferLocalBuild", defaults.preferLocalBuild), + .allowSubstitutes = getBoolAttr(env, parsed, "allowSubstitutes", defaults.allowSubstitutes), }; } diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index 3baf4babf..659cea444 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -2,6 +2,7 @@ ///@file #include "nix/store/parsed-derivations.hh" +#include "nix/store/derivations.hh" #include "nix/store/derivation-options.hh" #ifndef _WIN32 # include "nix/store/user-lock.hh" @@ -150,7 +151,7 @@ struct DerivationGoal : public Goal */ std::unique_ptr drv; - std::unique_ptr parsedDrv; + std::unique_ptr parsedDrv; std::unique_ptr drvOptions; /** diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index bc9ad3317..16730b5c9 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -13,16 +13,16 @@ namespace nix { class Store; struct BasicDerivation; -class ParsedDerivation; +struct StructuredAttrs; /** * This represents all the special options on a `Derivation`. * * Currently, these options are parsed from the environment variables - * with the aid of `ParsedDerivation`. + * with the aid of `StructuredAttrs`. * * The first goal of this data type is to make sure that no other code - * uses `ParsedDerivation` to ad-hoc parse some additional options. That + * uses `StructuredAttrs` to ad-hoc parse some additional options. That * ensures this data type is up to date and fully correct. * * The second goal of this data type is to allow an alternative to @@ -173,7 +173,8 @@ struct DerivationOptions * (e.g. JSON) but is necessary for supporing old formats (e.g. * ATerm). */ - static DerivationOptions fromParsedDerivation(const ParsedDerivation & parsed, bool shouldWarn = true); + static DerivationOptions + fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn = true); /** * @param drv Must be the same derivation we parsed this from. In diff --git a/src/libstore/include/nix/store/parsed-derivations.hh b/src/libstore/include/nix/store/parsed-derivations.hh index 30a294187..a7c053a8f 100644 --- a/src/libstore/include/nix/store/parsed-derivations.hh +++ b/src/libstore/include/nix/store/parsed-derivations.hh @@ -1,54 +1,43 @@ #pragma once ///@file -#include "nix/store/derivations.hh" -#include "nix/store/store-api.hh" +#include -#include +#include "nix/util/types.hh" +#include "nix/store/path.hh" namespace nix { +class Store; struct DerivationOptions; +struct DerivationOutput; -class ParsedDerivation +typedef std::map DerivationOutputs; + +struct StructuredAttrs { - const StringPairs & env; - std::unique_ptr structuredAttrs; + nlohmann::json structuredAttrs; - std::optional getStringAttr(const std::string & name) const; + static std::optional tryParse(const StringPairs & env); - bool getBoolAttr(const std::string & name, bool def = false) const; - - std::optional getStringsAttr(const std::string & name) const; - - std::optional getStringSetAttr(const std::string & name) const; - - /** - * Only `DerivationOptions` is allowed to parse individual fields - * from `ParsedDerivation`. This ensure that it includes all - * derivation options, and, the likes of `LocalDerivationGoal` are - * incapable of more ad-hoc options. - */ - friend struct DerivationOptions; - -public: - - ParsedDerivation(const StringPairs & env); - - ~ParsedDerivation(); - - bool hasStructuredAttrs() const - { - return static_cast(structuredAttrs); - } - - std::optional prepareStructuredAttrs( + nlohmann::json prepareStructuredAttrs( Store & store, const DerivationOptions & drvOptions, const StorePathSet & inputPaths, - const DerivationOutputs & outputs); + const DerivationOutputs & outputs) const; + + /** + * As a convenience to bash scripts, write a shell file that + * maps all attributes that are representable in bash - + * namely, strings, integers, nulls, Booleans, and arrays and + * objects consisting entirely of those values. (So nested + * arrays or objects are not supported.) + * + * @param prepared This should be the result of + * `prepareStructuredAttrs`, *not* the original `structuredAttrs` + * field. + */ + static std::string writeShell(const nlohmann::json & prepared); }; -std::string writeStructuredAttrsShell(const nlohmann::json & json); - } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index cfa82845d..c56b2cfe3 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -222,10 +222,12 @@ void Store::queryMissing(const std::vector & targets, if (knownOutputPaths && invalid.empty()) return; auto drv = make_ref(derivationFromPath(drvPath)); - ParsedDerivation parsedDrv(drv->env); + auto parsedDrv = StructuredAttrs::tryParse(drv->env); DerivationOptions drvOptions; try { - drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); + drvOptions = DerivationOptions::fromStructuredAttrs( + drv->env, + parsedDrv ? &*parsedDrv : nullptr); } catch (Error & e) { e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath)); throw; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 9aa16ff2c..61af101de 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,4 +1,6 @@ #include "nix/store/parsed-derivations.hh" +#include "nix/store/store-api.hh" +#include "nix/store/derivations.hh" #include "nix/store/derivation-options.hh" #include @@ -6,94 +8,20 @@ namespace nix { -ParsedDerivation::ParsedDerivation(const StringPairs & env) - : env(env) +std::optional StructuredAttrs::tryParse(const StringPairs & env) { /* Parse the __json attribute, if any. */ auto jsonAttr = env.find("__json"); if (jsonAttr != env.end()) { try { - structuredAttrs = std::make_unique(nlohmann::json::parse(jsonAttr->second)); + return StructuredAttrs { + .structuredAttrs = nlohmann::json::parse(jsonAttr->second), + }; } catch (std::exception & e) { throw Error("cannot process __json attribute: %s", e.what()); } } -} - -ParsedDerivation::~ParsedDerivation() { } - -std::optional ParsedDerivation::getStringAttr(const std::string & name) const -{ - if (structuredAttrs) { - auto i = structuredAttrs->find(name); - if (i == structuredAttrs->end()) - return {}; - else { - if (!i->is_string()) - throw Error("attribute '%s' of must be a string", name); - return i->get(); - } - } else { - auto i = env.find(name); - if (i == env.end()) - return {}; - else - return i->second; - } -} - -bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const -{ - if (structuredAttrs) { - auto i = structuredAttrs->find(name); - if (i == structuredAttrs->end()) - return def; - else { - if (!i->is_boolean()) - throw Error("attribute '%s' must be a Boolean", name); - return i->get(); - } - } else { - auto i = env.find(name); - if (i == env.end()) - return def; - else - return i->second == "1"; - } -} - -std::optional ParsedDerivation::getStringsAttr(const std::string & name) const -{ - if (structuredAttrs) { - auto i = structuredAttrs->find(name); - if (i == structuredAttrs->end()) - return {}; - else { - if (!i->is_array()) - throw Error("attribute '%s' must be a list of strings", name); - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' must be a list of strings", name); - res.push_back(j->get()); - } - return res; - } - } else { - auto i = env.find(name); - if (i == env.end()) - return {}; - else - return tokenizeString(i->second); - } -} - -std::optional ParsedDerivation::getStringSetAttr(const std::string & name) const -{ - auto ss = getStringsAttr(name); - return ss - ? (std::optional{StringSet{ss->begin(), ss->end()}}) - : (std::optional{}); + return {}; } static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); @@ -152,15 +80,14 @@ static nlohmann::json pathInfoToJSON( return jsonList; } -std::optional ParsedDerivation::prepareStructuredAttrs( +nlohmann::json StructuredAttrs::prepareStructuredAttrs( Store & store, const DerivationOptions & drvOptions, const StorePathSet & inputPaths, - const DerivationOutputs & outputs) + const DerivationOutputs & outputs) const { - if (!structuredAttrs) return std::nullopt; - - auto json = *structuredAttrs; + /* Copy to then modify */ + auto json = structuredAttrs; /* Add an "outputs" object containing the output paths. */ nlohmann::json outputsJson; @@ -180,12 +107,7 @@ std::optional ParsedDerivation::prepareStructuredAttrs( return json; } -/* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ -std::string writeStructuredAttrsShell(const nlohmann::json & json) +std::string StructuredAttrs::writeShell(const nlohmann::json & json) { auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 18300904e..7f921a23a 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -971,7 +971,7 @@ void LocalDerivationGoal::startBuilder() writeStructuredAttrs(); /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->hasStructuredAttrs()) { + if (!parsedDrv) { for (auto & [fileName, ss] : drvOptions->exportReferencesGraph) { StorePathSet storePathSet; for (auto & storePathS : ss) { @@ -1475,7 +1475,7 @@ void LocalDerivationGoal::initTmpDir() /* In non-structured mode, set all bindings either directory in the environment or via a file, as specified by `DerivationOptions::passAsFile`. */ - if (!parsedDrv->hasStructuredAttrs()) { + if (!parsedDrv) { for (auto & i : drv->env) { if (drvOptions->passAsFile.find(i.first) == drvOptions->passAsFile.end()) { env[i.first] = i.second; @@ -1576,13 +1576,12 @@ void LocalDerivationGoal::initEnv() void LocalDerivationGoal::writeStructuredAttrs() { - if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs( + if (parsedDrv) { + auto json = parsedDrv->prepareStructuredAttrs( worker.store, *drvOptions, inputPaths, - drv->outputs)) - { - auto json = structAttrsJson.value(); + drv->outputs); nlohmann::json rewritten; for (auto & [i, v] : json["outputs"].get()) { /* The placeholder must have a rewrite, so we use it to cover both the @@ -1592,7 +1591,7 @@ void LocalDerivationGoal::writeStructuredAttrs() json["outputs"] = rewritten; - auto jsonSh = writeStructuredAttrsShell(json); + auto jsonSh = StructuredAttrs::writeShell(json); writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); chownToBuilder(tmpDir + "/.attrs.sh"); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 7ace4d4b4..6815e0cdd 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -544,10 +544,12 @@ static void main_nix_build(int argc, char * * argv) env["NIX_STORE"] = store->storeDir; env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); - ParsedDerivation parsedDrv(drv.env); + auto parsedDrv = StructuredAttrs::tryParse(drv.env); DerivationOptions drvOptions; try { - drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); + drvOptions = DerivationOptions::fromStructuredAttrs( + drv.env, + parsedDrv ? &*parsedDrv : nullptr); } catch (Error & e) { e.addTrace({}, "while parsing derivation '%s'", store->printStorePath(packageInfo.requireDrvPath())); throw; @@ -566,7 +568,7 @@ static void main_nix_build(int argc, char * * argv) std::string structuredAttrsRC; - if (parsedDrv.hasStructuredAttrs()) { + if (parsedDrv) { StorePathSet inputs; std::function::ChildNode &)> accumInputClosure; @@ -584,24 +586,22 @@ static void main_nix_build(int argc, char * * argv) for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) accumInputClosure(inputDrv, inputNode); - if (auto structAttrs = parsedDrv.prepareStructuredAttrs( - *store, - drvOptions, - inputs, - drv.outputs)) - { - auto json = structAttrs.value(); - structuredAttrsRC = writeStructuredAttrsShell(json); + auto json = parsedDrv->prepareStructuredAttrs( + *store, + drvOptions, + inputs, + drv.outputs); - auto attrsJSON = (tmpDir.path() / ".attrs.json").string(); - writeFile(attrsJSON, json.dump()); + structuredAttrsRC = StructuredAttrs::writeShell(json); - auto attrsSH = (tmpDir.path() / ".attrs.sh").string(); - writeFile(attrsSH, structuredAttrsRC); + auto attrsJSON = (tmpDir.path() / ".attrs.json").string(); + writeFile(attrsJSON, json.dump()); - env["NIX_ATTRS_SH_FILE"] = attrsSH; - env["NIX_ATTRS_JSON_FILE"] = attrsJSON; - } + auto attrsSH = (tmpDir.path() / ".attrs.sh").string(); + writeFile(attrsSH, structuredAttrsRC); + + env["NIX_ATTRS_SH_FILE"] = attrsSH; + env["NIX_ATTRS_JSON_FILE"] = attrsJSON; } /* Run a shell using the derivation's environment. For From 674375b021ce9e229e575204395357f8d317bef5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 14 Apr 2025 14:09:30 +0200 Subject: [PATCH 226/396] call-flake.nix: refactor: Bring mapAttrs into scope --- src/libflake/call-flake.nix | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libflake/call-flake.nix b/src/libflake/call-flake.nix index 1e9e21048..03a52c87c 100644 --- a/src/libflake/call-flake.nix +++ b/src/libflake/call-flake.nix @@ -14,6 +14,7 @@ overrides: fetchTreeFinal: let + inherit (builtins) mapAttrs; lockFile = builtins.fromJSON lockFileStr; @@ -35,7 +36,7 @@ let (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path}) (builtins.tail path); - allNodes = builtins.mapAttrs ( + allNodes = mapAttrs ( key: node: let @@ -60,9 +61,7 @@ let flake = import (outPath + "/flake.nix"); - inputs = builtins.mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}) ( - node.inputs or { } - ); + inputs = mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}) (node.inputs or { }); outputs = flake.outputs (inputs // { self = result; }); From 9de9410f295a3daf5c97ea9fcbdcb0d3c5aafd5d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 15 Apr 2025 09:10:18 +0200 Subject: [PATCH 227/396] call-flake.nix: allNodes.${key} -> allNodes.${key}.result --- src/libflake/call-flake.nix | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/libflake/call-flake.nix b/src/libflake/call-flake.nix index 03a52c87c..430dfabdd 100644 --- a/src/libflake/call-flake.nix +++ b/src/libflake/call-flake.nix @@ -48,7 +48,7 @@ let else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" then parentNode.sourceInfo // { - outPath = parentNode.outPath + ("/" + node.locked.path); + outPath = parentNode.result.outPath + ("/" + node.locked.path); } else # FIXME: remove obsolete node.info. @@ -61,7 +61,9 @@ let flake = import (outPath + "/flake.nix"); - inputs = mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}) (node.inputs or { }); + inputs = mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}.result) ( + node.inputs or { } + ); outputs = flake.outputs (inputs // { self = result; }); @@ -84,12 +86,15 @@ let }; in - if node.flake or true then - assert builtins.isFunction flake.outputs; - result - else - sourceInfo + { + result = + if node.flake or true then + assert builtins.isFunction flake.outputs; + result + else + sourceInfo; + } ) lockFile.nodes; in -allNodes.${lockFile.root} +allNodes.${lockFile.root}.result From 2109a5a2066d0d49a1bcc5b44b2a4d84b5d313bd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 15 Apr 2025 09:28:23 +0200 Subject: [PATCH 228/396] fix: Evaluate flake parent source without evaluating its outputs This requires that we refer to the `sourceInfo` instead of the `result`. However, `sourceInfo` does not create a chain of basedir resolution, so we add that back with `flakeDir`. --- src/libflake/call-flake.nix | 11 ++++++++++- tests/functional/flakes/relative-paths.sh | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/libflake/call-flake.nix b/src/libflake/call-flake.nix index 430dfabdd..fe326291f 100644 --- a/src/libflake/call-flake.nix +++ b/src/libflake/call-flake.nix @@ -42,13 +42,20 @@ let parentNode = allNodes.${getInputByPath lockFile.root node.parent}; + flakeDir = + let + dir = overrides.${key}.dir or node.locked.path or ""; + parentDir = parentNode.flakeDir; + in + if node ? parent then parentDir + ("/" + dir) else dir; + sourceInfo = if overrides ? ${key} then overrides.${key}.sourceInfo else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" then parentNode.sourceInfo // { - outPath = parentNode.result.outPath + ("/" + node.locked.path); + outPath = parentNode.sourceInfo.outPath + ("/" + flakeDir); } else # FIXME: remove obsolete node.info. @@ -93,6 +100,8 @@ let result else sourceInfo; + + inherit flakeDir sourceInfo; } ) lockFile.nodes; diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 3f7ca3f46..4648ba98c 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -108,3 +108,24 @@ EOF [[ $(nix eval "$rootFlake#z") = 90 ]] fi + +# https://github.com/NixOS/nix/pull/10089#discussion_r2041984987 +# https://github.com/NixOS/nix/issues/13018 +mkdir -p "$TEST_ROOT/issue-13018/example" +( + cd "$TEST_ROOT/issue-13018" + git init + echo '{ outputs = _: { }; }' >flake.nix + cat >example/flake.nix < Date: Tue, 15 Apr 2025 11:53:17 -0400 Subject: [PATCH 229/396] Derivation "advanced attrs" test: Ensure fields are set to distinct values We had fields set to the same values before in our test data. This is not a problem per-se, but does mean we wouldn't catch certain mixups. Now, the fields are set to distinct values (where possible), which makes the test more robust. --- .../advanced-attributes-structured-attrs.json | 10 ++-- .../derivation/ca/advanced-attributes.json | 18 ++++--- .../advanced-attributes-structured-attrs.json | 22 ++++---- .../derivation/ia/advanced-attributes.json | 22 ++++---- .../derivation-advanced-attrs.cc | 51 ++++++++++--------- .../advanced-attributes-structured-attrs.nix | 12 ++++- .../derivation/advanced-attributes.nix | 12 ++++- .../advanced-attributes-structured-attrs.drv | 2 +- .../derivation/ca/advanced-attributes.drv | 2 +- .../advanced-attributes-structured-attrs.drv | 2 +- .../derivation/ia/advanced-attributes.drv | 2 +- 11 files changed, 91 insertions(+), 64 deletions(-) diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json index ddc887633..a421efea7 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -5,27 +5,29 @@ ], "builder": "/bin/bash", "env": { - "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"],\"refs2\":[\"/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"],\"disallowedRequisites\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"],\"allowedRequisites\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"refs2\":[\"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", "bin": "/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m", "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" }, "inputDrvs": { - "/nix/store/spfzlnkwb1v8s62yvh8vj1apd1kwjr5f-foo.drv": { + "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { "dynamicOutputs": {}, "outputs": [ + "dev", "out" ] }, - "/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv": { + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { "dynamicOutputs": {}, "outputs": [ + "dev", "out" ] } }, "inputSrcs": [ - "/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv" + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" ], "name": "advanced-attributes-structured-attrs", "outputs": { diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.json b/src/libstore-tests/data/derivation/ca/advanced-attributes.json index 57a1141b5..0ac0a9c5c 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.json @@ -10,12 +10,12 @@ "__noChroot": "1", "__sandboxProfile": "sandcastle", "allowSubstitutes": "", - "allowedReferences": "/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8", - "allowedRequisites": "/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8", + "allowedReferences": "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9", + "allowedRequisites": "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z", "builder": "/bin/bash", - "disallowedReferences": "/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99", - "disallowedRequisites": "/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99", - "exportReferencesGraph": "refs1 /08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8 refs2 /nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv", + "disallowedReferences": "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g", + "disallowedRequisites": "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8", + "exportReferencesGraph": "refs1 /164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9 refs2 /nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv", "impureEnvVars": "UNICORN", "name": "advanced-attributes", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9", @@ -26,21 +26,23 @@ "system": "my-system" }, "inputDrvs": { - "/nix/store/spfzlnkwb1v8s62yvh8vj1apd1kwjr5f-foo.drv": { + "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { "dynamicOutputs": {}, "outputs": [ + "dev", "out" ] }, - "/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv": { + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { "dynamicOutputs": {}, "outputs": [ + "dev", "out" ] } }, "inputSrcs": [ - "/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv" + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" ], "name": "advanced-attributes", "outputs": { diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json index 2ba83c21c..d68502d56 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json @@ -5,38 +5,40 @@ ], "builder": "/bin/bash", "env": { - "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"refs2\":[\"/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", - "bin": "/nix/store/spb9y9agq61rvq29fhkfw1ql00adjq7d-advanced-attributes-structured-attrs-bin", - "dev": "/nix/store/0v889x74f5d5swbjivcik2yw4gg2pm52-advanced-attributes-structured-attrs-dev", - "out": "/nix/store/vnzmd26f5hx5dp5xrfs2kshqvcpxhrhh-advanced-attributes-structured-attrs" + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"refs2\":[\"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "bin": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin", + "dev": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev", + "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" }, "inputDrvs": { - "/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv": { + "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { "dynamicOutputs": {}, "outputs": [ + "dev", "out" ] }, - "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv": { + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { "dynamicOutputs": {}, "outputs": [ + "dev", "out" ] } }, "inputSrcs": [ - "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv" + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { - "path": "/nix/store/spb9y9agq61rvq29fhkfw1ql00adjq7d-advanced-attributes-structured-attrs-bin" + "path": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin" }, "dev": { - "path": "/nix/store/0v889x74f5d5swbjivcik2yw4gg2pm52-advanced-attributes-structured-attrs-dev" + "path": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev" }, "out": { - "path": "/nix/store/vnzmd26f5hx5dp5xrfs2kshqvcpxhrhh-advanced-attributes-structured-attrs" + "path": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" } }, "system": "my-system" diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.json b/src/libstore-tests/data/derivation/ia/advanced-attributes.json index 06ae314d7..20ce5e1c2 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.json @@ -10,40 +10,42 @@ "__noChroot": "1", "__sandboxProfile": "sandcastle", "allowSubstitutes": "", - "allowedReferences": "/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo", - "allowedRequisites": "/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo", + "allowedReferences": "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo", + "allowedRequisites": "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev", "builder": "/bin/bash", - "disallowedReferences": "/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar", - "disallowedRequisites": "/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar", - "exportReferencesGraph": "refs1 /nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo refs2 /nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv", + "disallowedReferences": "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar", + "disallowedRequisites": "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev", + "exportReferencesGraph": "refs1 /nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo refs2 /nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv", "impureEnvVars": "UNICORN", "name": "advanced-attributes", - "out": "/nix/store/jvm2xsx0lm29byzr59yzjw7c14fa9z5f-advanced-attributes", + "out": "/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes", "preferLocalBuild": "1", "requiredSystemFeatures": "rainbow uid-range", "system": "my-system" }, "inputDrvs": { - "/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv": { + "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { "dynamicOutputs": {}, "outputs": [ + "dev", "out" ] }, - "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv": { + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { "dynamicOutputs": {}, "outputs": [ + "dev", "out" ] } }, "inputSrcs": [ - "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv" + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" ], "name": "advanced-attributes", "outputs": { "out": { - "path": "/nix/store/jvm2xsx0lm29byzr59yzjw7c14fa9z5f-advanced-attributes" + "path": "/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes" } }, "system": "my-system" diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 4408720e8..1144a9351 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -204,13 +204,13 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes) { "refs1", { - "/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo", + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo", }, }, { "refs2", { - "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv", + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv", }, }, })); @@ -221,13 +221,15 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes) auto & checksForAllOutputs = *checksForAllOutputs_; EXPECT_EQ( - checksForAllOutputs.allowedReferences, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + checksForAllOutputs.allowedReferences, StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}); EXPECT_EQ( - checksForAllOutputs.allowedRequisites, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + checksForAllOutputs.allowedRequisites, + StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}); EXPECT_EQ( - checksForAllOutputs.disallowedReferences, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + checksForAllOutputs.disallowedReferences, StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"}); EXPECT_EQ( - checksForAllOutputs.disallowedRequisites, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + checksForAllOutputs.disallowedRequisites, + StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}); } StringSet systemFeatures{"rainbow", "uid-range"}; @@ -252,13 +254,13 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) { "refs1", { - "/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8", + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9", }, }, { "refs2", { - "/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv", + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv", }, }, })); @@ -270,16 +272,16 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) EXPECT_EQ( checksForAllOutputs.allowedReferences, - StringSet{"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"}); + StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}); EXPECT_EQ( checksForAllOutputs.allowedRequisites, - StringSet{"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"}); + StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"}); EXPECT_EQ( checksForAllOutputs.disallowedReferences, - StringSet{"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"}); + StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"}); EXPECT_EQ( checksForAllOutputs.disallowedRequisites, - StringSet{"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"}); + StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"}); } StringSet systemFeatures{"rainbow", "uid-range"}; @@ -401,13 +403,13 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) { "refs1", { - "/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo", + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo", }, }, { "refs2", { - "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv", + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv", }, }, })); @@ -418,8 +420,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) ASSERT_TRUE(output_); auto & output = *output_; - EXPECT_EQ(output.allowedReferences, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); - EXPECT_EQ(output.allowedRequisites, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ(output.allowedReferences, StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}); + EXPECT_EQ(output.allowedRequisites, StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}); } { @@ -427,8 +429,9 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) ASSERT_TRUE(output_); auto & output = *output_; - EXPECT_EQ(output.disallowedReferences, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); - EXPECT_EQ(output.disallowedRequisites, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ(output.disallowedReferences, StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"}); + EXPECT_EQ( + output.disallowedRequisites, StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}); } } @@ -454,13 +457,13 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) { "refs1", { - "/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8", + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9", }, }, { "refs2", { - "/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv", + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv", }, }, })); @@ -471,8 +474,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) ASSERT_TRUE(output_); auto & output = *output_; - EXPECT_EQ(output.allowedReferences, StringSet{"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"}); - EXPECT_EQ(output.allowedRequisites, StringSet{"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"}); + EXPECT_EQ(output.allowedReferences, StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}); + EXPECT_EQ(output.allowedRequisites, StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"}); } { @@ -481,9 +484,9 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) auto & output = *output_; EXPECT_EQ( - output.disallowedReferences, StringSet{"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"}); + output.disallowedReferences, StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"}); EXPECT_EQ( - output.disallowedRequisites, StringSet{"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"}); + output.disallowedRequisites, StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"}); } } diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.nix b/tests/functional/derivation/advanced-attributes-structured-attrs.nix index 60bbee61c..46f619272 100644 --- a/tests/functional/derivation/advanced-attributes-structured-attrs.nix +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.nix @@ -23,6 +23,10 @@ let "-c" "echo foo > $out" ]; + outputs = [ + "out" + "dev" + ]; }; bar = derivation' { @@ -33,6 +37,10 @@ let "-c" "echo bar > $out" ]; + outputs = [ + "out" + "dev" + ]; }; in @@ -58,11 +66,11 @@ derivation' { outputChecks = { out = { allowedReferences = [ foo ]; - allowedRequisites = [ foo ]; + allowedRequisites = [ foo.dev ]; }; bin = { disallowedReferences = [ bar ]; - disallowedRequisites = [ bar ]; + disallowedRequisites = [ bar.dev ]; }; dev = { maxSize = 789; diff --git a/tests/functional/derivation/advanced-attributes.nix b/tests/functional/derivation/advanced-attributes.nix index 09d5c33c2..dd0c09e22 100644 --- a/tests/functional/derivation/advanced-attributes.nix +++ b/tests/functional/derivation/advanced-attributes.nix @@ -23,6 +23,10 @@ let "-c" "echo foo > $out" ]; + outputs = [ + "out" + "dev" + ]; }; bar = derivation' { @@ -33,6 +37,10 @@ let "-c" "echo bar > $out" ]; + outputs = [ + "out" + "dev" + ]; }; in @@ -50,9 +58,9 @@ derivation' { impureEnvVars = [ "UNICORN" ]; __darwinAllowLocalNetworking = true; allowedReferences = [ foo ]; - allowedRequisites = [ foo ]; + allowedRequisites = [ foo.dev ]; disallowedReferences = [ bar ]; - disallowedRequisites = [ bar ]; + disallowedRequisites = [ bar.dev ]; requiredSystemFeatures = [ "rainbow" "uid-range" diff --git a/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv index b22fbdb2d..cd02c2f86 100644 --- a/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv +++ b/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv @@ -1 +1 @@ -Derive([("bin","","r:sha256",""),("dev","","r:sha256",""),("out","","r:sha256","")],[("/nix/store/spfzlnkwb1v8s62yvh8vj1apd1kwjr5f-foo.drv",["out"]),("/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv",["out"])],["/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"],\"refs2\":[\"/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"],\"disallowedRequisites\":[\"/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"],\"allowedRequisites\":[\"/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m"),("dev","/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9")]) \ No newline at end of file +Derive([("bin","","r:sha256",""),("dev","","r:sha256",""),("out","","r:sha256","")],[("/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",["dev","out"]),("/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",["dev","out"])],["/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"refs2\":[\"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m"),("dev","/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9")]) \ No newline at end of file diff --git a/tests/functional/derivation/ca/advanced-attributes.drv b/tests/functional/derivation/ca/advanced-attributes.drv index 93996c15c..068cb593e 100644 --- a/tests/functional/derivation/ca/advanced-attributes.drv +++ b/tests/functional/derivation/ca/advanced-attributes.drv @@ -1 +1 @@ -Derive([("out","","r:sha256","")],[("/nix/store/spfzlnkwb1v8s62yvh8vj1apd1kwjr5f-foo.drv",["out"]),("/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv",["out"])],["/nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"),("allowedRequisites","/08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8"),("builder","/bin/bash"),("disallowedReferences","/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"),("disallowedRequisites","/05pdic30acaypbz73ivw4wlsi9whq08jxsimml2h0inwqya2hn99"),("exportReferencesGraph","refs1 /08cr1k2yfw44g21w1h850285vqhsciy7y3siqjdzz1m9yvwlqfm8 refs2 /nix/store/x1vpzav565aqr7ccmkn0wv0svkm1qrbl-bar.drv"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"),("outputHashAlgo","sha256"),("outputHashMode","recursive"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file +Derive([("out","","r:sha256","")],[("/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",["dev","out"]),("/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",["dev","out"])],["/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"),("allowedRequisites","/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"),("builder","/bin/bash"),("disallowedReferences","/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"),("disallowedRequisites","/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"),("exportReferencesGraph","refs1 /164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9 refs2 /nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"),("outputHashAlgo","sha256"),("outputHashMode","recursive"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv index b9359bbbd..1dfcac42d 100644 --- a/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv +++ b/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv @@ -1 +1 @@ -Derive([("bin","/nix/store/spb9y9agq61rvq29fhkfw1ql00adjq7d-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/0v889x74f5d5swbjivcik2yw4gg2pm52-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/vnzmd26f5hx5dp5xrfs2kshqvcpxhrhh-advanced-attributes-structured-attrs","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],["/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"refs2\":[\"/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/spb9y9agq61rvq29fhkfw1ql00adjq7d-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/0v889x74f5d5swbjivcik2yw4gg2pm52-advanced-attributes-structured-attrs-dev"),("out","/nix/store/vnzmd26f5hx5dp5xrfs2kshqvcpxhrhh-advanced-attributes-structured-attrs")]) \ No newline at end of file +Derive([("bin","/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs","","")],[("/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv",["dev","out"]),("/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",["dev","out"])],["/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"refs2\":[\"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev"),("out","/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs")]) \ No newline at end of file diff --git a/tests/functional/derivation/ia/advanced-attributes.drv b/tests/functional/derivation/ia/advanced-attributes.drv index 8e69b8c68..c71a88886 100644 --- a/tests/functional/derivation/ia/advanced-attributes.drv +++ b/tests/functional/derivation/ia/advanced-attributes.drv @@ -1 +1 @@ -Derive([("out","/nix/store/jvm2xsx0lm29byzr59yzjw7c14fa9z5f-advanced-attributes","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],["/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("allowedRequisites","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("builder","/bin/bash"),("disallowedReferences","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("disallowedRequisites","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("exportReferencesGraph","refs1 /nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo refs2 /nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/jvm2xsx0lm29byzr59yzjw7c14fa9z5f-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file +Derive([("out","/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes","","")],[("/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv",["dev","out"]),("/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",["dev","out"])],["/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"),("allowedRequisites","/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"),("builder","/bin/bash"),("disallowedReferences","/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"),("disallowedRequisites","/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"),("exportReferencesGraph","refs1 /nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo refs2 /nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file From 32409dd7d750576153657beb075bb303840c0c3a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 15 Apr 2025 11:54:11 -0400 Subject: [PATCH 230/396] Remove stray assignment side affect in lambda This was almost a bug! It wasn't simply because another assignment would clobber it later. --- src/libstore/derivation-options.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 526e682f7..ff777fbae 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -70,7 +70,6 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation throw Error("attribute '%s' must be a list of strings", name); res.insert(j->get()); } - checks.disallowedRequisites = res; return res; } return {}; From e83ef7a477b7be090145a57c87b1ca514a685d09 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 16 Mar 2025 16:48:41 -0400 Subject: [PATCH 231/396] Make `appendLogTailErrorMsg` as class method after all The other parameters it took were somewhat implementation-specific. --- src/libstore/build/derivation-goal.cc | 10 +++------- .../include/nix/store/build/derivation-goal.hh | 9 ++------- src/libstore/unix/build/local-derivation-goal.cc | 2 +- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1a3d23c48..6f02ee314 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -879,11 +879,7 @@ void runPostBuildHook( } -void appendLogTailErrorMsg( - const Store & store, - const StorePath & drvPath, - const std::list & logTail, - std::string & msg) +void DerivationGoal::appendLogTailErrorMsg(std::string & msg) { if (!logger->isVerbose() && !logTail.empty()) { msg += fmt(";\nlast %d log lines:\n", logTail.size()); @@ -900,7 +896,7 @@ void appendLogTailErrorMsg( // command will not put it at the start of the line unfortunately. msg += fmt("For full logs, run:\n " ANSI_BOLD "%s %s" ANSI_NORMAL, nixLogCommand, - store.printStorePath(drvPath)); + worker.store.printStorePath(drvPath)); } } @@ -947,7 +943,7 @@ Goal::Co DerivationGoal::hookDone() Magenta(worker.store.printStorePath(drvPath)), statusToString(status)); - appendLogTailErrorMsg(worker.store, drvPath, logTail, msg); + appendLogTailErrorMsg(msg); outputLocks.unlock(); diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index 659cea444..23675690d 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -63,13 +63,6 @@ void runPostBuildHook( const StorePath & drvPath, const StorePathSet & outputPaths); -/** Used internally */ -void appendLogTailErrorMsg( - const Store & store, - const StorePath & drvPath, - const std::list & logTail, - std::string & msg); - /** * A goal for building some or all of the outputs of a derivation. */ @@ -306,6 +299,8 @@ struct DerivationGoal : public Goal SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); + void appendLogTailErrorMsg(std::string & msg); + StorePathSet exportReferences(const StorePathSet & storePaths); JobCategory jobCategory() const override { diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 7f921a23a..859056f10 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -623,7 +623,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() Magenta(worker.store.printStorePath(drvPath)), statusToString(status)); - appendLogTailErrorMsg(worker.store, drvPath, logTail, msg); + appendLogTailErrorMsg(msg); if (diskFull) msg += "\nnote: build failure may have been caused by lack of free disk space"; From ae7f411a18697d5e9e00e1271df733b5b3f581df Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 16 Apr 2025 17:39:22 -0400 Subject: [PATCH 232/396] Remove some unused includes This is unreleated to the other commits in this PR. --- src/libstore/build/derivation-goal.cc | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6f02ee314..b19b0efeb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -5,31 +5,17 @@ #include "nix/util/processes.hh" #include "nix/util/config-global.hh" #include "nix/store/build/worker.hh" -#include "nix/store/builtins.hh" -#include "nix/store/builtins/buildenv.hh" -#include "nix/util/references.hh" -#include "nix/util/finally.hh" #include "nix/util/util.hh" -#include "nix/util/archive.hh" #include "nix/util/compression.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" -#include "nix/util/topo-sort.hh" -#include "nix/util/callback.hh" #include "nix/store/local-store.hh" // TODO remove, along with remaining downcasts -#include -#include - #include #include #include #include -#ifndef _WIN32 // TODO abstract over proc exit status -# include -#endif - #include #include "nix/util/strings.hh" From f81c06accfd58ff1649877d25b74ed89c993e5d4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 14 Mar 2025 15:37:59 -0400 Subject: [PATCH 233/396] Gut `LocalDerivationGoal::tryLocalBuild` Now, most of it is in two new functions: `LocalDerivationGoal::{,un}repareBuild`. This might seems like a step backwards from coroutines --- now we have more functions, and are stuck with class vars --- but I don't think it needs to be. There's a few options here: - (Re)introduce coroutines for the isolated building logic. We could use the same coroutines types, or simpler ones specialized to this use-case. The `tryLocalBuild` caller can still use `Goal::Co`, and just will manually "pump" this inner coroutine. - Return closures from each step. This is sort of like coroutines by hand, but it still allows us to stop writing down the local variables in each type. Being able to fully-use RAII again would be very nice! - Keep top-level first-order functions like now, but make more functional. Instead of having one state object (`DerivationBuilder`) for all steps (setup, run, teardown), we can have separate structs for the live variables at each point we consume and return. This at least avoids "are these variables active at this time?" questions, but doesn't give us the full benefit of RAII as we must manually ensure FIFO create/destroy orders still. One thing to note is that by keeping the `outputLock` unlocking in `tryLocalBuild`, we are arguably uncovering a rebuild scheduling vs building distinction, as the output locks are pretty squarely a scheduling concern. It's nice that the builder doesn't need to know about them at all. --- .../unix/build/local-derivation-goal.cc | 109 ++++++++++++------ 1 file changed, 76 insertions(+), 33 deletions(-) diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 859056f10..8c74ed4e1 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -252,11 +252,32 @@ struct LocalDerivationGoal : DerivationGoal, RestrictionContext */ Goal::Co tryLocalBuild() override; + /** + * Set up build environment / sandbox, acquiring resources (e.g. + * locks as needed). After this is run, the builder should be + * started. + * + * @returns true if successful, false if we could not acquire a build + * user. In that case, the caller must wait and then try again. + */ + bool prepareBuild(); + /** * Start building a derivation. */ void startBuilder(); + /** + * Tear down build environment after the builder exits (either on + * its own or if it is killed). + * + * @returns The first case indicates failure during output + * processing. A status code and exception are returned, providing + * more information. The second case indicates success, and + * realisations for each output of the derivation are returned. + */ + std::variant, SingleDrvOutputs> unprepareBuild(); + /** * Fill in the environment for the builder. */ @@ -488,6 +509,53 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() co_return tryToBuild(); } + if (!prepareBuild()) { + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); + co_await waitForAWhile(); + co_return tryLocalBuild(); + } + + actLock.reset(); + + try { + + /* Okay, we have to build. */ + startBuilder(); + + } catch (BuildError & e) { + outputLocks.unlock(); + buildUser.reset(); + worker.permanentFailure = true; + co_return done(BuildResult::InputRejected, {}, std::move(e)); + } + + started(); + co_await Suspend{}; + + trace("build done"); + + auto res = unprepareBuild(); + // N.B. cannot use `std::visit` with co-routine return + if (auto * ste = std::get_if<0>(&res)) { + outputLocks.unlock(); + co_return done(std::move(ste->first), {}, std::move(ste->second)); + } else if (auto * builtOutputs = std::get_if<1>(&res)) { + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + co_return done(BuildResult::Built, std::move(*builtOutputs)); + } else { + unreachable(); + } +} + +bool LocalDerivationGoal::prepareBuild() +{ /* Cache this */ derivationType = drv->type(); @@ -535,33 +603,16 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() buildUser = acquireUserLock(drvOptions->useUidRange(*drv) ? 65536 : 1, useChroot); if (!buildUser) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); - co_await waitForAWhile(); - co_return tryLocalBuild(); + return false; } } - actLock.reset(); + return true; +} - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - co_return done(BuildResult::InputRejected, {}, std::move(e)); - } - - started(); - co_await Suspend{}; - - trace("build done"); +std::variant, SingleDrvOutputs> LocalDerivationGoal::unprepareBuild() +{ Finally releaseBuildUser([&](){ /* Release the build user at the end of this function. We don't do it right away because we don't want another build grabbing this @@ -654,18 +705,9 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() deleteTmpDir(true); - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - co_return done(BuildResult::Built, std::move(builtOutputs)); + return std::move(builtOutputs); } catch (BuildError & e) { - outputLocks.unlock(); - assert(derivationType); BuildResult::Status st = dynamic_cast(&e) ? BuildResult::NotDeterministic : @@ -673,10 +715,11 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() !derivationType->isSandboxed() || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; - co_return done(st, {}, std::move(e)); + return std::pair{std::move(st), std::move(e)}; } } + static void chmod_(const Path & path, mode_t mode) { if (chmod(path.c_str(), mode) == -1) From d98c0dbe99464d5fe440ffd3f5d5a2ad55b5716a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 14 Mar 2025 04:40:02 -0400 Subject: [PATCH 234/396] Start separating scheduling from building We have a new `DerivationBuilder` struct, and `DerivationBuilderParams` `DerivationBuilderCallbacks` supporting it. `LocalDerivationGoal` doesn't subclass any of these, so we are ready to now move them out to a new file! --- .../unix/build/local-derivation-goal.cc | 553 +++++++++++++----- 1 file changed, 399 insertions(+), 154 deletions(-) diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 8c74ed4e1..973eef26e 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -80,9 +80,136 @@ extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, namespace nix { -struct LocalDerivationGoal : DerivationGoal, RestrictionContext +/** + * Parameters by (mostly) `const` reference for `DerivationBuilder`. + */ +struct DerivationBuilderParams { - LocalStore & getLocalStore(); + /** The path of the derivation. */ + const StorePath & drvPath; + + BuildResult & buildResult; + + /** + * The derivation stored at drvPath. + * + * @todo Remove double indirection by delaying when this is + * initialized. + */ + const std::unique_ptr & drv; + + const std::unique_ptr & parsedDrv; + const std::unique_ptr & drvOptions; + + // The remainder is state held during the build. + + /** + * All input paths (that is, the union of FS closures of the + * immediate input paths). + */ + const StorePathSet & inputPaths; + + /** + * @note we do in fact mutate this + */ + std::map & initialOutputs; + + const BuildMode & buildMode; + + DerivationBuilderParams( + const StorePath & drvPath, + const BuildMode & buildMode, + BuildResult & buildResult, + const std::unique_ptr & drv, + const std::unique_ptr & parsedDrv, + const std::unique_ptr & drvOptions, + const StorePathSet & inputPaths, + std::map & initialOutputs) + : drvPath{drvPath} + , buildResult{buildResult} + , drv{drv} + , parsedDrv{parsedDrv} + , drvOptions{drvOptions} + , inputPaths{inputPaths} + , initialOutputs{initialOutputs} + , buildMode{buildMode} + { } + + DerivationBuilderParams(DerivationBuilderParams &&) = default; +}; + +/** + * Callbacks that `DerivationBuilder` needs. + */ +struct DerivationBuilderCallbacks +{ + /** + * Open a log file and a pipe to it. + */ + virtual Path openLogFile() = 0; + + /** + * Close the log file. + */ + virtual void closeLogFile() = 0; + + /** + * Aborts if any output is not valid or corrupt, and otherwise + * returns a 'SingleDrvOutputs' structure containing all outputs. + * + * @todo Probably should just be in `DerivationGoal`. + */ + virtual SingleDrvOutputs assertPathValidity() = 0; + + virtual void appendLogTailErrorMsg(std::string & msg) = 0; + + /** + * Hook up `builderOut` to some mechanism to ingest the log + * + * @todo this should be reworked + */ + virtual void childStarted() = 0; + + /** + * @todo this should be reworked + */ + virtual void childTerminated() = 0; + + virtual void noteHashMismatch(void) = 0; + virtual void noteCheckMismatch(void) = 0; + + virtual void markContentsGood(const StorePath & path) = 0; +}; + +/** + * This class represents the state for building locally. + * + * @todo Ideally, it would not be a class, but a single function. + * However, besides the main entry point, there are a few more methods + * which are externally called, and need to be gotten rid of. There are + * also some virtual methods (either directly here or inherited from + * `DerivationBuilderCallbacks`, a stop-gap) that represent outgoing + * rather than incoming call edges that either should be removed, or + * become (higher order) function parameters. + */ +class DerivationBuilder : public RestrictionContext, DerivationBuilderParams +{ + Store & store; + + DerivationBuilderCallbacks & miscMethods; + +public: + + DerivationBuilder( + Store & store, + DerivationBuilderCallbacks & miscMethods, + DerivationBuilderParams params) + : DerivationBuilderParams{std::move(params)} + , store{store} + , miscMethods{miscMethods} + { } + + LocalStore & getLocalStore(); /** * User selected for running the builder. @@ -94,6 +221,8 @@ struct LocalDerivationGoal : DerivationGoal, RestrictionContext */ Pid pid; +private: + /** * The cgroup of the builder, if any. */ @@ -115,12 +244,16 @@ struct LocalDerivationGoal : DerivationGoal, RestrictionContext */ Path tmpDirInSandbox; +public: + /** * Master side of the pseudoterminal used for the builder's * standard output/error. */ AutoCloseFD builderOut; +private: + /** * Pipe for synchronising updates to the builder namespaces. */ @@ -238,19 +371,12 @@ struct LocalDerivationGoal : DerivationGoal, RestrictionContext friend struct RestrictedStore; - using DerivationGoal::DerivationGoal; - - virtual ~LocalDerivationGoal() override; - /** * Whether we need to perform hash rewriting if there are valid output paths. */ bool needsHashRewrite(); - /** - * The additional states. - */ - Goal::Co tryLocalBuild() override; +public: /** * Set up build environment / sandbox, acquiring resources (e.g. @@ -278,6 +404,8 @@ struct LocalDerivationGoal : DerivationGoal, RestrictionContext */ std::variant, SingleDrvOutputs> unprepareBuild(); +private: + /** * Fill in the environment for the builder. */ @@ -303,12 +431,16 @@ struct LocalDerivationGoal : DerivationGoal, RestrictionContext */ void startDaemon(); +public: + /** * Stop the in-process nix daemon thread. * @see startDaemon */ void stopDaemon(); +private: + void addDependency(const StorePath & path) override; /** @@ -334,26 +466,21 @@ struct LocalDerivationGoal : DerivationGoal, RestrictionContext */ void checkOutputs(const std::map & outputs); - bool isReadDesc(int fd) override; +public: /** * Delete the temporary directory, if we have one. */ void deleteTmpDir(bool force); - /** - * Forcibly kill the child process, if any. - * - * Called by destructor, can't be overridden - */ - void killChild() override final; - /** * Kill any processes running under the build user UID or in the * cgroup of the build. */ void killSandbox(bool getStats); +private: + bool cleanupDecideWhetherDiskFull(); /** @@ -373,6 +500,98 @@ struct LocalDerivationGoal : DerivationGoal, RestrictionContext StorePath makeFallbackPath(OutputNameView outputName); }; +/** + * This hooks up `DerivationBuilder` to the scheduler / goal machinary. + * + * @todo Eventually, this shouldn't exist, because `DerivationGoal` can + * just choose to use `DerivationBuilder` or its remote-building + * equalivalent directly, at the "value level" rather than "class + * inheritance hierarchy" level. + */ +struct LocalDerivationGoal : DerivationGoal, DerivationBuilderCallbacks +{ + DerivationBuilder builder; + + LocalDerivationGoal(const StorePath & drvPath, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode) + : DerivationGoal{drvPath, wantedOutputs, worker, buildMode} + , builder{ + worker.store, + static_cast(*this), + DerivationBuilderParams { + DerivationGoal::drvPath, + DerivationGoal::buildMode, + DerivationGoal::buildResult, + DerivationGoal::drv, + DerivationGoal::parsedDrv, + DerivationGoal::drvOptions, + DerivationGoal::inputPaths, + DerivationGoal::initialOutputs, + }, + } + {} + + LocalDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal) + : DerivationGoal{drvPath, drv, wantedOutputs, worker, buildMode} + , builder{ + worker.store, + static_cast(*this), + DerivationBuilderParams { + DerivationGoal::drvPath, + DerivationGoal::buildMode, + DerivationGoal::buildResult, + DerivationGoal::drv, + DerivationGoal::parsedDrv, + DerivationGoal::drvOptions, + DerivationGoal::inputPaths, + DerivationGoal::initialOutputs, + }, + } + {} + + virtual ~LocalDerivationGoal() override; + + /** + * The additional states. + */ + Goal::Co tryLocalBuild() override; + + bool isReadDesc(int fd) override; + + /** + * Forcibly kill the child process, if any. + * + * Called by destructor, can't be overridden + */ + void killChild() override final; + + void childStarted() override; + void childTerminated() override; + + void noteHashMismatch(void) override; + void noteCheckMismatch(void) override; + + void markContentsGood(const StorePath &) override; + + // Fake overrides to instantiate identically-named virtual methods + + Path openLogFile() override { + return DerivationGoal::openLogFile(); + } + void closeLogFile() override { + DerivationGoal::closeLogFile(); + } + SingleDrvOutputs assertPathValidity() override { + return DerivationGoal::assertPathValidity(); + } + void appendLogTailErrorMsg(std::string & msg) override { + DerivationGoal::appendLogTailErrorMsg(msg); + } +}; + std::shared_ptr makeLocalDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, @@ -423,20 +642,20 @@ void handleDiffHook( } } -const Path LocalDerivationGoal::homeDir = "/homeless-shelter"; +const Path DerivationBuilder::homeDir = "/homeless-shelter"; LocalDerivationGoal::~LocalDerivationGoal() { /* Careful: we should never ever throw an exception from a destructor. */ - try { deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } + try { builder.deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } - try { stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } + try { builder.stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } } -inline bool LocalDerivationGoal::needsHashRewrite() +inline bool DerivationBuilder::needsHashRewrite() { #ifdef __linux__ return !useChroot; @@ -447,9 +666,9 @@ inline bool LocalDerivationGoal::needsHashRewrite() } -LocalStore & LocalDerivationGoal::getLocalStore() +LocalStore & DerivationBuilder::getLocalStore() { - auto p = dynamic_cast(&worker.store); + auto p = dynamic_cast(&store); assert(p); return *p; } @@ -457,7 +676,7 @@ LocalStore & LocalDerivationGoal::getLocalStore() void LocalDerivationGoal::killChild() { - if (pid != -1) { + if (builder.pid != -1) { worker.childTerminated(this); /* If we're using a build user, then there is a tricky race @@ -465,18 +684,18 @@ void LocalDerivationGoal::killChild() done its setuid() to the build user uid, then it won't be killed, and we'll potentially lock up in pid.wait(). So also send a conventional kill to the child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ + ::kill(-builder.pid, SIGKILL); /* ignore the result */ - killSandbox(true); + builder.killSandbox(true); - pid.wait(); + builder.pid.wait(); } DerivationGoal::killChild(); } -void LocalDerivationGoal::killSandbox(bool getStats) +void DerivationBuilder::killSandbox(bool getStats) { if (cgroup) { #ifdef __linux__ @@ -498,6 +717,34 @@ void LocalDerivationGoal::killSandbox(bool getStats) } +void LocalDerivationGoal::childStarted() +{ + worker.childStarted(shared_from_this(), {builder.builderOut.get()}, true, true); +} + +void LocalDerivationGoal::childTerminated() +{ + worker.childTerminated(this); +} + +void LocalDerivationGoal::noteHashMismatch() +{ + worker.hashMismatch = true; +} + + +void LocalDerivationGoal::noteCheckMismatch() +{ + worker.checkMismatch = true; +} + + +void LocalDerivationGoal::markContentsGood(const StorePath & path) +{ + worker.markContentsGood(path); +} + + Goal::Co LocalDerivationGoal::tryLocalBuild() { assert(!hook); @@ -509,7 +756,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() co_return tryToBuild(); } - if (!prepareBuild()) { + if (!builder.prepareBuild()) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); @@ -522,11 +769,11 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() try { /* Okay, we have to build. */ - startBuilder(); + builder.startBuilder(); } catch (BuildError & e) { outputLocks.unlock(); - buildUser.reset(); + builder.buildUser.reset(); worker.permanentFailure = true; co_return done(BuildResult::InputRejected, {}, std::move(e)); } @@ -536,7 +783,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() trace("build done"); - auto res = unprepareBuild(); + auto res = builder.unprepareBuild(); // N.B. cannot use `std::visit` with co-routine return if (auto * ste = std::get_if<0>(&res)) { outputLocks.unlock(); @@ -554,7 +801,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() } } -bool LocalDerivationGoal::prepareBuild() +bool DerivationBuilder::prepareBuild() { /* Cache this */ derivationType = drv->type(); @@ -564,11 +811,11 @@ bool LocalDerivationGoal::prepareBuild() if (settings.sandboxMode == smEnabled) { if (drvOptions->noChroot) throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); + "but that's not allowed when 'sandbox' is 'true'", store.printStorePath(drvPath)); #ifdef __APPLE__ if (drvOptions->additionalSandboxProfile != "") throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); + "but this is only allowed when 'sandbox' is 'relaxed'", store.printStorePath(drvPath)); #endif useChroot = true; } @@ -611,7 +858,7 @@ bool LocalDerivationGoal::prepareBuild() } -std::variant, SingleDrvOutputs> LocalDerivationGoal::unprepareBuild() +std::variant, SingleDrvOutputs> DerivationBuilder::unprepareBuild() { Finally releaseBuildUser([&](){ /* Release the build user at the end of this function. We don't do @@ -629,19 +876,19 @@ std::variant, SingleDrvOutputs> LocalDeriv kill it. */ int status = pid.kill(); - debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); + debug("builder process for '%s' finished", store.printStorePath(drvPath)); buildResult.timesBuilt++; buildResult.stopTime = time(0); /* So the child is gone now. */ - worker.childTerminated(this); + miscMethods.childTerminated(); /* Close the read side of the logger pipe. */ builderOut.close(); /* Close the log file. */ - closeLogFile(); + miscMethods.closeLogFile(); /* When running under a build user, make sure that all processes running under that uid are gone. This is to prevent a @@ -655,7 +902,7 @@ std::variant, SingleDrvOutputs> LocalDeriv if (buildResult.cpuUser && buildResult.cpuSystem) { debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs", - worker.store.printStorePath(drvPath), + store.printStorePath(drvPath), status, ((double) buildResult.cpuUser->count()) / 1000000, ((double) buildResult.cpuSystem->count()) / 1000000); @@ -671,10 +918,10 @@ std::variant, SingleDrvOutputs> LocalDeriv diskFull |= cleanupDecideWhetherDiskFull(); auto msg = fmt("builder for '%s' %s", - Magenta(worker.store.printStorePath(drvPath)), + Magenta(store.printStorePath(drvPath)), statusToString(status)); - appendLogTailErrorMsg(msg); + miscMethods.appendLogTailErrorMsg(msg); if (diskFull) msg += "\nnote: build failure may have been caused by lack of free disk space"; @@ -690,7 +937,7 @@ std::variant, SingleDrvOutputs> LocalDeriv for (auto & [_, output] : builtOutputs) outputPaths.insert(output.outPath); runPostBuildHook( - worker.store, + store, *logger, drvPath, outputPaths @@ -698,7 +945,7 @@ std::variant, SingleDrvOutputs> LocalDeriv /* Delete unused redirected outputs (when doing hash rewriting). */ for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); + deletePath(store.Store::toRealPath(i.second)); /* Delete the chroot (if we were using one). */ autoDelChroot.reset(); /* this runs the destructor */ @@ -749,7 +996,7 @@ static void movePath(const Path & src, const Path & dst) extern void replaceValidPath(const Path & storePath, const Path & tmpPath); -bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() +bool DerivationBuilder::cleanupDecideWhetherDiskFull() { bool diskFull = false; @@ -780,7 +1027,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() for (auto & [_, status] : initialOutputs) { if (!status.known) continue; if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.toRealPath(status.known->path); + auto p = store.toRealPath(status.known->path); if (pathExists(chrootRootDir + p)) std::filesystem::rename((chrootRootDir + p), p); } @@ -842,7 +1089,7 @@ static void rethrowExceptionAsError() /** * Send the current exception to the parent in the format expected by - * `LocalDerivationGoal::processSandboxSetupMessages()`. + * `DerivationBuilder::processSandboxSetupMessages()`. */ static void handleChildException(bool sendException) { @@ -859,7 +1106,7 @@ static void handleChildException(bool sendException) } } -void LocalDerivationGoal::startBuilder() +void DerivationBuilder::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) #ifdef __linux__ @@ -917,7 +1164,7 @@ void LocalDerivationGoal::startBuilder() killSandbox(false); /* Right platform? */ - if (!drvOptions->canBuildLocally(worker.store, *drv)) { + if (!drvOptions->canBuildLocally(store, *drv)) { // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - we should tell them to run the command to install Darwin 2 if (drv->platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") { throw Error("run `/usr/sbin/softwareupdate --install-rosetta` to enable your %s to run programs for %s", settings.thisSystem, drv->platform); @@ -925,9 +1172,9 @@ void LocalDerivationGoal::startBuilder() throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", drv->platform, concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)), - worker.store.printStorePath(drvPath), + store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); + concatStringsSep(", ", store.systemFeatures)); } } @@ -985,7 +1232,7 @@ void LocalDerivationGoal::startBuilder() /* Substitute output placeholders with the scratch output paths. We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); + inputRewrites[hashPlaceholder(outputName)] = store.printStorePath(scratchPath); /* Additional tasks if we know the final path a priori. */ if (!status.known) continue; @@ -996,7 +1243,7 @@ void LocalDerivationGoal::startBuilder() if (fixedFinalPath == scratchPath) continue; /* Ensure scratch path is ours to use. */ - deletePath(worker.store.printStorePath(scratchPath)); + deletePath(store.printStorePath(scratchPath)); /* Rewrite and unrewrite paths */ { @@ -1018,14 +1265,14 @@ void LocalDerivationGoal::startBuilder() for (auto & [fileName, ss] : drvOptions->exportReferencesGraph) { StorePathSet storePathSet; for (auto & storePathS : ss) { - if (!worker.store.isInStore(storePathS)) + if (!store.isInStore(storePathS)) throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - storePathSet.insert(worker.store.toStorePath(storePathS).first); + storePathSet.insert(store.toStorePath(storePathS).first); } /* Write closure info to . */ writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration( - worker.store.exportReferences(storePathSet, inputPaths), false, false)); + store.makeValidityRegistration( + store.exportReferences(storePathSet, inputPaths), false, false)); } } @@ -1048,7 +1295,7 @@ void LocalDerivationGoal::startBuilder() else pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; } - if (hasPrefix(worker.store.storeDir, tmpDirInSandbox)) + if (hasPrefix(store.storeDir, tmpDirInSandbox)) { throw Error("`sandbox-build-dir` must not contain the storeDir"); } @@ -1058,15 +1305,15 @@ void LocalDerivationGoal::startBuilder() StorePathSet closure; for (auto & i : pathsInChroot) try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); + if (store.isInStore(i.second.source)) + store.computeFSClosure(store.toStorePath(i.second.source).first, closure); } catch (InvalidPath & e) { } catch (Error & e) { e.addTrace({}, "while processing 'sandbox-paths'"); throw; } for (auto & i : closure) { - auto p = worker.store.printStorePath(i); + auto p = store.printStorePath(i); pathsInChroot.insert_or_assign(p, p); } @@ -1091,7 +1338,7 @@ void LocalDerivationGoal::startBuilder() } if (!found) throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - worker.store.printStorePath(drvPath), i); + store.printStorePath(drvPath), i); /* Allow files in drvOptions->impureHostDeps to be missing; e.g. macOS 11+ has no /usr/lib/libSystem*.dylib */ @@ -1103,7 +1350,7 @@ void LocalDerivationGoal::startBuilder() environment using bind-mounts. We put it in the Nix store so that the build outputs can be moved efficiently from the chroot to their final location. */ - auto chrootParentDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; + auto chrootParentDir = store.Store::toRealPath(drvPath) + ".chroot"; deletePath(chrootParentDir); /* Clean up the chroot directory automatically. */ @@ -1157,7 +1404,7 @@ void LocalDerivationGoal::startBuilder() can be bind-mounted). !!! As an extra security precaution, make the fake Nix store only writable by the build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + Path chrootStoreDir = chrootRootDir + store.storeDir; createDirs(chrootStoreDir); chmod_(chrootStoreDir, 01775); @@ -1165,8 +1412,8 @@ void LocalDerivationGoal::startBuilder() throw SysError("cannot change ownership of '%1%'", chrootStoreDir); for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - Path r = worker.store.toRealPath(p); + auto p = store.printStorePath(i); + Path r = store.toRealPath(p); pathsInChroot.insert_or_assign(p, r); } @@ -1175,14 +1422,14 @@ void LocalDerivationGoal::startBuilder() rebuilding a path that is in settings.sandbox-paths (typically the dependencies of /bin/sh). Throw them out. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { + for (auto & i : drv->outputsAndOptPaths(store)) { /* If the name isn't known a priori (i.e. floating content-addressing derivation), the temporary location we use should be fresh. Freshness means it is impossible that the path is already in the sandbox, so we don't need to worry about removing it. */ if (i.second.second) - pathsInChroot.erase(worker.store.printStorePath(*i.second.second)); + pathsInChroot.erase(store.printStorePath(*i.second.second)); } if (cgroup) { @@ -1214,8 +1461,8 @@ void LocalDerivationGoal::startBuilder() if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); - auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : - Strings({ worker.store.printStorePath(drvPath) }); + auto args = useChroot ? Strings({store.printStorePath(drvPath), chrootRootDir}) : + Strings({ store.printStorePath(drvPath) }); enum BuildHookState { stBegin, stExtraChrootDirs @@ -1260,7 +1507,7 @@ void LocalDerivationGoal::startBuilder() printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); /* Create the log file. */ - [[maybe_unused]] Path logFile = openLogFile(); + [[maybe_unused]] Path logFile = miscMethods.openLogFile(); /* Create a pseudoterminal to get the output of the builder. */ builderOut = posix_openpt(O_RDWR | O_NOCTTY); @@ -1470,13 +1717,13 @@ void LocalDerivationGoal::startBuilder() /* parent */ pid.setSeparatePG(true); - worker.childStarted(shared_from_this(), {builderOut.get()}, true, true); + miscMethods.childStarted(); processSandboxSetupMessages(); } -void LocalDerivationGoal::processSandboxSetupMessages() +void DerivationBuilder::processSandboxSetupMessages() { std::vector msgs; while (true) { @@ -1486,7 +1733,7 @@ void LocalDerivationGoal::processSandboxSetupMessages() } catch (Error & e) { auto status = pid.wait(); e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", - worker.store.printStorePath(drvPath), + store.printStorePath(drvPath), statusToString(status), concatStringsSep("|", msgs)); throw; @@ -1505,7 +1752,7 @@ void LocalDerivationGoal::processSandboxSetupMessages() } -void LocalDerivationGoal::initTmpDir() +void DerivationBuilder::initTmpDir() { /* In a sandbox, for determinism, always use the same temporary directory. */ @@ -1549,7 +1796,7 @@ void LocalDerivationGoal::initTmpDir() } -void LocalDerivationGoal::initEnv() +void DerivationBuilder::initEnv() { env.clear(); @@ -1570,7 +1817,7 @@ void LocalDerivationGoal::initEnv() shouldn't care, but this is useful for purity checking (e.g., the compiler or linker might only want to accept paths to files in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; + env["NIX_STORE"] = store.storeDir; /* The maximum number of cores to utilize for parallel building. */ env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores); @@ -1617,11 +1864,11 @@ void LocalDerivationGoal::initEnv() } -void LocalDerivationGoal::writeStructuredAttrs() +void DerivationBuilder::writeStructuredAttrs() { if (parsedDrv) { auto json = parsedDrv->prepareStructuredAttrs( - worker.store, + store, *drvOptions, inputPaths, drv->outputs); @@ -1646,19 +1893,19 @@ void LocalDerivationGoal::writeStructuredAttrs() } -void LocalDerivationGoal::startDaemon() +void DerivationBuilder::startDaemon() { experimentalFeatureSettings.require(Xp::RecursiveNix); Store::Params params; params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; + params["store"] = store.storeDir; if (auto & optRoot = getLocalStore().rootDir.get()) params["root"] = *optRoot; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; auto store = makeRestrictedStore(params, - ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), + ref(std::dynamic_pointer_cast(this->store.shared_from_this())), *this); addedPaths.clear(); @@ -1714,7 +1961,7 @@ void LocalDerivationGoal::startDaemon() } -void LocalDerivationGoal::stopDaemon() +void DerivationBuilder::stopDaemon() { if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) { // According to the POSIX standard, the 'shutdown' function should @@ -1747,7 +1994,7 @@ void LocalDerivationGoal::stopDaemon() } -void LocalDerivationGoal::addDependency(const StorePath & path) +void DerivationBuilder::addDependency(const StorePath & path) { if (isAllowed(path)) return; @@ -1757,17 +2004,17 @@ void LocalDerivationGoal::addDependency(const StorePath & path) appear in the sandbox. */ if (useChroot) { - debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); + debug("materialising '%s' in the sandbox", store.printStorePath(path)); #ifdef __linux__ - Path source = worker.store.Store::toRealPath(path); - Path target = chrootRootDir + worker.store.printStorePath(path); + Path source = store.Store::toRealPath(path); + Path target = chrootRootDir + store.printStorePath(path); if (pathExists(target)) { // There is a similar debug message in doBind, so only run it in this block to not have double messages. debug("bind-mounting %s -> %s", target, source); - throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); + throw Error("store path '%s' already exists in the sandbox", store.printStorePath(path)); } /* Bind-mount the path into the sandbox. This requires @@ -1789,17 +2036,17 @@ void LocalDerivationGoal::addDependency(const StorePath & path) int status = child.wait(); if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); + throw Error("could not add path '%s' to sandbox", store.printStorePath(path)); #else throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - worker.store.printStorePath(path)); + store.printStorePath(path)); #endif } } -void LocalDerivationGoal::chownToBuilder(const Path & path) +void DerivationBuilder::chownToBuilder(const Path & path) { if (!buildUser) return; if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) @@ -1895,7 +2142,7 @@ void setupSeccomp() } -void LocalDerivationGoal::runChild() +void DerivationBuilder::runChild() { /* Warning: in the child we should absolutely not make any SQLite calls! */ @@ -1984,7 +2231,7 @@ void LocalDerivationGoal::runChild() Marking chrootRootDir as MS_SHARED causes pivot_root() to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + Path chrootStoreDir = chrootRootDir + store.storeDir; if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) throw SysError("unable to bind mount the Nix store", chrootStoreDir); @@ -1999,7 +2246,7 @@ void LocalDerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + if (store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); ss.push_back("/dev/null"); ss.push_back("/dev/random"); @@ -2224,7 +2471,7 @@ void LocalDerivationGoal::runChild() /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; + Path cur = store.storeDir; while (cur.compare("/") != 0) { ancestry.insert(cur); cur = dirOf(cur); @@ -2232,7 +2479,7 @@ void LocalDerivationGoal::runChild() /* Add all our input paths to the chroot */ for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); + auto p = store.printStorePath(i); pathsInChroot[p] = p; } @@ -2255,7 +2502,7 @@ void LocalDerivationGoal::runChild() /* Add the output paths we'll use at build-time to the chroot */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); + sandboxProfile += fmt("\t(subpath \"%s\")\n", store.printStorePath(path)); sandboxProfile += ")\n"; @@ -2348,7 +2595,7 @@ void LocalDerivationGoal::runChild() std::map outputs; for (auto & e : drv->outputs) outputs.insert_or_assign(e.first, - worker.store.printStorePath(scratchOutputs.at(e.first))); + store.printStorePath(scratchOutputs.at(e.first))); if (drv->builder == "builtin:fetchurl") builtinFetchurl(*drv, outputs, netrcData, caFileData); @@ -2408,10 +2655,8 @@ void LocalDerivationGoal::runChild() } -SingleDrvOutputs LocalDerivationGoal::registerOutputs() +SingleDrvOutputs DerivationBuilder::registerOutputs() { - assert(!hook); - std::map infos; /* Set of inodes seen during calls to canonicalisePathMetaData() @@ -2436,7 +2681,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto toRealPathChroot = [&](const Path & p) -> Path { return useChroot && !needsHashRewrite() ? chrootRootDir + p - : worker.store.toRealPath(p); + : store.toRealPath(p); }; /* Check whether the output paths were created, and make all @@ -2454,8 +2699,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (!scratchOutput) throw BuildError( "builder for '%s' has no scratch output for '%s'", - worker.store.printStorePath(drvPath), outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchOutput)); + store.printStorePath(drvPath), outputName); + auto actualPath = toRealPathChroot(store.printStorePath(*scratchOutput)); outputsToSort.insert(outputName); @@ -2464,7 +2709,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (!initialOutput) throw BuildError( "builder for '%s' has no initial output for '%s'", - worker.store.printStorePath(drvPath), outputName); + store.printStorePath(drvPath), outputName); auto & initialInfo = *initialOutput; /* Don't register if already valid, and not checking */ @@ -2481,7 +2726,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (!optSt) throw BuildError( "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); + store.printStorePath(drvPath), outputName, actualPath); struct stat & st = *optSt; #ifndef __CYGWIN__ @@ -2532,7 +2777,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (!orifu) throw BuildError( "no output reference for '%s' in build of '%s'", - name, worker.store.printStorePath(drvPath)); + name, store.printStorePath(drvPath)); return std::visit(overloaded { /* Since we'll use the already installed versions of these, we can treat them as leaves and ignore any references they @@ -2553,7 +2798,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() // TODO with more -vvvv also show the temporary paths for manual inspection. return BuildError( "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - worker.store.printStorePath(drvPath), path, parent); + store.printStorePath(drvPath), path, parent); }}); std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); @@ -2564,7 +2809,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto output = get(drv->outputs, outputName); auto scratchPath = get(scratchOutputs, outputName); assert(output && scratchPath); - auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchPath)); + auto actualPath = toRealPathChroot(store.printStorePath(*scratchPath)); auto finish = [&](StorePath finalStorePath) { /* Store the final path */ @@ -2682,7 +2927,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() }(); ValidPathInfo newInfo0 { - worker.store, + store, outputPathName(drv->name, outputName), ContentAddressWithReferences::fromParts( outputHash.method, @@ -2760,10 +3005,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (wanted != got) { /* Throw an error after registering the path as valid. */ - worker.hashMismatch = true; + miscMethods.noteHashMismatch(); delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", - worker.store.printStorePath(drvPath), + store.printStorePath(drvPath), wanted.to_string(HashFormat::SRI, true), got.to_string(HashFormat::SRI, true))); } @@ -2771,9 +3016,9 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto numViolations = newInfo.references.size(); delayedException = std::make_exception_ptr( BuildError("fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", - worker.store.printStorePath(drvPath), + store.printStorePath(drvPath), numViolations, - worker.store.printStorePath(*newInfo.references.begin()))); + store.printStorePath(*newInfo.references.begin()))); } return newInfo0; @@ -2805,36 +3050,36 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() /* Calculate where we'll move the output files. In the checking case we will leave leave them where they are, for now, rather than move to their usual "final destination" */ - auto finalDestPath = worker.store.printStorePath(newInfo.path); + auto finalDestPath = store.printStorePath(newInfo.path); /* Lock final output path, if not already locked. This happens with floating CA derivations and hash-mismatching fixed-output derivations. */ PathLocks dynamicOutputLock; dynamicOutputLock.setDeletion(true); - auto optFixedPath = output->path(worker.store, drv->name, outputName); + auto optFixedPath = output->path(store, drv->name, outputName); if (!optFixedPath || - worker.store.printStorePath(*optFixedPath) != finalDestPath) + store.printStorePath(*optFixedPath) != finalDestPath) { assert(newInfo.ca); - dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); + dynamicOutputLock.lockPaths({store.toRealPath(finalDestPath)}); } /* Move files, if needed */ - if (worker.store.toRealPath(finalDestPath) != actualPath) { + if (store.toRealPath(finalDestPath) != actualPath) { if (buildMode == bmRepair) { /* Path already exists, need to replace it */ - replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); - actualPath = worker.store.toRealPath(finalDestPath); + replaceValidPath(store.toRealPath(finalDestPath), actualPath); + actualPath = store.toRealPath(finalDestPath); } else if (buildMode == bmCheck) { /* Path already exists, and we want to compare, so we leave out new path in place. */ - } else if (worker.store.isValidPath(newInfo.path)) { + } else if (store.isValidPath(newInfo.path)) { /* Path already exists because CA path produced by something else. No moving needed. */ assert(newInfo.ca); } else { - auto destPath = worker.store.toRealPath(finalDestPath); + auto destPath = store.toRealPath(finalDestPath); deletePath(destPath); movePath(actualPath, destPath); actualPath = destPath; @@ -2845,25 +3090,25 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (buildMode == bmCheck) { - if (!worker.store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); + if (!store.isValidPath(newInfo.path)) continue; + ValidPathInfo oldInfo(*store.queryPathInfo(newInfo.path)); if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; + miscMethods.noteCheckMismatch(); if (settings.runDiffHook || settings.keepFailed) { - auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); + auto dst = store.toRealPath(finalDestPath + checkSuffix); deletePath(dst); movePath(actualPath, dst); handleDiffHook( buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); + finalDestPath, dst, store.printStorePath(drvPath), tmpDir); throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); + store.printStorePath(drvPath), store.toRealPath(finalDestPath), dst); } else throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); + store.printStorePath(drvPath), store.toRealPath(finalDestPath)); } /* Since we verified the build, it's now ultimately trusted. */ @@ -2879,13 +3124,13 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() /* For debugging, print out the referenced and unreferenced paths. */ for (auto & i : inputPaths) { if (references.count(i)) - debug("referenced input: '%1%'", worker.store.printStorePath(i)); + debug("referenced input: '%1%'", store.printStorePath(i)); else - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); + debug("unreferenced input: '%1%'", store.printStorePath(i)); } localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); + miscMethods.markContentsGood(newInfo.path); newInfo.deriver = drvPath; newInfo.ultimate = true; @@ -2908,7 +3153,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() also a source for non-determinism. */ if (delayedException) std::rethrow_exception(delayedException); - return assertPathValidity(); + return miscMethods.assertPathValidity(); } /* Apply output checks. */ @@ -2952,8 +3197,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv->type().isImpure()) { - worker.store.signRealisation(thisRealisation); - worker.store.registerDrvOutput(thisRealisation); + store.signRealisation(thisRealisation); + store.registerDrvOutput(thisRealisation); } builtOutputs.emplace(outputName, thisRealisation); } @@ -2962,11 +3207,11 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() } -void LocalDerivationGoal::checkOutputs(const std::map & outputs) +void DerivationBuilder::checkOutputs(const std::map & outputs) { std::map outputsByPath; for (auto & output : outputs) - outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); + outputsByPath.emplace(store.printStorePath(output.second.path), output.second); for (auto & output : outputs) { auto & outputName = output.first; @@ -2987,13 +3232,13 @@ void LocalDerivationGoal::checkOutputs(const std::mapsecond.narSize; for (auto & ref : i->second.references) pathsLeft.push(ref); } else { - auto info = worker.store.queryPathInfo(path); + auto info = store.queryPathInfo(path); closureSize += info->narSize; for (auto & ref : info->references) pathsLeft.push(ref); @@ -3007,13 +3252,13 @@ void LocalDerivationGoal::checkOutputs(const std::map *checks.maxSize) throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); + store.printStorePath(info.path), info.narSize, *checks.maxSize); if (checks.maxClosureSize) { uint64_t closureSize = getClosure(info.path).second; if (closureSize > *checks.maxClosureSize) throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); + store.printStorePath(info.path), closureSize, *checks.maxClosureSize); } auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive) @@ -3023,15 +3268,15 @@ void LocalDerivationGoal::checkOutputs(const std::mappath); else { std::string outputsListing = concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; }); throw BuildError("derivation '%s' output check for '%s' contains an illegal reference specifier '%s'," " expected store path or output name (one of [%s])", - worker.store.printStorePath(drvPath), outputName, i, outputsListing); + store.printStorePath(drvPath), outputName, i, outputsListing); } } @@ -3057,10 +3302,10 @@ void LocalDerivationGoal::checkOutputs(const std::mapname, outputName)); } -StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) +StorePath DerivationBuilder::makeFallbackPath(const StorePath & path) { // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path // See doc/manual/source/protocols/store-path.md for details auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()); - return worker.store.makeStorePath( + return store.makeStorePath( pathType, // pass an all-zeroes hash Hash(HashAlgorithm::SHA256), path.name()); From 6c2a7fdc496b9558985e8971eadfe415887c356c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 20 Mar 2025 19:03:21 -0400 Subject: [PATCH 235/396] Copy `local-derivation-goal.cc` to `derivation-builder.{cc,hh}` This is done to prior to splitting, just like 05cc5a858717c092e1835e2b0fec4c4b1a7fc97e for 68f4c728eca33f115f90e3f924c9081a4cd59896. --- src/libstore/unix/build/derivation-builder.cc | 3395 +++++++++++++++++ .../nix/store/build/derivation-builder.hh | 3395 +++++++++++++++++ 2 files changed, 6790 insertions(+) create mode 100644 src/libstore/unix/build/derivation-builder.cc create mode 100644 src/libstore/unix/include/nix/store/build/derivation-builder.hh diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc new file mode 100644 index 000000000..973eef26e --- /dev/null +++ b/src/libstore/unix/build/derivation-builder.cc @@ -0,0 +1,3395 @@ +#include "nix/store/build/local-derivation-goal.hh" +#include "nix/store/local-store.hh" +#include "nix/util/processes.hh" +#include "nix/store/indirect-root-store.hh" +#include "nix/store/build/hook-instance.hh" +#include "nix/store/build/worker.hh" +#include "nix/store/builtins.hh" +#include "nix/store/builtins/buildenv.hh" +#include "nix/store/path-references.hh" +#include "nix/util/finally.hh" +#include "nix/util/util.hh" +#include "nix/util/archive.hh" +#include "nix/util/git.hh" +#include "nix/util/compression.hh" +#include "nix/store/daemon.hh" +#include "nix/util/topo-sort.hh" +#include "nix/util/callback.hh" +#include "nix/util/json-utils.hh" +#include "nix/util/current-process.hh" +#include "nix/store/build/child.hh" +#include "nix/util/unix-domain-socket.hh" +#include "nix/store/posix-fs-canonicalise.hh" +#include "nix/util/posix-source-accessor.hh" +#include "nix/store/restricted-store.hh" +#include "nix/store/config.hh" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "store-config-private.hh" + +#if HAVE_STATVFS +#include +#endif + +/* Includes required for chroot support. */ +#ifdef __linux__ +# include "linux/fchmodat2-compat.hh" +# include +# include +# include +# include +# include +# include +# include +# include +# include "nix/util/namespaces.hh" +# if HAVE_SECCOMP +# include +# endif +# define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) +# include "nix/util/cgroup.hh" +# include "nix/store/personality.hh" +#endif + +#ifdef __APPLE__ +#include +#include +#include + +/* This definition is undocumented but depended upon by all major browsers. */ +extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf); +#endif + +#include +#include +#include + +#include "nix/util/strings.hh" +#include "nix/util/signals.hh" + +#include "store-config-private.hh" + +namespace nix { + +/** + * Parameters by (mostly) `const` reference for `DerivationBuilder`. + */ +struct DerivationBuilderParams +{ + /** The path of the derivation. */ + const StorePath & drvPath; + + BuildResult & buildResult; + + /** + * The derivation stored at drvPath. + * + * @todo Remove double indirection by delaying when this is + * initialized. + */ + const std::unique_ptr & drv; + + const std::unique_ptr & parsedDrv; + const std::unique_ptr & drvOptions; + + // The remainder is state held during the build. + + /** + * All input paths (that is, the union of FS closures of the + * immediate input paths). + */ + const StorePathSet & inputPaths; + + /** + * @note we do in fact mutate this + */ + std::map & initialOutputs; + + const BuildMode & buildMode; + + DerivationBuilderParams( + const StorePath & drvPath, + const BuildMode & buildMode, + BuildResult & buildResult, + const std::unique_ptr & drv, + const std::unique_ptr & parsedDrv, + const std::unique_ptr & drvOptions, + const StorePathSet & inputPaths, + std::map & initialOutputs) + : drvPath{drvPath} + , buildResult{buildResult} + , drv{drv} + , parsedDrv{parsedDrv} + , drvOptions{drvOptions} + , inputPaths{inputPaths} + , initialOutputs{initialOutputs} + , buildMode{buildMode} + { } + + DerivationBuilderParams(DerivationBuilderParams &&) = default; +}; + +/** + * Callbacks that `DerivationBuilder` needs. + */ +struct DerivationBuilderCallbacks +{ + /** + * Open a log file and a pipe to it. + */ + virtual Path openLogFile() = 0; + + /** + * Close the log file. + */ + virtual void closeLogFile() = 0; + + /** + * Aborts if any output is not valid or corrupt, and otherwise + * returns a 'SingleDrvOutputs' structure containing all outputs. + * + * @todo Probably should just be in `DerivationGoal`. + */ + virtual SingleDrvOutputs assertPathValidity() = 0; + + virtual void appendLogTailErrorMsg(std::string & msg) = 0; + + /** + * Hook up `builderOut` to some mechanism to ingest the log + * + * @todo this should be reworked + */ + virtual void childStarted() = 0; + + /** + * @todo this should be reworked + */ + virtual void childTerminated() = 0; + + virtual void noteHashMismatch(void) = 0; + virtual void noteCheckMismatch(void) = 0; + + virtual void markContentsGood(const StorePath & path) = 0; +}; + +/** + * This class represents the state for building locally. + * + * @todo Ideally, it would not be a class, but a single function. + * However, besides the main entry point, there are a few more methods + * which are externally called, and need to be gotten rid of. There are + * also some virtual methods (either directly here or inherited from + * `DerivationBuilderCallbacks`, a stop-gap) that represent outgoing + * rather than incoming call edges that either should be removed, or + * become (higher order) function parameters. + */ +class DerivationBuilder : public RestrictionContext, DerivationBuilderParams +{ + Store & store; + + DerivationBuilderCallbacks & miscMethods; + +public: + + DerivationBuilder( + Store & store, + DerivationBuilderCallbacks & miscMethods, + DerivationBuilderParams params) + : DerivationBuilderParams{std::move(params)} + , store{store} + , miscMethods{miscMethods} + { } + + LocalStore & getLocalStore(); + + /** + * User selected for running the builder. + */ + std::unique_ptr buildUser; + + /** + * The process ID of the builder. + */ + Pid pid; + +private: + + /** + * The cgroup of the builder, if any. + */ + std::optional cgroup; + + /** + * The temporary directory used for the build. + */ + Path tmpDir; + + /** + * The top-level temporary directory. `tmpDir` is either equal to + * or a child of this directory. + */ + Path topTmpDir; + + /** + * The path of the temporary directory in the sandbox. + */ + Path tmpDirInSandbox; + +public: + + /** + * Master side of the pseudoterminal used for the builder's + * standard output/error. + */ + AutoCloseFD builderOut; + +private: + + /** + * Pipe for synchronising updates to the builder namespaces. + */ + Pipe userNamespaceSync; + + /** + * The mount namespace and user namespace of the builder, used to add additional + * paths to the sandbox as a result of recursive Nix calls. + */ + AutoCloseFD sandboxMountNamespace; + AutoCloseFD sandboxUserNamespace; + + /** + * On Linux, whether we're doing the build in its own user + * namespace. + */ + bool usingUserNamespace = true; + + /** + * Whether we're currently doing a chroot build. + */ + bool useChroot = false; + + /** + * The root of the chroot environment. + */ + Path chrootRootDir; + + /** + * RAII object to delete the chroot directory. + */ + std::shared_ptr autoDelChroot; + + /** + * The sort of derivation we are building. + * + * Just a cached value, can be recomputed from `drv`. + */ + std::optional derivationType; + + /** + * Stuff we need to pass to initChild(). + */ + struct ChrootPath { + Path source; + bool optional; + ChrootPath(Path source = "", bool optional = false) + : source(source), optional(optional) + { } + }; + typedef map PathsInChroot; // maps target path to source path + PathsInChroot pathsInChroot; + + typedef map Environment; + Environment env; + + /** + * Hash rewriting. + */ + StringMap inputRewrites, outputRewrites; + typedef map RedirectedOutputs; + RedirectedOutputs redirectedOutputs; + + /** + * The output paths used during the build. + * + * - Input-addressed derivations or fixed content-addressed outputs are + * sometimes built when some of their outputs already exist, and can not + * be hidden via sandboxing. We use temporary locations instead and + * rewrite after the build. Otherwise the regular predetermined paths are + * put here. + * + * - Floating content-addressing derivations do not know their final build + * output paths until the outputs are hashed, so random locations are + * used, and then renamed. The randomness helps guard against hidden + * self-references. + */ + OutputPathMap scratchOutputs; + + uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } + + const static Path homeDir; + + /** + * The recursive Nix daemon socket. + */ + AutoCloseFD daemonSocket; + + /** + * The daemon main thread. + */ + std::thread daemonThread; + + /** + * The daemon worker threads. + */ + std::vector daemonWorkerThreads; + + const StorePathSet & originalPaths() override + { + return inputPaths; + } + + bool isAllowed(const StorePath & path) override + { + return inputPaths.count(path) || addedPaths.count(path); + } + bool isAllowed(const DrvOutput & id) override + { + return addedDrvOutputs.count(id); + } + + bool isAllowed(const DerivedPath & req); + + friend struct RestrictedStore; + + /** + * Whether we need to perform hash rewriting if there are valid output paths. + */ + bool needsHashRewrite(); + +public: + + /** + * Set up build environment / sandbox, acquiring resources (e.g. + * locks as needed). After this is run, the builder should be + * started. + * + * @returns true if successful, false if we could not acquire a build + * user. In that case, the caller must wait and then try again. + */ + bool prepareBuild(); + + /** + * Start building a derivation. + */ + void startBuilder(); + + /** + * Tear down build environment after the builder exits (either on + * its own or if it is killed). + * + * @returns The first case indicates failure during output + * processing. A status code and exception are returned, providing + * more information. The second case indicates success, and + * realisations for each output of the derivation are returned. + */ + std::variant, SingleDrvOutputs> unprepareBuild(); + +private: + + /** + * Fill in the environment for the builder. + */ + void initEnv(); + + /** + * Process messages send by the sandbox initialization. + */ + void processSandboxSetupMessages(); + + /** + * Setup tmp dir location. + */ + void initTmpDir(); + + /** + * Write a JSON file containing the derivation attributes. + */ + void writeStructuredAttrs(); + + /** + * Start an in-process nix daemon thread for recursive-nix. + */ + void startDaemon(); + +public: + + /** + * Stop the in-process nix daemon thread. + * @see startDaemon + */ + void stopDaemon(); + +private: + + void addDependency(const StorePath & path) override; + + /** + * Make a file owned by the builder. + */ + void chownToBuilder(const Path & path); + + /** + * Run the builder's process. + */ + void runChild(); + + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ + SingleDrvOutputs registerOutputs(); + + /** + * Check that an output meets the requirements specified by the + * 'outputChecks' attribute (or the legacy + * '{allowed,disallowed}{References,Requisites}' attributes). + */ + void checkOutputs(const std::map & outputs); + +public: + + /** + * Delete the temporary directory, if we have one. + */ + void deleteTmpDir(bool force); + + /** + * Kill any processes running under the build user UID or in the + * cgroup of the build. + */ + void killSandbox(bool getStats); + +private: + + bool cleanupDecideWhetherDiskFull(); + + /** + * Create alternative path calculated from but distinct from the + * input, so we can avoid overwriting outputs (or other store paths) + * that already exist. + */ + StorePath makeFallbackPath(const StorePath & path); + + /** + * Make a path to another based on the output name along with the + * derivation hash. + * + * @todo Add option to randomize, so we can audit whether our + * rewrites caught everything + */ + StorePath makeFallbackPath(OutputNameView outputName); +}; + +/** + * This hooks up `DerivationBuilder` to the scheduler / goal machinary. + * + * @todo Eventually, this shouldn't exist, because `DerivationGoal` can + * just choose to use `DerivationBuilder` or its remote-building + * equalivalent directly, at the "value level" rather than "class + * inheritance hierarchy" level. + */ +struct LocalDerivationGoal : DerivationGoal, DerivationBuilderCallbacks +{ + DerivationBuilder builder; + + LocalDerivationGoal(const StorePath & drvPath, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode) + : DerivationGoal{drvPath, wantedOutputs, worker, buildMode} + , builder{ + worker.store, + static_cast(*this), + DerivationBuilderParams { + DerivationGoal::drvPath, + DerivationGoal::buildMode, + DerivationGoal::buildResult, + DerivationGoal::drv, + DerivationGoal::parsedDrv, + DerivationGoal::drvOptions, + DerivationGoal::inputPaths, + DerivationGoal::initialOutputs, + }, + } + {} + + LocalDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal) + : DerivationGoal{drvPath, drv, wantedOutputs, worker, buildMode} + , builder{ + worker.store, + static_cast(*this), + DerivationBuilderParams { + DerivationGoal::drvPath, + DerivationGoal::buildMode, + DerivationGoal::buildResult, + DerivationGoal::drv, + DerivationGoal::parsedDrv, + DerivationGoal::drvOptions, + DerivationGoal::inputPaths, + DerivationGoal::initialOutputs, + }, + } + {} + + virtual ~LocalDerivationGoal() override; + + /** + * The additional states. + */ + Goal::Co tryLocalBuild() override; + + bool isReadDesc(int fd) override; + + /** + * Forcibly kill the child process, if any. + * + * Called by destructor, can't be overridden + */ + void killChild() override final; + + void childStarted() override; + void childTerminated() override; + + void noteHashMismatch(void) override; + void noteCheckMismatch(void) override; + + void markContentsGood(const StorePath &) override; + + // Fake overrides to instantiate identically-named virtual methods + + Path openLogFile() override { + return DerivationGoal::openLogFile(); + } + void closeLogFile() override { + DerivationGoal::closeLogFile(); + } + SingleDrvOutputs assertPathValidity() override { + return DerivationGoal::assertPathValidity(); + } + void appendLogTailErrorMsg(std::string & msg) override { + DerivationGoal::appendLogTailErrorMsg(msg); + } +}; + +std::shared_ptr makeLocalDerivationGoal( + const StorePath & drvPath, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode) +{ + return std::make_shared(drvPath, wantedOutputs, worker, buildMode); +} + +std::shared_ptr makeLocalDerivationGoal( + const StorePath & drvPath, const BasicDerivation & drv, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode) +{ + return std::make_shared(drvPath, drv, wantedOutputs, worker, buildMode); +} + +void handleDiffHook( + uid_t uid, uid_t gid, + const Path & tryA, const Path & tryB, + const Path & drvPath, const Path & tmpDir) +{ + auto & diffHookOpt = settings.diffHook.get(); + if (diffHookOpt && settings.runDiffHook) { + auto & diffHook = *diffHookOpt; + try { + auto diffRes = runProgram(RunOptions { + .program = diffHook, + .lookupPath = true, + .args = {tryA, tryB, drvPath, tmpDir}, + .uid = uid, + .gid = gid, + .chdir = "/" + }); + if (!statusOk(diffRes.first)) + throw ExecError(diffRes.first, + "diff-hook program '%1%' %2%", + diffHook, + statusToString(diffRes.first)); + + if (diffRes.second != "") + printError(chomp(diffRes.second)); + } catch (Error & error) { + ErrorInfo ei = error.info(); + // FIXME: wrap errors. + ei.msg = HintFmt("diff hook execution failed: %s", ei.msg.str()); + logError(ei); + } + } +} + +const Path DerivationBuilder::homeDir = "/homeless-shelter"; + + +LocalDerivationGoal::~LocalDerivationGoal() +{ + /* Careful: we should never ever throw an exception from a + destructor. */ + try { builder.deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } + try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } + try { builder.stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } +} + + +inline bool DerivationBuilder::needsHashRewrite() +{ +#ifdef __linux__ + return !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + return true; +#endif +} + + +LocalStore & DerivationBuilder::getLocalStore() +{ + auto p = dynamic_cast(&store); + assert(p); + return *p; +} + + +void LocalDerivationGoal::killChild() +{ + if (builder.pid != -1) { + worker.childTerminated(this); + + /* If we're using a build user, then there is a tricky race + condition: if we kill the build user before the child has + done its setuid() to the build user uid, then it won't be + killed, and we'll potentially lock up in pid.wait(). So + also send a conventional kill to the child. */ + ::kill(-builder.pid, SIGKILL); /* ignore the result */ + + builder.killSandbox(true); + + builder.pid.wait(); + } + + DerivationGoal::killChild(); +} + + +void DerivationBuilder::killSandbox(bool getStats) +{ + if (cgroup) { + #ifdef __linux__ + auto stats = destroyCgroup(*cgroup); + if (getStats) { + buildResult.cpuUser = stats.cpuUser; + buildResult.cpuSystem = stats.cpuSystem; + } + #else + unreachable(); + #endif + } + + else if (buildUser) { + auto uid = buildUser->getUID(); + assert(uid != 0); + killUser(uid); + } +} + + +void LocalDerivationGoal::childStarted() +{ + worker.childStarted(shared_from_this(), {builder.builderOut.get()}, true, true); +} + +void LocalDerivationGoal::childTerminated() +{ + worker.childTerminated(this); +} + +void LocalDerivationGoal::noteHashMismatch() +{ + worker.hashMismatch = true; +} + + +void LocalDerivationGoal::noteCheckMismatch() +{ + worker.checkMismatch = true; +} + + +void LocalDerivationGoal::markContentsGood(const StorePath & path) +{ + worker.markContentsGood(path); +} + + +Goal::Co LocalDerivationGoal::tryLocalBuild() +{ + assert(!hook); + + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= settings.maxBuildJobs) { + outputLocks.unlock(); + co_await waitForBuildSlot(); + co_return tryToBuild(); + } + + if (!builder.prepareBuild()) { + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); + co_await waitForAWhile(); + co_return tryLocalBuild(); + } + + actLock.reset(); + + try { + + /* Okay, we have to build. */ + builder.startBuilder(); + + } catch (BuildError & e) { + outputLocks.unlock(); + builder.buildUser.reset(); + worker.permanentFailure = true; + co_return done(BuildResult::InputRejected, {}, std::move(e)); + } + + started(); + co_await Suspend{}; + + trace("build done"); + + auto res = builder.unprepareBuild(); + // N.B. cannot use `std::visit` with co-routine return + if (auto * ste = std::get_if<0>(&res)) { + outputLocks.unlock(); + co_return done(std::move(ste->first), {}, std::move(ste->second)); + } else if (auto * builtOutputs = std::get_if<1>(&res)) { + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + co_return done(BuildResult::Built, std::move(*builtOutputs)); + } else { + unreachable(); + } +} + +bool DerivationBuilder::prepareBuild() +{ + /* Cache this */ + derivationType = drv->type(); + + /* Are we doing a chroot build? */ + { + if (settings.sandboxMode == smEnabled) { + if (drvOptions->noChroot) + throw Error("derivation '%s' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'", store.printStorePath(drvPath)); +#ifdef __APPLE__ + if (drvOptions->additionalSandboxProfile != "") + throw Error("derivation '%s' specifies a sandbox profile, " + "but this is only allowed when 'sandbox' is 'relaxed'", store.printStorePath(drvPath)); +#endif + useChroot = true; + } + else if (settings.sandboxMode == smDisabled) + useChroot = false; + else if (settings.sandboxMode == smRelaxed) + useChroot = derivationType->isSandboxed() && !drvOptions->noChroot; + } + + auto & localStore = getLocalStore(); + if (localStore.storeDir != localStore.realStoreDir.get()) { + #ifdef __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } + + #ifdef __linux__ + if (useChroot) { + if (!mountAndPidNamespacesSupported()) { + if (!settings.sandboxFallback) + throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); + debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); + useChroot = false; + } + } + #endif + + if (useBuildUsers()) { + if (!buildUser) + buildUser = acquireUserLock(drvOptions->useUidRange(*drv) ? 65536 : 1, useChroot); + + if (!buildUser) { + return false; + } + } + + return true; +} + + +std::variant, SingleDrvOutputs> DerivationBuilder::unprepareBuild() +{ + Finally releaseBuildUser([&](){ + /* Release the build user at the end of this function. We don't do + it right away because we don't want another build grabbing this + uid and then messing around with our output. */ + buildUser.reset(); + }); + + sandboxMountNamespace = -1; + sandboxUserNamespace = -1; + + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = pid.kill(); + + debug("builder process for '%s' finished", store.printStorePath(drvPath)); + + buildResult.timesBuilt++; + buildResult.stopTime = time(0); + + /* So the child is gone now. */ + miscMethods.childTerminated(); + + /* Close the read side of the logger pipe. */ + builderOut.close(); + + /* Close the log file. */ + miscMethods.closeLogFile(); + + /* When running under a build user, make sure that all processes + running under that uid are gone. This is to prevent a + malicious user from leaving behind a process that keeps files + open and modifies them after they have been chown'ed to + root. */ + killSandbox(true); + + /* Terminate the recursive Nix daemon. */ + stopDaemon(); + + if (buildResult.cpuUser && buildResult.cpuSystem) { + debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs", + store.printStorePath(drvPath), + status, + ((double) buildResult.cpuUser->count()) / 1000000, + ((double) buildResult.cpuSystem->count()) / 1000000); + } + + bool diskFull = false; + + try { + + /* Check the exit status. */ + if (!statusOk(status)) { + + diskFull |= cleanupDecideWhetherDiskFull(); + + auto msg = fmt("builder for '%s' %s", + Magenta(store.printStorePath(drvPath)), + statusToString(status)); + + miscMethods.appendLogTailErrorMsg(msg); + + if (diskFull) + msg += "\nnote: build failure may have been caused by lack of free disk space"; + + throw BuildError(msg); + } + + /* Compute the FS closure of the outputs and register them as + being valid. */ + auto builtOutputs = registerOutputs(); + + StorePathSet outputPaths; + for (auto & [_, output] : builtOutputs) + outputPaths.insert(output.outPath); + runPostBuildHook( + store, + *logger, + drvPath, + outputPaths + ); + + /* Delete unused redirected outputs (when doing hash rewriting). */ + for (auto & i : redirectedOutputs) + deletePath(store.Store::toRealPath(i.second)); + + /* Delete the chroot (if we were using one). */ + autoDelChroot.reset(); /* this runs the destructor */ + + deleteTmpDir(true); + + return std::move(builtOutputs); + + } catch (BuildError & e) { + assert(derivationType); + BuildResult::Status st = + dynamic_cast(&e) ? BuildResult::NotDeterministic : + statusOk(status) ? BuildResult::OutputRejected : + !derivationType->isSandboxed() || diskFull ? BuildResult::TransientFailure : + BuildResult::PermanentFailure; + + return std::pair{std::move(st), std::move(e)}; + } +} + + +static void chmod_(const Path & path, mode_t mode) +{ + if (chmod(path.c_str(), mode) == -1) + throw SysError("setting permissions on '%s'", path); +} + + +/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if + it's a directory and we're not root (to be able to update the + directory's parent link ".."). */ +static void movePath(const Path & src, const Path & dst) +{ + auto st = lstat(src); + + bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + + if (changePerm) + chmod_(src, st.st_mode | S_IWUSR); + + std::filesystem::rename(src, dst); + + if (changePerm) + chmod_(dst, st.st_mode); +} + + +extern void replaceValidPath(const Path & storePath, const Path & tmpPath); + + +bool DerivationBuilder::cleanupDecideWhetherDiskFull() +{ + bool diskFull = false; + + /* Heuristically check whether the build failure may have + been caused by a disk full condition. We have no way + of knowing whether the build actually got an ENOSPC. + So instead, check if the disk is (nearly) full now. If + so, we don't mark this build as a permanent failure. */ +#if HAVE_STATVFS + { + auto & localStore = getLocalStore(); + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable + struct statvfs st; + if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + if (statvfs(tmpDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + } +#endif + + deleteTmpDir(false); + + /* Move paths out of the chroot for easier debugging of + build failures. */ + if (useChroot && buildMode == bmNormal) + for (auto & [_, status] : initialOutputs) { + if (!status.known) continue; + if (buildMode != bmCheck && status.known->isValid()) continue; + auto p = store.toRealPath(status.known->path); + if (pathExists(chrootRootDir + p)) + std::filesystem::rename((chrootRootDir + p), p); + } + + return diskFull; +} + + +#ifdef __linux__ +static void doBind(const Path & source, const Path & target, bool optional = false) { + debug("bind mounting '%1%' to '%2%'", source, target); + + auto bindMount = [&]() { + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); + }; + + auto maybeSt = maybeLstat(source); + if (!maybeSt) { + if (optional) + return; + else + throw SysError("getting attributes of path '%1%'", source); + } + auto st = *maybeSt; + + if (S_ISDIR(st.st_mode)) { + createDirs(target); + bindMount(); + } else if (S_ISLNK(st.st_mode)) { + // Symlinks can (apparently) not be bind-mounted, so just copy it + createDirs(dirOf(target)); + copyFile( + std::filesystem::path(source), + std::filesystem::path(target), false); + } else { + createDirs(dirOf(target)); + writeFile(target, ""); + bindMount(); + } +}; +#endif + +/** + * Rethrow the current exception as a subclass of `Error`. + */ +static void rethrowExceptionAsError() +{ + try { + throw; + } catch (Error &) { + throw; + } catch (std::exception & e) { + throw Error(e.what()); + } catch (...) { + throw Error("unknown exception"); + } +} + +/** + * Send the current exception to the parent in the format expected by + * `DerivationBuilder::processSandboxSetupMessages()`. + */ +static void handleChildException(bool sendException) +{ + try { + rethrowExceptionAsError(); + } catch (Error & e) { + if (sendException) { + writeFull(STDERR_FILENO, "\1\n"); + FdSink sink(STDERR_FILENO); + sink << e; + sink.flush(); + } else + std::cerr << e.msg(); + } +} + +void DerivationBuilder::startBuilder() +{ + if ((buildUser && buildUser->getUIDCount() != 1) + #ifdef __linux__ + || settings.useCgroups + #endif + ) + { + #ifdef __linux__ + experimentalFeatureSettings.require(Xp::Cgroups); + + /* If we're running from the daemon, then this will return the + root cgroup of the service. Otherwise, it will return the + current cgroup. */ + auto rootCgroup = getRootCgroup(); + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) + throw Error("cannot determine the cgroups file system"); + auto rootCgroupPath = canonPath(*cgroupFS + "/" + rootCgroup); + if (!pathExists(rootCgroupPath)) + throw Error("expected cgroup directory '%s'", rootCgroupPath); + + static std::atomic counter{0}; + + cgroup = buildUser + ? fmt("%s/nix-build-uid-%d", rootCgroupPath, buildUser->getUID()) + : fmt("%s/nix-build-pid-%d-%d", rootCgroupPath, getpid(), counter++); + + debug("using cgroup '%s'", *cgroup); + + /* When using a build user, record the cgroup we used for that + user so that if we got interrupted previously, we can kill + any left-over cgroup first. */ + if (buildUser) { + auto cgroupsDir = settings.nixStateDir + "/cgroups"; + createDirs(cgroupsDir); + + auto cgroupFile = fmt("%s/%d", cgroupsDir, buildUser->getUID()); + + if (pathExists(cgroupFile)) { + auto prevCgroup = readFile(cgroupFile); + destroyCgroup(prevCgroup); + } + + writeFile(cgroupFile, *cgroup); + } + + #else + throw Error("cgroups are not supported on this platform"); + #endif + } + + /* Make sure that no other processes are executing under the + sandbox uids. This must be done before any chownToBuilder() + calls. */ + killSandbox(false); + + /* Right platform? */ + if (!drvOptions->canBuildLocally(store, *drv)) { + // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - we should tell them to run the command to install Darwin 2 + if (drv->platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") { + throw Error("run `/usr/sbin/softwareupdate --install-rosetta` to enable your %s to run programs for %s", settings.thisSystem, drv->platform); + } else { + throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", + drv->platform, + concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)), + store.printStorePath(drvPath), + settings.thisSystem, + concatStringsSep(", ", store.systemFeatures)); + } + } + + /* Create a temporary directory where the build will take + place. */ + topTmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); +#ifdef __APPLE__ + if (false) { +#else + if (useChroot) { +#endif + /* If sandboxing is enabled, put the actual TMPDIR underneath + an inaccessible root-owned directory, to prevent outside + access. + + On macOS, we don't use an actual chroot, so this isn't + possible. Any mitigation along these lines would have to be + done directly in the sandbox profile. */ + tmpDir = topTmpDir + "/build"; + createDir(tmpDir, 0700); + } else { + tmpDir = topTmpDir; + } + chownToBuilder(tmpDir); + + for (auto & [outputName, status] : initialOutputs) { + /* Set scratch path we'll actually use during the build. + + If we're not doing a chroot build, but we have some valid + output paths. Since we can't just overwrite or delete + them, we have to do hash rewriting: i.e. in the + environment/arguments passed to the build, we replace the + hashes of the valid outputs with unique dummy strings; + after the build, we discard the redirected outputs + corresponding to the valid outputs, and rewrite the + contents of the new outputs to replace the dummy strings + with the actual hashes. */ + auto scratchPath = + !status.known + ? makeFallbackPath(outputName) + : !needsHashRewrite() + /* Can always use original path in sandbox */ + ? status.known->path + : !status.known->isPresent() + /* If path doesn't yet exist can just use it */ + ? status.known->path + : buildMode != bmRepair && !status.known->isValid() + /* If we aren't repairing we'll delete a corrupted path, so we + can use original path */ + ? status.known->path + : /* If we are repairing or the path is totally valid, we'll need + to use a temporary path */ + makeFallbackPath(status.known->path); + scratchOutputs.insert_or_assign(outputName, scratchPath); + + /* Substitute output placeholders with the scratch output paths. + We'll use during the build. */ + inputRewrites[hashPlaceholder(outputName)] = store.printStorePath(scratchPath); + + /* Additional tasks if we know the final path a priori. */ + if (!status.known) continue; + auto fixedFinalPath = status.known->path; + + /* Additional tasks if the final and scratch are both known and + differ. */ + if (fixedFinalPath == scratchPath) continue; + + /* Ensure scratch path is ours to use. */ + deletePath(store.printStorePath(scratchPath)); + + /* Rewrite and unrewrite paths */ + { + std::string h1 { fixedFinalPath.hashPart() }; + std::string h2 { scratchPath.hashPart() }; + inputRewrites[h1] = h2; + } + + redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); + } + + /* Construct the environment passed to the builder. */ + initEnv(); + + writeStructuredAttrs(); + + /* Handle exportReferencesGraph(), if set. */ + if (!parsedDrv) { + for (auto & [fileName, ss] : drvOptions->exportReferencesGraph) { + StorePathSet storePathSet; + for (auto & storePathS : ss) { + if (!store.isInStore(storePathS)) + throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); + storePathSet.insert(store.toStorePath(storePathS).first); + } + /* Write closure info to . */ + writeFile(tmpDir + "/" + fileName, + store.makeValidityRegistration( + store.exportReferences(storePathSet, inputPaths), false, false)); + } + } + + if (useChroot) { + + /* Allow a user-configurable set of directories from the + host file system. */ + pathsInChroot.clear(); + + for (auto i : settings.sandboxPaths.get()) { + if (i.empty()) continue; + bool optional = false; + if (i[i.size() - 1] == '?') { + optional = true; + i.pop_back(); + } + size_t p = i.find('='); + if (p == std::string::npos) + pathsInChroot[i] = {i, optional}; + else + pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; + } + if (hasPrefix(store.storeDir, tmpDirInSandbox)) + { + throw Error("`sandbox-build-dir` must not contain the storeDir"); + } + pathsInChroot[tmpDirInSandbox] = tmpDir; + + /* Add the closure of store paths to the chroot. */ + StorePathSet closure; + for (auto & i : pathsInChroot) + try { + if (store.isInStore(i.second.source)) + store.computeFSClosure(store.toStorePath(i.second.source).first, closure); + } catch (InvalidPath & e) { + } catch (Error & e) { + e.addTrace({}, "while processing 'sandbox-paths'"); + throw; + } + for (auto & i : closure) { + auto p = store.printStorePath(i); + pathsInChroot.insert_or_assign(p, p); + } + + PathSet allowedPaths = settings.allowedImpureHostPrefixes; + + /* This works like the above, except on a per-derivation level */ + auto impurePaths = drvOptions->impureHostDeps; + + for (auto & i : impurePaths) { + bool found = false; + /* Note: we're not resolving symlinks here to prevent + giving a non-root user info about inaccessible + files. */ + Path canonI = canonPath(i); + /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ + for (auto & a : allowedPaths) { + Path canonA = canonPath(a); + if (isDirOrInDir(canonI, canonA)) { + found = true; + break; + } + } + if (!found) + throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", + store.printStorePath(drvPath), i); + + /* Allow files in drvOptions->impureHostDeps to be missing; e.g. + macOS 11+ has no /usr/lib/libSystem*.dylib */ + pathsInChroot[i] = {i, true}; + } + +#ifdef __linux__ + /* Create a temporary directory in which we set up the chroot + environment using bind-mounts. We put it in the Nix store + so that the build outputs can be moved efficiently from the + chroot to their final location. */ + auto chrootParentDir = store.Store::toRealPath(drvPath) + ".chroot"; + deletePath(chrootParentDir); + + /* Clean up the chroot directory automatically. */ + autoDelChroot = std::make_shared(chrootParentDir); + + printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootParentDir); + + if (mkdir(chrootParentDir.c_str(), 0700) == -1) + throw SysError("cannot create '%s'", chrootRootDir); + + chrootRootDir = chrootParentDir + "/root"; + + if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) + throw SysError("cannot create '%1%'", chrootRootDir); + + if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUIDCount() != 1 ? buildUser->getUID() : 0, buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", chrootRootDir); + + /* Create a writable /tmp in the chroot. Many builders need + this. (Of course they should really respect $TMPDIR + instead.) */ + Path chrootTmpDir = chrootRootDir + "/tmp"; + createDirs(chrootTmpDir); + chmod_(chrootTmpDir, 01777); + + /* Create a /etc/passwd with entries for the build user and the + nobody account. The latter is kind of a hack to support + Samba-in-QEMU. */ + createDirs(chrootRootDir + "/etc"); + if (drvOptions->useUidRange(*drv)) + chownToBuilder(chrootRootDir + "/etc"); + + if (drvOptions->useUidRange(*drv) && (!buildUser || buildUser->getUIDCount() < 65536)) + throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); + + /* Declare the build user's group so that programs get a consistent + view of the system (e.g., "id -gn"). */ + writeFile(chrootRootDir + "/etc/group", + fmt("root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n", sandboxGid())); + + /* Create /etc/hosts with localhost entry. */ + if (derivationType->isSandboxed()) + writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); + + /* Make the closure of the inputs available in the chroot, + rather than the whole Nix store. This prevents any access + to undeclared dependencies. Directories are bind-mounted, + while other inputs are hard-linked (since only directories + can be bind-mounted). !!! As an extra security + precaution, make the fake Nix store only writable by the + build user. */ + Path chrootStoreDir = chrootRootDir + store.storeDir; + createDirs(chrootStoreDir); + chmod_(chrootStoreDir, 01775); + + if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", chrootStoreDir); + + for (auto & i : inputPaths) { + auto p = store.printStorePath(i); + Path r = store.toRealPath(p); + pathsInChroot.insert_or_assign(p, r); + } + + /* If we're repairing, checking or rebuilding part of a + multiple-outputs derivation, it's possible that we're + rebuilding a path that is in settings.sandbox-paths + (typically the dependencies of /bin/sh). Throw them + out. */ + for (auto & i : drv->outputsAndOptPaths(store)) { + /* If the name isn't known a priori (i.e. floating + content-addressing derivation), the temporary location we use + should be fresh. Freshness means it is impossible that the path + is already in the sandbox, so we don't need to worry about + removing it. */ + if (i.second.second) + pathsInChroot.erase(store.printStorePath(*i.second.second)); + } + + if (cgroup) { + if (mkdir(cgroup->c_str(), 0755) != 0) + throw SysError("creating cgroup '%s'", *cgroup); + chownToBuilder(*cgroup); + chownToBuilder(*cgroup + "/cgroup.procs"); + chownToBuilder(*cgroup + "/cgroup.threads"); + //chownToBuilder(*cgroup + "/cgroup.subtree_control"); + } + +#else + if (drvOptions->useUidRange(*drv)) + throw Error("feature 'uid-range' is not supported on this platform"); + #ifdef __APPLE__ + /* We don't really have any parent prep work to do (yet?) + All work happens in the child, instead. */ + #else + throw Error("sandboxing builds is not supported on this platform"); + #endif +#endif + } else { + if (drvOptions->useUidRange(*drv)) + throw Error("feature 'uid-range' is only supported in sandboxed builds"); + } + + if (needsHashRewrite() && pathExists(homeDir)) + throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); + + if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { + printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); + auto args = useChroot ? Strings({store.printStorePath(drvPath), chrootRootDir}) : + Strings({ store.printStorePath(drvPath) }); + enum BuildHookState { + stBegin, + stExtraChrootDirs + }; + auto state = stBegin; + auto lines = runProgram(settings.preBuildHook, false, args); + auto lastPos = std::string::size_type{0}; + for (auto nlPos = lines.find('\n'); nlPos != std::string::npos; + nlPos = lines.find('\n', lastPos)) + { + auto line = lines.substr(lastPos, nlPos - lastPos); + lastPos = nlPos + 1; + if (state == stBegin) { + if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { + state = stExtraChrootDirs; + } else { + throw Error("unknown pre-build hook command '%1%'", line); + } + } else if (state == stExtraChrootDirs) { + if (line == "") { + state = stBegin; + } else { + auto p = line.find('='); + if (p == std::string::npos) + pathsInChroot[line] = line; + else + pathsInChroot[line.substr(0, p)] = line.substr(p + 1); + } + } + } + } + + /* Fire up a Nix daemon to process recursive Nix calls from the + builder. */ + if (drvOptions->getRequiredSystemFeatures(*drv).count("recursive-nix")) + startDaemon(); + + /* Run the builder. */ + printMsg(lvlChatty, "executing builder '%1%'", drv->builder); + printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args)); + for (auto & i : drv->env) + printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); + + /* Create the log file. */ + [[maybe_unused]] Path logFile = miscMethods.openLogFile(); + + /* Create a pseudoterminal to get the output of the builder. */ + builderOut = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut) + throw SysError("opening pseudoterminal master"); + + // FIXME: not thread-safe, use ptsname_r + std::string slaveName = ptsname(builderOut.get()); + + if (buildUser) { + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); + + if (chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + } +#ifdef __APPLE__ + else { + if (grantpt(builderOut.get())) + throw SysError("granting access to pseudoterminal slave"); + } +#endif + + if (unlockpt(builderOut.get())) + throw SysError("unlocking pseudoterminal"); + + /* Open the slave side of the pseudoterminal and use it as stderr. */ + auto openSlave = [&]() + { + AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); + + if (dup2(builderOut.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + }; + + buildResult.startTime = time(0); + + /* Fork a child to build the package. */ + +#ifdef __linux__ + if (useChroot) { + /* Set up private namespaces for the build: + + - The PID namespace causes the build to start as PID 1. + Processes outside of the chroot are not visible to those + on the inside, but processes inside the chroot are + visible from the outside (though with different PIDs). + + - The private mount namespace ensures that all the bind + mounts we do will only show up in this process and its + children, and will disappear automatically when we're + done. + + - The private network namespace ensures that the builder + cannot talk to the outside world (or vice versa). It + only has a private loopback interface. (Fixed-output + derivations are not run in a private network namespace + to allow functions like fetchurl to work.) + + - The IPC namespace prevents the builder from communicating + with outside processes using SysV IPC mechanisms (shared + memory, message queues, semaphores). It also ensures + that all IPC objects are destroyed when the builder + exits. + + - The UTS namespace ensures that builders see a hostname of + localhost rather than the actual hostname. + + We use a helper process to do the clone() to work around + clone() being broken in multi-threaded programs due to + at-fork handlers not being run. Note that we use + CLONE_PARENT to ensure that the real builder is parented to + us. + */ + + userNamespaceSync.create(); + + usingUserNamespace = userNamespacesSupported(); + + Pipe sendPid; + sendPid.create(); + + Pid helper = startProcess([&]() { + sendPid.readSide.close(); + + /* We need to open the slave early, before + CLONE_NEWUSER. Otherwise we get EPERM when running as + root. */ + openSlave(); + + try { + /* Drop additional groups here because we can't do it + after we've created the new user namespace. */ + if (setgroups(0, 0) == -1) { + if (errno != EPERM) + throw SysError("setgroups failed"); + if (settings.requireDropSupplementaryGroups) + throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); + } + + ProcessOptions options; + options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + if (derivationType->isSandboxed()) + options.cloneFlags |= CLONE_NEWNET; + if (usingUserNamespace) + options.cloneFlags |= CLONE_NEWUSER; + + pid_t child = startProcess([&]() { runChild(); }, options); + + writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); + _exit(0); + } catch (...) { + handleChildException(true); + _exit(1); + } + }); + + sendPid.writeSide.close(); + + if (helper.wait() != 0) { + processSandboxSetupMessages(); + // Only reached if the child process didn't send an exception. + throw Error("unable to start build process"); + } + + userNamespaceSync.readSide = -1; + + /* Close the write side to prevent runChild() from hanging + reading from this. */ + Finally cleanup([&]() { + userNamespaceSync.writeSide = -1; + }); + + auto ss = tokenizeString>(readLine(sendPid.readSide.get())); + assert(ss.size() == 1); + pid = string2Int(ss[0]).value(); + + if (usingUserNamespace) { + /* Set the UID/GID mapping of the builder's user namespace + such that the sandbox user maps to the build user, or to + the calling user (if build users are disabled). */ + uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); + uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); + uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1; + + writeFile("/proc/" + std::to_string(pid) + "/uid_map", + fmt("%d %d %d", sandboxUid(), hostUid, nrIds)); + + if (!buildUser || buildUser->getUIDCount() == 1) + writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + + writeFile("/proc/" + std::to_string(pid) + "/gid_map", + fmt("%d %d %d", sandboxGid(), hostGid, nrIds)); + } else { + debug("note: not using a user namespace"); + if (!buildUser) + throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); + } + + /* Now that we now the sandbox uid, we can write + /etc/passwd. */ + writeFile(chrootRootDir + "/etc/passwd", fmt( + "root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); + + /* Save the mount- and user namespace of the child. We have to do this + *before* the child does a chroot. */ + sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); + if (sandboxMountNamespace.get() == -1) + throw SysError("getting sandbox mount namespace"); + + if (usingUserNamespace) { + sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY); + if (sandboxUserNamespace.get() == -1) + throw SysError("getting sandbox user namespace"); + } + + /* Move the child into its own cgroup. */ + if (cgroup) + writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + + /* Signal the builder that we've updated its user namespace. */ + writeFull(userNamespaceSync.writeSide.get(), "1"); + + } else +#endif + { + pid = startProcess([&]() { + openSlave(); + runChild(); + }); + } + + /* parent */ + pid.setSeparatePG(true); + miscMethods.childStarted(); + + processSandboxSetupMessages(); +} + + +void DerivationBuilder::processSandboxSetupMessages() +{ + std::vector msgs; + while (true) { + std::string msg = [&]() { + try { + return readLine(builderOut.get()); + } catch (Error & e) { + auto status = pid.wait(); + e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", + store.printStorePath(drvPath), + statusToString(status), + concatStringsSep("|", msgs)); + throw; + } + }(); + if (msg.substr(0, 1) == "\2") break; + if (msg.substr(0, 1) == "\1") { + FdSource source(builderOut.get()); + auto ex = readError(source); + ex.addTrace({}, "while setting up the build environment"); + throw ex; + } + debug("sandbox setup: " + msg); + msgs.push_back(std::move(msg)); + } +} + + +void DerivationBuilder::initTmpDir() +{ + /* In a sandbox, for determinism, always use the same temporary + directory. */ +#ifdef __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif + + /* In non-structured mode, set all bindings either directory in the + environment or via a file, as specified by + `DerivationOptions::passAsFile`. */ + if (!parsedDrv) { + for (auto & i : drv->env) { + if (drvOptions->passAsFile.find(i.first) == drvOptions->passAsFile.end()) { + env[i.first] = i.second; + } else { + auto hash = hashString(HashAlgorithm::SHA256, i.first); + std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false); + Path p = tmpDir + "/" + fn; + writeFile(p, rewriteStrings(i.second, inputRewrites)); + chownToBuilder(p); + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; + } + } + + } + + /* For convenience, set an environment pointing to the top build + directory. */ + env["NIX_BUILD_TOP"] = tmpDirInSandbox; + + /* Also set TMPDIR and variants to point to this directory. */ + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; + + /* Explicitly set PWD to prevent problems with chroot builds. In + particular, dietlibc cannot figure out the cwd because the + inode of the current directory doesn't appear in .. (because + getdents returns the inode of the mount point). */ + env["PWD"] = tmpDirInSandbox; +} + + +void DerivationBuilder::initEnv() +{ + env.clear(); + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = store.storeDir; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores); + + initTmpDir(); + + /* Compatibility hack with Nix <= 0.7: if this is a fixed-output + derivation, tell the builder, so that for instance `fetchurl' + can skip checking the output. On older Nixes, this environment + variable won't be set, so `fetchurl' will do the check. */ + if (derivationType->isFixed()) env["NIX_OUTPUT_CHECKED"] = "1"; + + /* *Only* if this is a fixed-output derivation, propagate the + values of the environment variables specified in the + `impureEnvVars' attribute to the builder. This allows for + instance environment variables for proxy configuration such as + `http_proxy' to be easily passed to downloaders like + `fetchurl'. Passing such environment variables from the caller + to the builder is generally impure, but the output of + fixed-output derivations is by definition pure (since we + already know the cryptographic hash of the output). */ + if (!derivationType->isSandboxed()) { + auto & impureEnv = settings.impureEnv.get(); + if (!impureEnv.empty()) + experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv); + + for (auto & i : drvOptions->impureEnvVars){ + auto envVar = impureEnv.find(i); + if (envVar != impureEnv.end()) { + env[i] = envVar->second; + } else { + env[i] = getEnv(i).value_or(""); + } + } + } + + /* Currently structured log messages piggyback on stderr, but we + may change that in the future. So tell the builder which file + descriptor to use for that. */ + env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; +} + + +void DerivationBuilder::writeStructuredAttrs() +{ + if (parsedDrv) { + auto json = parsedDrv->prepareStructuredAttrs( + store, + *drvOptions, + inputPaths, + drv->outputs); + nlohmann::json rewritten; + for (auto & [i, v] : json["outputs"].get()) { + /* The placeholder must have a rewrite, so we use it to cover both the + cases where we know or don't know the output path ahead of time. */ + rewritten[i] = rewriteStrings((std::string) v, inputRewrites); + } + + json["outputs"] = rewritten; + + auto jsonSh = StructuredAttrs::writeShell(json); + + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.sh"); + env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh"; + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.json"); + env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json"; + } +} + + +void DerivationBuilder::startDaemon() +{ + experimentalFeatureSettings.require(Xp::RecursiveNix); + + Store::Params params; + params["path-info-cache-size"] = "0"; + params["store"] = store.storeDir; + if (auto & optRoot = getLocalStore().rootDir.get()) + params["root"] = *optRoot; + params["state"] = "/no-such-path"; + params["log"] = "/no-such-path"; + auto store = makeRestrictedStore(params, + ref(std::dynamic_pointer_cast(this->store.shared_from_this())), + *this); + + addedPaths.clear(); + + auto socketName = ".nix-socket"; + Path socketPath = tmpDir + "/" + socketName; + env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; + + daemonSocket = createUnixDomainSocket(socketPath, 0600); + + chownToBuilder(socketPath); + + daemonThread = std::thread([this, store]() { + + while (true) { + + /* Accept a connection. */ + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = accept(daemonSocket.get(), + (struct sockaddr *) &remoteAddr, &remoteAddrLen); + if (!remote) { + if (errno == EINTR || errno == EAGAIN) continue; + if (errno == EINVAL || errno == ECONNABORTED) break; + throw SysError("accepting connection"); + } + + unix::closeOnExec(remote.get()); + + debug("received daemon connection"); + + auto workerThread = std::thread([store, remote{std::move(remote)}]() { + try { + daemon::processConnection( + store, + FdSource(remote.get()), + FdSink(remote.get()), + NotTrusted, daemon::Recursive); + debug("terminated daemon connection"); + } catch (const Interrupted &) { + debug("interrupted daemon connection"); + } catch (SystemError &) { + ignoreExceptionExceptInterrupt(); + } + }); + + daemonWorkerThreads.push_back(std::move(workerThread)); + } + + debug("daemon shutting down"); + }); +} + + +void DerivationBuilder::stopDaemon() +{ + if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) { + // According to the POSIX standard, the 'shutdown' function should + // return an ENOTCONN error when attempting to shut down a socket that + // hasn't been connected yet. This situation occurs when the 'accept' + // function is called on a socket without any accepted connections, + // leaving the socket unconnected. While Linux doesn't seem to produce + // an error for sockets that have only been accepted, more + // POSIX-compliant operating systems like OpenBSD, macOS, and others do + // return the ENOTCONN error. Therefore, we handle this error here to + // avoid raising an exception for compliant behaviour. + if (errno == ENOTCONN) { + daemonSocket.close(); + } else { + throw SysError("shutting down daemon socket"); + } + } + + if (daemonThread.joinable()) + daemonThread.join(); + + // FIXME: should prune worker threads more quickly. + // FIXME: shutdown the client socket to speed up worker termination. + for (auto & thread : daemonWorkerThreads) + thread.join(); + daemonWorkerThreads.clear(); + + // release the socket. + daemonSocket.close(); +} + + +void DerivationBuilder::addDependency(const StorePath & path) +{ + if (isAllowed(path)) return; + + addedPaths.insert(path); + + /* If we're doing a sandbox build, then we have to make the path + appear in the sandbox. */ + if (useChroot) { + + debug("materialising '%s' in the sandbox", store.printStorePath(path)); + + #ifdef __linux__ + + Path source = store.Store::toRealPath(path); + Path target = chrootRootDir + store.printStorePath(path); + + if (pathExists(target)) { + // There is a similar debug message in doBind, so only run it in this block to not have double messages. + debug("bind-mounting %s -> %s", target, source); + throw Error("store path '%s' already exists in the sandbox", store.printStorePath(path)); + } + + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { + + if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) + throw SysError("entering sandbox user namespace"); + + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); + + doBind(source, target); + + _exit(0); + })); + + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", store.printStorePath(path)); + + #else + throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", + store.printStorePath(path)); + #endif + + } +} + +void DerivationBuilder::chownToBuilder(const Path & path) +{ + if (!buildUser) return; + if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", path); +} + + +void setupSeccomp() +{ +#ifdef __linux__ + if (!settings.filterSyscalls) return; +#if HAVE_SECCOMP + scmp_filter_ctx ctx; + + if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) + throw SysError("unable to initialize seccomp mode 2"); + + Finally cleanup([&]() { + seccomp_release(ctx); + }); + + constexpr std::string_view nativeSystem = NIX_LOCAL_SYSTEM; + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) + throw SysError("unable to add 32-bit seccomp architecture"); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) + throw SysError("unable to add X32 seccomp architecture"); + + if (nativeSystem == "aarch64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) + printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); + + if (nativeSystem == "mips64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPS) != 0) + printError("unable to add mips seccomp architecture"); + + if (nativeSystem == "mips64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPS64N32) != 0) + printError("unable to add mips64-*abin32 seccomp architecture"); + + if (nativeSystem == "mips64el-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL) != 0) + printError("unable to add mipsel seccomp architecture"); + + if (nativeSystem == "mips64el-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL64N32) != 0) + printError("unable to add mips64el-*abin32 seccomp architecture"); + + /* Prevent builders from creating setuid/setgid binaries. */ + for (int perm : { S_ISUID, S_ISGID }) { + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, + SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), NIX_SYSCALL_FCHMODAT2, 1, + SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + } + + /* Prevent builders from using EAs or ACLs. Not all filesystems + support these, and they're not allowed in the Nix store because + they're not representable in the NAR serialisation. */ + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) + throw SysError("unable to set 'no new privileges' seccomp attribute"); + + if (seccomp_load(ctx) != 0) + throw SysError("unable to load seccomp BPF program"); +#else + throw Error( + "seccomp is not supported on this platform; " + "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); +#endif +#endif +} + + +void DerivationBuilder::runChild() +{ + /* Warning: in the child we should absolutely not make any SQLite + calls! */ + + bool sendException = true; + + try { /* child */ + + commonChildInit(); + + try { + setupSeccomp(); + } catch (...) { + if (buildUser) throw; + } + + bool setUser = true; + + /* Make the contents of netrc and the CA certificate bundle + available to builtin:fetchurl (which may run under a + different uid and/or in a sandbox). */ + std::string netrcData; + std::string caFileData; + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") { + try { + netrcData = readFile(settings.netrcFile); + } catch (SystemError &) { } + + try { + caFileData = readFile(settings.caFile); + } catch (SystemError &) { } + } + +#ifdef __linux__ + if (useChroot) { + + userNamespaceSync.writeSide = -1; + + if (drainFD(userNamespaceSync.readSide.get()) != "1") + throw Error("user namespace initialisation failed"); + + userNamespaceSync.readSide = -1; + + if (derivationType->isSandboxed()) { + + /* Initialise the loopback interface. */ + AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); + if (!fd) throw SysError("cannot open IP socket"); + + struct ifreq ifr; + strcpy(ifr.ifr_name, "lo"); + ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; + if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) + throw SysError("cannot set loopback interface flags"); + } + + /* Set the hostname etc. to fixed values. */ + char hostname[] = "localhost"; + if (sethostname(hostname, sizeof(hostname)) == -1) + throw SysError("cannot set host name"); + char domainname[] = "(none)"; // kernel default + if (setdomainname(domainname, sizeof(domainname)) == -1) + throw SysError("cannot set domain name"); + + /* Make all filesystems private. This is necessary + because subtrees may have been mounted as "shared" + (MS_SHARED). (Systemd does this, for instance.) Even + though we have a private mount namespace, mounting + filesystems on top of a shared subtree still propagates + outside of the namespace. Making a subtree private is + local to the namespace, though, so setting MS_PRIVATE + does not affect the outside world. */ + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) + throw SysError("unable to make '/' private"); + + /* Bind-mount chroot directory to itself, to treat it as a + different filesystem from /, as needed for pivot_root. */ + if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) + throw SysError("unable to bind mount '%1%'", chrootRootDir); + + /* Bind-mount the sandbox's Nix store onto itself so that + we can mark it as a "shared" subtree, allowing bind + mounts made in *this* mount namespace to be propagated + into the child namespace created by the + unshare(CLONE_NEWNS) call below. + + Marking chrootRootDir as MS_SHARED causes pivot_root() + to fail with EINVAL. Don't know why. */ + Path chrootStoreDir = chrootRootDir + store.storeDir; + + if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) + throw SysError("unable to bind mount the Nix store", chrootStoreDir); + + if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) + throw SysError("unable to make '%s' shared", chrootStoreDir); + + /* Set up a nearly empty /dev, unless the user asked to + bind-mount the host /dev. */ + Strings ss; + if (pathsInChroot.find("/dev") == pathsInChroot.end()) { + createDirs(chrootRootDir + "/dev/shm"); + createDirs(chrootRootDir + "/dev/pts"); + ss.push_back("/dev/full"); + if (store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + ss.push_back("/dev/kvm"); + ss.push_back("/dev/null"); + ss.push_back("/dev/random"); + ss.push_back("/dev/tty"); + ss.push_back("/dev/urandom"); + ss.push_back("/dev/zero"); + createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); + createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); + createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); + createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); + } + + /* Fixed-output derivations typically need to access the + network, so give them access to /etc/resolv.conf and so + on. */ + if (!derivationType->isSandboxed()) { + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed-outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + + /* N.B. it is realistic that these paths might not exist. It + happens when testing Nix building fixed-output derivations + within a pure derivation. */ + for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" }) + if (pathExists(path)) + ss.push_back(path); + + if (settings.caFile != "" && pathExists(settings.caFile)) { + Path caFile = settings.caFile; + pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true); + } + } + + for (auto & i : ss) { + // For backwards-compatibiliy, resolve all the symlinks in the + // chroot paths + auto canonicalPath = canonPath(i, true); + pathsInChroot.emplace(i, canonicalPath); + } + + /* Bind-mount all the directories from the "host" + filesystem that we want in the chroot + environment. */ + for (auto & i : pathsInChroot) { + if (i.second.source == "/proc") continue; // backwards compatibility + + #if HAVE_EMBEDDED_SANDBOX_SHELL + if (i.second.source == "__embedded_sandbox_shell__") { + static unsigned char sh[] = { + #include "embedded-sandbox-shell.gen.hh" + }; + auto dst = chrootRootDir + i.first; + createDirs(dirOf(dst)); + writeFile(dst, std::string_view((const char *) sh, sizeof(sh))); + chmod_(dst, 0555); + } else + #endif + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); + } + + /* Bind a new instance of procfs on /proc. */ + createDirs(chrootRootDir + "/proc"); + if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) + throw SysError("mounting /proc"); + + /* Mount sysfs on /sys. */ + if (buildUser && buildUser->getUIDCount() != 1) { + createDirs(chrootRootDir + "/sys"); + if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) + throw SysError("mounting /sys"); + } + + /* Mount a new tmpfs on /dev/shm to ensure that whatever + the builder puts in /dev/shm is cleaned up automatically. */ + if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, + fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) + throw SysError("mounting /dev/shm"); + + /* Mount a new devpts on /dev/pts. Note that this + requires the kernel to be compiled with + CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case + if /dev/ptx/ptmx exists). */ + if (pathExists("/dev/pts/ptmx") && + !pathExists(chrootRootDir + "/dev/ptmx") + && !pathsInChroot.count("/dev/pts")) + { + if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) + { + createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); + + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + } else { + if (errno != EINVAL) + throw SysError("mounting /dev/pts"); + doBind("/dev/pts", chrootRootDir + "/dev/pts"); + doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); + } + } + + /* Make /etc unwritable */ + if (!drvOptions->useUidRange(*drv)) + chmod_(chrootRootDir + "/etc", 0555); + + /* Unshare this mount namespace. This is necessary because + pivot_root() below changes the root of the mount + namespace. This means that the call to setns() in + addDependency() would hide the host's filesystem, + making it impossible to bind-mount paths from the host + Nix store into the sandbox. Therefore, we save the + pre-pivot_root namespace in + sandboxMountNamespace. Since we made /nix/store a + shared subtree above, this allows addDependency() to + make paths appear in the sandbox. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("unsharing mount namespace"); + + /* Unshare the cgroup namespace. This means + /proc/self/cgroup will show the child's cgroup as '/' + rather than whatever it is in the parent. */ + if (cgroup && unshare(CLONE_NEWCGROUP) == -1) + throw SysError("unsharing cgroup namespace"); + + /* Do the chroot(). */ + if (chdir(chrootRootDir.c_str()) == -1) + throw SysError("cannot change directory to '%1%'", chrootRootDir); + + if (mkdir("real-root", 0500) == -1) + throw SysError("cannot create real-root directory"); + + if (pivot_root(".", "real-root") == -1) + throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); + + if (chroot(".") == -1) + throw SysError("cannot change root directory to '%1%'", chrootRootDir); + + if (umount2("real-root", MNT_DETACH) == -1) + throw SysError("cannot unmount real root filesystem"); + + if (rmdir("real-root") == -1) + throw SysError("cannot remove real-root directory"); + + /* Switch to the sandbox uid/gid in the user namespace, + which corresponds to the build user or calling user in + the parent namespace. */ + if (setgid(sandboxGid()) == -1) + throw SysError("setgid failed"); + if (setuid(sandboxUid()) == -1) + throw SysError("setuid failed"); + + setUser = false; + } +#endif + + if (chdir(tmpDirInSandbox.c_str()) == -1) + throw SysError("changing into '%1%'", tmpDir); + + /* Close all other file descriptors. */ + unix::closeExtraFDs(); + +#ifdef __linux__ + linux::setPersonality(drv->platform); +#endif + + /* Disable core dumps by default. */ + struct rlimit limit = { 0, RLIM_INFINITY }; + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + + /* Fill in the environment. */ + Strings envStrs; + for (auto & i : env) + envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); + + /* If we are running in `build-users' mode, then switch to the + user we allocated above. Make sure that we drop all root + privileges. Note that above we have closed all file + descriptors except std*, so that's safe. Also note that + setuid() when run as root sets the real, effective and + saved UIDs. */ + if (setUser && buildUser) { + /* Preserve supplementary groups of the build user, to allow + admins to specify groups such as "kvm". */ + auto gids = buildUser->getSupplementaryGIDs(); + if (setgroups(gids.size(), gids.data()) == -1) + throw SysError("cannot set supplementary groups of build user"); + + if (setgid(buildUser->getGID()) == -1 || + getgid() != buildUser->getGID() || + getegid() != buildUser->getGID()) + throw SysError("setgid failed"); + + if (setuid(buildUser->getUID()) == -1 || + getuid() != buildUser->getUID() || + geteuid() != buildUser->getUID()) + throw SysError("setuid failed"); + } + +#ifdef __APPLE__ + /* This has to appear before import statements. */ + std::string sandboxProfile = "(version 1)\n"; + + if (useChroot) { + + /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ + PathSet ancestry; + + /* We build the ancestry before adding all inputPaths to the store because we know they'll + all have the same parents (the store), and there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice */ + for (auto & i : pathsInChroot) { + Path cur = i.first; + while (cur.compare("/") != 0) { + cur = dirOf(cur); + ancestry.insert(cur); + } + } + + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost + path component this time, since it's typically /nix/store and we care about that. */ + Path cur = store.storeDir; + while (cur.compare("/") != 0) { + ancestry.insert(cur); + cur = dirOf(cur); + } + + /* Add all our input paths to the chroot */ + for (auto & i : inputPaths) { + auto p = store.printStorePath(i); + pathsInChroot[p] = p; + } + + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ + if (settings.darwinLogSandboxViolations) { + sandboxProfile += "(deny default)\n"; + } else { + sandboxProfile += "(deny default (with no-log))\n"; + } + + sandboxProfile += + #include "sandbox-defaults.sb" + ; + + if (!derivationType->isSandboxed()) + sandboxProfile += + #include "sandbox-network.sb" + ; + + /* Add the output paths we'll use at build-time to the chroot */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & [_, path] : scratchOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", store.printStorePath(path)); + + sandboxProfile += ")\n"; + + /* Our inputs (transitive dependencies and any impurities computed above) + + without file-write* allowed, access() incorrectly returns EPERM + */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + + // We create multiple allow lists, to avoid exceeding a limit in the darwin sandbox interpreter. + // See https://github.com/NixOS/nix/issues/4119 + // We split our allow groups approximately at half the actual limit, 1 << 16 + const size_t breakpoint = sandboxProfile.length() + (1 << 14); + for (auto & i : pathsInChroot) { + + if (sandboxProfile.length() >= breakpoint) { + debug("Sandbox break: %d %d", sandboxProfile.length(), breakpoint); + sandboxProfile += ")\n(allow file-read* file-write* process-exec\n"; + } + + if (i.first != i.second.source) + throw Error( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", + i.first, i.second.source); + + std::string path = i.first; + auto optSt = maybeLstat(path.c_str()); + if (!optSt) { + if (i.second.optional) + continue; + throw SysError("getting attributes of required path '%s", path); + } + if (S_ISDIR(optSt->st_mode)) + sandboxProfile += fmt("\t(subpath \"%s\")\n", path); + else + sandboxProfile += fmt("\t(literal \"%s\")\n", path); + } + sandboxProfile += ")\n"; + + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; + for (auto & i : ancestry) { + sandboxProfile += fmt("\t(literal \"%s\")\n", i); + } + sandboxProfile += ")\n"; + + sandboxProfile += drvOptions->additionalSandboxProfile; + } else + sandboxProfile += + #include "sandbox-minimal.sb" + ; + + debug("Generated sandbox profile:"); + debug(sandboxProfile); + + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms + to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */ + Path globalTmpDir = canonPath(defaultTempDir(), true); + + /* They don't like trailing slashes on subpath directives */ + while (!globalTmpDir.empty() && globalTmpDir.back() == '/') + globalTmpDir.pop_back(); + + if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { + Strings sandboxArgs; + sandboxArgs.push_back("_GLOBAL_TMP_DIR"); + sandboxArgs.push_back(globalTmpDir); + if (drvOptions->allowLocalNetworking) { + sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); + sandboxArgs.push_back("1"); + } + char * sandbox_errbuf = nullptr; + if (sandbox_init_with_parameters(sandboxProfile.c_str(), 0, stringsToCharPtrs(sandboxArgs).data(), &sandbox_errbuf)) { + writeFull(STDERR_FILENO, fmt("failed to configure sandbox: %s\n", sandbox_errbuf ? sandbox_errbuf : "(null)")); + _exit(1); + } + } +#endif + + /* Indicate that we managed to set up the build environment. */ + writeFull(STDERR_FILENO, std::string("\2\n")); + + sendException = false; + + /* Execute the program. This should not return. */ + if (drv->isBuiltin()) { + try { + logger = makeJSONLogger(getStandardError()); + + std::map outputs; + for (auto & e : drv->outputs) + outputs.insert_or_assign(e.first, + store.printStorePath(scratchOutputs.at(e.first))); + + if (drv->builder == "builtin:fetchurl") + builtinFetchurl(*drv, outputs, netrcData, caFileData); + else if (drv->builder == "builtin:buildenv") + builtinBuildenv(*drv, outputs); + else if (drv->builder == "builtin:unpack-channel") + builtinUnpackChannel(*drv, outputs); + else + throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8)); + _exit(0); + } catch (std::exception & e) { + writeFull(STDERR_FILENO, e.what() + std::string("\n")); + _exit(1); + } + } + + // Now builder is not builtin + + Strings args; + args.push_back(std::string(baseNameOf(drv->builder))); + + for (auto & i : drv->args) + args.push_back(rewriteStrings(i, inputRewrites)); + +#ifdef __APPLE__ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv->platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv->platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn(NULL, drv->builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#else + execve(drv->builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#endif + + throw SysError("executing '%1%'", drv->builder); + + } catch (...) { + handleChildException(sendException); + _exit(1); + } +} + + +SingleDrvOutputs DerivationBuilder::registerOutputs() +{ + std::map infos; + + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; + + Path checkSuffix = ".check"; + + std::exception_ptr delayedException; + + /* The paths that can be referenced are the input closures, the + output paths, and any paths that have been built via recursive + Nix calls. */ + StorePathSet referenceablePaths; + for (auto & p : inputPaths) referenceablePaths.insert(p); + for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); + for (auto & p : addedPaths) referenceablePaths.insert(p); + + /* FIXME `needsHashRewrite` should probably be removed and we get to the + real reason why we aren't using the chroot dir */ + auto toRealPathChroot = [&](const Path & p) -> Path { + return useChroot && !needsHashRewrite() + ? chrootRootDir + p + : store.toRealPath(p); + }; + + /* Check whether the output paths were created, and make all + output paths read-only. Then get the references of each output (that we + might need to register), so we can topologically sort them. For the ones + that are most definitely already installed, we just store their final + name so we can also use it in rewrites. */ + StringSet outputsToSort; + struct AlreadyRegistered { StorePath path; }; + struct PerhapsNeedToRegister { StorePathSet refs; }; + std::map> outputReferencesIfUnregistered; + std::map outputStats; + for (auto & [outputName, _] : drv->outputs) { + auto scratchOutput = get(scratchOutputs, outputName); + if (!scratchOutput) + throw BuildError( + "builder for '%s' has no scratch output for '%s'", + store.printStorePath(drvPath), outputName); + auto actualPath = toRealPathChroot(store.printStorePath(*scratchOutput)); + + outputsToSort.insert(outputName); + + /* Updated wanted info to remove the outputs we definitely don't need to register */ + auto initialOutput = get(initialOutputs, outputName); + if (!initialOutput) + throw BuildError( + "builder for '%s' has no initial output for '%s'", + store.printStorePath(drvPath), outputName); + auto & initialInfo = *initialOutput; + + /* Don't register if already valid, and not checking */ + initialInfo.wanted = buildMode == bmCheck + || !(initialInfo.known && initialInfo.known->isValid()); + if (!initialInfo.wanted) { + outputReferencesIfUnregistered.insert_or_assign( + outputName, + AlreadyRegistered { .path = initialInfo.known->path }); + continue; + } + + auto optSt = maybeLstat(actualPath.c_str()); + if (!optSt) + throw BuildError( + "builder for '%s' failed to produce output path for output '%s' at '%s'", + store.printStorePath(drvPath), outputName, actualPath); + struct stat & st = *optSt; + +#ifndef __CYGWIN__ + /* Check that the output is not group or world writable, as + that means that someone else can have interfered with the + build. Also, the output should be owned by the build + user. */ + if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || + (buildUser && st.st_uid != buildUser->getUID())) + throw BuildError( + "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", + actualPath, outputName); +#endif + + /* Canonicalise first. This ensures that the path we're + rewriting doesn't contain a hard link to /etc/shadow or + something like that. */ + canonicalisePathMetaData( + actualPath, + buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, + inodesSeen); + + bool discardReferences = false; + if (auto udr = get(drvOptions->unsafeDiscardReferences, outputName)) { + discardReferences = *udr; + } + + StorePathSet references; + if (discardReferences) + debug("discarding references of output '%s'", outputName); + else { + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + references = scanForReferences(blank, actualPath, referenceablePaths); + } + + outputReferencesIfUnregistered.insert_or_assign( + outputName, + PerhapsNeedToRegister { .refs = references }); + outputStats.insert_or_assign(outputName, std::move(st)); + } + + auto sortedOutputNames = topoSort(outputsToSort, + {[&](const std::string & name) { + auto orifu = get(outputReferencesIfUnregistered, name); + if (!orifu) + throw BuildError( + "no output reference for '%s' in build of '%s'", + name, store.printStorePath(drvPath)); + return std::visit(overloaded { + /* Since we'll use the already installed versions of these, we + can treat them as leaves and ignore any references they + have. */ + [&](const AlreadyRegistered &) { return StringSet {}; }, + [&](const PerhapsNeedToRegister & refs) { + StringSet referencedOutputs; + /* FIXME build inverted map up front so no quadratic waste here */ + for (auto & r : refs.refs) + for (auto & [o, p] : scratchOutputs) + if (r == p) + referencedOutputs.insert(o); + return referencedOutputs; + }, + }, *orifu); + }}, + {[&](const std::string & path, const std::string & parent) { + // TODO with more -vvvv also show the temporary paths for manual inspection. + return BuildError( + "cycle detected in build of '%s' in the references of output '%s' from output '%s'", + store.printStorePath(drvPath), path, parent); + }}); + + std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); + + OutputPathMap finalOutputs; + + for (auto & outputName : sortedOutputNames) { + auto output = get(drv->outputs, outputName); + auto scratchPath = get(scratchOutputs, outputName); + assert(output && scratchPath); + auto actualPath = toRealPathChroot(store.printStorePath(*scratchPath)); + + auto finish = [&](StorePath finalStorePath) { + /* Store the final path */ + finalOutputs.insert_or_assign(outputName, finalStorePath); + /* The rewrite rule will be used in downstream outputs that refer to + use. This is why the topological sort is essential to do first + before this for loop. */ + if (*scratchPath != finalStorePath) + outputRewrites[std::string { scratchPath->hashPart() }] = std::string { finalStorePath.hashPart() }; + }; + + auto orifu = get(outputReferencesIfUnregistered, outputName); + assert(orifu); + + std::optional referencesOpt = std::visit(overloaded { + [&](const AlreadyRegistered & skippedFinalPath) -> std::optional { + finish(skippedFinalPath.path); + return std::nullopt; + }, + [&](const PerhapsNeedToRegister & r) -> std::optional { + return r.refs; + }, + }, *orifu); + + if (!referencesOpt) + continue; + auto references = *referencesOpt; + + auto rewriteOutput = [&](const StringMap & rewrites) { + /* Apply hash rewriting if necessary. */ + if (!rewrites.empty()) { + debug("rewriting hashes in '%1%'; cross fingers", actualPath); + + /* FIXME: Is this actually streaming? */ + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink(rewrites, nextSink); + dumpPath(actualPath, rsink); + rsink.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); + deletePath(actualPath); + movePath(tmpPath, actualPath); + + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, {}, inodesSeen); + } + }; + + auto rewriteRefs = [&]() -> StoreReferences { + /* In the CA case, we need the rewritten refs to calculate the + final path, therefore we look for a *non-rewritten + self-reference, and use a bool rather try to solve the + computationally intractable fixed point. */ + StoreReferences res { + .self = false, + }; + for (auto & r : references) { + auto name = r.name(); + auto origHash = std::string { r.hashPart() }; + if (r == *scratchPath) { + res.self = true; + } else if (auto outputRewrite = get(outputRewrites, origHash)) { + std::string newRef = *outputRewrite; + newRef += '-'; + newRef += name; + res.others.insert(StorePath { newRef }); + } else { + res.others.insert(r); + } + } + return res; + }; + + auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo { + auto st = get(outputStats, outputName); + if (!st) + throw BuildError( + "output path %1% without valid stats info", + actualPath); + if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat) + { + /* The output path should be a regular file without execute permission. */ + if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) + throw BuildError( + "output path '%1%' should be a non-executable regular file " + "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", + actualPath); + } + rewriteOutput(outputRewrites); + /* FIXME optimize and deduplicate with addToStore */ + std::string oldHashPart { scratchPath->hashPart() }; + auto got = [&]{ + auto fim = outputHash.method.getFileIngestionMethod(); + switch (fim) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::NixArchive: + { + HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; + auto fim = outputHash.method.getFileIngestionMethod(); + dumpPath( + {getFSSourceAccessor(), CanonPath(actualPath)}, + caSink, + (FileSerialisationMethod) fim); + return caSink.finish().first; + } + case FileIngestionMethod::Git: { + return git::dumpHash( + outputHash.hashAlgo, + {getFSSourceAccessor(), CanonPath(actualPath)}).hash; + } + } + assert(false); + }(); + + ValidPathInfo newInfo0 { + store, + outputPathName(drv->name, outputName), + ContentAddressWithReferences::fromParts( + outputHash.method, + std::move(got), + rewriteRefs()), + Hash::dummy, + }; + if (*scratchPath != newInfo0.path) { + // If the path has some self-references, we need to rewrite + // them. + // (note that this doesn't invalidate the ca hash we calculated + // above because it's computed *modulo the self-references*, so + // it already takes this rewrite into account). + rewriteOutput( + StringMap{{oldHashPart, + std::string(newInfo0.path.hashPart())}}); + } + + { + HashResult narHashAndSize = hashPath( + {getFSSourceAccessor(), CanonPath(actualPath)}, + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); + newInfo0.narHash = narHashAndSize.first; + newInfo0.narSize = narHashAndSize.second; + } + + assert(newInfo0.ca); + return newInfo0; + }; + + ValidPathInfo newInfo = std::visit(overloaded { + + [&](const DerivationOutput::InputAddressed & output) { + /* input-addressed case */ + auto requiredFinalPath = output.path; + /* Preemptively add rewrite rule for final hash, as that is + what the NAR hash will use rather than normalized-self references */ + if (*scratchPath != requiredFinalPath) + outputRewrites.insert_or_assign( + std::string { scratchPath->hashPart() }, + std::string { requiredFinalPath.hashPart() }); + rewriteOutput(outputRewrites); + HashResult narHashAndSize = hashPath( + {getFSSourceAccessor(), CanonPath(actualPath)}, + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); + ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; + newInfo0.narSize = narHashAndSize.second; + auto refs = rewriteRefs(); + newInfo0.references = std::move(refs.others); + if (refs.self) + newInfo0.references.insert(newInfo0.path); + return newInfo0; + }, + + [&](const DerivationOutput::CAFixed & dof) { + auto & wanted = dof.ca.hash; + + // Replace the output by a fresh copy of itself to make sure + // that there's no stale file descriptor pointing to it + Path tmpOutput = actualPath + ".tmp"; + copyFile( + std::filesystem::path(actualPath), + std::filesystem::path(tmpOutput), true); + + std::filesystem::rename(tmpOutput, actualPath); + + auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { + .method = dof.ca.method, + .hashAlgo = wanted.algo, + }); + + /* Check wanted hash */ + assert(newInfo0.ca); + auto & got = newInfo0.ca->hash; + if (wanted != got) { + /* Throw an error after registering the path as + valid. */ + miscMethods.noteHashMismatch(); + delayedException = std::make_exception_ptr( + BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", + store.printStorePath(drvPath), + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true))); + } + if (!newInfo0.references.empty()) { + auto numViolations = newInfo.references.size(); + delayedException = std::make_exception_ptr( + BuildError("fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", + store.printStorePath(drvPath), + numViolations, + store.printStorePath(*newInfo.references.begin()))); + } + + return newInfo0; + }, + + [&](const DerivationOutput::CAFloating & dof) { + return newInfoFromCA(dof); + }, + + [&](const DerivationOutput::Deferred &) -> ValidPathInfo { + // No derivation should reach that point without having been + // rewritten first + assert(false); + }, + + [&](const DerivationOutput::Impure & doi) { + return newInfoFromCA(DerivationOutput::CAFloating { + .method = doi.method, + .hashAlgo = doi.hashAlgo, + }); + }, + + }, output->raw); + + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, {}, inodesSeen); + + /* Calculate where we'll move the output files. In the checking case we + will leave leave them where they are, for now, rather than move to + their usual "final destination" */ + auto finalDestPath = store.printStorePath(newInfo.path); + + /* Lock final output path, if not already locked. This happens with + floating CA derivations and hash-mismatching fixed-output + derivations. */ + PathLocks dynamicOutputLock; + dynamicOutputLock.setDeletion(true); + auto optFixedPath = output->path(store, drv->name, outputName); + if (!optFixedPath || + store.printStorePath(*optFixedPath) != finalDestPath) + { + assert(newInfo.ca); + dynamicOutputLock.lockPaths({store.toRealPath(finalDestPath)}); + } + + /* Move files, if needed */ + if (store.toRealPath(finalDestPath) != actualPath) { + if (buildMode == bmRepair) { + /* Path already exists, need to replace it */ + replaceValidPath(store.toRealPath(finalDestPath), actualPath); + actualPath = store.toRealPath(finalDestPath); + } else if (buildMode == bmCheck) { + /* Path already exists, and we want to compare, so we leave out + new path in place. */ + } else if (store.isValidPath(newInfo.path)) { + /* Path already exists because CA path produced by something + else. No moving needed. */ + assert(newInfo.ca); + } else { + auto destPath = store.toRealPath(finalDestPath); + deletePath(destPath); + movePath(actualPath, destPath); + actualPath = destPath; + } + } + + auto & localStore = getLocalStore(); + + if (buildMode == bmCheck) { + + if (!store.isValidPath(newInfo.path)) continue; + ValidPathInfo oldInfo(*store.queryPathInfo(newInfo.path)); + if (newInfo.narHash != oldInfo.narHash) { + miscMethods.noteCheckMismatch(); + if (settings.runDiffHook || settings.keepFailed) { + auto dst = store.toRealPath(finalDestPath + checkSuffix); + deletePath(dst); + movePath(actualPath, dst); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + finalDestPath, dst, store.printStorePath(drvPath), tmpDir); + + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", + store.printStorePath(drvPath), store.toRealPath(finalDestPath), dst); + } else + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", + store.printStorePath(drvPath), store.toRealPath(finalDestPath)); + } + + /* Since we verified the build, it's now ultimately trusted. */ + if (!oldInfo.ultimate) { + oldInfo.ultimate = true; + localStore.signPathInfo(oldInfo); + localStore.registerValidPaths({{oldInfo.path, oldInfo}}); + } + + continue; + } + + /* For debugging, print out the referenced and unreferenced paths. */ + for (auto & i : inputPaths) { + if (references.count(i)) + debug("referenced input: '%1%'", store.printStorePath(i)); + else + debug("unreferenced input: '%1%'", store.printStorePath(i)); + } + + localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() + miscMethods.markContentsGood(newInfo.path); + + newInfo.deriver = drvPath; + newInfo.ultimate = true; + localStore.signPathInfo(newInfo); + + finish(newInfo.path); + + /* If it's a CA path, register it right away. This is necessary if it + isn't statically known so that we can safely unlock the path before + the next iteration */ + if (newInfo.ca) + localStore.registerValidPaths({{newInfo.path, newInfo}}); + + infos.emplace(outputName, std::move(newInfo)); + } + + if (buildMode == bmCheck) { + /* In case of fixed-output derivations, if there are + mismatches on `--check` an error must be thrown as this is + also a source for non-determinism. */ + if (delayedException) + std::rethrow_exception(delayedException); + return miscMethods.assertPathValidity(); + } + + /* Apply output checks. */ + checkOutputs(infos); + + /* Register each output path as valid, and register the sets of + paths referenced by each of them. If there are cycles in the + outputs, this will fail. */ + { + auto & localStore = getLocalStore(); + + ValidPathInfos infos2; + for (auto & [outputName, newInfo] : infos) { + infos2.insert_or_assign(newInfo.path, newInfo); + } + localStore.registerValidPaths(infos2); + } + + /* In case of a fixed-output derivation hash mismatch, throw an + exception now that we have registered the output as valid. */ + if (delayedException) + std::rethrow_exception(delayedException); + + /* If we made it this far, we are sure the output matches the derivation + (since the delayedException would be a fixed output CA mismatch). That + means it's safe to link the derivation to the output hash. We must do + that for floating CA derivations, which otherwise couldn't be cached, + but it's fine to do in all cases. */ + SingleDrvOutputs builtOutputs; + + for (auto & [outputName, newInfo] : infos) { + auto oldinfo = get(initialOutputs, outputName); + assert(oldinfo); + auto thisRealisation = Realisation { + .id = DrvOutput { + oldinfo->outputHash, + outputName + }, + .outPath = newInfo.path + }; + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) + && !drv->type().isImpure()) + { + store.signRealisation(thisRealisation); + store.registerDrvOutput(thisRealisation); + } + builtOutputs.emplace(outputName, thisRealisation); + } + + return builtOutputs; +} + + +void DerivationBuilder::checkOutputs(const std::map & outputs) +{ + std::map outputsByPath; + for (auto & output : outputs) + outputsByPath.emplace(store.printStorePath(output.second.path), output.second); + + for (auto & output : outputs) { + auto & outputName = output.first; + auto & info = output.second; + + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const StorePath & path) + { + uint64_t closureSize = 0; + StorePathSet pathsDone; + std::queue pathsLeft; + pathsLeft.push(path); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front(); + pathsLeft.pop(); + if (!pathsDone.insert(path).second) continue; + + auto i = outputsByPath.find(store.printStorePath(path)); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto & ref : i->second.references) + pathsLeft.push(ref); + } else { + auto info = store.queryPathInfo(path); + closureSize += info->narSize; + for (auto & ref : info->references) + pathsLeft.push(ref); + } + } + + return std::make_pair(std::move(pathsDone), closureSize); + }; + + auto applyChecks = [&](const DerivationOptions::OutputChecks & checks) + { + if (checks.maxSize && info.narSize > *checks.maxSize) + throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", + store.printStorePath(info.path), info.narSize, *checks.maxSize); + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) + throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", + store.printStorePath(info.path), closureSize, *checks.maxClosureSize); + } + + auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive) + { + /* Parse a list of reference specifiers. Each element must + either be a store path, or the symbolic name of the output + of the derivation (such as `out'). */ + StorePathSet spec; + for (auto & i : value) { + if (store.isStorePath(i)) + spec.insert(store.parseStorePath(i)); + else if (auto output = get(outputs, i)) + spec.insert(output->path); + else { + std::string outputsListing = concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; }); + throw BuildError("derivation '%s' output check for '%s' contains an illegal reference specifier '%s'," + " expected store path or output name (one of [%s])", + store.printStorePath(drvPath), outputName, i, outputsListing); + } + } + + auto used = recursive + ? getClosure(info.path).first + : info.references; + + if (recursive && checks.ignoreSelfRefs) + used.erase(info.path); + + StorePathSet badPaths; + + for (auto & i : used) + if (allowed) { + if (!spec.count(i)) + badPaths.insert(i); + } else { + if (spec.count(i)) + badPaths.insert(i); + } + + if (!badPaths.empty()) { + std::string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n "; + badPathsStr += store.printStorePath(i); + } + throw BuildError("output '%s' is not allowed to refer to the following paths:%s", + store.printStorePath(info.path), badPathsStr); + } + }; + + /* Mandatory check: absent whitelist, and present but empty + whitelist mean very different things. */ + if (auto & refs = checks.allowedReferences) { + checkRefs(*refs, true, false); + } + if (auto & refs = checks.allowedRequisites) { + checkRefs(*refs, true, true); + } + + /* Optimization: don't need to do anything when + disallowed and empty set. */ + if (!checks.disallowedReferences.empty()) { + checkRefs(checks.disallowedReferences, false, false); + } + if (!checks.disallowedRequisites.empty()) { + checkRefs(checks.disallowedRequisites, false, true); + } + }; + + std::visit(overloaded{ + [&](const DerivationOptions::OutputChecks & checks) { + applyChecks(checks); + }, + [&](const std::map & checksPerOutput) { + if (auto outputChecks = get(checksPerOutput, outputName)) + + applyChecks(*outputChecks); + }, + }, drvOptions->outputChecks); + } +} + + +void DerivationBuilder::deleteTmpDir(bool force) +{ + if (topTmpDir != "") { + /* Don't keep temporary directories for builtins because they + might have privileged stuff (like a copy of netrc). */ + if (settings.keepFailed && !force && !drv->isBuiltin()) { + printError("note: keeping build directory '%s'", tmpDir); + chmod(topTmpDir.c_str(), 0755); + chmod(tmpDir.c_str(), 0755); + } + else + deletePath(topTmpDir); + topTmpDir = ""; + tmpDir = ""; + } +} + + +bool LocalDerivationGoal::isReadDesc(int fd) +{ + return (hook && DerivationGoal::isReadDesc(fd)) || + (!hook && fd == builder.builderOut.get()); +} + + +StorePath DerivationBuilder::makeFallbackPath(OutputNameView outputName) +{ + // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path + // See doc/manual/source/protocols/store-path.md for details + // TODO: We may want to separate the responsibilities of constructing the path fingerprint and of actually doing the hashing + auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName); + return store.makeStorePath( + pathType, + // pass an all-zeroes hash + Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName)); +} + + +StorePath DerivationBuilder::makeFallbackPath(const StorePath & path) +{ + // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path + // See doc/manual/source/protocols/store-path.md for details + auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()); + return store.makeStorePath( + pathType, + // pass an all-zeroes hash + Hash(HashAlgorithm::SHA256), path.name()); +} + + +} diff --git a/src/libstore/unix/include/nix/store/build/derivation-builder.hh b/src/libstore/unix/include/nix/store/build/derivation-builder.hh new file mode 100644 index 000000000..973eef26e --- /dev/null +++ b/src/libstore/unix/include/nix/store/build/derivation-builder.hh @@ -0,0 +1,3395 @@ +#include "nix/store/build/local-derivation-goal.hh" +#include "nix/store/local-store.hh" +#include "nix/util/processes.hh" +#include "nix/store/indirect-root-store.hh" +#include "nix/store/build/hook-instance.hh" +#include "nix/store/build/worker.hh" +#include "nix/store/builtins.hh" +#include "nix/store/builtins/buildenv.hh" +#include "nix/store/path-references.hh" +#include "nix/util/finally.hh" +#include "nix/util/util.hh" +#include "nix/util/archive.hh" +#include "nix/util/git.hh" +#include "nix/util/compression.hh" +#include "nix/store/daemon.hh" +#include "nix/util/topo-sort.hh" +#include "nix/util/callback.hh" +#include "nix/util/json-utils.hh" +#include "nix/util/current-process.hh" +#include "nix/store/build/child.hh" +#include "nix/util/unix-domain-socket.hh" +#include "nix/store/posix-fs-canonicalise.hh" +#include "nix/util/posix-source-accessor.hh" +#include "nix/store/restricted-store.hh" +#include "nix/store/config.hh" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "store-config-private.hh" + +#if HAVE_STATVFS +#include +#endif + +/* Includes required for chroot support. */ +#ifdef __linux__ +# include "linux/fchmodat2-compat.hh" +# include +# include +# include +# include +# include +# include +# include +# include +# include "nix/util/namespaces.hh" +# if HAVE_SECCOMP +# include +# endif +# define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) +# include "nix/util/cgroup.hh" +# include "nix/store/personality.hh" +#endif + +#ifdef __APPLE__ +#include +#include +#include + +/* This definition is undocumented but depended upon by all major browsers. */ +extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf); +#endif + +#include +#include +#include + +#include "nix/util/strings.hh" +#include "nix/util/signals.hh" + +#include "store-config-private.hh" + +namespace nix { + +/** + * Parameters by (mostly) `const` reference for `DerivationBuilder`. + */ +struct DerivationBuilderParams +{ + /** The path of the derivation. */ + const StorePath & drvPath; + + BuildResult & buildResult; + + /** + * The derivation stored at drvPath. + * + * @todo Remove double indirection by delaying when this is + * initialized. + */ + const std::unique_ptr & drv; + + const std::unique_ptr & parsedDrv; + const std::unique_ptr & drvOptions; + + // The remainder is state held during the build. + + /** + * All input paths (that is, the union of FS closures of the + * immediate input paths). + */ + const StorePathSet & inputPaths; + + /** + * @note we do in fact mutate this + */ + std::map & initialOutputs; + + const BuildMode & buildMode; + + DerivationBuilderParams( + const StorePath & drvPath, + const BuildMode & buildMode, + BuildResult & buildResult, + const std::unique_ptr & drv, + const std::unique_ptr & parsedDrv, + const std::unique_ptr & drvOptions, + const StorePathSet & inputPaths, + std::map & initialOutputs) + : drvPath{drvPath} + , buildResult{buildResult} + , drv{drv} + , parsedDrv{parsedDrv} + , drvOptions{drvOptions} + , inputPaths{inputPaths} + , initialOutputs{initialOutputs} + , buildMode{buildMode} + { } + + DerivationBuilderParams(DerivationBuilderParams &&) = default; +}; + +/** + * Callbacks that `DerivationBuilder` needs. + */ +struct DerivationBuilderCallbacks +{ + /** + * Open a log file and a pipe to it. + */ + virtual Path openLogFile() = 0; + + /** + * Close the log file. + */ + virtual void closeLogFile() = 0; + + /** + * Aborts if any output is not valid or corrupt, and otherwise + * returns a 'SingleDrvOutputs' structure containing all outputs. + * + * @todo Probably should just be in `DerivationGoal`. + */ + virtual SingleDrvOutputs assertPathValidity() = 0; + + virtual void appendLogTailErrorMsg(std::string & msg) = 0; + + /** + * Hook up `builderOut` to some mechanism to ingest the log + * + * @todo this should be reworked + */ + virtual void childStarted() = 0; + + /** + * @todo this should be reworked + */ + virtual void childTerminated() = 0; + + virtual void noteHashMismatch(void) = 0; + virtual void noteCheckMismatch(void) = 0; + + virtual void markContentsGood(const StorePath & path) = 0; +}; + +/** + * This class represents the state for building locally. + * + * @todo Ideally, it would not be a class, but a single function. + * However, besides the main entry point, there are a few more methods + * which are externally called, and need to be gotten rid of. There are + * also some virtual methods (either directly here or inherited from + * `DerivationBuilderCallbacks`, a stop-gap) that represent outgoing + * rather than incoming call edges that either should be removed, or + * become (higher order) function parameters. + */ +class DerivationBuilder : public RestrictionContext, DerivationBuilderParams +{ + Store & store; + + DerivationBuilderCallbacks & miscMethods; + +public: + + DerivationBuilder( + Store & store, + DerivationBuilderCallbacks & miscMethods, + DerivationBuilderParams params) + : DerivationBuilderParams{std::move(params)} + , store{store} + , miscMethods{miscMethods} + { } + + LocalStore & getLocalStore(); + + /** + * User selected for running the builder. + */ + std::unique_ptr buildUser; + + /** + * The process ID of the builder. + */ + Pid pid; + +private: + + /** + * The cgroup of the builder, if any. + */ + std::optional cgroup; + + /** + * The temporary directory used for the build. + */ + Path tmpDir; + + /** + * The top-level temporary directory. `tmpDir` is either equal to + * or a child of this directory. + */ + Path topTmpDir; + + /** + * The path of the temporary directory in the sandbox. + */ + Path tmpDirInSandbox; + +public: + + /** + * Master side of the pseudoterminal used for the builder's + * standard output/error. + */ + AutoCloseFD builderOut; + +private: + + /** + * Pipe for synchronising updates to the builder namespaces. + */ + Pipe userNamespaceSync; + + /** + * The mount namespace and user namespace of the builder, used to add additional + * paths to the sandbox as a result of recursive Nix calls. + */ + AutoCloseFD sandboxMountNamespace; + AutoCloseFD sandboxUserNamespace; + + /** + * On Linux, whether we're doing the build in its own user + * namespace. + */ + bool usingUserNamespace = true; + + /** + * Whether we're currently doing a chroot build. + */ + bool useChroot = false; + + /** + * The root of the chroot environment. + */ + Path chrootRootDir; + + /** + * RAII object to delete the chroot directory. + */ + std::shared_ptr autoDelChroot; + + /** + * The sort of derivation we are building. + * + * Just a cached value, can be recomputed from `drv`. + */ + std::optional derivationType; + + /** + * Stuff we need to pass to initChild(). + */ + struct ChrootPath { + Path source; + bool optional; + ChrootPath(Path source = "", bool optional = false) + : source(source), optional(optional) + { } + }; + typedef map PathsInChroot; // maps target path to source path + PathsInChroot pathsInChroot; + + typedef map Environment; + Environment env; + + /** + * Hash rewriting. + */ + StringMap inputRewrites, outputRewrites; + typedef map RedirectedOutputs; + RedirectedOutputs redirectedOutputs; + + /** + * The output paths used during the build. + * + * - Input-addressed derivations or fixed content-addressed outputs are + * sometimes built when some of their outputs already exist, and can not + * be hidden via sandboxing. We use temporary locations instead and + * rewrite after the build. Otherwise the regular predetermined paths are + * put here. + * + * - Floating content-addressing derivations do not know their final build + * output paths until the outputs are hashed, so random locations are + * used, and then renamed. The randomness helps guard against hidden + * self-references. + */ + OutputPathMap scratchOutputs; + + uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } + + const static Path homeDir; + + /** + * The recursive Nix daemon socket. + */ + AutoCloseFD daemonSocket; + + /** + * The daemon main thread. + */ + std::thread daemonThread; + + /** + * The daemon worker threads. + */ + std::vector daemonWorkerThreads; + + const StorePathSet & originalPaths() override + { + return inputPaths; + } + + bool isAllowed(const StorePath & path) override + { + return inputPaths.count(path) || addedPaths.count(path); + } + bool isAllowed(const DrvOutput & id) override + { + return addedDrvOutputs.count(id); + } + + bool isAllowed(const DerivedPath & req); + + friend struct RestrictedStore; + + /** + * Whether we need to perform hash rewriting if there are valid output paths. + */ + bool needsHashRewrite(); + +public: + + /** + * Set up build environment / sandbox, acquiring resources (e.g. + * locks as needed). After this is run, the builder should be + * started. + * + * @returns true if successful, false if we could not acquire a build + * user. In that case, the caller must wait and then try again. + */ + bool prepareBuild(); + + /** + * Start building a derivation. + */ + void startBuilder(); + + /** + * Tear down build environment after the builder exits (either on + * its own or if it is killed). + * + * @returns The first case indicates failure during output + * processing. A status code and exception are returned, providing + * more information. The second case indicates success, and + * realisations for each output of the derivation are returned. + */ + std::variant, SingleDrvOutputs> unprepareBuild(); + +private: + + /** + * Fill in the environment for the builder. + */ + void initEnv(); + + /** + * Process messages send by the sandbox initialization. + */ + void processSandboxSetupMessages(); + + /** + * Setup tmp dir location. + */ + void initTmpDir(); + + /** + * Write a JSON file containing the derivation attributes. + */ + void writeStructuredAttrs(); + + /** + * Start an in-process nix daemon thread for recursive-nix. + */ + void startDaemon(); + +public: + + /** + * Stop the in-process nix daemon thread. + * @see startDaemon + */ + void stopDaemon(); + +private: + + void addDependency(const StorePath & path) override; + + /** + * Make a file owned by the builder. + */ + void chownToBuilder(const Path & path); + + /** + * Run the builder's process. + */ + void runChild(); + + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ + SingleDrvOutputs registerOutputs(); + + /** + * Check that an output meets the requirements specified by the + * 'outputChecks' attribute (or the legacy + * '{allowed,disallowed}{References,Requisites}' attributes). + */ + void checkOutputs(const std::map & outputs); + +public: + + /** + * Delete the temporary directory, if we have one. + */ + void deleteTmpDir(bool force); + + /** + * Kill any processes running under the build user UID or in the + * cgroup of the build. + */ + void killSandbox(bool getStats); + +private: + + bool cleanupDecideWhetherDiskFull(); + + /** + * Create alternative path calculated from but distinct from the + * input, so we can avoid overwriting outputs (or other store paths) + * that already exist. + */ + StorePath makeFallbackPath(const StorePath & path); + + /** + * Make a path to another based on the output name along with the + * derivation hash. + * + * @todo Add option to randomize, so we can audit whether our + * rewrites caught everything + */ + StorePath makeFallbackPath(OutputNameView outputName); +}; + +/** + * This hooks up `DerivationBuilder` to the scheduler / goal machinary. + * + * @todo Eventually, this shouldn't exist, because `DerivationGoal` can + * just choose to use `DerivationBuilder` or its remote-building + * equalivalent directly, at the "value level" rather than "class + * inheritance hierarchy" level. + */ +struct LocalDerivationGoal : DerivationGoal, DerivationBuilderCallbacks +{ + DerivationBuilder builder; + + LocalDerivationGoal(const StorePath & drvPath, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode) + : DerivationGoal{drvPath, wantedOutputs, worker, buildMode} + , builder{ + worker.store, + static_cast(*this), + DerivationBuilderParams { + DerivationGoal::drvPath, + DerivationGoal::buildMode, + DerivationGoal::buildResult, + DerivationGoal::drv, + DerivationGoal::parsedDrv, + DerivationGoal::drvOptions, + DerivationGoal::inputPaths, + DerivationGoal::initialOutputs, + }, + } + {} + + LocalDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal) + : DerivationGoal{drvPath, drv, wantedOutputs, worker, buildMode} + , builder{ + worker.store, + static_cast(*this), + DerivationBuilderParams { + DerivationGoal::drvPath, + DerivationGoal::buildMode, + DerivationGoal::buildResult, + DerivationGoal::drv, + DerivationGoal::parsedDrv, + DerivationGoal::drvOptions, + DerivationGoal::inputPaths, + DerivationGoal::initialOutputs, + }, + } + {} + + virtual ~LocalDerivationGoal() override; + + /** + * The additional states. + */ + Goal::Co tryLocalBuild() override; + + bool isReadDesc(int fd) override; + + /** + * Forcibly kill the child process, if any. + * + * Called by destructor, can't be overridden + */ + void killChild() override final; + + void childStarted() override; + void childTerminated() override; + + void noteHashMismatch(void) override; + void noteCheckMismatch(void) override; + + void markContentsGood(const StorePath &) override; + + // Fake overrides to instantiate identically-named virtual methods + + Path openLogFile() override { + return DerivationGoal::openLogFile(); + } + void closeLogFile() override { + DerivationGoal::closeLogFile(); + } + SingleDrvOutputs assertPathValidity() override { + return DerivationGoal::assertPathValidity(); + } + void appendLogTailErrorMsg(std::string & msg) override { + DerivationGoal::appendLogTailErrorMsg(msg); + } +}; + +std::shared_ptr makeLocalDerivationGoal( + const StorePath & drvPath, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode) +{ + return std::make_shared(drvPath, wantedOutputs, worker, buildMode); +} + +std::shared_ptr makeLocalDerivationGoal( + const StorePath & drvPath, const BasicDerivation & drv, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode) +{ + return std::make_shared(drvPath, drv, wantedOutputs, worker, buildMode); +} + +void handleDiffHook( + uid_t uid, uid_t gid, + const Path & tryA, const Path & tryB, + const Path & drvPath, const Path & tmpDir) +{ + auto & diffHookOpt = settings.diffHook.get(); + if (diffHookOpt && settings.runDiffHook) { + auto & diffHook = *diffHookOpt; + try { + auto diffRes = runProgram(RunOptions { + .program = diffHook, + .lookupPath = true, + .args = {tryA, tryB, drvPath, tmpDir}, + .uid = uid, + .gid = gid, + .chdir = "/" + }); + if (!statusOk(diffRes.first)) + throw ExecError(diffRes.first, + "diff-hook program '%1%' %2%", + diffHook, + statusToString(diffRes.first)); + + if (diffRes.second != "") + printError(chomp(diffRes.second)); + } catch (Error & error) { + ErrorInfo ei = error.info(); + // FIXME: wrap errors. + ei.msg = HintFmt("diff hook execution failed: %s", ei.msg.str()); + logError(ei); + } + } +} + +const Path DerivationBuilder::homeDir = "/homeless-shelter"; + + +LocalDerivationGoal::~LocalDerivationGoal() +{ + /* Careful: we should never ever throw an exception from a + destructor. */ + try { builder.deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } + try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } + try { builder.stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } +} + + +inline bool DerivationBuilder::needsHashRewrite() +{ +#ifdef __linux__ + return !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + return true; +#endif +} + + +LocalStore & DerivationBuilder::getLocalStore() +{ + auto p = dynamic_cast(&store); + assert(p); + return *p; +} + + +void LocalDerivationGoal::killChild() +{ + if (builder.pid != -1) { + worker.childTerminated(this); + + /* If we're using a build user, then there is a tricky race + condition: if we kill the build user before the child has + done its setuid() to the build user uid, then it won't be + killed, and we'll potentially lock up in pid.wait(). So + also send a conventional kill to the child. */ + ::kill(-builder.pid, SIGKILL); /* ignore the result */ + + builder.killSandbox(true); + + builder.pid.wait(); + } + + DerivationGoal::killChild(); +} + + +void DerivationBuilder::killSandbox(bool getStats) +{ + if (cgroup) { + #ifdef __linux__ + auto stats = destroyCgroup(*cgroup); + if (getStats) { + buildResult.cpuUser = stats.cpuUser; + buildResult.cpuSystem = stats.cpuSystem; + } + #else + unreachable(); + #endif + } + + else if (buildUser) { + auto uid = buildUser->getUID(); + assert(uid != 0); + killUser(uid); + } +} + + +void LocalDerivationGoal::childStarted() +{ + worker.childStarted(shared_from_this(), {builder.builderOut.get()}, true, true); +} + +void LocalDerivationGoal::childTerminated() +{ + worker.childTerminated(this); +} + +void LocalDerivationGoal::noteHashMismatch() +{ + worker.hashMismatch = true; +} + + +void LocalDerivationGoal::noteCheckMismatch() +{ + worker.checkMismatch = true; +} + + +void LocalDerivationGoal::markContentsGood(const StorePath & path) +{ + worker.markContentsGood(path); +} + + +Goal::Co LocalDerivationGoal::tryLocalBuild() +{ + assert(!hook); + + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= settings.maxBuildJobs) { + outputLocks.unlock(); + co_await waitForBuildSlot(); + co_return tryToBuild(); + } + + if (!builder.prepareBuild()) { + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); + co_await waitForAWhile(); + co_return tryLocalBuild(); + } + + actLock.reset(); + + try { + + /* Okay, we have to build. */ + builder.startBuilder(); + + } catch (BuildError & e) { + outputLocks.unlock(); + builder.buildUser.reset(); + worker.permanentFailure = true; + co_return done(BuildResult::InputRejected, {}, std::move(e)); + } + + started(); + co_await Suspend{}; + + trace("build done"); + + auto res = builder.unprepareBuild(); + // N.B. cannot use `std::visit` with co-routine return + if (auto * ste = std::get_if<0>(&res)) { + outputLocks.unlock(); + co_return done(std::move(ste->first), {}, std::move(ste->second)); + } else if (auto * builtOutputs = std::get_if<1>(&res)) { + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + co_return done(BuildResult::Built, std::move(*builtOutputs)); + } else { + unreachable(); + } +} + +bool DerivationBuilder::prepareBuild() +{ + /* Cache this */ + derivationType = drv->type(); + + /* Are we doing a chroot build? */ + { + if (settings.sandboxMode == smEnabled) { + if (drvOptions->noChroot) + throw Error("derivation '%s' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'", store.printStorePath(drvPath)); +#ifdef __APPLE__ + if (drvOptions->additionalSandboxProfile != "") + throw Error("derivation '%s' specifies a sandbox profile, " + "but this is only allowed when 'sandbox' is 'relaxed'", store.printStorePath(drvPath)); +#endif + useChroot = true; + } + else if (settings.sandboxMode == smDisabled) + useChroot = false; + else if (settings.sandboxMode == smRelaxed) + useChroot = derivationType->isSandboxed() && !drvOptions->noChroot; + } + + auto & localStore = getLocalStore(); + if (localStore.storeDir != localStore.realStoreDir.get()) { + #ifdef __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } + + #ifdef __linux__ + if (useChroot) { + if (!mountAndPidNamespacesSupported()) { + if (!settings.sandboxFallback) + throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); + debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); + useChroot = false; + } + } + #endif + + if (useBuildUsers()) { + if (!buildUser) + buildUser = acquireUserLock(drvOptions->useUidRange(*drv) ? 65536 : 1, useChroot); + + if (!buildUser) { + return false; + } + } + + return true; +} + + +std::variant, SingleDrvOutputs> DerivationBuilder::unprepareBuild() +{ + Finally releaseBuildUser([&](){ + /* Release the build user at the end of this function. We don't do + it right away because we don't want another build grabbing this + uid and then messing around with our output. */ + buildUser.reset(); + }); + + sandboxMountNamespace = -1; + sandboxUserNamespace = -1; + + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = pid.kill(); + + debug("builder process for '%s' finished", store.printStorePath(drvPath)); + + buildResult.timesBuilt++; + buildResult.stopTime = time(0); + + /* So the child is gone now. */ + miscMethods.childTerminated(); + + /* Close the read side of the logger pipe. */ + builderOut.close(); + + /* Close the log file. */ + miscMethods.closeLogFile(); + + /* When running under a build user, make sure that all processes + running under that uid are gone. This is to prevent a + malicious user from leaving behind a process that keeps files + open and modifies them after they have been chown'ed to + root. */ + killSandbox(true); + + /* Terminate the recursive Nix daemon. */ + stopDaemon(); + + if (buildResult.cpuUser && buildResult.cpuSystem) { + debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs", + store.printStorePath(drvPath), + status, + ((double) buildResult.cpuUser->count()) / 1000000, + ((double) buildResult.cpuSystem->count()) / 1000000); + } + + bool diskFull = false; + + try { + + /* Check the exit status. */ + if (!statusOk(status)) { + + diskFull |= cleanupDecideWhetherDiskFull(); + + auto msg = fmt("builder for '%s' %s", + Magenta(store.printStorePath(drvPath)), + statusToString(status)); + + miscMethods.appendLogTailErrorMsg(msg); + + if (diskFull) + msg += "\nnote: build failure may have been caused by lack of free disk space"; + + throw BuildError(msg); + } + + /* Compute the FS closure of the outputs and register them as + being valid. */ + auto builtOutputs = registerOutputs(); + + StorePathSet outputPaths; + for (auto & [_, output] : builtOutputs) + outputPaths.insert(output.outPath); + runPostBuildHook( + store, + *logger, + drvPath, + outputPaths + ); + + /* Delete unused redirected outputs (when doing hash rewriting). */ + for (auto & i : redirectedOutputs) + deletePath(store.Store::toRealPath(i.second)); + + /* Delete the chroot (if we were using one). */ + autoDelChroot.reset(); /* this runs the destructor */ + + deleteTmpDir(true); + + return std::move(builtOutputs); + + } catch (BuildError & e) { + assert(derivationType); + BuildResult::Status st = + dynamic_cast(&e) ? BuildResult::NotDeterministic : + statusOk(status) ? BuildResult::OutputRejected : + !derivationType->isSandboxed() || diskFull ? BuildResult::TransientFailure : + BuildResult::PermanentFailure; + + return std::pair{std::move(st), std::move(e)}; + } +} + + +static void chmod_(const Path & path, mode_t mode) +{ + if (chmod(path.c_str(), mode) == -1) + throw SysError("setting permissions on '%s'", path); +} + + +/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if + it's a directory and we're not root (to be able to update the + directory's parent link ".."). */ +static void movePath(const Path & src, const Path & dst) +{ + auto st = lstat(src); + + bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + + if (changePerm) + chmod_(src, st.st_mode | S_IWUSR); + + std::filesystem::rename(src, dst); + + if (changePerm) + chmod_(dst, st.st_mode); +} + + +extern void replaceValidPath(const Path & storePath, const Path & tmpPath); + + +bool DerivationBuilder::cleanupDecideWhetherDiskFull() +{ + bool diskFull = false; + + /* Heuristically check whether the build failure may have + been caused by a disk full condition. We have no way + of knowing whether the build actually got an ENOSPC. + So instead, check if the disk is (nearly) full now. If + so, we don't mark this build as a permanent failure. */ +#if HAVE_STATVFS + { + auto & localStore = getLocalStore(); + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable + struct statvfs st; + if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + if (statvfs(tmpDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + } +#endif + + deleteTmpDir(false); + + /* Move paths out of the chroot for easier debugging of + build failures. */ + if (useChroot && buildMode == bmNormal) + for (auto & [_, status] : initialOutputs) { + if (!status.known) continue; + if (buildMode != bmCheck && status.known->isValid()) continue; + auto p = store.toRealPath(status.known->path); + if (pathExists(chrootRootDir + p)) + std::filesystem::rename((chrootRootDir + p), p); + } + + return diskFull; +} + + +#ifdef __linux__ +static void doBind(const Path & source, const Path & target, bool optional = false) { + debug("bind mounting '%1%' to '%2%'", source, target); + + auto bindMount = [&]() { + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); + }; + + auto maybeSt = maybeLstat(source); + if (!maybeSt) { + if (optional) + return; + else + throw SysError("getting attributes of path '%1%'", source); + } + auto st = *maybeSt; + + if (S_ISDIR(st.st_mode)) { + createDirs(target); + bindMount(); + } else if (S_ISLNK(st.st_mode)) { + // Symlinks can (apparently) not be bind-mounted, so just copy it + createDirs(dirOf(target)); + copyFile( + std::filesystem::path(source), + std::filesystem::path(target), false); + } else { + createDirs(dirOf(target)); + writeFile(target, ""); + bindMount(); + } +}; +#endif + +/** + * Rethrow the current exception as a subclass of `Error`. + */ +static void rethrowExceptionAsError() +{ + try { + throw; + } catch (Error &) { + throw; + } catch (std::exception & e) { + throw Error(e.what()); + } catch (...) { + throw Error("unknown exception"); + } +} + +/** + * Send the current exception to the parent in the format expected by + * `DerivationBuilder::processSandboxSetupMessages()`. + */ +static void handleChildException(bool sendException) +{ + try { + rethrowExceptionAsError(); + } catch (Error & e) { + if (sendException) { + writeFull(STDERR_FILENO, "\1\n"); + FdSink sink(STDERR_FILENO); + sink << e; + sink.flush(); + } else + std::cerr << e.msg(); + } +} + +void DerivationBuilder::startBuilder() +{ + if ((buildUser && buildUser->getUIDCount() != 1) + #ifdef __linux__ + || settings.useCgroups + #endif + ) + { + #ifdef __linux__ + experimentalFeatureSettings.require(Xp::Cgroups); + + /* If we're running from the daemon, then this will return the + root cgroup of the service. Otherwise, it will return the + current cgroup. */ + auto rootCgroup = getRootCgroup(); + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) + throw Error("cannot determine the cgroups file system"); + auto rootCgroupPath = canonPath(*cgroupFS + "/" + rootCgroup); + if (!pathExists(rootCgroupPath)) + throw Error("expected cgroup directory '%s'", rootCgroupPath); + + static std::atomic counter{0}; + + cgroup = buildUser + ? fmt("%s/nix-build-uid-%d", rootCgroupPath, buildUser->getUID()) + : fmt("%s/nix-build-pid-%d-%d", rootCgroupPath, getpid(), counter++); + + debug("using cgroup '%s'", *cgroup); + + /* When using a build user, record the cgroup we used for that + user so that if we got interrupted previously, we can kill + any left-over cgroup first. */ + if (buildUser) { + auto cgroupsDir = settings.nixStateDir + "/cgroups"; + createDirs(cgroupsDir); + + auto cgroupFile = fmt("%s/%d", cgroupsDir, buildUser->getUID()); + + if (pathExists(cgroupFile)) { + auto prevCgroup = readFile(cgroupFile); + destroyCgroup(prevCgroup); + } + + writeFile(cgroupFile, *cgroup); + } + + #else + throw Error("cgroups are not supported on this platform"); + #endif + } + + /* Make sure that no other processes are executing under the + sandbox uids. This must be done before any chownToBuilder() + calls. */ + killSandbox(false); + + /* Right platform? */ + if (!drvOptions->canBuildLocally(store, *drv)) { + // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - we should tell them to run the command to install Darwin 2 + if (drv->platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") { + throw Error("run `/usr/sbin/softwareupdate --install-rosetta` to enable your %s to run programs for %s", settings.thisSystem, drv->platform); + } else { + throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", + drv->platform, + concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)), + store.printStorePath(drvPath), + settings.thisSystem, + concatStringsSep(", ", store.systemFeatures)); + } + } + + /* Create a temporary directory where the build will take + place. */ + topTmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); +#ifdef __APPLE__ + if (false) { +#else + if (useChroot) { +#endif + /* If sandboxing is enabled, put the actual TMPDIR underneath + an inaccessible root-owned directory, to prevent outside + access. + + On macOS, we don't use an actual chroot, so this isn't + possible. Any mitigation along these lines would have to be + done directly in the sandbox profile. */ + tmpDir = topTmpDir + "/build"; + createDir(tmpDir, 0700); + } else { + tmpDir = topTmpDir; + } + chownToBuilder(tmpDir); + + for (auto & [outputName, status] : initialOutputs) { + /* Set scratch path we'll actually use during the build. + + If we're not doing a chroot build, but we have some valid + output paths. Since we can't just overwrite or delete + them, we have to do hash rewriting: i.e. in the + environment/arguments passed to the build, we replace the + hashes of the valid outputs with unique dummy strings; + after the build, we discard the redirected outputs + corresponding to the valid outputs, and rewrite the + contents of the new outputs to replace the dummy strings + with the actual hashes. */ + auto scratchPath = + !status.known + ? makeFallbackPath(outputName) + : !needsHashRewrite() + /* Can always use original path in sandbox */ + ? status.known->path + : !status.known->isPresent() + /* If path doesn't yet exist can just use it */ + ? status.known->path + : buildMode != bmRepair && !status.known->isValid() + /* If we aren't repairing we'll delete a corrupted path, so we + can use original path */ + ? status.known->path + : /* If we are repairing or the path is totally valid, we'll need + to use a temporary path */ + makeFallbackPath(status.known->path); + scratchOutputs.insert_or_assign(outputName, scratchPath); + + /* Substitute output placeholders with the scratch output paths. + We'll use during the build. */ + inputRewrites[hashPlaceholder(outputName)] = store.printStorePath(scratchPath); + + /* Additional tasks if we know the final path a priori. */ + if (!status.known) continue; + auto fixedFinalPath = status.known->path; + + /* Additional tasks if the final and scratch are both known and + differ. */ + if (fixedFinalPath == scratchPath) continue; + + /* Ensure scratch path is ours to use. */ + deletePath(store.printStorePath(scratchPath)); + + /* Rewrite and unrewrite paths */ + { + std::string h1 { fixedFinalPath.hashPart() }; + std::string h2 { scratchPath.hashPart() }; + inputRewrites[h1] = h2; + } + + redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); + } + + /* Construct the environment passed to the builder. */ + initEnv(); + + writeStructuredAttrs(); + + /* Handle exportReferencesGraph(), if set. */ + if (!parsedDrv) { + for (auto & [fileName, ss] : drvOptions->exportReferencesGraph) { + StorePathSet storePathSet; + for (auto & storePathS : ss) { + if (!store.isInStore(storePathS)) + throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); + storePathSet.insert(store.toStorePath(storePathS).first); + } + /* Write closure info to . */ + writeFile(tmpDir + "/" + fileName, + store.makeValidityRegistration( + store.exportReferences(storePathSet, inputPaths), false, false)); + } + } + + if (useChroot) { + + /* Allow a user-configurable set of directories from the + host file system. */ + pathsInChroot.clear(); + + for (auto i : settings.sandboxPaths.get()) { + if (i.empty()) continue; + bool optional = false; + if (i[i.size() - 1] == '?') { + optional = true; + i.pop_back(); + } + size_t p = i.find('='); + if (p == std::string::npos) + pathsInChroot[i] = {i, optional}; + else + pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; + } + if (hasPrefix(store.storeDir, tmpDirInSandbox)) + { + throw Error("`sandbox-build-dir` must not contain the storeDir"); + } + pathsInChroot[tmpDirInSandbox] = tmpDir; + + /* Add the closure of store paths to the chroot. */ + StorePathSet closure; + for (auto & i : pathsInChroot) + try { + if (store.isInStore(i.second.source)) + store.computeFSClosure(store.toStorePath(i.second.source).first, closure); + } catch (InvalidPath & e) { + } catch (Error & e) { + e.addTrace({}, "while processing 'sandbox-paths'"); + throw; + } + for (auto & i : closure) { + auto p = store.printStorePath(i); + pathsInChroot.insert_or_assign(p, p); + } + + PathSet allowedPaths = settings.allowedImpureHostPrefixes; + + /* This works like the above, except on a per-derivation level */ + auto impurePaths = drvOptions->impureHostDeps; + + for (auto & i : impurePaths) { + bool found = false; + /* Note: we're not resolving symlinks here to prevent + giving a non-root user info about inaccessible + files. */ + Path canonI = canonPath(i); + /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ + for (auto & a : allowedPaths) { + Path canonA = canonPath(a); + if (isDirOrInDir(canonI, canonA)) { + found = true; + break; + } + } + if (!found) + throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", + store.printStorePath(drvPath), i); + + /* Allow files in drvOptions->impureHostDeps to be missing; e.g. + macOS 11+ has no /usr/lib/libSystem*.dylib */ + pathsInChroot[i] = {i, true}; + } + +#ifdef __linux__ + /* Create a temporary directory in which we set up the chroot + environment using bind-mounts. We put it in the Nix store + so that the build outputs can be moved efficiently from the + chroot to their final location. */ + auto chrootParentDir = store.Store::toRealPath(drvPath) + ".chroot"; + deletePath(chrootParentDir); + + /* Clean up the chroot directory automatically. */ + autoDelChroot = std::make_shared(chrootParentDir); + + printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootParentDir); + + if (mkdir(chrootParentDir.c_str(), 0700) == -1) + throw SysError("cannot create '%s'", chrootRootDir); + + chrootRootDir = chrootParentDir + "/root"; + + if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) + throw SysError("cannot create '%1%'", chrootRootDir); + + if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUIDCount() != 1 ? buildUser->getUID() : 0, buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", chrootRootDir); + + /* Create a writable /tmp in the chroot. Many builders need + this. (Of course they should really respect $TMPDIR + instead.) */ + Path chrootTmpDir = chrootRootDir + "/tmp"; + createDirs(chrootTmpDir); + chmod_(chrootTmpDir, 01777); + + /* Create a /etc/passwd with entries for the build user and the + nobody account. The latter is kind of a hack to support + Samba-in-QEMU. */ + createDirs(chrootRootDir + "/etc"); + if (drvOptions->useUidRange(*drv)) + chownToBuilder(chrootRootDir + "/etc"); + + if (drvOptions->useUidRange(*drv) && (!buildUser || buildUser->getUIDCount() < 65536)) + throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); + + /* Declare the build user's group so that programs get a consistent + view of the system (e.g., "id -gn"). */ + writeFile(chrootRootDir + "/etc/group", + fmt("root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n", sandboxGid())); + + /* Create /etc/hosts with localhost entry. */ + if (derivationType->isSandboxed()) + writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); + + /* Make the closure of the inputs available in the chroot, + rather than the whole Nix store. This prevents any access + to undeclared dependencies. Directories are bind-mounted, + while other inputs are hard-linked (since only directories + can be bind-mounted). !!! As an extra security + precaution, make the fake Nix store only writable by the + build user. */ + Path chrootStoreDir = chrootRootDir + store.storeDir; + createDirs(chrootStoreDir); + chmod_(chrootStoreDir, 01775); + + if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", chrootStoreDir); + + for (auto & i : inputPaths) { + auto p = store.printStorePath(i); + Path r = store.toRealPath(p); + pathsInChroot.insert_or_assign(p, r); + } + + /* If we're repairing, checking or rebuilding part of a + multiple-outputs derivation, it's possible that we're + rebuilding a path that is in settings.sandbox-paths + (typically the dependencies of /bin/sh). Throw them + out. */ + for (auto & i : drv->outputsAndOptPaths(store)) { + /* If the name isn't known a priori (i.e. floating + content-addressing derivation), the temporary location we use + should be fresh. Freshness means it is impossible that the path + is already in the sandbox, so we don't need to worry about + removing it. */ + if (i.second.second) + pathsInChroot.erase(store.printStorePath(*i.second.second)); + } + + if (cgroup) { + if (mkdir(cgroup->c_str(), 0755) != 0) + throw SysError("creating cgroup '%s'", *cgroup); + chownToBuilder(*cgroup); + chownToBuilder(*cgroup + "/cgroup.procs"); + chownToBuilder(*cgroup + "/cgroup.threads"); + //chownToBuilder(*cgroup + "/cgroup.subtree_control"); + } + +#else + if (drvOptions->useUidRange(*drv)) + throw Error("feature 'uid-range' is not supported on this platform"); + #ifdef __APPLE__ + /* We don't really have any parent prep work to do (yet?) + All work happens in the child, instead. */ + #else + throw Error("sandboxing builds is not supported on this platform"); + #endif +#endif + } else { + if (drvOptions->useUidRange(*drv)) + throw Error("feature 'uid-range' is only supported in sandboxed builds"); + } + + if (needsHashRewrite() && pathExists(homeDir)) + throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); + + if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { + printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); + auto args = useChroot ? Strings({store.printStorePath(drvPath), chrootRootDir}) : + Strings({ store.printStorePath(drvPath) }); + enum BuildHookState { + stBegin, + stExtraChrootDirs + }; + auto state = stBegin; + auto lines = runProgram(settings.preBuildHook, false, args); + auto lastPos = std::string::size_type{0}; + for (auto nlPos = lines.find('\n'); nlPos != std::string::npos; + nlPos = lines.find('\n', lastPos)) + { + auto line = lines.substr(lastPos, nlPos - lastPos); + lastPos = nlPos + 1; + if (state == stBegin) { + if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { + state = stExtraChrootDirs; + } else { + throw Error("unknown pre-build hook command '%1%'", line); + } + } else if (state == stExtraChrootDirs) { + if (line == "") { + state = stBegin; + } else { + auto p = line.find('='); + if (p == std::string::npos) + pathsInChroot[line] = line; + else + pathsInChroot[line.substr(0, p)] = line.substr(p + 1); + } + } + } + } + + /* Fire up a Nix daemon to process recursive Nix calls from the + builder. */ + if (drvOptions->getRequiredSystemFeatures(*drv).count("recursive-nix")) + startDaemon(); + + /* Run the builder. */ + printMsg(lvlChatty, "executing builder '%1%'", drv->builder); + printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args)); + for (auto & i : drv->env) + printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); + + /* Create the log file. */ + [[maybe_unused]] Path logFile = miscMethods.openLogFile(); + + /* Create a pseudoterminal to get the output of the builder. */ + builderOut = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut) + throw SysError("opening pseudoterminal master"); + + // FIXME: not thread-safe, use ptsname_r + std::string slaveName = ptsname(builderOut.get()); + + if (buildUser) { + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); + + if (chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + } +#ifdef __APPLE__ + else { + if (grantpt(builderOut.get())) + throw SysError("granting access to pseudoterminal slave"); + } +#endif + + if (unlockpt(builderOut.get())) + throw SysError("unlocking pseudoterminal"); + + /* Open the slave side of the pseudoterminal and use it as stderr. */ + auto openSlave = [&]() + { + AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); + + if (dup2(builderOut.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + }; + + buildResult.startTime = time(0); + + /* Fork a child to build the package. */ + +#ifdef __linux__ + if (useChroot) { + /* Set up private namespaces for the build: + + - The PID namespace causes the build to start as PID 1. + Processes outside of the chroot are not visible to those + on the inside, but processes inside the chroot are + visible from the outside (though with different PIDs). + + - The private mount namespace ensures that all the bind + mounts we do will only show up in this process and its + children, and will disappear automatically when we're + done. + + - The private network namespace ensures that the builder + cannot talk to the outside world (or vice versa). It + only has a private loopback interface. (Fixed-output + derivations are not run in a private network namespace + to allow functions like fetchurl to work.) + + - The IPC namespace prevents the builder from communicating + with outside processes using SysV IPC mechanisms (shared + memory, message queues, semaphores). It also ensures + that all IPC objects are destroyed when the builder + exits. + + - The UTS namespace ensures that builders see a hostname of + localhost rather than the actual hostname. + + We use a helper process to do the clone() to work around + clone() being broken in multi-threaded programs due to + at-fork handlers not being run. Note that we use + CLONE_PARENT to ensure that the real builder is parented to + us. + */ + + userNamespaceSync.create(); + + usingUserNamespace = userNamespacesSupported(); + + Pipe sendPid; + sendPid.create(); + + Pid helper = startProcess([&]() { + sendPid.readSide.close(); + + /* We need to open the slave early, before + CLONE_NEWUSER. Otherwise we get EPERM when running as + root. */ + openSlave(); + + try { + /* Drop additional groups here because we can't do it + after we've created the new user namespace. */ + if (setgroups(0, 0) == -1) { + if (errno != EPERM) + throw SysError("setgroups failed"); + if (settings.requireDropSupplementaryGroups) + throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); + } + + ProcessOptions options; + options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + if (derivationType->isSandboxed()) + options.cloneFlags |= CLONE_NEWNET; + if (usingUserNamespace) + options.cloneFlags |= CLONE_NEWUSER; + + pid_t child = startProcess([&]() { runChild(); }, options); + + writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); + _exit(0); + } catch (...) { + handleChildException(true); + _exit(1); + } + }); + + sendPid.writeSide.close(); + + if (helper.wait() != 0) { + processSandboxSetupMessages(); + // Only reached if the child process didn't send an exception. + throw Error("unable to start build process"); + } + + userNamespaceSync.readSide = -1; + + /* Close the write side to prevent runChild() from hanging + reading from this. */ + Finally cleanup([&]() { + userNamespaceSync.writeSide = -1; + }); + + auto ss = tokenizeString>(readLine(sendPid.readSide.get())); + assert(ss.size() == 1); + pid = string2Int(ss[0]).value(); + + if (usingUserNamespace) { + /* Set the UID/GID mapping of the builder's user namespace + such that the sandbox user maps to the build user, or to + the calling user (if build users are disabled). */ + uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); + uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); + uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1; + + writeFile("/proc/" + std::to_string(pid) + "/uid_map", + fmt("%d %d %d", sandboxUid(), hostUid, nrIds)); + + if (!buildUser || buildUser->getUIDCount() == 1) + writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + + writeFile("/proc/" + std::to_string(pid) + "/gid_map", + fmt("%d %d %d", sandboxGid(), hostGid, nrIds)); + } else { + debug("note: not using a user namespace"); + if (!buildUser) + throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); + } + + /* Now that we now the sandbox uid, we can write + /etc/passwd. */ + writeFile(chrootRootDir + "/etc/passwd", fmt( + "root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); + + /* Save the mount- and user namespace of the child. We have to do this + *before* the child does a chroot. */ + sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); + if (sandboxMountNamespace.get() == -1) + throw SysError("getting sandbox mount namespace"); + + if (usingUserNamespace) { + sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY); + if (sandboxUserNamespace.get() == -1) + throw SysError("getting sandbox user namespace"); + } + + /* Move the child into its own cgroup. */ + if (cgroup) + writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + + /* Signal the builder that we've updated its user namespace. */ + writeFull(userNamespaceSync.writeSide.get(), "1"); + + } else +#endif + { + pid = startProcess([&]() { + openSlave(); + runChild(); + }); + } + + /* parent */ + pid.setSeparatePG(true); + miscMethods.childStarted(); + + processSandboxSetupMessages(); +} + + +void DerivationBuilder::processSandboxSetupMessages() +{ + std::vector msgs; + while (true) { + std::string msg = [&]() { + try { + return readLine(builderOut.get()); + } catch (Error & e) { + auto status = pid.wait(); + e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", + store.printStorePath(drvPath), + statusToString(status), + concatStringsSep("|", msgs)); + throw; + } + }(); + if (msg.substr(0, 1) == "\2") break; + if (msg.substr(0, 1) == "\1") { + FdSource source(builderOut.get()); + auto ex = readError(source); + ex.addTrace({}, "while setting up the build environment"); + throw ex; + } + debug("sandbox setup: " + msg); + msgs.push_back(std::move(msg)); + } +} + + +void DerivationBuilder::initTmpDir() +{ + /* In a sandbox, for determinism, always use the same temporary + directory. */ +#ifdef __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif + + /* In non-structured mode, set all bindings either directory in the + environment or via a file, as specified by + `DerivationOptions::passAsFile`. */ + if (!parsedDrv) { + for (auto & i : drv->env) { + if (drvOptions->passAsFile.find(i.first) == drvOptions->passAsFile.end()) { + env[i.first] = i.second; + } else { + auto hash = hashString(HashAlgorithm::SHA256, i.first); + std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false); + Path p = tmpDir + "/" + fn; + writeFile(p, rewriteStrings(i.second, inputRewrites)); + chownToBuilder(p); + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; + } + } + + } + + /* For convenience, set an environment pointing to the top build + directory. */ + env["NIX_BUILD_TOP"] = tmpDirInSandbox; + + /* Also set TMPDIR and variants to point to this directory. */ + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; + + /* Explicitly set PWD to prevent problems with chroot builds. In + particular, dietlibc cannot figure out the cwd because the + inode of the current directory doesn't appear in .. (because + getdents returns the inode of the mount point). */ + env["PWD"] = tmpDirInSandbox; +} + + +void DerivationBuilder::initEnv() +{ + env.clear(); + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = store.storeDir; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores); + + initTmpDir(); + + /* Compatibility hack with Nix <= 0.7: if this is a fixed-output + derivation, tell the builder, so that for instance `fetchurl' + can skip checking the output. On older Nixes, this environment + variable won't be set, so `fetchurl' will do the check. */ + if (derivationType->isFixed()) env["NIX_OUTPUT_CHECKED"] = "1"; + + /* *Only* if this is a fixed-output derivation, propagate the + values of the environment variables specified in the + `impureEnvVars' attribute to the builder. This allows for + instance environment variables for proxy configuration such as + `http_proxy' to be easily passed to downloaders like + `fetchurl'. Passing such environment variables from the caller + to the builder is generally impure, but the output of + fixed-output derivations is by definition pure (since we + already know the cryptographic hash of the output). */ + if (!derivationType->isSandboxed()) { + auto & impureEnv = settings.impureEnv.get(); + if (!impureEnv.empty()) + experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv); + + for (auto & i : drvOptions->impureEnvVars){ + auto envVar = impureEnv.find(i); + if (envVar != impureEnv.end()) { + env[i] = envVar->second; + } else { + env[i] = getEnv(i).value_or(""); + } + } + } + + /* Currently structured log messages piggyback on stderr, but we + may change that in the future. So tell the builder which file + descriptor to use for that. */ + env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; +} + + +void DerivationBuilder::writeStructuredAttrs() +{ + if (parsedDrv) { + auto json = parsedDrv->prepareStructuredAttrs( + store, + *drvOptions, + inputPaths, + drv->outputs); + nlohmann::json rewritten; + for (auto & [i, v] : json["outputs"].get()) { + /* The placeholder must have a rewrite, so we use it to cover both the + cases where we know or don't know the output path ahead of time. */ + rewritten[i] = rewriteStrings((std::string) v, inputRewrites); + } + + json["outputs"] = rewritten; + + auto jsonSh = StructuredAttrs::writeShell(json); + + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.sh"); + env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh"; + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.json"); + env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json"; + } +} + + +void DerivationBuilder::startDaemon() +{ + experimentalFeatureSettings.require(Xp::RecursiveNix); + + Store::Params params; + params["path-info-cache-size"] = "0"; + params["store"] = store.storeDir; + if (auto & optRoot = getLocalStore().rootDir.get()) + params["root"] = *optRoot; + params["state"] = "/no-such-path"; + params["log"] = "/no-such-path"; + auto store = makeRestrictedStore(params, + ref(std::dynamic_pointer_cast(this->store.shared_from_this())), + *this); + + addedPaths.clear(); + + auto socketName = ".nix-socket"; + Path socketPath = tmpDir + "/" + socketName; + env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; + + daemonSocket = createUnixDomainSocket(socketPath, 0600); + + chownToBuilder(socketPath); + + daemonThread = std::thread([this, store]() { + + while (true) { + + /* Accept a connection. */ + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = accept(daemonSocket.get(), + (struct sockaddr *) &remoteAddr, &remoteAddrLen); + if (!remote) { + if (errno == EINTR || errno == EAGAIN) continue; + if (errno == EINVAL || errno == ECONNABORTED) break; + throw SysError("accepting connection"); + } + + unix::closeOnExec(remote.get()); + + debug("received daemon connection"); + + auto workerThread = std::thread([store, remote{std::move(remote)}]() { + try { + daemon::processConnection( + store, + FdSource(remote.get()), + FdSink(remote.get()), + NotTrusted, daemon::Recursive); + debug("terminated daemon connection"); + } catch (const Interrupted &) { + debug("interrupted daemon connection"); + } catch (SystemError &) { + ignoreExceptionExceptInterrupt(); + } + }); + + daemonWorkerThreads.push_back(std::move(workerThread)); + } + + debug("daemon shutting down"); + }); +} + + +void DerivationBuilder::stopDaemon() +{ + if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) { + // According to the POSIX standard, the 'shutdown' function should + // return an ENOTCONN error when attempting to shut down a socket that + // hasn't been connected yet. This situation occurs when the 'accept' + // function is called on a socket without any accepted connections, + // leaving the socket unconnected. While Linux doesn't seem to produce + // an error for sockets that have only been accepted, more + // POSIX-compliant operating systems like OpenBSD, macOS, and others do + // return the ENOTCONN error. Therefore, we handle this error here to + // avoid raising an exception for compliant behaviour. + if (errno == ENOTCONN) { + daemonSocket.close(); + } else { + throw SysError("shutting down daemon socket"); + } + } + + if (daemonThread.joinable()) + daemonThread.join(); + + // FIXME: should prune worker threads more quickly. + // FIXME: shutdown the client socket to speed up worker termination. + for (auto & thread : daemonWorkerThreads) + thread.join(); + daemonWorkerThreads.clear(); + + // release the socket. + daemonSocket.close(); +} + + +void DerivationBuilder::addDependency(const StorePath & path) +{ + if (isAllowed(path)) return; + + addedPaths.insert(path); + + /* If we're doing a sandbox build, then we have to make the path + appear in the sandbox. */ + if (useChroot) { + + debug("materialising '%s' in the sandbox", store.printStorePath(path)); + + #ifdef __linux__ + + Path source = store.Store::toRealPath(path); + Path target = chrootRootDir + store.printStorePath(path); + + if (pathExists(target)) { + // There is a similar debug message in doBind, so only run it in this block to not have double messages. + debug("bind-mounting %s -> %s", target, source); + throw Error("store path '%s' already exists in the sandbox", store.printStorePath(path)); + } + + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { + + if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) + throw SysError("entering sandbox user namespace"); + + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); + + doBind(source, target); + + _exit(0); + })); + + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", store.printStorePath(path)); + + #else + throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", + store.printStorePath(path)); + #endif + + } +} + +void DerivationBuilder::chownToBuilder(const Path & path) +{ + if (!buildUser) return; + if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", path); +} + + +void setupSeccomp() +{ +#ifdef __linux__ + if (!settings.filterSyscalls) return; +#if HAVE_SECCOMP + scmp_filter_ctx ctx; + + if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) + throw SysError("unable to initialize seccomp mode 2"); + + Finally cleanup([&]() { + seccomp_release(ctx); + }); + + constexpr std::string_view nativeSystem = NIX_LOCAL_SYSTEM; + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) + throw SysError("unable to add 32-bit seccomp architecture"); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) + throw SysError("unable to add X32 seccomp architecture"); + + if (nativeSystem == "aarch64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) + printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); + + if (nativeSystem == "mips64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPS) != 0) + printError("unable to add mips seccomp architecture"); + + if (nativeSystem == "mips64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPS64N32) != 0) + printError("unable to add mips64-*abin32 seccomp architecture"); + + if (nativeSystem == "mips64el-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL) != 0) + printError("unable to add mipsel seccomp architecture"); + + if (nativeSystem == "mips64el-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL64N32) != 0) + printError("unable to add mips64el-*abin32 seccomp architecture"); + + /* Prevent builders from creating setuid/setgid binaries. */ + for (int perm : { S_ISUID, S_ISGID }) { + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, + SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), NIX_SYSCALL_FCHMODAT2, 1, + SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + } + + /* Prevent builders from using EAs or ACLs. Not all filesystems + support these, and they're not allowed in the Nix store because + they're not representable in the NAR serialisation. */ + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) + throw SysError("unable to set 'no new privileges' seccomp attribute"); + + if (seccomp_load(ctx) != 0) + throw SysError("unable to load seccomp BPF program"); +#else + throw Error( + "seccomp is not supported on this platform; " + "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); +#endif +#endif +} + + +void DerivationBuilder::runChild() +{ + /* Warning: in the child we should absolutely not make any SQLite + calls! */ + + bool sendException = true; + + try { /* child */ + + commonChildInit(); + + try { + setupSeccomp(); + } catch (...) { + if (buildUser) throw; + } + + bool setUser = true; + + /* Make the contents of netrc and the CA certificate bundle + available to builtin:fetchurl (which may run under a + different uid and/or in a sandbox). */ + std::string netrcData; + std::string caFileData; + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") { + try { + netrcData = readFile(settings.netrcFile); + } catch (SystemError &) { } + + try { + caFileData = readFile(settings.caFile); + } catch (SystemError &) { } + } + +#ifdef __linux__ + if (useChroot) { + + userNamespaceSync.writeSide = -1; + + if (drainFD(userNamespaceSync.readSide.get()) != "1") + throw Error("user namespace initialisation failed"); + + userNamespaceSync.readSide = -1; + + if (derivationType->isSandboxed()) { + + /* Initialise the loopback interface. */ + AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); + if (!fd) throw SysError("cannot open IP socket"); + + struct ifreq ifr; + strcpy(ifr.ifr_name, "lo"); + ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; + if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) + throw SysError("cannot set loopback interface flags"); + } + + /* Set the hostname etc. to fixed values. */ + char hostname[] = "localhost"; + if (sethostname(hostname, sizeof(hostname)) == -1) + throw SysError("cannot set host name"); + char domainname[] = "(none)"; // kernel default + if (setdomainname(domainname, sizeof(domainname)) == -1) + throw SysError("cannot set domain name"); + + /* Make all filesystems private. This is necessary + because subtrees may have been mounted as "shared" + (MS_SHARED). (Systemd does this, for instance.) Even + though we have a private mount namespace, mounting + filesystems on top of a shared subtree still propagates + outside of the namespace. Making a subtree private is + local to the namespace, though, so setting MS_PRIVATE + does not affect the outside world. */ + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) + throw SysError("unable to make '/' private"); + + /* Bind-mount chroot directory to itself, to treat it as a + different filesystem from /, as needed for pivot_root. */ + if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) + throw SysError("unable to bind mount '%1%'", chrootRootDir); + + /* Bind-mount the sandbox's Nix store onto itself so that + we can mark it as a "shared" subtree, allowing bind + mounts made in *this* mount namespace to be propagated + into the child namespace created by the + unshare(CLONE_NEWNS) call below. + + Marking chrootRootDir as MS_SHARED causes pivot_root() + to fail with EINVAL. Don't know why. */ + Path chrootStoreDir = chrootRootDir + store.storeDir; + + if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) + throw SysError("unable to bind mount the Nix store", chrootStoreDir); + + if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) + throw SysError("unable to make '%s' shared", chrootStoreDir); + + /* Set up a nearly empty /dev, unless the user asked to + bind-mount the host /dev. */ + Strings ss; + if (pathsInChroot.find("/dev") == pathsInChroot.end()) { + createDirs(chrootRootDir + "/dev/shm"); + createDirs(chrootRootDir + "/dev/pts"); + ss.push_back("/dev/full"); + if (store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + ss.push_back("/dev/kvm"); + ss.push_back("/dev/null"); + ss.push_back("/dev/random"); + ss.push_back("/dev/tty"); + ss.push_back("/dev/urandom"); + ss.push_back("/dev/zero"); + createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); + createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); + createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); + createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); + } + + /* Fixed-output derivations typically need to access the + network, so give them access to /etc/resolv.conf and so + on. */ + if (!derivationType->isSandboxed()) { + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed-outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + + /* N.B. it is realistic that these paths might not exist. It + happens when testing Nix building fixed-output derivations + within a pure derivation. */ + for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" }) + if (pathExists(path)) + ss.push_back(path); + + if (settings.caFile != "" && pathExists(settings.caFile)) { + Path caFile = settings.caFile; + pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true); + } + } + + for (auto & i : ss) { + // For backwards-compatibiliy, resolve all the symlinks in the + // chroot paths + auto canonicalPath = canonPath(i, true); + pathsInChroot.emplace(i, canonicalPath); + } + + /* Bind-mount all the directories from the "host" + filesystem that we want in the chroot + environment. */ + for (auto & i : pathsInChroot) { + if (i.second.source == "/proc") continue; // backwards compatibility + + #if HAVE_EMBEDDED_SANDBOX_SHELL + if (i.second.source == "__embedded_sandbox_shell__") { + static unsigned char sh[] = { + #include "embedded-sandbox-shell.gen.hh" + }; + auto dst = chrootRootDir + i.first; + createDirs(dirOf(dst)); + writeFile(dst, std::string_view((const char *) sh, sizeof(sh))); + chmod_(dst, 0555); + } else + #endif + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); + } + + /* Bind a new instance of procfs on /proc. */ + createDirs(chrootRootDir + "/proc"); + if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) + throw SysError("mounting /proc"); + + /* Mount sysfs on /sys. */ + if (buildUser && buildUser->getUIDCount() != 1) { + createDirs(chrootRootDir + "/sys"); + if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) + throw SysError("mounting /sys"); + } + + /* Mount a new tmpfs on /dev/shm to ensure that whatever + the builder puts in /dev/shm is cleaned up automatically. */ + if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, + fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) + throw SysError("mounting /dev/shm"); + + /* Mount a new devpts on /dev/pts. Note that this + requires the kernel to be compiled with + CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case + if /dev/ptx/ptmx exists). */ + if (pathExists("/dev/pts/ptmx") && + !pathExists(chrootRootDir + "/dev/ptmx") + && !pathsInChroot.count("/dev/pts")) + { + if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) + { + createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); + + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + } else { + if (errno != EINVAL) + throw SysError("mounting /dev/pts"); + doBind("/dev/pts", chrootRootDir + "/dev/pts"); + doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); + } + } + + /* Make /etc unwritable */ + if (!drvOptions->useUidRange(*drv)) + chmod_(chrootRootDir + "/etc", 0555); + + /* Unshare this mount namespace. This is necessary because + pivot_root() below changes the root of the mount + namespace. This means that the call to setns() in + addDependency() would hide the host's filesystem, + making it impossible to bind-mount paths from the host + Nix store into the sandbox. Therefore, we save the + pre-pivot_root namespace in + sandboxMountNamespace. Since we made /nix/store a + shared subtree above, this allows addDependency() to + make paths appear in the sandbox. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("unsharing mount namespace"); + + /* Unshare the cgroup namespace. This means + /proc/self/cgroup will show the child's cgroup as '/' + rather than whatever it is in the parent. */ + if (cgroup && unshare(CLONE_NEWCGROUP) == -1) + throw SysError("unsharing cgroup namespace"); + + /* Do the chroot(). */ + if (chdir(chrootRootDir.c_str()) == -1) + throw SysError("cannot change directory to '%1%'", chrootRootDir); + + if (mkdir("real-root", 0500) == -1) + throw SysError("cannot create real-root directory"); + + if (pivot_root(".", "real-root") == -1) + throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); + + if (chroot(".") == -1) + throw SysError("cannot change root directory to '%1%'", chrootRootDir); + + if (umount2("real-root", MNT_DETACH) == -1) + throw SysError("cannot unmount real root filesystem"); + + if (rmdir("real-root") == -1) + throw SysError("cannot remove real-root directory"); + + /* Switch to the sandbox uid/gid in the user namespace, + which corresponds to the build user or calling user in + the parent namespace. */ + if (setgid(sandboxGid()) == -1) + throw SysError("setgid failed"); + if (setuid(sandboxUid()) == -1) + throw SysError("setuid failed"); + + setUser = false; + } +#endif + + if (chdir(tmpDirInSandbox.c_str()) == -1) + throw SysError("changing into '%1%'", tmpDir); + + /* Close all other file descriptors. */ + unix::closeExtraFDs(); + +#ifdef __linux__ + linux::setPersonality(drv->platform); +#endif + + /* Disable core dumps by default. */ + struct rlimit limit = { 0, RLIM_INFINITY }; + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + + /* Fill in the environment. */ + Strings envStrs; + for (auto & i : env) + envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); + + /* If we are running in `build-users' mode, then switch to the + user we allocated above. Make sure that we drop all root + privileges. Note that above we have closed all file + descriptors except std*, so that's safe. Also note that + setuid() when run as root sets the real, effective and + saved UIDs. */ + if (setUser && buildUser) { + /* Preserve supplementary groups of the build user, to allow + admins to specify groups such as "kvm". */ + auto gids = buildUser->getSupplementaryGIDs(); + if (setgroups(gids.size(), gids.data()) == -1) + throw SysError("cannot set supplementary groups of build user"); + + if (setgid(buildUser->getGID()) == -1 || + getgid() != buildUser->getGID() || + getegid() != buildUser->getGID()) + throw SysError("setgid failed"); + + if (setuid(buildUser->getUID()) == -1 || + getuid() != buildUser->getUID() || + geteuid() != buildUser->getUID()) + throw SysError("setuid failed"); + } + +#ifdef __APPLE__ + /* This has to appear before import statements. */ + std::string sandboxProfile = "(version 1)\n"; + + if (useChroot) { + + /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ + PathSet ancestry; + + /* We build the ancestry before adding all inputPaths to the store because we know they'll + all have the same parents (the store), and there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice */ + for (auto & i : pathsInChroot) { + Path cur = i.first; + while (cur.compare("/") != 0) { + cur = dirOf(cur); + ancestry.insert(cur); + } + } + + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost + path component this time, since it's typically /nix/store and we care about that. */ + Path cur = store.storeDir; + while (cur.compare("/") != 0) { + ancestry.insert(cur); + cur = dirOf(cur); + } + + /* Add all our input paths to the chroot */ + for (auto & i : inputPaths) { + auto p = store.printStorePath(i); + pathsInChroot[p] = p; + } + + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ + if (settings.darwinLogSandboxViolations) { + sandboxProfile += "(deny default)\n"; + } else { + sandboxProfile += "(deny default (with no-log))\n"; + } + + sandboxProfile += + #include "sandbox-defaults.sb" + ; + + if (!derivationType->isSandboxed()) + sandboxProfile += + #include "sandbox-network.sb" + ; + + /* Add the output paths we'll use at build-time to the chroot */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & [_, path] : scratchOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", store.printStorePath(path)); + + sandboxProfile += ")\n"; + + /* Our inputs (transitive dependencies and any impurities computed above) + + without file-write* allowed, access() incorrectly returns EPERM + */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + + // We create multiple allow lists, to avoid exceeding a limit in the darwin sandbox interpreter. + // See https://github.com/NixOS/nix/issues/4119 + // We split our allow groups approximately at half the actual limit, 1 << 16 + const size_t breakpoint = sandboxProfile.length() + (1 << 14); + for (auto & i : pathsInChroot) { + + if (sandboxProfile.length() >= breakpoint) { + debug("Sandbox break: %d %d", sandboxProfile.length(), breakpoint); + sandboxProfile += ")\n(allow file-read* file-write* process-exec\n"; + } + + if (i.first != i.second.source) + throw Error( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", + i.first, i.second.source); + + std::string path = i.first; + auto optSt = maybeLstat(path.c_str()); + if (!optSt) { + if (i.second.optional) + continue; + throw SysError("getting attributes of required path '%s", path); + } + if (S_ISDIR(optSt->st_mode)) + sandboxProfile += fmt("\t(subpath \"%s\")\n", path); + else + sandboxProfile += fmt("\t(literal \"%s\")\n", path); + } + sandboxProfile += ")\n"; + + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; + for (auto & i : ancestry) { + sandboxProfile += fmt("\t(literal \"%s\")\n", i); + } + sandboxProfile += ")\n"; + + sandboxProfile += drvOptions->additionalSandboxProfile; + } else + sandboxProfile += + #include "sandbox-minimal.sb" + ; + + debug("Generated sandbox profile:"); + debug(sandboxProfile); + + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms + to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */ + Path globalTmpDir = canonPath(defaultTempDir(), true); + + /* They don't like trailing slashes on subpath directives */ + while (!globalTmpDir.empty() && globalTmpDir.back() == '/') + globalTmpDir.pop_back(); + + if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { + Strings sandboxArgs; + sandboxArgs.push_back("_GLOBAL_TMP_DIR"); + sandboxArgs.push_back(globalTmpDir); + if (drvOptions->allowLocalNetworking) { + sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); + sandboxArgs.push_back("1"); + } + char * sandbox_errbuf = nullptr; + if (sandbox_init_with_parameters(sandboxProfile.c_str(), 0, stringsToCharPtrs(sandboxArgs).data(), &sandbox_errbuf)) { + writeFull(STDERR_FILENO, fmt("failed to configure sandbox: %s\n", sandbox_errbuf ? sandbox_errbuf : "(null)")); + _exit(1); + } + } +#endif + + /* Indicate that we managed to set up the build environment. */ + writeFull(STDERR_FILENO, std::string("\2\n")); + + sendException = false; + + /* Execute the program. This should not return. */ + if (drv->isBuiltin()) { + try { + logger = makeJSONLogger(getStandardError()); + + std::map outputs; + for (auto & e : drv->outputs) + outputs.insert_or_assign(e.first, + store.printStorePath(scratchOutputs.at(e.first))); + + if (drv->builder == "builtin:fetchurl") + builtinFetchurl(*drv, outputs, netrcData, caFileData); + else if (drv->builder == "builtin:buildenv") + builtinBuildenv(*drv, outputs); + else if (drv->builder == "builtin:unpack-channel") + builtinUnpackChannel(*drv, outputs); + else + throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8)); + _exit(0); + } catch (std::exception & e) { + writeFull(STDERR_FILENO, e.what() + std::string("\n")); + _exit(1); + } + } + + // Now builder is not builtin + + Strings args; + args.push_back(std::string(baseNameOf(drv->builder))); + + for (auto & i : drv->args) + args.push_back(rewriteStrings(i, inputRewrites)); + +#ifdef __APPLE__ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv->platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv->platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn(NULL, drv->builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#else + execve(drv->builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#endif + + throw SysError("executing '%1%'", drv->builder); + + } catch (...) { + handleChildException(sendException); + _exit(1); + } +} + + +SingleDrvOutputs DerivationBuilder::registerOutputs() +{ + std::map infos; + + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; + + Path checkSuffix = ".check"; + + std::exception_ptr delayedException; + + /* The paths that can be referenced are the input closures, the + output paths, and any paths that have been built via recursive + Nix calls. */ + StorePathSet referenceablePaths; + for (auto & p : inputPaths) referenceablePaths.insert(p); + for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); + for (auto & p : addedPaths) referenceablePaths.insert(p); + + /* FIXME `needsHashRewrite` should probably be removed and we get to the + real reason why we aren't using the chroot dir */ + auto toRealPathChroot = [&](const Path & p) -> Path { + return useChroot && !needsHashRewrite() + ? chrootRootDir + p + : store.toRealPath(p); + }; + + /* Check whether the output paths were created, and make all + output paths read-only. Then get the references of each output (that we + might need to register), so we can topologically sort them. For the ones + that are most definitely already installed, we just store their final + name so we can also use it in rewrites. */ + StringSet outputsToSort; + struct AlreadyRegistered { StorePath path; }; + struct PerhapsNeedToRegister { StorePathSet refs; }; + std::map> outputReferencesIfUnregistered; + std::map outputStats; + for (auto & [outputName, _] : drv->outputs) { + auto scratchOutput = get(scratchOutputs, outputName); + if (!scratchOutput) + throw BuildError( + "builder for '%s' has no scratch output for '%s'", + store.printStorePath(drvPath), outputName); + auto actualPath = toRealPathChroot(store.printStorePath(*scratchOutput)); + + outputsToSort.insert(outputName); + + /* Updated wanted info to remove the outputs we definitely don't need to register */ + auto initialOutput = get(initialOutputs, outputName); + if (!initialOutput) + throw BuildError( + "builder for '%s' has no initial output for '%s'", + store.printStorePath(drvPath), outputName); + auto & initialInfo = *initialOutput; + + /* Don't register if already valid, and not checking */ + initialInfo.wanted = buildMode == bmCheck + || !(initialInfo.known && initialInfo.known->isValid()); + if (!initialInfo.wanted) { + outputReferencesIfUnregistered.insert_or_assign( + outputName, + AlreadyRegistered { .path = initialInfo.known->path }); + continue; + } + + auto optSt = maybeLstat(actualPath.c_str()); + if (!optSt) + throw BuildError( + "builder for '%s' failed to produce output path for output '%s' at '%s'", + store.printStorePath(drvPath), outputName, actualPath); + struct stat & st = *optSt; + +#ifndef __CYGWIN__ + /* Check that the output is not group or world writable, as + that means that someone else can have interfered with the + build. Also, the output should be owned by the build + user. */ + if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || + (buildUser && st.st_uid != buildUser->getUID())) + throw BuildError( + "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", + actualPath, outputName); +#endif + + /* Canonicalise first. This ensures that the path we're + rewriting doesn't contain a hard link to /etc/shadow or + something like that. */ + canonicalisePathMetaData( + actualPath, + buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, + inodesSeen); + + bool discardReferences = false; + if (auto udr = get(drvOptions->unsafeDiscardReferences, outputName)) { + discardReferences = *udr; + } + + StorePathSet references; + if (discardReferences) + debug("discarding references of output '%s'", outputName); + else { + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + references = scanForReferences(blank, actualPath, referenceablePaths); + } + + outputReferencesIfUnregistered.insert_or_assign( + outputName, + PerhapsNeedToRegister { .refs = references }); + outputStats.insert_or_assign(outputName, std::move(st)); + } + + auto sortedOutputNames = topoSort(outputsToSort, + {[&](const std::string & name) { + auto orifu = get(outputReferencesIfUnregistered, name); + if (!orifu) + throw BuildError( + "no output reference for '%s' in build of '%s'", + name, store.printStorePath(drvPath)); + return std::visit(overloaded { + /* Since we'll use the already installed versions of these, we + can treat them as leaves and ignore any references they + have. */ + [&](const AlreadyRegistered &) { return StringSet {}; }, + [&](const PerhapsNeedToRegister & refs) { + StringSet referencedOutputs; + /* FIXME build inverted map up front so no quadratic waste here */ + for (auto & r : refs.refs) + for (auto & [o, p] : scratchOutputs) + if (r == p) + referencedOutputs.insert(o); + return referencedOutputs; + }, + }, *orifu); + }}, + {[&](const std::string & path, const std::string & parent) { + // TODO with more -vvvv also show the temporary paths for manual inspection. + return BuildError( + "cycle detected in build of '%s' in the references of output '%s' from output '%s'", + store.printStorePath(drvPath), path, parent); + }}); + + std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); + + OutputPathMap finalOutputs; + + for (auto & outputName : sortedOutputNames) { + auto output = get(drv->outputs, outputName); + auto scratchPath = get(scratchOutputs, outputName); + assert(output && scratchPath); + auto actualPath = toRealPathChroot(store.printStorePath(*scratchPath)); + + auto finish = [&](StorePath finalStorePath) { + /* Store the final path */ + finalOutputs.insert_or_assign(outputName, finalStorePath); + /* The rewrite rule will be used in downstream outputs that refer to + use. This is why the topological sort is essential to do first + before this for loop. */ + if (*scratchPath != finalStorePath) + outputRewrites[std::string { scratchPath->hashPart() }] = std::string { finalStorePath.hashPart() }; + }; + + auto orifu = get(outputReferencesIfUnregistered, outputName); + assert(orifu); + + std::optional referencesOpt = std::visit(overloaded { + [&](const AlreadyRegistered & skippedFinalPath) -> std::optional { + finish(skippedFinalPath.path); + return std::nullopt; + }, + [&](const PerhapsNeedToRegister & r) -> std::optional { + return r.refs; + }, + }, *orifu); + + if (!referencesOpt) + continue; + auto references = *referencesOpt; + + auto rewriteOutput = [&](const StringMap & rewrites) { + /* Apply hash rewriting if necessary. */ + if (!rewrites.empty()) { + debug("rewriting hashes in '%1%'; cross fingers", actualPath); + + /* FIXME: Is this actually streaming? */ + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink(rewrites, nextSink); + dumpPath(actualPath, rsink); + rsink.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); + deletePath(actualPath); + movePath(tmpPath, actualPath); + + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, {}, inodesSeen); + } + }; + + auto rewriteRefs = [&]() -> StoreReferences { + /* In the CA case, we need the rewritten refs to calculate the + final path, therefore we look for a *non-rewritten + self-reference, and use a bool rather try to solve the + computationally intractable fixed point. */ + StoreReferences res { + .self = false, + }; + for (auto & r : references) { + auto name = r.name(); + auto origHash = std::string { r.hashPart() }; + if (r == *scratchPath) { + res.self = true; + } else if (auto outputRewrite = get(outputRewrites, origHash)) { + std::string newRef = *outputRewrite; + newRef += '-'; + newRef += name; + res.others.insert(StorePath { newRef }); + } else { + res.others.insert(r); + } + } + return res; + }; + + auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo { + auto st = get(outputStats, outputName); + if (!st) + throw BuildError( + "output path %1% without valid stats info", + actualPath); + if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat) + { + /* The output path should be a regular file without execute permission. */ + if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) + throw BuildError( + "output path '%1%' should be a non-executable regular file " + "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", + actualPath); + } + rewriteOutput(outputRewrites); + /* FIXME optimize and deduplicate with addToStore */ + std::string oldHashPart { scratchPath->hashPart() }; + auto got = [&]{ + auto fim = outputHash.method.getFileIngestionMethod(); + switch (fim) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::NixArchive: + { + HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; + auto fim = outputHash.method.getFileIngestionMethod(); + dumpPath( + {getFSSourceAccessor(), CanonPath(actualPath)}, + caSink, + (FileSerialisationMethod) fim); + return caSink.finish().first; + } + case FileIngestionMethod::Git: { + return git::dumpHash( + outputHash.hashAlgo, + {getFSSourceAccessor(), CanonPath(actualPath)}).hash; + } + } + assert(false); + }(); + + ValidPathInfo newInfo0 { + store, + outputPathName(drv->name, outputName), + ContentAddressWithReferences::fromParts( + outputHash.method, + std::move(got), + rewriteRefs()), + Hash::dummy, + }; + if (*scratchPath != newInfo0.path) { + // If the path has some self-references, we need to rewrite + // them. + // (note that this doesn't invalidate the ca hash we calculated + // above because it's computed *modulo the self-references*, so + // it already takes this rewrite into account). + rewriteOutput( + StringMap{{oldHashPart, + std::string(newInfo0.path.hashPart())}}); + } + + { + HashResult narHashAndSize = hashPath( + {getFSSourceAccessor(), CanonPath(actualPath)}, + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); + newInfo0.narHash = narHashAndSize.first; + newInfo0.narSize = narHashAndSize.second; + } + + assert(newInfo0.ca); + return newInfo0; + }; + + ValidPathInfo newInfo = std::visit(overloaded { + + [&](const DerivationOutput::InputAddressed & output) { + /* input-addressed case */ + auto requiredFinalPath = output.path; + /* Preemptively add rewrite rule for final hash, as that is + what the NAR hash will use rather than normalized-self references */ + if (*scratchPath != requiredFinalPath) + outputRewrites.insert_or_assign( + std::string { scratchPath->hashPart() }, + std::string { requiredFinalPath.hashPart() }); + rewriteOutput(outputRewrites); + HashResult narHashAndSize = hashPath( + {getFSSourceAccessor(), CanonPath(actualPath)}, + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); + ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; + newInfo0.narSize = narHashAndSize.second; + auto refs = rewriteRefs(); + newInfo0.references = std::move(refs.others); + if (refs.self) + newInfo0.references.insert(newInfo0.path); + return newInfo0; + }, + + [&](const DerivationOutput::CAFixed & dof) { + auto & wanted = dof.ca.hash; + + // Replace the output by a fresh copy of itself to make sure + // that there's no stale file descriptor pointing to it + Path tmpOutput = actualPath + ".tmp"; + copyFile( + std::filesystem::path(actualPath), + std::filesystem::path(tmpOutput), true); + + std::filesystem::rename(tmpOutput, actualPath); + + auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { + .method = dof.ca.method, + .hashAlgo = wanted.algo, + }); + + /* Check wanted hash */ + assert(newInfo0.ca); + auto & got = newInfo0.ca->hash; + if (wanted != got) { + /* Throw an error after registering the path as + valid. */ + miscMethods.noteHashMismatch(); + delayedException = std::make_exception_ptr( + BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", + store.printStorePath(drvPath), + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true))); + } + if (!newInfo0.references.empty()) { + auto numViolations = newInfo.references.size(); + delayedException = std::make_exception_ptr( + BuildError("fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", + store.printStorePath(drvPath), + numViolations, + store.printStorePath(*newInfo.references.begin()))); + } + + return newInfo0; + }, + + [&](const DerivationOutput::CAFloating & dof) { + return newInfoFromCA(dof); + }, + + [&](const DerivationOutput::Deferred &) -> ValidPathInfo { + // No derivation should reach that point without having been + // rewritten first + assert(false); + }, + + [&](const DerivationOutput::Impure & doi) { + return newInfoFromCA(DerivationOutput::CAFloating { + .method = doi.method, + .hashAlgo = doi.hashAlgo, + }); + }, + + }, output->raw); + + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, {}, inodesSeen); + + /* Calculate where we'll move the output files. In the checking case we + will leave leave them where they are, for now, rather than move to + their usual "final destination" */ + auto finalDestPath = store.printStorePath(newInfo.path); + + /* Lock final output path, if not already locked. This happens with + floating CA derivations and hash-mismatching fixed-output + derivations. */ + PathLocks dynamicOutputLock; + dynamicOutputLock.setDeletion(true); + auto optFixedPath = output->path(store, drv->name, outputName); + if (!optFixedPath || + store.printStorePath(*optFixedPath) != finalDestPath) + { + assert(newInfo.ca); + dynamicOutputLock.lockPaths({store.toRealPath(finalDestPath)}); + } + + /* Move files, if needed */ + if (store.toRealPath(finalDestPath) != actualPath) { + if (buildMode == bmRepair) { + /* Path already exists, need to replace it */ + replaceValidPath(store.toRealPath(finalDestPath), actualPath); + actualPath = store.toRealPath(finalDestPath); + } else if (buildMode == bmCheck) { + /* Path already exists, and we want to compare, so we leave out + new path in place. */ + } else if (store.isValidPath(newInfo.path)) { + /* Path already exists because CA path produced by something + else. No moving needed. */ + assert(newInfo.ca); + } else { + auto destPath = store.toRealPath(finalDestPath); + deletePath(destPath); + movePath(actualPath, destPath); + actualPath = destPath; + } + } + + auto & localStore = getLocalStore(); + + if (buildMode == bmCheck) { + + if (!store.isValidPath(newInfo.path)) continue; + ValidPathInfo oldInfo(*store.queryPathInfo(newInfo.path)); + if (newInfo.narHash != oldInfo.narHash) { + miscMethods.noteCheckMismatch(); + if (settings.runDiffHook || settings.keepFailed) { + auto dst = store.toRealPath(finalDestPath + checkSuffix); + deletePath(dst); + movePath(actualPath, dst); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + finalDestPath, dst, store.printStorePath(drvPath), tmpDir); + + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", + store.printStorePath(drvPath), store.toRealPath(finalDestPath), dst); + } else + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", + store.printStorePath(drvPath), store.toRealPath(finalDestPath)); + } + + /* Since we verified the build, it's now ultimately trusted. */ + if (!oldInfo.ultimate) { + oldInfo.ultimate = true; + localStore.signPathInfo(oldInfo); + localStore.registerValidPaths({{oldInfo.path, oldInfo}}); + } + + continue; + } + + /* For debugging, print out the referenced and unreferenced paths. */ + for (auto & i : inputPaths) { + if (references.count(i)) + debug("referenced input: '%1%'", store.printStorePath(i)); + else + debug("unreferenced input: '%1%'", store.printStorePath(i)); + } + + localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() + miscMethods.markContentsGood(newInfo.path); + + newInfo.deriver = drvPath; + newInfo.ultimate = true; + localStore.signPathInfo(newInfo); + + finish(newInfo.path); + + /* If it's a CA path, register it right away. This is necessary if it + isn't statically known so that we can safely unlock the path before + the next iteration */ + if (newInfo.ca) + localStore.registerValidPaths({{newInfo.path, newInfo}}); + + infos.emplace(outputName, std::move(newInfo)); + } + + if (buildMode == bmCheck) { + /* In case of fixed-output derivations, if there are + mismatches on `--check` an error must be thrown as this is + also a source for non-determinism. */ + if (delayedException) + std::rethrow_exception(delayedException); + return miscMethods.assertPathValidity(); + } + + /* Apply output checks. */ + checkOutputs(infos); + + /* Register each output path as valid, and register the sets of + paths referenced by each of them. If there are cycles in the + outputs, this will fail. */ + { + auto & localStore = getLocalStore(); + + ValidPathInfos infos2; + for (auto & [outputName, newInfo] : infos) { + infos2.insert_or_assign(newInfo.path, newInfo); + } + localStore.registerValidPaths(infos2); + } + + /* In case of a fixed-output derivation hash mismatch, throw an + exception now that we have registered the output as valid. */ + if (delayedException) + std::rethrow_exception(delayedException); + + /* If we made it this far, we are sure the output matches the derivation + (since the delayedException would be a fixed output CA mismatch). That + means it's safe to link the derivation to the output hash. We must do + that for floating CA derivations, which otherwise couldn't be cached, + but it's fine to do in all cases. */ + SingleDrvOutputs builtOutputs; + + for (auto & [outputName, newInfo] : infos) { + auto oldinfo = get(initialOutputs, outputName); + assert(oldinfo); + auto thisRealisation = Realisation { + .id = DrvOutput { + oldinfo->outputHash, + outputName + }, + .outPath = newInfo.path + }; + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) + && !drv->type().isImpure()) + { + store.signRealisation(thisRealisation); + store.registerDrvOutput(thisRealisation); + } + builtOutputs.emplace(outputName, thisRealisation); + } + + return builtOutputs; +} + + +void DerivationBuilder::checkOutputs(const std::map & outputs) +{ + std::map outputsByPath; + for (auto & output : outputs) + outputsByPath.emplace(store.printStorePath(output.second.path), output.second); + + for (auto & output : outputs) { + auto & outputName = output.first; + auto & info = output.second; + + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const StorePath & path) + { + uint64_t closureSize = 0; + StorePathSet pathsDone; + std::queue pathsLeft; + pathsLeft.push(path); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front(); + pathsLeft.pop(); + if (!pathsDone.insert(path).second) continue; + + auto i = outputsByPath.find(store.printStorePath(path)); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto & ref : i->second.references) + pathsLeft.push(ref); + } else { + auto info = store.queryPathInfo(path); + closureSize += info->narSize; + for (auto & ref : info->references) + pathsLeft.push(ref); + } + } + + return std::make_pair(std::move(pathsDone), closureSize); + }; + + auto applyChecks = [&](const DerivationOptions::OutputChecks & checks) + { + if (checks.maxSize && info.narSize > *checks.maxSize) + throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", + store.printStorePath(info.path), info.narSize, *checks.maxSize); + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) + throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", + store.printStorePath(info.path), closureSize, *checks.maxClosureSize); + } + + auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive) + { + /* Parse a list of reference specifiers. Each element must + either be a store path, or the symbolic name of the output + of the derivation (such as `out'). */ + StorePathSet spec; + for (auto & i : value) { + if (store.isStorePath(i)) + spec.insert(store.parseStorePath(i)); + else if (auto output = get(outputs, i)) + spec.insert(output->path); + else { + std::string outputsListing = concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; }); + throw BuildError("derivation '%s' output check for '%s' contains an illegal reference specifier '%s'," + " expected store path or output name (one of [%s])", + store.printStorePath(drvPath), outputName, i, outputsListing); + } + } + + auto used = recursive + ? getClosure(info.path).first + : info.references; + + if (recursive && checks.ignoreSelfRefs) + used.erase(info.path); + + StorePathSet badPaths; + + for (auto & i : used) + if (allowed) { + if (!spec.count(i)) + badPaths.insert(i); + } else { + if (spec.count(i)) + badPaths.insert(i); + } + + if (!badPaths.empty()) { + std::string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n "; + badPathsStr += store.printStorePath(i); + } + throw BuildError("output '%s' is not allowed to refer to the following paths:%s", + store.printStorePath(info.path), badPathsStr); + } + }; + + /* Mandatory check: absent whitelist, and present but empty + whitelist mean very different things. */ + if (auto & refs = checks.allowedReferences) { + checkRefs(*refs, true, false); + } + if (auto & refs = checks.allowedRequisites) { + checkRefs(*refs, true, true); + } + + /* Optimization: don't need to do anything when + disallowed and empty set. */ + if (!checks.disallowedReferences.empty()) { + checkRefs(checks.disallowedReferences, false, false); + } + if (!checks.disallowedRequisites.empty()) { + checkRefs(checks.disallowedRequisites, false, true); + } + }; + + std::visit(overloaded{ + [&](const DerivationOptions::OutputChecks & checks) { + applyChecks(checks); + }, + [&](const std::map & checksPerOutput) { + if (auto outputChecks = get(checksPerOutput, outputName)) + + applyChecks(*outputChecks); + }, + }, drvOptions->outputChecks); + } +} + + +void DerivationBuilder::deleteTmpDir(bool force) +{ + if (topTmpDir != "") { + /* Don't keep temporary directories for builtins because they + might have privileged stuff (like a copy of netrc). */ + if (settings.keepFailed && !force && !drv->isBuiltin()) { + printError("note: keeping build directory '%s'", tmpDir); + chmod(topTmpDir.c_str(), 0755); + chmod(tmpDir.c_str(), 0755); + } + else + deletePath(topTmpDir); + topTmpDir = ""; + tmpDir = ""; + } +} + + +bool LocalDerivationGoal::isReadDesc(int fd) +{ + return (hook && DerivationGoal::isReadDesc(fd)) || + (!hook && fd == builder.builderOut.get()); +} + + +StorePath DerivationBuilder::makeFallbackPath(OutputNameView outputName) +{ + // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path + // See doc/manual/source/protocols/store-path.md for details + // TODO: We may want to separate the responsibilities of constructing the path fingerprint and of actually doing the hashing + auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName); + return store.makeStorePath( + pathType, + // pass an all-zeroes hash + Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName)); +} + + +StorePath DerivationBuilder::makeFallbackPath(const StorePath & path) +{ + // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path + // See doc/manual/source/protocols/store-path.md for details + auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()); + return store.makeStorePath( + pathType, + // pass an all-zeroes hash + Hash(HashAlgorithm::SHA256), path.name()); +} + + +} From 9792d6bbd98bfeb872c14fc97af9b3ce2b08b0c8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 16 Mar 2025 23:50:48 -0400 Subject: [PATCH 236/396] Move `DerivationBuilder` to its own file/header The building logic is now free of the scheduling logic! (The interface between them is just what is in the new header. This makes it much easier to audit, and shrink over time.) --- maintainers/flake-module.nix | 2 + .../store/build/derivation-building-misc.hh | 52 + .../nix/store/build/derivation-goal.hh | 40 +- src/libstore/include/nix/store/meson.build | 1 + src/libstore/unix/build/derivation-builder.cc | 427 +-- .../unix/build/local-derivation-goal.cc | 3174 +--------------- .../nix/store/build/derivation-builder.hh | 3242 +---------------- .../unix/include/nix/store/meson.build | 1 + src/libstore/unix/meson.build | 1 + 9 files changed, 142 insertions(+), 6798 deletions(-) create mode 100644 src/libstore/include/nix/store/build/derivation-building-misc.hh diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index a8c52eb46..a9d10386f 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -284,6 +284,8 @@ ''^src/libstore/build/goal\.cc$'' ''^src/libstore/include/nix/store/build/goal\.hh$'' ''^src/libstore/unix/build/hook-instance\.cc$'' + ''^src/libstore/unix/build/derivation-builder\.cc$'' + ''^src/libstore/unix/include/nix/store/build/derivation-builder\.hh$'' ''^src/libstore/unix/build/local-derivation-goal\.cc$'' ''^src/libstore/unix/include/nix/store/build/local-derivation-goal\.hh$'' ''^src/libstore/build/substitution-goal\.cc$'' diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh new file mode 100644 index 000000000..caf94844d --- /dev/null +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -0,0 +1,52 @@ +#pragma once +/** + * @file Misc type defitions for both local building and remote (RPC building) + */ + +#include "nix/util/hash.hh" +#include "nix/store/path.hh" + +namespace nix { + +class Store; + +/** + * Unless we are repairing, we don't both to test validity and just assume it, + * so the choices are `Absent` or `Valid`. + */ +enum struct PathStatus { + Corrupt, + Absent, + Valid, +}; + +struct InitialOutputStatus +{ + StorePath path; + PathStatus status; + /** + * Valid in the store, and additionally non-corrupt if we are repairing + */ + bool isValid() const + { + return status == PathStatus::Valid; + } + /** + * Merely present, allowed to be corrupt + */ + bool isPresent() const + { + return status == PathStatus::Corrupt || status == PathStatus::Valid; + } +}; + +struct InitialOutput +{ + bool wanted; + Hash outputHash; + std::optional known; +}; + +void runPostBuildHook(Store & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths); + +} diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index 23675690d..ecd7e7b97 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -4,9 +4,7 @@ #include "nix/store/parsed-derivations.hh" #include "nix/store/derivations.hh" #include "nix/store/derivation-options.hh" -#ifndef _WIN32 -# include "nix/store/user-lock.hh" -#endif +#include "nix/store/build/derivation-building-misc.hh" #include "nix/store/outputs-spec.hh" #include "nix/store/store-api.hh" #include "nix/store/pathlocks.hh" @@ -22,40 +20,6 @@ struct HookInstance; typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -/** - * Unless we are repairing, we don't both to test validity and just assume it, - * so the choices are `Absent` or `Valid`. - */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /** - * Valid in the store, and additionally non-corrupt if we are repairing - */ - bool isValid() const { - return status == PathStatus::Valid; - } - /** - * Merely present, allowed to be corrupt - */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - Hash outputHash; - std::optional known; -}; - /** Used internally */ void runPostBuildHook( Store & store, @@ -308,6 +272,4 @@ struct DerivationGoal : public Goal }; }; -MakeError(NotDeterministic, BuildError); - } diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index 551031b32..5298b77a6 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -13,6 +13,7 @@ headers = [config_pub_h] + files( 'binary-cache-store.hh', 'build-result.hh', 'build/derivation-goal.hh', + 'build/derivation-building-misc.hh', 'build/drv-output-substitution-goal.hh', 'build/goal.hh', 'build/substitution-goal.hh', diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 973eef26e..4a51441b7 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1,4 +1,4 @@ -#include "nix/store/build/local-derivation-goal.hh" +#include "nix/store/build/derivation-builder.hh" #include "nix/store/local-store.hh" #include "nix/util/processes.hh" #include "nix/store/indirect-root-store.hh" @@ -22,7 +22,6 @@ #include "nix/store/posix-fs-canonicalise.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/store/restricted-store.hh" -#include "nix/store/config.hh" #include @@ -80,106 +79,7 @@ extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, namespace nix { -/** - * Parameters by (mostly) `const` reference for `DerivationBuilder`. - */ -struct DerivationBuilderParams -{ - /** The path of the derivation. */ - const StorePath & drvPath; - - BuildResult & buildResult; - - /** - * The derivation stored at drvPath. - * - * @todo Remove double indirection by delaying when this is - * initialized. - */ - const std::unique_ptr & drv; - - const std::unique_ptr & parsedDrv; - const std::unique_ptr & drvOptions; - - // The remainder is state held during the build. - - /** - * All input paths (that is, the union of FS closures of the - * immediate input paths). - */ - const StorePathSet & inputPaths; - - /** - * @note we do in fact mutate this - */ - std::map & initialOutputs; - - const BuildMode & buildMode; - - DerivationBuilderParams( - const StorePath & drvPath, - const BuildMode & buildMode, - BuildResult & buildResult, - const std::unique_ptr & drv, - const std::unique_ptr & parsedDrv, - const std::unique_ptr & drvOptions, - const StorePathSet & inputPaths, - std::map & initialOutputs) - : drvPath{drvPath} - , buildResult{buildResult} - , drv{drv} - , parsedDrv{parsedDrv} - , drvOptions{drvOptions} - , inputPaths{inputPaths} - , initialOutputs{initialOutputs} - , buildMode{buildMode} - { } - - DerivationBuilderParams(DerivationBuilderParams &&) = default; -}; - -/** - * Callbacks that `DerivationBuilder` needs. - */ -struct DerivationBuilderCallbacks -{ - /** - * Open a log file and a pipe to it. - */ - virtual Path openLogFile() = 0; - - /** - * Close the log file. - */ - virtual void closeLogFile() = 0; - - /** - * Aborts if any output is not valid or corrupt, and otherwise - * returns a 'SingleDrvOutputs' structure containing all outputs. - * - * @todo Probably should just be in `DerivationGoal`. - */ - virtual SingleDrvOutputs assertPathValidity() = 0; - - virtual void appendLogTailErrorMsg(std::string & msg) = 0; - - /** - * Hook up `builderOut` to some mechanism to ingest the log - * - * @todo this should be reworked - */ - virtual void childStarted() = 0; - - /** - * @todo this should be reworked - */ - virtual void childTerminated() = 0; - - virtual void noteHashMismatch(void) = 0; - virtual void noteCheckMismatch(void) = 0; - - virtual void markContentsGood(const StorePath & path) = 0; -}; +MakeError(NotDeterministic, BuildError); /** * This class represents the state for building locally. @@ -192,7 +92,7 @@ struct DerivationBuilderCallbacks * rather than incoming call edges that either should be removed, or * become (higher order) function parameters. */ -class DerivationBuilder : public RestrictionContext, DerivationBuilderParams +class DerivationBuilderImpl : public DerivationBuilder, DerivationBuilderParams { Store & store; @@ -200,7 +100,7 @@ class DerivationBuilder : public RestrictionContext, DerivationBuilderParams public: - DerivationBuilder( + DerivationBuilderImpl( Store & store, DerivationBuilderCallbacks & miscMethods, DerivationBuilderParams params) @@ -211,16 +111,6 @@ public: LocalStore & getLocalStore(); - /** - * User selected for running the builder. - */ - std::unique_ptr buildUser; - - /** - * The process ID of the builder. - */ - Pid pid; - private: /** @@ -244,16 +134,6 @@ private: */ Path tmpDirInSandbox; -public: - - /** - * Master side of the pseudoterminal used for the builder's - * standard output/error. - */ - AutoCloseFD builderOut; - -private: - /** * Pipe for synchronising updates to the builder namespaces. */ @@ -304,17 +184,17 @@ private: : source(source), optional(optional) { } }; - typedef map PathsInChroot; // maps target path to source path + typedef std::map PathsInChroot; // maps target path to source path PathsInChroot pathsInChroot; - typedef map Environment; + typedef std::map Environment; Environment env; /** * Hash rewriting. */ StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; + typedef std::map RedirectedOutputs; RedirectedOutputs redirectedOutputs; /** @@ -386,12 +266,12 @@ public: * @returns true if successful, false if we could not acquire a build * user. In that case, the caller must wait and then try again. */ - bool prepareBuild(); + bool prepareBuild() override; /** * Start building a derivation. */ - void startBuilder(); + void startBuilder() override;; /** * Tear down build environment after the builder exits (either on @@ -402,7 +282,7 @@ public: * more information. The second case indicates success, and * realisations for each output of the derivation are returned. */ - std::variant, SingleDrvOutputs> unprepareBuild(); + std::variant, SingleDrvOutputs> unprepareBuild() override; private: @@ -437,7 +317,7 @@ public: * Stop the in-process nix daemon thread. * @see startDaemon */ - void stopDaemon(); + void stopDaemon() override; private: @@ -471,13 +351,13 @@ public: /** * Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); + void deleteTmpDir(bool force) override; /** * Kill any processes running under the build user UID or in the * cgroup of the build. */ - void killSandbox(bool getStats); + void killSandbox(bool getStats) override; private: @@ -500,112 +380,15 @@ private: StorePath makeFallbackPath(OutputNameView outputName); }; -/** - * This hooks up `DerivationBuilder` to the scheduler / goal machinary. - * - * @todo Eventually, this shouldn't exist, because `DerivationGoal` can - * just choose to use `DerivationBuilder` or its remote-building - * equalivalent directly, at the "value level" rather than "class - * inheritance hierarchy" level. - */ -struct LocalDerivationGoal : DerivationGoal, DerivationBuilderCallbacks +std::unique_ptr makeDerivationBuilder( + Store & store, + DerivationBuilderCallbacks & miscMethods, + DerivationBuilderParams params) { - DerivationBuilder builder; - - LocalDerivationGoal(const StorePath & drvPath, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode) - : DerivationGoal{drvPath, wantedOutputs, worker, buildMode} - , builder{ - worker.store, - static_cast(*this), - DerivationBuilderParams { - DerivationGoal::drvPath, - DerivationGoal::buildMode, - DerivationGoal::buildResult, - DerivationGoal::drv, - DerivationGoal::parsedDrv, - DerivationGoal::drvOptions, - DerivationGoal::inputPaths, - DerivationGoal::initialOutputs, - }, - } - {} - - LocalDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal) - : DerivationGoal{drvPath, drv, wantedOutputs, worker, buildMode} - , builder{ - worker.store, - static_cast(*this), - DerivationBuilderParams { - DerivationGoal::drvPath, - DerivationGoal::buildMode, - DerivationGoal::buildResult, - DerivationGoal::drv, - DerivationGoal::parsedDrv, - DerivationGoal::drvOptions, - DerivationGoal::inputPaths, - DerivationGoal::initialOutputs, - }, - } - {} - - virtual ~LocalDerivationGoal() override; - - /** - * The additional states. - */ - Goal::Co tryLocalBuild() override; - - bool isReadDesc(int fd) override; - - /** - * Forcibly kill the child process, if any. - * - * Called by destructor, can't be overridden - */ - void killChild() override final; - - void childStarted() override; - void childTerminated() override; - - void noteHashMismatch(void) override; - void noteCheckMismatch(void) override; - - void markContentsGood(const StorePath &) override; - - // Fake overrides to instantiate identically-named virtual methods - - Path openLogFile() override { - return DerivationGoal::openLogFile(); - } - void closeLogFile() override { - DerivationGoal::closeLogFile(); - } - SingleDrvOutputs assertPathValidity() override { - return DerivationGoal::assertPathValidity(); - } - void appendLogTailErrorMsg(std::string & msg) override { - DerivationGoal::appendLogTailErrorMsg(msg); - } -}; - -std::shared_ptr makeLocalDerivationGoal( - const StorePath & drvPath, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode) -{ - return std::make_shared(drvPath, wantedOutputs, worker, buildMode); -} - -std::shared_ptr makeLocalDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode) -{ - return std::make_shared(drvPath, drv, wantedOutputs, worker, buildMode); + return std::make_unique( + store, + miscMethods, + std::move(params)); } void handleDiffHook( @@ -642,20 +425,10 @@ void handleDiffHook( } } -const Path DerivationBuilder::homeDir = "/homeless-shelter"; +const Path DerivationBuilderImpl::homeDir = "/homeless-shelter"; -LocalDerivationGoal::~LocalDerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { builder.deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } - try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } - try { builder.stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } -} - - -inline bool DerivationBuilder::needsHashRewrite() +inline bool DerivationBuilderImpl::needsHashRewrite() { #ifdef __linux__ return !useChroot; @@ -666,7 +439,7 @@ inline bool DerivationBuilder::needsHashRewrite() } -LocalStore & DerivationBuilder::getLocalStore() +LocalStore & DerivationBuilderImpl::getLocalStore() { auto p = dynamic_cast(&store); assert(p); @@ -674,28 +447,7 @@ LocalStore & DerivationBuilder::getLocalStore() } -void LocalDerivationGoal::killChild() -{ - if (builder.pid != -1) { - worker.childTerminated(this); - - /* If we're using a build user, then there is a tricky race - condition: if we kill the build user before the child has - done its setuid() to the build user uid, then it won't be - killed, and we'll potentially lock up in pid.wait(). So - also send a conventional kill to the child. */ - ::kill(-builder.pid, SIGKILL); /* ignore the result */ - - builder.killSandbox(true); - - builder.pid.wait(); - } - - DerivationGoal::killChild(); -} - - -void DerivationBuilder::killSandbox(bool getStats) +void DerivationBuilderImpl::killSandbox(bool getStats) { if (cgroup) { #ifdef __linux__ @@ -717,91 +469,7 @@ void DerivationBuilder::killSandbox(bool getStats) } -void LocalDerivationGoal::childStarted() -{ - worker.childStarted(shared_from_this(), {builder.builderOut.get()}, true, true); -} - -void LocalDerivationGoal::childTerminated() -{ - worker.childTerminated(this); -} - -void LocalDerivationGoal::noteHashMismatch() -{ - worker.hashMismatch = true; -} - - -void LocalDerivationGoal::noteCheckMismatch() -{ - worker.checkMismatch = true; -} - - -void LocalDerivationGoal::markContentsGood(const StorePath & path) -{ - worker.markContentsGood(path); -} - - -Goal::Co LocalDerivationGoal::tryLocalBuild() -{ - assert(!hook); - - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs) { - outputLocks.unlock(); - co_await waitForBuildSlot(); - co_return tryToBuild(); - } - - if (!builder.prepareBuild()) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); - co_await waitForAWhile(); - co_return tryLocalBuild(); - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - builder.startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - builder.buildUser.reset(); - worker.permanentFailure = true; - co_return done(BuildResult::InputRejected, {}, std::move(e)); - } - - started(); - co_await Suspend{}; - - trace("build done"); - - auto res = builder.unprepareBuild(); - // N.B. cannot use `std::visit` with co-routine return - if (auto * ste = std::get_if<0>(&res)) { - outputLocks.unlock(); - co_return done(std::move(ste->first), {}, std::move(ste->second)); - } else if (auto * builtOutputs = std::get_if<1>(&res)) { - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - co_return done(BuildResult::Built, std::move(*builtOutputs)); - } else { - unreachable(); - } -} - -bool DerivationBuilder::prepareBuild() +bool DerivationBuilderImpl::prepareBuild() { /* Cache this */ derivationType = drv->type(); @@ -858,7 +526,7 @@ bool DerivationBuilder::prepareBuild() } -std::variant, SingleDrvOutputs> DerivationBuilder::unprepareBuild() +std::variant, SingleDrvOutputs> DerivationBuilderImpl::unprepareBuild() { Finally releaseBuildUser([&](){ /* Release the build user at the end of this function. We don't do @@ -996,7 +664,7 @@ static void movePath(const Path & src, const Path & dst) extern void replaceValidPath(const Path & storePath, const Path & tmpPath); -bool DerivationBuilder::cleanupDecideWhetherDiskFull() +bool DerivationBuilderImpl::cleanupDecideWhetherDiskFull() { bool diskFull = false; @@ -1089,7 +757,7 @@ static void rethrowExceptionAsError() /** * Send the current exception to the parent in the format expected by - * `DerivationBuilder::processSandboxSetupMessages()`. + * `DerivationBuilderImpl::processSandboxSetupMessages()`. */ static void handleChildException(bool sendException) { @@ -1106,7 +774,7 @@ static void handleChildException(bool sendException) } } -void DerivationBuilder::startBuilder() +void DerivationBuilderImpl::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) #ifdef __linux__ @@ -1723,7 +1391,7 @@ void DerivationBuilder::startBuilder() } -void DerivationBuilder::processSandboxSetupMessages() +void DerivationBuilderImpl::processSandboxSetupMessages() { std::vector msgs; while (true) { @@ -1752,7 +1420,7 @@ void DerivationBuilder::processSandboxSetupMessages() } -void DerivationBuilder::initTmpDir() +void DerivationBuilderImpl::initTmpDir() { /* In a sandbox, for determinism, always use the same temporary directory. */ @@ -1796,7 +1464,7 @@ void DerivationBuilder::initTmpDir() } -void DerivationBuilder::initEnv() +void DerivationBuilderImpl::initEnv() { env.clear(); @@ -1864,7 +1532,7 @@ void DerivationBuilder::initEnv() } -void DerivationBuilder::writeStructuredAttrs() +void DerivationBuilderImpl::writeStructuredAttrs() { if (parsedDrv) { auto json = parsedDrv->prepareStructuredAttrs( @@ -1893,7 +1561,7 @@ void DerivationBuilder::writeStructuredAttrs() } -void DerivationBuilder::startDaemon() +void DerivationBuilderImpl::startDaemon() { experimentalFeatureSettings.require(Xp::RecursiveNix); @@ -1961,7 +1629,7 @@ void DerivationBuilder::startDaemon() } -void DerivationBuilder::stopDaemon() +void DerivationBuilderImpl::stopDaemon() { if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) { // According to the POSIX standard, the 'shutdown' function should @@ -1994,7 +1662,7 @@ void DerivationBuilder::stopDaemon() } -void DerivationBuilder::addDependency(const StorePath & path) +void DerivationBuilderImpl::addDependency(const StorePath & path) { if (isAllowed(path)) return; @@ -2046,7 +1714,7 @@ void DerivationBuilder::addDependency(const StorePath & path) } } -void DerivationBuilder::chownToBuilder(const Path & path) +void DerivationBuilderImpl::chownToBuilder(const Path & path) { if (!buildUser) return; if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) @@ -2142,7 +1810,7 @@ void setupSeccomp() } -void DerivationBuilder::runChild() +void DerivationBuilderImpl::runChild() { /* Warning: in the child we should absolutely not make any SQLite calls! */ @@ -2655,7 +2323,7 @@ void DerivationBuilder::runChild() } -SingleDrvOutputs DerivationBuilder::registerOutputs() +SingleDrvOutputs DerivationBuilderImpl::registerOutputs() { std::map infos; @@ -3207,7 +2875,7 @@ SingleDrvOutputs DerivationBuilder::registerOutputs() } -void DerivationBuilder::checkOutputs(const std::map & outputs) +void DerivationBuilderImpl::checkOutputs(const std::map & outputs) { std::map outputsByPath; for (auto & output : outputs) @@ -3342,7 +3010,7 @@ void DerivationBuilder::checkOutputs(const std::map } -void DerivationBuilder::deleteTmpDir(bool force) +void DerivationBuilderImpl::deleteTmpDir(bool force) { if (topTmpDir != "") { /* Don't keep temporary directories for builtins because they @@ -3360,14 +3028,7 @@ void DerivationBuilder::deleteTmpDir(bool force) } -bool LocalDerivationGoal::isReadDesc(int fd) -{ - return (hook && DerivationGoal::isReadDesc(fd)) || - (!hook && fd == builder.builderOut.get()); -} - - -StorePath DerivationBuilder::makeFallbackPath(OutputNameView outputName) +StorePath DerivationBuilderImpl::makeFallbackPath(OutputNameView outputName) { // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path // See doc/manual/source/protocols/store-path.md for details @@ -3380,7 +3041,7 @@ StorePath DerivationBuilder::makeFallbackPath(OutputNameView outputName) } -StorePath DerivationBuilder::makeFallbackPath(const StorePath & path) +StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path) { // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path // See doc/manual/source/protocols/store-path.md for details diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 973eef26e..285f9cc2e 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1,30 +1,10 @@ #include "nix/store/build/local-derivation-goal.hh" #include "nix/store/local-store.hh" #include "nix/util/processes.hh" -#include "nix/store/indirect-root-store.hh" -#include "nix/store/build/hook-instance.hh" #include "nix/store/build/worker.hh" -#include "nix/store/builtins.hh" -#include "nix/store/builtins/buildenv.hh" -#include "nix/store/path-references.hh" -#include "nix/util/finally.hh" #include "nix/util/util.hh" -#include "nix/util/archive.hh" -#include "nix/util/git.hh" -#include "nix/util/compression.hh" -#include "nix/store/daemon.hh" -#include "nix/util/topo-sort.hh" -#include "nix/util/callback.hh" -#include "nix/util/json-utils.hh" -#include "nix/util/current-process.hh" -#include "nix/store/build/child.hh" -#include "nix/util/unix-domain-socket.hh" -#include "nix/store/posix-fs-canonicalise.hh" -#include "nix/util/posix-source-accessor.hh" #include "nix/store/restricted-store.hh" -#include "nix/store/config.hh" - -#include +#include "nix/store/build/derivation-builder.hh" #include #include @@ -40,466 +20,11 @@ #include #endif -/* Includes required for chroot support. */ -#ifdef __linux__ -# include "linux/fchmodat2-compat.hh" -# include -# include -# include -# include -# include -# include -# include -# include -# include "nix/util/namespaces.hh" -# if HAVE_SECCOMP -# include -# endif -# define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) -# include "nix/util/cgroup.hh" -# include "nix/store/personality.hh" -#endif - -#ifdef __APPLE__ -#include -#include -#include - -/* This definition is undocumented but depended upon by all major browsers. */ -extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf); -#endif - #include #include -#include - -#include "nix/util/strings.hh" -#include "nix/util/signals.hh" - -#include "store-config-private.hh" namespace nix { -/** - * Parameters by (mostly) `const` reference for `DerivationBuilder`. - */ -struct DerivationBuilderParams -{ - /** The path of the derivation. */ - const StorePath & drvPath; - - BuildResult & buildResult; - - /** - * The derivation stored at drvPath. - * - * @todo Remove double indirection by delaying when this is - * initialized. - */ - const std::unique_ptr & drv; - - const std::unique_ptr & parsedDrv; - const std::unique_ptr & drvOptions; - - // The remainder is state held during the build. - - /** - * All input paths (that is, the union of FS closures of the - * immediate input paths). - */ - const StorePathSet & inputPaths; - - /** - * @note we do in fact mutate this - */ - std::map & initialOutputs; - - const BuildMode & buildMode; - - DerivationBuilderParams( - const StorePath & drvPath, - const BuildMode & buildMode, - BuildResult & buildResult, - const std::unique_ptr & drv, - const std::unique_ptr & parsedDrv, - const std::unique_ptr & drvOptions, - const StorePathSet & inputPaths, - std::map & initialOutputs) - : drvPath{drvPath} - , buildResult{buildResult} - , drv{drv} - , parsedDrv{parsedDrv} - , drvOptions{drvOptions} - , inputPaths{inputPaths} - , initialOutputs{initialOutputs} - , buildMode{buildMode} - { } - - DerivationBuilderParams(DerivationBuilderParams &&) = default; -}; - -/** - * Callbacks that `DerivationBuilder` needs. - */ -struct DerivationBuilderCallbacks -{ - /** - * Open a log file and a pipe to it. - */ - virtual Path openLogFile() = 0; - - /** - * Close the log file. - */ - virtual void closeLogFile() = 0; - - /** - * Aborts if any output is not valid or corrupt, and otherwise - * returns a 'SingleDrvOutputs' structure containing all outputs. - * - * @todo Probably should just be in `DerivationGoal`. - */ - virtual SingleDrvOutputs assertPathValidity() = 0; - - virtual void appendLogTailErrorMsg(std::string & msg) = 0; - - /** - * Hook up `builderOut` to some mechanism to ingest the log - * - * @todo this should be reworked - */ - virtual void childStarted() = 0; - - /** - * @todo this should be reworked - */ - virtual void childTerminated() = 0; - - virtual void noteHashMismatch(void) = 0; - virtual void noteCheckMismatch(void) = 0; - - virtual void markContentsGood(const StorePath & path) = 0; -}; - -/** - * This class represents the state for building locally. - * - * @todo Ideally, it would not be a class, but a single function. - * However, besides the main entry point, there are a few more methods - * which are externally called, and need to be gotten rid of. There are - * also some virtual methods (either directly here or inherited from - * `DerivationBuilderCallbacks`, a stop-gap) that represent outgoing - * rather than incoming call edges that either should be removed, or - * become (higher order) function parameters. - */ -class DerivationBuilder : public RestrictionContext, DerivationBuilderParams -{ - Store & store; - - DerivationBuilderCallbacks & miscMethods; - -public: - - DerivationBuilder( - Store & store, - DerivationBuilderCallbacks & miscMethods, - DerivationBuilderParams params) - : DerivationBuilderParams{std::move(params)} - , store{store} - , miscMethods{miscMethods} - { } - - LocalStore & getLocalStore(); - - /** - * User selected for running the builder. - */ - std::unique_ptr buildUser; - - /** - * The process ID of the builder. - */ - Pid pid; - -private: - - /** - * The cgroup of the builder, if any. - */ - std::optional cgroup; - - /** - * The temporary directory used for the build. - */ - Path tmpDir; - - /** - * The top-level temporary directory. `tmpDir` is either equal to - * or a child of this directory. - */ - Path topTmpDir; - - /** - * The path of the temporary directory in the sandbox. - */ - Path tmpDirInSandbox; - -public: - - /** - * Master side of the pseudoterminal used for the builder's - * standard output/error. - */ - AutoCloseFD builderOut; - -private: - - /** - * Pipe for synchronising updates to the builder namespaces. - */ - Pipe userNamespaceSync; - - /** - * The mount namespace and user namespace of the builder, used to add additional - * paths to the sandbox as a result of recursive Nix calls. - */ - AutoCloseFD sandboxMountNamespace; - AutoCloseFD sandboxUserNamespace; - - /** - * On Linux, whether we're doing the build in its own user - * namespace. - */ - bool usingUserNamespace = true; - - /** - * Whether we're currently doing a chroot build. - */ - bool useChroot = false; - - /** - * The root of the chroot environment. - */ - Path chrootRootDir; - - /** - * RAII object to delete the chroot directory. - */ - std::shared_ptr autoDelChroot; - - /** - * The sort of derivation we are building. - * - * Just a cached value, can be recomputed from `drv`. - */ - std::optional derivationType; - - /** - * Stuff we need to pass to initChild(). - */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map PathsInChroot; // maps target path to source path - PathsInChroot pathsInChroot; - - typedef map Environment; - Environment env; - - /** - * Hash rewriting. - */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /** - * The output paths used during the build. - * - * - Input-addressed derivations or fixed content-addressed outputs are - * sometimes built when some of their outputs already exist, and can not - * be hidden via sandboxing. We use temporary locations instead and - * rewrite after the build. Otherwise the regular predetermined paths are - * put here. - * - * - Floating content-addressing derivations do not know their final build - * output paths until the outputs are hashed, so random locations are - * used, and then renamed. The randomness helps guard against hidden - * self-references. - */ - OutputPathMap scratchOutputs; - - uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } - - const static Path homeDir; - - /** - * The recursive Nix daemon socket. - */ - AutoCloseFD daemonSocket; - - /** - * The daemon main thread. - */ - std::thread daemonThread; - - /** - * The daemon worker threads. - */ - std::vector daemonWorkerThreads; - - const StorePathSet & originalPaths() override - { - return inputPaths; - } - - bool isAllowed(const StorePath & path) override - { - return inputPaths.count(path) || addedPaths.count(path); - } - bool isAllowed(const DrvOutput & id) override - { - return addedDrvOutputs.count(id); - } - - bool isAllowed(const DerivedPath & req); - - friend struct RestrictedStore; - - /** - * Whether we need to perform hash rewriting if there are valid output paths. - */ - bool needsHashRewrite(); - -public: - - /** - * Set up build environment / sandbox, acquiring resources (e.g. - * locks as needed). After this is run, the builder should be - * started. - * - * @returns true if successful, false if we could not acquire a build - * user. In that case, the caller must wait and then try again. - */ - bool prepareBuild(); - - /** - * Start building a derivation. - */ - void startBuilder(); - - /** - * Tear down build environment after the builder exits (either on - * its own or if it is killed). - * - * @returns The first case indicates failure during output - * processing. A status code and exception are returned, providing - * more information. The second case indicates success, and - * realisations for each output of the derivation are returned. - */ - std::variant, SingleDrvOutputs> unprepareBuild(); - -private: - - /** - * Fill in the environment for the builder. - */ - void initEnv(); - - /** - * Process messages send by the sandbox initialization. - */ - void processSandboxSetupMessages(); - - /** - * Setup tmp dir location. - */ - void initTmpDir(); - - /** - * Write a JSON file containing the derivation attributes. - */ - void writeStructuredAttrs(); - - /** - * Start an in-process nix daemon thread for recursive-nix. - */ - void startDaemon(); - -public: - - /** - * Stop the in-process nix daemon thread. - * @see startDaemon - */ - void stopDaemon(); - -private: - - void addDependency(const StorePath & path) override; - - /** - * Make a file owned by the builder. - */ - void chownToBuilder(const Path & path); - - /** - * Run the builder's process. - */ - void runChild(); - - /** - * Check that the derivation outputs all exist and register them - * as valid. - */ - SingleDrvOutputs registerOutputs(); - - /** - * Check that an output meets the requirements specified by the - * 'outputChecks' attribute (or the legacy - * '{allowed,disallowed}{References,Requisites}' attributes). - */ - void checkOutputs(const std::map & outputs); - -public: - - /** - * Delete the temporary directory, if we have one. - */ - void deleteTmpDir(bool force); - - /** - * Kill any processes running under the build user UID or in the - * cgroup of the build. - */ - void killSandbox(bool getStats); - -private: - - bool cleanupDecideWhetherDiskFull(); - - /** - * Create alternative path calculated from but distinct from the - * input, so we can avoid overwriting outputs (or other store paths) - * that already exist. - */ - StorePath makeFallbackPath(const StorePath & path); - - /** - * Make a path to another based on the output name along with the - * derivation hash. - * - * @todo Add option to randomize, so we can audit whether our - * rewrites caught everything - */ - StorePath makeFallbackPath(OutputNameView outputName); -}; - /** * This hooks up `DerivationBuilder` to the scheduler / goal machinary. * @@ -510,13 +35,13 @@ private: */ struct LocalDerivationGoal : DerivationGoal, DerivationBuilderCallbacks { - DerivationBuilder builder; + std::unique_ptr builder; LocalDerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) : DerivationGoal{drvPath, wantedOutputs, worker, buildMode} - , builder{ + , builder{makeDerivationBuilder( worker.store, static_cast(*this), DerivationBuilderParams { @@ -528,15 +53,14 @@ struct LocalDerivationGoal : DerivationGoal, DerivationBuilderCallbacks DerivationGoal::drvOptions, DerivationGoal::inputPaths, DerivationGoal::initialOutputs, - }, - } + })} {} LocalDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal) : DerivationGoal{drvPath, drv, wantedOutputs, worker, buildMode} - , builder{ + , builder{makeDerivationBuilder( worker.store, static_cast(*this), DerivationBuilderParams { @@ -548,8 +72,7 @@ struct LocalDerivationGoal : DerivationGoal, DerivationBuilderCallbacks DerivationGoal::drvOptions, DerivationGoal::inputPaths, DerivationGoal::initialOutputs, - }, - } + })} {} virtual ~LocalDerivationGoal() override; @@ -608,75 +131,20 @@ std::shared_ptr makeLocalDerivationGoal( return std::make_shared(drvPath, drv, wantedOutputs, worker, buildMode); } -void handleDiffHook( - uid_t uid, uid_t gid, - const Path & tryA, const Path & tryB, - const Path & drvPath, const Path & tmpDir) -{ - auto & diffHookOpt = settings.diffHook.get(); - if (diffHookOpt && settings.runDiffHook) { - auto & diffHook = *diffHookOpt; - try { - auto diffRes = runProgram(RunOptions { - .program = diffHook, - .lookupPath = true, - .args = {tryA, tryB, drvPath, tmpDir}, - .uid = uid, - .gid = gid, - .chdir = "/" - }); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, - "diff-hook program '%1%' %2%", - diffHook, - statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - // FIXME: wrap errors. - ei.msg = HintFmt("diff hook execution failed: %s", ei.msg.str()); - logError(ei); - } - } -} - -const Path DerivationBuilder::homeDir = "/homeless-shelter"; - LocalDerivationGoal::~LocalDerivationGoal() { /* Careful: we should never ever throw an exception from a destructor. */ - try { builder.deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } + try { builder->deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } - try { builder.stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } -} - - -inline bool DerivationBuilder::needsHashRewrite() -{ -#ifdef __linux__ - return !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; -#endif -} - - -LocalStore & DerivationBuilder::getLocalStore() -{ - auto p = dynamic_cast(&store); - assert(p); - return *p; + try { builder->stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } } void LocalDerivationGoal::killChild() { - if (builder.pid != -1) { + if (builder->pid != -1) { worker.childTerminated(this); /* If we're using a build user, then there is a tricky race @@ -684,42 +152,20 @@ void LocalDerivationGoal::killChild() done its setuid() to the build user uid, then it won't be killed, and we'll potentially lock up in pid.wait(). So also send a conventional kill to the child. */ - ::kill(-builder.pid, SIGKILL); /* ignore the result */ + ::kill(-builder->pid, SIGKILL); /* ignore the result */ - builder.killSandbox(true); + builder->killSandbox(true); - builder.pid.wait(); + builder->pid.wait(); } DerivationGoal::killChild(); } -void DerivationBuilder::killSandbox(bool getStats) -{ - if (cgroup) { - #ifdef __linux__ - auto stats = destroyCgroup(*cgroup); - if (getStats) { - buildResult.cpuUser = stats.cpuUser; - buildResult.cpuSystem = stats.cpuSystem; - } - #else - unreachable(); - #endif - } - - else if (buildUser) { - auto uid = buildUser->getUID(); - assert(uid != 0); - killUser(uid); - } -} - - void LocalDerivationGoal::childStarted() { - worker.childStarted(shared_from_this(), {builder.builderOut.get()}, true, true); + worker.childStarted(shared_from_this(), {builder->builderOut.get()}, true, true); } void LocalDerivationGoal::childTerminated() @@ -756,7 +202,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() co_return tryToBuild(); } - if (!builder.prepareBuild()) { + if (!builder->prepareBuild()) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); @@ -769,11 +215,11 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() try { /* Okay, we have to build. */ - builder.startBuilder(); + builder->startBuilder(); } catch (BuildError & e) { outputLocks.unlock(); - builder.buildUser.reset(); + builder->buildUser.reset(); worker.permanentFailure = true; co_return done(BuildResult::InputRejected, {}, std::move(e)); } @@ -783,7 +229,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() trace("build done"); - auto res = builder.unprepareBuild(); + auto res = builder->unprepareBuild(); // N.B. cannot use `std::visit` with co-routine return if (auto * ste = std::get_if<0>(&res)) { outputLocks.unlock(); @@ -801,2595 +247,11 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() } } -bool DerivationBuilder::prepareBuild() -{ - /* Cache this */ - derivationType = drv->type(); - - /* Are we doing a chroot build? */ - { - if (settings.sandboxMode == smEnabled) { - if (drvOptions->noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", store.printStorePath(drvPath)); -#ifdef __APPLE__ - if (drvOptions->additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = derivationType->isSandboxed() && !drvOptions->noChroot; - } - - auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir.get()) { - #ifdef __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - - #ifdef __linux__ - if (useChroot) { - if (!mountAndPidNamespacesSupported()) { - if (!settings.sandboxFallback) - throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); - debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); - useChroot = false; - } - } - #endif - - if (useBuildUsers()) { - if (!buildUser) - buildUser = acquireUserLock(drvOptions->useUidRange(*drv) ? 65536 : 1, useChroot); - - if (!buildUser) { - return false; - } - } - - return true; -} - - -std::variant, SingleDrvOutputs> DerivationBuilder::unprepareBuild() -{ - Finally releaseBuildUser([&](){ - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - buildUser.reset(); - }); - - sandboxMountNamespace = -1; - sandboxUserNamespace = -1; - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = pid.kill(); - - debug("builder process for '%s' finished", store.printStorePath(drvPath)); - - buildResult.timesBuilt++; - buildResult.stopTime = time(0); - - /* So the child is gone now. */ - miscMethods.childTerminated(); - - /* Close the read side of the logger pipe. */ - builderOut.close(); - - /* Close the log file. */ - miscMethods.closeLogFile(); - - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - killSandbox(true); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); - - if (buildResult.cpuUser && buildResult.cpuSystem) { - debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs", - store.printStorePath(drvPath), - status, - ((double) buildResult.cpuUser->count()) / 1000000, - ((double) buildResult.cpuSystem->count()) / 1000000); - } - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - diskFull |= cleanupDecideWhetherDiskFull(); - - auto msg = fmt("builder for '%s' %s", - Magenta(store.printStorePath(drvPath)), - statusToString(status)); - - miscMethods.appendLogTailErrorMsg(msg); - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - auto builtOutputs = registerOutputs(); - - StorePathSet outputPaths; - for (auto & [_, output] : builtOutputs) - outputPaths.insert(output.outPath); - runPostBuildHook( - store, - *logger, - drvPath, - outputPaths - ); - - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); - - return std::move(builtOutputs); - - } catch (BuildError & e) { - assert(derivationType); - BuildResult::Status st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - !derivationType->isSandboxed() || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - - return std::pair{std::move(st), std::move(e)}; - } -} - - -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - std::filesystem::rename(src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - -extern void replaceValidPath(const Path & storePath, const Path & tmpPath); - - -bool DerivationBuilder::cleanupDecideWhetherDiskFull() -{ - bool diskFull = false; - - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - { - auto & localStore = getLocalStore(); - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - } -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = store.toRealPath(status.known->path); - if (pathExists(chrootRootDir + p)) - std::filesystem::rename((chrootRootDir + p), p); - } - - return diskFull; -} - - -#ifdef __linux__ -static void doBind(const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - - auto bindMount = [&]() { - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - auto maybeSt = maybeLstat(source); - if (!maybeSt) { - if (optional) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - auto st = *maybeSt; - - if (S_ISDIR(st.st_mode)) { - createDirs(target); - bindMount(); - } else if (S_ISLNK(st.st_mode)) { - // Symlinks can (apparently) not be bind-mounted, so just copy it - createDirs(dirOf(target)); - copyFile( - std::filesystem::path(source), - std::filesystem::path(target), false); - } else { - createDirs(dirOf(target)); - writeFile(target, ""); - bindMount(); - } -}; -#endif - -/** - * Rethrow the current exception as a subclass of `Error`. - */ -static void rethrowExceptionAsError() -{ - try { - throw; - } catch (Error &) { - throw; - } catch (std::exception & e) { - throw Error(e.what()); - } catch (...) { - throw Error("unknown exception"); - } -} - -/** - * Send the current exception to the parent in the format expected by - * `DerivationBuilder::processSandboxSetupMessages()`. - */ -static void handleChildException(bool sendException) -{ - try { - rethrowExceptionAsError(); - } catch (Error & e) { - if (sendException) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - } else - std::cerr << e.msg(); - } -} - -void DerivationBuilder::startBuilder() -{ - if ((buildUser && buildUser->getUIDCount() != 1) - #ifdef __linux__ - || settings.useCgroups - #endif - ) - { - #ifdef __linux__ - experimentalFeatureSettings.require(Xp::Cgroups); - - /* If we're running from the daemon, then this will return the - root cgroup of the service. Otherwise, it will return the - current cgroup. */ - auto rootCgroup = getRootCgroup(); - auto cgroupFS = getCgroupFS(); - if (!cgroupFS) - throw Error("cannot determine the cgroups file system"); - auto rootCgroupPath = canonPath(*cgroupFS + "/" + rootCgroup); - if (!pathExists(rootCgroupPath)) - throw Error("expected cgroup directory '%s'", rootCgroupPath); - - static std::atomic counter{0}; - - cgroup = buildUser - ? fmt("%s/nix-build-uid-%d", rootCgroupPath, buildUser->getUID()) - : fmt("%s/nix-build-pid-%d-%d", rootCgroupPath, getpid(), counter++); - - debug("using cgroup '%s'", *cgroup); - - /* When using a build user, record the cgroup we used for that - user so that if we got interrupted previously, we can kill - any left-over cgroup first. */ - if (buildUser) { - auto cgroupsDir = settings.nixStateDir + "/cgroups"; - createDirs(cgroupsDir); - - auto cgroupFile = fmt("%s/%d", cgroupsDir, buildUser->getUID()); - - if (pathExists(cgroupFile)) { - auto prevCgroup = readFile(cgroupFile); - destroyCgroup(prevCgroup); - } - - writeFile(cgroupFile, *cgroup); - } - - #else - throw Error("cgroups are not supported on this platform"); - #endif - } - - /* Make sure that no other processes are executing under the - sandbox uids. This must be done before any chownToBuilder() - calls. */ - killSandbox(false); - - /* Right platform? */ - if (!drvOptions->canBuildLocally(store, *drv)) { - // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - we should tell them to run the command to install Darwin 2 - if (drv->platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") { - throw Error("run `/usr/sbin/softwareupdate --install-rosetta` to enable your %s to run programs for %s", settings.thisSystem, drv->platform); - } else { - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)), - store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", store.systemFeatures)); - } - } - - /* Create a temporary directory where the build will take - place. */ - topTmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); -#ifdef __APPLE__ - if (false) { -#else - if (useChroot) { -#endif - /* If sandboxing is enabled, put the actual TMPDIR underneath - an inaccessible root-owned directory, to prevent outside - access. - - On macOS, we don't use an actual chroot, so this isn't - possible. Any mitigation along these lines would have to be - done directly in the sandbox profile. */ - tmpDir = topTmpDir + "/build"; - createDir(tmpDir, 0700); - } else { - tmpDir = topTmpDir; - } - chownToBuilder(tmpDir); - - for (auto & [outputName, status] : initialOutputs) { - /* Set scratch path we'll actually use during the build. - - If we're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - auto scratchPath = - !status.known - ? makeFallbackPath(outputName) - : !needsHashRewrite() - /* Can always use original path in sandbox */ - ? status.known->path - : !status.known->isPresent() - /* If path doesn't yet exist can just use it */ - ? status.known->path - : buildMode != bmRepair && !status.known->isValid() - /* If we aren't repairing we'll delete a corrupted path, so we - can use original path */ - ? status.known->path - : /* If we are repairing or the path is totally valid, we'll need - to use a temporary path */ - makeFallbackPath(status.known->path); - scratchOutputs.insert_or_assign(outputName, scratchPath); - - /* Substitute output placeholders with the scratch output paths. - We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = store.printStorePath(scratchPath); - - /* Additional tasks if we know the final path a priori. */ - if (!status.known) continue; - auto fixedFinalPath = status.known->path; - - /* Additional tasks if the final and scratch are both known and - differ. */ - if (fixedFinalPath == scratchPath) continue; - - /* Ensure scratch path is ours to use. */ - deletePath(store.printStorePath(scratchPath)); - - /* Rewrite and unrewrite paths */ - { - std::string h1 { fixedFinalPath.hashPart() }; - std::string h2 { scratchPath.hashPart() }; - inputRewrites[h1] = h2; - } - - redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); - } - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv) { - for (auto & [fileName, ss] : drvOptions->exportReferencesGraph) { - StorePathSet storePathSet; - for (auto & storePathS : ss) { - if (!store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - storePathSet.insert(store.toStorePath(storePathS).first); - } - /* Write closure info to . */ - writeFile(tmpDir + "/" + fileName, - store.makeValidityRegistration( - store.exportReferences(storePathSet, inputPaths), false, false)); - } - } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - pathsInChroot.clear(); - - for (auto i : settings.sandboxPaths.get()) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == std::string::npos) - pathsInChroot[i] = {i, optional}; - else - pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; - } - if (hasPrefix(store.storeDir, tmpDirInSandbox)) - { - throw Error("`sandbox-build-dir` must not contain the storeDir"); - } - pathsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - StorePathSet closure; - for (auto & i : pathsInChroot) - try { - if (store.isInStore(i.second.source)) - store.computeFSClosure(store.toStorePath(i.second.source).first, closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - e.addTrace({}, "while processing 'sandbox-paths'"); - throw; - } - for (auto & i : closure) { - auto p = store.printStorePath(i); - pathsInChroot.insert_or_assign(p, p); - } - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = drvOptions->impureHostDeps; - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (isDirOrInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - store.printStorePath(drvPath), i); - - /* Allow files in drvOptions->impureHostDeps to be missing; e.g. - macOS 11+ has no /usr/lib/libSystem*.dylib */ - pathsInChroot[i] = {i, true}; - } - -#ifdef __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - so that the build outputs can be moved efficiently from the - chroot to their final location. */ - auto chrootParentDir = store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootParentDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootParentDir); - - printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootParentDir); - - if (mkdir(chrootParentDir.c_str(), 0700) == -1) - throw SysError("cannot create '%s'", chrootRootDir); - - chrootRootDir = chrootParentDir + "/root"; - - if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUIDCount() != 1 ? buildUser->getUID() : 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - if (drvOptions->useUidRange(*drv)) - chownToBuilder(chrootRootDir + "/etc"); - - if (drvOptions->useUidRange(*drv) && (!buildUser || buildUser->getUIDCount() < 65536)) - throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", sandboxGid())); - - /* Create /etc/hosts with localhost entry. */ - if (derivationType->isSandboxed()) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); - - for (auto & i : inputPaths) { - auto p = store.printStorePath(i); - Path r = store.toRealPath(p); - pathsInChroot.insert_or_assign(p, r); - } - - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.sandbox-paths - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputsAndOptPaths(store)) { - /* If the name isn't known a priori (i.e. floating - content-addressing derivation), the temporary location we use - should be fresh. Freshness means it is impossible that the path - is already in the sandbox, so we don't need to worry about - removing it. */ - if (i.second.second) - pathsInChroot.erase(store.printStorePath(*i.second.second)); - } - - if (cgroup) { - if (mkdir(cgroup->c_str(), 0755) != 0) - throw SysError("creating cgroup '%s'", *cgroup); - chownToBuilder(*cgroup); - chownToBuilder(*cgroup + "/cgroup.procs"); - chownToBuilder(*cgroup + "/cgroup.threads"); - //chownToBuilder(*cgroup + "/cgroup.subtree_control"); - } - -#else - if (drvOptions->useUidRange(*drv)) - throw Error("feature 'uid-range' is not supported on this platform"); - #ifdef __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ - #else - throw Error("sandboxing builds is not supported on this platform"); - #endif -#endif - } else { - if (drvOptions->useUidRange(*drv)) - throw Error("feature 'uid-range' is only supported in sandboxed builds"); - } - - if (needsHashRewrite() && pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); - auto args = useChroot ? Strings({store.printStorePath(drvPath), chrootRootDir}) : - Strings({ store.printStorePath(drvPath) }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != std::string::npos; - nlPos = lines.find('\n', lastPos)) - { - auto line = lines.substr(lastPos, nlPos - lastPos); - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error("unknown pre-build hook command '%1%'", line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == std::string::npos) - pathsInChroot[line] = line; - else - pathsInChroot[line.substr(0, p)] = line.substr(p + 1); - } - } - } - } - - /* Fire up a Nix daemon to process recursive Nix calls from the - builder. */ - if (drvOptions->getRequiredSystemFeatures(*drv).count("recursive-nix")) - startDaemon(); - - /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args)); - for (auto & i : drv->env) - printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); - - /* Create the log file. */ - [[maybe_unused]] Path logFile = miscMethods.openLogFile(); - - /* Create a pseudoterminal to get the output of the builder. */ - builderOut = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut) - throw SysError("opening pseudoterminal master"); - - // FIXME: not thread-safe, use ptsname_r - std::string slaveName = ptsname(builderOut.get()); - - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); - - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } -#ifdef __APPLE__ - else { - if (grantpt(builderOut.get())) - throw SysError("granting access to pseudoterminal slave"); - } -#endif - - if (unlockpt(builderOut.get())) - throw SysError("unlocking pseudoterminal"); - - /* Open the slave side of the pseudoterminal and use it as stderr. */ - auto openSlave = [&]() - { - AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - if (dup2(builderOut.get(), STDERR_FILENO) == -1) - throw SysError("cannot pipe standard error into log file"); - }; - - buildResult.startTime = time(0); - - /* Fork a child to build the package. */ - -#ifdef __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - userNamespaceSync.create(); - - usingUserNamespace = userNamespacesSupported(); - - Pipe sendPid; - sendPid.create(); - - Pid helper = startProcess([&]() { - sendPid.readSide.close(); - - /* We need to open the slave early, before - CLONE_NEWUSER. Otherwise we get EPERM when running as - root. */ - openSlave(); - - try { - /* Drop additional groups here because we can't do it - after we've created the new user namespace. */ - if (setgroups(0, 0) == -1) { - if (errno != EPERM) - throw SysError("setgroups failed"); - if (settings.requireDropSupplementaryGroups) - throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); - } - - ProcessOptions options; - options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (derivationType->isSandboxed()) - options.cloneFlags |= CLONE_NEWNET; - if (usingUserNamespace) - options.cloneFlags |= CLONE_NEWUSER; - - pid_t child = startProcess([&]() { runChild(); }, options); - - writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); - _exit(0); - } catch (...) { - handleChildException(true); - _exit(1); - } - }); - - sendPid.writeSide.close(); - - if (helper.wait() != 0) { - processSandboxSetupMessages(); - // Only reached if the child process didn't send an exception. - throw Error("unable to start build process"); - } - - userNamespaceSync.readSide = -1; - - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { - userNamespaceSync.writeSide = -1; - }); - - auto ss = tokenizeString>(readLine(sendPid.readSide.get())); - assert(ss.size() == 1); - pid = string2Int(ss[0]).value(); - - if (usingUserNamespace) { - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1; - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d %d", sandboxUid(), hostUid, nrIds)); - - if (!buildUser || buildUser->getUIDCount() == 1) - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d %d", sandboxGid(), hostGid, nrIds)); - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - } - - /* Now that we now the sandbox uid, we can write - /etc/passwd. */ - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - - /* Save the mount- and user namespace of the child. We have to do this - *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxMountNamespace.get() == -1) - throw SysError("getting sandbox mount namespace"); - - if (usingUserNamespace) { - sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxUserNamespace.get() == -1) - throw SysError("getting sandbox user namespace"); - } - - /* Move the child into its own cgroup. */ - if (cgroup) - writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); - - /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - - } else -#endif - { - pid = startProcess([&]() { - openSlave(); - runChild(); - }); - } - - /* parent */ - pid.setSeparatePG(true); - miscMethods.childStarted(); - - processSandboxSetupMessages(); -} - - -void DerivationBuilder::processSandboxSetupMessages() -{ - std::vector msgs; - while (true) { - std::string msg = [&]() { - try { - return readLine(builderOut.get()); - } catch (Error & e) { - auto status = pid.wait(); - e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", - store.printStorePath(drvPath), - statusToString(status), - concatStringsSep("|", msgs)); - throw; - } - }(); - if (msg.substr(0, 1) == "\2") break; - if (msg.substr(0, 1) == "\1") { - FdSource source(builderOut.get()); - auto ex = readError(source); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: " + msg); - msgs.push_back(std::move(msg)); - } -} - - -void DerivationBuilder::initTmpDir() -{ - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#ifdef __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif - - /* In non-structured mode, set all bindings either directory in the - environment or via a file, as specified by - `DerivationOptions::passAsFile`. */ - if (!parsedDrv) { - for (auto & i : drv->env) { - if (drvOptions->passAsFile.find(i.first) == drvOptions->passAsFile.end()) { - env[i.first] = i.second; - } else { - auto hash = hashString(HashAlgorithm::SHA256, i.first); - std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; -} - - -void DerivationBuilder::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (derivationType->isFixed()) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (!derivationType->isSandboxed()) { - auto & impureEnv = settings.impureEnv.get(); - if (!impureEnv.empty()) - experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv); - - for (auto & i : drvOptions->impureEnvVars){ - auto envVar = impureEnv.find(i); - if (envVar != impureEnv.end()) { - env[i] = envVar->second; - } else { - env[i] = getEnv(i).value_or(""); - } - } - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; -} - - -void DerivationBuilder::writeStructuredAttrs() -{ - if (parsedDrv) { - auto json = parsedDrv->prepareStructuredAttrs( - store, - *drvOptions, - inputPaths, - drv->outputs); - nlohmann::json rewritten; - for (auto & [i, v] : json["outputs"].get()) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - rewritten[i] = rewriteStrings((std::string) v, inputRewrites); - } - - json["outputs"] = rewritten; - - auto jsonSh = StructuredAttrs::writeShell(json); - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); - env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh"; - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json"; - } -} - - -void DerivationBuilder::startDaemon() -{ - experimentalFeatureSettings.require(Xp::RecursiveNix); - - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = store.storeDir; - if (auto & optRoot = getLocalStore().rootDir.get()) - params["root"] = *optRoot; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = makeRestrictedStore(params, - ref(std::dynamic_pointer_cast(this->store.shared_from_this())), - *this); - - addedPaths.clear(); - - auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; - - daemonSocket = createUnixDomainSocket(socketPath, 0600); - - chownToBuilder(socketPath); - - daemonThread = std::thread([this, store]() { - - while (true) { - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - if (!remote) { - if (errno == EINTR || errno == EAGAIN) continue; - if (errno == EINVAL || errno == ECONNABORTED) break; - throw SysError("accepting connection"); - } - - unix::closeOnExec(remote.get()); - - debug("received daemon connection"); - - auto workerThread = std::thread([store, remote{std::move(remote)}]() { - try { - daemon::processConnection( - store, - FdSource(remote.get()), - FdSink(remote.get()), - NotTrusted, daemon::Recursive); - debug("terminated daemon connection"); - } catch (const Interrupted &) { - debug("interrupted daemon connection"); - } catch (SystemError &) { - ignoreExceptionExceptInterrupt(); - } - }); - - daemonWorkerThreads.push_back(std::move(workerThread)); - } - - debug("daemon shutting down"); - }); -} - - -void DerivationBuilder::stopDaemon() -{ - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) { - // According to the POSIX standard, the 'shutdown' function should - // return an ENOTCONN error when attempting to shut down a socket that - // hasn't been connected yet. This situation occurs when the 'accept' - // function is called on a socket without any accepted connections, - // leaving the socket unconnected. While Linux doesn't seem to produce - // an error for sockets that have only been accepted, more - // POSIX-compliant operating systems like OpenBSD, macOS, and others do - // return the ENOTCONN error. Therefore, we handle this error here to - // avoid raising an exception for compliant behaviour. - if (errno == ENOTCONN) { - daemonSocket.close(); - } else { - throw SysError("shutting down daemon socket"); - } - } - - if (daemonThread.joinable()) - daemonThread.join(); - - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) - thread.join(); - daemonWorkerThreads.clear(); - - // release the socket. - daemonSocket.close(); -} - - -void DerivationBuilder::addDependency(const StorePath & path) -{ - if (isAllowed(path)) return; - - addedPaths.insert(path); - - /* If we're doing a sandbox build, then we have to make the path - appear in the sandbox. */ - if (useChroot) { - - debug("materialising '%s' in the sandbox", store.printStorePath(path)); - - #ifdef __linux__ - - Path source = store.Store::toRealPath(path); - Path target = chrootRootDir + store.printStorePath(path); - - if (pathExists(target)) { - // There is a similar debug message in doBind, so only run it in this block to not have double messages. - debug("bind-mounting %s -> %s", target, source); - throw Error("store path '%s' already exists in the sandbox", store.printStorePath(path)); - } - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) - throw SysError("entering sandbox user namespace"); - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); - - doBind(source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", store.printStorePath(path)); - - #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - store.printStorePath(path)); - #endif - - } -} - -void DerivationBuilder::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); -} - - -void setupSeccomp() -{ -#ifdef __linux__ - if (!settings.filterSyscalls) return; -#if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - constexpr std::string_view nativeSystem = NIX_LOCAL_SYSTEM; - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - if (nativeSystem == "mips64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_MIPS) != 0) - printError("unable to add mips seccomp architecture"); - - if (nativeSystem == "mips64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_MIPS64N32) != 0) - printError("unable to add mips64-*abin32 seccomp architecture"); - - if (nativeSystem == "mips64el-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL) != 0) - printError("unable to add mipsel seccomp architecture"); - - if (nativeSystem == "mips64el-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL64N32) != 0) - printError("unable to add mips64el-*abin32 seccomp architecture"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), NIX_SYSCALL_FCHMODAT2, 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from using EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); -#else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); -#endif -#endif -} - - -void DerivationBuilder::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - bool sendException = true; - - try { /* child */ - - commonChildInit(); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } - - bool setUser = true; - - /* Make the contents of netrc and the CA certificate bundle - available to builtin:fetchurl (which may run under a - different uid and/or in a sandbox). */ - std::string netrcData; - std::string caFileData; - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") { - try { - netrcData = readFile(settings.netrcFile); - } catch (SystemError &) { } - - try { - caFileData = readFile(settings.caFile); - } catch (SystemError &) { } - } - -#ifdef __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; - - if (derivationType->isSandboxed()) { - - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) - throw SysError("unable to make '/' private"); - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + store.storeDir; - - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); - - if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (pathsInChroot.find("/dev") == pathsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (!derivationType->isSandboxed()) { - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - /* N.B. it is realistic that these paths might not exist. It - happens when testing Nix building fixed-output derivations - within a pure derivation. */ - for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" }) - if (pathExists(path)) - ss.push_back(path); - - if (settings.caFile != "" && pathExists(settings.caFile)) { - Path caFile = settings.caFile; - pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true); - } - } - - for (auto & i : ss) { - // For backwards-compatibiliy, resolve all the symlinks in the - // chroot paths - auto canonicalPath = canonPath(i, true); - pathsInChroot.emplace(i, canonicalPath); - } - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - for (auto & i : pathsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - - #if HAVE_EMBEDDED_SANDBOX_SHELL - if (i.second.source == "__embedded_sandbox_shell__") { - static unsigned char sh[] = { - #include "embedded-sandbox-shell.gen.hh" - }; - auto dst = chrootRootDir + i.first; - createDirs(dirOf(dst)); - writeFile(dst, std::string_view((const char *) sh, sizeof(sh))); - chmod_(dst, 0555); - } else - #endif - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } - - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount sysfs on /sys. */ - if (buildUser && buildUser->getUIDCount() != 1) { - createDirs(chrootRootDir + "/sys"); - if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) - throw SysError("mounting /sys"); - } - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !pathsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Make /etc unwritable */ - if (!drvOptions->useUidRange(*drv)) - chmod_(chrootRootDir + "/etc", 0555); - - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("unsharing mount namespace"); - - /* Unshare the cgroup namespace. This means - /proc/self/cgroup will show the child's cgroup as '/' - rather than whatever it is in the parent. */ - if (cgroup && unshare(CLONE_NEWCGROUP) == -1) - throw SysError("unsharing cgroup namespace"); - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); - - if (mkdir("real-root", 0500) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); - - if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); - - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); - - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); - - setUser = false; - } -#endif - - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); - - /* Close all other file descriptors. */ - unix::closeExtraFDs(); - -#ifdef __linux__ - linux::setPersonality(drv->platform); -#endif - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - auto gids = buildUser->getSupplementaryGIDs(); - if (setgroups(gids.size(), gids.data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); - } - -#ifdef __APPLE__ - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : pathsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = store.printStorePath(i); - pathsInChroot[p] = p; - } - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += - #include "sandbox-defaults.sb" - ; - - if (!derivationType->isSandboxed()) - sandboxProfile += - #include "sandbox-network.sb" - ; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - - // We create multiple allow lists, to avoid exceeding a limit in the darwin sandbox interpreter. - // See https://github.com/NixOS/nix/issues/4119 - // We split our allow groups approximately at half the actual limit, 1 << 16 - const size_t breakpoint = sandboxProfile.length() + (1 << 14); - for (auto & i : pathsInChroot) { - - if (sandboxProfile.length() >= breakpoint) { - debug("Sandbox break: %d %d", sandboxProfile.length(), breakpoint); - sandboxProfile += ")\n(allow file-read* file-write* process-exec\n"; - } - - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - std::string path = i.first; - auto optSt = maybeLstat(path.c_str()); - if (!optSt) { - if (i.second.optional) - continue; - throw SysError("getting attributes of required path '%s", path); - } - if (S_ISDIR(optSt->st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += drvOptions->additionalSandboxProfile; - } else - sandboxProfile += - #include "sandbox-minimal.sb" - ; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */ - Path globalTmpDir = canonPath(defaultTempDir(), true); - - /* They don't like trailing slashes on subpath directives */ - while (!globalTmpDir.empty() && globalTmpDir.back() == '/') - globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - Strings sandboxArgs; - sandboxArgs.push_back("_GLOBAL_TMP_DIR"); - sandboxArgs.push_back(globalTmpDir); - if (drvOptions->allowLocalNetworking) { - sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); - sandboxArgs.push_back("1"); - } - char * sandbox_errbuf = nullptr; - if (sandbox_init_with_parameters(sandboxProfile.c_str(), 0, stringsToCharPtrs(sandboxArgs).data(), &sandbox_errbuf)) { - writeFull(STDERR_FILENO, fmt("failed to configure sandbox: %s\n", sandbox_errbuf ? sandbox_errbuf : "(null)")); - _exit(1); - } - } -#endif - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, std::string("\2\n")); - - sendException = false; - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(getStandardError()); - - std::map outputs; - for (auto & e : drv->outputs) - outputs.insert_or_assign(e.first, - store.printStorePath(scratchOutputs.at(e.first))); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(*drv, outputs, netrcData, caFileData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(*drv, outputs); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(*drv, outputs); - else - throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + std::string("\n")); - _exit(1); - } - } - - // Now builder is not builtin - - Strings args; - args.push_back(std::string(baseNameOf(drv->builder))); - - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - -#ifdef __APPLE__ - posix_spawnattr_t attrp; - - if (posix_spawnattr_init(&attrp)) - throw SysError("failed to initialize builder"); - - if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) - throw SysError("failed to initialize builder"); - - if (drv->platform == "aarch64-darwin") { - // Unset kern.curproc_arch_affinity so we can escape Rosetta - int affinity = 0; - sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); - - cpu_type_t cpu = CPU_TYPE_ARM64; - posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); - } else if (drv->platform == "x86_64-darwin") { - cpu_type_t cpu = CPU_TYPE_X86_64; - posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); - } - - posix_spawn(NULL, drv->builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); -#else - execve(drv->builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); -#endif - - throw SysError("executing '%1%'", drv->builder); - - } catch (...) { - handleChildException(sendException); - _exit(1); - } -} - - -SingleDrvOutputs DerivationBuilder::registerOutputs() -{ - std::map infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - - std::exception_ptr delayedException; - - /* The paths that can be referenced are the input closures, the - output paths, and any paths that have been built via recursive - Nix calls. */ - StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) referenceablePaths.insert(p); - - /* FIXME `needsHashRewrite` should probably be removed and we get to the - real reason why we aren't using the chroot dir */ - auto toRealPathChroot = [&](const Path & p) -> Path { - return useChroot && !needsHashRewrite() - ? chrootRootDir + p - : store.toRealPath(p); - }; - - /* Check whether the output paths were created, and make all - output paths read-only. Then get the references of each output (that we - might need to register), so we can topologically sort them. For the ones - that are most definitely already installed, we just store their final - name so we can also use it in rewrites. */ - StringSet outputsToSort; - struct AlreadyRegistered { StorePath path; }; - struct PerhapsNeedToRegister { StorePathSet refs; }; - std::map> outputReferencesIfUnregistered; - std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { - auto scratchOutput = get(scratchOutputs, outputName); - if (!scratchOutput) - throw BuildError( - "builder for '%s' has no scratch output for '%s'", - store.printStorePath(drvPath), outputName); - auto actualPath = toRealPathChroot(store.printStorePath(*scratchOutput)); - - outputsToSort.insert(outputName); - - /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto initialOutput = get(initialOutputs, outputName); - if (!initialOutput) - throw BuildError( - "builder for '%s' has no initial output for '%s'", - store.printStorePath(drvPath), outputName); - auto & initialInfo = *initialOutput; - - /* Don't register if already valid, and not checking */ - initialInfo.wanted = buildMode == bmCheck - || !(initialInfo.known && initialInfo.known->isValid()); - if (!initialInfo.wanted) { - outputReferencesIfUnregistered.insert_or_assign( - outputName, - AlreadyRegistered { .path = initialInfo.known->path }); - continue; - } - - auto optSt = maybeLstat(actualPath.c_str()); - if (!optSt) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - store.printStorePath(drvPath), outputName, actualPath); - struct stat & st = *optSt; - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError( - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, outputName); -#endif - - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData( - actualPath, - buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, - inodesSeen); - - bool discardReferences = false; - if (auto udr = get(drvOptions->unsafeDiscardReferences, outputName)) { - discardReferences = *udr; - } - - StorePathSet references; - if (discardReferences) - debug("discarding references of output '%s'", outputName); - else { - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - references = scanForReferences(blank, actualPath, referenceablePaths); - } - - outputReferencesIfUnregistered.insert_or_assign( - outputName, - PerhapsNeedToRegister { .refs = references }); - outputStats.insert_or_assign(outputName, std::move(st)); - } - - auto sortedOutputNames = topoSort(outputsToSort, - {[&](const std::string & name) { - auto orifu = get(outputReferencesIfUnregistered, name); - if (!orifu) - throw BuildError( - "no output reference for '%s' in build of '%s'", - name, store.printStorePath(drvPath)); - return std::visit(overloaded { - /* Since we'll use the already installed versions of these, we - can treat them as leaves and ignore any references they - have. */ - [&](const AlreadyRegistered &) { return StringSet {}; }, - [&](const PerhapsNeedToRegister & refs) { - StringSet referencedOutputs; - /* FIXME build inverted map up front so no quadratic waste here */ - for (auto & r : refs.refs) - for (auto & [o, p] : scratchOutputs) - if (r == p) - referencedOutputs.insert(o); - return referencedOutputs; - }, - }, *orifu); - }}, - {[&](const std::string & path, const std::string & parent) { - // TODO with more -vvvv also show the temporary paths for manual inspection. - return BuildError( - "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - store.printStorePath(drvPath), path, parent); - }}); - - std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); - - OutputPathMap finalOutputs; - - for (auto & outputName : sortedOutputNames) { - auto output = get(drv->outputs, outputName); - auto scratchPath = get(scratchOutputs, outputName); - assert(output && scratchPath); - auto actualPath = toRealPathChroot(store.printStorePath(*scratchPath)); - - auto finish = [&](StorePath finalStorePath) { - /* Store the final path */ - finalOutputs.insert_or_assign(outputName, finalStorePath); - /* The rewrite rule will be used in downstream outputs that refer to - use. This is why the topological sort is essential to do first - before this for loop. */ - if (*scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath->hashPart() }] = std::string { finalStorePath.hashPart() }; - }; - - auto orifu = get(outputReferencesIfUnregistered, outputName); - assert(orifu); - - std::optional referencesOpt = std::visit(overloaded { - [&](const AlreadyRegistered & skippedFinalPath) -> std::optional { - finish(skippedFinalPath.path); - return std::nullopt; - }, - [&](const PerhapsNeedToRegister & r) -> std::optional { - return r.refs; - }, - }, *orifu); - - if (!referencesOpt) - continue; - auto references = *referencesOpt; - - auto rewriteOutput = [&](const StringMap & rewrites) { - /* Apply hash rewriting if necessary. */ - if (!rewrites.empty()) { - debug("rewriting hashes in '%1%'; cross fingers", actualPath); - - /* FIXME: Is this actually streaming? */ - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink(rewrites, nextSink); - dumpPath(actualPath, rsink); - rsink.flush(); - }); - Path tmpPath = actualPath + ".tmp"; - restorePath(tmpPath, *source); - deletePath(actualPath); - movePath(tmpPath, actualPath); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, {}, inodesSeen); - } - }; - - auto rewriteRefs = [&]() -> StoreReferences { - /* In the CA case, we need the rewritten refs to calculate the - final path, therefore we look for a *non-rewritten - self-reference, and use a bool rather try to solve the - computationally intractable fixed point. */ - StoreReferences res { - .self = false, - }; - for (auto & r : references) { - auto name = r.name(); - auto origHash = std::string { r.hashPart() }; - if (r == *scratchPath) { - res.self = true; - } else if (auto outputRewrite = get(outputRewrites, origHash)) { - std::string newRef = *outputRewrite; - newRef += '-'; - newRef += name; - res.others.insert(StorePath { newRef }); - } else { - res.others.insert(r); - } - } - return res; - }; - - auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo { - auto st = get(outputStats, outputName); - if (!st) - throw BuildError( - "output path %1% without valid stats info", - actualPath); - if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat) - { - /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) - throw BuildError( - "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", - actualPath); - } - rewriteOutput(outputRewrites); - /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath->hashPart() }; - auto got = [&]{ - auto fim = outputHash.method.getFileIngestionMethod(); - switch (fim) { - case FileIngestionMethod::Flat: - case FileIngestionMethod::NixArchive: - { - HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; - auto fim = outputHash.method.getFileIngestionMethod(); - dumpPath( - {getFSSourceAccessor(), CanonPath(actualPath)}, - caSink, - (FileSerialisationMethod) fim); - return caSink.finish().first; - } - case FileIngestionMethod::Git: { - return git::dumpHash( - outputHash.hashAlgo, - {getFSSourceAccessor(), CanonPath(actualPath)}).hash; - } - } - assert(false); - }(); - - ValidPathInfo newInfo0 { - store, - outputPathName(drv->name, outputName), - ContentAddressWithReferences::fromParts( - outputHash.method, - std::move(got), - rewriteRefs()), - Hash::dummy, - }; - if (*scratchPath != newInfo0.path) { - // If the path has some self-references, we need to rewrite - // them. - // (note that this doesn't invalidate the ca hash we calculated - // above because it's computed *modulo the self-references*, so - // it already takes this rewrite into account). - rewriteOutput( - StringMap{{oldHashPart, - std::string(newInfo0.path.hashPart())}}); - } - - { - HashResult narHashAndSize = hashPath( - {getFSSourceAccessor(), CanonPath(actualPath)}, - FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); - newInfo0.narHash = narHashAndSize.first; - newInfo0.narSize = narHashAndSize.second; - } - - assert(newInfo0.ca); - return newInfo0; - }; - - ValidPathInfo newInfo = std::visit(overloaded { - - [&](const DerivationOutput::InputAddressed & output) { - /* input-addressed case */ - auto requiredFinalPath = output.path; - /* Preemptively add rewrite rule for final hash, as that is - what the NAR hash will use rather than normalized-self references */ - if (*scratchPath != requiredFinalPath) - outputRewrites.insert_or_assign( - std::string { scratchPath->hashPart() }, - std::string { requiredFinalPath.hashPart() }); - rewriteOutput(outputRewrites); - HashResult narHashAndSize = hashPath( - {getFSSourceAccessor(), CanonPath(actualPath)}, - FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); - ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; - newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = std::move(refs.others); - if (refs.self) - newInfo0.references.insert(newInfo0.path); - return newInfo0; - }, - - [&](const DerivationOutput::CAFixed & dof) { - auto & wanted = dof.ca.hash; - - // Replace the output by a fresh copy of itself to make sure - // that there's no stale file descriptor pointing to it - Path tmpOutput = actualPath + ".tmp"; - copyFile( - std::filesystem::path(actualPath), - std::filesystem::path(tmpOutput), true); - - std::filesystem::rename(tmpOutput, actualPath); - - auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { - .method = dof.ca.method, - .hashAlgo = wanted.algo, - }); - - /* Check wanted hash */ - assert(newInfo0.ca); - auto & got = newInfo0.ca->hash; - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - miscMethods.noteHashMismatch(); - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", - store.printStorePath(drvPath), - wanted.to_string(HashFormat::SRI, true), - got.to_string(HashFormat::SRI, true))); - } - if (!newInfo0.references.empty()) { - auto numViolations = newInfo.references.size(); - delayedException = std::make_exception_ptr( - BuildError("fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", - store.printStorePath(drvPath), - numViolations, - store.printStorePath(*newInfo.references.begin()))); - } - - return newInfo0; - }, - - [&](const DerivationOutput::CAFloating & dof) { - return newInfoFromCA(dof); - }, - - [&](const DerivationOutput::Deferred &) -> ValidPathInfo { - // No derivation should reach that point without having been - // rewritten first - assert(false); - }, - - [&](const DerivationOutput::Impure & doi) { - return newInfoFromCA(DerivationOutput::CAFloating { - .method = doi.method, - .hashAlgo = doi.hashAlgo, - }); - }, - - }, output->raw); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, {}, inodesSeen); - - /* Calculate where we'll move the output files. In the checking case we - will leave leave them where they are, for now, rather than move to - their usual "final destination" */ - auto finalDestPath = store.printStorePath(newInfo.path); - - /* Lock final output path, if not already locked. This happens with - floating CA derivations and hash-mismatching fixed-output - derivations. */ - PathLocks dynamicOutputLock; - dynamicOutputLock.setDeletion(true); - auto optFixedPath = output->path(store, drv->name, outputName); - if (!optFixedPath || - store.printStorePath(*optFixedPath) != finalDestPath) - { - assert(newInfo.ca); - dynamicOutputLock.lockPaths({store.toRealPath(finalDestPath)}); - } - - /* Move files, if needed */ - if (store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(store.toRealPath(finalDestPath), actualPath); - actualPath = store.toRealPath(finalDestPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - } else { - auto destPath = store.toRealPath(finalDestPath); - deletePath(destPath); - movePath(actualPath, destPath); - actualPath = destPath; - } - } - - auto & localStore = getLocalStore(); - - if (buildMode == bmCheck) { - - if (!store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - miscMethods.noteCheckMismatch(); - if (settings.runDiffHook || settings.keepFailed) { - auto dst = store.toRealPath(finalDestPath + checkSuffix); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, store.printStorePath(drvPath), tmpDir); - - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - store.printStorePath(drvPath), store.toRealPath(finalDestPath), dst); - } else - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - store.printStorePath(drvPath), store.toRealPath(finalDestPath)); - } - - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - localStore.signPathInfo(oldInfo); - localStore.registerValidPaths({{oldInfo.path, oldInfo}}); - } - - continue; - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - if (references.count(i)) - debug("referenced input: '%1%'", store.printStorePath(i)); - else - debug("unreferenced input: '%1%'", store.printStorePath(i)); - } - - localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() - miscMethods.markContentsGood(newInfo.path); - - newInfo.deriver = drvPath; - newInfo.ultimate = true; - localStore.signPathInfo(newInfo); - - finish(newInfo.path); - - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - localStore.registerValidPaths({{newInfo.path, newInfo}}); - - infos.emplace(outputName, std::move(newInfo)); - } - - if (buildMode == bmCheck) { - /* In case of fixed-output derivations, if there are - mismatches on `--check` an error must be thrown as this is - also a source for non-determinism. */ - if (delayedException) - std::rethrow_exception(delayedException); - return miscMethods.assertPathValidity(); - } - - /* Apply output checks. */ - checkOutputs(infos); - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - auto & localStore = getLocalStore(); - - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.insert_or_assign(newInfo.path, newInfo); - } - localStore.registerValidPaths(infos2); - } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ - SingleDrvOutputs builtOutputs; - - for (auto & [outputName, newInfo] : infos) { - auto oldinfo = get(initialOutputs, outputName); - assert(oldinfo); - auto thisRealisation = Realisation { - .id = DrvOutput { - oldinfo->outputHash, - outputName - }, - .outPath = newInfo.path - }; - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) - && !drv->type().isImpure()) - { - store.signRealisation(thisRealisation); - store.registerDrvOutput(thisRealisation); - } - builtOutputs.emplace(outputName, thisRealisation); - } - - return builtOutputs; -} - - -void DerivationBuilder::checkOutputs(const std::map & outputs) -{ - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) - { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const DerivationOptions::OutputChecks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - store.printStorePath(info.path), info.narSize, *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - store.printStorePath(info.path), closureSize, *checks.maxClosureSize); - } - - auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive) - { - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : value) { - if (store.isStorePath(i)) - spec.insert(store.parseStorePath(i)); - else if (auto output = get(outputs, i)) - spec.insert(output->path); - else { - std::string outputsListing = concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; }); - throw BuildError("derivation '%s' output check for '%s' contains an illegal reference specifier '%s'," - " expected store path or output name (one of [%s])", - store.printStorePath(drvPath), outputName, i, outputsListing); - } - } - - auto used = recursive - ? getClosure(info.path).first - : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - std::string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += store.printStorePath(i); - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", - store.printStorePath(info.path), badPathsStr); - } - }; - - /* Mandatory check: absent whitelist, and present but empty - whitelist mean very different things. */ - if (auto & refs = checks.allowedReferences) { - checkRefs(*refs, true, false); - } - if (auto & refs = checks.allowedRequisites) { - checkRefs(*refs, true, true); - } - - /* Optimization: don't need to do anything when - disallowed and empty set. */ - if (!checks.disallowedReferences.empty()) { - checkRefs(checks.disallowedReferences, false, false); - } - if (!checks.disallowedRequisites.empty()) { - checkRefs(checks.disallowedRequisites, false, true); - } - }; - - std::visit(overloaded{ - [&](const DerivationOptions::OutputChecks & checks) { - applyChecks(checks); - }, - [&](const std::map & checksPerOutput) { - if (auto outputChecks = get(checksPerOutput, outputName)) - - applyChecks(*outputChecks); - }, - }, drvOptions->outputChecks); - } -} - - -void DerivationBuilder::deleteTmpDir(bool force) -{ - if (topTmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(topTmpDir.c_str(), 0755); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(topTmpDir); - topTmpDir = ""; - tmpDir = ""; - } -} - bool LocalDerivationGoal::isReadDesc(int fd) { return (hook && DerivationGoal::isReadDesc(fd)) || - (!hook && fd == builder.builderOut.get()); + (!hook && fd == builder->builderOut.get()); } - -StorePath DerivationBuilder::makeFallbackPath(OutputNameView outputName) -{ - // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path - // See doc/manual/source/protocols/store-path.md for details - // TODO: We may want to separate the responsibilities of constructing the path fingerprint and of actually doing the hashing - auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName); - return store.makeStorePath( - pathType, - // pass an all-zeroes hash - Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName)); -} - - -StorePath DerivationBuilder::makeFallbackPath(const StorePath & path) -{ - // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path - // See doc/manual/source/protocols/store-path.md for details - auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()); - return store.makeStorePath( - pathType, - // pass an all-zeroes hash - Hash(HashAlgorithm::SHA256), path.name()); -} - - } diff --git a/src/libstore/unix/include/nix/store/build/derivation-builder.hh b/src/libstore/unix/include/nix/store/build/derivation-builder.hh index 973eef26e..5fe528f65 100644 --- a/src/libstore/unix/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/unix/include/nix/store/build/derivation-builder.hh @@ -1,82 +1,14 @@ -#include "nix/store/build/local-derivation-goal.hh" -#include "nix/store/local-store.hh" +#pragma once +///@file + +#include "nix/store/build-result.hh" +#include "nix/store/derivation-options.hh" +#include "nix/store/build/derivation-building-misc.hh" +#include "nix/store/derivations.hh" +#include "nix/store/parsed-derivations.hh" #include "nix/util/processes.hh" -#include "nix/store/indirect-root-store.hh" -#include "nix/store/build/hook-instance.hh" -#include "nix/store/build/worker.hh" -#include "nix/store/builtins.hh" -#include "nix/store/builtins/buildenv.hh" -#include "nix/store/path-references.hh" -#include "nix/util/finally.hh" -#include "nix/util/util.hh" -#include "nix/util/archive.hh" -#include "nix/util/git.hh" -#include "nix/util/compression.hh" -#include "nix/store/daemon.hh" -#include "nix/util/topo-sort.hh" -#include "nix/util/callback.hh" -#include "nix/util/json-utils.hh" -#include "nix/util/current-process.hh" -#include "nix/store/build/child.hh" -#include "nix/util/unix-domain-socket.hh" -#include "nix/store/posix-fs-canonicalise.hh" -#include "nix/util/posix-source-accessor.hh" #include "nix/store/restricted-store.hh" -#include "nix/store/config.hh" - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "store-config-private.hh" - -#if HAVE_STATVFS -#include -#endif - -/* Includes required for chroot support. */ -#ifdef __linux__ -# include "linux/fchmodat2-compat.hh" -# include -# include -# include -# include -# include -# include -# include -# include -# include "nix/util/namespaces.hh" -# if HAVE_SECCOMP -# include -# endif -# define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) -# include "nix/util/cgroup.hh" -# include "nix/store/personality.hh" -#endif - -#ifdef __APPLE__ -#include -#include -#include - -/* This definition is undocumented but depended upon by all major browsers. */ -extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf); -#endif - -#include -#include -#include - -#include "nix/util/strings.hh" -#include "nix/util/signals.hh" - -#include "store-config-private.hh" +#include "nix/store/user-lock.hh" namespace nix { @@ -192,25 +124,8 @@ struct DerivationBuilderCallbacks * rather than incoming call edges that either should be removed, or * become (higher order) function parameters. */ -class DerivationBuilder : public RestrictionContext, DerivationBuilderParams +struct DerivationBuilder : RestrictionContext { - Store & store; - - DerivationBuilderCallbacks & miscMethods; - -public: - - DerivationBuilder( - Store & store, - DerivationBuilderCallbacks & miscMethods, - DerivationBuilderParams params) - : DerivationBuilderParams{std::move(params)} - , store{store} - , miscMethods{miscMethods} - { } - - LocalStore & getLocalStore(); - /** * User selected for running the builder. */ @@ -221,30 +136,8 @@ public: */ Pid pid; -private: - - /** - * The cgroup of the builder, if any. - */ - std::optional cgroup; - - /** - * The temporary directory used for the build. - */ - Path tmpDir; - - /** - * The top-level temporary directory. `tmpDir` is either equal to - * or a child of this directory. - */ - Path topTmpDir; - - /** - * The path of the temporary directory in the sandbox. - */ - Path tmpDirInSandbox; - -public: + DerivationBuilder() = default; + virtual ~DerivationBuilder() = default; /** * Master side of the pseudoterminal used for the builder's @@ -252,132 +145,6 @@ public: */ AutoCloseFD builderOut; -private: - - /** - * Pipe for synchronising updates to the builder namespaces. - */ - Pipe userNamespaceSync; - - /** - * The mount namespace and user namespace of the builder, used to add additional - * paths to the sandbox as a result of recursive Nix calls. - */ - AutoCloseFD sandboxMountNamespace; - AutoCloseFD sandboxUserNamespace; - - /** - * On Linux, whether we're doing the build in its own user - * namespace. - */ - bool usingUserNamespace = true; - - /** - * Whether we're currently doing a chroot build. - */ - bool useChroot = false; - - /** - * The root of the chroot environment. - */ - Path chrootRootDir; - - /** - * RAII object to delete the chroot directory. - */ - std::shared_ptr autoDelChroot; - - /** - * The sort of derivation we are building. - * - * Just a cached value, can be recomputed from `drv`. - */ - std::optional derivationType; - - /** - * Stuff we need to pass to initChild(). - */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map PathsInChroot; // maps target path to source path - PathsInChroot pathsInChroot; - - typedef map Environment; - Environment env; - - /** - * Hash rewriting. - */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /** - * The output paths used during the build. - * - * - Input-addressed derivations or fixed content-addressed outputs are - * sometimes built when some of their outputs already exist, and can not - * be hidden via sandboxing. We use temporary locations instead and - * rewrite after the build. Otherwise the regular predetermined paths are - * put here. - * - * - Floating content-addressing derivations do not know their final build - * output paths until the outputs are hashed, so random locations are - * used, and then renamed. The randomness helps guard against hidden - * self-references. - */ - OutputPathMap scratchOutputs; - - uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } - - const static Path homeDir; - - /** - * The recursive Nix daemon socket. - */ - AutoCloseFD daemonSocket; - - /** - * The daemon main thread. - */ - std::thread daemonThread; - - /** - * The daemon worker threads. - */ - std::vector daemonWorkerThreads; - - const StorePathSet & originalPaths() override - { - return inputPaths; - } - - bool isAllowed(const StorePath & path) override - { - return inputPaths.count(path) || addedPaths.count(path); - } - bool isAllowed(const DrvOutput & id) override - { - return addedDrvOutputs.count(id); - } - - bool isAllowed(const DerivedPath & req); - - friend struct RestrictedStore; - - /** - * Whether we need to perform hash rewriting if there are valid output paths. - */ - bool needsHashRewrite(); - -public: - /** * Set up build environment / sandbox, acquiring resources (e.g. * locks as needed). After this is run, the builder should be @@ -386,12 +153,12 @@ public: * @returns true if successful, false if we could not acquire a build * user. In that case, the caller must wait and then try again. */ - bool prepareBuild(); + virtual bool prepareBuild() = 0; /** * Start building a derivation. */ - void startBuilder(); + virtual void startBuilder() = 0; /** * Tear down build environment after the builder exits (either on @@ -402,2994 +169,29 @@ public: * more information. The second case indicates success, and * realisations for each output of the derivation are returned. */ - std::variant, SingleDrvOutputs> unprepareBuild(); - -private: - - /** - * Fill in the environment for the builder. - */ - void initEnv(); - - /** - * Process messages send by the sandbox initialization. - */ - void processSandboxSetupMessages(); - - /** - * Setup tmp dir location. - */ - void initTmpDir(); - - /** - * Write a JSON file containing the derivation attributes. - */ - void writeStructuredAttrs(); - - /** - * Start an in-process nix daemon thread for recursive-nix. - */ - void startDaemon(); - -public: + virtual std::variant, SingleDrvOutputs> unprepareBuild() = 0; /** * Stop the in-process nix daemon thread. * @see startDaemon */ - void stopDaemon(); - -private: - - void addDependency(const StorePath & path) override; - - /** - * Make a file owned by the builder. - */ - void chownToBuilder(const Path & path); - - /** - * Run the builder's process. - */ - void runChild(); - - /** - * Check that the derivation outputs all exist and register them - * as valid. - */ - SingleDrvOutputs registerOutputs(); - - /** - * Check that an output meets the requirements specified by the - * 'outputChecks' attribute (or the legacy - * '{allowed,disallowed}{References,Requisites}' attributes). - */ - void checkOutputs(const std::map & outputs); - -public: + virtual void stopDaemon() = 0; /** * Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); + virtual void deleteTmpDir(bool force) = 0; /** * Kill any processes running under the build user UID or in the * cgroup of the build. */ - void killSandbox(bool getStats); - -private: - - bool cleanupDecideWhetherDiskFull(); - - /** - * Create alternative path calculated from but distinct from the - * input, so we can avoid overwriting outputs (or other store paths) - * that already exist. - */ - StorePath makeFallbackPath(const StorePath & path); - - /** - * Make a path to another based on the output name along with the - * derivation hash. - * - * @todo Add option to randomize, so we can audit whether our - * rewrites caught everything - */ - StorePath makeFallbackPath(OutputNameView outputName); + virtual void killSandbox(bool getStats) = 0; }; -/** - * This hooks up `DerivationBuilder` to the scheduler / goal machinary. - * - * @todo Eventually, this shouldn't exist, because `DerivationGoal` can - * just choose to use `DerivationBuilder` or its remote-building - * equalivalent directly, at the "value level" rather than "class - * inheritance hierarchy" level. - */ -struct LocalDerivationGoal : DerivationGoal, DerivationBuilderCallbacks -{ - DerivationBuilder builder; - - LocalDerivationGoal(const StorePath & drvPath, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode) - : DerivationGoal{drvPath, wantedOutputs, worker, buildMode} - , builder{ - worker.store, - static_cast(*this), - DerivationBuilderParams { - DerivationGoal::drvPath, - DerivationGoal::buildMode, - DerivationGoal::buildResult, - DerivationGoal::drv, - DerivationGoal::parsedDrv, - DerivationGoal::drvOptions, - DerivationGoal::inputPaths, - DerivationGoal::initialOutputs, - }, - } - {} - - LocalDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal) - : DerivationGoal{drvPath, drv, wantedOutputs, worker, buildMode} - , builder{ - worker.store, - static_cast(*this), - DerivationBuilderParams { - DerivationGoal::drvPath, - DerivationGoal::buildMode, - DerivationGoal::buildResult, - DerivationGoal::drv, - DerivationGoal::parsedDrv, - DerivationGoal::drvOptions, - DerivationGoal::inputPaths, - DerivationGoal::initialOutputs, - }, - } - {} - - virtual ~LocalDerivationGoal() override; - - /** - * The additional states. - */ - Goal::Co tryLocalBuild() override; - - bool isReadDesc(int fd) override; - - /** - * Forcibly kill the child process, if any. - * - * Called by destructor, can't be overridden - */ - void killChild() override final; - - void childStarted() override; - void childTerminated() override; - - void noteHashMismatch(void) override; - void noteCheckMismatch(void) override; - - void markContentsGood(const StorePath &) override; - - // Fake overrides to instantiate identically-named virtual methods - - Path openLogFile() override { - return DerivationGoal::openLogFile(); - } - void closeLogFile() override { - DerivationGoal::closeLogFile(); - } - SingleDrvOutputs assertPathValidity() override { - return DerivationGoal::assertPathValidity(); - } - void appendLogTailErrorMsg(std::string & msg) override { - DerivationGoal::appendLogTailErrorMsg(msg); - } -}; - -std::shared_ptr makeLocalDerivationGoal( - const StorePath & drvPath, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode) -{ - return std::make_shared(drvPath, wantedOutputs, worker, buildMode); -} - -std::shared_ptr makeLocalDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode) -{ - return std::make_shared(drvPath, drv, wantedOutputs, worker, buildMode); -} - -void handleDiffHook( - uid_t uid, uid_t gid, - const Path & tryA, const Path & tryB, - const Path & drvPath, const Path & tmpDir) -{ - auto & diffHookOpt = settings.diffHook.get(); - if (diffHookOpt && settings.runDiffHook) { - auto & diffHook = *diffHookOpt; - try { - auto diffRes = runProgram(RunOptions { - .program = diffHook, - .lookupPath = true, - .args = {tryA, tryB, drvPath, tmpDir}, - .uid = uid, - .gid = gid, - .chdir = "/" - }); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, - "diff-hook program '%1%' %2%", - diffHook, - statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - // FIXME: wrap errors. - ei.msg = HintFmt("diff hook execution failed: %s", ei.msg.str()); - logError(ei); - } - } -} - -const Path DerivationBuilder::homeDir = "/homeless-shelter"; - - -LocalDerivationGoal::~LocalDerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { builder.deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } - try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } - try { builder.stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } -} - - -inline bool DerivationBuilder::needsHashRewrite() -{ -#ifdef __linux__ - return !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; -#endif -} - - -LocalStore & DerivationBuilder::getLocalStore() -{ - auto p = dynamic_cast(&store); - assert(p); - return *p; -} - - -void LocalDerivationGoal::killChild() -{ - if (builder.pid != -1) { - worker.childTerminated(this); - - /* If we're using a build user, then there is a tricky race - condition: if we kill the build user before the child has - done its setuid() to the build user uid, then it won't be - killed, and we'll potentially lock up in pid.wait(). So - also send a conventional kill to the child. */ - ::kill(-builder.pid, SIGKILL); /* ignore the result */ - - builder.killSandbox(true); - - builder.pid.wait(); - } - - DerivationGoal::killChild(); -} - - -void DerivationBuilder::killSandbox(bool getStats) -{ - if (cgroup) { - #ifdef __linux__ - auto stats = destroyCgroup(*cgroup); - if (getStats) { - buildResult.cpuUser = stats.cpuUser; - buildResult.cpuSystem = stats.cpuSystem; - } - #else - unreachable(); - #endif - } - - else if (buildUser) { - auto uid = buildUser->getUID(); - assert(uid != 0); - killUser(uid); - } -} - - -void LocalDerivationGoal::childStarted() -{ - worker.childStarted(shared_from_this(), {builder.builderOut.get()}, true, true); -} - -void LocalDerivationGoal::childTerminated() -{ - worker.childTerminated(this); -} - -void LocalDerivationGoal::noteHashMismatch() -{ - worker.hashMismatch = true; -} - - -void LocalDerivationGoal::noteCheckMismatch() -{ - worker.checkMismatch = true; -} - - -void LocalDerivationGoal::markContentsGood(const StorePath & path) -{ - worker.markContentsGood(path); -} - - -Goal::Co LocalDerivationGoal::tryLocalBuild() -{ - assert(!hook); - - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs) { - outputLocks.unlock(); - co_await waitForBuildSlot(); - co_return tryToBuild(); - } - - if (!builder.prepareBuild()) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); - co_await waitForAWhile(); - co_return tryLocalBuild(); - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - builder.startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - builder.buildUser.reset(); - worker.permanentFailure = true; - co_return done(BuildResult::InputRejected, {}, std::move(e)); - } - - started(); - co_await Suspend{}; - - trace("build done"); - - auto res = builder.unprepareBuild(); - // N.B. cannot use `std::visit` with co-routine return - if (auto * ste = std::get_if<0>(&res)) { - outputLocks.unlock(); - co_return done(std::move(ste->first), {}, std::move(ste->second)); - } else if (auto * builtOutputs = std::get_if<1>(&res)) { - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - co_return done(BuildResult::Built, std::move(*builtOutputs)); - } else { - unreachable(); - } -} - -bool DerivationBuilder::prepareBuild() -{ - /* Cache this */ - derivationType = drv->type(); - - /* Are we doing a chroot build? */ - { - if (settings.sandboxMode == smEnabled) { - if (drvOptions->noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", store.printStorePath(drvPath)); -#ifdef __APPLE__ - if (drvOptions->additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = derivationType->isSandboxed() && !drvOptions->noChroot; - } - - auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir.get()) { - #ifdef __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - - #ifdef __linux__ - if (useChroot) { - if (!mountAndPidNamespacesSupported()) { - if (!settings.sandboxFallback) - throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); - debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); - useChroot = false; - } - } - #endif - - if (useBuildUsers()) { - if (!buildUser) - buildUser = acquireUserLock(drvOptions->useUidRange(*drv) ? 65536 : 1, useChroot); - - if (!buildUser) { - return false; - } - } - - return true; -} - - -std::variant, SingleDrvOutputs> DerivationBuilder::unprepareBuild() -{ - Finally releaseBuildUser([&](){ - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - buildUser.reset(); - }); - - sandboxMountNamespace = -1; - sandboxUserNamespace = -1; - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = pid.kill(); - - debug("builder process for '%s' finished", store.printStorePath(drvPath)); - - buildResult.timesBuilt++; - buildResult.stopTime = time(0); - - /* So the child is gone now. */ - miscMethods.childTerminated(); - - /* Close the read side of the logger pipe. */ - builderOut.close(); - - /* Close the log file. */ - miscMethods.closeLogFile(); - - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - killSandbox(true); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); - - if (buildResult.cpuUser && buildResult.cpuSystem) { - debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs", - store.printStorePath(drvPath), - status, - ((double) buildResult.cpuUser->count()) / 1000000, - ((double) buildResult.cpuSystem->count()) / 1000000); - } - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - diskFull |= cleanupDecideWhetherDiskFull(); - - auto msg = fmt("builder for '%s' %s", - Magenta(store.printStorePath(drvPath)), - statusToString(status)); - - miscMethods.appendLogTailErrorMsg(msg); - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - auto builtOutputs = registerOutputs(); - - StorePathSet outputPaths; - for (auto & [_, output] : builtOutputs) - outputPaths.insert(output.outPath); - runPostBuildHook( - store, - *logger, - drvPath, - outputPaths - ); - - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); - - return std::move(builtOutputs); - - } catch (BuildError & e) { - assert(derivationType); - BuildResult::Status st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - !derivationType->isSandboxed() || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - - return std::pair{std::move(st), std::move(e)}; - } -} - - -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - std::filesystem::rename(src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - -extern void replaceValidPath(const Path & storePath, const Path & tmpPath); - - -bool DerivationBuilder::cleanupDecideWhetherDiskFull() -{ - bool diskFull = false; - - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - { - auto & localStore = getLocalStore(); - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - } -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = store.toRealPath(status.known->path); - if (pathExists(chrootRootDir + p)) - std::filesystem::rename((chrootRootDir + p), p); - } - - return diskFull; -} - - -#ifdef __linux__ -static void doBind(const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - - auto bindMount = [&]() { - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - auto maybeSt = maybeLstat(source); - if (!maybeSt) { - if (optional) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - auto st = *maybeSt; - - if (S_ISDIR(st.st_mode)) { - createDirs(target); - bindMount(); - } else if (S_ISLNK(st.st_mode)) { - // Symlinks can (apparently) not be bind-mounted, so just copy it - createDirs(dirOf(target)); - copyFile( - std::filesystem::path(source), - std::filesystem::path(target), false); - } else { - createDirs(dirOf(target)); - writeFile(target, ""); - bindMount(); - } -}; -#endif - -/** - * Rethrow the current exception as a subclass of `Error`. - */ -static void rethrowExceptionAsError() -{ - try { - throw; - } catch (Error &) { - throw; - } catch (std::exception & e) { - throw Error(e.what()); - } catch (...) { - throw Error("unknown exception"); - } -} - -/** - * Send the current exception to the parent in the format expected by - * `DerivationBuilder::processSandboxSetupMessages()`. - */ -static void handleChildException(bool sendException) -{ - try { - rethrowExceptionAsError(); - } catch (Error & e) { - if (sendException) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - } else - std::cerr << e.msg(); - } -} - -void DerivationBuilder::startBuilder() -{ - if ((buildUser && buildUser->getUIDCount() != 1) - #ifdef __linux__ - || settings.useCgroups - #endif - ) - { - #ifdef __linux__ - experimentalFeatureSettings.require(Xp::Cgroups); - - /* If we're running from the daemon, then this will return the - root cgroup of the service. Otherwise, it will return the - current cgroup. */ - auto rootCgroup = getRootCgroup(); - auto cgroupFS = getCgroupFS(); - if (!cgroupFS) - throw Error("cannot determine the cgroups file system"); - auto rootCgroupPath = canonPath(*cgroupFS + "/" + rootCgroup); - if (!pathExists(rootCgroupPath)) - throw Error("expected cgroup directory '%s'", rootCgroupPath); - - static std::atomic counter{0}; - - cgroup = buildUser - ? fmt("%s/nix-build-uid-%d", rootCgroupPath, buildUser->getUID()) - : fmt("%s/nix-build-pid-%d-%d", rootCgroupPath, getpid(), counter++); - - debug("using cgroup '%s'", *cgroup); - - /* When using a build user, record the cgroup we used for that - user so that if we got interrupted previously, we can kill - any left-over cgroup first. */ - if (buildUser) { - auto cgroupsDir = settings.nixStateDir + "/cgroups"; - createDirs(cgroupsDir); - - auto cgroupFile = fmt("%s/%d", cgroupsDir, buildUser->getUID()); - - if (pathExists(cgroupFile)) { - auto prevCgroup = readFile(cgroupFile); - destroyCgroup(prevCgroup); - } - - writeFile(cgroupFile, *cgroup); - } - - #else - throw Error("cgroups are not supported on this platform"); - #endif - } - - /* Make sure that no other processes are executing under the - sandbox uids. This must be done before any chownToBuilder() - calls. */ - killSandbox(false); - - /* Right platform? */ - if (!drvOptions->canBuildLocally(store, *drv)) { - // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - we should tell them to run the command to install Darwin 2 - if (drv->platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") { - throw Error("run `/usr/sbin/softwareupdate --install-rosetta` to enable your %s to run programs for %s", settings.thisSystem, drv->platform); - } else { - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)), - store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", store.systemFeatures)); - } - } - - /* Create a temporary directory where the build will take - place. */ - topTmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); -#ifdef __APPLE__ - if (false) { -#else - if (useChroot) { -#endif - /* If sandboxing is enabled, put the actual TMPDIR underneath - an inaccessible root-owned directory, to prevent outside - access. - - On macOS, we don't use an actual chroot, so this isn't - possible. Any mitigation along these lines would have to be - done directly in the sandbox profile. */ - tmpDir = topTmpDir + "/build"; - createDir(tmpDir, 0700); - } else { - tmpDir = topTmpDir; - } - chownToBuilder(tmpDir); - - for (auto & [outputName, status] : initialOutputs) { - /* Set scratch path we'll actually use during the build. - - If we're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - auto scratchPath = - !status.known - ? makeFallbackPath(outputName) - : !needsHashRewrite() - /* Can always use original path in sandbox */ - ? status.known->path - : !status.known->isPresent() - /* If path doesn't yet exist can just use it */ - ? status.known->path - : buildMode != bmRepair && !status.known->isValid() - /* If we aren't repairing we'll delete a corrupted path, so we - can use original path */ - ? status.known->path - : /* If we are repairing or the path is totally valid, we'll need - to use a temporary path */ - makeFallbackPath(status.known->path); - scratchOutputs.insert_or_assign(outputName, scratchPath); - - /* Substitute output placeholders with the scratch output paths. - We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = store.printStorePath(scratchPath); - - /* Additional tasks if we know the final path a priori. */ - if (!status.known) continue; - auto fixedFinalPath = status.known->path; - - /* Additional tasks if the final and scratch are both known and - differ. */ - if (fixedFinalPath == scratchPath) continue; - - /* Ensure scratch path is ours to use. */ - deletePath(store.printStorePath(scratchPath)); - - /* Rewrite and unrewrite paths */ - { - std::string h1 { fixedFinalPath.hashPart() }; - std::string h2 { scratchPath.hashPart() }; - inputRewrites[h1] = h2; - } - - redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); - } - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv) { - for (auto & [fileName, ss] : drvOptions->exportReferencesGraph) { - StorePathSet storePathSet; - for (auto & storePathS : ss) { - if (!store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - storePathSet.insert(store.toStorePath(storePathS).first); - } - /* Write closure info to . */ - writeFile(tmpDir + "/" + fileName, - store.makeValidityRegistration( - store.exportReferences(storePathSet, inputPaths), false, false)); - } - } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - pathsInChroot.clear(); - - for (auto i : settings.sandboxPaths.get()) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == std::string::npos) - pathsInChroot[i] = {i, optional}; - else - pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; - } - if (hasPrefix(store.storeDir, tmpDirInSandbox)) - { - throw Error("`sandbox-build-dir` must not contain the storeDir"); - } - pathsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - StorePathSet closure; - for (auto & i : pathsInChroot) - try { - if (store.isInStore(i.second.source)) - store.computeFSClosure(store.toStorePath(i.second.source).first, closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - e.addTrace({}, "while processing 'sandbox-paths'"); - throw; - } - for (auto & i : closure) { - auto p = store.printStorePath(i); - pathsInChroot.insert_or_assign(p, p); - } - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = drvOptions->impureHostDeps; - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (isDirOrInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - store.printStorePath(drvPath), i); - - /* Allow files in drvOptions->impureHostDeps to be missing; e.g. - macOS 11+ has no /usr/lib/libSystem*.dylib */ - pathsInChroot[i] = {i, true}; - } - -#ifdef __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - so that the build outputs can be moved efficiently from the - chroot to their final location. */ - auto chrootParentDir = store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootParentDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootParentDir); - - printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootParentDir); - - if (mkdir(chrootParentDir.c_str(), 0700) == -1) - throw SysError("cannot create '%s'", chrootRootDir); - - chrootRootDir = chrootParentDir + "/root"; - - if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUIDCount() != 1 ? buildUser->getUID() : 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - if (drvOptions->useUidRange(*drv)) - chownToBuilder(chrootRootDir + "/etc"); - - if (drvOptions->useUidRange(*drv) && (!buildUser || buildUser->getUIDCount() < 65536)) - throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", sandboxGid())); - - /* Create /etc/hosts with localhost entry. */ - if (derivationType->isSandboxed()) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); - - for (auto & i : inputPaths) { - auto p = store.printStorePath(i); - Path r = store.toRealPath(p); - pathsInChroot.insert_or_assign(p, r); - } - - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.sandbox-paths - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputsAndOptPaths(store)) { - /* If the name isn't known a priori (i.e. floating - content-addressing derivation), the temporary location we use - should be fresh. Freshness means it is impossible that the path - is already in the sandbox, so we don't need to worry about - removing it. */ - if (i.second.second) - pathsInChroot.erase(store.printStorePath(*i.second.second)); - } - - if (cgroup) { - if (mkdir(cgroup->c_str(), 0755) != 0) - throw SysError("creating cgroup '%s'", *cgroup); - chownToBuilder(*cgroup); - chownToBuilder(*cgroup + "/cgroup.procs"); - chownToBuilder(*cgroup + "/cgroup.threads"); - //chownToBuilder(*cgroup + "/cgroup.subtree_control"); - } - -#else - if (drvOptions->useUidRange(*drv)) - throw Error("feature 'uid-range' is not supported on this platform"); - #ifdef __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ - #else - throw Error("sandboxing builds is not supported on this platform"); - #endif -#endif - } else { - if (drvOptions->useUidRange(*drv)) - throw Error("feature 'uid-range' is only supported in sandboxed builds"); - } - - if (needsHashRewrite() && pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); - auto args = useChroot ? Strings({store.printStorePath(drvPath), chrootRootDir}) : - Strings({ store.printStorePath(drvPath) }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != std::string::npos; - nlPos = lines.find('\n', lastPos)) - { - auto line = lines.substr(lastPos, nlPos - lastPos); - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error("unknown pre-build hook command '%1%'", line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == std::string::npos) - pathsInChroot[line] = line; - else - pathsInChroot[line.substr(0, p)] = line.substr(p + 1); - } - } - } - } - - /* Fire up a Nix daemon to process recursive Nix calls from the - builder. */ - if (drvOptions->getRequiredSystemFeatures(*drv).count("recursive-nix")) - startDaemon(); - - /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args)); - for (auto & i : drv->env) - printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); - - /* Create the log file. */ - [[maybe_unused]] Path logFile = miscMethods.openLogFile(); - - /* Create a pseudoterminal to get the output of the builder. */ - builderOut = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut) - throw SysError("opening pseudoterminal master"); - - // FIXME: not thread-safe, use ptsname_r - std::string slaveName = ptsname(builderOut.get()); - - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); - - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } -#ifdef __APPLE__ - else { - if (grantpt(builderOut.get())) - throw SysError("granting access to pseudoterminal slave"); - } -#endif - - if (unlockpt(builderOut.get())) - throw SysError("unlocking pseudoterminal"); - - /* Open the slave side of the pseudoterminal and use it as stderr. */ - auto openSlave = [&]() - { - AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - if (dup2(builderOut.get(), STDERR_FILENO) == -1) - throw SysError("cannot pipe standard error into log file"); - }; - - buildResult.startTime = time(0); - - /* Fork a child to build the package. */ - -#ifdef __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - userNamespaceSync.create(); - - usingUserNamespace = userNamespacesSupported(); - - Pipe sendPid; - sendPid.create(); - - Pid helper = startProcess([&]() { - sendPid.readSide.close(); - - /* We need to open the slave early, before - CLONE_NEWUSER. Otherwise we get EPERM when running as - root. */ - openSlave(); - - try { - /* Drop additional groups here because we can't do it - after we've created the new user namespace. */ - if (setgroups(0, 0) == -1) { - if (errno != EPERM) - throw SysError("setgroups failed"); - if (settings.requireDropSupplementaryGroups) - throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); - } - - ProcessOptions options; - options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (derivationType->isSandboxed()) - options.cloneFlags |= CLONE_NEWNET; - if (usingUserNamespace) - options.cloneFlags |= CLONE_NEWUSER; - - pid_t child = startProcess([&]() { runChild(); }, options); - - writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); - _exit(0); - } catch (...) { - handleChildException(true); - _exit(1); - } - }); - - sendPid.writeSide.close(); - - if (helper.wait() != 0) { - processSandboxSetupMessages(); - // Only reached if the child process didn't send an exception. - throw Error("unable to start build process"); - } - - userNamespaceSync.readSide = -1; - - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { - userNamespaceSync.writeSide = -1; - }); - - auto ss = tokenizeString>(readLine(sendPid.readSide.get())); - assert(ss.size() == 1); - pid = string2Int(ss[0]).value(); - - if (usingUserNamespace) { - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1; - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d %d", sandboxUid(), hostUid, nrIds)); - - if (!buildUser || buildUser->getUIDCount() == 1) - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d %d", sandboxGid(), hostGid, nrIds)); - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - } - - /* Now that we now the sandbox uid, we can write - /etc/passwd. */ - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - - /* Save the mount- and user namespace of the child. We have to do this - *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxMountNamespace.get() == -1) - throw SysError("getting sandbox mount namespace"); - - if (usingUserNamespace) { - sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxUserNamespace.get() == -1) - throw SysError("getting sandbox user namespace"); - } - - /* Move the child into its own cgroup. */ - if (cgroup) - writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); - - /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - - } else -#endif - { - pid = startProcess([&]() { - openSlave(); - runChild(); - }); - } - - /* parent */ - pid.setSeparatePG(true); - miscMethods.childStarted(); - - processSandboxSetupMessages(); -} - - -void DerivationBuilder::processSandboxSetupMessages() -{ - std::vector msgs; - while (true) { - std::string msg = [&]() { - try { - return readLine(builderOut.get()); - } catch (Error & e) { - auto status = pid.wait(); - e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", - store.printStorePath(drvPath), - statusToString(status), - concatStringsSep("|", msgs)); - throw; - } - }(); - if (msg.substr(0, 1) == "\2") break; - if (msg.substr(0, 1) == "\1") { - FdSource source(builderOut.get()); - auto ex = readError(source); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: " + msg); - msgs.push_back(std::move(msg)); - } -} - - -void DerivationBuilder::initTmpDir() -{ - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#ifdef __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif - - /* In non-structured mode, set all bindings either directory in the - environment or via a file, as specified by - `DerivationOptions::passAsFile`. */ - if (!parsedDrv) { - for (auto & i : drv->env) { - if (drvOptions->passAsFile.find(i.first) == drvOptions->passAsFile.end()) { - env[i.first] = i.second; - } else { - auto hash = hashString(HashAlgorithm::SHA256, i.first); - std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; -} - - -void DerivationBuilder::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (derivationType->isFixed()) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (!derivationType->isSandboxed()) { - auto & impureEnv = settings.impureEnv.get(); - if (!impureEnv.empty()) - experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv); - - for (auto & i : drvOptions->impureEnvVars){ - auto envVar = impureEnv.find(i); - if (envVar != impureEnv.end()) { - env[i] = envVar->second; - } else { - env[i] = getEnv(i).value_or(""); - } - } - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; -} - - -void DerivationBuilder::writeStructuredAttrs() -{ - if (parsedDrv) { - auto json = parsedDrv->prepareStructuredAttrs( - store, - *drvOptions, - inputPaths, - drv->outputs); - nlohmann::json rewritten; - for (auto & [i, v] : json["outputs"].get()) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - rewritten[i] = rewriteStrings((std::string) v, inputRewrites); - } - - json["outputs"] = rewritten; - - auto jsonSh = StructuredAttrs::writeShell(json); - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); - env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh"; - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json"; - } -} - - -void DerivationBuilder::startDaemon() -{ - experimentalFeatureSettings.require(Xp::RecursiveNix); - - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = store.storeDir; - if (auto & optRoot = getLocalStore().rootDir.get()) - params["root"] = *optRoot; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = makeRestrictedStore(params, - ref(std::dynamic_pointer_cast(this->store.shared_from_this())), - *this); - - addedPaths.clear(); - - auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; - - daemonSocket = createUnixDomainSocket(socketPath, 0600); - - chownToBuilder(socketPath); - - daemonThread = std::thread([this, store]() { - - while (true) { - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - if (!remote) { - if (errno == EINTR || errno == EAGAIN) continue; - if (errno == EINVAL || errno == ECONNABORTED) break; - throw SysError("accepting connection"); - } - - unix::closeOnExec(remote.get()); - - debug("received daemon connection"); - - auto workerThread = std::thread([store, remote{std::move(remote)}]() { - try { - daemon::processConnection( - store, - FdSource(remote.get()), - FdSink(remote.get()), - NotTrusted, daemon::Recursive); - debug("terminated daemon connection"); - } catch (const Interrupted &) { - debug("interrupted daemon connection"); - } catch (SystemError &) { - ignoreExceptionExceptInterrupt(); - } - }); - - daemonWorkerThreads.push_back(std::move(workerThread)); - } - - debug("daemon shutting down"); - }); -} - - -void DerivationBuilder::stopDaemon() -{ - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) { - // According to the POSIX standard, the 'shutdown' function should - // return an ENOTCONN error when attempting to shut down a socket that - // hasn't been connected yet. This situation occurs when the 'accept' - // function is called on a socket without any accepted connections, - // leaving the socket unconnected. While Linux doesn't seem to produce - // an error for sockets that have only been accepted, more - // POSIX-compliant operating systems like OpenBSD, macOS, and others do - // return the ENOTCONN error. Therefore, we handle this error here to - // avoid raising an exception for compliant behaviour. - if (errno == ENOTCONN) { - daemonSocket.close(); - } else { - throw SysError("shutting down daemon socket"); - } - } - - if (daemonThread.joinable()) - daemonThread.join(); - - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) - thread.join(); - daemonWorkerThreads.clear(); - - // release the socket. - daemonSocket.close(); -} - - -void DerivationBuilder::addDependency(const StorePath & path) -{ - if (isAllowed(path)) return; - - addedPaths.insert(path); - - /* If we're doing a sandbox build, then we have to make the path - appear in the sandbox. */ - if (useChroot) { - - debug("materialising '%s' in the sandbox", store.printStorePath(path)); - - #ifdef __linux__ - - Path source = store.Store::toRealPath(path); - Path target = chrootRootDir + store.printStorePath(path); - - if (pathExists(target)) { - // There is a similar debug message in doBind, so only run it in this block to not have double messages. - debug("bind-mounting %s -> %s", target, source); - throw Error("store path '%s' already exists in the sandbox", store.printStorePath(path)); - } - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) - throw SysError("entering sandbox user namespace"); - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); - - doBind(source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", store.printStorePath(path)); - - #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - store.printStorePath(path)); - #endif - - } -} - -void DerivationBuilder::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); -} - - -void setupSeccomp() -{ -#ifdef __linux__ - if (!settings.filterSyscalls) return; -#if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - constexpr std::string_view nativeSystem = NIX_LOCAL_SYSTEM; - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - if (nativeSystem == "mips64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_MIPS) != 0) - printError("unable to add mips seccomp architecture"); - - if (nativeSystem == "mips64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_MIPS64N32) != 0) - printError("unable to add mips64-*abin32 seccomp architecture"); - - if (nativeSystem == "mips64el-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL) != 0) - printError("unable to add mipsel seccomp architecture"); - - if (nativeSystem == "mips64el-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL64N32) != 0) - printError("unable to add mips64el-*abin32 seccomp architecture"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), NIX_SYSCALL_FCHMODAT2, 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from using EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); -#else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); -#endif -#endif -} - - -void DerivationBuilder::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - bool sendException = true; - - try { /* child */ - - commonChildInit(); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } - - bool setUser = true; - - /* Make the contents of netrc and the CA certificate bundle - available to builtin:fetchurl (which may run under a - different uid and/or in a sandbox). */ - std::string netrcData; - std::string caFileData; - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") { - try { - netrcData = readFile(settings.netrcFile); - } catch (SystemError &) { } - - try { - caFileData = readFile(settings.caFile); - } catch (SystemError &) { } - } - -#ifdef __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; - - if (derivationType->isSandboxed()) { - - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) - throw SysError("unable to make '/' private"); - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + store.storeDir; - - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); - - if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (pathsInChroot.find("/dev") == pathsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (!derivationType->isSandboxed()) { - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - /* N.B. it is realistic that these paths might not exist. It - happens when testing Nix building fixed-output derivations - within a pure derivation. */ - for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" }) - if (pathExists(path)) - ss.push_back(path); - - if (settings.caFile != "" && pathExists(settings.caFile)) { - Path caFile = settings.caFile; - pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true); - } - } - - for (auto & i : ss) { - // For backwards-compatibiliy, resolve all the symlinks in the - // chroot paths - auto canonicalPath = canonPath(i, true); - pathsInChroot.emplace(i, canonicalPath); - } - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - for (auto & i : pathsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - - #if HAVE_EMBEDDED_SANDBOX_SHELL - if (i.second.source == "__embedded_sandbox_shell__") { - static unsigned char sh[] = { - #include "embedded-sandbox-shell.gen.hh" - }; - auto dst = chrootRootDir + i.first; - createDirs(dirOf(dst)); - writeFile(dst, std::string_view((const char *) sh, sizeof(sh))); - chmod_(dst, 0555); - } else - #endif - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } - - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount sysfs on /sys. */ - if (buildUser && buildUser->getUIDCount() != 1) { - createDirs(chrootRootDir + "/sys"); - if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) - throw SysError("mounting /sys"); - } - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !pathsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Make /etc unwritable */ - if (!drvOptions->useUidRange(*drv)) - chmod_(chrootRootDir + "/etc", 0555); - - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("unsharing mount namespace"); - - /* Unshare the cgroup namespace. This means - /proc/self/cgroup will show the child's cgroup as '/' - rather than whatever it is in the parent. */ - if (cgroup && unshare(CLONE_NEWCGROUP) == -1) - throw SysError("unsharing cgroup namespace"); - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); - - if (mkdir("real-root", 0500) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); - - if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); - - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); - - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); - - setUser = false; - } -#endif - - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); - - /* Close all other file descriptors. */ - unix::closeExtraFDs(); - -#ifdef __linux__ - linux::setPersonality(drv->platform); -#endif - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - auto gids = buildUser->getSupplementaryGIDs(); - if (setgroups(gids.size(), gids.data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); - } - -#ifdef __APPLE__ - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : pathsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = store.printStorePath(i); - pathsInChroot[p] = p; - } - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += - #include "sandbox-defaults.sb" - ; - - if (!derivationType->isSandboxed()) - sandboxProfile += - #include "sandbox-network.sb" - ; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - - // We create multiple allow lists, to avoid exceeding a limit in the darwin sandbox interpreter. - // See https://github.com/NixOS/nix/issues/4119 - // We split our allow groups approximately at half the actual limit, 1 << 16 - const size_t breakpoint = sandboxProfile.length() + (1 << 14); - for (auto & i : pathsInChroot) { - - if (sandboxProfile.length() >= breakpoint) { - debug("Sandbox break: %d %d", sandboxProfile.length(), breakpoint); - sandboxProfile += ")\n(allow file-read* file-write* process-exec\n"; - } - - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - std::string path = i.first; - auto optSt = maybeLstat(path.c_str()); - if (!optSt) { - if (i.second.optional) - continue; - throw SysError("getting attributes of required path '%s", path); - } - if (S_ISDIR(optSt->st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += drvOptions->additionalSandboxProfile; - } else - sandboxProfile += - #include "sandbox-minimal.sb" - ; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */ - Path globalTmpDir = canonPath(defaultTempDir(), true); - - /* They don't like trailing slashes on subpath directives */ - while (!globalTmpDir.empty() && globalTmpDir.back() == '/') - globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - Strings sandboxArgs; - sandboxArgs.push_back("_GLOBAL_TMP_DIR"); - sandboxArgs.push_back(globalTmpDir); - if (drvOptions->allowLocalNetworking) { - sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); - sandboxArgs.push_back("1"); - } - char * sandbox_errbuf = nullptr; - if (sandbox_init_with_parameters(sandboxProfile.c_str(), 0, stringsToCharPtrs(sandboxArgs).data(), &sandbox_errbuf)) { - writeFull(STDERR_FILENO, fmt("failed to configure sandbox: %s\n", sandbox_errbuf ? sandbox_errbuf : "(null)")); - _exit(1); - } - } -#endif - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, std::string("\2\n")); - - sendException = false; - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(getStandardError()); - - std::map outputs; - for (auto & e : drv->outputs) - outputs.insert_or_assign(e.first, - store.printStorePath(scratchOutputs.at(e.first))); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(*drv, outputs, netrcData, caFileData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(*drv, outputs); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(*drv, outputs); - else - throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + std::string("\n")); - _exit(1); - } - } - - // Now builder is not builtin - - Strings args; - args.push_back(std::string(baseNameOf(drv->builder))); - - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - -#ifdef __APPLE__ - posix_spawnattr_t attrp; - - if (posix_spawnattr_init(&attrp)) - throw SysError("failed to initialize builder"); - - if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) - throw SysError("failed to initialize builder"); - - if (drv->platform == "aarch64-darwin") { - // Unset kern.curproc_arch_affinity so we can escape Rosetta - int affinity = 0; - sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); - - cpu_type_t cpu = CPU_TYPE_ARM64; - posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); - } else if (drv->platform == "x86_64-darwin") { - cpu_type_t cpu = CPU_TYPE_X86_64; - posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); - } - - posix_spawn(NULL, drv->builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); -#else - execve(drv->builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); -#endif - - throw SysError("executing '%1%'", drv->builder); - - } catch (...) { - handleChildException(sendException); - _exit(1); - } -} - - -SingleDrvOutputs DerivationBuilder::registerOutputs() -{ - std::map infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - - std::exception_ptr delayedException; - - /* The paths that can be referenced are the input closures, the - output paths, and any paths that have been built via recursive - Nix calls. */ - StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) referenceablePaths.insert(p); - - /* FIXME `needsHashRewrite` should probably be removed and we get to the - real reason why we aren't using the chroot dir */ - auto toRealPathChroot = [&](const Path & p) -> Path { - return useChroot && !needsHashRewrite() - ? chrootRootDir + p - : store.toRealPath(p); - }; - - /* Check whether the output paths were created, and make all - output paths read-only. Then get the references of each output (that we - might need to register), so we can topologically sort them. For the ones - that are most definitely already installed, we just store their final - name so we can also use it in rewrites. */ - StringSet outputsToSort; - struct AlreadyRegistered { StorePath path; }; - struct PerhapsNeedToRegister { StorePathSet refs; }; - std::map> outputReferencesIfUnregistered; - std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { - auto scratchOutput = get(scratchOutputs, outputName); - if (!scratchOutput) - throw BuildError( - "builder for '%s' has no scratch output for '%s'", - store.printStorePath(drvPath), outputName); - auto actualPath = toRealPathChroot(store.printStorePath(*scratchOutput)); - - outputsToSort.insert(outputName); - - /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto initialOutput = get(initialOutputs, outputName); - if (!initialOutput) - throw BuildError( - "builder for '%s' has no initial output for '%s'", - store.printStorePath(drvPath), outputName); - auto & initialInfo = *initialOutput; - - /* Don't register if already valid, and not checking */ - initialInfo.wanted = buildMode == bmCheck - || !(initialInfo.known && initialInfo.known->isValid()); - if (!initialInfo.wanted) { - outputReferencesIfUnregistered.insert_or_assign( - outputName, - AlreadyRegistered { .path = initialInfo.known->path }); - continue; - } - - auto optSt = maybeLstat(actualPath.c_str()); - if (!optSt) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - store.printStorePath(drvPath), outputName, actualPath); - struct stat & st = *optSt; - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError( - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, outputName); -#endif - - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData( - actualPath, - buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, - inodesSeen); - - bool discardReferences = false; - if (auto udr = get(drvOptions->unsafeDiscardReferences, outputName)) { - discardReferences = *udr; - } - - StorePathSet references; - if (discardReferences) - debug("discarding references of output '%s'", outputName); - else { - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - references = scanForReferences(blank, actualPath, referenceablePaths); - } - - outputReferencesIfUnregistered.insert_or_assign( - outputName, - PerhapsNeedToRegister { .refs = references }); - outputStats.insert_or_assign(outputName, std::move(st)); - } - - auto sortedOutputNames = topoSort(outputsToSort, - {[&](const std::string & name) { - auto orifu = get(outputReferencesIfUnregistered, name); - if (!orifu) - throw BuildError( - "no output reference for '%s' in build of '%s'", - name, store.printStorePath(drvPath)); - return std::visit(overloaded { - /* Since we'll use the already installed versions of these, we - can treat them as leaves and ignore any references they - have. */ - [&](const AlreadyRegistered &) { return StringSet {}; }, - [&](const PerhapsNeedToRegister & refs) { - StringSet referencedOutputs; - /* FIXME build inverted map up front so no quadratic waste here */ - for (auto & r : refs.refs) - for (auto & [o, p] : scratchOutputs) - if (r == p) - referencedOutputs.insert(o); - return referencedOutputs; - }, - }, *orifu); - }}, - {[&](const std::string & path, const std::string & parent) { - // TODO with more -vvvv also show the temporary paths for manual inspection. - return BuildError( - "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - store.printStorePath(drvPath), path, parent); - }}); - - std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); - - OutputPathMap finalOutputs; - - for (auto & outputName : sortedOutputNames) { - auto output = get(drv->outputs, outputName); - auto scratchPath = get(scratchOutputs, outputName); - assert(output && scratchPath); - auto actualPath = toRealPathChroot(store.printStorePath(*scratchPath)); - - auto finish = [&](StorePath finalStorePath) { - /* Store the final path */ - finalOutputs.insert_or_assign(outputName, finalStorePath); - /* The rewrite rule will be used in downstream outputs that refer to - use. This is why the topological sort is essential to do first - before this for loop. */ - if (*scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath->hashPart() }] = std::string { finalStorePath.hashPart() }; - }; - - auto orifu = get(outputReferencesIfUnregistered, outputName); - assert(orifu); - - std::optional referencesOpt = std::visit(overloaded { - [&](const AlreadyRegistered & skippedFinalPath) -> std::optional { - finish(skippedFinalPath.path); - return std::nullopt; - }, - [&](const PerhapsNeedToRegister & r) -> std::optional { - return r.refs; - }, - }, *orifu); - - if (!referencesOpt) - continue; - auto references = *referencesOpt; - - auto rewriteOutput = [&](const StringMap & rewrites) { - /* Apply hash rewriting if necessary. */ - if (!rewrites.empty()) { - debug("rewriting hashes in '%1%'; cross fingers", actualPath); - - /* FIXME: Is this actually streaming? */ - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink(rewrites, nextSink); - dumpPath(actualPath, rsink); - rsink.flush(); - }); - Path tmpPath = actualPath + ".tmp"; - restorePath(tmpPath, *source); - deletePath(actualPath); - movePath(tmpPath, actualPath); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, {}, inodesSeen); - } - }; - - auto rewriteRefs = [&]() -> StoreReferences { - /* In the CA case, we need the rewritten refs to calculate the - final path, therefore we look for a *non-rewritten - self-reference, and use a bool rather try to solve the - computationally intractable fixed point. */ - StoreReferences res { - .self = false, - }; - for (auto & r : references) { - auto name = r.name(); - auto origHash = std::string { r.hashPart() }; - if (r == *scratchPath) { - res.self = true; - } else if (auto outputRewrite = get(outputRewrites, origHash)) { - std::string newRef = *outputRewrite; - newRef += '-'; - newRef += name; - res.others.insert(StorePath { newRef }); - } else { - res.others.insert(r); - } - } - return res; - }; - - auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo { - auto st = get(outputStats, outputName); - if (!st) - throw BuildError( - "output path %1% without valid stats info", - actualPath); - if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat) - { - /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) - throw BuildError( - "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", - actualPath); - } - rewriteOutput(outputRewrites); - /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath->hashPart() }; - auto got = [&]{ - auto fim = outputHash.method.getFileIngestionMethod(); - switch (fim) { - case FileIngestionMethod::Flat: - case FileIngestionMethod::NixArchive: - { - HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; - auto fim = outputHash.method.getFileIngestionMethod(); - dumpPath( - {getFSSourceAccessor(), CanonPath(actualPath)}, - caSink, - (FileSerialisationMethod) fim); - return caSink.finish().first; - } - case FileIngestionMethod::Git: { - return git::dumpHash( - outputHash.hashAlgo, - {getFSSourceAccessor(), CanonPath(actualPath)}).hash; - } - } - assert(false); - }(); - - ValidPathInfo newInfo0 { - store, - outputPathName(drv->name, outputName), - ContentAddressWithReferences::fromParts( - outputHash.method, - std::move(got), - rewriteRefs()), - Hash::dummy, - }; - if (*scratchPath != newInfo0.path) { - // If the path has some self-references, we need to rewrite - // them. - // (note that this doesn't invalidate the ca hash we calculated - // above because it's computed *modulo the self-references*, so - // it already takes this rewrite into account). - rewriteOutput( - StringMap{{oldHashPart, - std::string(newInfo0.path.hashPart())}}); - } - - { - HashResult narHashAndSize = hashPath( - {getFSSourceAccessor(), CanonPath(actualPath)}, - FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); - newInfo0.narHash = narHashAndSize.first; - newInfo0.narSize = narHashAndSize.second; - } - - assert(newInfo0.ca); - return newInfo0; - }; - - ValidPathInfo newInfo = std::visit(overloaded { - - [&](const DerivationOutput::InputAddressed & output) { - /* input-addressed case */ - auto requiredFinalPath = output.path; - /* Preemptively add rewrite rule for final hash, as that is - what the NAR hash will use rather than normalized-self references */ - if (*scratchPath != requiredFinalPath) - outputRewrites.insert_or_assign( - std::string { scratchPath->hashPart() }, - std::string { requiredFinalPath.hashPart() }); - rewriteOutput(outputRewrites); - HashResult narHashAndSize = hashPath( - {getFSSourceAccessor(), CanonPath(actualPath)}, - FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); - ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; - newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = std::move(refs.others); - if (refs.self) - newInfo0.references.insert(newInfo0.path); - return newInfo0; - }, - - [&](const DerivationOutput::CAFixed & dof) { - auto & wanted = dof.ca.hash; - - // Replace the output by a fresh copy of itself to make sure - // that there's no stale file descriptor pointing to it - Path tmpOutput = actualPath + ".tmp"; - copyFile( - std::filesystem::path(actualPath), - std::filesystem::path(tmpOutput), true); - - std::filesystem::rename(tmpOutput, actualPath); - - auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { - .method = dof.ca.method, - .hashAlgo = wanted.algo, - }); - - /* Check wanted hash */ - assert(newInfo0.ca); - auto & got = newInfo0.ca->hash; - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - miscMethods.noteHashMismatch(); - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", - store.printStorePath(drvPath), - wanted.to_string(HashFormat::SRI, true), - got.to_string(HashFormat::SRI, true))); - } - if (!newInfo0.references.empty()) { - auto numViolations = newInfo.references.size(); - delayedException = std::make_exception_ptr( - BuildError("fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", - store.printStorePath(drvPath), - numViolations, - store.printStorePath(*newInfo.references.begin()))); - } - - return newInfo0; - }, - - [&](const DerivationOutput::CAFloating & dof) { - return newInfoFromCA(dof); - }, - - [&](const DerivationOutput::Deferred &) -> ValidPathInfo { - // No derivation should reach that point without having been - // rewritten first - assert(false); - }, - - [&](const DerivationOutput::Impure & doi) { - return newInfoFromCA(DerivationOutput::CAFloating { - .method = doi.method, - .hashAlgo = doi.hashAlgo, - }); - }, - - }, output->raw); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, {}, inodesSeen); - - /* Calculate where we'll move the output files. In the checking case we - will leave leave them where they are, for now, rather than move to - their usual "final destination" */ - auto finalDestPath = store.printStorePath(newInfo.path); - - /* Lock final output path, if not already locked. This happens with - floating CA derivations and hash-mismatching fixed-output - derivations. */ - PathLocks dynamicOutputLock; - dynamicOutputLock.setDeletion(true); - auto optFixedPath = output->path(store, drv->name, outputName); - if (!optFixedPath || - store.printStorePath(*optFixedPath) != finalDestPath) - { - assert(newInfo.ca); - dynamicOutputLock.lockPaths({store.toRealPath(finalDestPath)}); - } - - /* Move files, if needed */ - if (store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(store.toRealPath(finalDestPath), actualPath); - actualPath = store.toRealPath(finalDestPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - } else { - auto destPath = store.toRealPath(finalDestPath); - deletePath(destPath); - movePath(actualPath, destPath); - actualPath = destPath; - } - } - - auto & localStore = getLocalStore(); - - if (buildMode == bmCheck) { - - if (!store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - miscMethods.noteCheckMismatch(); - if (settings.runDiffHook || settings.keepFailed) { - auto dst = store.toRealPath(finalDestPath + checkSuffix); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, store.printStorePath(drvPath), tmpDir); - - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - store.printStorePath(drvPath), store.toRealPath(finalDestPath), dst); - } else - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - store.printStorePath(drvPath), store.toRealPath(finalDestPath)); - } - - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - localStore.signPathInfo(oldInfo); - localStore.registerValidPaths({{oldInfo.path, oldInfo}}); - } - - continue; - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - if (references.count(i)) - debug("referenced input: '%1%'", store.printStorePath(i)); - else - debug("unreferenced input: '%1%'", store.printStorePath(i)); - } - - localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() - miscMethods.markContentsGood(newInfo.path); - - newInfo.deriver = drvPath; - newInfo.ultimate = true; - localStore.signPathInfo(newInfo); - - finish(newInfo.path); - - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - localStore.registerValidPaths({{newInfo.path, newInfo}}); - - infos.emplace(outputName, std::move(newInfo)); - } - - if (buildMode == bmCheck) { - /* In case of fixed-output derivations, if there are - mismatches on `--check` an error must be thrown as this is - also a source for non-determinism. */ - if (delayedException) - std::rethrow_exception(delayedException); - return miscMethods.assertPathValidity(); - } - - /* Apply output checks. */ - checkOutputs(infos); - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - auto & localStore = getLocalStore(); - - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.insert_or_assign(newInfo.path, newInfo); - } - localStore.registerValidPaths(infos2); - } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ - SingleDrvOutputs builtOutputs; - - for (auto & [outputName, newInfo] : infos) { - auto oldinfo = get(initialOutputs, outputName); - assert(oldinfo); - auto thisRealisation = Realisation { - .id = DrvOutput { - oldinfo->outputHash, - outputName - }, - .outPath = newInfo.path - }; - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) - && !drv->type().isImpure()) - { - store.signRealisation(thisRealisation); - store.registerDrvOutput(thisRealisation); - } - builtOutputs.emplace(outputName, thisRealisation); - } - - return builtOutputs; -} - - -void DerivationBuilder::checkOutputs(const std::map & outputs) -{ - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) - { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const DerivationOptions::OutputChecks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - store.printStorePath(info.path), info.narSize, *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - store.printStorePath(info.path), closureSize, *checks.maxClosureSize); - } - - auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive) - { - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : value) { - if (store.isStorePath(i)) - spec.insert(store.parseStorePath(i)); - else if (auto output = get(outputs, i)) - spec.insert(output->path); - else { - std::string outputsListing = concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; }); - throw BuildError("derivation '%s' output check for '%s' contains an illegal reference specifier '%s'," - " expected store path or output name (one of [%s])", - store.printStorePath(drvPath), outputName, i, outputsListing); - } - } - - auto used = recursive - ? getClosure(info.path).first - : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - std::string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += store.printStorePath(i); - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", - store.printStorePath(info.path), badPathsStr); - } - }; - - /* Mandatory check: absent whitelist, and present but empty - whitelist mean very different things. */ - if (auto & refs = checks.allowedReferences) { - checkRefs(*refs, true, false); - } - if (auto & refs = checks.allowedRequisites) { - checkRefs(*refs, true, true); - } - - /* Optimization: don't need to do anything when - disallowed and empty set. */ - if (!checks.disallowedReferences.empty()) { - checkRefs(checks.disallowedReferences, false, false); - } - if (!checks.disallowedRequisites.empty()) { - checkRefs(checks.disallowedRequisites, false, true); - } - }; - - std::visit(overloaded{ - [&](const DerivationOptions::OutputChecks & checks) { - applyChecks(checks); - }, - [&](const std::map & checksPerOutput) { - if (auto outputChecks = get(checksPerOutput, outputName)) - - applyChecks(*outputChecks); - }, - }, drvOptions->outputChecks); - } -} - - -void DerivationBuilder::deleteTmpDir(bool force) -{ - if (topTmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(topTmpDir.c_str(), 0755); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(topTmpDir); - topTmpDir = ""; - tmpDir = ""; - } -} - - -bool LocalDerivationGoal::isReadDesc(int fd) -{ - return (hook && DerivationGoal::isReadDesc(fd)) || - (!hook && fd == builder.builderOut.get()); -} - - -StorePath DerivationBuilder::makeFallbackPath(OutputNameView outputName) -{ - // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path - // See doc/manual/source/protocols/store-path.md for details - // TODO: We may want to separate the responsibilities of constructing the path fingerprint and of actually doing the hashing - auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName); - return store.makeStorePath( - pathType, - // pass an all-zeroes hash - Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName)); -} - - -StorePath DerivationBuilder::makeFallbackPath(const StorePath & path) -{ - // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path - // See doc/manual/source/protocols/store-path.md for details - auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()); - return store.makeStorePath( - pathType, - // pass an all-zeroes hash - Hash(HashAlgorithm::SHA256), path.name()); -} - +std::unique_ptr makeDerivationBuilder( + Store & store, + DerivationBuilderCallbacks & miscMethods, + DerivationBuilderParams params); } diff --git a/src/libstore/unix/include/nix/store/meson.build b/src/libstore/unix/include/nix/store/meson.build index 9f12440cd..03101f4bb 100644 --- a/src/libstore/unix/include/nix/store/meson.build +++ b/src/libstore/unix/include/nix/store/meson.build @@ -2,6 +2,7 @@ include_dirs += include_directories('../..') headers += files( 'build/child.hh', + 'build/derivation-builder.hh', 'build/hook-instance.hh', 'build/local-derivation-goal.hh', 'user-lock.hh', diff --git a/src/libstore/unix/meson.build b/src/libstore/unix/meson.build index f06c9aa95..08d38742b 100644 --- a/src/libstore/unix/meson.build +++ b/src/libstore/unix/meson.build @@ -1,5 +1,6 @@ sources += files( 'build/child.cc', + 'build/derivation-builder.cc', 'build/hook-instance.cc', 'build/local-derivation-goal.cc', 'pathlocks.cc', From fc8c11be481ce0fa2b50af1f7bedb9b9a1f1ac1f Mon Sep 17 00:00:00 2001 From: Illia Bobyr Date: Mon, 13 Jan 2025 19:17:09 -0800 Subject: [PATCH 237/396] nix-profile.fish: Add local state dir bin to $PATH It seems reasonable to add both `$HOME/.profile/bin` and `@localstatedir@/nix/profiles/default/bin` to `$PATH` for both user local and daemon based nix execution. Nix daemon execution mode does not affect these path. --- scripts/nix-profile.fish.in | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index 59b3c695e..e1c001dc0 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -69,6 +69,7 @@ if set --query MANPATH set --export --prepend --path MANPATH "$NIX_LINK/share/man" end +add_path "@localstatedir@/nix/profiles/default/bin" add_path "$NIX_LINK/bin" # Cleanup From 470c521bccd8c97ed33edb43278d6476ac6f4322 Mon Sep 17 00:00:00 2001 From: Illia Bobyr Date: Mon, 13 Jan 2025 19:23:06 -0800 Subject: [PATCH 238/396] nix-profile-daemon.fish: Set MANPATH There seems to be no good reason for `nix-profile.fish` to set `$MANPATH` while it being unset when `nix-profile-daemon.fish` is used. --- scripts/nix-profile-daemon.fish.in | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 2ecab0077..3fb727151 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -62,6 +62,13 @@ else end end +# Only use MANPATH if it is already set. In general `man` will just simply +# pick up `.nix-profile/share/man` because is it close to `.nix-profile/bin` +# which is in the $PATH. For more info, run `manpath -d`. +if set --query MANPATH + set --export --prepend --path MANPATH "$NIX_LINK/share/man" +end + add_path "@localstatedir@/nix/profiles/default/bin" add_path "$NIX_LINK/bin" From 66ae8f4f443d45ac33f95c26ee5a2a8f816a25ec Mon Sep 17 00:00:00 2001 From: Illia Bobyr Date: Mon, 13 Jan 2025 19:25:06 -0800 Subject: [PATCH 239/396] nix-profile.fish: Do not check $USER While it seems unlikely that `$USER` will be unset while `$HOME` is set, as `$USER` is not used in the script and as `nix-profile-daemon.fish` is not checking `$USER`, it seems better to remove this check. `nix-profile.fish` and `nix-profile-daemon.fish` now become identical. --- scripts/nix-profile.fish.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index 59b3c695e..c1421b097 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -1,5 +1,5 @@ # Only execute this file once per shell. -if test -z "$HOME" || test -z "$USER" || \ +if test -z "$HOME" || \ test -n "$__ETC_PROFILE_NIX_SOURCED" exit end From b3bbbd2e71b506f84d1bd1d2862a16cc62db975f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 17 Apr 2025 13:54:14 +0200 Subject: [PATCH 240/396] fix armv7/i686 build --- src/libfetchers/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 4a916929e..80c5884f7 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -38,7 +38,7 @@ const std::string gitInitialBranch = "__nix_dummy_branch"; bool isCacheFileWithinTtl(time_t now, const struct stat & st) { - return st.st_mtime + settings.tarballTtl > now; + return st.st_mtime + static_cast(settings.tarballTtl) > now; } Path getCachePath(std::string_view key, bool shallow) From 27907e6cac22a8b1070554fb6e02e918c2f0a8d4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 18 Apr 2025 12:24:24 -0400 Subject: [PATCH 241/396] Add documentation for `DerivationBuilder::{parsedDrv, drvOptions}` --- .../include/nix/store/build/derivation-builder.hh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/libstore/unix/include/nix/store/build/derivation-builder.hh b/src/libstore/unix/include/nix/store/build/derivation-builder.hh index 5fe528f65..c5382c90e 100644 --- a/src/libstore/unix/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/unix/include/nix/store/build/derivation-builder.hh @@ -30,7 +30,20 @@ struct DerivationBuilderParams */ const std::unique_ptr & drv; + /** + * The "structured attrs" of `drv`, if it has them. + * + * @todo this should be part of `Derivation`. + * + * @todo this should be renamed from `parsedDrv`. + */ const std::unique_ptr & parsedDrv; + + /** + * The derivation options of `drv`. + * + * @todo this should be part of `Derivation`. + */ const std::unique_ptr & drvOptions; // The remainder is state held during the build. From 2b2ea218d97a71df09c28b349abf6170ff090a0f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 18 Apr 2025 12:13:06 -0400 Subject: [PATCH 242/396] Delay constructing a `DerivationBuilder` until we're about to build This makes the simplification in the following commit possible. --- .../unix/build/local-derivation-goal.cc | 57 +++++++++---------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 285f9cc2e..24cd1a028 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -41,38 +41,12 @@ struct LocalDerivationGoal : DerivationGoal, DerivationBuilderCallbacks const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) : DerivationGoal{drvPath, wantedOutputs, worker, buildMode} - , builder{makeDerivationBuilder( - worker.store, - static_cast(*this), - DerivationBuilderParams { - DerivationGoal::drvPath, - DerivationGoal::buildMode, - DerivationGoal::buildResult, - DerivationGoal::drv, - DerivationGoal::parsedDrv, - DerivationGoal::drvOptions, - DerivationGoal::inputPaths, - DerivationGoal::initialOutputs, - })} {} LocalDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal) : DerivationGoal{drvPath, drv, wantedOutputs, worker, buildMode} - , builder{makeDerivationBuilder( - worker.store, - static_cast(*this), - DerivationBuilderParams { - DerivationGoal::drvPath, - DerivationGoal::buildMode, - DerivationGoal::buildResult, - DerivationGoal::drv, - DerivationGoal::parsedDrv, - DerivationGoal::drvOptions, - DerivationGoal::inputPaths, - DerivationGoal::initialOutputs, - })} {} virtual ~LocalDerivationGoal() override; @@ -136,15 +110,19 @@ LocalDerivationGoal::~LocalDerivationGoal() { /* Careful: we should never ever throw an exception from a destructor. */ - try { builder->deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } + if (builder) { + try { builder->deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } + } try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } - try { builder->stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } + if (builder) { + try { builder->stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } + } } void LocalDerivationGoal::killChild() { - if (builder->pid != -1) { + if (builder && builder->pid != -1) { worker.childTerminated(this); /* If we're using a build user, then there is a tricky race @@ -165,6 +143,7 @@ void LocalDerivationGoal::killChild() void LocalDerivationGoal::childStarted() { + assert(builder); worker.childStarted(shared_from_this(), {builder->builderOut.get()}, true, true); } @@ -202,6 +181,24 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() co_return tryToBuild(); } + if (!builder) { + /* If we have to wait and retry (see below), then `builder` will + already be created, so we don't need to create it again. */ + builder = makeDerivationBuilder( + worker.store, + static_cast(*this), + DerivationBuilderParams { + DerivationGoal::drvPath, + DerivationGoal::buildMode, + DerivationGoal::buildResult, + DerivationGoal::drv, + DerivationGoal::parsedDrv, + DerivationGoal::drvOptions, + DerivationGoal::inputPaths, + DerivationGoal::initialOutputs, + }); + } + if (!builder->prepareBuild()) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, @@ -251,7 +248,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() bool LocalDerivationGoal::isReadDesc(int fd) { return (hook && DerivationGoal::isReadDesc(fd)) || - (!hook && fd == builder->builderOut.get()); + (!hook && builder && fd == builder->builderOut.get()); } } From 5eeeb44f7920e1909eb540611fef484ef8a2888e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 18 Apr 2025 12:16:23 -0400 Subject: [PATCH 243/396] Remove double indirection from `DerivationBuilder` params Now that `DerivationBuilder` is created after the underlying data has already been initialized, we can just refer this data normally, with a direct reference. Only `parsedDrv` takes a (borrowing) pointer, because independent of initialization the derivation may or may not have structured attrs. --- src/libstore/unix/build/derivation-builder.cc | 116 +++++++++--------- .../unix/build/local-derivation-goal.cc | 6 +- .../nix/store/build/derivation-builder.hh | 15 +-- 3 files changed, 67 insertions(+), 70 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 4a51441b7..95588bf08 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -472,16 +472,16 @@ void DerivationBuilderImpl::killSandbox(bool getStats) bool DerivationBuilderImpl::prepareBuild() { /* Cache this */ - derivationType = drv->type(); + derivationType = drv.type(); /* Are we doing a chroot build? */ { if (settings.sandboxMode == smEnabled) { - if (drvOptions->noChroot) + if (drvOptions.noChroot) throw Error("derivation '%s' has '__noChroot' set, " "but that's not allowed when 'sandbox' is 'true'", store.printStorePath(drvPath)); #ifdef __APPLE__ - if (drvOptions->additionalSandboxProfile != "") + if (drvOptions.additionalSandboxProfile != "") throw Error("derivation '%s' specifies a sandbox profile, " "but this is only allowed when 'sandbox' is 'relaxed'", store.printStorePath(drvPath)); #endif @@ -490,7 +490,7 @@ bool DerivationBuilderImpl::prepareBuild() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = derivationType->isSandboxed() && !drvOptions->noChroot; + useChroot = derivationType->isSandboxed() && !drvOptions.noChroot; } auto & localStore = getLocalStore(); @@ -515,7 +515,7 @@ bool DerivationBuilderImpl::prepareBuild() if (useBuildUsers()) { if (!buildUser) - buildUser = acquireUserLock(drvOptions->useUidRange(*drv) ? 65536 : 1, useChroot); + buildUser = acquireUserLock(drvOptions.useUidRange(drv) ? 65536 : 1, useChroot); if (!buildUser) { return false; @@ -832,14 +832,14 @@ void DerivationBuilderImpl::startBuilder() killSandbox(false); /* Right platform? */ - if (!drvOptions->canBuildLocally(store, *drv)) { + if (!drvOptions.canBuildLocally(store, drv)) { // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - we should tell them to run the command to install Darwin 2 - if (drv->platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") { - throw Error("run `/usr/sbin/softwareupdate --install-rosetta` to enable your %s to run programs for %s", settings.thisSystem, drv->platform); + if (drv.platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") { + throw Error("run `/usr/sbin/softwareupdate --install-rosetta` to enable your %s to run programs for %s", settings.thisSystem, drv.platform); } else { throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)), + drv.platform, + concatStringsSep(", ", drvOptions.getRequiredSystemFeatures(drv)), store.printStorePath(drvPath), settings.thisSystem, concatStringsSep(", ", store.systemFeatures)); @@ -930,7 +930,7 @@ void DerivationBuilderImpl::startBuilder() /* Handle exportReferencesGraph(), if set. */ if (!parsedDrv) { - for (auto & [fileName, ss] : drvOptions->exportReferencesGraph) { + for (auto & [fileName, ss] : drvOptions.exportReferencesGraph) { StorePathSet storePathSet; for (auto & storePathS : ss) { if (!store.isInStore(storePathS)) @@ -988,7 +988,7 @@ void DerivationBuilderImpl::startBuilder() PathSet allowedPaths = settings.allowedImpureHostPrefixes; /* This works like the above, except on a per-derivation level */ - auto impurePaths = drvOptions->impureHostDeps; + auto impurePaths = drvOptions.impureHostDeps; for (auto & i : impurePaths) { bool found = false; @@ -1008,7 +1008,7 @@ void DerivationBuilderImpl::startBuilder() throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", store.printStorePath(drvPath), i); - /* Allow files in drvOptions->impureHostDeps to be missing; e.g. + /* Allow files in drvOptions.impureHostDeps to be missing; e.g. macOS 11+ has no /usr/lib/libSystem*.dylib */ pathsInChroot[i] = {i, true}; } @@ -1048,10 +1048,10 @@ void DerivationBuilderImpl::startBuilder() nobody account. The latter is kind of a hack to support Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); - if (drvOptions->useUidRange(*drv)) + if (drvOptions.useUidRange(drv)) chownToBuilder(chrootRootDir + "/etc"); - if (drvOptions->useUidRange(*drv) && (!buildUser || buildUser->getUIDCount() < 65536)) + if (drvOptions.useUidRange(drv) && (!buildUser || buildUser->getUIDCount() < 65536)) throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); /* Declare the build user's group so that programs get a consistent @@ -1090,7 +1090,7 @@ void DerivationBuilderImpl::startBuilder() rebuilding a path that is in settings.sandbox-paths (typically the dependencies of /bin/sh). Throw them out. */ - for (auto & i : drv->outputsAndOptPaths(store)) { + for (auto & i : drv.outputsAndOptPaths(store)) { /* If the name isn't known a priori (i.e. floating content-addressing derivation), the temporary location we use should be fresh. Freshness means it is impossible that the path @@ -1110,7 +1110,7 @@ void DerivationBuilderImpl::startBuilder() } #else - if (drvOptions->useUidRange(*drv)) + if (drvOptions.useUidRange(drv)) throw Error("feature 'uid-range' is not supported on this platform"); #ifdef __APPLE__ /* We don't really have any parent prep work to do (yet?) @@ -1120,14 +1120,14 @@ void DerivationBuilderImpl::startBuilder() #endif #endif } else { - if (drvOptions->useUidRange(*drv)) + if (drvOptions.useUidRange(drv)) throw Error("feature 'uid-range' is only supported in sandboxed builds"); } if (needsHashRewrite() && pathExists(homeDir)) throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { + if (useChroot && settings.preBuildHook != "" && dynamic_cast(&drv)) { printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); auto args = useChroot ? Strings({store.printStorePath(drvPath), chrootRootDir}) : Strings({ store.printStorePath(drvPath) }); @@ -1165,13 +1165,13 @@ void DerivationBuilderImpl::startBuilder() /* Fire up a Nix daemon to process recursive Nix calls from the builder. */ - if (drvOptions->getRequiredSystemFeatures(*drv).count("recursive-nix")) + if (drvOptions.getRequiredSystemFeatures(drv).count("recursive-nix")) startDaemon(); /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args)); - for (auto & i : drv->env) + printMsg(lvlChatty, "executing builder '%1%'", drv.builder); + printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv.args)); + for (auto & i : drv.env) printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); /* Create the log file. */ @@ -1434,8 +1434,8 @@ void DerivationBuilderImpl::initTmpDir() environment or via a file, as specified by `DerivationOptions::passAsFile`. */ if (!parsedDrv) { - for (auto & i : drv->env) { - if (drvOptions->passAsFile.find(i.first) == drvOptions->passAsFile.end()) { + for (auto & i : drv.env) { + if (drvOptions.passAsFile.find(i.first) == drvOptions.passAsFile.end()) { env[i.first] = i.second; } else { auto hash = hashString(HashAlgorithm::SHA256, i.first); @@ -1512,7 +1512,7 @@ void DerivationBuilderImpl::initEnv() if (!impureEnv.empty()) experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv); - for (auto & i : drvOptions->impureEnvVars){ + for (auto & i : drvOptions.impureEnvVars){ auto envVar = impureEnv.find(i); if (envVar != impureEnv.end()) { env[i] = envVar->second; @@ -1537,9 +1537,9 @@ void DerivationBuilderImpl::writeStructuredAttrs() if (parsedDrv) { auto json = parsedDrv->prepareStructuredAttrs( store, - *drvOptions, + drvOptions, inputPaths, - drv->outputs); + drv.outputs); nlohmann::json rewritten; for (auto & [i, v] : json["outputs"].get()) { /* The placeholder must have a rewrite, so we use it to cover both the @@ -1834,7 +1834,7 @@ void DerivationBuilderImpl::runChild() different uid and/or in a sandbox). */ std::string netrcData; std::string caFileData; - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") { + if (drv.isBuiltin() && drv.builder == "builtin:fetchurl") { try { netrcData = readFile(settings.netrcFile); } catch (SystemError &) { } @@ -2019,7 +2019,7 @@ void DerivationBuilderImpl::runChild() } /* Make /etc unwritable */ - if (!drvOptions->useUidRange(*drv)) + if (!drvOptions.useUidRange(drv)) chmod_(chrootRootDir + "/etc", 0555); /* Unshare this mount namespace. This is necessary because @@ -2079,7 +2079,7 @@ void DerivationBuilderImpl::runChild() unix::closeExtraFDs(); #ifdef __linux__ - linux::setPersonality(drv->platform); + linux::setPersonality(drv.platform); #endif /* Disable core dumps by default. */ @@ -2217,7 +2217,7 @@ void DerivationBuilderImpl::runChild() } sandboxProfile += ")\n"; - sandboxProfile += drvOptions->additionalSandboxProfile; + sandboxProfile += drvOptions.additionalSandboxProfile; } else sandboxProfile += #include "sandbox-minimal.sb" @@ -2238,7 +2238,7 @@ void DerivationBuilderImpl::runChild() Strings sandboxArgs; sandboxArgs.push_back("_GLOBAL_TMP_DIR"); sandboxArgs.push_back(globalTmpDir); - if (drvOptions->allowLocalNetworking) { + if (drvOptions.allowLocalNetworking) { sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); sandboxArgs.push_back("1"); } @@ -2256,23 +2256,23 @@ void DerivationBuilderImpl::runChild() sendException = false; /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { + if (drv.isBuiltin()) { try { logger = makeJSONLogger(getStandardError()); std::map outputs; - for (auto & e : drv->outputs) + for (auto & e : drv.outputs) outputs.insert_or_assign(e.first, store.printStorePath(scratchOutputs.at(e.first))); - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(*drv, outputs, netrcData, caFileData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(*drv, outputs); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(*drv, outputs); + if (drv.builder == "builtin:fetchurl") + builtinFetchurl(drv, outputs, netrcData, caFileData); + else if (drv.builder == "builtin:buildenv") + builtinBuildenv(drv, outputs); + else if (drv.builder == "builtin:unpack-channel") + builtinUnpackChannel(drv, outputs); else - throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8)); + throw Error("unsupported builtin builder '%1%'", drv.builder.substr(8)); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, e.what() + std::string("\n")); @@ -2283,9 +2283,9 @@ void DerivationBuilderImpl::runChild() // Now builder is not builtin Strings args; - args.push_back(std::string(baseNameOf(drv->builder))); + args.push_back(std::string(baseNameOf(drv.builder))); - for (auto & i : drv->args) + for (auto & i : drv.args) args.push_back(rewriteStrings(i, inputRewrites)); #ifdef __APPLE__ @@ -2297,24 +2297,24 @@ void DerivationBuilderImpl::runChild() if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) throw SysError("failed to initialize builder"); - if (drv->platform == "aarch64-darwin") { + if (drv.platform == "aarch64-darwin") { // Unset kern.curproc_arch_affinity so we can escape Rosetta int affinity = 0; sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); cpu_type_t cpu = CPU_TYPE_ARM64; posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); - } else if (drv->platform == "x86_64-darwin") { + } else if (drv.platform == "x86_64-darwin") { cpu_type_t cpu = CPU_TYPE_X86_64; posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); } - posix_spawn(NULL, drv->builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); + posix_spawn(NULL, drv.builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); #else - execve(drv->builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); + execve(drv.builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); #endif - throw SysError("executing '%1%'", drv->builder); + throw SysError("executing '%1%'", drv.builder); } catch (...) { handleChildException(sendException); @@ -2362,7 +2362,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() struct PerhapsNeedToRegister { StorePathSet refs; }; std::map> outputReferencesIfUnregistered; std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { + for (auto & [outputName, _] : drv.outputs) { auto scratchOutput = get(scratchOutputs, outputName); if (!scratchOutput) throw BuildError( @@ -2418,7 +2418,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() inodesSeen); bool discardReferences = false; - if (auto udr = get(drvOptions->unsafeDiscardReferences, outputName)) { + if (auto udr = get(drvOptions.unsafeDiscardReferences, outputName)) { discardReferences = *udr; } @@ -2474,7 +2474,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() OutputPathMap finalOutputs; for (auto & outputName : sortedOutputNames) { - auto output = get(drv->outputs, outputName); + auto output = get(drv.outputs, outputName); auto scratchPath = get(scratchOutputs, outputName); assert(output && scratchPath); auto actualPath = toRealPathChroot(store.printStorePath(*scratchPath)); @@ -2596,7 +2596,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() ValidPathInfo newInfo0 { store, - outputPathName(drv->name, outputName), + outputPathName(drv.name, outputName), ContentAddressWithReferences::fromParts( outputHash.method, std::move(got), @@ -2725,7 +2725,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() derivations. */ PathLocks dynamicOutputLock; dynamicOutputLock.setDeletion(true); - auto optFixedPath = output->path(store, drv->name, outputName); + auto optFixedPath = output->path(store, drv.name, outputName); if (!optFixedPath || store.printStorePath(*optFixedPath) != finalDestPath) { @@ -2863,7 +2863,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() .outPath = newInfo.path }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) - && !drv->type().isImpure()) + && !drv.type().isImpure()) { store.signRealisation(thisRealisation); store.registerDrvOutput(thisRealisation); @@ -3005,7 +3005,7 @@ void DerivationBuilderImpl::checkOutputs(const std::mapoutputChecks); + }, drvOptions.outputChecks); } } @@ -3015,7 +3015,7 @@ void DerivationBuilderImpl::deleteTmpDir(bool force) if (topTmpDir != "") { /* Don't keep temporary directories for builtins because they might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { + if (settings.keepFailed && !force && !drv.isBuiltin()) { printError("note: keeping build directory '%s'", tmpDir); chmod(topTmpDir.c_str(), 0755); chmod(tmpDir.c_str(), 0755); @@ -3037,7 +3037,7 @@ StorePath DerivationBuilderImpl::makeFallbackPath(OutputNameView outputName) return store.makeStorePath( pathType, // pass an all-zeroes hash - Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName)); + Hash(HashAlgorithm::SHA256), outputPathName(drv.name, outputName)); } diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 24cd1a028..188066679 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -191,9 +191,9 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() DerivationGoal::drvPath, DerivationGoal::buildMode, DerivationGoal::buildResult, - DerivationGoal::drv, - DerivationGoal::parsedDrv, - DerivationGoal::drvOptions, + *DerivationGoal::drv, + DerivationGoal::parsedDrv.get(), + *DerivationGoal::drvOptions, DerivationGoal::inputPaths, DerivationGoal::initialOutputs, }); diff --git a/src/libstore/unix/include/nix/store/build/derivation-builder.hh b/src/libstore/unix/include/nix/store/build/derivation-builder.hh index c5382c90e..5e5532fe0 100644 --- a/src/libstore/unix/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/unix/include/nix/store/build/derivation-builder.hh @@ -24,11 +24,8 @@ struct DerivationBuilderParams /** * The derivation stored at drvPath. - * - * @todo Remove double indirection by delaying when this is - * initialized. */ - const std::unique_ptr & drv; + const Derivation & drv; /** * The "structured attrs" of `drv`, if it has them. @@ -37,14 +34,14 @@ struct DerivationBuilderParams * * @todo this should be renamed from `parsedDrv`. */ - const std::unique_ptr & parsedDrv; + const StructuredAttrs * parsedDrv; /** * The derivation options of `drv`. * * @todo this should be part of `Derivation`. */ - const std::unique_ptr & drvOptions; + const DerivationOptions & drvOptions; // The remainder is state held during the build. @@ -65,9 +62,9 @@ struct DerivationBuilderParams const StorePath & drvPath, const BuildMode & buildMode, BuildResult & buildResult, - const std::unique_ptr & drv, - const std::unique_ptr & parsedDrv, - const std::unique_ptr & drvOptions, + const Derivation & drv, + const StructuredAttrs * parsedDrv, + const DerivationOptions & drvOptions, const StorePathSet & inputPaths, std::map & initialOutputs) : drvPath{drvPath} From 85420b85379d42d03c225db91e0eeee4ac77ffde Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Sun, 20 Apr 2025 14:49:03 -0400 Subject: [PATCH 244/396] libstore: increase retry delay for 429 A 429 (Too Many Requests) error should not be retried after a quarter of a second; that's just silly. GitHub recommends a minute. --- src/libstore/filetransfer.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index a917188d9..5a298a658 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -399,6 +399,8 @@ struct curlFileTransfer : public FileTransfer { auto finishTime = std::chrono::steady_clock::now(); + auto retryTimeMs = request.baseRetryTimeMs; + auto httpStatus = getHTTPStatus(); debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", @@ -449,10 +451,12 @@ struct curlFileTransfer : public FileTransfer } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) { // Don't retry on authentication/authorization failures err = Forbidden; - } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) { + } else if (httpStatus == 429) { + // 429 means too many requests, so we retry (with a substantially longer delay) + retryTimeMs = 60000; + } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408) { // Most 4xx errors are client errors and are probably not worth retrying: // * 408 means the server timed out waiting for us, so we try again - // * 429 means too many requests, so we retry (with a delay) err = Misc; } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) { // Let's treat most 5xx (server) errors as transient, except for a handful: @@ -518,7 +522,7 @@ struct curlFileTransfer : public FileTransfer || writtenToSink == 0 || (acceptRanges && encoding.empty()))) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); + int ms = retryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); if (writtenToSink) warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); else From b257ea94e32652b2f822f85e5b8e6a9524c47fe1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 20 Apr 2025 22:20:52 +0200 Subject: [PATCH 245/396] Fix pkgs.nixVersions and installTests ... by moving our stuff out of the way from upstream's `nixComponents` and `nixDependencies` attrsets. (I prefer not to use overlays, but let's make it work this way first) --- flake.nix | 36 +++++++++++----------- packaging/dev-shell.nix | 50 +++++++++++++++---------------- packaging/hydra.nix | 26 ++++++++-------- tests/nixos/default.nix | 2 +- tests/nixos/functional/common.nix | 4 +-- 5 files changed, 59 insertions(+), 59 deletions(-) diff --git a/flake.nix b/flake.nix index 193cd3875..084cec010 100644 --- a/flake.nix +++ b/flake.nix @@ -143,14 +143,14 @@ # without "polluting" the top level "`pkgs`" attrset. # This also has the benefit of providing us with a distinct set of packages # we can iterate over. - nixComponents = + nixComponents2 = lib.makeScopeWithSplicing' { inherit (final) splicePackages; - inherit (final.nixDependencies) newScope; + inherit (final.nixDependencies2) newScope; } { - otherSplices = final.generateSplicesForMkScope "nixComponents"; + otherSplices = final.generateSplicesForMkScope "nixComponents2"; f = import ./packaging/components.nix { inherit (final) lib; inherit officialRelease; @@ -161,22 +161,22 @@ }; # The dependencies are in their own scope, so that they don't have to be - # in Nixpkgs top level `pkgs` or `nixComponents`. - nixDependencies = + # in Nixpkgs top level `pkgs` or `nixComponents2`. + nixDependencies2 = lib.makeScopeWithSplicing' { inherit (final) splicePackages; - inherit (final) newScope; # layered directly on pkgs, unlike nixComponents above + inherit (final) newScope; # layered directly on pkgs, unlike nixComponents2 above } { - otherSplices = final.generateSplicesForMkScope "nixDependencies"; + otherSplices = final.generateSplicesForMkScope "nixDependencies2"; f = import ./packaging/dependencies.nix { inherit inputs stdenv; pkgs = final; }; }; - nix = final.nixComponents.nix-cli; + nix = final.nixComponents2.nix-cli; # See https://github.com/NixOS/nixpkgs/pull/214409 # Remove when fixed in this flake's nixpkgs @@ -277,7 +277,7 @@ # memory leaks with detect_leaks=0. "" = rec { nixpkgs = nixpkgsFor.${system}.native; - nixComponents = nixpkgs.nixComponents.overrideScope ( + nixComponents = nixpkgs.nixComponents2.overrideScope ( nixCompFinal: nixCompPrev: { mesonComponentOverrides = _finalAttrs: prevAttrs: { mesonFlags = @@ -305,7 +305,7 @@ nixpkgsPrefix: { nixpkgs, - nixComponents ? nixpkgs.nixComponents, + nixComponents ? nixpkgs.nixComponents2, }: flatMapAttrs nixComponents ( pkgName: pkg: @@ -335,9 +335,9 @@ binaryTarball = self.hydraJobs.binaryTarball.${system}; # TODO probably should be `nix-cli` nix = self.packages.${system}.nix-everything; - nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual; - nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; - nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + nix-manual = nixpkgsFor.${system}.native.nixComponents2.nix-manual; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-external-api-docs; } # We need to flatten recursive attribute sets of derivations to pass `flake check`. // @@ -391,9 +391,9 @@ }: { # These attributes go right into `packages.`. - "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; - "${pkgName}-static" = nixpkgsFor.${system}.native.pkgsStatic.nixComponents.${pkgName}; - "${pkgName}-llvm" = nixpkgsFor.${system}.native.pkgsLLVM.nixComponents.${pkgName}; + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents2.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.native.pkgsStatic.nixComponents2.${pkgName}; + "${pkgName}-llvm" = nixpkgsFor.${system}.native.pkgsLLVM.nixComponents2.${pkgName}; } // lib.optionalAttrs supportsCross ( flatMapAttrs (lib.genAttrs crossSystems (_: { })) ( @@ -401,7 +401,7 @@ { }: { # These attributes go right into `packages.`. - "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents2.${pkgName}; } ) ) @@ -411,7 +411,7 @@ { # These attributes go right into `packages.`. "${pkgName}-${stdenvName}" = - nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.nixComponents.${pkgName}; + nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.nixComponents2.${pkgName}; } ) ) diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index ae491f5b6..be760496a 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -5,11 +5,11 @@ { pkgs }: -pkgs.nixComponents.nix-util.overrideAttrs ( +pkgs.nixComponents2.nix-util.overrideAttrs ( attrs: let - stdenv = pkgs.nixDependencies.stdenv; + stdenv = pkgs.nixDependencies2.stdenv; buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; modular = devFlake.getSystem stdenv.buildPlatform.system; transformFlag = @@ -84,26 +84,26 @@ pkgs.nixComponents.nix-util.overrideAttrs ( }; mesonFlags = - map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) - ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) - ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags) ++ lib.optionals havePerl ( - map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags) + map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags) ) - ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) - ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags); + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags); nativeBuildInputs = attrs.nativeBuildInputs or [ ] - ++ pkgs.nixComponents.nix-util.nativeBuildInputs - ++ pkgs.nixComponents.nix-store.nativeBuildInputs - ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs - ++ pkgs.nixComponents.nix-expr.nativeBuildInputs - ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs - ++ lib.optionals buildCanExecuteHost pkgs.nixComponents.nix-manual.externalNativeBuildInputs - ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs - ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs - ++ pkgs.nixComponents.nix-functional-tests.externalNativeBuildInputs + ++ pkgs.nixComponents2.nix-util.nativeBuildInputs + ++ pkgs.nixComponents2.nix-store.nativeBuildInputs + ++ pkgs.nixComponents2.nix-fetchers.nativeBuildInputs + ++ pkgs.nixComponents2.nix-expr.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents2.nix-perl-bindings.nativeBuildInputs + ++ lib.optionals buildCanExecuteHost pkgs.nixComponents2.nix-manual.externalNativeBuildInputs + ++ pkgs.nixComponents2.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents2.nix-external-api-docs.nativeBuildInputs + ++ pkgs.nixComponents2.nix-functional-tests.externalNativeBuildInputs ++ lib.optional ( !buildCanExecuteHost # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 @@ -129,14 +129,14 @@ pkgs.nixComponents.nix-util.overrideAttrs ( buildInputs = attrs.buildInputs or [ ] - ++ pkgs.nixComponents.nix-util.buildInputs - ++ pkgs.nixComponents.nix-store.buildInputs - ++ pkgs.nixComponents.nix-store-tests.externalBuildInputs - ++ pkgs.nixComponents.nix-fetchers.buildInputs - ++ pkgs.nixComponents.nix-expr.buildInputs - ++ pkgs.nixComponents.nix-expr.externalPropagatedBuildInputs - ++ pkgs.nixComponents.nix-cmd.buildInputs - ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.externalBuildInputs + ++ pkgs.nixComponents2.nix-util.buildInputs + ++ pkgs.nixComponents2.nix-store.buildInputs + ++ pkgs.nixComponents2.nix-store-tests.externalBuildInputs + ++ pkgs.nixComponents2.nix-fetchers.buildInputs + ++ pkgs.nixComponents2.nix-expr.buildInputs + ++ pkgs.nixComponents2.nix-expr.externalPropagatedBuildInputs + ++ pkgs.nixComponents2.nix-cmd.buildInputs + ++ lib.optionals havePerl pkgs.nixComponents2.nix-perl-bindings.externalBuildInputs ++ lib.optional havePerl pkgs.perl; } ) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index ee4cabe62..27c09d9c9 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -19,14 +19,14 @@ let testNixVersions = pkgs: daemon: - pkgs.nixComponents.nix-functional-tests.override { + pkgs.nixComponents2.nix-functional-tests.override { pname = "nix-daemon-compat-tests"; version = "${pkgs.nix.version}-with-daemon-${daemon.version}"; test-daemon = daemon; }; - # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # Technically we could just return `pkgs.nixComponents2`, but for Hydra it's # convention to transpose it, and to transpose it efficiently, we need to # enumerate them manually, so that we don't evaluate unnecessary package sets. # See listingIsComplete below. @@ -86,7 +86,7 @@ in } (_: null); actualPkgs = lib.concatMapAttrs ( k: v: if lib.strings.hasPrefix "nix-" k then { ${k} = null; } else { } - ) nixpkgsFor.${arbitrarySystem}.native.nixComponents; + ) nixpkgsFor.${arbitrarySystem}.native.nixComponents2; diff = lib.concatStringsSep "\n" ( lib.concatLists ( lib.mapAttrsToList ( @@ -112,7 +112,7 @@ in # Binary package for various platforms. build = forAllPackages ( - pkgName: forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName}) + pkgName: forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.${pkgName}) ); shellInputs = removeAttrs (forAllSystems ( @@ -122,7 +122,7 @@ in buildStatic = forAllPackages ( pkgName: lib.genAttrs linux64BitSystems ( - system: nixpkgsFor.${system}.native.pkgsStatic.nixComponents.${pkgName} + system: nixpkgsFor.${system}.native.pkgsStatic.nixComponents2.${pkgName} ) ); @@ -139,7 +139,7 @@ in forAllCrossSystems ( crossSystem: lib.genAttrs [ "x86_64-linux" ] ( - system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName} + system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents2.${pkgName} ) ) ) @@ -149,7 +149,7 @@ in let components = forAllSystems ( system: - nixpkgsFor.${system}.native.nixComponents.overrideScope ( + nixpkgsFor.${system}.native.nixComponents2.overrideScope ( self: super: { nix-expr = super.nix-expr.override { enableGC = false; }; } @@ -158,7 +158,7 @@ in in forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName})); - buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-cli); + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-cli); # Toggles some settings for better coverage. Windows needs these # library combinations, and Debian build Nix with GNU readline too. @@ -166,7 +166,7 @@ in let components = forAllSystems ( system: - nixpkgsFor.${system}.native.nixComponents.overrideScope ( + nixpkgsFor.${system}.native.nixComponents2.overrideScope ( self: super: { nix-cmd = super.nix-cmd.override { enableMarkdown = false; @@ -179,7 +179,7 @@ in forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName})); # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-perl-bindings); # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of @@ -229,13 +229,13 @@ in # }; # Nix's manual - manual = nixpkgsFor.x86_64-linux.native.nixComponents.nix-manual; + manual = nixpkgsFor.x86_64-linux.native.nixComponents2.nix-manual; # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents2.nix-internal-api-docs; # API docs for Nix's C bindings. - external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents2.nix-external-api-docs; # System tests. tests = diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 92f89d8db..3e2d20a71 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -31,7 +31,7 @@ let nixpkgs.pkgs = nixpkgsFor.${system}.native; nix.checkAllErrors = false; # TODO: decide which packaging stage to use. `nix-cli` is efficient, but not the same as the user-facing `everything.nix` package (`default`). Perhaps a good compromise is `everything.nix` + `noTests` defined above? - nix.package = nixpkgsFor.${system}.native.nixComponents.nix-cli; + nix.package = nixpkgsFor.${system}.native.nixComponents2.nix-cli; # Evaluate VMs faster documentation.enable = false; diff --git a/tests/nixos/functional/common.nix b/tests/nixos/functional/common.nix index f3cab4725..a2067c07d 100644 --- a/tests/nixos/functional/common.nix +++ b/tests/nixos/functional/common.nix @@ -49,11 +49,11 @@ in cd ~ - cp -r ${pkgs.nixComponents.nix-functional-tests.src} nix + cp -r ${pkgs.nixComponents2.nix-functional-tests.src} nix chmod -R +w nix chmod u+w nix/.version - echo ${pkgs.nixComponents.version} > nix/.version + echo ${pkgs.nixComponents2.version} > nix/.version export isTestOnNixOS=1 From 363ee6dfcb0029d13a6bd16223b19db3ffe9e32e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 20 Apr 2025 22:43:06 +0200 Subject: [PATCH 246/396] Fix non-virtual destructor warning --- src/libfetchers/include/nix/fetchers/input-cache.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libfetchers/include/nix/fetchers/input-cache.hh b/src/libfetchers/include/nix/fetchers/input-cache.hh index a7ca34487..869c7d41e 100644 --- a/src/libfetchers/include/nix/fetchers/input-cache.hh +++ b/src/libfetchers/include/nix/fetchers/input-cache.hh @@ -26,6 +26,8 @@ struct InputCache virtual void clear() = 0; static ref create(); + + virtual ~InputCache() = default; }; } From ef368068984feb73bae4fef5ecef5c9419a5a4de Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Apr 2025 17:20:54 -0400 Subject: [PATCH 247/396] Explain the use of "2" in the overlay Co-authored-by: Robert Hensing --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index 084cec010..a8759d042 100644 --- a/flake.nix +++ b/flake.nix @@ -143,6 +143,7 @@ # without "polluting" the top level "`pkgs`" attrset. # This also has the benefit of providing us with a distinct set of packages # we can iterate over. + # The `2` suffix is here because otherwise it interferes with `nixVersions.latest`, which is used in daemon compat tests. nixComponents2 = lib.makeScopeWithSplicing' { @@ -162,6 +163,7 @@ # The dependencies are in their own scope, so that they don't have to be # in Nixpkgs top level `pkgs` or `nixComponents2`. + # The `2` suffix is here because otherwise it interferes with `nixVersions.latest`, which is used in daemon compat tests. nixDependencies2 = lib.makeScopeWithSplicing' { From 047f2bc1af17bc173ebe0e7c7f55e4741bc2b131 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 20 Apr 2025 22:42:12 +0200 Subject: [PATCH 248/396] refactor: Extract RETRY_TIME constants in filetransfer --- src/libstore/filetransfer.cc | 5 ++++- src/libstore/include/nix/store/filetransfer.hh | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 5a298a658..8fc4f14f2 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -33,6 +33,9 @@ using namespace std::string_literals; namespace nix { +const unsigned int RETRY_TIME_MS_DEFAULT = 250; +const unsigned int RETRY_TIME_MS_TOO_MANY_REQUESTS = 60000; + FileTransferSettings fileTransferSettings; static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); @@ -453,7 +456,7 @@ struct curlFileTransfer : public FileTransfer err = Forbidden; } else if (httpStatus == 429) { // 429 means too many requests, so we retry (with a substantially longer delay) - retryTimeMs = 60000; + retryTimeMs = RETRY_TIME_MS_TOO_MANY_REQUESTS; } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408) { // Most 4xx errors are client errors and are probably not worth retrying: // * 408 means the server timed out waiting for us, so we try again diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index f9b1f620f..10c3ec7ef 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -58,6 +58,8 @@ struct FileTransferSettings : Config extern FileTransferSettings fileTransferSettings; +extern const unsigned int RETRY_TIME_MS_DEFAULT; + struct FileTransferRequest { std::string uri; @@ -67,7 +69,7 @@ struct FileTransferRequest bool head = false; bool post = false; size_t tries = fileTransferSettings.tries; - unsigned int baseRetryTimeMs = 250; + unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT; ActivityId parentAct; bool decompress = true; std::optional data; From 4e586149df4dd3eb4c021ca0bcfa3ab286058cb5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Apr 2025 16:16:34 -0400 Subject: [PATCH 249/396] Get rid of `LocalDerivationGoal` I split it out before to try to separate the building logic, but now we have the much better `DerivationBuilder` abstraction for that. With that change, I think `LocalDerivationGoal` has outlived its usefulness. We just inline it back into `DerivationGoal`, and do so with minimal `#ifdef` for Windows. Note that the order of statements in `~DerivationGoal` is different than it was after the `~LocalDerivationGoal` split, but it is *restored* to the way it original was before --- evidently I did the split slightly wrong, but nobody noticed, probably because the order doesn't actually matter. Co-authored-by: Robert Hensing --- maintainers/flake-module.nix | 2 - src/libstore/build/derivation-goal.cc | 183 ++++++++++++- src/libstore/build/worker.cc | 17 +- .../nix/store/build/derivation-goal.hh | 10 +- src/libstore/include/nix/store/local-store.hh | 1 - src/libstore/unix/build/derivation-builder.cc | 28 +- .../unix/build/local-derivation-goal.cc | 254 ------------------ .../nix/store/build/derivation-builder.hh | 6 +- .../nix/store/build/local-derivation-goal.hh | 26 -- .../unix/include/nix/store/meson.build | 1 - src/libstore/unix/meson.build | 1 - 11 files changed, 198 insertions(+), 331 deletions(-) delete mode 100644 src/libstore/unix/build/local-derivation-goal.cc delete mode 100644 src/libstore/unix/include/nix/store/build/local-derivation-goal.hh diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index a9d10386f..5e414c697 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -286,8 +286,6 @@ ''^src/libstore/unix/build/hook-instance\.cc$'' ''^src/libstore/unix/build/derivation-builder\.cc$'' ''^src/libstore/unix/include/nix/store/build/derivation-builder\.hh$'' - ''^src/libstore/unix/build/local-derivation-goal\.cc$'' - ''^src/libstore/unix/include/nix/store/build/local-derivation-goal\.hh$'' ''^src/libstore/build/substitution-goal\.cc$'' ''^src/libstore/include/nix/store/build/substitution-goal\.hh$'' ''^src/libstore/build/worker\.cc$'' diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b19b0efeb..11162a1ee 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1,6 +1,7 @@ #include "nix/store/build/derivation-goal.hh" #ifndef _WIN32 // TODO enable build hook on Windows # include "nix/store/build/hook-instance.hh" +# include "nix/store/build/derivation-builder.hh" #endif #include "nix/util/processes.hh" #include "nix/util/config-global.hh" @@ -68,6 +69,13 @@ DerivationGoal::~DerivationGoal() { /* Careful: we should never ever throw an exception from a destructor. */ + try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } +#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows + if (builder) { + try { builder->stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } + try { builder->deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } + } +#endif try { closeLogFile(); } catch (...) { ignoreExceptionInDestructor(); } } @@ -87,6 +95,22 @@ void DerivationGoal::killChild() #ifndef _WIN32 // TODO enable build hook on Windows hook.reset(); #endif +#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows + if (builder && builder->pid != -1) { + worker.childTerminated(this); + + /* If we're using a build user, then there is a tricky race + condition: if we kill the build user before the child has + done its setuid() to the build user uid, then it won't be + killed, and we'll potentially lock up in pid.wait(). So + also send a conventional kill to the child. */ + ::kill(-builder->pid, SIGKILL); /* ignore the result */ + + builder->killSandbox(true); + + builder->pid.wait(); + } +#endif } @@ -666,18 +690,152 @@ Goal::Co DerivationGoal::tryToBuild() actLock.reset(); co_await yield(); - co_return tryLocalBuild(); -} -Goal::Co DerivationGoal::tryLocalBuild() { - throw Error( - R"( - Unable to build with a primary store that isn't a local store; - either pass a different '--store' or enable remote builds. + if (!dynamic_cast(&worker.store)) { + throw Error( + R"( + Unable to build with a primary store that isn't a local store; + either pass a different '--store' or enable remote builds. - For more information check 'man nix.conf' and search for '/machines'. - )" - ); + For more information check 'man nix.conf' and search for '/machines'. + )" + ); + } + +#ifdef _WIN32 // TODO enable `DerivationBuilder` on Windows + throw UnimplementedError("building derivations is not yet implemented on Windows"); +#else + + // Will continue here while waiting for a build user below + while (true) { + + assert(!hook); + + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= settings.maxBuildJobs) { + outputLocks.unlock(); + co_await waitForBuildSlot(); + co_return tryToBuild(); + } + + if (!builder) { + /** + * Local implementation of these virtual methods, consider + * this just a record of lambdas. + */ + struct DerivationGoalCallbacks : DerivationBuilderCallbacks + { + DerivationGoal & goal; + + DerivationGoalCallbacks(DerivationGoal & goal, std::unique_ptr & builder) + : goal{goal} + {} + + ~DerivationGoalCallbacks() override = default; + + void childStarted(Descriptor builderOut) override + { + goal.worker.childStarted(goal.shared_from_this(), {builderOut}, true, true); + } + + void childTerminated() override + { + goal.worker.childTerminated(&goal); + } + + void noteHashMismatch() override + { + goal.worker.hashMismatch = true; + } + + void noteCheckMismatch() override + { + goal.worker.checkMismatch = true; + } + + void markContentsGood(const StorePath & path) override + { + goal.worker.markContentsGood(path); + } + + Path openLogFile() override { + return goal.openLogFile(); + } + void closeLogFile() override { + goal.closeLogFile(); + } + SingleDrvOutputs assertPathValidity() override { + return goal.assertPathValidity(); + } + void appendLogTailErrorMsg(std::string & msg) override { + goal.appendLogTailErrorMsg(msg); + } + }; + + /* If we have to wait and retry (see below), then `builder` will + already be created, so we don't need to create it again. */ + builder = makeDerivationBuilder( + worker.store, + std::make_unique(*this, builder), + DerivationBuilderParams { + drvPath, + buildMode, + buildResult, + *drv, + parsedDrv.get(), + *drvOptions, + inputPaths, + initialOutputs, + }); + } + + if (!builder->prepareBuild()) { + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); + co_await waitForAWhile(); + continue; + } + + break; + } + + actLock.reset(); + + try { + + /* Okay, we have to build. */ + builder->startBuilder(); + + } catch (BuildError & e) { + outputLocks.unlock(); + builder->buildUser.reset(); + worker.permanentFailure = true; + co_return done(BuildResult::InputRejected, {}, std::move(e)); + } + + started(); + co_await Suspend{}; + + trace("build done"); + + auto res = builder->unprepareBuild(); + // N.B. cannot use `std::visit` with co-routine return + if (auto * ste = std::get_if<0>(&res)) { + outputLocks.unlock(); + co_return done(std::move(ste->first), {}, std::move(ste->second)); + } else if (auto * builtOutputs = std::get_if<1>(&res)) { + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + co_return done(BuildResult::Built, std::move(*builtOutputs)); + } else { + unreachable(); + } +#endif } @@ -1207,7 +1365,10 @@ bool DerivationGoal::isReadDesc(Descriptor fd) #ifdef _WIN32 // TODO enable build hook on Windows return false; #else - return fd == hook->builderOut.readSide.get(); + return + (hook && fd == hook->builderOut.readSide.get()) + || + (builder && fd == builder->builderOut.get()); #endif } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 66c31d39e..dd3692f41 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -5,7 +5,6 @@ #include "nix/store/build/drv-output-substitution-goal.hh" #include "nix/store/build/derivation-goal.hh" #ifndef _WIN32 // TODO Enable building on Windows -# include "nix/store/build/local-derivation-goal.hh" # include "nix/store/build/hook-instance.hh" #endif #include "nix/util/signals.hh" @@ -65,13 +64,7 @@ std::shared_ptr Worker::makeDerivationGoal(const StorePath & drv const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { - return -#ifndef _WIN32 // TODO Enable building on Windows - dynamic_cast(&store) - ? makeLocalDerivationGoal(drvPath, wantedOutputs, *this, buildMode) - : -#endif - std::make_shared(drvPath, wantedOutputs, *this, buildMode); + return std::make_shared(drvPath, wantedOutputs, *this, buildMode); }); } @@ -79,13 +72,7 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { - return -#ifndef _WIN32 // TODO Enable building on Windows - dynamic_cast(&store) - ? makeLocalDerivationGoal(drvPath, drv, wantedOutputs, *this, buildMode) - : -#endif - std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); + return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); }); } diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index ecd7e7b97..e14ab9dcf 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -16,6 +16,7 @@ using std::map; #ifndef _WIN32 // TODO enable build hook on Windows struct HookInstance; +struct DerivationBuilder; #endif typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; @@ -154,6 +155,8 @@ struct DerivationGoal : public Goal * The build hook. */ std::unique_ptr hook; + + std::unique_ptr builder; #endif BuildMode buildMode; @@ -180,7 +183,7 @@ struct DerivationGoal : public Goal DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); - virtual ~DerivationGoal(); + ~DerivationGoal(); void timedOut(Error && ex) override; @@ -198,7 +201,6 @@ struct DerivationGoal : public Goal Co haveDerivation(); Co gaveUpOnSubstitution(); Co tryToBuild(); - virtual Co tryLocalBuild(); Co hookDone(); Co resolvedFinished(); @@ -218,7 +220,7 @@ struct DerivationGoal : public Goal */ void closeLogFile(); - virtual bool isReadDesc(Descriptor fd); + bool isReadDesc(Descriptor fd); /** * Callback used by the worker to write to the log. @@ -252,7 +254,7 @@ struct DerivationGoal : public Goal /** * Forcibly kill the child process, if any. */ - virtual void killChild(); + void killChild(); Co repairClosure(); diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index a9109b43d..204720c34 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -398,7 +398,6 @@ private: void addBuildLog(const StorePath & drvPath, std::string_view log) override; - friend struct LocalDerivationGoal; friend struct PathSubstitutionGoal; friend struct SubstitutionGoal; friend struct DerivationGoal; diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 95588bf08..01e4bca8e 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -96,17 +96,17 @@ class DerivationBuilderImpl : public DerivationBuilder, DerivationBuilderParams { Store & store; - DerivationBuilderCallbacks & miscMethods; + std::unique_ptr miscMethods; public: DerivationBuilderImpl( Store & store, - DerivationBuilderCallbacks & miscMethods, + std::unique_ptr miscMethods, DerivationBuilderParams params) : DerivationBuilderParams{std::move(params)} , store{store} - , miscMethods{miscMethods} + , miscMethods{std::move(miscMethods)} { } LocalStore & getLocalStore(); @@ -382,12 +382,12 @@ private: std::unique_ptr makeDerivationBuilder( Store & store, - DerivationBuilderCallbacks & miscMethods, + std::unique_ptr miscMethods, DerivationBuilderParams params) { return std::make_unique( store, - miscMethods, + std::move(miscMethods), std::move(params)); } @@ -550,13 +550,13 @@ std::variant, SingleDrvOutputs> Derivation buildResult.stopTime = time(0); /* So the child is gone now. */ - miscMethods.childTerminated(); + miscMethods->childTerminated(); /* Close the read side of the logger pipe. */ builderOut.close(); /* Close the log file. */ - miscMethods.closeLogFile(); + miscMethods->closeLogFile(); /* When running under a build user, make sure that all processes running under that uid are gone. This is to prevent a @@ -589,7 +589,7 @@ std::variant, SingleDrvOutputs> Derivation Magenta(store.printStorePath(drvPath)), statusToString(status)); - miscMethods.appendLogTailErrorMsg(msg); + miscMethods->appendLogTailErrorMsg(msg); if (diskFull) msg += "\nnote: build failure may have been caused by lack of free disk space"; @@ -1175,7 +1175,7 @@ void DerivationBuilderImpl::startBuilder() printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); /* Create the log file. */ - [[maybe_unused]] Path logFile = miscMethods.openLogFile(); + [[maybe_unused]] Path logFile = miscMethods->openLogFile(); /* Create a pseudoterminal to get the output of the builder. */ builderOut = posix_openpt(O_RDWR | O_NOCTTY); @@ -1385,7 +1385,7 @@ void DerivationBuilderImpl::startBuilder() /* parent */ pid.setSeparatePG(true); - miscMethods.childStarted(); + miscMethods->childStarted(builderOut.get()); processSandboxSetupMessages(); } @@ -2673,7 +2673,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() if (wanted != got) { /* Throw an error after registering the path as valid. */ - miscMethods.noteHashMismatch(); + miscMethods->noteHashMismatch(); delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", store.printStorePath(drvPath), @@ -2761,7 +2761,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() if (!store.isValidPath(newInfo.path)) continue; ValidPathInfo oldInfo(*store.queryPathInfo(newInfo.path)); if (newInfo.narHash != oldInfo.narHash) { - miscMethods.noteCheckMismatch(); + miscMethods->noteCheckMismatch(); if (settings.runDiffHook || settings.keepFailed) { auto dst = store.toRealPath(finalDestPath + checkSuffix); deletePath(dst); @@ -2798,7 +2798,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() } localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() - miscMethods.markContentsGood(newInfo.path); + miscMethods->markContentsGood(newInfo.path); newInfo.deriver = drvPath; newInfo.ultimate = true; @@ -2821,7 +2821,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() also a source for non-determinism. */ if (delayedException) std::rethrow_exception(delayedException); - return miscMethods.assertPathValidity(); + return miscMethods->assertPathValidity(); } /* Apply output checks. */ diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc deleted file mode 100644 index 188066679..000000000 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ /dev/null @@ -1,254 +0,0 @@ -#include "nix/store/build/local-derivation-goal.hh" -#include "nix/store/local-store.hh" -#include "nix/util/processes.hh" -#include "nix/store/build/worker.hh" -#include "nix/util/util.hh" -#include "nix/store/restricted-store.hh" -#include "nix/store/build/derivation-builder.hh" - -#include -#include -#include -#include -#include -#include -#include - -#include "store-config-private.hh" - -#if HAVE_STATVFS -#include -#endif - -#include -#include - -namespace nix { - -/** - * This hooks up `DerivationBuilder` to the scheduler / goal machinary. - * - * @todo Eventually, this shouldn't exist, because `DerivationGoal` can - * just choose to use `DerivationBuilder` or its remote-building - * equalivalent directly, at the "value level" rather than "class - * inheritance hierarchy" level. - */ -struct LocalDerivationGoal : DerivationGoal, DerivationBuilderCallbacks -{ - std::unique_ptr builder; - - LocalDerivationGoal(const StorePath & drvPath, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode) - : DerivationGoal{drvPath, wantedOutputs, worker, buildMode} - {} - - LocalDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal) - : DerivationGoal{drvPath, drv, wantedOutputs, worker, buildMode} - {} - - virtual ~LocalDerivationGoal() override; - - /** - * The additional states. - */ - Goal::Co tryLocalBuild() override; - - bool isReadDesc(int fd) override; - - /** - * Forcibly kill the child process, if any. - * - * Called by destructor, can't be overridden - */ - void killChild() override final; - - void childStarted() override; - void childTerminated() override; - - void noteHashMismatch(void) override; - void noteCheckMismatch(void) override; - - void markContentsGood(const StorePath &) override; - - // Fake overrides to instantiate identically-named virtual methods - - Path openLogFile() override { - return DerivationGoal::openLogFile(); - } - void closeLogFile() override { - DerivationGoal::closeLogFile(); - } - SingleDrvOutputs assertPathValidity() override { - return DerivationGoal::assertPathValidity(); - } - void appendLogTailErrorMsg(std::string & msg) override { - DerivationGoal::appendLogTailErrorMsg(msg); - } -}; - -std::shared_ptr makeLocalDerivationGoal( - const StorePath & drvPath, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode) -{ - return std::make_shared(drvPath, wantedOutputs, worker, buildMode); -} - -std::shared_ptr makeLocalDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode) -{ - return std::make_shared(drvPath, drv, wantedOutputs, worker, buildMode); -} - - -LocalDerivationGoal::~LocalDerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - if (builder) { - try { builder->deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } - } - try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } - if (builder) { - try { builder->stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } - } -} - - -void LocalDerivationGoal::killChild() -{ - if (builder && builder->pid != -1) { - worker.childTerminated(this); - - /* If we're using a build user, then there is a tricky race - condition: if we kill the build user before the child has - done its setuid() to the build user uid, then it won't be - killed, and we'll potentially lock up in pid.wait(). So - also send a conventional kill to the child. */ - ::kill(-builder->pid, SIGKILL); /* ignore the result */ - - builder->killSandbox(true); - - builder->pid.wait(); - } - - DerivationGoal::killChild(); -} - - -void LocalDerivationGoal::childStarted() -{ - assert(builder); - worker.childStarted(shared_from_this(), {builder->builderOut.get()}, true, true); -} - -void LocalDerivationGoal::childTerminated() -{ - worker.childTerminated(this); -} - -void LocalDerivationGoal::noteHashMismatch() -{ - worker.hashMismatch = true; -} - - -void LocalDerivationGoal::noteCheckMismatch() -{ - worker.checkMismatch = true; -} - - -void LocalDerivationGoal::markContentsGood(const StorePath & path) -{ - worker.markContentsGood(path); -} - - -Goal::Co LocalDerivationGoal::tryLocalBuild() -{ - assert(!hook); - - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs) { - outputLocks.unlock(); - co_await waitForBuildSlot(); - co_return tryToBuild(); - } - - if (!builder) { - /* If we have to wait and retry (see below), then `builder` will - already be created, so we don't need to create it again. */ - builder = makeDerivationBuilder( - worker.store, - static_cast(*this), - DerivationBuilderParams { - DerivationGoal::drvPath, - DerivationGoal::buildMode, - DerivationGoal::buildResult, - *DerivationGoal::drv, - DerivationGoal::parsedDrv.get(), - *DerivationGoal::drvOptions, - DerivationGoal::inputPaths, - DerivationGoal::initialOutputs, - }); - } - - if (!builder->prepareBuild()) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); - co_await waitForAWhile(); - co_return tryLocalBuild(); - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - builder->startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - builder->buildUser.reset(); - worker.permanentFailure = true; - co_return done(BuildResult::InputRejected, {}, std::move(e)); - } - - started(); - co_await Suspend{}; - - trace("build done"); - - auto res = builder->unprepareBuild(); - // N.B. cannot use `std::visit` with co-routine return - if (auto * ste = std::get_if<0>(&res)) { - outputLocks.unlock(); - co_return done(std::move(ste->first), {}, std::move(ste->second)); - } else if (auto * builtOutputs = std::get_if<1>(&res)) { - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - co_return done(BuildResult::Built, std::move(*builtOutputs)); - } else { - unreachable(); - } -} - - -bool LocalDerivationGoal::isReadDesc(int fd) -{ - return (hook && DerivationGoal::isReadDesc(fd)) || - (!hook && builder && fd == builder->builderOut.get()); -} - -} diff --git a/src/libstore/unix/include/nix/store/build/derivation-builder.hh b/src/libstore/unix/include/nix/store/build/derivation-builder.hh index 5e5532fe0..d6c40060a 100644 --- a/src/libstore/unix/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/unix/include/nix/store/build/derivation-builder.hh @@ -85,6 +85,8 @@ struct DerivationBuilderParams */ struct DerivationBuilderCallbacks { + virtual ~DerivationBuilderCallbacks() = default; + /** * Open a log file and a pipe to it. */ @@ -110,7 +112,7 @@ struct DerivationBuilderCallbacks * * @todo this should be reworked */ - virtual void childStarted() = 0; + virtual void childStarted(Descriptor builderOut) = 0; /** * @todo this should be reworked @@ -201,7 +203,7 @@ struct DerivationBuilder : RestrictionContext std::unique_ptr makeDerivationBuilder( Store & store, - DerivationBuilderCallbacks & miscMethods, + std::unique_ptr miscMethods, DerivationBuilderParams params); } diff --git a/src/libstore/unix/include/nix/store/build/local-derivation-goal.hh b/src/libstore/unix/include/nix/store/build/local-derivation-goal.hh deleted file mode 100644 index 36aaa7857..000000000 --- a/src/libstore/unix/include/nix/store/build/local-derivation-goal.hh +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once -///@file - -#include "nix/store/build/derivation-goal.hh" - -namespace nix { - -/** - * Create a local derivation goal, see `DerivationGoal` for info on each - * constructor variant. - */ -std::shared_ptr makeLocalDerivationGoal( - const StorePath & drvPath, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - -/** - * Create a local derivation goal, see `DerivationGoal` for info on each - * constructor variant. - */ -std::shared_ptr makeLocalDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - -} diff --git a/src/libstore/unix/include/nix/store/meson.build b/src/libstore/unix/include/nix/store/meson.build index 03101f4bb..7cf973223 100644 --- a/src/libstore/unix/include/nix/store/meson.build +++ b/src/libstore/unix/include/nix/store/meson.build @@ -4,6 +4,5 @@ headers += files( 'build/child.hh', 'build/derivation-builder.hh', 'build/hook-instance.hh', - 'build/local-derivation-goal.hh', 'user-lock.hh', ) diff --git a/src/libstore/unix/meson.build b/src/libstore/unix/meson.build index 08d38742b..4b8a6b105 100644 --- a/src/libstore/unix/meson.build +++ b/src/libstore/unix/meson.build @@ -2,7 +2,6 @@ sources += files( 'build/child.cc', 'build/derivation-builder.cc', 'build/hook-instance.cc', - 'build/local-derivation-goal.cc', 'pathlocks.cc', 'user-lock.cc', ) From 16f640a9b2a2b7316f85a280a509607c9adc9a85 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Apr 2025 18:31:30 -0400 Subject: [PATCH 250/396] Inline `DerivationGoal::resolvedFinished` `resolvedDrvGoal` can just become a local variable! --- src/libstore/build/derivation-goal.cc | 140 +++++++++--------- .../nix/store/build/derivation-goal.hh | 7 - 2 files changed, 67 insertions(+), 80 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b19b0efeb..0b289d6ce 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -492,14 +492,79 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() worker.store.printStorePath(pathResolved), }); - resolvedDrvGoal = worker.makeDerivationGoal( + auto resolvedDrvGoal = worker.makeDerivationGoal( pathResolved, wantedOutputs, buildMode); { Goals waitees{resolvedDrvGoal}; co_await await(std::move(waitees)); } - co_return resolvedFinished(); + trace("resolved derivation finished"); + + auto resolvedDrv = *resolvedDrvGoal->drv; + auto & resolvedResult = resolvedDrvGoal->buildResult; + + SingleDrvOutputs builtOutputs; + + if (resolvedResult.success()) { + auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); + + StorePathSet outputPaths; + + for (auto & outputName : resolvedDrv.outputNames()) { + auto initialOutput = get(initialOutputs, outputName); + auto resolvedHash = get(resolvedHashes, outputName); + if ((!initialOutput) || (!resolvedHash)) + throw Error( + "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", + worker.store.printStorePath(drvPath), outputName); + + auto realisation = [&]{ + auto take1 = get(resolvedResult.builtOutputs, outputName); + if (take1) return *take1; + + /* The above `get` should work. But sateful tracking of + outputs in resolvedResult, this can get out of sync with the + store, which is our actual source of truth. For now we just + check the store directly if it fails. */ + auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, outputName }); + if (take2) return *take2; + + throw Error( + "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/realisation)", + worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName); + }(); + + if (!drv->type().isImpure()) { + auto newRealisation = realisation; + newRealisation.id = DrvOutput { initialOutput->outputHash, outputName }; + newRealisation.signatures.clear(); + if (!drv->type().isFixed()) { + auto & drvStore = worker.evalStore.isValidPath(drvPath) + ? worker.evalStore + : worker.store; + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore); + } + worker.store.signRealisation(newRealisation); + worker.store.registerDrvOutput(newRealisation); + } + outputPaths.insert(realisation.outPath); + builtOutputs.emplace(outputName, realisation); + } + + runPostBuildHook( + worker.store, + *logger, + drvPath, + outputPaths + ); + } + + auto status = resolvedResult.status; + if (status == BuildResult::AlreadyValid) + status = BuildResult::ResolvesToAlreadyValid; + + co_return done(status, std::move(builtOutputs)); } /* If we get this far, we know no dynamic drvs inputs */ @@ -970,77 +1035,6 @@ Goal::Co DerivationGoal::hookDone() co_return done(BuildResult::Built, std::move(builtOutputs)); } -Goal::Co DerivationGoal::resolvedFinished() -{ - trace("resolved derivation finished"); - - assert(resolvedDrvGoal); - auto resolvedDrv = *resolvedDrvGoal->drv; - auto & resolvedResult = resolvedDrvGoal->buildResult; - - SingleDrvOutputs builtOutputs; - - if (resolvedResult.success()) { - auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); - - StorePathSet outputPaths; - - for (auto & outputName : resolvedDrv.outputNames()) { - auto initialOutput = get(initialOutputs, outputName); - auto resolvedHash = get(resolvedHashes, outputName); - if ((!initialOutput) || (!resolvedHash)) - throw Error( - "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)", - worker.store.printStorePath(drvPath), outputName); - - auto realisation = [&]{ - auto take1 = get(resolvedResult.builtOutputs, outputName); - if (take1) return *take1; - - /* The above `get` should work. But sateful tracking of - outputs in resolvedResult, this can get out of sync with the - store, which is our actual source of truth. For now we just - check the store directly if it fails. */ - auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, outputName }); - if (take2) return *take2; - - throw Error( - "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", - worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName); - }(); - - if (!drv->type().isImpure()) { - auto newRealisation = realisation; - newRealisation.id = DrvOutput { initialOutput->outputHash, outputName }; - newRealisation.signatures.clear(); - if (!drv->type().isFixed()) { - auto & drvStore = worker.evalStore.isValidPath(drvPath) - ? worker.evalStore - : worker.store; - newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore); - } - worker.store.signRealisation(newRealisation); - worker.store.registerDrvOutput(newRealisation); - } - outputPaths.insert(realisation.outPath); - builtOutputs.emplace(outputName, realisation); - } - - runPostBuildHook( - worker.store, - *logger, - drvPath, - outputPaths - ); - } - - auto status = resolvedResult.status; - if (status == BuildResult::AlreadyValid) - status = BuildResult::ResolvesToAlreadyValid; - - co_return done(status, std::move(builtOutputs)); -} - HookReply DerivationGoal::tryBuildHook() { #ifdef _WIN32 // TODO enable build hook on Windows diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index ecd7e7b97..5f76206bd 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -40,11 +40,6 @@ struct DerivationGoal : public Goal /** The path of the derivation. */ StorePath drvPath; - /** - * The goal for the corresponding resolved derivation - */ - std::shared_ptr resolvedDrvGoal; - /** * The specific outputs that we need to build. */ @@ -201,8 +196,6 @@ struct DerivationGoal : public Goal virtual Co tryLocalBuild(); Co hookDone(); - Co resolvedFinished(); - /** * Is the build hook willing to perform the build? */ From aff7facdf61d9c5cec6000988fe925d447d1e079 Mon Sep 17 00:00:00 2001 From: Vit Gottwald Date: Tue, 22 Apr 2025 09:23:56 +0200 Subject: [PATCH 251/396] Use python3 packages in deps To be consistent with the sample that uses the packages directly in inline shebang. --- doc/manual/source/command-ref/nix-shell.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/command-ref/nix-shell.md b/doc/manual/source/command-ref/nix-shell.md index e95db9bea..03585fa2d 100644 --- a/doc/manual/source/command-ref/nix-shell.md +++ b/doc/manual/source/command-ref/nix-shell.md @@ -316,7 +316,7 @@ contains: ```nix with import {}; -runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" +runCommand "dummy" { buildInputs = [ python3 python3Packages.prettytable ]; } "" ``` The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. From aa96bf2faf1a6291b1fb47a8f1581b61f8c0ec6a Mon Sep 17 00:00:00 2001 From: Vit Gottwald Date: Tue, 22 Apr 2025 09:49:34 +0200 Subject: [PATCH 252/396] Fix perl example The perl example does not work with http://nixos.org because it redirects. Updating the url to https requires additional package. --- doc/manual/source/command-ref/nix-shell.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/command-ref/nix-shell.md b/doc/manual/source/command-ref/nix-shell.md index e95db9bea..9f371bff3 100644 --- a/doc/manual/source/command-ref/nix-shell.md +++ b/doc/manual/source/command-ref/nix-shell.md @@ -242,16 +242,18 @@ print(t) ``` Similarly, the following is a Perl script that specifies that it -requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages: +requires Perl and the `HTML::TokeParser::Simple`, `LWP` and +`LWP::Protocol::Https` packages: ```perl #! /usr/bin/env nix-shell #! nix-shell -i perl --packages perl perlPackages.HTMLTokeParserSimple perlPackages.LWP +#! nix-shell -i perl --packages perl perlPackages.HTMLTokeParserSimple perlPackages.LWP perlPackages.LWPProtocolHttps use HTML::TokeParser::Simple; # Fetch nixos.org and print all hrefs. -my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/'); +my $p = HTML::TokeParser::Simple->new(url => 'https://nixos.org/'); while (my $token = $p->get_tag("a")) { my $href = $token->get_attr("href"); From 9f94a1b9fb87a8d56c5d2d676551a23c589933a5 Mon Sep 17 00:00:00 2001 From: Vit Gottwald Date: Tue, 22 Apr 2025 10:45:15 +0200 Subject: [PATCH 253/396] Put every package on its own line --- doc/manual/source/command-ref/nix-shell.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/command-ref/nix-shell.md b/doc/manual/source/command-ref/nix-shell.md index 9f371bff3..455899d56 100644 --- a/doc/manual/source/command-ref/nix-shell.md +++ b/doc/manual/source/command-ref/nix-shell.md @@ -247,8 +247,11 @@ requires Perl and the `HTML::TokeParser::Simple`, `LWP` and ```perl #! /usr/bin/env nix-shell -#! nix-shell -i perl --packages perl perlPackages.HTMLTokeParserSimple perlPackages.LWP -#! nix-shell -i perl --packages perl perlPackages.HTMLTokeParserSimple perlPackages.LWP perlPackages.LWPProtocolHttps +#! nix-shell -i perl +#! nix-shell --packages perl +#! nix-shell --packages perlPackages.HTMLTokeParserSimple +#! nix-shell --packages perlPackages.LWP +#! nix-shell --packages perlPackages.LWPProtocolHttps use HTML::TokeParser::Simple; From bc67e47298022d6d0bcc270421b01e0697d63030 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Apr 2025 17:11:55 +0200 Subject: [PATCH 254/396] Move libflake/flake/* to libflake --- maintainers/flake-module.nix | 10 +++++----- src/libflake/{flake => }/config.cc | 0 src/libflake/{flake => }/flake-primops.cc | 0 src/libflake/{flake => }/flake.cc | 0 src/libflake/{flake => }/flakeref.cc | 0 src/libflake/{flake => }/lockfile.cc | 0 src/libflake/meson.build | 14 +++++++------- src/libflake/{flake => }/settings.cc | 0 src/libflake/{flake => }/url-name.cc | 0 9 files changed, 12 insertions(+), 12 deletions(-) rename src/libflake/{flake => }/config.cc (100%) rename src/libflake/{flake => }/flake-primops.cc (100%) rename src/libflake/{flake => }/flake.cc (100%) rename src/libflake/{flake => }/flakeref.cc (100%) rename src/libflake/{flake => }/lockfile.cc (100%) rename src/libflake/{flake => }/settings.cc (100%) rename src/libflake/{flake => }/url-name.cc (100%) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 5e414c697..de71be0d7 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -172,14 +172,14 @@ ''^src/libfetchers/include/nix/fetchers/tarball\.hh$'' ''^src/libfetchers/git\.cc$'' ''^src/libfetchers/mercurial\.cc$'' - ''^src/libflake/flake/config\.cc$'' - ''^src/libflake/flake/flake\.cc$'' + ''^src/libflake/config\.cc$'' + ''^src/libflake/flake\.cc$'' ''^src/libflake/include/nix/flake/flake\.hh$'' - ''^src/libflake/flake/flakeref\.cc$'' + ''^src/libflake/flakeref\.cc$'' ''^src/libflake/include/nix/flake/flakeref\.hh$'' - ''^src/libflake/flake/lockfile\.cc$'' + ''^src/libflake/lockfile\.cc$'' ''^src/libflake/include/nix/flake/lockfile\.hh$'' - ''^src/libflake/flake/url-name\.cc$'' + ''^src/libflake/url-name\.cc$'' ''^src/libmain/common-args\.cc$'' ''^src/libmain/include/nix/main/common-args\.hh$'' ''^src/libmain/loggers\.cc$'' diff --git a/src/libflake/flake/config.cc b/src/libflake/config.cc similarity index 100% rename from src/libflake/flake/config.cc rename to src/libflake/config.cc diff --git a/src/libflake/flake/flake-primops.cc b/src/libflake/flake-primops.cc similarity index 100% rename from src/libflake/flake/flake-primops.cc rename to src/libflake/flake-primops.cc diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake.cc similarity index 100% rename from src/libflake/flake/flake.cc rename to src/libflake/flake.cc diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flakeref.cc similarity index 100% rename from src/libflake/flake/flakeref.cc rename to src/libflake/flakeref.cc diff --git a/src/libflake/flake/lockfile.cc b/src/libflake/lockfile.cc similarity index 100% rename from src/libflake/flake/lockfile.cc rename to src/libflake/lockfile.cc diff --git a/src/libflake/meson.build b/src/libflake/meson.build index f4c034490..bc8533e15 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -39,13 +39,13 @@ foreach header : [ endforeach sources = files( - 'flake/config.cc', - 'flake/flake.cc', - 'flake/flakeref.cc', - 'flake/lockfile.cc', - 'flake/flake-primops.cc', - 'flake/settings.cc', - 'flake/url-name.cc', + 'config.cc', + 'flake.cc', + 'flakeref.cc', + 'lockfile.cc', + 'flake-primops.cc', + 'settings.cc', + 'url-name.cc', ) subdir('include/nix/flake') diff --git a/src/libflake/flake/settings.cc b/src/libflake/settings.cc similarity index 100% rename from src/libflake/flake/settings.cc rename to src/libflake/settings.cc diff --git a/src/libflake/flake/url-name.cc b/src/libflake/url-name.cc similarity index 100% rename from src/libflake/flake/url-name.cc rename to src/libflake/url-name.cc From d07541be29ba62c1cf6a83279ac3d41e9eabf839 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Apr 2025 17:28:09 +0200 Subject: [PATCH 255/396] DerivationBuilder: Fix compiler warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes ../src/libstore/unix/build/derivation-builder.cc:1130:86: warning: the compiler can assume that the address of ‘nix::DerivationBuilderParams::drv’ will never be NULL [-Waddress] 1130 | if (useChroot && settings.preBuildHook != "" && dynamic_cast(&drv)) { | ^~~~ Assuming this check was left over from the time `drv` could be a `BasicDerivation`. --- src/libstore/unix/build/derivation-builder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 01e4bca8e..2131b4e88 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1127,7 +1127,7 @@ void DerivationBuilderImpl::startBuilder() if (needsHashRewrite() && pathExists(homeDir)) throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - if (useChroot && settings.preBuildHook != "" && dynamic_cast(&drv)) { + if (useChroot && settings.preBuildHook != "") { printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); auto args = useChroot ? Strings({store.printStorePath(drvPath), chrootRootDir}) : Strings({ store.printStorePath(drvPath) }); From fd0835e7d7dcacc41f96306c5c9fb2075c37e31a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Apr 2025 17:41:35 +0200 Subject: [PATCH 256/396] Fix signedness error on FreeBSD https://hydra.nixos.org/build/295398446 --- src/libutil/current-process.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 926714ae8..4cc5a4218 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -57,7 +57,7 @@ size_t savedStackSize = 0; void setStackSize(size_t stackSize) { struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + if (getrlimit(RLIMIT_STACK, &limit) == 0 && static_cast(limit.rlim_cur) < stackSize) { savedStackSize = limit.rlim_cur; limit.rlim_cur = std::min(static_cast(stackSize), limit.rlim_max); if (setrlimit(RLIMIT_STACK, &limit) != 0) { From a9b62132210beadbd3905e42260b85bec7205de1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Apr 2025 17:50:45 +0200 Subject: [PATCH 257/396] Don't build MonitorFdHup on Windows https://hydra.nixos.org/build/295398462 --- src/libutil-tests/monitorfdhup.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libutil-tests/monitorfdhup.cc b/src/libutil-tests/monitorfdhup.cc index f9da4022d..8e6fed6f0 100644 --- a/src/libutil-tests/monitorfdhup.cc +++ b/src/libutil-tests/monitorfdhup.cc @@ -1,8 +1,10 @@ -#include "nix/util/util.hh" -#include "nix/util/monitor-fd.hh" +#ifndef _WIN32 -#include -#include +# include "nix/util/util.hh" +# include "nix/util/monitor-fd.hh" + +# include +# include namespace nix { TEST(MonitorFdHup, shouldNotBlock) @@ -16,3 +18,5 @@ TEST(MonitorFdHup, shouldNotBlock) } } } + +#endif From 1e5b1d99734933a8c2eec04037b420f1d9140b07 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 23 Apr 2025 13:14:28 +0200 Subject: [PATCH 258/396] Rename shellEscape -> escapeShellArgAlways This name is close to the Nixpkgs lib function `escapeShellArg`, making it easier to find. A friendlier function with the same behavior as lib could be added later. --- src/libstore/parsed-derivations.cc | 4 ++-- src/libutil/include/nix/util/util.hh | 7 ++++++- src/libutil/util.cc | 2 +- src/nix-build/nix-build.cc | 14 +++++++------- src/nix-store/nix-store.cc | 4 ++-- src/nix/develop.cc | 14 +++++++------- 6 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 61af101de..d6453c6db 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -112,7 +112,7 @@ std::string StructuredAttrs::writeShell(const nlohmann::json & json) auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { if (value.is_string()) - return shellEscape(value.get()); + return escapeShellArgAlways(value.get()); if (value.is_number()) { auto f = value.get(); @@ -160,7 +160,7 @@ std::string StructuredAttrs::writeShell(const nlohmann::json & json) for (auto & [key2, value2] : value.items()) { auto s3 = handleSimpleType(value2); if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(key2), *s3); + s2 += fmt("[%s]=%s ", escapeShellArgAlways(key2), *s3); } if (good) diff --git a/src/libutil/include/nix/util/util.hh b/src/libutil/include/nix/util/util.hh index 5a4530798..2361bf2e7 100644 --- a/src/libutil/include/nix/util/util.hh +++ b/src/libutil/include/nix/util/util.hh @@ -152,8 +152,13 @@ std::string toLower(std::string s); /** * Escape a string as a shell word. + * + * This always adds single quotes, even if escaping is not strictly necessary. + * So both + * - `"hello world"` -> `"'hello world'"`, which needs escaping because of the space + * - `"echo"` -> `"'echo'"`, which doesn't need escaping */ -std::string shellEscape(const std::string_view s); +std::string escapeShellArgAlways(const std::string_view s); /** diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ffd85ffbb..c9cc80fef 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -171,7 +171,7 @@ std::string toLower(std::string s) } -std::string shellEscape(const std::string_view s) +std::string escapeShellArgAlways(const std::string_view s) { std::string r; r.reserve(s.size() + 2); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 6815e0cdd..231b320ac 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -255,16 +255,16 @@ static void main_nix_build(int argc, char * * argv) std::ostringstream joined; for (const auto & i : savedArgs) - joined << shellEscape(i) << ' '; + joined << escapeShellArgAlways(i) << ' '; if (std::regex_search(interpreter, std::regex("ruby"))) { // Hack for Ruby. Ruby also examines the shebang. It tries to // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, shellEscape(script), toView(joined)); + envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, escapeShellArgAlways(script), toView(joined)); } else { - envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, shellEscape(script), toView(joined)); + envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, escapeShellArgAlways(script), toView(joined)); } } @@ -637,12 +637,12 @@ static void main_nix_build(int argc, char * * argv) "unset TZ; %6%" "shopt -s execfail;" "%7%", - shellEscape(tmpDir.path().string()), + escapeShellArgAlways(tmpDir.path().string()), (pure ? "" : "p=$PATH; "), (pure ? "" : "PATH=$PATH:$p; unset p; "), - shellEscape(dirOf(*shell)), - shellEscape(*shell), - (getenv("TZ") ? (std::string("export TZ=") + shellEscape(getenv("TZ")) + "; ") : ""), + escapeShellArgAlways(dirOf(*shell)), + escapeShellArgAlways(*shell), + (getenv("TZ") ? (std::string("export TZ=") + escapeShellArgAlways(getenv("TZ")) + "; ") : ""), envCommand); vomit("Sourcing nix-shell with file %s and contents:\n%s", rcfile, rc); writeFile(rcfile, rc); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 23d4071e9..128d57443 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -497,7 +497,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) /* Print each environment variable in the derivation in a format * that can be sourced by the shell. */ for (auto & i : drv.env) - logger->cout("export %1%; %1%=%2%\n", i.first, shellEscape(i.second)); + logger->cout("export %1%; %1%=%2%\n", i.first, escapeShellArgAlways(i.second)); /* Also output the arguments. This doesn't preserve whitespace in arguments. */ @@ -506,7 +506,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) for (auto & i : drv.args) { if (!first) cout << ' '; first = false; - cout << shellEscape(i); + cout << escapeShellArgAlways(i); } cout << "'\n"; } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 7e20addf7..a0b863792 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -156,20 +156,20 @@ struct BuildEnvironment for (auto & [name, value] : vars) { if (!ignoreVars.count(name)) { if (auto str = std::get_if(&value)) { - out << fmt("%s=%s\n", name, shellEscape(str->value)); + out << fmt("%s=%s\n", name, escapeShellArgAlways(str->value)); if (str->exported) out << fmt("export %s\n", name); } else if (auto arr = std::get_if(&value)) { out << "declare -a " << name << "=("; for (auto & s : *arr) - out << shellEscape(s) << " "; + out << escapeShellArgAlways(s) << " "; out << ")\n"; } else if (auto arr = std::get_if(&value)) { out << "declare -A " << name << "=("; for (auto & [n, v] : *arr) - out << "[" << shellEscape(n) << "]=" << shellEscape(v) << " "; + out << "[" << escapeShellArgAlways(n) << "]=" << escapeShellArgAlways(v) << " "; out << ")\n"; } } @@ -614,7 +614,7 @@ struct CmdDevelop : Common, MixEnvironment std::vector args; args.reserve(command.size()); for (const auto & s : command) - args.push_back(shellEscape(s)); + args.push_back(escapeShellArgAlways(s)); script += fmt("exec %s\n", concatStringsSep(" ", args)); } @@ -622,13 +622,13 @@ struct CmdDevelop : Common, MixEnvironment script = "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\nshopt -u expand_aliases\n" + script + "\nshopt -s expand_aliases\n"; if (developSettings.bashPrompt != "") script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", - shellEscape(developSettings.bashPrompt.get())); + escapeShellArgAlways(developSettings.bashPrompt.get())); if (developSettings.bashPromptPrefix != "") script += fmt("[ -n \"$PS1\" ] && PS1=%s\"$PS1\";\n", - shellEscape(developSettings.bashPromptPrefix.get())); + escapeShellArgAlways(developSettings.bashPromptPrefix.get())); if (developSettings.bashPromptSuffix != "") script += fmt("[ -n \"$PS1\" ] && PS1+=%s;\n", - shellEscape(developSettings.bashPromptSuffix.get())); + escapeShellArgAlways(developSettings.bashPromptSuffix.get())); } writeFull(rcFileFd.get(), script); From 3b5f0d9fb3af870b832bdcadcf8080649bcd0cd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 24 Apr 2025 11:28:02 +0200 Subject: [PATCH 259/396] Revert "Actually ignore system/user registries during locking" This reverts commit 77d4316353deaf8f429025738891b625eb0b5d8a. --- src/libflake/flakeref.cc | 2 +- tests/functional/flakes/flakes.sh | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index 1580c2846..6e95eb767 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -39,7 +39,7 @@ FlakeRef FlakeRef::resolve( ref store, const fetchers::RegistryFilter & filter) const { - auto [input2, extraAttrs] = lookupInRegistries(store, input, filter); + auto [input2, extraAttrs] = lookupInRegistries(store, input); return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); } diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index b67a0964a..d8c9f254d 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -220,13 +220,6 @@ nix store gc nix registry list --flake-registry "file://$registry" --refresh | grepQuiet flake3 mv "$registry.tmp" "$registry" -# Ensure that locking ignores the user registry. -mkdir -p "$TEST_HOME/.config/nix" -ln -sfn "$registry" "$TEST_HOME/.config/nix/registry.json" -nix flake metadata flake1 -expectStderr 1 nix flake update --flake-registry '' --flake "$flake3Dir" | grepQuiet "cannot find flake 'flake:flake1' in the flake registries" -rm "$TEST_HOME/.config/nix/registry.json" - # Test whether flakes are registered as GC roots for offline use. # FIXME: use tarballs rather than git. rm -rf "$TEST_HOME/.cache" From dda265f09a69938e38c66ae1285a67038822f006 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 24 Apr 2025 17:33:27 +0200 Subject: [PATCH 260/396] Reapply "Actually ignore system/user registries during locking" This reverts commit 3b5f0d9fb3af870b832bdcadcf8080649bcd0cd5. --- src/libflake/flakeref.cc | 2 +- tests/functional/flakes/flakes.sh | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index 6e95eb767..1580c2846 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -39,7 +39,7 @@ FlakeRef FlakeRef::resolve( ref store, const fetchers::RegistryFilter & filter) const { - auto [input2, extraAttrs] = lookupInRegistries(store, input); + auto [input2, extraAttrs] = lookupInRegistries(store, input, filter); return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); } diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index d8c9f254d..b67a0964a 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -220,6 +220,13 @@ nix store gc nix registry list --flake-registry "file://$registry" --refresh | grepQuiet flake3 mv "$registry.tmp" "$registry" +# Ensure that locking ignores the user registry. +mkdir -p "$TEST_HOME/.config/nix" +ln -sfn "$registry" "$TEST_HOME/.config/nix/registry.json" +nix flake metadata flake1 +expectStderr 1 nix flake update --flake-registry '' --flake "$flake3Dir" | grepQuiet "cannot find flake 'flake:flake1' in the flake registries" +rm "$TEST_HOME/.config/nix/registry.json" + # Test whether flakes are registered as GC roots for offline use. # FIXME: use tarballs rather than git. rm -rf "$TEST_HOME/.cache" From e1b68244ade89a0e3ad9ea5da3e41eb77aba1b15 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 24 Apr 2025 07:56:11 -0700 Subject: [PATCH 261/396] nix-cli: restore binary-dist artifact to Hydra static builds --- src/nix/package.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/nix/package.nix b/src/nix/package.nix index 40a280437..3d4f6f40b 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -1,4 +1,5 @@ { + stdenv, lib, mkMesonExecutable, @@ -94,6 +95,11 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; + postInstall = lib.optionalString stdenv.hostPlatform.isStatic '' + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products + ''; + meta = { mainProgram = "nix"; platforms = lib.platforms.unix ++ lib.platforms.windows; From 68de26d38afea6b87460afec77c85e1642a269ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 24 Apr 2025 18:54:16 +0200 Subject: [PATCH 262/396] lockFlake(): Allow registry lookups for the top-level flake Fixes #13050. --- src/libcmd/installables.cc | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- .../include/nix/fetchers/input-cache.hh | 4 ++- .../include/nix/fetchers/registry.hh | 8 ++++-- src/libfetchers/input-cache.cc | 12 +++----- src/libfetchers/registry.cc | 13 +++++++-- src/libflake/flake.cc | 28 +++++++++++++++---- src/libflake/flakeref.cc | 4 +-- src/libflake/include/nix/flake/flake.hh | 2 +- src/libflake/include/nix/flake/flakeref.hh | 2 +- tests/functional/flakes/flakes.sh | 2 +- 11 files changed, 53 insertions(+), 26 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 2ebfac3e6..1c414e9e2 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -40,7 +40,7 @@ void completeFlakeInputAttrPath( std::string_view prefix) { for (auto & flakeRef : flakeRefs) { - auto flake = flake::getFlake(*evalState, flakeRef, true); + auto flake = flake::getFlake(*evalState, flakeRef, fetchers::UseRegistries::All); for (auto & input : flake.inputs) if (hasPrefix(input.first, prefix)) completions.add(input.first); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 0be9f4bdc..745705e04 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -181,7 +181,7 @@ static void fetchTree( } if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) - input = lookupInRegistries(state.store, input).first; + input = lookupInRegistries(state.store, input, fetchers::UseRegistries::Limited).first; if (state.settings.pureEval && !input.isLocked()) { if (input.getNarHash()) diff --git a/src/libfetchers/include/nix/fetchers/input-cache.hh b/src/libfetchers/include/nix/fetchers/input-cache.hh index 869c7d41e..9b1c5a310 100644 --- a/src/libfetchers/include/nix/fetchers/input-cache.hh +++ b/src/libfetchers/include/nix/fetchers/input-cache.hh @@ -2,6 +2,8 @@ namespace nix::fetchers { +enum class UseRegistries : int; + struct InputCache { struct CachedResult @@ -11,7 +13,7 @@ struct InputCache Input lockedInput; }; - CachedResult getAccessor(ref store, const Input & originalInput, bool useRegistries); + CachedResult getAccessor(ref store, const Input & originalInput, UseRegistries useRegistries); struct CachedInput { diff --git a/src/libfetchers/include/nix/fetchers/registry.hh b/src/libfetchers/include/nix/fetchers/registry.hh index 47ff9e86f..efbfe07c8 100644 --- a/src/libfetchers/include/nix/fetchers/registry.hh +++ b/src/libfetchers/include/nix/fetchers/registry.hh @@ -65,7 +65,11 @@ void overrideRegistry( const Input & to, const Attrs & extraAttrs); -using RegistryFilter = std::function; +enum class UseRegistries : int { + No, + All, + Limited, // global and flag registry only +}; /** * Rewrite a flakeref using the registries. If `filter` is set, only @@ -74,6 +78,6 @@ using RegistryFilter = std::function; std::pair lookupInRegistries( ref store, const Input & input, - const RegistryFilter & filter = {}); + UseRegistries useRegistries); } diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 716143899..1a4bb28a3 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -5,7 +5,8 @@ namespace nix::fetchers { -InputCache::CachedResult InputCache::getAccessor(ref store, const Input & originalInput, bool useRegistries) +InputCache::CachedResult +InputCache::getAccessor(ref store, const Input & originalInput, UseRegistries useRegistries) { auto fetched = lookup(originalInput); Input resolvedInput = originalInput; @@ -15,13 +16,8 @@ InputCache::CachedResult InputCache::getAccessor(ref store, const Input & auto [accessor, lockedInput] = originalInput.getAccessor(store); fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor}); } else { - if (useRegistries) { - auto [res, extraAttrs] = - lookupInRegistries(store, originalInput, [](fetchers::Registry::RegistryType type) { - /* Only use the global registry and CLI flags - to resolve indirect flakerefs. */ - return type == fetchers::Registry::Flag || type == fetchers::Registry::Global; - }); + if (useRegistries != UseRegistries::No) { + auto [res, extraAttrs] = lookupInRegistries(store, originalInput, useRegistries); resolvedInput = std::move(res); fetched = lookup(resolvedInput); if (!fetched) { diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index e9b55f7f2..bfaf9569a 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -14,6 +14,8 @@ std::shared_ptr Registry::read( const Settings & settings, const Path & path, RegistryType type) { + debug("reading registry '%s'", path); + auto registry = std::make_shared(settings, type); if (!pathExists(path)) @@ -179,29 +181,36 @@ Registries getRegistries(const Settings & settings, ref store) std::pair lookupInRegistries( ref store, const Input & _input, - const RegistryFilter & filter) + UseRegistries useRegistries) { Attrs extraAttrs; int n = 0; Input input(_input); + if (useRegistries == UseRegistries::No) + return {input, extraAttrs}; + restart: n++; if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); for (auto & registry : getRegistries(*input.settings, store)) { - if (filter && !filter(registry->type)) continue; + if (useRegistries == UseRegistries::Limited + && !(registry->type == fetchers::Registry::Flag || registry->type == fetchers::Registry::Global)) + continue; // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { if (entry.from == input) { + debug("resolved flakeref '%s' against registry %d exactly", input.to_string(), registry->type); input = entry.to; extraAttrs = entry.extraAttrs; goto restart; } } else { if (entry.from.contains(input)) { + debug("resolved flakeref '%s' against registry %d", input.to_string(), registry->type); input = entry.to.applyOverrides( !entry.from.getRef() && input.getRef() ? input.getRef() : std::optional(), !entry.from.getRev() && input.getRev() ? input.getRev() : std::optional()); diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index afeefdaec..818b2fb75 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -337,7 +337,7 @@ static FlakeRef applySelfAttrs( static Flake getFlake( EvalState & state, const FlakeRef & originalRef, - bool useRegistries, + fetchers::UseRegistries useRegistries, const InputAttrPath & lockRootAttrPath) { // Fetch a lazy tree first. @@ -368,7 +368,7 @@ static Flake getFlake( return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath); } -Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool useRegistries) +Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries) { return getFlake(state, originalRef, useRegistries, {}); } @@ -393,8 +393,14 @@ LockedFlake lockFlake( experimentalFeatureSettings.require(Xp::Flakes); auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); + auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; + auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; - auto flake = getFlake(state, topRef, useRegistries, {}); + auto flake = getFlake( + state, + topRef, + useRegistriesTop, + {}); if (lockFlags.applyNixConfig) { flake.config.apply(settings); @@ -569,7 +575,11 @@ LockedFlake lockFlake( if (auto resolvedPath = resolveRelativePath()) { return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath); } else { - return getFlake(state, ref, useRegistries, inputAttrPath); + return getFlake( + state, + ref, + useRegistriesInputs, + inputAttrPath); } }; @@ -717,7 +727,10 @@ LockedFlake lockFlake( if (auto resolvedPath = resolveRelativePath()) { return {*resolvedPath, *input.ref}; } else { - auto cachedInput = state.inputCache->getAccessor(state.store, input.ref->input, useRegistries); + auto cachedInput = state.inputCache->getAccessor( + state.store, + input.ref->input, + useRegistriesInputs); auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); @@ -834,7 +847,10 @@ LockedFlake lockFlake( repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ auto prevLockedRef = flake.lockedRef; - flake = getFlake(state, topRef, useRegistries); + flake = getFlake( + state, + topRef, + useRegistriesTop); if (lockFlags.commitLockFile && flake.lockedRef.input.getRev() && diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index 1580c2846..a8b139d65 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -37,9 +37,9 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) FlakeRef FlakeRef::resolve( ref store, - const fetchers::RegistryFilter & filter) const + fetchers::UseRegistries useRegistries) const { - auto [input2, extraAttrs] = lookupInRegistries(store, input, filter); + auto [input2, extraAttrs] = lookupInRegistries(store, input, useRegistries); return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); } diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 3b98f1400..f90498b7c 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -115,7 +115,7 @@ struct Flake } }; -Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool useRegistries); +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, fetchers::UseRegistries useRegistries); /** * Fingerprint of a locked flake; used as a cache key. diff --git a/src/libflake/include/nix/flake/flakeref.hh b/src/libflake/include/nix/flake/flakeref.hh index 0fd1fec4d..8c15f9d95 100644 --- a/src/libflake/include/nix/flake/flakeref.hh +++ b/src/libflake/include/nix/flake/flakeref.hh @@ -65,7 +65,7 @@ struct FlakeRef FlakeRef resolve( ref store, - const fetchers::RegistryFilter & filter = {}) const; + fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const; static FlakeRef fromAttrs( const fetchers::Settings & fetchSettings, diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index b67a0964a..aac505d41 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -223,7 +223,7 @@ mv "$registry.tmp" "$registry" # Ensure that locking ignores the user registry. mkdir -p "$TEST_HOME/.config/nix" ln -sfn "$registry" "$TEST_HOME/.config/nix/registry.json" -nix flake metadata flake1 +nix flake metadata --flake-registry '' flake1 expectStderr 1 nix flake update --flake-registry '' --flake "$flake3Dir" | grepQuiet "cannot find flake 'flake:flake1' in the flake registries" rm "$TEST_HOME/.config/nix/registry.json" From 3e7d85dfdb7f52ad16868003b28f9ed822628060 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 24 Apr 2025 13:43:53 -0400 Subject: [PATCH 263/396] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/f675531bc7e6657c10a18b565cfebd8aa9e24c14?narHash=sha256-gbl9hE39nQRpZaLjhWKmEu5ejtQsgI5TWYrIVVJn30U%3D' (2025-04-09) → 'github:NixOS/nixpkgs/8a2f738d9d1f1d986b5a4cd2fd2061a7127237d7?narHash=sha256-sPwcCYuiEopaafePqlG826tBhctuJsLx/mhKKM5Fmjo%3D' (2025-04-23) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 5154c1685..7abe8bd62 100644 --- a/flake.lock +++ b/flake.lock @@ -63,11 +63,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1744232761, - "narHash": "sha256-gbl9hE39nQRpZaLjhWKmEu5ejtQsgI5TWYrIVVJn30U=", + "lastModified": 1745391562, + "narHash": "sha256-sPwcCYuiEopaafePqlG826tBhctuJsLx/mhKKM5Fmjo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f675531bc7e6657c10a18b565cfebd8aa9e24c14", + "rev": "8a2f738d9d1f1d986b5a4cd2fd2061a7127237d7", "type": "github" }, "original": { From bfb357c40b289490ad841cc7271f2afa92081d34 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 24 Apr 2025 21:00:24 +0000 Subject: [PATCH 264/396] libutil: Fix invalid boost format string in infinite symlink recursion error Found while working on an automated migration to `std::format`. --- src/libutil/file-system.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 39b03268d..071aecc9d 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -111,7 +111,7 @@ Path canonPath(PathView path, bool resolveSymlinks) (std::string & result, std::string_view & remaining) { if (resolveSymlinks && fs::is_symlink(result)) { if (++followCount >= maxFollow) - throw Error("infinite symlink recursion in path '%0%'", remaining); + throw Error("infinite symlink recursion in path '%1%'", remaining); remaining = (temp = concatStrings(readLink(result), remaining)); if (isAbsolute(remaining)) { /* restart for symlinks pointing to absolute path */ From 1b5c8aac123d96b907972a9cbb67891ff17caf7a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 24 Apr 2025 22:27:03 +0000 Subject: [PATCH 265/396] libutil: Use correct argument to Error format ctor It seems that the intention was to format a number in base 8 (as suggested by the %o format specifier), but `perms` is a `std::string` and not a number. Looks like `rawMode` is the correct thing to use here. --- src/libutil/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 45cda1c2c..edeef71b7 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -134,7 +134,7 @@ void parseTree( RawMode rawMode = std::stoi(perms, 0, 8); auto modeOpt = decodeMode(rawMode); if (!modeOpt) - throw Error("Unknown Git permission: %o", perms); + throw Error("Unknown Git permission: %o", rawMode); auto mode = std::move(*modeOpt); std::string name = getStringUntil(source, '\0'); From 6405d6822d0139ea02123919eb40d0b57786b7f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 25 Apr 2025 10:10:06 +0200 Subject: [PATCH 266/396] tests/flakes: add regression test for resolving user flakes --- tests/functional/flakes/flakes.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index aac505d41..72fe79838 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -266,6 +266,7 @@ nix registry add user-flake2 "git+file://$percentEncodedFlake2Dir" [[ $(nix --flake-registry "" registry list | wc -l) == 2 ]] nix --flake-registry "" registry list | grepQuietInverse '^global' # nothing in global registry nix --flake-registry "" registry list | grepQuiet '^user' +nix flake metadata --flake-registry "" user-flake1 | grepQuiet 'URL:.*flake1.*' nix registry remove user-flake1 nix registry remove user-flake2 [[ $(nix registry list | wc -l) == 4 ]] From 9fff868e39ddbeeee4c1aece452cf0d9c9cc8019 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 25 Apr 2025 13:35:16 +0300 Subject: [PATCH 267/396] libutil: Add missing format arguments to UsageError ctor Once again found by an automated migration to `std::format`. I've tested that boost::format works fine with `std::string_view` arguments. --- src/libutil-tests/file-content-address.cc | 15 +++++++++++---- src/libutil/file-content-address.cc | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/libutil-tests/file-content-address.cc b/src/libutil-tests/file-content-address.cc index 5cdf94edc..92c6059a4 100644 --- a/src/libutil-tests/file-content-address.cc +++ b/src/libutil-tests/file-content-address.cc @@ -1,3 +1,4 @@ +#include #include #include "nix/util/file-content-address.hh" @@ -26,8 +27,11 @@ TEST(FileSerialisationMethod, testRoundTripPrintParse_2) { } } -TEST(FileSerialisationMethod, testParseFileSerialisationMethodOptException) { - EXPECT_THROW(parseFileSerialisationMethod("narwhal"), UsageError); +TEST(FileSerialisationMethod, testParseFileSerialisationMethodOptException) +{ + EXPECT_THAT( + []() { parseFileSerialisationMethod("narwhal"); }, + testing::ThrowsMessage(testing::HasSubstr("narwhal"))); } /* ---------------------------------------------------------------------------- @@ -54,8 +58,11 @@ TEST(FileIngestionMethod, testRoundTripPrintParse_2) { } } -TEST(FileIngestionMethod, testParseFileIngestionMethodOptException) { - EXPECT_THROW(parseFileIngestionMethod("narwhal"), UsageError); +TEST(FileIngestionMethod, testParseFileIngestionMethodOptException) +{ + EXPECT_THAT( + []() { parseFileIngestionMethod("narwhal"); }, + testing::ThrowsMessage(testing::HasSubstr("narwhal"))); } } diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index 673e1dff1..142bc70d5 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -22,7 +22,7 @@ FileSerialisationMethod parseFileSerialisationMethod(std::string_view input) if (ret) return *ret; else - throw UsageError("Unknown file serialiation method '%s', expect `flat` or `nar`"); + throw UsageError("Unknown file serialiation method '%s', expect `flat` or `nar`", input); } @@ -35,7 +35,7 @@ FileIngestionMethod parseFileIngestionMethod(std::string_view input) if (ret) return static_cast(*ret); else - throw UsageError("Unknown file ingestion method '%s', expect `flat`, `nar`, or `git`"); + throw UsageError("Unknown file ingestion method '%s', expect `flat`, `nar`, or `git`", input); } } From 953ec00794d631867a915050c72ea63de3069c65 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 25 Apr 2025 13:46:42 +0200 Subject: [PATCH 268/396] getFlake(): Don't use registries for refetching `newLockedRef` is already resolved so there is no need to re-resolve it. --- src/libflake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 818b2fb75..28e3c4b93 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -356,7 +356,7 @@ static Flake getFlake( debug("refetching input '%s' due to self attribute", newLockedRef); // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'. newLockedRef.input.attrs.erase("narHash"); - auto cachedInput2 = state.inputCache->getAccessor(state.store, newLockedRef.input, useRegistries); + auto cachedInput2 = state.inputCache->getAccessor(state.store, newLockedRef.input, fetchers::UseRegistries::No); cachedInput.accessor = cachedInput2.accessor; lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); } From 94916136dcee1d33973a52da3c9a93922ee80dad Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 25 Apr 2025 17:03:44 +0200 Subject: [PATCH 269/396] Fix flake-c out of bounds access The explicit include is needed for clangd to not get confused somehow, which is also what threw me off initially and made me pick the wrong constructor. The (pointer, number, number) constructor first constructs a C string and then takes a substring from that, but we didn't specify that the buffer needs to be NUL-terminated, and then what would be the point of the size argument anyway... basic_string.h: > basic_string(const _Tp& __t, size_type __pos, size_type __n, > const _Alloc& __a = _Alloc()) > : basic_string(_S_to_string_view(__t).substr(__pos, __n), __a) { } Valgrind on nixops4/rust/nix-flake tests: ==1344422== Conditional jump or move depends on uninitialised value(s) ==1344422== at 0x48513E8: strlen (vg_replace_strmem.c:505) ==1344422== by 0x488E941: UnknownInlinedFun (char_traits.h:391) ==1344422== by 0x488E941: UnknownInlinedFun (string_view:141) ==1344422== by 0x488E941: UnknownInlinedFun (basic_string.h:790) ==1344422== by 0x488E941: nix_flake_reference_and_fragment_from_string (nix_api_flake.cc:81) ==1344422== by 0x127332: nix_flake::FlakeReference::parse_with_fragment (lib.rs:123) --- src/libflake-c/nix_api_flake.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc index f100bf146..ad8f0bf4e 100644 --- a/src/libflake-c/nix_api_flake.cc +++ b/src/libflake-c/nix_api_flake.cc @@ -1,3 +1,5 @@ +#include + #include "nix_api_flake.h" #include "nix_api_flake_internal.hh" #include "nix_api_util.h" @@ -78,7 +80,7 @@ nix_err nix_flake_reference_and_fragment_from_string( nix_clear_err(context); *flakeReferenceOut = nullptr; try { - std::string str(strData, 0, strSize); + std::string str(strData, strSize); auto [flakeRef, fragment] = nix::parseFlakeRefWithFragment(*fetchSettings->settings, str, parseFlags->baseDirectory, true); From a525c7e9910f93eed1b92337823cf835bfb1b92d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 25 Apr 2025 17:50:00 +0200 Subject: [PATCH 270/396] doc/manual: Add language/evaluation --- doc/manual/source/SUMMARY.md.in | 1 + doc/manual/source/language/evaluation.md | 77 ++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 doc/manual/source/language/evaluation.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 5932e0999..140be98ad 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -33,6 +33,7 @@ - [Data Types](language/types.md) - [String context](language/string-context.md) - [Syntax and semantics](language/syntax.md) + - [Evaluation](language/evaluation.md) - [Variables](language/variables.md) - [String literals](language/string-literals.md) - [Identifiers](language/identifiers.md) diff --git a/doc/manual/source/language/evaluation.md b/doc/manual/source/language/evaluation.md new file mode 100644 index 000000000..980942c92 --- /dev/null +++ b/doc/manual/source/language/evaluation.md @@ -0,0 +1,77 @@ +# Evaluation + +Evaluation is the process of turning a Nix expression into a [Nix value](types.md). + +This happens by a number of rules, such as: +- Constructing values from literals. + For example the number literal `1` is turned into the number value `1`. +- Applying operators + For example the addition operator `+` is applied to two number values to produce a new number value. +- Applying built-in functions + For example the expression `builtins.isInt 1` is evaluated to `true`. +- Applying user-defined functions + For example the expression `(x: x + 1) 10` can[*](#laziness) be thought of rewriting `x` in the function body to the argument, `10 + 1`, which is then evaluated to `11`. + +These rules are applied as needed, driven by the specific use of the expression. For example, this can occur in the Nix command line interface or interactively with the [repl (read-eval-print loop)](@docroot@/command-ref/new-cli/nix3-repl.md), which is a useful tool when learning about evaluation. + +# Details + +## Values {#values} + +Nix values can be thought of as a subset of Nix expressions. +For example, the expression `1 + 2` is not a value, because it can be reduced to `3`. The expression `3` is a value, because it cannot be reduced any further. + +Evaluation normally happens by applying rules to the "head" of the expression, which is the outermost part of the expression. The head of an expression like `[ 1 2 ]` is the list literal (`[ a1 a2 ]`), for `1 + 2` it is the addition operator (`+`), and for `f 1` it is the function application "operator" (` `). + +After applying all possible rules to the head until no rules can be applied, the expression is in "weak head normal form" (WHNF). This means that the outermost constructor of the expression is evaluated, but the inner values may or may not be. "Weak" only signifies that the expression may be a function. This is an historical or academic artifact, and Nix has no use for the non-weak "head normal form". + +## Laziness and thunks {#laziness} + +The Nix language implements _call by need_ (as opposed to _call by value_ or _call by reference_). Call by need is commonly known as laziness in functional programming, as it is a specific implementation of the concept where evaluation is deferred until the result is required, aiming to only evaluate the parts of an expression that are needed to produce the final result. + +Furthermore, the result of evaluation is preserved, in values, in `let` bindings, in function _parameters_, which behave a lot like `let` bindings, but with the notable exception of function _calls_. Results of function calls rely on being put into `let` bindings, etc to be reused. + +When discussing the process of evaluation in lower level terms, we may define values not as a subset of expressions, but separately, where each "value" is either a data constructor, a function or a _thunk_. A thunk is a delayed computation, represented by an expression reference and a "closure" – the values for the lexical scope around the delayed expression. + +As a user of the language, you generally don't have to think about thunks, as they are not part of the language semantics, but you may encounter them in the repl, in the [C API] or in discussions. + +## Strictness + +Instead of thinking about thunks, it is often more productive to think in terms of _strictness_. +This term is used in functional programming to refer to the opposite of laziness, i.e. not just for something like error propagation. It refers to the need to evaluate certain expressions before evaluation can produce any result. + +Statements about strictness usually implicitly refer to weak head normal form. +For example, we can say that the following function is strict in its argument: + +```nix +x: isAttrs x || isFunction x +``` + +The above function must be strict in its argument `x` because determining its type requires evaluating `x` to at least some degree. + +The following function is not strict in its argument: + +```nix +x: { isOk = isAttrs x || isFunction x; } +``` + +It is not strict, because it can return the attribute set before evaluating `x`. +The attribute value for `isOk` _is_ strict in `x`. + +A function with a _set pattern_ is always strict in its argument, as a consequence of checking the argument's type and/or attribute names: + +```nix +let f = { ... }: "ok"; +in f (throw "kablam") +=> error: kablam +``` + +However, a set pattern does not add any strictness beyond WHNF of the attribute set argument. + +```nix +let f = orig@{ x, ... }: "ok"; +in f { x = throw "error"; y = throw "error"; } +=> "ok" +``` + +[C API]: @docroot@/c-api.md From 4e95f662db38d219609361697ae48a2b02352c20 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Mon, 28 Apr 2025 00:46:44 +0200 Subject: [PATCH 271/396] allocate SimpleLogger before forking --- src/libutil/unix/processes.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 198243c20..0d50fc303 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -202,6 +202,7 @@ static int childEntry(void * arg) pid_t startProcess(std::function fun, const ProcessOptions & options) { + auto newLogger = makeSimpleLogger(); ChildWrapperFunction wrapper = [&] { if (!options.allowVfork) { /* Set a simple logger, while releasing (not destroying) @@ -210,7 +211,7 @@ pid_t startProcess(std::function fun, const ProcessOptions & options) ~ProgressBar() tries to join a thread that doesn't exist. */ logger.release(); - logger = makeSimpleLogger(); + logger = std::move(newLogger); } try { #ifdef __linux__ From 9d1f00e31deb486392f6953d6b99e1adac0a90c5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 28 Apr 2025 16:53:53 +0200 Subject: [PATCH 272/396] Factor out MixOutLinkByDefault --- src/libcmd/command.cc | 7 +++++ src/libcmd/include/nix/cmd/command.hh | 39 ++++++++++++++++++++++++++- src/nix/build.cc | 22 ++------------- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 56541fa57..047906786 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -397,4 +397,11 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu } } +void MixOutLinkBase::createOutLinksMaybe(const std::vector & buildables, ref & store) +{ + if (outLink != "") + if (auto store2 = store.dynamic_pointer_cast()) + createOutLinks(outLink, toBuiltPaths(buildables), *store2); +} + } diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index 6b6418f51..92377be6b 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -374,4 +374,41 @@ void printClosureDiff( */ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store); -} +/** `outLink` parameter, `createOutLinksMaybe` method. See `MixOutLinkByDefault`. */ +struct MixOutLinkBase : virtual Args +{ + /** Prefix for any output symlinks. Empty means do not write an output symlink. */ + Path outLink; + + MixOutLinkBase(const std::string & defaultOutLink) + : outLink(defaultOutLink) + { + } + + void createOutLinksMaybe(const std::vector & buildables, ref & store); +}; + +/** `--out-link`, `--no-link`, `createOutLinksMaybe` */ +struct MixOutLinkByDefault : MixOutLinkBase, virtual Args +{ + MixOutLinkByDefault() + : MixOutLinkBase("result") + { + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.", + .labels = {"path"}, + .handler = {&outLink}, + .completer = completePath, + }); + + addFlag({ + .longName = "no-link", + .description = "Do not create symlinks to the build results.", + .handler = {&outLink, Path("")}, + }); + } +}; + +} // namespace nix diff --git a/src/nix/build.cc b/src/nix/build.cc index daf50082e..bd0c8862b 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -41,29 +41,13 @@ static nlohmann::json builtPathsWithResultToJSON(const std::vectorcout("%s", builtPathsWithResultToJSON(buildables, *store).dump()); - if (outLink != "") - if (auto store2 = store.dynamic_pointer_cast()) - createOutLinks(outLink, toBuiltPaths(buildables), *store2); + createOutLinksMaybe(buildables, store); if (printOutputPaths) { logger->stop(); From 46030181d4725f5bf3fe46f1ebfd032f99906c92 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 28 Apr 2025 11:19:36 -0400 Subject: [PATCH 273/396] Delete dead code We had multiple copies of some static functions after splitting out `DerivationBuilder` by mistake. --- src/libstore/build/derivation-goal.cc | 58 ------------------- src/libstore/unix/build/derivation-builder.cc | 27 ++++++++- 2 files changed, 26 insertions(+), 59 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index a08a3ea74..344d1c7ea 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -971,64 +971,6 @@ Goal::Co DerivationGoal::repairClosure() } -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = ( -#ifndef _WIN32 - geteuid() -#else - !isRootUser() -#endif - && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - std::filesystem::rename(src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - -void replaceValidPath(const Path & storePath, const Path & tmpPath) -{ - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), rand()); - if (pathExists(storePath)) - movePath(storePath, oldPath); - - try { - movePath(tmpPath, storePath); - } catch (...) { - try { - // attempt to recover - movePath(oldPath, storePath); - } catch (...) { - ignoreExceptionExceptInterrupt(); - } - throw; - } - - deletePath(oldPath); -} - - void runPostBuildHook( Store & store, Logger & logger, diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 2131b4e88..76f69671c 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -661,7 +661,32 @@ static void movePath(const Path & src, const Path & dst) } -extern void replaceValidPath(const Path & storePath, const Path & tmpPath); +static void replaceValidPath(const Path & storePath, const Path & tmpPath) +{ + /* We can't atomically replace storePath (the original) with + tmpPath (the replacement), so we have to move it out of the + way first. We'd better not be interrupted here, because if + we're repairing (say) Glibc, we end up with a broken system. */ + Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), rand()); + if (pathExists(storePath)) + movePath(storePath, oldPath); + + try { + movePath(tmpPath, storePath); + } catch (...) { + try { + // attempt to recover + movePath(oldPath, storePath); + } catch (...) { + ignoreExceptionExceptInterrupt(); + } + throw; + } + + deletePath(oldPath); +} + + bool DerivationBuilderImpl::cleanupDecideWhetherDiskFull() From cb9182f9eda2d04918fde12aa0fa072d46e2234b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 29 Apr 2025 10:58:34 +0200 Subject: [PATCH 274/396] docs: don't mention Haskell sometimes it's these little things that let beginners stumble at the first step... mentioning one potentially foreign concept while introducing an entirely new concept is asking enough already. --- doc/manual/source/introduction.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index 76489bc1b..e70411c11 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -1,8 +1,8 @@ # Introduction Nix is a _purely functional package manager_. This means that it -treats packages like values in purely functional programming languages -such as Haskell — they are built by functions that don’t have +treats packages like values in a purely functional programming language +— packages are built by functions that don’t have side-effects, and they never change after they have been built. Nix stores packages in the _Nix store_, usually the directory `/nix/store`, where each package has its own unique subdirectory such From e322b714dc88c2c8ce6b81e529e382ba4c81df1b Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Tue, 29 Apr 2025 15:35:53 +0100 Subject: [PATCH 275/396] libutil: amend OSC 8 escape stripping for xterm-style separator Before the change `nix` was stripping warning flags reported by `gcc-14` too eagerly: $ nix build -f. texinfo4 error: builder for '/nix/store/i9948l91s3df44ip5jlpp6imbrcs646x-texinfo-4.13a.drv' failed with exit code 2; last 25 log lines: > 1495 | info_tag (mbi_iterator_t iter, int handle, size_t *plen) > | ~~~~~~~~^~~~ > window.c:1887:39: error: passing argument 4 of 'printed_representation' from incompatible pointer type [] > 1887 | &replen); > | ^~~~~~~ > | | > | int * After the change the compiler flag remains: $ ~/patched.nix build -f. texinfo4 error: builder for '/nix/store/i9948l91s3df44ip5jlpp6imbrcs646x-texinfo-4.13a.drv' failed with exit code 2; last 25 log lines: > 1495 | info_tag (mbi_iterator_t iter, int handle, size_t *plen) > | ~~~~~~~~^~~~ > window.c:1887:39: error: passing argument 4 of 'printed_representation' from incompatible pointer type [-Wincompatible-pointer-types] > 1887 | &replen); > | ^~~~~~~ > | | > | int * Note the difference in flag rendering around the warning. https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda has a good sumamry of why it happens. Befomre the change `nix` was handling just one form or URL separator: $ printf '\e]8;;http://example.com\e\\This is a link\e]8;;\e\\\n' Now it also handled another for (used by gcc-14`): printf '\e]8;;http://example.com\aThis is a link\e]8;;\a\n' While at it fixed accumulation of trailing escape `\e\\` symbol. --- src/libutil-tests/terminal.cc | 8 ++++++++ src/libutil/terminal.cc | 17 +++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libutil-tests/terminal.cc b/src/libutil-tests/terminal.cc index 329c1a186..198b20951 100644 --- a/src/libutil-tests/terminal.cc +++ b/src/libutil-tests/terminal.cc @@ -66,4 +66,12 @@ TEST(filterANSIEscapes, osc8) ASSERT_EQ(filterANSIEscapes("\e]8;;http://example.com\e\\This is a link\e]8;;\e\\"), "This is a link"); } +TEST(filterANSIEscapes, osc8_bell_as_sep) +{ + // gcc-14 uses \a as a separator, xterm style: + // https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda + ASSERT_EQ(filterANSIEscapes("\e]8;;http://example.com\aThis is a link\e]8;;\a"), "This is a link"); + ASSERT_EQ(filterANSIEscapes("\e]8;;http://example.com\a\\This is a link\e]8;;\a"), "\\This is a link"); +} + } // namespace nix diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index fa0f7e871..63473d1a9 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -95,10 +95,19 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w } else if (i != s.end() && *i == ']') { // OSC e += *i++; - // eat ESC - while (i != s.end() && *i != '\e') e += *i++; - // eat backslash - if (i != s.end() && *i == '\\') e += last = *i++; + // https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda defines + // two forms of a URI separator: + // 1. ESC '\' (standard) + // 2. BEL ('\a') (xterm-style, used by gcc) + + // eat ESC or BEL + while (i != s.end() && *i != '\e' && *i != '\a') e += *i++; + if (i != s.end()) { + char v = *i; + e += *i++; + // eat backslash after ESC + if (i != s.end() && v == '\e' && *i == '\\') e += last = *i++; + } } else { if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; } From 788be3f964939601677281c4cf2ede9df3dbec39 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Wed, 30 Apr 2025 01:38:48 +0200 Subject: [PATCH 276/396] bugfix in getInteger(const nlohmann::json &) and add bounds checks improve error messages, too --- src/libstore/nar-info.cc | 2 +- src/libstore/path-info.cc | 4 ++-- src/libutil-tests/json-utils.cc | 28 +++++++++++++++------- src/libutil/include/nix/util/json-utils.hh | 22 ++++++++++++++++- src/libutil/json-utils.cc | 13 ++++++++-- 5 files changed, 54 insertions(+), 15 deletions(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ba80652d0..ef7af6126 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -176,7 +176,7 @@ NarInfo NarInfo::fromJSON( std::nullopt); if (json.contains("downloadSize")) - res.fileSize = getInteger(valueAt(json, "downloadSize")); + res.fileSize = getUnsigned(valueAt(json, "downloadSize")); return res; } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 5400a9da1..175146435 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -200,7 +200,7 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( auto & json = getObject(_json); res.narHash = Hash::parseAny(getString(valueAt(json, "narHash")), std::nullopt); - res.narSize = getInteger(valueAt(json, "narSize")); + res.narSize = getUnsigned(valueAt(json, "narSize")); try { auto references = getStringList(valueAt(json, "references")); @@ -224,7 +224,7 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( if (json.contains("registrationTime")) if (auto * rawRegistrationTime = getNullable(valueAt(json, "registrationTime"))) - res.registrationTime = getInteger(*rawRegistrationTime); + res.registrationTime = getInteger(*rawRegistrationTime); if (json.contains("ultimate")) res.ultimate = getBoolean(valueAt(json, "ultimate")); diff --git a/src/libutil-tests/json-utils.cc b/src/libutil-tests/json-utils.cc index 71c069d66..eae67b4b3 100644 --- a/src/libutil-tests/json-utils.cc +++ b/src/libutil-tests/json-utils.cc @@ -128,19 +128,29 @@ TEST(getString, wrongAssertions) { ASSERT_THROW(getString(valueAt(json, "boolean")), Error); } -TEST(getInteger, rightAssertions) { - auto simple = R"({ "int": 0 })"_json; +TEST(getIntegralNumber, rightAssertions) { + auto simple = R"({ "int": 0, "signed": -1 })"_json; - ASSERT_EQ(getInteger(valueAt(getObject(simple), "int")), 0); + ASSERT_EQ(getUnsigned(valueAt(getObject(simple), "int")), 0); + ASSERT_EQ(getInteger(valueAt(getObject(simple), "int")), 0); + ASSERT_EQ(getInteger(valueAt(getObject(simple), "signed")), -1); } -TEST(getInteger, wrongAssertions) { - auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; +TEST(getIntegralNumber, wrongAssertions) { + auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "signed": -256, "large": 128, "boolean": false })"_json; - ASSERT_THROW(getInteger(valueAt(json, "object")), Error); - ASSERT_THROW(getInteger(valueAt(json, "array")), Error); - ASSERT_THROW(getInteger(valueAt(json, "string")), Error); - ASSERT_THROW(getInteger(valueAt(json, "boolean")), Error); + ASSERT_THROW(getUnsigned(valueAt(json, "object")), Error); + ASSERT_THROW(getUnsigned(valueAt(json, "array")), Error); + ASSERT_THROW(getUnsigned(valueAt(json, "string")), Error); + ASSERT_THROW(getUnsigned(valueAt(json, "boolean")), Error); + ASSERT_THROW(getUnsigned(valueAt(json, "signed")), Error); + + ASSERT_THROW(getInteger(valueAt(json, "object")), Error); + ASSERT_THROW(getInteger(valueAt(json, "array")), Error); + ASSERT_THROW(getInteger(valueAt(json, "string")), Error); + ASSERT_THROW(getInteger(valueAt(json, "boolean")), Error); + ASSERT_THROW(getInteger(valueAt(json, "large")), Error); + ASSERT_THROW(getInteger(valueAt(json, "signed")), Error); } TEST(getBoolean, rightAssertions) { diff --git a/src/libutil/include/nix/util/json-utils.hh b/src/libutil/include/nix/util/json-utils.hh index 9308d4392..bcae46a0a 100644 --- a/src/libutil/include/nix/util/json-utils.hh +++ b/src/libutil/include/nix/util/json-utils.hh @@ -4,6 +4,7 @@ #include #include +#include "nix/util/error.hh" #include "nix/util/types.hh" namespace nix { @@ -35,7 +36,26 @@ const nlohmann::json * getNullable(const nlohmann::json & value); const nlohmann::json::object_t & getObject(const nlohmann::json & value); const nlohmann::json::array_t & getArray(const nlohmann::json & value); const nlohmann::json::string_t & getString(const nlohmann::json & value); -const nlohmann::json::number_integer_t & getInteger(const nlohmann::json & value); +const nlohmann::json::number_unsigned_t & getUnsigned(const nlohmann::json & value); + +template +auto getInteger(const nlohmann::json & value) -> std::enable_if_t && std::is_integral_v, T> +{ + if (auto ptr = value.get_ptr()) { + if (*ptr <= std::make_unsigned_t(std::numeric_limits::max())) { + return *ptr; + } + } else if (auto ptr = value.get_ptr()) { + if (*ptr >= std::numeric_limits::min() && *ptr <= std::numeric_limits::max()) { + return *ptr; + } + } else { + auto typeName = value.is_number_float() ? "floating point number" : value.type_name(); + throw Error("Expected JSON value to be an integral number but it is of type '%s': %s", typeName, value.dump()); + } + throw Error("Out of range: JSON value '%s' cannot be casted to %d-bit integer", value.dump(), 8 * sizeof(T)); +} + const nlohmann::json::boolean_t & getBoolean(const nlohmann::json & value); Strings getStringList(const nlohmann::json & value); StringMap getStringMap(const nlohmann::json & value); diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index 2c8edfce8..34da83a2c 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -92,9 +92,18 @@ const nlohmann::json::string_t & getString(const nlohmann::json & value) return ensureType(value, nlohmann::json::value_t::string).get_ref(); } -const nlohmann::json::number_integer_t & getInteger(const nlohmann::json & value) +const nlohmann::json::number_unsigned_t & getUnsigned(const nlohmann::json & value) { - return ensureType(value, nlohmann::json::value_t::number_integer).get_ref(); + if (auto ptr = value.get()) { + return *ptr; + } + const char * typeName = value.type_name(); + if (typeName == nlohmann::json(0).type_name()) { + typeName = value.is_number_float() ? "floating point number" : "signed integral number"; + } + throw Error( + "Expected JSON value to be an unsigned integral number but it is of type '%s': %s", + typeName, value.dump()); } const nlohmann::json::boolean_t & getBoolean(const nlohmann::json & value) From 5ea7b97147744580bc5d5f7d07f764dfc2cc175b Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Tue, 22 Apr 2025 17:07:08 -0700 Subject: [PATCH 277/396] refactor: create a new `nix formatter run` command alias for `nix fmt` This refactor shouldn't change much except add a new `nix formatter run` command. This creates space for the new `nix formatter build` command, which I'll be introducing in the next commit. --- maintainers/flake-module.nix | 4 +- src/nix/fmt.cc | 55 ----------- src/nix/{fmt.md => formatter-run.md} | 2 +- src/nix/formatter.cc | 98 +++++++++++++++++++ src/nix/meson.build | 2 +- tests/functional/{fmt.sh => formatter.sh} | 0 .../{fmt.simple.sh => formatter.simple.sh} | 0 tests/functional/meson.build | 2 +- 8 files changed, 103 insertions(+), 60 deletions(-) delete mode 100644 src/nix/fmt.cc rename src/nix/{fmt.md => formatter-run.md} (90%) create mode 100644 src/nix/formatter.cc rename tests/functional/{fmt.sh => formatter.sh} (100%) rename tests/functional/{fmt.simple.sh => formatter.simple.sh} (100%) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index de71be0d7..2dbeaf889 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -605,8 +605,8 @@ ''^tests/functional/flakes/prefetch\.sh$'' ''^tests/functional/flakes/run\.sh$'' ''^tests/functional/flakes/show\.sh$'' - ''^tests/functional/fmt\.sh$'' - ''^tests/functional/fmt\.simple\.sh$'' + ''^tests/functional/formatter\.sh$'' + ''^tests/functional/formatter\.simple\.sh$'' ''^tests/functional/gc-auto\.sh$'' ''^tests/functional/gc-concurrent\.builder\.sh$'' ''^tests/functional/gc-concurrent\.sh$'' diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc deleted file mode 100644 index dc270fb8c..000000000 --- a/src/nix/fmt.cc +++ /dev/null @@ -1,55 +0,0 @@ -#include "nix/cmd/command.hh" -#include "nix/cmd/installable-value.hh" -#include "nix/expr/eval.hh" -#include "run.hh" - -using namespace nix; - -struct CmdFmt : SourceExprCommand { - std::vector args; - - CmdFmt() { expectArgs({.label = "args", .handler = {&args}}); } - - std::string description() override { - return "reformat your code in the standard style"; - } - - std::string doc() override { - return - #include "fmt.md" - ; - } - - Category category() override { return catSecondary; } - - Strings getDefaultFlakeAttrPaths() override { - return Strings{"formatter." + settings.thisSystem.get()}; - } - - Strings getDefaultFlakeAttrPathPrefixes() override { return Strings{}; } - - void run(ref store) override - { - auto evalState = getEvalState(); - auto evalStore = getEvalStore(); - - auto installable_ = parseInstallable(store, "."); - auto & installable = InstallableValue::require(*installable_); - auto app = installable.toApp(*evalState).resolve(evalStore, store); - - Strings programArgs{app.program}; - - // Propagate arguments from the CLI - for (auto &i : args) { - programArgs.push_back(i); - } - - // Release our references to eval caches to ensure they are persisted to disk, because - // we are about to exec out of this process without running C++ destructors. - evalState->evalCaches.clear(); - - execProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); - }; -}; - -static auto r2 = registerCommand("fmt"); diff --git a/src/nix/fmt.md b/src/nix/formatter-run.md similarity index 90% rename from src/nix/fmt.md rename to src/nix/formatter-run.md index b4693eb65..a01b52a9d 100644 --- a/src/nix/fmt.md +++ b/src/nix/formatter-run.md @@ -2,7 +2,7 @@ R""( # Description -`nix fmt` calls the formatter specified in the flake. +`nix fmt` (an alias for `nix formatter run`) calls the formatter specified in the flake. Flags can be forwarded to the formatter by using `--` followed by the flags. diff --git a/src/nix/formatter.cc b/src/nix/formatter.cc new file mode 100644 index 000000000..80ce64d28 --- /dev/null +++ b/src/nix/formatter.cc @@ -0,0 +1,98 @@ +#include "nix/cmd/command.hh" +#include "nix/cmd/installable-value.hh" +#include "nix/expr/eval.hh" +#include "run.hh" + +using namespace nix; + +struct CmdFormatter : NixMultiCommand +{ + CmdFormatter() + : NixMultiCommand("formatter", RegisterCommand::getCommandsFor({"formatter"})) + { + } + + std::string description() override + { + return "build or run the formatter"; + } + + Category category() override + { + return catSecondary; + } +}; + +static auto rCmdFormatter = registerCommand("formatter"); + +struct CmdFormatterRun : SourceExprCommand +{ + std::vector args; + + CmdFormatterRun() + { + expectArgs({.label = "args", .handler = {&args}}); + } + + std::string description() override + { + return "reformat your code in the standard style"; + } + + std::string doc() override + { + return +#include "formatter-run.md" + ; + } + + Category category() override + { + return catSecondary; + } + + Strings getDefaultFlakeAttrPaths() override + { + return Strings{"formatter." + settings.thisSystem.get()}; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + return Strings{}; + } + + void run(ref store) override + { + auto evalState = getEvalState(); + auto evalStore = getEvalStore(); + + auto installable_ = parseInstallable(store, "."); + auto & installable = InstallableValue::require(*installable_); + auto app = installable.toApp(*evalState).resolve(evalStore, store); + + Strings programArgs{app.program}; + + // Propagate arguments from the CLI + for (auto & i : args) { + programArgs.push_back(i); + } + + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + evalState->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); + }; +}; + +static auto rFormatterRun = registerCommand2({"formatter", "run"}); + +struct CmdFmt : CmdFormatterRun +{ + void run(ref store) override + { + CmdFormatterRun::run(store); + } +}; + +static auto rFmt = registerCommand("fmt"); diff --git a/src/nix/meson.build b/src/nix/meson.build index 901021330..11c30914b 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -78,7 +78,7 @@ nix_sources = [config_priv_h] + files( 'env.cc', 'eval.cc', 'flake.cc', - 'fmt.cc', + 'formatter.cc', 'hash.cc', 'log.cc', 'ls.cc', diff --git a/tests/functional/fmt.sh b/tests/functional/formatter.sh similarity index 100% rename from tests/functional/fmt.sh rename to tests/functional/formatter.sh diff --git a/tests/functional/fmt.simple.sh b/tests/functional/formatter.simple.sh similarity index 100% rename from tests/functional/fmt.simple.sh rename to tests/functional/formatter.simple.sh diff --git a/tests/functional/meson.build b/tests/functional/meson.build index f2d6a64ea..b2005d9d9 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -132,7 +132,7 @@ suites = [ 'nix-copy-ssh-ng.sh', 'post-hook.sh', 'function-trace.sh', - 'fmt.sh', + 'formatter.sh', 'eval-store.sh', 'why-depends.sh', 'derivation-json.sh', From ba6b617e755e229f44f8f1d37fe969de1de6e32c Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Wed, 23 Apr 2025 16:54:01 -0700 Subject: [PATCH 278/396] Add `nix formatter build` command `nix formatter build` is sort of like `nix build`: it builds, links, and prints a path to the formatter program: $ nix formatter build /nix/store/cb9w44vkhk2x4adfxwgdkkf5gjmm856j-treefmt/bin/treefmt Note that unlike `nix build`, this prints the full path to the program, not just the store path (in the example above that would be `/nix/store/cb9w44vkhk2x4adfxwgdkkf5gjmm856j-treefmt`). Motivation ---------- I maintain a vim plugin that automatically runs `nix fmt` on files on save. Since `nix fmt` can be quite slow due to nix evaluation, I choose to cache the `nix fmt `entrypoint. This was very awkward to do, see the implementation for details: https://github.com/nvimtools/none-ls.nvim/blob/786460723170bda9e9f95c55a382d21436575297/lua/null-ls/builtins/formatting/nix_flake_fmt.lua#L83-L110. I recently discovered that my implementation was buggy (it didn't handle flakes that expose a `formatter` package, such as nixpkgs), so I had to rework the implementation: https://github.com/nvimtools/none-ls.nvim/pull/272. With the new `nix formatter build` command, I can delete all this akward code, and it will be easier for other folks to build performant editor integrations for `nix fmt`. --- src/nix/formatter-build.md | 23 +++++++++++ src/nix/formatter.cc | 78 ++++++++++++++++++++++++++++++++++- tests/functional/formatter.sh | 30 ++++++++++++-- 3 files changed, 127 insertions(+), 4 deletions(-) create mode 100644 src/nix/formatter-build.md diff --git a/src/nix/formatter-build.md b/src/nix/formatter-build.md new file mode 100644 index 000000000..fbb6adb1c --- /dev/null +++ b/src/nix/formatter-build.md @@ -0,0 +1,23 @@ +R""( + +# Description + +`nix formatter build` builds the formatter specified in the flake. + +Similar to [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md), +unless `--no-link` is specified, after a successful +build, it creates a symlink to the store path of the formatter. This symlink is +named `./result` by default; this can be overridden using the +`--out-link` option. + +It always prints the command to standard output. + +# Examples + +* Build the formatter: + + ```console + # nix formatter build + /nix/store/cb9w44vkhk2x4adfxwgdkkf5gjmm856j-treefmt/bin/treefmt + ``` +)"" diff --git a/src/nix/formatter.cc b/src/nix/formatter.cc index 80ce64d28..0f915e7c9 100644 --- a/src/nix/formatter.cc +++ b/src/nix/formatter.cc @@ -1,6 +1,8 @@ #include "nix/cmd/command.hh" #include "nix/cmd/installable-value.hh" #include "nix/expr/eval.hh" +#include "nix/store/local-fs-store.hh" +#include "nix/cmd/installable-derived-path.hh" #include "run.hh" using namespace nix; @@ -25,7 +27,7 @@ struct CmdFormatter : NixMultiCommand static auto rCmdFormatter = registerCommand("formatter"); -struct CmdFormatterRun : SourceExprCommand +struct CmdFormatterRun : SourceExprCommand, MixJSON { std::vector args; @@ -87,6 +89,80 @@ struct CmdFormatterRun : SourceExprCommand static auto rFormatterRun = registerCommand2({"formatter", "run"}); +struct CmdFormatterBuild : SourceExprCommand +{ + Path outLink = "result"; + + CmdFormatterBuild() + { + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "Use *path* as prefix for the symlink to the build result. It defaults to `result`.", + .labels = {"path"}, + .handler = {&outLink}, + .completer = completePath, + }); + + addFlag({ + .longName = "no-link", + .description = "Do not create symlink to the build results.", + .handler = {&outLink, Path("")}, + }); + } + + std::string description() override + { + return "build the current flake's formatter"; + } + + std::string doc() override + { + return +#include "formatter-build.md" + ; + } + + Category category() override + { + return catSecondary; + } + + Strings getDefaultFlakeAttrPaths() override + { + return Strings{"formatter." + settings.thisSystem.get()}; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + return Strings{}; + } + + void run(ref store) override + { + auto evalState = getEvalState(); + auto evalStore = getEvalStore(); + + auto installable_ = parseInstallable(store, "."); + auto & installable = InstallableValue::require(*installable_); + auto unresolvedApp = installable.toApp(*evalState); + auto app = unresolvedApp.resolve(evalStore, store); + + Installables installableContext; + for (auto & ctxElt : unresolvedApp.unresolved.context) + installableContext.push_back(make_ref(store, DerivedPath{ctxElt})); + auto buildables = Installable::build(evalStore, store, Realise::Outputs, installableContext); + + if (outLink != "") + if (auto store2 = store.dynamic_pointer_cast()) + createOutLinks(outLink, toBuiltPaths(buildables), *store2); + + logger->cout("%s", app.program); + }; +}; + +static auto rFormatterBuild = registerCommand2({"formatter", "build"}); + struct CmdFmt : CmdFormatterRun { void run(ref store) override diff --git a/tests/functional/formatter.sh b/tests/functional/formatter.sh index e9bff50d5..ea6f9e1ce 100755 --- a/tests/functional/formatter.sh +++ b/tests/functional/formatter.sh @@ -7,11 +7,14 @@ TODO_NixOS # Provide a `shell` variable. Try not to `export` it, perhaps. clearStoreIfPossible rm -rf "$TEST_HOME"/.cache "$TEST_HOME"/.config "$TEST_HOME"/.local -cp ./simple.nix ./simple.builder.sh ./fmt.simple.sh "${config_nix}" "$TEST_HOME" +cp ./simple.nix ./simple.builder.sh ./formatter.simple.sh "${config_nix}" "$TEST_HOME" cd "$TEST_HOME" -nix fmt --help | grep "forward" +nix formatter --help | grep "build or run the formatter" +nix fmt --help | grep "reformat your code" +nix fmt run --help | grep "reformat your code" +nix fmt build --help | grep "build" cat << EOF > flake.nix { @@ -23,16 +26,37 @@ cat << EOF > flake.nix buildCommand = '' mkdir -p \$out/bin echo "#! ${shell}" > \$out/bin/formatter - cat \${./fmt.simple.sh} >> \$out/bin/formatter + cat \${./formatter.simple.sh} >> \$out/bin/formatter chmod +x \$out/bin/formatter ''; }; }; } EOF + # No arguments check [[ "$(nix fmt)" = "Formatting(0):" ]] +[[ "$(nix formatter run)" = "Formatting(0):" ]] + # Argument forwarding check nix fmt ./file ./folder | grep 'Formatting(2): ./file ./folder' +nix formatter run ./file ./folder | grep 'Formatting(2): ./file ./folder' + +# Build checks +## Defaults to a ./result. +nix formatter build | grep ".\+/bin/formatter" +[[ -L ./result ]] +rm result + +## Can prevent the symlink. +nix formatter build --no-link +[[ ! -e ./result ]] + +## Can change the symlink name. +nix formatter build --out-link my-result | grep ".\+/bin/formatter" +[[ -L ./my-result ]] +rm ./my-result + +# Flake outputs check. nix flake check nix flake show | grep -P "package 'formatter'" From e14346c7daad22782eb872ab5daf42e812090104 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 28 Apr 2025 15:08:15 +0200 Subject: [PATCH 279/396] Refactor, dedup nix formatter attribute methods --- src/nix/formatter.cc | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/src/nix/formatter.cc b/src/nix/formatter.cc index 0f915e7c9..31275eb35 100644 --- a/src/nix/formatter.cc +++ b/src/nix/formatter.cc @@ -27,7 +27,21 @@ struct CmdFormatter : NixMultiCommand static auto rCmdFormatter = registerCommand("formatter"); -struct CmdFormatterRun : SourceExprCommand, MixJSON +/** Common implementation bits for the `nix formatter` subcommands. */ +struct MixFormatter : SourceExprCommand +{ + Strings getDefaultFlakeAttrPaths() override + { + return Strings{"formatter." + settings.thisSystem.get()}; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + return Strings{}; + } +}; + +struct CmdFormatterRun : MixFormatter, MixJSON { std::vector args; @@ -53,16 +67,6 @@ struct CmdFormatterRun : SourceExprCommand, MixJSON return catSecondary; } - Strings getDefaultFlakeAttrPaths() override - { - return Strings{"formatter." + settings.thisSystem.get()}; - } - - Strings getDefaultFlakeAttrPathPrefixes() override - { - return Strings{}; - } - void run(ref store) override { auto evalState = getEvalState(); @@ -89,7 +93,7 @@ struct CmdFormatterRun : SourceExprCommand, MixJSON static auto rFormatterRun = registerCommand2({"formatter", "run"}); -struct CmdFormatterBuild : SourceExprCommand +struct CmdFormatterBuild : MixFormatter { Path outLink = "result"; @@ -128,16 +132,6 @@ struct CmdFormatterBuild : SourceExprCommand return catSecondary; } - Strings getDefaultFlakeAttrPaths() override - { - return Strings{"formatter." + settings.thisSystem.get()}; - } - - Strings getDefaultFlakeAttrPathPrefixes() override - { - return Strings{}; - } - void run(ref store) override { auto evalState = getEvalState(); From 7df7bde3065e3ffbadf8ef6c9c3d83f7f868209f Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Tue, 29 Apr 2025 18:36:56 -0700 Subject: [PATCH 280/396] Refactor, extract some shared code into `UnresolvedApp::build` --- src/libcmd/include/nix/cmd/installable-value.hh | 1 + src/nix/app.cc | 15 ++++++++++----- src/nix/formatter.cc | 6 +----- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/libcmd/include/nix/cmd/installable-value.hh b/src/libcmd/include/nix/cmd/installable-value.hh index 9c8f1a9fb..e65c199a5 100644 --- a/src/libcmd/include/nix/cmd/installable-value.hh +++ b/src/libcmd/include/nix/cmd/installable-value.hh @@ -21,6 +21,7 @@ struct App struct UnresolvedApp { App unresolved; + std::vector build(ref evalStore, ref store); App resolve(ref evalStore, ref store); }; diff --git a/src/nix/app.cc b/src/nix/app.cc index 75ef874ba..c9a9f9caf 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -129,18 +129,23 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) throw Error("attribute '%s' has unsupported type '%s'", cursor->getAttrPathStr(), type); } -// FIXME: move to libcmd -App UnresolvedApp::resolve(ref evalStore, ref store) +std::vector UnresolvedApp::build(ref evalStore, ref store) { - auto res = unresolved; - Installables installableContext; for (auto & ctxElt : unresolved.context) installableContext.push_back( make_ref(store, DerivedPath { ctxElt })); - auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); + return Installable::build(evalStore, store, Realise::Outputs, installableContext); +} + +// FIXME: move to libcmd +App UnresolvedApp::resolve(ref evalStore, ref store) +{ + auto res = unresolved; + + auto builtContext = build(evalStore, store); res.program = resolveString(*store, unresolved.program, builtContext); if (!store->isInStore(res.program)) throw Error("app program '%s' is not in the Nix store", res.program); diff --git a/src/nix/formatter.cc b/src/nix/formatter.cc index 31275eb35..f4cde7d68 100644 --- a/src/nix/formatter.cc +++ b/src/nix/formatter.cc @@ -141,11 +141,7 @@ struct CmdFormatterBuild : MixFormatter auto & installable = InstallableValue::require(*installable_); auto unresolvedApp = installable.toApp(*evalState); auto app = unresolvedApp.resolve(evalStore, store); - - Installables installableContext; - for (auto & ctxElt : unresolvedApp.unresolved.context) - installableContext.push_back(make_ref(store, DerivedPath{ctxElt})); - auto buildables = Installable::build(evalStore, store, Realise::Outputs, installableContext); + auto buildables = unresolvedApp.build(evalStore, store); if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) From 5089f1292dabd01e9d27c5752e498d9a00f13b0d Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Tue, 29 Apr 2025 18:37:20 -0700 Subject: [PATCH 281/396] Refactor, use `MixOutLinkByDefault` --- src/nix/formatter.cc | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/src/nix/formatter.cc b/src/nix/formatter.cc index f4cde7d68..8b171b244 100644 --- a/src/nix/formatter.cc +++ b/src/nix/formatter.cc @@ -93,27 +93,9 @@ struct CmdFormatterRun : MixFormatter, MixJSON static auto rFormatterRun = registerCommand2({"formatter", "run"}); -struct CmdFormatterBuild : MixFormatter +struct CmdFormatterBuild : MixFormatter, MixOutLinkByDefault { - Path outLink = "result"; - - CmdFormatterBuild() - { - addFlag({ - .longName = "out-link", - .shortName = 'o', - .description = "Use *path* as prefix for the symlink to the build result. It defaults to `result`.", - .labels = {"path"}, - .handler = {&outLink}, - .completer = completePath, - }); - - addFlag({ - .longName = "no-link", - .description = "Do not create symlink to the build results.", - .handler = {&outLink, Path("")}, - }); - } + CmdFormatterBuild() {} std::string description() override { @@ -142,10 +124,7 @@ struct CmdFormatterBuild : MixFormatter auto unresolvedApp = installable.toApp(*evalState); auto app = unresolvedApp.resolve(evalStore, store); auto buildables = unresolvedApp.build(evalStore, store); - - if (outLink != "") - if (auto store2 = store.dynamic_pointer_cast()) - createOutLinks(outLink, toBuiltPaths(buildables), *store2); + createOutLinksMaybe(buildables, store); logger->cout("%s", app.program); }; From d46f741cdfcb4b9b6aed5532dec7fe62f1e3cb72 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 30 Apr 2025 22:29:27 +0200 Subject: [PATCH 282/396] Complain when using --pure-eval with --file This never worked and cannot work because in pure eval mode, the evaluator doesn't have access to the file. --- src/libcmd/installables.cc | 6 +++++- tests/functional/eval.sh | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 1c414e9e2..080557d4f 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -491,7 +491,11 @@ Installables SourceExprCommand::parseInstallables( throw UsageError("'--file' and '--expr' are exclusive"); // FIXME: backward compatibility hack - if (file) evalSettings.pureEval = false; + if (file) { + if (evalSettings.pureEval && evalSettings.pureEval.overridden) + throw UsageError("'--file' is not compatible with '--pure-eval'"); + evalSettings.pureEval = false; + } auto state = getEvalState(); auto vFile = state->allocValue(); diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index ed9c214f5..f876f5ac4 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -39,6 +39,9 @@ nix-instantiate --eval -E 'assert 1 + 2 == 3; true' ln -sfn cycle.nix "$TEST_ROOT/cycle.nix" (! nix eval --file "$TEST_ROOT/cycle.nix") +# --file and --pure-eval don't mix. +expectStderr 1 nix eval --pure-eval --file "$TEST_ROOT/cycle.nix" | grepQuiet "not compatible" + # Check that relative symlinks are resolved correctly. mkdir -p "$TEST_ROOT/xyzzy" "$TEST_ROOT/foo" ln -sfn ../xyzzy "$TEST_ROOT/foo/bar" From 408746cba6128bad91730946d8b45e9548267d92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 30 Apr 2025 10:20:17 +0200 Subject: [PATCH 283/396] replaceSymlink: fix quoting in error message --- src/libutil/file-system.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 071aecc9d..83c692e49 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -629,14 +629,14 @@ void replaceSymlink(const fs::path & target, const fs::path & link) fs::create_symlink(target, tmp); } catch (fs::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; - throw SysError("creating symlink '%1%' -> '%2%'", tmp, target); + throw SysError("creating symlink %1% -> %2%", tmp, target); } try { fs::rename(tmp, link); } catch (fs::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; - throw SysError("renaming '%1%' to '%2%'", tmp, link); + throw SysError("renaming %1% to %2%", tmp, link); } From 7ccc0d591f3eb879af8eae95c994405a1172d210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 1 May 2025 09:50:53 +0200 Subject: [PATCH 284/396] add DirectoryIterator to re-throw std::filesystem::filesystem_error Co-authored-by: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> --- src/libutil-tests/file-system.cc | 21 ++++++++ src/libutil/file-system.cc | 28 ++++++++++ src/libutil/include/nix/util/file-system.hh | 60 +++++++++++++++++++++ 3 files changed, 109 insertions(+) diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 9e4a27750..2d1058c4f 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -297,4 +297,25 @@ TEST(chmodIfNeeded, nonexistent) ASSERT_THROW(chmodIfNeeded("/schnitzel/darmstadt/pommes", 0755), SysError); } +/* ---------------------------------------------------------------------------- + * DirectoryIterator + * --------------------------------------------------------------------------*/ + +TEST(DirectoryIterator, works) +{ + auto tmpDir = nix::createTempDir(); + nix::AutoDelete delTmpDir(tmpDir, true); + + nix::writeFile(tmpDir + "/somefile", ""); + + for (auto path : DirectoryIterator(tmpDir)) { + ASSERT_EQ(path.path().string(), tmpDir + "/somefile"); + } +} + +TEST(DirectoryIterator, nonexistent) +{ + ASSERT_THROW(DirectoryIterator("/schnitzel/darmstadt/pommes"), SysError); +} + } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 071aecc9d..b677c8ccb 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -43,6 +43,34 @@ namespace fs { } } +DirectoryIterator::DirectoryIterator(const std::filesystem::path& p) { + try { + // **Attempt to create the underlying directory_iterator** + it_ = std::filesystem::directory_iterator(p); + } catch (const std::filesystem::filesystem_error& e) { + // **Catch filesystem_error and throw SysError** + // Adapt the error message as needed for SysError + throw SysError("cannot read directory %s", p); + } +} + +DirectoryIterator& DirectoryIterator::operator++() { + // **Attempt to increment the underlying iterator** + std::error_code ec; + it_.increment(ec); + if (ec) { + // Try to get path info if possible, might fail if iterator is bad + try { + if (it_ != std::filesystem::directory_iterator{}) { + throw SysError("cannot read directory past %s: %s", it_->path(), ec.message()); + } + } catch (...) { + throw SysError("cannot read directory"); + } + } + return *this; +} + bool isAbsolute(PathView path) { return fs::path { path }.is_absolute(); diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 5c2416750..9afab46ab 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -376,4 +376,64 @@ extern PathFilter defaultPathFilter; */ bool chmodIfNeeded(const std::filesystem::path & path, mode_t mode, mode_t mask = S_IRWXU | S_IRWXG | S_IRWXO); +/** + * @brief A directory iterator that can be used to iterate over the + * contents of a directory. It is similar to std::filesystem::directory_iterator + * but throws NixError on failure instead of std::filesystem::filesystem_error. + */ +class DirectoryIterator { +public: + // --- Iterator Traits --- + using iterator_category = std::input_iterator_tag; + using value_type = std::filesystem::directory_entry; + using difference_type = std::ptrdiff_t; + using pointer = const std::filesystem::directory_entry*; + using reference = const std::filesystem::directory_entry&; + + // Default constructor (represents end iterator) + DirectoryIterator() noexcept = default; + + // Constructor taking a path + explicit DirectoryIterator(const std::filesystem::path& p); + + reference operator*() const { + // Accessing the value itself doesn't typically throw filesystem_error + // after successful construction/increment, but underlying operations might. + // If directory_entry methods called via -> could throw, add try-catch there. + return *it_; + } + + pointer operator->() const { + return &(*it_); + } + + + DirectoryIterator& operator++(); + + // Postfix increment operator + DirectoryIterator operator++(int) { + DirectoryIterator temp = *this; + ++(*this); // Uses the prefix increment's try-catch logic + return temp; + } + + // Equality comparison + friend bool operator==(const DirectoryIterator& a, const DirectoryIterator& b) noexcept { + return a.it_ == b.it_; + } + + // Inequality comparison + friend bool operator!=(const DirectoryIterator& a, const DirectoryIterator& b) noexcept { + return !(a == b); + } + + // Allow direct use in range-based for loops if iterating over an instance + DirectoryIterator begin() const { return *this; } + DirectoryIterator end() const { return DirectoryIterator{}; } + + +private: + std::filesystem::directory_iterator it_; +}; + } From 1c4496f4e5c51b1036087f95889033a71b33edf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 1 May 2025 09:54:14 +0200 Subject: [PATCH 285/396] replace all instances of std::filesystem::directory_iterator with DirectoryIterator --- src/libcmd/repl.cc | 3 +- src/libmain/plugin.cc | 7 ++- src/libstore/builtins/buildenv.cc | 8 +-- src/libstore/builtins/unpack-channel.cc | 10 +--- src/libstore/gc.cc | 4 +- src/libstore/local-binary-cache-store.cc | 2 +- src/libstore/local-store.cc | 4 +- src/libstore/posix-fs-canonicalise.cc | 2 +- src/libstore/profiles.cc | 2 +- src/libutil/file-system.cc | 4 +- src/libutil/linux/cgroup.cc | 2 +- src/libutil/posix-source-accessor.cc | 60 +++++++++---------- src/libutil/unix/file-descriptor.cc | 3 +- .../nix-collect-garbage.cc | 2 +- src/nix/prefetch.cc | 4 +- src/nix/run.cc | 2 +- 16 files changed, 55 insertions(+), 64 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index c5a95268b..8bbda1a4b 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -244,14 +244,13 @@ StringSet NixRepl::completePrefix(const std::string & prefix) try { auto dir = std::string(cur, 0, slash); auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) { + for (auto & entry : DirectoryIterator{dir == "" ? "/" : dir}) { checkInterrupt(); auto name = entry.path().filename().string(); if (name[0] != '.' && hasPrefix(name, prefix2)) completions.insert(prev + entry.path().string()); } } catch (Error &) { - } catch (std::filesystem::filesystem_error &) { } } else if ((dot = cur.rfind('.')) == std::string::npos) { /* This is a variable name; look it up in the current scope. */ diff --git a/src/libmain/plugin.cc b/src/libmain/plugin.cc index 63ed650a7..812506a86 100644 --- a/src/libmain/plugin.cc +++ b/src/libmain/plugin.cc @@ -6,6 +6,7 @@ #include "nix/util/config-global.hh" #include "nix/util/signals.hh" +#include "nix/util/file-system.hh" namespace nix { @@ -77,13 +78,13 @@ void initPlugins() for (const auto & pluginFile : pluginSettings.pluginFiles.get()) { std::vector pluginFiles; try { - auto ents = std::filesystem::directory_iterator{pluginFile}; + auto ents = DirectoryIterator{pluginFile}; for (const auto & ent : ents) { checkInterrupt(); pluginFiles.emplace_back(ent.path()); } - } catch (std::filesystem::filesystem_error & e) { - if (e.code() != std::errc::not_a_directory) + } catch (SysError & e) { + if (e.errNo != ENOTDIR) throw; pluginFiles.emplace_back(pluginFile); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c3b80bb0b..74036d22e 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -18,12 +18,12 @@ struct State /* For each activated package, create symlinks */ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority) { - std::filesystem::directory_iterator srcFiles; + DirectoryIterator srcFiles; try { - srcFiles = std::filesystem::directory_iterator{srcDir}; - } catch (std::filesystem::filesystem_error & e) { - if (e.code() == std::errc::not_a_directory) { + srcFiles = DirectoryIterator{srcDir}; + } catch (SysError & e) { + if (e.errNo == ENOTDIR) { warn("not including '%s' in the user environment because it's not a directory", srcDir); return; } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index f6be21e35..07b1d5773 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -29,13 +29,9 @@ void builtinUnpackChannel( size_t fileCount; std::string fileName; - try { - auto entries = fs::directory_iterator{out}; - fileName = entries->path().string(); - fileCount = std::distance(fs::begin(entries), fs::end(entries)); - } catch (fs::filesystem_error &) { - throw SysError("failed to read directory %1%", out.string()); - } + auto entries = DirectoryIterator{out}; + fileName = entries->path().string(); + fileCount = std::distance(entries.begin(), entries.end()); if (fileCount != 1) throw Error("channel tarball '%s' contains more than one file", src); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index db91e5213..f6a4124ff 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -164,7 +164,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) { /* Read the `temproots' directory for per-process temporary root files. */ - for (auto & i : std::filesystem::directory_iterator{tempRootsDir}) { + for (auto & i : DirectoryIterator{tempRootsDir}) { checkInterrupt(); auto name = i.path().filename().string(); if (name[0] == '.') { @@ -232,7 +232,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R type = std::filesystem::symlink_status(path).type(); if (type == std::filesystem::file_type::directory) { - for (auto & i : std::filesystem::directory_iterator{path}) { + for (auto & i : DirectoryIterator{path}) { checkInterrupt(); findRoots(i.path().string(), i.symlink_status().type(), roots); } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 212eacc8c..4d1be5785 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -84,7 +84,7 @@ protected: { StorePathSet paths; - for (auto & entry : std::filesystem::directory_iterator{binaryCacheDir}) { + for (auto & entry : DirectoryIterator{binaryCacheDir}) { checkInterrupt(); auto name = entry.path().filename().string(); if (name.size() != 40 || diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1c6d6bced..ade209b8a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1382,7 +1382,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking link hashes..."); - for (auto & link : std::filesystem::directory_iterator{linksDir}) { + for (auto & link : DirectoryIterator{linksDir}) { checkInterrupt(); auto name = link.path().filename(); printMsg(lvlTalkative, "checking contents of '%s'", name); @@ -1475,7 +1475,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair database and the filesystem) in the loop below, in order to catch invalid states. */ - for (auto & i : std::filesystem::directory_iterator{realStoreDir.to_string()}) { + for (auto & i : DirectoryIterator{realStoreDir.to_string()}) { checkInterrupt(); try { storePathsInStoreDir.insert({i.path().filename().string()}); diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index aeb35eab5..792fe5c76 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -136,7 +136,7 @@ static void canonicalisePathMetaData_( #endif if (S_ISDIR(st.st_mode)) { - for (auto & i : std::filesystem::directory_iterator{path}) { + for (auto & i : DirectoryIterator{path}) { checkInterrupt(); canonicalisePathMetaData_( i.path().string(), diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index bd24332cb..b5161b79f 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -38,7 +38,7 @@ std::pair> findGenerations(Path pro std::filesystem::path profileDir = dirOf(profile); auto profileName = std::string(baseNameOf(profile)); - for (auto & i : std::filesystem::directory_iterator{profileDir}) { + for (auto & i : DirectoryIterator{profileDir}) { checkInterrupt(); if (auto n = parseName(profileName, i.path().filename().string())) { auto path = i.path().string(); diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index b677c8ccb..839e6ff29 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -387,7 +387,7 @@ void recursiveSync(const Path & path) while (!dirsToEnumerate.empty()) { auto currentDir = dirsToEnumerate.back(); dirsToEnumerate.pop_back(); - for (auto & entry : std::filesystem::directory_iterator(currentDir)) { + for (auto & entry : DirectoryIterator(currentDir)) { auto st = entry.symlink_status(); if (fs::is_directory(st)) { dirsToEnumerate.emplace_back(entry.path()); @@ -691,7 +691,7 @@ void copyFile(const fs::path & from, const fs::path & to, bool andDelete) fs::copy(from, to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); } else if (fs::is_directory(fromStatus)) { fs::create_directory(to); - for (auto & entry : fs::directory_iterator(from)) { + for (auto & entry : DirectoryIterator(from)) { copyFile(entry, to / entry.path().filename(), andDelete); } } else { diff --git a/src/libutil/linux/cgroup.cc b/src/libutil/linux/cgroup.cc index e8e2fdfc7..4acfe82f1 100644 --- a/src/libutil/linux/cgroup.cc +++ b/src/libutil/linux/cgroup.cc @@ -65,7 +65,7 @@ static CgroupStats destroyCgroup(const std::filesystem::path & cgroup, bool retu /* Otherwise, manually kill every process in the subcgroups and this cgroup. */ - for (auto & entry : std::filesystem::directory_iterator{cgroup}) { + for (auto & entry : DirectoryIterator{cgroup}) { checkInterrupt(); if (entry.symlink_status().type() != std::filesystem::file_type::directory) continue; destroyCgroup(cgroup / entry.path().filename(), false); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 5c7b4654b..773540e6a 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -138,42 +138,38 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & { assertNoSymlinks(path); DirEntries res; - try { - for (auto & entry : std::filesystem::directory_iterator{makeAbsPath(path)}) { - checkInterrupt(); - auto type = [&]() -> std::optional { - std::filesystem::file_type nativeType; - try { - nativeType = entry.symlink_status().type(); - } catch (std::filesystem::filesystem_error & e) { - // We cannot always stat the child. (Ideally there is no - // stat because the native directory entry has the type - // already, but this isn't always the case.) - if (e.code() == std::errc::permission_denied || e.code() == std::errc::operation_not_permitted) - return std::nullopt; - else throw; - } + for (auto & entry : DirectoryIterator{makeAbsPath(path)}) { + checkInterrupt(); + auto type = [&]() -> std::optional { + std::filesystem::file_type nativeType; + try { + nativeType = entry.symlink_status().type(); + } catch (std::filesystem::filesystem_error & e) { + // We cannot always stat the child. (Ideally there is no + // stat because the native directory entry has the type + // already, but this isn't always the case.) + if (e.code() == std::errc::permission_denied || e.code() == std::errc::operation_not_permitted) + return std::nullopt; + else throw; + } - // cannot exhaustively enumerate because implementation-specific - // additional file types are allowed. + // cannot exhaustively enumerate because implementation-specific + // additional file types are allowed. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wswitch-enum" - switch (nativeType) { - case std::filesystem::file_type::regular: return Type::tRegular; break; - case std::filesystem::file_type::symlink: return Type::tSymlink; break; - case std::filesystem::file_type::directory: return Type::tDirectory; break; - case std::filesystem::file_type::character: return Type::tChar; break; - case std::filesystem::file_type::block: return Type::tBlock; break; - case std::filesystem::file_type::fifo: return Type::tFifo; break; - case std::filesystem::file_type::socket: return Type::tSocket; break; - default: return tUnknown; - } -#pragma GCC diagnostic pop - }(); - res.emplace(entry.path().filename().string(), type); + switch (nativeType) { + case std::filesystem::file_type::regular: return Type::tRegular; break; + case std::filesystem::file_type::symlink: return Type::tSymlink; break; + case std::filesystem::file_type::directory: return Type::tDirectory; break; + case std::filesystem::file_type::character: return Type::tChar; break; + case std::filesystem::file_type::block: return Type::tBlock; break; + case std::filesystem::file_type::fifo: return Type::tFifo; break; + case std::filesystem::file_type::socket: return Type::tSocket; break; + default: return tUnknown; } - } catch (std::filesystem::filesystem_error & e) { - throw SysError("reading directory %1%", showPath(path)); +#pragma GCC diagnostic pop + }(); + res.emplace(entry.path().filename().string(), type); } return res; } diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index 73ee49982..e6d0c255d 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -191,7 +191,7 @@ void unix::closeExtraFDs() #ifdef __linux__ try { - for (auto & s : std::filesystem::directory_iterator{"/proc/self/fd"}) { + for (auto & s : DirectoryIterator{"/proc/self/fd"}) { checkInterrupt(); auto fd = std::stoi(s.path().filename()); if (fd > MAX_KEPT_FD) { @@ -201,7 +201,6 @@ void unix::closeExtraFDs() } return; } catch (SysError &) { - } catch (std::filesystem::filesystem_error &) { } #endif diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 3a84d97aa..8e4fc30fb 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -30,7 +30,7 @@ void removeOldGenerations(fs::path dir) bool canWrite = access(dir.string().c_str(), W_OK) == 0; - for (auto & i : fs::directory_iterator{dir}) { + for (auto & i : DirectoryIterator{dir}) { checkInterrupt(); auto path = i.path().string(); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 6819e596d..d56b73bf1 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -116,11 +116,11 @@ std::tuple prefetchFile( createDirs(unpacked); unpackTarfile(tmpFile.string(), unpacked); - auto entries = std::filesystem::directory_iterator{unpacked}; + auto entries = DirectoryIterator{unpacked}; /* If the archive unpacks to a single file/directory, then use that as the top-level. */ tmpFile = entries->path(); - auto fileCount = std::distance(entries, std::filesystem::directory_iterator{}); + auto fileCount = std::distance(entries, DirectoryIterator{}); if (fileCount != 1) { /* otherwise, use the directory itself */ tmpFile = unpacked; diff --git a/src/nix/run.cc b/src/nix/run.cc index 146ae9ec9..10f5026cd 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -179,7 +179,7 @@ void chrootHelper(int argc, char * * argv) if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - for (const auto & entry : fs::directory_iterator{"/"}) { + for (const auto & entry : DirectoryIterator{"/"}) { checkInterrupt(); const auto & src = entry.path(); fs::path dst = tmpDir / entry.path().filename(); From 5b59be914da9f7183719521e619f9f661b094adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 1 May 2025 11:43:37 +0200 Subject: [PATCH 286/396] Replace symlink_exists with pathExists As it turns out the orignal implementation of symlink_exists cannot be used in Nix because it did now std::filesystem::filesystem_error. The new implementation fixes that but is now actually the same as pathExists except for the path type. --- src/libfetchers/git.cc | 2 +- src/libflake/flake.cc | 2 +- src/libstore/unix/build/derivation-builder.cc | 5 +++-- src/libutil/file-system.cc | 16 ++------------- src/libutil/include/nix/util/file-system.hh | 20 +------------------ src/nix/eval.cc | 2 +- 6 files changed, 9 insertions(+), 38 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 80c5884f7..492867dc4 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -533,7 +533,7 @@ struct GitInputScheme : InputScheme static MakeNotAllowedError makeNotAllowedError(std::filesystem::path repoPath) { return [repoPath{std::move(repoPath)}](const CanonPath & path) -> RestrictedPathError { - if (fs::symlink_exists(repoPath / path.rel())) + if (pathExists(repoPath / path.rel())) return RestrictedPathError( "Path '%1%' in the repository %2% is not tracked by Git.\n" "\n" diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 28e3c4b93..a086251b9 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -811,7 +811,7 @@ LockedFlake lockFlake( auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; auto outputLockFilePath = *sourcePath / relPath; - bool lockFileExists = fs::symlink_exists(outputLockFilePath); + bool lockFileExists = pathExists(outputLockFilePath); auto s = chomp(diff); if (lockFileExists) { diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 76f69671c..4bde9750d 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1969,9 +1969,10 @@ void DerivationBuilderImpl::runChild() if (pathExists(path)) ss.push_back(path); - if (settings.caFile != "" && pathExists(settings.caFile)) { + if (settings.caFile != "") { Path caFile = settings.caFile; - pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true); + if (pathExists(caFile)) + pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true); } } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index f6e399774..edd68d6e5 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -31,18 +31,6 @@ namespace nix { -namespace fs { - using namespace std::filesystem; - - bool symlink_exists(const std::filesystem::path & path) { - try { - return std::filesystem::exists(std::filesystem::symlink_status(path)); - } catch (const std::filesystem::filesystem_error & e) { - throw SysError("cannot check existence of %1%", path); - } - } -} - DirectoryIterator::DirectoryIterator(const std::filesystem::path& p) { try { // **Attempt to create the underlying directory_iterator** @@ -243,9 +231,9 @@ std::optional maybeLstat(const Path & path) } -bool pathExists(const Path & path) +bool pathExists(const std::filesystem::path & path) { - return maybeLstat(path).has_value(); + return maybeLstat(path.string()).has_value(); } bool pathAccessible(const std::filesystem::path & path) diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 9afab46ab..1d7b5e3aa 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -126,26 +126,8 @@ std::optional maybeLstat(const Path & path); /** * @return true iff the given path exists. - * - * In the process of being deprecated for `fs::symlink_exists`. */ -bool pathExists(const Path & path); - -namespace fs { - -/** - * TODO: we may actually want to use pathExists instead of this function - * ``` - * symlink_exists(p) = std::filesystem::exists(std::filesystem::symlink_status(p)) - * ``` - * Missing convenience analogous to - * ``` - * std::filesystem::exists(p) = std::filesystem::exists(std::filesystem::status(p)) - * ``` - */ -bool symlink_exists(const std::filesystem::path & path); - -} // namespace fs +bool pathExists(const std::filesystem::path & path); /** * Canonicalize a path except for the last component. diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 6a9bd2f16..858531642 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -76,7 +76,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption if (writeTo) { logger->stop(); - if (fs::symlink_exists(*writeTo)) + if (pathExists(*writeTo)) throw Error("path '%s' already exists", writeTo->string()); std::function recurse; From 979d5a7caee31a356714b14022597ce23f954e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 1 May 2025 11:49:06 +0200 Subject: [PATCH 287/396] Drop fs alias in favour of std::filesystem Since we dropped fs::symlink_exists, we no longer have a need for the fs namespace. Having less abstractions makes it easier to lookup the functions in reference documentations. --- src/libcmd/common-eval-args.cc | 5 +- src/libcmd/installables.cc | 4 +- src/libfetchers-tests/git-utils.cc | 6 +- .../include/nix/store/tests/nix_api_store.hh | 10 +- src/libstore-tests/machines.cc | 14 +-- src/libstore/builtins/unpack-channel.cc | 10 +- src/libutil/executable-path.cc | 17 ++-- src/libutil/file-system.cc | 99 +++++++++---------- src/libutil/tarfile.cc | 10 +- src/libutil/unix/file-system.cc | 9 +- src/libutil/unix/users.cc | 2 - src/libutil/windows/file-system.cc | 9 +- .../nix-collect-garbage.cc | 14 +-- src/nix/bundle.cc | 2 +- src/nix/config-check.cc | 14 +-- src/nix/develop.cc | 2 +- src/nix/eval.cc | 8 +- src/nix/flake.cc | 24 ++--- src/nix/run.cc | 14 +-- src/nix/self-exe.cc | 16 ++- 20 files changed, 129 insertions(+), 160 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 1c7c70a30..f2881ca2b 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -18,7 +18,6 @@ namespace nix { -namespace fs { using namespace std::filesystem; } fetchers::Settings fetchSettings; @@ -123,8 +122,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(fetchSettings, _from, fs::current_path().string()); - auto to = parseFlakeRef(fetchSettings, _to, fs::current_path().string()); + auto from = parseFlakeRef(fetchSettings, _from, std::filesystem::current_path().string()); + auto to = parseFlakeRef(fetchSettings, _to, std::filesystem::current_path().string()); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 080557d4f..2d6024890 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -31,8 +31,6 @@ namespace nix { -namespace fs { using namespace std::filesystem; } - void completeFlakeInputAttrPath( AddCompletions & completions, ref evalState, @@ -343,7 +341,7 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), fs::current_path().string()); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), std::filesystem::current_path().string()); auto evalCache = openEvalCache(*evalState, std::make_shared(lockFlake( diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index 2f407dbd0..c2c7f9da0 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -11,14 +11,10 @@ namespace nix { -namespace fs { -using namespace std::filesystem; -} - class GitUtilsTest : public ::testing::Test { // We use a single repository for all tests. - fs::path tmpDir; + std::filesystem::path tmpDir; std::unique_ptr delTmpDir; public: diff --git a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh index bc0f31d05..63f80cf91 100644 --- a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh +++ b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh @@ -11,8 +11,6 @@ #include #include -namespace fs { using namespace std::filesystem; } - namespace nixC { class nix_api_store_test : public nix_api_util_context { @@ -27,10 +25,10 @@ public: { nix_store_free(store); - for (auto & path : fs::recursive_directory_iterator(nixDir)) { - fs::permissions(path, fs::perms::owner_all); + for (auto & path : std::filesystem::recursive_directory_iterator(nixDir)) { + std::filesystem::permissions(path, std::filesystem::perms::owner_all); } - fs::remove_all(nixDir); + std::filesystem::remove_all(nixDir); } Store * store; @@ -45,7 +43,7 @@ protected: auto tmpl = nix::defaultTempDir() + "/tests_nix-store."; for (size_t i = 0; true; ++i) { nixDir = tmpl + std::string { i }; - if (fs::create_directory(nixDir)) break; + if (std::filesystem::create_directory(nixDir)) break; } #else // resolve any symlinks in i.e. on macOS /tmp -> /private/tmp diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index 084807130..8873ff183 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -163,8 +163,8 @@ TEST(machines, getMachinesWithIncorrectFormat) { } TEST(machines, getMachinesWithCorrectFileReference) { - auto path = fs::weakly_canonical(getUnitTestData() / "machines/valid"); - ASSERT_TRUE(fs::exists(path)); + auto path = std::filesystem::weakly_canonical(getUnitTestData() / "machines/valid"); + ASSERT_TRUE(std::filesystem::exists(path)); auto actual = Machine::parseConfig({}, "@" + path.string()); ASSERT_THAT(actual, SizeIs(3)); @@ -174,22 +174,22 @@ TEST(machines, getMachinesWithCorrectFileReference) { } TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { - fs::path path = "/dev/null"; - ASSERT_TRUE(fs::exists(path)); + std::filesystem::path path = "/dev/null"; + ASSERT_TRUE(std::filesystem::exists(path)); auto actual = Machine::parseConfig({}, "@" + path.string()); ASSERT_THAT(actual, SizeIs(0)); } TEST(machines, getMachinesWithIncorrectFileReference) { - auto path = fs::weakly_canonical("/not/a/file"); - ASSERT_TRUE(!fs::exists(path)); + auto path = std::filesystem::weakly_canonical("/not/a/file"); + ASSERT_TRUE(!std::filesystem::exists(path)); auto actual = Machine::parseConfig({}, "@" + path.string()); ASSERT_THAT(actual, SizeIs(0)); } TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { EXPECT_THROW( - Machine::parseConfig({}, "@" + fs::weakly_canonical(getUnitTestData() / "machines" / "bad_format").string()), + Machine::parseConfig({}, "@" + std::filesystem::weakly_canonical(getUnitTestData() / "machines" / "bad_format").string()), FormatError); } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 07b1d5773..e03f3076b 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -3,8 +3,6 @@ namespace nix { -namespace fs { using namespace std::filesystem; } - void builtinUnpackChannel( const BasicDerivation & drv, const std::map & outputs) @@ -15,11 +13,11 @@ void builtinUnpackChannel( return i->second; }; - fs::path out{outputs.at("out")}; + std::filesystem::path out{outputs.at("out")}; auto & channelName = getAttr("channelName"); auto & src = getAttr("src"); - if (fs::path{channelName}.filename().string() != channelName) { + if (std::filesystem::path{channelName}.filename().string() != channelName) { throw Error("channelName is not allowed to contain filesystem separators, got %1%", channelName); } @@ -38,8 +36,8 @@ void builtinUnpackChannel( auto target = out / channelName; try { - fs::rename(fileName, target); - } catch (fs::filesystem_error &) { + std::filesystem::rename(fileName, target); + } catch (std::filesystem::filesystem_error &) { throw SysError("failed to rename %1% to %2%", fileName, target.string()); } } diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index ed1ac49ce..75ab91f3a 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -6,10 +6,6 @@ namespace nix { -namespace fs { -using namespace std::filesystem; -} - constexpr static const OsStringView path_var_separator{ &ExecutablePath::separator, 1, @@ -28,7 +24,7 @@ ExecutablePath ExecutablePath::parse(const OsString & path) auto strings = path.empty() ? (std::list{}) : basicSplitString, OsChar>(path, path_var_separator); - std::vector ret; + std::vector ret; ret.reserve(strings.size()); std::transform( @@ -36,7 +32,7 @@ ExecutablePath ExecutablePath::parse(const OsString & path) std::make_move_iterator(strings.end()), std::back_inserter(ret), [](OsString && str) { - return fs::path{ + return std::filesystem::path{ str.empty() // "A zero-length prefix is a legacy feature that // indicates the current working directory. It @@ -62,13 +58,13 @@ OsString ExecutablePath::render() const return basicConcatStringsSep(path_var_separator, path2); } -std::optional -ExecutablePath::findName(const OsString & exe, std::function isExecutable) const +std::optional +ExecutablePath::findName(const OsString & exe, std::function isExecutable) const { // "If the pathname being sought contains a , the search // through the path prefixes shall not be performed." // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 - assert(OsPathTrait::rfindPathSep(exe) == exe.npos); + assert(OsPathTrait::rfindPathSep(exe) == exe.npos); for (auto & dir : directories) { auto candidate = dir / exe; @@ -79,7 +75,8 @@ ExecutablePath::findName(const OsString & exe, std::function isExecutable) const +std::filesystem::path ExecutablePath::findPath( + const std::filesystem::path & exe, std::function isExecutable) const { // "If the pathname being sought contains a , the search // through the path prefixes shall not be performed." diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index edd68d6e5..ad17f837f 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -61,7 +61,7 @@ DirectoryIterator& DirectoryIterator::operator++() { bool isAbsolute(PathView path) { - return fs::path { path }.is_absolute(); + return std::filesystem::path { path }.is_absolute(); } @@ -110,7 +110,7 @@ Path canonPath(PathView path, bool resolveSymlinks) throw Error("not an absolute path: '%1%'", path); // For Windows - auto rootName = fs::path { path }.root_name(); + auto rootName = std::filesystem::path { path }.root_name(); /* This just exists because we cannot set the target of `remaining` (the callback parameter) directly to a newly-constructed string, @@ -125,7 +125,7 @@ Path canonPath(PathView path, bool resolveSymlinks) path, [&followCount, &temp, maxFollow, resolveSymlinks] (std::string & result, std::string_view & remaining) { - if (resolveSymlinks && fs::is_symlink(result)) { + if (resolveSymlinks && std::filesystem::is_symlink(result)) { if (++followCount >= maxFollow) throw Error("infinite symlink recursion in path '%1%'", remaining); remaining = (temp = concatStrings(readLink(result), remaining)); @@ -154,7 +154,7 @@ Path dirOf(const PathView path) Path::size_type pos = OsPathTrait::rfindPathSep(path); if (pos == path.npos) return "."; - return fs::path{path}.parent_path().string(); + return std::filesystem::path{path}.parent_path().string(); } @@ -177,7 +177,7 @@ std::string_view baseNameOf(std::string_view path) } -bool isInDir(const fs::path & path, const fs::path & dir) +bool isInDir(const std::filesystem::path & path, const std::filesystem::path & dir) { /* Note that while the standard doesn't guarantee this, the `lexically_*` functions should do no IO and not throw. */ @@ -188,7 +188,7 @@ bool isInDir(const fs::path & path, const fs::path & dir) } -bool isDirOrInDir(const fs::path & path, const fs::path & dir) +bool isDirOrInDir(const std::filesystem::path & path, const std::filesystem::path & dir) { return path == dir || isInDir(path, dir); } @@ -251,7 +251,7 @@ bool pathAccessible(const std::filesystem::path & path) Path readLink(const Path & path) { checkInterrupt(); - return fs::read_symlink(path).string(); + return std::filesystem::read_symlink(path).string(); } @@ -369,17 +369,17 @@ void recursiveSync(const Path & path) /* Otherwise, perform a depth-first traversal of the directory and fsync all the files. */ - std::deque dirsToEnumerate; + std::deque dirsToEnumerate; dirsToEnumerate.push_back(path); - std::vector dirsToFsync; + std::vector dirsToFsync; while (!dirsToEnumerate.empty()) { auto currentDir = dirsToEnumerate.back(); dirsToEnumerate.pop_back(); for (auto & entry : DirectoryIterator(currentDir)) { auto st = entry.symlink_status(); - if (fs::is_directory(st)) { + if (std::filesystem::is_directory(st)) { dirsToEnumerate.emplace_back(entry.path()); - } else if (fs::is_regular_file(st)) { + } else if (std::filesystem::is_regular_file(st)) { AutoCloseFD fd = toDescriptor(open(entry.path().string().c_str(), O_RDONLY, 0)); if (!fd) throw SysError("opening file '%1%'", entry.path()); @@ -399,7 +399,7 @@ void recursiveSync(const Path & path) } -static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & bytesFreed) +static void _deletePath(Descriptor parentfd, const std::filesystem::path & path, uint64_t & bytesFreed) { #ifndef _WIN32 checkInterrupt(); @@ -473,7 +473,7 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b #endif } -static void _deletePath(const fs::path & path, uint64_t & bytesFreed) +static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFreed) { Path dir = dirOf(path.string()); if (dir == "") @@ -489,7 +489,7 @@ static void _deletePath(const fs::path & path, uint64_t & bytesFreed) } -void deletePath(const fs::path & path) +void deletePath(const std::filesystem::path & path) { uint64_t dummy; deletePath(path, dummy); @@ -505,17 +505,17 @@ void createDir(const Path & path, mode_t mode) throw SysError("creating directory '%1%'", path); } -void createDirs(const fs::path & path) +void createDirs(const std::filesystem::path & path) { try { - fs::create_directories(path); - } catch (fs::filesystem_error & e) { + std::filesystem::create_directories(path); + } catch (std::filesystem::filesystem_error & e) { throw SysError("creating directory '%1%'", path.string()); } } -void deletePath(const fs::path & path, uint64_t & bytesFreed) +void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed) { //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); bytesFreed = 0; @@ -540,7 +540,7 @@ AutoDelete::~AutoDelete() if (recursive) deletePath(_path); else { - fs::remove(_path); + std::filesystem::remove(_path); } } } catch (...) { @@ -553,7 +553,7 @@ void AutoDelete::cancel() del = false; } -void AutoDelete::reset(const fs::path & p, bool recursive) { +void AutoDelete::reset(const std::filesystem::path & p, bool recursive) { _path = p; this->recursive = recursive; del = true; @@ -629,28 +629,28 @@ std::pair createTempFile(const Path & prefix) void createSymlink(const Path & target, const Path & link) { try { - fs::create_symlink(target, link); - } catch (fs::filesystem_error & e) { + std::filesystem::create_symlink(target, link); + } catch (std::filesystem::filesystem_error & e) { throw SysError("creating symlink '%1%' -> '%2%'", link, target); } } -void replaceSymlink(const fs::path & target, const fs::path & link) +void replaceSymlink(const std::filesystem::path & target, const std::filesystem::path & link) { for (unsigned int n = 0; true; n++) { - auto tmp = link.parent_path() / fs::path{fmt(".%d_%s", n, link.filename().string())}; + auto tmp = link.parent_path() /std::filesystem::path{fmt(".%d_%s", n, link.filename().string())}; tmp = tmp.lexically_normal(); try { - fs::create_symlink(target, tmp); - } catch (fs::filesystem_error & e) { + std::filesystem::create_symlink(target, tmp); + } catch (std::filesystem::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; throw SysError("creating symlink %1% -> %2%", tmp, target); } try { - fs::rename(tmp, link); - } catch (fs::filesystem_error & e) { + std::filesystem::rename(tmp, link); + } catch (std::filesystem::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; throw SysError("renaming %1% to %2%", tmp, link); } @@ -660,25 +660,24 @@ void replaceSymlink(const fs::path & target, const fs::path & link) } } -void setWriteTime(const fs::path & path, const struct stat & st) +void setWriteTime(const std::filesystem::path & path, const struct stat & st) { setWriteTime(path, st.st_atime, st.st_mtime, S_ISLNK(st.st_mode)); } -void copyFile(const fs::path & from, const fs::path & to, bool andDelete) +void copyFile(const std::filesystem::path & from, const std::filesystem::path & to, bool andDelete) { - auto fromStatus = fs::symlink_status(from); + auto fromStatus =std::filesystem::symlink_status(from); // Mark the directory as writable so that we can delete its children - if (andDelete && fs::is_directory(fromStatus)) { - fs::permissions(from, fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + if (andDelete &&std::filesystem::is_directory(fromStatus)) { + std::filesystem::permissions(from, std::filesystem::perms::owner_write, std::filesystem::perm_options::add | std::filesystem::perm_options::nofollow); } - - if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { - fs::copy(from, to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); - } else if (fs::is_directory(fromStatus)) { - fs::create_directory(to); + if (std::filesystem::is_symlink(fromStatus) ||std::filesystem::is_regular_file(fromStatus)) { + std::filesystem::copy(from, to, std::filesystem::copy_options::copy_symlinks | std::filesystem::copy_options::overwrite_existing); + } else if (std::filesystem::is_directory(fromStatus)) { + std::filesystem::create_directory(to); for (auto & entry : DirectoryIterator(from)) { copyFile(entry, to / entry.path().filename(), andDelete); } @@ -688,9 +687,9 @@ void copyFile(const fs::path & from, const fs::path & to, bool andDelete) setWriteTime(to, lstat(from.string().c_str())); if (andDelete) { - if (!fs::is_symlink(fromStatus)) - fs::permissions(from, fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - fs::remove(from); + if (!std::filesystem::is_symlink(fromStatus)) + std::filesystem::permissions(from, std::filesystem::perms::owner_write, std::filesystem::perm_options::add | std::filesystem::perm_options::nofollow); + std::filesystem::remove(from); } } @@ -698,18 +697,18 @@ void moveFile(const Path & oldName, const Path & newName) { try { std::filesystem::rename(oldName, newName); - } catch (fs::filesystem_error & e) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); + } catch (std::filesystem::filesystem_error & e) { + auto oldPath = std::filesystem::path(oldName); + auto newPath = std::filesystem::path(newName); // For the move to be as atomic as possible, copy to a temporary // directory - fs::path temp = createTempDir( + std::filesystem::path temp = createTempDir( os_string_to_string(PathViewNG { newPath.parent_path() }), "rename-tmp"); - Finally removeTemp = [&]() { fs::remove(temp); }; + Finally removeTemp = [&]() { std::filesystem::remove(temp); }; auto tempCopyTarget = temp / "copy-target"; if (e.code().value() == EXDEV) { - fs::remove(newPath); + std::filesystem::remove(newPath); warn("can’t rename %s as %s, copying instead", oldName, newName); copyFile(oldPath, tempCopyTarget, true); std::filesystem::rename( @@ -721,7 +720,7 @@ void moveFile(const Path & oldName, const Path & newName) ////////////////////////////////////////////////////////////////////// -bool isExecutableFileAmbient(const fs::path & exe) { +bool isExecutableFileAmbient(const std::filesystem::path & exe) { // Check file type, because directory being executable means // something completely different. // `is_regular_file` follows symlinks before checking. @@ -745,12 +744,12 @@ std::filesystem::path makeParentCanonical(const std::filesystem::path & rawPath) return parent; } return std::filesystem::canonical(parent) / path.filename(); - } catch (fs::filesystem_error & e) { + } catch (std::filesystem::filesystem_error & e) { throw SysError("canonicalising parent path of '%1%'", path); } } -bool chmodIfNeeded(const fs::path & path, mode_t mode, mode_t mask) +bool chmodIfNeeded(const std::filesystem::path & path, mode_t mode, mode_t mask) { auto pathString = path.string(); auto prevMode = lstat(pathString).st_mode; diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index eb5cd8288..75373e3eb 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -8,10 +8,6 @@ namespace nix { -namespace fs { -using namespace std::filesystem; -} - namespace { int callback_open(struct archive *, void * self) @@ -127,7 +123,7 @@ TarArchive::~TarArchive() archive_read_free(this->archive); } -static void extract_archive(TarArchive & archive, const fs::path & destDir) +static void extract_archive(TarArchive & archive, const std::filesystem::path & destDir) { int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS | ARCHIVE_EXTRACT_SECURE_NODOTDOT; @@ -162,7 +158,7 @@ static void extract_archive(TarArchive & archive, const fs::path & destDir) archive.close(); } -void unpackTarfile(Source & source, const fs::path & destDir) +void unpackTarfile(Source & source, const std::filesystem::path & destDir) { auto archive = TarArchive(source); @@ -170,7 +166,7 @@ void unpackTarfile(Source & source, const fs::path & destDir) extract_archive(archive, destDir); } -void unpackTarfile(const fs::path & tarFile, const fs::path & destDir) +void unpackTarfile(const std::filesystem::path & tarFile, const std::filesystem::path & destDir) { auto archive = TarArchive(tarFile); diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index e62b7d1c2..7865de2e9 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -14,16 +14,13 @@ namespace nix { -namespace fs { -using namespace std::filesystem; -} - Descriptor openDirectory(const std::filesystem::path & path) { return open(path.c_str(), O_RDONLY | O_DIRECTORY); } -void setWriteTime(const fs::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) +void setWriteTime( + const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) { // Would be nice to use std::filesystem unconditionally, but // doesn't support access time just modification time. @@ -57,7 +54,7 @@ void setWriteTime(const fs::path & path, time_t accessedTime, time_t modificatio if (lutimes(path.c_str(), times) == -1) throw SysError("changing modification time of %s", path); # else - bool isSymlink = optIsSymlink ? *optIsSymlink : fs::is_symlink(path); + bool isSymlink = optIsSymlink ? *optIsSymlink : std::filesystem::is_symlink(path); if (!isSymlink) { if (utimes(path.c_str(), times) == -1) diff --git a/src/libutil/unix/users.cc b/src/libutil/unix/users.cc index 18df7fdf2..5ac851e95 100644 --- a/src/libutil/unix/users.cc +++ b/src/libutil/unix/users.cc @@ -9,8 +9,6 @@ namespace nix { -namespace fs { using namespace std::filesystem; } - std::string getUserName() { auto pw = getpwuid(geteuid()); diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 1dac7e754..a73fa223a 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -3,13 +3,10 @@ #ifdef _WIN32 namespace nix { -namespace fs { -using namespace std::filesystem; -} - -void setWriteTime(const fs::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) +void setWriteTime( + const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) { - // FIXME use `fs::last_write_time`. + // FIXME use `std::filesystem::last_write_time`. // // Would be nice to use std::filesystem unconditionally, but // doesn't support access time just modification time. diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 8e4fc30fb..3140bb809 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -24,7 +24,7 @@ bool dryRun = false; * Of course, this makes rollbacks to before this point in time * impossible. */ -void removeOldGenerations(fs::path dir) +void removeOldGenerations(std::filesystem::path dir) { if (access(dir.string().c_str(), R_OK) != 0) return; @@ -36,11 +36,11 @@ void removeOldGenerations(fs::path dir) auto path = i.path().string(); auto type = i.symlink_status().type(); - if (type == fs::file_type::symlink && canWrite) { + if (type == std::filesystem::file_type::symlink && canWrite) { std::string link; try { link = readLink(path); - } catch (fs::filesystem_error & e) { + } catch (std::filesystem::filesystem_error & e) { if (e.code() == std::errc::no_such_file_or_directory) continue; throw; } @@ -52,7 +52,7 @@ void removeOldGenerations(fs::path dir) } else deleteOldGenerations(path, dryRun); } - } else if (type == fs::file_type::directory) { + } else if (type == std::filesystem::file_type::directory) { removeOldGenerations(path); } } @@ -84,10 +84,10 @@ static int main_nix_collect_garbage(int argc, char * * argv) }); if (removeOld) { - std::set dirsToClean = { + std::set dirsToClean = { profilesDir(), - fs::path{settings.nixStateDir} / "profiles", - fs::path{getDefaultProfile()}.parent_path(), + std::filesystem::path{settings.nixStateDir} / "profiles", + std::filesystem::path{getDefaultProfile()}.parent_path(), }; for (auto & dir : dirsToClean) removeOldGenerations(dir); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index c334469b5..50d7bf6a3 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -80,7 +80,7 @@ struct CmdBundle : InstallableValueCommand auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec( - fetchSettings, bundler, fs::current_path().string()); + fetchSettings, bundler, std::filesystem::current_path().string()); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, evalState, std::move(bundlerFlakeRef), bundlerName, std::move(extendedOutputsSpec), diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index deac8e560..27d053b9f 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -78,12 +78,12 @@ struct CmdConfigCheck : StoreCommand bool checkNixInPath() { - std::set dirs; + std::set dirs; for (auto & dir : ExecutablePath::load().directories) { auto candidate = dir / "nix-env"; - if (fs::exists(candidate)) - dirs.insert(fs::canonical(candidate).parent_path() ); + if (std::filesystem::exists(candidate)) + dirs.insert(std::filesystem::canonical(candidate).parent_path() ); } if (dirs.size() != 1) { @@ -99,12 +99,12 @@ struct CmdConfigCheck : StoreCommand bool checkProfileRoots(ref store) { - std::set dirs; + std::set dirs; for (auto & dir : ExecutablePath::load().directories) { auto profileDir = dir.parent_path(); try { - auto userEnv = fs::weakly_canonical(profileDir); + auto userEnv = std::filesystem::weakly_canonical(profileDir); auto noContainsProfiles = [&]{ for (auto && part : profileDir) @@ -114,8 +114,8 @@ struct CmdConfigCheck : StoreCommand if (store->isStorePath(userEnv.string()) && hasSuffix(userEnv.string(), "user-environment")) { while (noContainsProfiles() && std::filesystem::is_symlink(profileDir)) - profileDir = fs::weakly_canonical( - profileDir.parent_path() / fs::read_symlink(profileDir)); + profileDir = std::filesystem::weakly_canonical( + profileDir.parent_path() / std::filesystem::read_symlink(profileDir)); if (noContainsProfiles()) dirs.insert(dir); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index a0b863792..af32c8386 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -345,7 +345,7 @@ struct Common : InstallableCommand, MixProfile ref store, const BuildEnvironment & buildEnvironment, const std::filesystem::path & tmpDir, - const std::filesystem::path & outputsDir = fs::path { fs::current_path() } / "outputs") + const std::filesystem::path & outputsDir = std::filesystem::path { std::filesystem::current_path() } / "outputs") { // A list of colon-separated environment variables that should be // prepended to, rather than overwritten, in order to keep the shell usable. diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 858531642..be064e552 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -16,7 +16,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption { bool raw = false; std::optional apply; - std::optional writeTo; + std::optional writeTo; CmdEval() : InstallableValueCommand() { @@ -79,16 +79,16 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption if (pathExists(*writeTo)) throw Error("path '%s' already exists", writeTo->string()); - std::function recurse; + std::function recurse; - recurse = [&](Value & v, const PosIdx pos, const fs::path & path) + recurse = [&](Value & v, const PosIdx pos, const std::filesystem::path & path) { state->forceValue(v, pos); if (v.type() == nString) // FIXME: disallow strings with contexts? writeFile(path.string(), v.string_view()); else if (v.type() == nAttrs) { - [[maybe_unused]] bool directoryCreated = fs::create_directory(path); + [[maybe_unused]] bool directoryCreated = std::filesystem::create_directory(path); // Directory should not already exist assert(directoryCreated); for (auto & attr : *v.attrs()) { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3d174dc53..4586ebfbe 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -54,7 +54,7 @@ public: FlakeRef getFlakeRef() { - return parseFlakeRef(fetchSettings, flakeUrl, fs::current_path().string()); //FIXME + return parseFlakeRef(fetchSettings, flakeUrl, std::filesystem::current_path().string()); //FIXME } LockedFlake lockFlake() @@ -66,7 +66,7 @@ public: { return { // Like getFlakeRef but with expandTilde calld first - parseFlakeRef(fetchSettings, expandTilde(flakeUrl), fs::current_path().string()) + parseFlakeRef(fetchSettings, expandTilde(flakeUrl), std::filesystem::current_path().string()) }; } }; @@ -885,7 +885,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto evalState = getEvalState(); auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment( - fetchSettings, templateUrl, fs::current_path().string()); + fetchSettings, templateUrl, std::filesystem::current_path().string()); auto installable = InstallableFlake(nullptr, evalState, std::move(templateFlakeRef), templateName, ExtendedOutputsSpec::Default(), @@ -899,11 +899,11 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand NixStringContext context; auto templateDir = evalState->coerceToPath(noPos, templateDirAttr, context, ""); - std::vector changedFiles; - std::vector conflictedFiles; + std::vector changedFiles; + std::vector conflictedFiles; - std::function copyDir; - copyDir = [&](const SourcePath & from, const fs::path & to) + std::function copyDir; + copyDir = [&](const SourcePath & from, const std::filesystem::path & to) { createDirs(to); @@ -912,12 +912,12 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto from2 = from / name; auto to2 = to / name; auto st = from2.lstat(); - auto to_st = fs::symlink_status(to2); + auto to_st = std::filesystem::symlink_status(to2); if (st.type == SourceAccessor::tDirectory) copyDir(from2, to2); else if (st.type == SourceAccessor::tRegular) { auto contents = from2.readFile(); - if (fs::exists(to_st)) { + if (std::filesystem::exists(to_st)) { auto contents2 = readFile(to2.string()); if (contents != contents2) { printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2); @@ -931,8 +931,8 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } else if (st.type == SourceAccessor::tSymlink) { auto target = from2.readLink(); - if (fs::exists(to_st)) { - if (fs::read_symlink(to2) != target) { + if (std::filesystem::exists(to_st)) { + if (std::filesystem::read_symlink(to2) != target) { printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2); conflictedFiles.push_back(to2); } else { @@ -951,7 +951,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand copyDir(templateDir, flakeDir); - if (!changedFiles.empty() && fs::exists(std::filesystem::path{flakeDir} / ".git")) { + if (!changedFiles.empty() && std::filesystem::exists(std::filesystem::path{flakeDir} / ".git")) { Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" }; for (auto & s : changedFiles) args.emplace_back(s.string()); runProgram("git", true, args); diff --git a/src/nix/run.cc b/src/nix/run.cc index 10f5026cd..0473c99b7 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -172,7 +172,7 @@ void chrootHelper(int argc, char * * argv) if (!pathExists(storeDir)) { // FIXME: Use overlayfs? - fs::path tmpDir = createTempDir(); + std::filesystem::path tmpDir = createTempDir(); createDirs(tmpDir + storeDir); @@ -182,15 +182,15 @@ void chrootHelper(int argc, char * * argv) for (const auto & entry : DirectoryIterator{"/"}) { checkInterrupt(); const auto & src = entry.path(); - fs::path dst = tmpDir / entry.path().filename(); + std::filesystem::path dst = tmpDir / entry.path().filename(); if (pathExists(dst)) continue; auto st = entry.symlink_status(); - if (fs::is_directory(st)) { + if (std::filesystem::is_directory(st)) { if (mkdir(dst.c_str(), 0700) == -1) throw SysError("creating directory '%s'", dst); if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) throw SysError("mounting '%s' on '%s'", src, dst); - } else if (fs::is_symlink(st)) + } else if (std::filesystem::is_symlink(st)) createSymlink(readLink(src), dst); } @@ -208,9 +208,9 @@ void chrootHelper(int argc, char * * argv) if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - writeFile(fs::path{"/proc/self/setgroups"}, "deny"); - writeFile(fs::path{"/proc/self/uid_map"}, fmt("%d %d %d", uid, uid, 1)); - writeFile(fs::path{"/proc/self/gid_map"}, fmt("%d %d %d", gid, gid, 1)); + writeFile(std::filesystem::path{"/proc/self/setgroups"}, "deny"); + writeFile(std::filesystem::path{"/proc/self/uid_map"}, fmt("%d %d %d", uid, uid, 1)); + writeFile(std::filesystem::path{"/proc/self/gid_map"}, fmt("%d %d %d", gid, gid, 1)); #ifdef __linux__ if (system != "") diff --git a/src/nix/self-exe.cc b/src/nix/self-exe.cc index 5cc2326be..b5eb1190d 100644 --- a/src/nix/self-exe.cc +++ b/src/nix/self-exe.cc @@ -7,32 +7,28 @@ namespace nix { -namespace fs { -using namespace std::filesystem; -} - -fs::path getNixBin(std::optional binaryNameOpt) +std::filesystem::path getNixBin(std::optional binaryNameOpt) { auto getBinaryName = [&] { return binaryNameOpt ? *binaryNameOpt : "nix"; }; // If the environment variable is set, use it unconditionally. if (auto envOpt = getEnvNonEmpty("NIX_BIN_DIR")) - return fs::path{*envOpt} / std::string{getBinaryName()}; + return std::filesystem::path{*envOpt} / std::string{getBinaryName()}; // Try OS tricks, if available, to get to the path of this Nix, and // see if we can find the right executable next to that. if (auto selfOpt = getSelfExe()) { - fs::path path{*selfOpt}; + std::filesystem::path path{*selfOpt}; if (binaryNameOpt) path = path.parent_path() / std::string{*binaryNameOpt}; - if (fs::exists(path)) + if (std::filesystem::exists(path)) return path; } // If `nix` exists at the hardcoded fallback path, use it. { - auto path = fs::path{NIX_BIN_DIR} / std::string{getBinaryName()}; - if (fs::exists(path)) + auto path = std::filesystem::path{NIX_BIN_DIR} / std::string{getBinaryName()}; + if (std::filesystem::exists(path)) return path; } From 6f71d8a9c2d36be2bde7f9c58b81b13eca20ce0d Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Sun, 13 Apr 2025 11:30:18 -0700 Subject: [PATCH 288/396] Update `nix fmt` man page with official formatter example The current example relies upon [nixfmt's deprecated tree traversal behavior](https://github.com/NixOS/nixfmt/pull/240). The simplest alternative is the new `nixfmt-tree` wrapper for `nixfmt`/`treefmt`. --- src/nix/formatter-run.md | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/src/nix/formatter-run.md b/src/nix/formatter-run.md index a01b52a9d..db8583c95 100644 --- a/src/nix/formatter-run.md +++ b/src/nix/formatter-run.md @@ -9,37 +9,15 @@ Flags can be forwarded to the formatter by using `--` followed by the flags. Any arguments will be forwarded to the formatter. Typically these are the files to format. -# Examples +# Example -With [nixpkgs-fmt](https://github.com/nix-community/nixpkgs-fmt): +To use the [official Nix formatter](https://github.com/NixOS/nixfmt): ```nix # flake.nix { outputs = { nixpkgs, self }: { - formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixpkgs-fmt; - }; -} -``` - -With [nixfmt](https://github.com/NixOS/nixfmt): - -```nix -# flake.nix -{ - outputs = { nixpkgs, self }: { - formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt-rfc-style; - }; -} -``` - -With [Alejandra](https://github.com/kamadorueda/alejandra): - -```nix -# flake.nix -{ - outputs = { nixpkgs, self }: { - formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.alejandra; + formatter.x86_64-linux = nixpkgs.legacyPackages.${system}.nixfmt-tree; }; } ``` From 90deb665eb58ee981389cd3dedc031140dbf52aa Mon Sep 17 00:00:00 2001 From: Thomas Bereknyei Date: Thu, 1 May 2025 02:28:17 -0400 Subject: [PATCH 289/396] fix: allow redirected HTTP uploads When a PUT is redirected, some of the data can be sent by curl before headers are read. This means the subsequent PUT operation needs to seek back to origin. --- src/libstore/filetransfer.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8fc4f14f2..f77881cf7 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -312,6 +312,23 @@ struct curlFileTransfer : public FileTransfer } #endif + size_t seekCallback(curl_off_t offset, int origin) + { + if (origin == SEEK_SET) { + readOffset = offset; + } else if (origin == SEEK_CUR) { + readOffset += offset; + } else if (origin == SEEK_END) { + readOffset = request.data->length() + offset; + } + return CURL_SEEKFUNC_OK; + } + + static size_t seekCallbackWrapper(void *clientp, curl_off_t offset, int origin) + { + return ((TransferItem *) clientp)->seekCallback(offset, origin); + } + void init() { if (!req) req = curl_easy_init(); @@ -364,6 +381,8 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); curl_easy_setopt(req, CURLOPT_READDATA, this); curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length()); + curl_easy_setopt(req, CURLOPT_SEEKFUNCTION, seekCallbackWrapper); + curl_easy_setopt(req, CURLOPT_SEEKDATA, this); } if (request.verifyTLS) { From 86a3fad0856a1755a128c645057c69d2b11b6bec Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 1 May 2025 23:10:04 +0000 Subject: [PATCH 290/396] libexpr: Improve lexer performance by using full scanner tables (-Cf) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This trades off some executable size for measurable lexer performance improvements. Note on the explicitly enabling 8bit scanner. This is needed due to the default behavior of flex (excerpt from the manual [1]): > Flex’s default behavior is to generate an 8-bit scanner unless you > use the ‘-Cf’ or ‘-CF’, in which case flex defaults to generating > 7-bit scanners unless your site was always configured to generate 8-bit > scanners. Some quantifyable metrics: Nixpkgs revision: a6e3f45acf4e817532a861ab0eda4ab5485fecc1 Parsing the largest file in nixpkgs: pkgs/development/haskell-modules/hackage-packages.nix. (Before this patch) ``` $ nix build github:nixos/nix/9fe3077d4#nix-expr $ du --apparent-size result/lib/libnixexpr.so 2518 result/lib/libnixexpr.so $ nix build github:nixos/nix/9fe3077d4#nix-cli $ taskset -c 2,3 hyperfine "GC_INITIAL_HEAP_SIZE=16g \ result/bin/nix-instantiate --parse \ ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix > /dev/null" Time (mean ± σ): 375.5 ms ± 6.3 ms [User: 316.9 ms, System: 56.7 ms] Range (min … max): 368.5 ms … 388.3 ms 10 runs ``` (After the patch) ``` $ nix build .#nix-expr $ du --apparent-size result/lib/libnixexpr.so 2685 result/lib/libnixexpr.so $ nix build .#nix-cli $ taskset -c 2,3 hyperfine "GC_INITIAL_HEAP_SIZE=16g \ result/bin/nix-instantiate --parse \ ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix > /dev/null" Time (mean ± σ): 326.8 ms ± 4.9 ms [User: 269.5 ms, System: 55.3 ms] Range (min … max): 319.7 ms … 335.5 ms 10 runs ``` Overall, the change is roughly: - 2518KiB -> 2685KiB ~ 150 KiB of machine code - 375ms -> 325ms ~ 50ms The perf uplift for eval-heavy test cases is obviously less noticeable, but it doesn't make sense not to take this free perf win. [1]: https://westes.github.io/flex/manual/Options-Affecting-Scanner-Behavior.html#Options-Affecting-Scanner-Behavior --- src/libexpr/lexer.l | 1 + src/libexpr/meson.build | 1 + 2 files changed, 2 insertions(+) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 1e196741d..1005f9f7e 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -1,3 +1,4 @@ +%option 8bit %option reentrant bison-bridge bison-locations %option align %option noyywrap diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 2e773938d..2b465b85a 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -112,6 +112,7 @@ lexer_tab = custom_target( ], command : [ 'flex', + '-Cf', # Use full scanner tables '--outfile', '@OUTPUT0@', '--header-file=' + '@OUTPUT1@', From d8c97d8073cbeb5f62face1d905dd9b3b05e43ca Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 2 May 2025 17:40:29 +0000 Subject: [PATCH 291/396] treewide: Use StringSet alias consistently instead of std::set The intention is to switch to transparent comparators from N3657 for ordered set containers for strings and using the alias consistently would simplify things. --- src/build-remote/build-remote.cc | 4 ++-- src/libcmd/command.cc | 2 +- src/libcmd/include/nix/cmd/command.hh | 2 +- src/libcmd/installable-attr-path.cc | 2 +- src/libcmd/installable-flake.cc | 2 +- src/libexpr/attr-path.cc | 2 +- src/libexpr/eval-cache.cc | 2 +- src/libexpr/eval.cc | 4 ++-- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/tarball.cc | 4 ++-- src/libflake/config.cc | 2 +- src/libflake/flake.cc | 2 +- src/libmain/plugin.cc | 2 +- src/libstore-tests/common-protocol.cc | 2 +- src/libstore-tests/outputs-spec.cc | 2 +- src/libstore-tests/serve-protocol.cc | 2 +- src/libstore-tests/worker-protocol.cc | 2 +- src/libstore/derived-path-map.cc | 12 ++++++------ src/libstore/dummy-store.cc | 2 +- .../include/nix/store/derived-path-map.hh | 14 +++++++------- src/libstore/include/nix/store/globals.hh | 2 +- .../include/nix/store/http-binary-cache-store.hh | 4 ++-- .../include/nix/store/legacy-ssh-store.hh | 2 +- .../nix/store/local-binary-cache-store.hh | 2 +- .../include/nix/store/local-overlay-store.hh | 2 +- src/libstore/include/nix/store/local-store.hh | 2 +- src/libstore/include/nix/store/machines.hh | 12 ++++++------ .../include/nix/store/path-with-outputs.hh | 2 +- .../include/nix/store/s3-binary-cache-store.hh | 2 +- src/libstore/include/nix/store/ssh-store.hh | 4 ++-- src/libstore/include/nix/store/store-api.hh | 2 +- .../include/nix/store/uds-remote-store.hh | 2 +- src/libstore/local-binary-cache-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/machines.cc | 16 ++++++++-------- src/libstore/path-with-outputs.cc | 4 ++-- src/libutil/args.cc | 2 +- src/libutil/configuration.cc | 6 +++--- src/libutil/experimental-features.cc | 2 +- src/libutil/hash.cc | 4 ++-- src/libutil/include/nix/util/args.hh | 4 ++-- src/libutil/include/nix/util/configuration.hh | 12 ++++++------ .../include/nix/util/experimental-features.hh | 2 +- src/libutil/include/nix/util/hash.hh | 4 ++-- src/libutil/include/nix/util/strings.hh | 10 ++++++---- src/libutil/include/nix/util/suggestions.hh | 2 +- src/libutil/strings.cc | 8 ++++---- src/libutil/suggestions.cc | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-env/nix-env.cc | 4 ++-- src/nix/develop.cc | 6 +++--- src/nix/diff-closures.cc | 10 +++++----- src/nix/flake.cc | 2 +- src/nix/profile.cc | 6 +++--- src/nix/why-depends.cc | 2 +- 55 files changed, 111 insertions(+), 109 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 60247b735..d967548f5 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -42,7 +42,7 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } -static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { +static bool allSupportedLocally(Store & store, const StringSet& requiredFeatures) { for (auto & feature : requiredFeatures) if (!store.systemFeatures.get().count(feature)) return false; return true; @@ -113,7 +113,7 @@ static int main_build_remote(int argc, char * * argv) auto amWilling = readInt(source); auto neededSystem = readString(source); drvPath = store->parseStorePath(readString(source)); - auto requiredFeatures = readStrings>(source); + auto requiredFeatures = readStrings(source); /* It would be possible to build locally after some builds clear out, so don't show the warning now: */ diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 047906786..ffb745742 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -40,7 +40,7 @@ nlohmann::json NixMultiCommand::toJSON() void NixMultiCommand::run() { if (!command) { - std::set subCommandTextLines; + StringSet subCommandTextLines; for (auto & [name, _] : commands) subCommandTextLines.insert(fmt("- `%s`", name)); std::string markdownError = diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index 92377be6b..45739fdb2 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -363,7 +363,7 @@ void completeFlakeRefWithFragment( const Strings & defaultFlakeAttrPaths, std::string_view prefix); -std::string showVersions(const std::set & versions); +std::string showVersions(const StringSet & versions); void printClosureDiff( ref store, const StorePath & beforePath, const StorePath & afterPath, std::string_view indent); diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index fcbfe1482..7783b4f40 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -72,7 +72,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() auto newOutputs = std::visit(overloaded { [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { - std::set outputsToInstall; + StringSet outputsToInstall; for (auto & output : packageInfo.queryOutputs(false, true)) outputsToInstall.insert(output.first); if (outputsToInstall.empty()) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 83285b739..85a4188a7 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -117,7 +117,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() .drvPath = makeConstantStorePathRef(std::move(drvPath)), .outputs = std::visit(overloaded { [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { - std::set outputsToInstall; + StringSet outputsToInstall; if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { if (aOutputSpecified->getBool()) { if (auto aOutputName = attr->maybeGetAttr("outputName")) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index cee805d14..722b57bbf 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -74,7 +74,7 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin auto a = v->attrs()->get(state.symbols.create(attr)); if (!a) { - std::set attrNames; + StringSet attrNames; for (auto & attr : *v->attrs()) attrNames.insert(std::string(state.symbols[attr.name])); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 30aa6076a..0d732f59c 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -492,7 +492,7 @@ Value & AttrCursor::forceValue() Suggestions AttrCursor::getSuggestionsForAttr(Symbol name) { auto attrNames = getAttrs(); - std::set strAttrNames; + StringSet strAttrNames; for (auto & name : attrNames) strAttrNames.insert(std::string(root->state.symbols[name])); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 10a33c042..ab7f33d5f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1446,7 +1446,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } else { state.forceAttrs(*vAttrs, pos, "while selecting an attribute"); if (!(j = vAttrs->attrs()->get(name))) { - std::set allAttrNames; + StringSet allAttrNames; for (auto & attr : *vAttrs->attrs()) allAttrNames.insert(std::string(state.symbols[attr.name])); auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); @@ -1603,7 +1603,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, user. */ for (auto & i : *args[0]->attrs()) if (!lambda.formals->has(i.name)) { - std::set formalNames; + StringSet formalNames; for (auto & formal : lambda.formals->formals) formalNames.insert(std::string(symbols[formal.name])); auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index eb6bdd1eb..74e9fd089 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -194,7 +194,7 @@ struct MercurialInputScheme : InputScheme input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl }))); - auto files = tokenizeString>( + auto files = tokenizeString( runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); Path actualPath(absPath(actualUrl)); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ef91d6b25..1bd7e3e59 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -224,7 +224,7 @@ ref downloadTarball( // An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { - const std::set transportUrlSchemes = {"file", "http", "https"}; + const StringSet transportUrlSchemes = {"file", "http", "https"}; bool hasTarballExtension(std::string_view path) const { @@ -236,7 +236,7 @@ struct CurlInputScheme : InputScheme virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0; - static const std::set specialParams; + static const StringSet specialParams; std::optional inputFromURL( const Settings & settings, diff --git a/src/libflake/config.cc b/src/libflake/config.cc index a67f7884c..030104e7f 100644 --- a/src/libflake/config.cc +++ b/src/libflake/config.cc @@ -32,7 +32,7 @@ static void writeTrustedList(const TrustedList & trustedList) void ConfigFile::apply(const Settings & flakeSettings) { - std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"}; + StringSet whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"}; for (auto & [name, value] : settings) { diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index a086251b9..516b38131 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -323,7 +323,7 @@ static FlakeRef applySelfAttrs( { auto newRef(ref); - std::set allowedAttrs{"submodules", "lfs"}; + StringSet allowedAttrs{"submodules", "lfs"}; for (auto & attr : flake.selfAttrs) { if (!allowedAttrs.contains(attr.first)) diff --git a/src/libmain/plugin.cc b/src/libmain/plugin.cc index 812506a86..db686a251 100644 --- a/src/libmain/plugin.cc +++ b/src/libmain/plugin.cc @@ -19,7 +19,7 @@ struct PluginFilesSetting : public BaseSetting const Paths & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) + const StringSet & aliases = {}) : BaseSetting(def, true, name, description, aliases) { options->addSetting(this); diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 6bfb8bd80..5164f154a 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -154,7 +154,7 @@ CHARACTERIZATION_TEST( CHARACTERIZATION_TEST( set, "set", - (std::tuple, std::set, std::set, std::set>> { + (std::tuple> { { }, { "" }, { "", "foo", "bar" }, diff --git a/src/libstore-tests/outputs-spec.cc b/src/libstore-tests/outputs-spec.cc index a17922c46..a1c13d2f8 100644 --- a/src/libstore-tests/outputs-spec.cc +++ b/src/libstore-tests/outputs-spec.cc @@ -7,7 +7,7 @@ namespace nix { TEST(OutputsSpec, no_empty_names) { - ASSERT_DEATH(OutputsSpec::Names { std::set { } }, ""); + ASSERT_DEATH(OutputsSpec::Names { StringSet { } }, ""); } #define TEST_DONT_PARSE(NAME, STR) \ diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 9297d46ea..69dab5488 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -374,7 +374,7 @@ VERSIONED_CHARACTERIZATION_TEST( set, "set", defaultVersion, - (std::tuple, std::set, std::set, std::set>> { + (std::tuple> { { }, { "" }, { "", "foo", "bar" }, diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index acf83cb60..80d314b1c 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -574,7 +574,7 @@ VERSIONED_CHARACTERIZATION_TEST( set, "set", defaultVersion, - (std::tuple, std::set, std::set, std::set>> { + (std::tuple> { { }, { "" }, { "", "foo", "bar" }, diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index d4234d92c..b785dddd9 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -55,17 +55,17 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single namespace nix { template<> -bool DerivedPathMap>::ChildNode::operator == ( - const DerivedPathMap>::ChildNode &) const noexcept = default; +bool DerivedPathMap::ChildNode::operator == ( + const DerivedPathMap::ChildNode &) const noexcept = default; // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. #if 0 template<> -std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( - const DerivedPathMap>::ChildNode &) const noexcept = default; +std::strong_ordering DerivedPathMap::ChildNode::operator <=> ( + const DerivedPathMap::ChildNode &) const noexcept = default; #endif -template struct DerivedPathMap>::ChildNode; -template struct DerivedPathMap>; +template struct DerivedPathMap::ChildNode; +template struct DerivedPathMap; }; diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 80367d597..3e1d168a1 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -22,7 +22,7 @@ struct DummyStoreConfig : virtual StoreConfig { ; } - static std::set uriSchemes() { + static StringSet uriSchemes() { return {"dummy"}; } }; diff --git a/src/libstore/include/nix/store/derived-path-map.hh b/src/libstore/include/nix/store/derived-path-map.hh index 956f8bb0b..cad86d1b4 100644 --- a/src/libstore/include/nix/store/derived-path-map.hh +++ b/src/libstore/include/nix/store/derived-path-map.hh @@ -91,20 +91,20 @@ struct DerivedPathMap { }; template<> -bool DerivedPathMap>::ChildNode::operator == ( - const DerivedPathMap>::ChildNode &) const noexcept; +bool DerivedPathMap::ChildNode::operator == ( + const DerivedPathMap::ChildNode &) const noexcept; // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. #if 0 template<> -std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( - const DerivedPathMap>::ChildNode &) const noexcept; +std::strong_ordering DerivedPathMap::ChildNode::operator <=> ( + const DerivedPathMap::ChildNode &) const noexcept; template<> -inline auto DerivedPathMap>::operator <=> (const DerivedPathMap> &) const noexcept = default; +inline auto DerivedPathMap::operator <=> (const DerivedPathMap &) const noexcept = default; #endif -extern template struct DerivedPathMap>::ChildNode; -extern template struct DerivedPathMap>; +extern template struct DerivedPathMap::ChildNode; +extern template struct DerivedPathMap; } diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index 82211d8dc..3a677216a 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -24,7 +24,7 @@ struct MaxBuildJobsSetting : public BaseSetting unsigned int def, const std::string & name, const std::string & description, - const std::set & aliases = {}) + const StringSet & aliases = {}) : BaseSetting(def, true, name, description, aliases) { options->addSetting(this); diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index aaec3116d..592def4f8 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -15,10 +15,10 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig return "HTTP Binary Cache Store"; } - static std::set uriSchemes() + static StringSet uriSchemes() { static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - auto ret = std::set({"http", "https"}); + auto ret = StringSet({"http", "https"}); if (forceHttp) ret.insert("file"); return ret; diff --git a/src/libstore/include/nix/store/legacy-ssh-store.hh b/src/libstore/include/nix/store/legacy-ssh-store.hh index a1fbf3f1e..7859abbc9 100644 --- a/src/libstore/include/nix/store/legacy-ssh-store.hh +++ b/src/libstore/include/nix/store/legacy-ssh-store.hh @@ -37,7 +37,7 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig const std::string name() override { return "SSH Store"; } - static std::set uriSchemes() { return {"ssh"}; } + static StringSet uriSchemes() { return {"ssh"}; } std::string doc() override; }; diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index dde4701da..a46ce07d2 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -15,7 +15,7 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig return "Local Binary Cache Store"; } - static std::set uriSchemes(); + static StringSet uriSchemes(); std::string doc() override; }; diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index 825214cb6..f3658d603 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -63,7 +63,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig return ExperimentalFeature::LocalOverlayStore; } - static std::set uriSchemes() + static StringSet uriSchemes() { return { "local-overlay" }; } diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 204720c34..baa82e7f6 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -67,7 +67,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig const std::string name() override { return "Local Store"; } - static std::set uriSchemes() + static StringSet uriSchemes() { return {"local"}; } std::string doc() override; diff --git a/src/libstore/include/nix/store/machines.hh b/src/libstore/include/nix/store/machines.hh index f07d6b63b..2bf7408f6 100644 --- a/src/libstore/include/nix/store/machines.hh +++ b/src/libstore/include/nix/store/machines.hh @@ -15,12 +15,12 @@ typedef std::vector Machines; struct Machine { const StoreReference storeUri; - const std::set systemTypes; + const StringSet systemTypes; const std::string sshKey; const unsigned int maxJobs; const float speedFactor; - const std::set supportedFeatures; - const std::set mandatoryFeatures; + const StringSet supportedFeatures; + const StringSet mandatoryFeatures; const std::string sshPublicHostKey; bool enabled = true; @@ -34,12 +34,12 @@ struct Machine { * @return Whether `features` is a subset of the union of `supportedFeatures` and * `mandatoryFeatures`. */ - bool allSupported(const std::set & features) const; + bool allSupported(const StringSet & features) const; /** * @return Whether `mandatoryFeatures` is a subset of `features`. */ - bool mandatoryMet(const std::set & features) const; + bool mandatoryMet(const StringSet & features) const; Machine( const std::string & storeUri, @@ -75,7 +75,7 @@ struct Machine { * with `@` are interpreted as paths to other configuration files in * the same format. */ - static Machines parseConfig(const std::set & defaultSystems, const std::string & config); + static Machines parseConfig(const StringSet & defaultSystems, const std::string & config); }; /** diff --git a/src/libstore/include/nix/store/path-with-outputs.hh b/src/libstore/include/nix/store/path-with-outputs.hh index 76c1f9f8f..368667c47 100644 --- a/src/libstore/include/nix/store/path-with-outputs.hh +++ b/src/libstore/include/nix/store/path-with-outputs.hh @@ -19,7 +19,7 @@ struct StoreDirConfig; struct StorePathWithOutputs { StorePath path; - std::set outputs; + StringSet outputs; std::string to_string(const StoreDirConfig & store) const; diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index 7bc04aa4a..e3c95ce78 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -98,7 +98,7 @@ public: return "S3 Binary Cache Store"; } - static std::set uriSchemes() + static StringSet uriSchemes() { return {"s3"}; } diff --git a/src/libstore/include/nix/store/ssh-store.hh b/src/libstore/include/nix/store/ssh-store.hh index 76e8e33a4..a0f73cc8e 100644 --- a/src/libstore/include/nix/store/ssh-store.hh +++ b/src/libstore/include/nix/store/ssh-store.hh @@ -23,7 +23,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig return "Experimental SSH Store"; } - static std::set uriSchemes() + static StringSet uriSchemes() { return {"ssh-ng"}; } @@ -45,7 +45,7 @@ struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfi return "Experimental SSH Store with filesystem mounted"; } - static std::set uriSchemes() + static StringSet uriSchemes() { return {"mounted-ssh-ng"}; } diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 070abbe97..d5e3d5214 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -888,7 +888,7 @@ std::list> getDefaultSubstituters(); struct StoreFactory { - std::set uriSchemes; + StringSet uriSchemes; /** * The `authorityPath` parameter is `/`, or really * whatever comes after `://` and before `?`. diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index f7ef76058..9c85fbfb2 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -38,7 +38,7 @@ protected: static constexpr char const * scheme = "unix"; public: - static std::set uriSchemes() + static StringSet uriSchemes() { return {scheme}; } }; diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 4d1be5785..30396d7b6 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -119,7 +119,7 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path) return pathExists(binaryCacheDir + "/" + path); } -std::set LocalBinaryCacheStoreConfig::uriSchemes() +StringSet LocalBinaryCacheStoreConfig::uriSchemes() { if (getEnv("_NIX_FORCE_HTTP") == "1") return {}; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ade209b8a..f1cf466a6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -535,7 +535,7 @@ void LocalStore::upgradeDBSchema(State & state) { state.db.exec("create table if not exists SchemaMigrations (migration text primary key not null);"); - std::set schemaMigrations; + StringSet schemaMigrations; { SQLiteStmt querySchemaMigrations; diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index d98d06651..0325a6a3a 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -47,7 +47,7 @@ bool Machine::systemSupported(const std::string & system) const return system == "builtin" || (systemTypes.count(system) > 0); } -bool Machine::allSupported(const std::set & features) const +bool Machine::allSupported(const StringSet & features) const { return std::all_of(features.begin(), features.end(), [&](const std::string & feature) { @@ -56,7 +56,7 @@ bool Machine::allSupported(const std::set & features) const }); } -bool Machine::mandatoryMet(const std::set & features) const +bool Machine::mandatoryMet(const StringSet & features) const { return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), [&](const std::string & feature) { @@ -134,7 +134,7 @@ static std::vector expandBuilderLines(const std::string & builders) return result; } -static Machine parseBuilderLine(const std::set & defaultSystems, const std::string & line) +static Machine parseBuilderLine(const StringSet & defaultSystems, const std::string & line) { const auto tokens = tokenizeString>(line); @@ -178,7 +178,7 @@ static Machine parseBuilderLine(const std::set & defaultSystems, co // `storeUri` tokens[0], // `systemTypes` - isSet(1) ? tokenizeString>(tokens[1], ",") : defaultSystems, + isSet(1) ? tokenizeString(tokens[1], ",") : defaultSystems, // `sshKey` isSet(2) ? tokens[2] : "", // `maxJobs` @@ -186,15 +186,15 @@ static Machine parseBuilderLine(const std::set & defaultSystems, co // `speedFactor` isSet(4) ? parseFloatField(4) : 1.0f, // `supportedFeatures` - isSet(5) ? tokenizeString>(tokens[5], ",") : std::set{}, + isSet(5) ? tokenizeString(tokens[5], ",") : StringSet{}, // `mandatoryFeatures` - isSet(6) ? tokenizeString>(tokens[6], ",") : std::set{}, + isSet(6) ? tokenizeString(tokens[6], ",") : StringSet{}, // `sshPublicHostKey` isSet(7) ? ensureBase64(7) : "" }; } -static Machines parseBuilderLines(const std::set & defaultSystems, const std::vector & builders) +static Machines parseBuilderLines(const StringSet & defaultSystems, const std::vector & builders) { Machines result; std::transform( @@ -203,7 +203,7 @@ static Machines parseBuilderLines(const std::set & defaultSystems, return result; } -Machines Machine::parseConfig(const std::set & defaultSystems, const std::string & s) +Machines Machine::parseConfig(const StringSet & defaultSystems, const std::string & s) { const auto builderLines = expandBuilderLines(s); return parseBuilderLines(defaultSystems, builderLines); diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 9fbbc8f46..f3fc534ef 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -82,9 +82,9 @@ std::pair parsePathWithOutputs(std::string_view s) { size_t n = s.find("!"); return n == s.npos - ? std::make_pair(s, std::set()) + ? std::make_pair(s, StringSet()) : std::make_pair(s.substr(0, n), - tokenizeString>(s.substr(n + 1), ",")); + tokenizeString(s.substr(n + 1), ",")); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index fb9d7163c..d8d004e6f 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -593,7 +593,7 @@ MultiCommand::MultiCommand(std::string_view commandName, const Commands & comman assert(!command); auto i = commands.find(s); if (i == commands.end()) { - std::set commandNames; + StringSet commandNames; for (auto & [name, _] : commands) commandNames.insert(name); auto suggestions = Suggestions::bestMatches(commandNames, s); diff --git a/src/libutil/configuration.cc b/src/libutil/configuration.cc index 0f5a6a432..314ae34db 100644 --- a/src/libutil/configuration.cc +++ b/src/libutil/configuration.cc @@ -220,7 +220,7 @@ void Config::convertToArgs(Args & args, const std::string & category) AbstractSetting::AbstractSetting( const std::string & name, const std::string & description, - const std::set & aliases, + const StringSet & aliases, std::optional experimentalFeature) : name(name) , description(stripIndentation(description)) @@ -428,7 +428,7 @@ PathSetting::PathSetting(Config * options, const Path & def, const std::string & name, const std::string & description, - const std::set & aliases) + const StringSet & aliases) : BaseSetting(def, true, name, description, aliases) { options->addSetting(this); @@ -444,7 +444,7 @@ OptionalPathSetting::OptionalPathSetting(Config * options, const std::optional & def, const std::string & name, const std::string & description, - const std::set & aliases) + const StringSet & aliases) : BaseSetting>(def, true, name, description, aliases) { options->addSetting(this); diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 348caa44e..7dee1f5c7 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -358,7 +358,7 @@ nlohmann::json documentExperimentalFeatures() return (nlohmann::json) res; } -std::set parseFeatures(const std::set & rawFeatures) +std::set parseFeatures(const StringSet & rawFeatures) { std::set res; for (auto & rawFeature : rawFeatures) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 0a654b914..50516ac71 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -32,9 +32,9 @@ static size_t regularHashSize(HashAlgorithm type) { } -const std::set hashAlgorithms = {"blake3", "md5", "sha1", "sha256", "sha512" }; +const StringSet hashAlgorithms = {"blake3", "md5", "sha1", "sha256", "sha512" }; -const std::set hashFormats = {"base64", "nix32", "base16", "sri" }; +const StringSet hashFormats = {"base64", "nix32", "base16", "sri" }; Hash::Hash(HashAlgorithm algo, const ExperimentalFeatureSettings & xpSettings) : algo(algo) { diff --git a/src/libutil/include/nix/util/args.hh b/src/libutil/include/nix/util/args.hh index 463270374..f1eb96675 100644 --- a/src/libutil/include/nix/util/args.hh +++ b/src/libutil/include/nix/util/args.hh @@ -179,7 +179,7 @@ public: using ptr = std::shared_ptr; std::string longName; - std::set aliases; + StringSet aliases; char shortName = 0; std::string description; std::string category; @@ -263,7 +263,7 @@ protected: virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) { return pos; } - std::set hiddenCategories; + StringSet hiddenCategories; /** * Called after all command line flags before the first non-flag diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 34cefd73b..24b42f02c 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -179,7 +179,7 @@ public: const std::string name; const std::string description; - const std::set aliases; + const StringSet aliases; int created = 123; @@ -192,7 +192,7 @@ protected: AbstractSetting( const std::string & name, const std::string & description, - const std::set & aliases, + const StringSet & aliases, std::optional experimentalFeature = std::nullopt); virtual ~AbstractSetting(); @@ -251,7 +251,7 @@ public: const bool documentDefault, const std::string & name, const std::string & description, - const std::set & aliases = {}, + const StringSet & aliases = {}, std::optional experimentalFeature = std::nullopt) : AbstractSetting(name, description, aliases, experimentalFeature) , value(def) @@ -323,7 +323,7 @@ public: const T & def, const std::string & name, const std::string & description, - const std::set & aliases = {}, + const StringSet & aliases = {}, const bool documentDefault = true, std::optional experimentalFeature = std::nullopt) : BaseSetting(def, documentDefault, name, description, aliases, std::move(experimentalFeature)) @@ -349,7 +349,7 @@ public: const Path & def, const std::string & name, const std::string & description, - const std::set & aliases = {}); + const StringSet & aliases = {}); Path parse(const std::string & str) const override; @@ -371,7 +371,7 @@ public: const std::optional & def, const std::string & name, const std::string & description, - const std::set & aliases = {}); + const StringSet & aliases = {}); std::optional parse(const std::string & str) const override; diff --git a/src/libutil/include/nix/util/experimental-features.hh b/src/libutil/include/nix/util/experimental-features.hh index 06dd7062b..8923517ba 100644 --- a/src/libutil/include/nix/util/experimental-features.hh +++ b/src/libutil/include/nix/util/experimental-features.hh @@ -76,7 +76,7 @@ std::ostream & operator<<( * Parse a set of strings to the corresponding set of experimental * features, ignoring (but warning for) any unknown feature. */ -std::set parseFeatures(const std::set &); +std::set parseFeatures(const StringSet &); /** * An experimental feature was required for some (experimental) diff --git a/src/libutil/include/nix/util/hash.hh b/src/libutil/include/nix/util/hash.hh index f3cc4cc6c..0ae0a9564 100644 --- a/src/libutil/include/nix/util/hash.hh +++ b/src/libutil/include/nix/util/hash.hh @@ -20,7 +20,7 @@ const int sha1HashSize = 20; const int sha256HashSize = 32; const int sha512HashSize = 64; -extern const std::set hashAlgorithms; +extern const StringSet hashAlgorithms; extern const std::string nix32Chars; @@ -40,7 +40,7 @@ enum struct HashFormat : int { SRI }; -extern const std::set hashFormats; +extern const StringSet hashFormats; struct Hash { diff --git a/src/libutil/include/nix/util/strings.hh b/src/libutil/include/nix/util/strings.hh index 521e3425f..4c213de87 100644 --- a/src/libutil/include/nix/util/strings.hh +++ b/src/libutil/include/nix/util/strings.hh @@ -1,5 +1,7 @@ #pragma once +#include "nix/util/types.hh" + #include #include #include @@ -30,7 +32,7 @@ template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); extern template std::list tokenizeString(std::string_view s, std::string_view separators); -extern template std::set tokenizeString(std::string_view s, std::string_view separators); +extern template StringSet tokenizeString(std::string_view s, std::string_view separators); extern template std::vector tokenizeString(std::string_view s, std::string_view separators); /** @@ -44,7 +46,7 @@ template C splitString(std::string_view s, std::string_view separators); extern template std::list splitString(std::string_view s, std::string_view separators); -extern template std::set splitString(std::string_view s, std::string_view separators); +extern template StringSet splitString(std::string_view s, std::string_view separators); extern template std::vector splitString(std::string_view s, std::string_view separators); /** @@ -54,7 +56,7 @@ template std::string concatStringsSep(const std::string_view sep, const C & ss); extern template std::string concatStringsSep(std::string_view, const std::list &); -extern template std::string concatStringsSep(std::string_view, const std::set &); +extern template std::string concatStringsSep(std::string_view, const StringSet &); extern template std::string concatStringsSep(std::string_view, const std::vector &); extern template std::string concatStringsSep(std::string_view, const boost::container::small_vector &); @@ -85,7 +87,7 @@ template dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss); extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); -extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const StringSet &); extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); /** diff --git a/src/libutil/include/nix/util/suggestions.hh b/src/libutil/include/nix/util/suggestions.hh index 16496379c..6a76eb9d9 100644 --- a/src/libutil/include/nix/util/suggestions.hh +++ b/src/libutil/include/nix/util/suggestions.hh @@ -35,7 +35,7 @@ public: ) const; static Suggestions bestMatches ( - const std::set & allMatches, + const StringSet & allMatches, std::string_view query ); diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index 7ce37d73c..a95390089 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -26,18 +26,18 @@ __attribute__((no_sanitize("undefined"))) std::string_view toView(const std::ost } template std::list tokenizeString(std::string_view s, std::string_view separators); -template std::set tokenizeString(std::string_view s, std::string_view separators); +template StringSet tokenizeString(std::string_view s, std::string_view separators); template std::vector tokenizeString(std::string_view s, std::string_view separators); template std::list splitString(std::string_view s, std::string_view separators); -template std::set splitString(std::string_view s, std::string_view separators); +template StringSet splitString(std::string_view s, std::string_view separators); template std::vector splitString(std::string_view s, std::string_view separators); template std::list basicSplitString(std::basic_string_view s, std::basic_string_view separators); template std::string concatStringsSep(std::string_view, const std::list &); -template std::string concatStringsSep(std::string_view, const std::set &); +template std::string concatStringsSep(std::string_view, const StringSet &); template std::string concatStringsSep(std::string_view, const std::vector &); template std::string concatStringsSep(std::string_view, const boost::container::small_vector &); @@ -49,7 +49,7 @@ typedef std::string_view strings_4[4]; template std::string concatStringsSep(std::string_view, const strings_4 &); template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); -template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const StringSet &); template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); /** diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 0105c30e7..aee23d45e 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -38,7 +38,7 @@ int levenshteinDistance(std::string_view first, std::string_view second) } Suggestions Suggestions::bestMatches ( - const std::set & allMatches, + const StringSet & allMatches, std::string_view query) { std::set res; diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 231b320ac..1becac2b8 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -147,7 +147,7 @@ static void main_nix_build(int argc, char * * argv) std::string outLink = "./result"; // List of environment variables kept for --pure - std::set keepVars{ + StringSet keepVars{ "HOME", "XDG_RUNTIME_DIR", "USER", "LOGNAME", "DISPLAY", "WAYLAND_DISPLAY", "WAYLAND_SOCKET", "PATH", "TERM", "IN_NIX_SHELL", "NIX_SHELL_PRESERVE_PROMPT", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL", diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 021619ada..ff629d430 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -238,9 +238,9 @@ static void checkSelectorUse(DrvNames & selectors) namespace { -std::set searchByPrefix(const PackageInfos & allElems, std::string_view prefix) { +StringSet searchByPrefix(const PackageInfos & allElems, std::string_view prefix) { constexpr std::size_t maxResults = 3; - std::set result; + StringSet result; for (const auto & packageInfo : allElems) { const auto drvName = DrvName { packageInfo.queryName() }; if (hasPrefix(drvName.name, prefix)) { diff --git a/src/nix/develop.cc b/src/nix/develop.cc index af32c8386..b525e5de3 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -67,7 +67,7 @@ struct BuildEnvironment { BuildEnvironment res; - std::set exported; + StringSet exported; for (auto & [name, info] : json["variables"].items()) { std::string type = info["type"]; @@ -151,7 +151,7 @@ struct BuildEnvironment return structuredAttrs->second; } - void toBash(std::ostream & out, const std::set & ignoreVars) const + void toBash(std::ostream & out, const StringSet & ignoreVars) const { for (auto & [name, value] : vars) { if (!ignoreVars.count(name)) { @@ -308,7 +308,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore struct Common : InstallableCommand, MixProfile { - std::set ignoreVars{ + StringSet ignoreVars{ "BASHOPTS", "HOME", // FIXME: don't ignore in pure mode? "NIX_BUILD_TOP", diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index c4d21db6f..ff9f9db40 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -47,10 +47,10 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) return groupedPaths; } -std::string showVersions(const std::set & versions) +std::string showVersions(const StringSet & versions) { if (versions.empty()) return "∅"; - std::set versions2; + StringSet versions2; for (auto & version : versions) versions2.insert(version.empty() ? "ε" : version); return concatStringsSep(", ", versions2); @@ -65,7 +65,7 @@ void printClosureDiff( auto beforeClosure = getClosureInfo(store, beforePath); auto afterClosure = getClosureInfo(store, afterPath); - std::set allNames; + StringSet allNames; for (auto & [name, _] : beforeClosure) allNames.insert(name); for (auto & [name, _] : afterClosure) allNames.insert(name); @@ -87,11 +87,11 @@ void printClosureDiff( auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize; auto showDelta = std::abs(sizeDelta) >= 8 * 1024; - std::set removed, unchanged; + StringSet removed, unchanged; for (auto & [version, _] : beforeVersions) if (!afterVersions.count(version)) removed.insert(version); else unchanged.insert(version); - std::set added; + StringSet added; for (auto & [version, _] : afterVersions) if (!beforeVersions.count(version)) added.insert(version); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 4586ebfbe..351b2caaf 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -386,7 +386,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::set omittedSystems; + StringSet omittedSystems; // FIXME: rewrite to use EvalCache. diff --git a/src/nix/profile.cc b/src/nix/profile.cc index a0b6ee6c2..51de2f789 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -67,7 +67,7 @@ struct ProfileElement * Return a string representing an installable corresponding to the current * element, either a flakeref or a plain store path */ - std::set toInstallables(Store & store) + StringSet toInstallables(Store & store) { if (source) return {source->to_string()}; @@ -600,7 +600,7 @@ public: }); } - std::set getMatchingElementNames(ProfileManifest & manifest) { + StringSet getMatchingElementNames(ProfileManifest & manifest) { if (_matchers.empty()) { throw UsageError("No packages specified."); } @@ -614,7 +614,7 @@ public: return {}; } - std::set result; + StringSet result; for (auto & matcher : _matchers) { bool foundMatch = false; for (auto & [name, element] : manifest.elements) { diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 5de32caae..3aac45d34 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -193,7 +193,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions /* Sort the references by distance to `dependency` to ensure that the shortest path is printed first. */ std::multimap refs; - std::set hashes; + StringSet hashes; for (auto & ref : node.refs) { if (ref == node.path && packagePath != dependencyPath) continue; From 55815ec22545a560435cc8b5963bf4efb0f3fca0 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 2 May 2025 17:40:31 +0000 Subject: [PATCH 292/396] treewide: Use PathSet alias consistently instead of std::set --- src/libstore/builtins/buildenv.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 74036d22e..eeaccb24c 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -123,7 +123,7 @@ void buildProfile(const Path & out, Packages && pkgs) { State state; - std::set done, postponed; + PathSet done, postponed; auto addPkg = [&](const Path & pkgDir, int priority) { if (!done.insert(pkgDir).second) return; @@ -157,7 +157,7 @@ void buildProfile(const Path & out, Packages && pkgs) */ auto priorityCounter = 1000; while (!postponed.empty()) { - std::set pkgDirs; + PathSet pkgDirs; postponed.swap(pkgDirs); for (const auto & pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++); From 5278cd23966359a3971858dca01491cf3a9bba6a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 2 May 2025 17:40:34 +0000 Subject: [PATCH 293/396] libstore: Introduce WorkerProto::FeatureSet alias Unfortunately Feature is just an alias to `std::string` and not a new-type, so a ton of code relies on it being exactly a `std::string`. Using transparent comparators just for StringSet necessitates using it here as well. --- src/libstore-tests/worker-protocol.cc | 4 ++-- .../nix/store/worker-protocol-connection.hh | 16 ++++--------- .../include/nix/store/worker-protocol.hh | 3 ++- src/libstore/worker-protocol-connection.cc | 23 +++++++++---------- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index 80d314b1c..4baf8a325 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -685,7 +685,7 @@ TEST_F(WorkerProtoTest, handshake_features) toClient.create(); toServer.create(); - std::tuple> clientResult; + std::tuple clientResult; auto clientThread = std::thread([&]() { FdSink out { toServer.writeSide.get() }; @@ -703,7 +703,7 @@ TEST_F(WorkerProtoTest, handshake_features) EXPECT_EQ(clientResult, daemonResult); EXPECT_EQ(std::get<0>(clientResult), 123u); - EXPECT_EQ(std::get<1>(clientResult), std::set({"bar", "xyzzy"})); + EXPECT_EQ(std::get<1>(clientResult), WorkerProto::FeatureSet({"bar", "xyzzy"})); } /// Has to be a `BufferedSink` for handshake. diff --git a/src/libstore/include/nix/store/worker-protocol-connection.hh b/src/libstore/include/nix/store/worker-protocol-connection.hh index df2fe0ec2..11f112a71 100644 --- a/src/libstore/include/nix/store/worker-protocol-connection.hh +++ b/src/libstore/include/nix/store/worker-protocol-connection.hh @@ -26,7 +26,7 @@ struct WorkerProto::BasicConnection /** * The set of features that both sides support. */ - std::set features; + FeatureSet features; /** * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the @@ -92,11 +92,8 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection * @param supportedFeatures The protocol features that we support. */ // FIXME: this should probably be a constructor. - static std::tuple> handshake( - BufferedSink & to, - Source & from, - WorkerProto::Version localVersion, - const std::set & supportedFeatures); + static std::tuple handshake( + BufferedSink & to, Source & from, WorkerProto::Version localVersion, const FeatureSet & supportedFeatures); /** * After calling handshake, must call this to exchange some basic @@ -155,11 +152,8 @@ struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection * @param supportedFeatures The protocol features that we support. */ // FIXME: this should probably be a constructor. - static std::tuple> handshake( - BufferedSink & to, - Source & from, - WorkerProto::Version localVersion, - const std::set & supportedFeatures); + static std::tuple handshake( + BufferedSink & to, Source & from, WorkerProto::Version localVersion, const FeatureSet & supportedFeatures); /** * After calling handshake, must call this to exchange some basic diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 3060681b8..f96d41c71 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -135,8 +135,9 @@ struct WorkerProto } using Feature = std::string; + using FeatureSet = std::set; - static const std::set allFeatures; + static const FeatureSet allFeatures; }; enum struct WorkerProto::Op : uint64_t diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index d83be10e6..d07dc8163 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -5,7 +5,7 @@ namespace nix { -const std::set WorkerProto::allFeatures{}; +const WorkerProto::FeatureSet WorkerProto::allFeatures{}; WorkerProto::BasicClientConnection::~BasicClientConnection() { @@ -146,21 +146,20 @@ void WorkerProto::BasicClientConnection::processStderr( } } -static std::set -intersectFeatures(const std::set & a, const std::set & b) +static WorkerProto::FeatureSet intersectFeatures(const WorkerProto::FeatureSet & a, const WorkerProto::FeatureSet & b) { - std::set res; + WorkerProto::FeatureSet res; for (auto & x : a) if (b.contains(x)) res.insert(x); return res; } -std::tuple> WorkerProto::BasicClientConnection::handshake( +std::tuple WorkerProto::BasicClientConnection::handshake( BufferedSink & to, Source & from, WorkerProto::Version localVersion, - const std::set & supportedFeatures) + const WorkerProto::FeatureSet & supportedFeatures) { to << WORKER_MAGIC_1 << localVersion; to.flush(); @@ -178,21 +177,21 @@ std::tuple> WorkerProto::Ba auto protoVersion = std::min(daemonVersion, localVersion); /* Exchange features. */ - std::set daemonFeatures; + WorkerProto::FeatureSet daemonFeatures; if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { to << supportedFeatures; to.flush(); - daemonFeatures = readStrings>(from); + daemonFeatures = readStrings(from); } return {protoVersion, intersectFeatures(daemonFeatures, supportedFeatures)}; } -std::tuple> WorkerProto::BasicServerConnection::handshake( +std::tuple WorkerProto::BasicServerConnection::handshake( BufferedSink & to, Source & from, WorkerProto::Version localVersion, - const std::set & supportedFeatures) + const WorkerProto::FeatureSet & supportedFeatures) { unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_1) @@ -204,9 +203,9 @@ std::tuple> WorkerProto::Ba auto protoVersion = std::min(clientVersion, localVersion); /* Exchange features. */ - std::set clientFeatures; + WorkerProto::FeatureSet clientFeatures; if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { - clientFeatures = readStrings>(from); + clientFeatures = readStrings(from); to << supportedFeatures; to.flush(); } From ebb836d499e8395bb1f6cb18de09346fc7e761ac Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 2 May 2025 17:43:02 +0000 Subject: [PATCH 294/396] Use transparent comparators for std::set (NFC) This patch finally applies the transition to std::less<>, which is a transparent comparator. There's no functional change and string lookups in sets are now more efficient and don't produce temporaries (e.g. set.find(std::string_view{"key"})). --- .../include/nix/store/common-protocol-impl.hh | 4 ++-- .../include/nix/store/common-protocol.hh | 6 ++--- src/libstore/include/nix/store/derivations.hh | 2 +- .../store/length-prefixed-protocol-helper.hh | 20 ++++++++--------- .../include/nix/store/outputs-spec.hh | 16 +++++++++----- .../include/nix/store/serve-protocol-impl.hh | 4 +++- .../include/nix/store/serve-protocol.hh | 6 ++--- .../include/nix/store/worker-protocol-impl.hh | 4 +++- .../include/nix/store/worker-protocol.hh | 8 +++---- src/libutil/include/nix/util/json-utils.hh | 4 ++-- src/libutil/include/nix/util/topo-sort.hh | 10 ++++----- src/libutil/include/nix/util/types.hh | 22 +++++++++++++++++-- 12 files changed, 66 insertions(+), 40 deletions(-) diff --git a/src/libstore/include/nix/store/common-protocol-impl.hh b/src/libstore/include/nix/store/common-protocol-impl.hh index 171b4c6a5..18e63ac33 100644 --- a/src/libstore/include/nix/store/common-protocol-impl.hh +++ b/src/libstore/include/nix/store/common-protocol-impl.hh @@ -25,11 +25,11 @@ namespace nix { LengthPrefixedProtoHelper::write(store, conn, t); \ } +#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) -COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) -#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER( template, std::map) diff --git a/src/libstore/include/nix/store/common-protocol.hh b/src/libstore/include/nix/store/common-protocol.hh index b464cda67..7887120b5 100644 --- a/src/libstore/include/nix/store/common-protocol.hh +++ b/src/libstore/include/nix/store/common-protocol.hh @@ -72,14 +72,14 @@ DECLARE_COMMON_SERIALISER(DrvOutput); template<> DECLARE_COMMON_SERIALISER(Realisation); +#define COMMA_ , template DECLARE_COMMON_SERIALISER(std::vector); -template -DECLARE_COMMON_SERIALISER(std::set); +template +DECLARE_COMMON_SERIALISER(std::set); template DECLARE_COMMON_SERIALISER(std::tuple); -#define COMMA_ , template DECLARE_COMMON_SERIALISER(std::map); #undef COMMA_ diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 01ff337f6..46a9e2d02 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -343,7 +343,7 @@ struct Derivation : BasicDerivation /** * inputs that are sub-derivations */ - DerivedPathMap> inputDrvs; + DerivedPathMap>> inputDrvs; /** * Print a derivation. diff --git a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh index 664841aae..a83635aa4 100644 --- a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh +++ b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh @@ -52,8 +52,10 @@ struct LengthPrefixedProtoHelper; template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); -template -LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); +#define COMMA_ , +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); +#undef COMMA_ template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); @@ -86,12 +88,11 @@ LengthPrefixedProtoHelper>::write( } } -template -std::set -LengthPrefixedProtoHelper>::read( +template +std::set LengthPrefixedProtoHelper>::read( const StoreDirConfig & store, typename Inner::ReadConn conn) { - std::set resSet; + std::set resSet; auto size = readNum(conn.from); while (size--) { resSet.insert(S::read(store, conn)); @@ -99,10 +100,9 @@ LengthPrefixedProtoHelper>::read( return resSet; } -template -void -LengthPrefixedProtoHelper>::write( - const StoreDirConfig & store, typename Inner::WriteConn conn, const std::set & resSet) +template +void LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::set & resSet) { conn.to << resSet.size(); for (auto & key : resSet) { diff --git a/src/libstore/include/nix/store/outputs-spec.hh b/src/libstore/include/nix/store/outputs-spec.hh index b89f425c2..b47f26542 100644 --- a/src/libstore/include/nix/store/outputs-spec.hh +++ b/src/libstore/include/nix/store/outputs-spec.hh @@ -27,20 +27,24 @@ struct OutputsSpec { /** * A non-empty set of outputs, specified by name */ - struct Names : std::set { - using std::set::set; + struct Names : std::set> { + private: + using BaseType = std::set>; + + public: + using BaseType::BaseType; /* These need to be "inherited manually" */ - Names(const std::set & s) - : std::set(s) + Names(const BaseType & s) + : BaseType(s) { assert(!empty()); } /** * Needs to be "inherited manually" */ - Names(std::set && s) - : std::set(s) + Names(BaseType && s) + : BaseType(std::move(s)) { assert(!empty()); } /* This set should always be non-empty, so we delete this diff --git a/src/libstore/include/nix/store/serve-protocol-impl.hh b/src/libstore/include/nix/store/serve-protocol-impl.hh index 769b9ae2b..4ab164721 100644 --- a/src/libstore/include/nix/store/serve-protocol-impl.hh +++ b/src/libstore/include/nix/store/serve-protocol-impl.hh @@ -26,7 +26,9 @@ namespace nix { } SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) -SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +#define COMMA_ , +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +#undef COMMA_ SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA , diff --git a/src/libstore/include/nix/store/serve-protocol.hh b/src/libstore/include/nix/store/serve-protocol.hh index 76f0ecd49..6f6bf6b60 100644 --- a/src/libstore/include/nix/store/serve-protocol.hh +++ b/src/libstore/include/nix/store/serve-protocol.hh @@ -180,12 +180,12 @@ DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); template DECLARE_SERVE_SERIALISER(std::vector); -template -DECLARE_SERVE_SERIALISER(std::set); +#define COMMA_ , +template +DECLARE_SERVE_SERIALISER(std::set); template DECLARE_SERVE_SERIALISER(std::tuple); -#define COMMA_ , template DECLARE_SERVE_SERIALISER(std::map); #undef COMMA_ diff --git a/src/libstore/include/nix/store/worker-protocol-impl.hh b/src/libstore/include/nix/store/worker-protocol-impl.hh index 337c245e2..908a9323e 100644 --- a/src/libstore/include/nix/store/worker-protocol-impl.hh +++ b/src/libstore/include/nix/store/worker-protocol-impl.hh @@ -26,7 +26,9 @@ namespace nix { } WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) -WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +#define COMMA_ , +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +#undef COMMA_ WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA , diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index f96d41c71..1b188806d 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -135,7 +135,7 @@ struct WorkerProto } using Feature = std::string; - using FeatureSet = std::set; + using FeatureSet = std::set>; static const FeatureSet allFeatures; }; @@ -273,12 +273,12 @@ DECLARE_WORKER_SERIALISER(WorkerProto::ClientHandshakeInfo); template DECLARE_WORKER_SERIALISER(std::vector); -template -DECLARE_WORKER_SERIALISER(std::set); +#define COMMA_ , +template +DECLARE_WORKER_SERIALISER(std::set); template DECLARE_WORKER_SERIALISER(std::tuple); -#define COMMA_ , template DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ diff --git a/src/libutil/include/nix/util/json-utils.hh b/src/libutil/include/nix/util/json-utils.hh index bcae46a0a..37f4d58f8 100644 --- a/src/libutil/include/nix/util/json-utils.hh +++ b/src/libutil/include/nix/util/json-utils.hh @@ -90,8 +90,8 @@ struct json_avoids_null> : std::true_type {}; template struct json_avoids_null> : std::true_type {}; -template -struct json_avoids_null> : std::true_type {}; +template +struct json_avoids_null> : std::true_type {}; template struct json_avoids_null> : std::true_type {}; diff --git a/src/libutil/include/nix/util/topo-sort.hh b/src/libutil/include/nix/util/topo-sort.hh index 77a9ce421..6ba6fda71 100644 --- a/src/libutil/include/nix/util/topo-sort.hh +++ b/src/libutil/include/nix/util/topo-sort.hh @@ -5,13 +5,13 @@ namespace nix { -template -std::vector topoSort(std::set items, - std::function(const T &)> getChildren, +template +std::vector topoSort(std::set items, + std::function(const T &)> getChildren, std::function makeCycleError) { std::vector sorted; - std::set visited, parents; + decltype(items) visited, parents; std::function dfsVisit; @@ -21,7 +21,7 @@ std::vector topoSort(std::set items, if (!visited.insert(path).second) return; parents.insert(path); - std::set references = getChildren(path); + auto references = getChildren(path); for (auto & i : references) /* Don't traverse into items that don't exist in our starting set. */ diff --git a/src/libutil/include/nix/util/types.hh b/src/libutil/include/nix/util/types.hh index 9f5c75827..5139256ca 100644 --- a/src/libutil/include/nix/util/types.hh +++ b/src/libutil/include/nix/util/types.hh @@ -12,17 +12,35 @@ namespace nix { typedef std::list Strings; -typedef std::set StringSet; typedef std::map StringMap; typedef std::map StringPairs; +/** + * Alias to ordered set container with transparent comparator. + * + * Used instead of std::set to use C++14 N3657 [1] + * heterogenous lookup consistently across the whole codebase. + * Transparent comparators get rid of creation of unnecessary + * temporary variables when looking up keys by `std::string_view` + * or C-style `const char *` strings. + * + * [1]: https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2013/n3657.htm + */ +using StringSet = std::set>; + /** * Paths are just strings. */ typedef std::string Path; typedef std::string_view PathView; typedef std::list Paths; -typedef std::set PathSet; + +/** + * Alias to an ordered set of `Path`s. Uses transparent comparator. + * + * @see StringSet + */ +using PathSet = std::set>; typedef std::vector> Headers; From 161c5dbf39c789ad5bd397109f8858e175501483 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 2 May 2025 20:42:47 +0000 Subject: [PATCH 295/396] libexpr: Remove unused field from SymbolTable::symbols and emplace into the ChunkedVector Remove outdated and no longer relevant TODO. It's more confusing now, since symbol table must now be addressed by uint32_t indices in order to keep Attr size down to 16 bytes on 64 bit machines. --- src/libexpr/include/nix/expr/symbol-table.hh | 17 ++++++++++------- src/libutil/include/nix/util/chunked-vector.hh | 14 ++++++++++---- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/libexpr/include/nix/expr/symbol-table.hh b/src/libexpr/include/nix/expr/symbol-table.hh index 018465bf5..c04cc041b 100644 --- a/src/libexpr/include/nix/expr/symbol-table.hh +++ b/src/libexpr/include/nix/expr/symbol-table.hh @@ -81,26 +81,29 @@ public: class SymbolTable { private: - std::unordered_map> symbols; + /** + * Map from string view (backed by ChunkedVector) -> offset into the store. + * ChunkedVector references are never invalidated. + */ + std::unordered_map symbols; ChunkedVector store{16}; public: /** - * converts a string into a symbol. + * Converts a string into a symbol. */ Symbol create(std::string_view s) { // Most symbols are looked up more than once, so we trade off insertion performance // for lookup performance. - // TODO: could probably be done more efficiently with transparent Hash and Equals - // on the original implementation using unordered_set // FIXME: make this thread-safe. auto it = symbols.find(s); - if (it != symbols.end()) return Symbol(it->second.second + 1); + if (it != symbols.end()) + return Symbol(it->second + 1); - const auto & [rawSym, idx] = store.add(std::string(s)); - symbols.emplace(rawSym, std::make_pair(&rawSym, idx)); + const auto & [rawSym, idx] = store.add(s); + symbols.emplace(rawSym, idx); return Symbol(idx + 1); } diff --git a/src/libutil/include/nix/util/chunked-vector.hh b/src/libutil/include/nix/util/chunked-vector.hh index 96a717556..2c21183ac 100644 --- a/src/libutil/include/nix/util/chunked-vector.hh +++ b/src/libutil/include/nix/util/chunked-vector.hh @@ -45,9 +45,10 @@ public: addChunk(); } - uint32_t size() const { return size_; } + uint32_t size() const noexcept { return size_; } - std::pair add(T value) + template + std::pair add(Args &&... args) { const auto idx = size_++; auto & chunk = [&] () -> auto & { @@ -55,11 +56,16 @@ public: return back; return addChunk(); }(); - auto & result = chunk.emplace_back(std::move(value)); + auto & result = chunk.emplace_back(std::forward(args)...); return {result, idx}; } - const T & operator[](uint32_t idx) const + /** + * Unchecked subscript operator. + * @pre add must have been called at least idx + 1 times. + * @throws nothing + */ + const T & operator[](uint32_t idx) const noexcept { return chunks[idx / ChunkSize][idx % ChunkSize]; } From a87c3711b6eec35b498f8c71ebd7d4038e5b4545 Mon Sep 17 00:00:00 2001 From: silvanshade Date: Wed, 30 Apr 2025 08:25:02 -0600 Subject: [PATCH 296/396] Update flake nixpkgs --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 7abe8bd62..b8ff29a0c 100644 --- a/flake.lock +++ b/flake.lock @@ -63,11 +63,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1745391562, - "narHash": "sha256-sPwcCYuiEopaafePqlG826tBhctuJsLx/mhKKM5Fmjo=", + "lastModified": 1746141548, + "narHash": "sha256-IgBWhX7A2oJmZFIrpRuMnw5RAufVnfvOgHWgIdds+hc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8a2f738d9d1f1d986b5a4cd2fd2061a7127237d7", + "rev": "f02fddb8acef29a8b32f10a335d44828d7825b78", "type": "github" }, "original": { From b1783ff6151d24344cb22cb39e5f2975424fb22f Mon Sep 17 00:00:00 2001 From: silvanshade Date: Tue, 18 Mar 2025 14:22:15 -0600 Subject: [PATCH 297/396] Implement memory-mapped IO for Sinks --- packaging/dependencies.nix | 1 + src/libutil/file-system.cc | 19 +++++++++++++++++-- src/libutil/include/nix/util/file-system.hh | 2 +- src/libutil/meson.build | 2 +- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 16d1f1376..a90ef1b4a 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -61,6 +61,7 @@ scope: { "--with-container" "--with-context" "--with-coroutine" + "--with-iostreams" ]; }).overrideAttrs (old: { diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index ad17f837f..90ec5eda5 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -21,6 +21,8 @@ #include #include +#include + #ifdef _WIN32 # include #endif @@ -273,9 +275,22 @@ std::string readFile(const std::filesystem::path & path) return readFile(os_string_to_string(PathViewNG { path })); } - -void readFile(const Path & path, Sink & sink) +void readFile(const Path & path, Sink & sink, bool memory_map) { + // Memory-map the file for faster processing where possible. + if (memory_map) { + try { + boost::iostreams::mapped_file_source mmap(path); + if (mmap.is_open()) { + sink({mmap.data(), mmap.size()}); + return; + } + } catch (const boost::exception & e) { + } + debug("memory-mapping failed for path: %s", path); + } + + // Stream the file instead if memory-mapping fails or is disabled. AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY // TODO #ifndef _WIN32 diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 1d7b5e3aa..b8fa4cfa0 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -173,7 +173,7 @@ Descriptor openDirectory(const std::filesystem::path & path); */ std::string readFile(const Path & path); std::string readFile(const std::filesystem::path & path); -void readFile(const Path & path, Sink & sink); +void readFile(const Path & path, Sink & sink, bool memory_map = true); /** * Write a string to a file. diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 782c361e0..944198970 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -56,7 +56,7 @@ deps_private += blake3 boost = dependency( 'boost', - modules : ['context', 'coroutine'], + modules : ['context', 'coroutine', 'iostreams'], include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we From 7db388f597f0db5b2b7e6217a931908d31df1299 Mon Sep 17 00:00:00 2001 From: silvanshade Date: Tue, 18 Mar 2025 15:16:56 -0600 Subject: [PATCH 298/396] Implement multi-threaded BLAKE3 hashing --- src/libutil/hash.cc | 22 +++++++++++++++++++++- src/libutil/meson.build | 3 ++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 0a654b914..50b050e3b 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -307,11 +307,31 @@ static void start(HashAlgorithm ha, Ctx & ctx) else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512); } +// BLAKE3 data size threshold beyond which parallel hashing with TBB is likely faster. +// +// NOTE: This threshold is based on the recommended rule-of-thumb from the official BLAKE3 documentation for typical +// x86_64 hardware as of 2025. In the future it may make sense to allow the user to tune this through nix.conf. +const size_t blake3TbbThreshold = 128000; + +// Decide which BLAKE3 update strategy to use based on some heuristics. Currently this just checks the data size but in +// the future it might also take into consideration available system resources or the presence of a shared-memory +// capable GPU for a heterogenous compute implementation. +void blake3_hasher_update_with_heuristics(blake3_hasher * blake3, std::string_view data) +{ +#ifdef BLAKE3_USE_TBB + if (data.size() >= blake3TbbThreshold) { + blake3_hasher_update_tbb(blake3, data.data(), data.size()); + } else +#endif + { + blake3_hasher_update(blake3, data.data(), data.size()); + } +} static void update(HashAlgorithm ha, Ctx & ctx, std::string_view data) { - if (ha == HashAlgorithm::BLAKE3) blake3_hasher_update(&ctx.blake3, data.data(), data.size()); + if (ha == HashAlgorithm::BLAKE3) blake3_hasher_update_with_heuristics(&ctx.blake3, data); else if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size()); else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 944198970..b0e82e46a 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -50,7 +50,8 @@ endif blake3 = dependency( 'libblake3', - version: '>= 1.5.5', + version: '>= 1.8.2', + method : 'pkg-config', ) deps_private += blake3 From 81683a845bcd7e0d1c9af0294d4cdab31949ca5b Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Sun, 4 May 2025 12:17:48 +0200 Subject: [PATCH 299/396] fix(docs): update Matrix channel links --- .github/STALE-BOT.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md index 383717bfc..bc0005413 100644 --- a/.github/STALE-BOT.md +++ b/.github/STALE-BOT.md @@ -3,7 +3,7 @@ - Thanks for your contribution! - To remove the stale label, just leave a new comment. - _How to find the right people to ping?_ → [`git blame`](https://git-scm.com/docs/git-blame) to the rescue! (or GitHub's history and blame buttons.) -- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org). +- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on [Matrix - #users:nixos.org](https://matrix.to/#/#users:nixos.org). ## Suggestions for PRs diff --git a/README.md b/README.md index 54a6fcc39..02498944c 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Today, a world-wide developer community contributes to Nix and the ecosystem tha - [Nixpkgs](https://github.com/NixOS/nixpkgs) is [the largest, most up-to-date free software repository in the world](https://repology.org/repositories/graphs) - [NixOS](https://github.com/NixOS/nixpkgs/tree/master/nixos) is a Linux distribution that can be configured fully declaratively - [Discourse](https://discourse.nixos.org/) -- [Matrix](https://matrix.to/#/#nix:nixos.org) +- Matrix: [#users:nixos.org](https://matrix.to/#/#users:nixos.org) for user support and [#nix-dev:nixos.org](https://matrix.to/#/#nix-dev:nixos.org) for development ## License From 36c583dae08907f38ea3ab6216047ab91682b1cc Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 4 May 2025 16:03:57 +0000 Subject: [PATCH 300/396] libexpr: Use C++20 heterogeneous lookup for RegexCache --- src/libexpr/primops.cc | 14 ++++++---- src/libutil/include/nix/util/strings.hh | 35 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5b24849d2..b7b027fba 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4311,9 +4311,7 @@ struct RegexCache { struct State { - // TODO use C++20 transparent comparison when available - std::unordered_map cache; - std::list keys; + std::unordered_map> cache; }; Sync state_; @@ -4324,8 +4322,14 @@ struct RegexCache auto it = state->cache.find(re); if (it != state->cache.end()) return it->second; - state->keys.emplace_back(re); - return state->cache.emplace(state->keys.back(), std::regex(state->keys.back(), std::regex::extended)).first->second; + /* No std::regex constructor overload from std::string_view, but can be constructed + from a pointer + size or an iterator range. */ + return state->cache + .emplace( + std::piecewise_construct, + std::forward_as_tuple(re), + std::forward_as_tuple(/*s=*/re.data(), /*count=*/re.size(), std::regex::extended)) + .first->second; } }; diff --git a/src/libutil/include/nix/util/strings.hh b/src/libutil/include/nix/util/strings.hh index 4c213de87..4c77516a3 100644 --- a/src/libutil/include/nix/util/strings.hh +++ b/src/libutil/include/nix/util/strings.hh @@ -97,4 +97,39 @@ extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, * Arguments that need to be passed to ssh with spaces in them. */ std::list shellSplitString(std::string_view s); + +/** + * Hash implementation that can be used for zero-copy heterogenous lookup from + * P1690R1[1] in unordered containers. + * + * [1]: https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1690r1.html + */ +struct StringViewHash +{ +private: + using HashType = std::hash; + +public: + using is_transparent = void; + + auto operator()(const char * str) const + { + /* This has a slight overhead due to an implicit strlen, but there isn't + a good way around it because the hash value of all overloads must be + consistent. Delegating to string_view is the solution initially proposed + in P0919R3. */ + return HashType{}(std::string_view{str}); + } + + auto operator()(std::string_view str) const + { + return HashType{}(str); + } + + auto operator()(const std::string & str) const + { + return HashType{}(std::string_view{str}); + } +}; + } From 40bbad3be59e71d67f93efb10bcd2fcaf01de6a3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 May 2025 10:53:47 +0200 Subject: [PATCH 301/396] Allow dynamic registration of builtin builders --- src/libstore/builtins/buildenv.cc | 5 ++++- src/libstore/builtins/unpack-channel.cc | 4 +++- src/libstore/include/nix/store/builtins.hh | 17 ++++++++++++++--- .../include/nix/store/builtins/buildenv.hh | 4 ---- src/libstore/unix/build/derivation-builder.cc | 17 +++++++++++------ 5 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index eeaccb24c..204912278 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,5 @@ #include "nix/store/builtins/buildenv.hh" +#include "nix/store/builtins.hh" #include "nix/store/derivations.hh" #include "nix/util/signals.hh" @@ -166,7 +167,7 @@ void buildProfile(const Path & out, Packages && pkgs) debug("created %d symlinks in user environment", state.symlinks); } -void builtinBuildenv( +static void builtinBuildenv( const BasicDerivation & drv, const std::map & outputs) { @@ -203,4 +204,6 @@ void builtinBuildenv( createSymlink(getAttr("manifest"), out + "/manifest.nix"); } +static RegisterBuiltinBuilder registerBuildenv("buildenv", builtinBuildenv); + } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index e03f3076b..47bf25fbf 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -3,7 +3,7 @@ namespace nix { -void builtinUnpackChannel( +static void builtinUnpackChannel( const BasicDerivation & drv, const std::map & outputs) { @@ -42,4 +42,6 @@ void builtinUnpackChannel( } } +static RegisterBuiltinBuilder registerUnpackChannel("unpack-channel", builtinUnpackChannel); + } diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index 004e9ef64..6d54c2a22 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -12,8 +12,19 @@ void builtinFetchurl( const std::string & netrcData, const std::string & caFileData); -void builtinUnpackChannel( - const BasicDerivation & drv, - const std::map & outputs); +using BuiltinBuilder = + std::function & outputs)>; + +struct RegisterBuiltinBuilder +{ + typedef std::map BuiltinBuilders; + static BuiltinBuilders * builtinBuilders; + + RegisterBuiltinBuilder(const std::string & name, BuiltinBuilder && fun) + { + if (!builtinBuilders) builtinBuilders = new BuiltinBuilders; + builtinBuilders->insert_or_assign(name, std::move(fun)); + } +}; } diff --git a/src/libstore/include/nix/store/builtins/buildenv.hh b/src/libstore/include/nix/store/builtins/buildenv.hh index a0a262037..163666c0b 100644 --- a/src/libstore/include/nix/store/builtins/buildenv.hh +++ b/src/libstore/include/nix/store/builtins/buildenv.hh @@ -45,8 +45,4 @@ typedef std::vector Packages; void buildProfile(const Path & out, Packages && pkgs); -void builtinBuildenv( - const BasicDerivation & drv, - const std::map & outputs); - } diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 4bde9750d..c40d5251b 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1835,6 +1835,9 @@ void setupSeccomp() } +RegisterBuiltinBuilder::BuiltinBuilders * RegisterBuiltinBuilder::builtinBuilders = nullptr; + + void DerivationBuilderImpl::runChild() { /* Warning: in the child we should absolutely not make any SQLite @@ -2293,12 +2296,14 @@ void DerivationBuilderImpl::runChild() if (drv.builder == "builtin:fetchurl") builtinFetchurl(drv, outputs, netrcData, caFileData); - else if (drv.builder == "builtin:buildenv") - builtinBuildenv(drv, outputs); - else if (drv.builder == "builtin:unpack-channel") - builtinUnpackChannel(drv, outputs); - else - throw Error("unsupported builtin builder '%1%'", drv.builder.substr(8)); + else { + std::string builtinName = drv.builder.substr(8); + assert(RegisterBuiltinBuilder::builtinBuilders); + if (auto builtin = get(*RegisterBuiltinBuilder::builtinBuilders, builtinName)) + (*builtin)(drv, outputs); + else + throw Error("unsupported builtin builder '%1%'", builtinName); + } _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, e.what() + std::string("\n")); From fe0124fe171430a644144ae3a6cf9d95e6ca270e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 May 2025 12:46:40 +0200 Subject: [PATCH 302/396] Put the builder context in a struct --- src/libstore/builtins/buildenv.cc | 10 +++---- src/libstore/builtins/fetchurl.cc | 28 +++++++++---------- src/libstore/builtins/unpack-channel.cc | 10 +++---- src/libstore/include/nix/store/builtins.hh | 16 +++++------ src/libstore/unix/build/derivation-builder.cc | 27 ++++++++---------- 5 files changed, 40 insertions(+), 51 deletions(-) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 204912278..0e99ca0e5 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -167,17 +167,15 @@ void buildProfile(const Path & out, Packages && pkgs) debug("created %d symlinks in user environment", state.symlinks); } -static void builtinBuildenv( - const BasicDerivation & drv, - const std::map & outputs) +static void builtinBuildenv(const BuiltinBuilderContext & ctx) { auto getAttr = [&](const std::string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + auto i = ctx.drv.env.find(name); + if (i == ctx.drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; }; - auto out = outputs.at("out"); + auto out = ctx.outputs.at("out"); createDirs(out); /* Convert the stuff we get from the environment back into a diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 82f268d80..18fa75558 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -6,33 +6,29 @@ namespace nix { -void builtinFetchurl( - const BasicDerivation & drv, - const std::map & outputs, - const std::string & netrcData, - const std::string & caFileData) +static void builtinFetchurl(const BuiltinBuilderContext & ctx) { /* Make the host's netrc data available. Too bad curl requires this to be stored in a file. It would be nice if we could just pass a pointer to the data. */ - if (netrcData != "") { + if (ctx.netrcData != "") { settings.netrcFile = "netrc"; - writeFile(settings.netrcFile, netrcData, 0600); + writeFile(settings.netrcFile, ctx.netrcData, 0600); } settings.caFile = "ca-certificates.crt"; - writeFile(settings.caFile, caFileData, 0600); + writeFile(settings.caFile, ctx.caFileData, 0600); - auto out = get(drv.outputs, "out"); + auto out = get(ctx.drv.outputs, "out"); if (!out) throw Error("'builtin:fetchurl' requires an 'out' output"); - if (!(drv.type().isFixed() || drv.type().isImpure())) + if (!(ctx.drv.type().isFixed() || ctx.drv.type().isImpure())) throw Error("'builtin:fetchurl' must be a fixed-output or impure derivation"); - auto storePath = outputs.at("out"); - auto mainUrl = drv.env.at("url"); - bool unpack = getOr(drv.env, "unpack", "") == "1"; + auto storePath = ctx.outputs.at("out"); + auto mainUrl = ctx.drv.env.at("url"); + bool unpack = getOr(ctx.drv.env, "unpack", "") == "1"; /* Note: have to use a fresh fileTransfer here because we're in a forked process. */ @@ -56,8 +52,8 @@ void builtinFetchurl( else writeFile(storePath, *source); - auto executable = drv.env.find("executable"); - if (executable != drv.env.end() && executable->second == "1") { + auto executable = ctx.drv.env.find("executable"); + if (executable != ctx.drv.env.end() && executable->second == "1") { if (chmod(storePath.c_str(), 0755) == -1) throw SysError("making '%1%' executable", storePath); } @@ -79,4 +75,6 @@ void builtinFetchurl( fetch(mainUrl); } +static RegisterBuiltinBuilder registerFetchurl("fetchurl", builtinFetchurl); + } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 47bf25fbf..dd6b8bb71 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -3,17 +3,15 @@ namespace nix { -static void builtinUnpackChannel( - const BasicDerivation & drv, - const std::map & outputs) +static void builtinUnpackChannel(const BuiltinBuilderContext & ctx) { auto getAttr = [&](const std::string & name) -> const std::string & { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + auto i = ctx.drv.env.find(name); + if (i == ctx.drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; }; - std::filesystem::path out{outputs.at("out")}; + std::filesystem::path out{ctx.outputs.at("out")}; auto & channelName = getAttr("channelName"); auto & src = getAttr("src"); diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index 6d54c2a22..7d9863e00 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -5,15 +5,15 @@ namespace nix { -// TODO: make pluggable. -void builtinFetchurl( - const BasicDerivation & drv, - const std::map & outputs, - const std::string & netrcData, - const std::string & caFileData); +struct BuiltinBuilderContext +{ + const BasicDerivation & drv; + std::map outputs; + std::string netrcData; + std::string caFileData; +}; -using BuiltinBuilder = - std::function & outputs)>; +using BuiltinBuilder = std::function; struct RegisterBuiltinBuilder { diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index c40d5251b..4288f0367 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1860,15 +1860,15 @@ void DerivationBuilderImpl::runChild() /* Make the contents of netrc and the CA certificate bundle available to builtin:fetchurl (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - std::string caFileData; + BuiltinBuilderContext ctx{.drv = drv}; + if (drv.isBuiltin() && drv.builder == "builtin:fetchurl") { try { - netrcData = readFile(settings.netrcFile); + ctx.netrcData = readFile(settings.netrcFile); } catch (SystemError &) { } try { - caFileData = readFile(settings.caFile); + ctx.caFileData = readFile(settings.caFile); } catch (SystemError &) { } } @@ -2289,21 +2289,16 @@ void DerivationBuilderImpl::runChild() try { logger = makeJSONLogger(getStandardError()); - std::map outputs; for (auto & e : drv.outputs) - outputs.insert_or_assign(e.first, + ctx.outputs.insert_or_assign(e.first, store.printStorePath(scratchOutputs.at(e.first))); - if (drv.builder == "builtin:fetchurl") - builtinFetchurl(drv, outputs, netrcData, caFileData); - else { - std::string builtinName = drv.builder.substr(8); - assert(RegisterBuiltinBuilder::builtinBuilders); - if (auto builtin = get(*RegisterBuiltinBuilder::builtinBuilders, builtinName)) - (*builtin)(drv, outputs); - else - throw Error("unsupported builtin builder '%1%'", builtinName); - } + std::string builtinName = drv.builder.substr(8); + assert(RegisterBuiltinBuilder::builtinBuilders); + if (auto builtin = get(*RegisterBuiltinBuilder::builtinBuilders, builtinName)) + (*builtin)(ctx); + else + throw Error("unsupported builtin builder '%1%'", builtinName); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, e.what() + std::string("\n")); From c7a84b9160b81a8594a2f235faf77d5e9e47a323 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 May 2025 17:49:58 +0200 Subject: [PATCH 303/396] Pass tmpDirInSandbox to the builtin builders --- src/libstore/include/nix/store/builtins.hh | 1 + src/libstore/unix/build/derivation-builder.cc | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index 7d9863e00..3385fd9fc 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -11,6 +11,7 @@ struct BuiltinBuilderContext std::map outputs; std::string netrcData; std::string caFileData; + Path tmpDirInSandbox; }; using BuiltinBuilder = std::function; diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 4288f0367..dd08ff8bd 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1860,7 +1860,10 @@ void DerivationBuilderImpl::runChild() /* Make the contents of netrc and the CA certificate bundle available to builtin:fetchurl (which may run under a different uid and/or in a sandbox). */ - BuiltinBuilderContext ctx{.drv = drv}; + BuiltinBuilderContext ctx{ + .drv = drv, + .tmpDirInSandbox = tmpDirInSandbox, + }; if (drv.isBuiltin() && drv.builder == "builtin:fetchurl") { try { From 147930500117504445d24f78c3bfe0411fa771e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 May 2025 08:16:09 +0200 Subject: [PATCH 304/396] Simplify RegisterBuiltinBuilder --- src/libstore/include/nix/store/builtins.hh | 9 ++++++--- src/libstore/unix/build/derivation-builder.cc | 5 +---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index 3385fd9fc..096c8af7b 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -19,12 +19,15 @@ using BuiltinBuilder = std::function; struct RegisterBuiltinBuilder { typedef std::map BuiltinBuilders; - static BuiltinBuilders * builtinBuilders; + + static BuiltinBuilders & builtinBuilders() { + static BuiltinBuilders builders; + return builders; + } RegisterBuiltinBuilder(const std::string & name, BuiltinBuilder && fun) { - if (!builtinBuilders) builtinBuilders = new BuiltinBuilders; - builtinBuilders->insert_or_assign(name, std::move(fun)); + builtinBuilders().insert_or_assign(name, std::move(fun)); } }; diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index dd08ff8bd..d416228dc 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1835,9 +1835,6 @@ void setupSeccomp() } -RegisterBuiltinBuilder::BuiltinBuilders * RegisterBuiltinBuilder::builtinBuilders = nullptr; - - void DerivationBuilderImpl::runChild() { /* Warning: in the child we should absolutely not make any SQLite @@ -2298,7 +2295,7 @@ void DerivationBuilderImpl::runChild() std::string builtinName = drv.builder.substr(8); assert(RegisterBuiltinBuilder::builtinBuilders); - if (auto builtin = get(*RegisterBuiltinBuilder::builtinBuilders, builtinName)) + if (auto builtin = get(RegisterBuiltinBuilder::builtinBuilders(), builtinName)) (*builtin)(ctx); else throw Error("unsupported builtin builder '%1%'", builtinName); From b7add9736cde93534572f294e3f4cb4d0573b356 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 May 2025 08:22:53 +0200 Subject: [PATCH 305/396] Simplify RegisterLegacyCommand --- src/libcmd/include/nix/cmd/legacy.hh | 9 ++++++--- src/libcmd/legacy.cc | 7 ------- src/libcmd/meson.build | 1 - src/nix/main.cc | 2 +- 4 files changed, 7 insertions(+), 12 deletions(-) delete mode 100644 src/libcmd/legacy.cc diff --git a/src/libcmd/include/nix/cmd/legacy.hh b/src/libcmd/include/nix/cmd/legacy.hh index 357500a4d..0c375a7d2 100644 --- a/src/libcmd/include/nix/cmd/legacy.hh +++ b/src/libcmd/include/nix/cmd/legacy.hh @@ -12,12 +12,15 @@ typedef std::function MainFunction; struct RegisterLegacyCommand { typedef std::map Commands; - static Commands * commands; + + static Commands & commands() { + static Commands commands; + return commands; + } RegisterLegacyCommand(const std::string & name, MainFunction fun) { - if (!commands) commands = new Commands; - (*commands)[name] = fun; + commands()[name] = fun; } }; diff --git a/src/libcmd/legacy.cc b/src/libcmd/legacy.cc deleted file mode 100644 index 69b066831..000000000 --- a/src/libcmd/legacy.cc +++ /dev/null @@ -1,7 +0,0 @@ -#include "nix/cmd/legacy.hh" - -namespace nix { - -RegisterLegacyCommand::Commands * RegisterLegacyCommand::commands = 0; - -} diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 32f44697d..216d4df9c 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -71,7 +71,6 @@ sources = files( 'installable-flake.cc', 'installable-value.cc', 'installables.cc', - 'legacy.cc', 'markdown.cc', 'misc-store-flags.cc', 'network-proxy.cc', diff --git a/src/nix/main.cc b/src/nix/main.cc index f229ba2a4..eff2634e2 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -373,7 +373,7 @@ void mainWrapped(int argc, char * * argv) } { - auto legacy = (*RegisterLegacyCommand::commands)[programName]; + auto legacy = RegisterLegacyCommand::commands()[programName]; if (legacy) return legacy(argc, argv); } From 4de7a986d49334c0662deba144e2d096d78024ef Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 May 2025 08:26:29 +0200 Subject: [PATCH 306/396] Simplify RegisterPrimOp --- src/libexpr/include/nix/expr/primops.hh | 7 ++++++- src/libexpr/primops.cc | 20 +++++++------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/libexpr/include/nix/expr/primops.hh b/src/libexpr/include/nix/expr/primops.hh index f0742a138..0b4ecdd50 100644 --- a/src/libexpr/include/nix/expr/primops.hh +++ b/src/libexpr/include/nix/expr/primops.hh @@ -27,7 +27,12 @@ constexpr size_t conservativeStackReservation = 16; struct RegisterPrimOp { typedef std::vector PrimOps; - static PrimOps * primOps; + + static PrimOps & primOps() + { + static PrimOps primOps; + return primOps; + } /** * You can register a constant by passing an arity of 0. fun diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index b7b027fba..535a9a501 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4713,13 +4713,9 @@ static RegisterPrimOp primop_splitVersion({ *************************************************************/ -RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; - - RegisterPrimOp::RegisterPrimOp(PrimOp && primOp) { - if (!primOps) primOps = new PrimOps; - primOps->push_back(std::move(primOp)); + primOps().push_back(std::move(primOp)); } @@ -4973,14 +4969,12 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) )", }); - if (RegisterPrimOp::primOps) - for (auto & primOp : *RegisterPrimOp::primOps) - if (experimentalFeatureSettings.isEnabled(primOp.experimentalFeature)) - { - auto primOpAdjusted = primOp; - primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity); - addPrimOp(std::move(primOpAdjusted)); - } + for (auto & primOp : RegisterPrimOp::primOps()) + if (experimentalFeatureSettings.isEnabled(primOp.experimentalFeature)) { + auto primOpAdjusted = primOp; + primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity); + addPrimOp(std::move(primOpAdjusted)); + } for (auto & primOp : evalSettings.extraPrimOps) { auto primOpAdjusted = primOp; From e7c0906521f19b645b2972c38ce94fe32da84c8c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 May 2025 08:28:12 +0200 Subject: [PATCH 307/396] Simplify RegisterCommand --- src/libcmd/command.cc | 4 +--- src/libcmd/include/nix/cmd/command.hh | 11 +++++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index ffb745742..f7aeb739b 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -14,12 +14,10 @@ namespace nix { -RegisterCommand::Commands * RegisterCommand::commands = nullptr; - nix::Commands RegisterCommand::getCommandsFor(const std::vector & prefix) { nix::Commands res; - for (auto & [name, command] : *RegisterCommand::commands) + for (auto & [name, command] : RegisterCommand::commands()) if (name.size() == prefix.size() + 1) { bool equal = true; for (size_t i = 0; i < prefix.size(); ++i) diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index 45739fdb2..9139594ac 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -285,13 +285,16 @@ struct StorePathCommand : public StorePathsCommand struct RegisterCommand { typedef std::map, std::function()>> Commands; - static Commands * commands; + + static Commands & commands() + { + static Commands commands; + return commands; + } RegisterCommand(std::vector && name, std::function()> command) { - if (!commands) - commands = new Commands; - commands->emplace(name, command); + commands().emplace(name, command); } static nix::Commands getCommandsFor(const std::vector & prefix); From 93844a5998348c7fa3a0ec146afd59de67d72781 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 May 2025 08:35:59 +0200 Subject: [PATCH 308/396] Simplify registerInputScheme() --- src/libfetchers/fetchers.cc | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 3ae45dcf8..e0161dee8 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -12,24 +12,26 @@ namespace nix::fetchers { using InputSchemeMap = std::map>; -std::unique_ptr inputSchemes = nullptr; +static InputSchemeMap & inputSchemes() +{ + static InputSchemeMap inputSchemeMap; + return inputSchemeMap; +} void registerInputScheme(std::shared_ptr && inputScheme) { - if (!inputSchemes) - inputSchemes = std::make_unique(); auto schemeName = inputScheme->schemeName(); - if (inputSchemes->count(schemeName) > 0) + if (!inputSchemes().emplace(schemeName, std::move(inputScheme)).second) throw Error("Input scheme with name %s already registered", schemeName); - inputSchemes->insert_or_assign(schemeName, std::move(inputScheme)); } -nlohmann::json dumpRegisterInputSchemeInfo() { +nlohmann::json dumpRegisterInputSchemeInfo() +{ using nlohmann::json; auto res = json::object(); - for (auto & [name, scheme] : *inputSchemes) { + for (auto & [name, scheme] : inputSchemes()) { auto & r = res[name] = json::object(); r["allowedAttrs"] = scheme->allowedAttrs(); } @@ -57,7 +59,7 @@ Input Input::fromURL( const Settings & settings, const ParsedURL & url, bool requireTree) { - for (auto & [_, inputScheme] : *inputSchemes) { + for (auto & [_, inputScheme] : inputSchemes()) { auto res = inputScheme->inputFromURL(settings, url, requireTree); if (res) { experimentalFeatureSettings.require(inputScheme->experimentalFeature()); @@ -91,8 +93,8 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs) }; std::shared_ptr inputScheme = ({ - auto i = inputSchemes->find(schemeName); - i == inputSchemes->end() ? nullptr : i->second; + auto i = get(inputSchemes(), schemeName); + i ? *i : nullptr; }); if (!inputScheme) return raw(); From f59ccb468e186877ac08f76facc428d1e24ede06 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 May 2025 08:41:23 +0200 Subject: [PATCH 309/396] Simplify Implementations registration --- src/libstore/include/nix/store/store-api.hh | 9 ++++++--- src/libstore/store-api.cc | 4 +--- src/nix/main.cc | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index d5e3d5214..678c00d8d 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -902,12 +902,15 @@ struct StoreFactory struct Implementations { - static std::vector * registered; + static std::vector & registered() + { + static std::vector registered; + return registered; + } template static void add() { - if (!registered) registered = new std::vector(); StoreFactory factory{ .uriSchemes = TConfig::uriSchemes(), .create = @@ -919,7 +922,7 @@ struct Implementations -> std::shared_ptr { return std::make_shared(StringMap({})); }) }; - registered->push_back(factory); + registered().push_back(factory); } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e9e982e61..49bbaecd7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1355,7 +1355,7 @@ ref openStore(StoreReference && storeURI) return std::make_shared(params); }, [&](const StoreReference::Specified & g) { - for (const auto & implem : *Implementations::registered) + for (const auto & implem : Implementations::registered()) if (implem.uriSchemes.count(g.scheme)) return implem.create(g.scheme, g.authority, params); @@ -1399,6 +1399,4 @@ std::list> getDefaultSubstituters() return stores; } -std::vector * Implementations::registered = 0; - } diff --git a/src/nix/main.cc b/src/nix/main.cc index eff2634e2..351fcd1b6 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -193,7 +193,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs res["args"] = toJSON(); auto stores = nlohmann::json::object(); - for (auto & implem : *Implementations::registered) { + for (auto & implem : Implementations::registered()) { auto storeConfig = implem.getConfig(); auto storeName = storeConfig->name(); auto & j = stores[storeName]; From 47989a21244f0e5f704a2e81ed507ed798d2b254 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 May 2025 08:45:50 +0200 Subject: [PATCH 310/396] Simplify ConfigRegistrations --- src/libutil/config-global.cc | 16 ++++++---------- src/libutil/include/nix/util/config-global.hh | 7 ++++++- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/libutil/config-global.cc b/src/libutil/config-global.cc index 10d176c51..94d715443 100644 --- a/src/libutil/config-global.cc +++ b/src/libutil/config-global.cc @@ -6,7 +6,7 @@ namespace nix { bool GlobalConfig::set(const std::string & name, const std::string & value) { - for (auto & config : *configRegistrations) + for (auto & config : configRegistrations()) if (config->set(name, value)) return true; @@ -17,20 +17,20 @@ bool GlobalConfig::set(const std::string & name, const std::string & value) void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) { - for (auto & config : *configRegistrations) + for (auto & config : configRegistrations()) config->getSettings(res, overriddenOnly); } void GlobalConfig::resetOverridden() { - for (auto & config : *configRegistrations) + for (auto & config : configRegistrations()) config->resetOverridden(); } nlohmann::json GlobalConfig::toJSON() { auto res = nlohmann::json::object(); - for (const auto & config : *configRegistrations) + for (const auto & config : configRegistrations()) res.update(config->toJSON()); return res; } @@ -47,19 +47,15 @@ std::string GlobalConfig::toKeyValue() void GlobalConfig::convertToArgs(Args & args, const std::string & category) { - for (auto & config : *configRegistrations) + for (auto & config : configRegistrations()) config->convertToArgs(args, category); } GlobalConfig globalConfig; -GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; - GlobalConfig::Register::Register(Config * config) { - if (!configRegistrations) - configRegistrations = new ConfigRegistrations; - configRegistrations->emplace_back(config); + configRegistrations().emplace_back(config); } ExperimentalFeatureSettings experimentalFeatureSettings; diff --git a/src/libutil/include/nix/util/config-global.hh b/src/libutil/include/nix/util/config-global.hh index b47ee0ad1..44f89e06d 100644 --- a/src/libutil/include/nix/util/config-global.hh +++ b/src/libutil/include/nix/util/config-global.hh @@ -8,7 +8,12 @@ namespace nix { struct GlobalConfig : public AbstractConfig { typedef std::vector ConfigRegistrations; - static ConfigRegistrations * configRegistrations; + + static ConfigRegistrations & configRegistrations() + { + static ConfigRegistrations configRegistrations; + return configRegistrations; + } bool set(const std::string & name, const std::string & value) override; From 060c34b6647af007b5f97b943bc3ed97ee06afce Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 May 2025 08:50:00 +0200 Subject: [PATCH 311/396] Attempt to fix macOS build --- src/libstore/include/nix/store/store-api.hh | 6 +----- src/libstore/store-api.cc | 6 ++++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 678c00d8d..fa1873502 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -902,11 +902,7 @@ struct StoreFactory struct Implementations { - static std::vector & registered() - { - static std::vector registered; - return registered; - } + static std::vector & registered(); template static void add() diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 49bbaecd7..7e5ce6e2e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1399,4 +1399,10 @@ std::list> getDefaultSubstituters() return stores; } +std::vector & Implementations::registered() +{ + static std::vector registered; + return registered; +} + } From 1594d4b8799cbdaba1d1eeffa52c0ac70c4bf0b8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 6 May 2025 15:02:10 -0400 Subject: [PATCH 312/396] Fix windows warning --- src/libutil/windows/include/nix/util/signals-impl.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/windows/include/nix/util/signals-impl.hh b/src/libutil/windows/include/nix/util/signals-impl.hh index f716ffd1a..55606debd 100644 --- a/src/libutil/windows/include/nix/util/signals-impl.hh +++ b/src/libutil/windows/include/nix/util/signals-impl.hh @@ -25,6 +25,7 @@ inline void setInterruptThrown() static inline bool isInterrupted() { /* Do nothing for now */ + return false; } inline void checkInterrupt() From 6d0f174cd9571c49ede89638b8bd8b292f9fc746 Mon Sep 17 00:00:00 2001 From: Vincent Breitmoser Date: Thu, 8 May 2025 09:56:14 +0200 Subject: [PATCH 313/396] Reduce maxLayers to 70 in docker build The nixos/nix docker image is built using `buildLayeredImage`, which spreads the nix store over a configured number of layers. This number was set to create an image with 100 layers. Because there is a limit of (typically) 127 layers in AUFS, this only left 27 layers to build on top. At the same time, nearly half of the created layers were only <100kb in size, many even <10kb, negating the intended advantage in cachability. This commit moves the tradeoff a bit by reducing the number of layers to 70. Layer sizes for the 2.28.3 nixos/nix image: https://hub.docker.com/layers/nixos/nix/2.28.3/images/sha256-d078d7153763895fce17c5fbbdeb86fcfcac414ca0ba875d413c1df57be19931 --- docker.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker.nix b/docker.nix index d52c317d6..a92d478d9 100644 --- a/docker.nix +++ b/docker.nix @@ -7,7 +7,7 @@ channelName ? "nixpkgs", channelURL ? "https://nixos.org/channels/nixpkgs-unstable", extraPkgs ? [ ], - maxLayers ? 100, + maxLayers ? 70, nixConf ? { }, flake-registry ? null, uid ? 0, From d00682beb2fc0ba62fb87752ba429cc0f4c4345e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 May 2025 19:01:34 +0200 Subject: [PATCH 314/396] Backward compatibility hack for dealing with `dir` in URL-style flakerefs --- src/libflake/flake.cc | 2 +- src/libflake/flakeref.cc | 49 ++++++++++++++++++ src/libflake/include/nix/flake/flakeref.hh | 6 +++ tests/functional/flakes/meson.build | 1 + tests/functional/flakes/old-lockfiles.sh | 60 ++++++++++++++++++++++ 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 tests/functional/flakes/old-lockfiles.sh diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 516b38131..987c9f610 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -595,7 +595,7 @@ LockedFlake lockFlake( oldLock = *oldLock3; if (oldLock - && oldLock->originalRef == *input.ref + && oldLock->originalRef.canonicalize() == input.ref->canonicalize() && oldLock->parentInputAttrPath == overridenParentPath && !hasCliOverride) { diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index a8b139d65..12bddf578 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -289,6 +289,55 @@ std::pair, FlakeRef> FlakeRef::lazyFetch(ref store) c return {accessor, FlakeRef(std::move(lockedInput), subdir)}; } +FlakeRef FlakeRef::canonicalize() const +{ + auto flakeRef(*this); + + /* Backward compatibility hack: In old versions of Nix, if you had + a flake input like + + inputs.foo.url = "git+https://foo/bar?dir=subdir"; + + it would result in a lock file entry like + + "original": { + "dir": "subdir", + "type": "git", + "url": "https://foo/bar?dir=subdir" + } + + New versions of Nix remove `?dir=subdir` from the `url` field, + since the subdirectory is intended for `FlakeRef`, not the + fetcher (and specifically the remote server), that is, the + flakeref is parsed into + + "original": { + "dir": "subdir", + "type": "git", + "url": "https://foo/bar" + } + + However, this causes new versions of Nix to consider the lock + file entry to be stale since the `original` ref no longer + matches exactly. + + For this reason, we canonicalise the `original` ref by + filtering the `dir` query parameter from the URL. */ + if (auto url = fetchers::maybeGetStrAttr(flakeRef.input.attrs, "url")) { + try { + auto parsed = parseURL(*url); + if (auto dir2 = get(parsed.query, "dir")) { + if (flakeRef.subdir != "" && flakeRef.subdir == *dir2) + parsed.query.erase("dir"); + } + flakeRef.input.attrs.insert_or_assign("url", parsed.to_string()); + } catch (BadURL &) { + } + } + + return flakeRef; +} + std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( const fetchers::Settings & fetchSettings, const std::string & url, diff --git a/src/libflake/include/nix/flake/flakeref.hh b/src/libflake/include/nix/flake/flakeref.hh index 8c15f9d95..6184d2363 100644 --- a/src/libflake/include/nix/flake/flakeref.hh +++ b/src/libflake/include/nix/flake/flakeref.hh @@ -72,6 +72,12 @@ struct FlakeRef const fetchers::Attrs & attrs); std::pair, FlakeRef> lazyFetch(ref store) const; + + /** + * Canonicalize a flakeref for the purpose of comparing "old" and + * "new" `original` fields in lock files. + */ + FlakeRef canonicalize() const; }; std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build index 368c43876..213c388a6 100644 --- a/tests/functional/flakes/meson.build +++ b/tests/functional/flakes/meson.build @@ -32,6 +32,7 @@ suites += { 'symlink-paths.sh', 'debugger.sh', 'source-paths.sh', + 'old-lockfiles.sh', ], 'workdir': meson.current_source_dir(), } diff --git a/tests/functional/flakes/old-lockfiles.sh b/tests/functional/flakes/old-lockfiles.sh new file mode 100644 index 000000000..fd36abdcc --- /dev/null +++ b/tests/functional/flakes/old-lockfiles.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +source ./common.sh + +requireGit + +repo="$TEST_ROOT/repo" + +createGitRepo "$repo" + +cat > "$repo/flake.nix" < "$repo/flake.lock" < Date: Fri, 9 May 2025 16:44:48 +0200 Subject: [PATCH 315/396] nix flake prefetch: Remove __final This is currently an internal attribute, not intended to be shown to users. Fixes #13150. --- src/nix/flake.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 351b2caaf..80ff111f1 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1497,6 +1497,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON res["hash"] = hash.to_string(HashFormat::SRI, true); res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); + res["locked"].erase("__final"); // internal for now printJSON(res); } else { notice("Downloaded '%s' to '%s' (hash '%s').", From da953d6d39f830f7edd2e9c2505bc86af4fabda3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 May 2025 16:55:13 +0200 Subject: [PATCH 316/396] Add test --- tests/functional/tarball.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index 53807603c..6b09cf6a5 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -73,7 +73,8 @@ test_tarball .gz gzip # All entries in tree.tar.gz refer to the same file, and all have the same inode when unpacked by GNU tar. # We don't preserve the hard links, because that's an optimization we think is not worth the complexity, # so we only make sure that the contents are copied correctly. -nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" --out-link "$TEST_ROOT/result" +json=$(nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" --out-link "$TEST_ROOT/result") +[[ $json =~ ^'{"hash":"sha256-'.*'","locked":{"lastModified":'.*',"narHash":"sha256-'.*'","type":"tarball","url":"file:///'.*'/tree.tar.gz"},"original":{"type":"tarball","url":"file:///'.*'/tree.tar.gz"},"storePath":"'.*'/store/'.*'-source"}'$ ]] [[ $(cat "$TEST_ROOT/result/a/b/foo") = bar ]] [[ $(cat "$TEST_ROOT/result/a/b/xyzzy") = bar ]] [[ $(cat "$TEST_ROOT/result/a/yyy") = bar ]] From 3cc16d13f0223651fc99783de6ea4e1c2aa7c091 Mon Sep 17 00:00:00 2001 From: Andrey Butirsky Date: Sat, 10 May 2025 18:13:58 +0300 Subject: [PATCH 317/396] index.md: add warning for installing as root warning: installing Nix as root is not supported by this script! performing a single-user installation of Nix... copying Nix to /nix/store....................................................... ...... warning: the group 'nixbld' specified in 'build-users-group' does not exist warning: the group 'nixbld' specified in 'build-users-group' does not exist installing 'nix-2.28.3' error: the group 'nixbld' specified in 'build-users-group' does not exist /tmp/nix-binary-tarball-unpack.2j3lCU0A89/unpack/nix-2.28.3-x86_64-linux/install: unable to install Nix into your default profile --- doc/manual/source/installation/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/source/installation/index.md b/doc/manual/source/installation/index.md index 48725c1ba..3c09f1031 100644 --- a/doc/manual/source/installation/index.md +++ b/doc/manual/source/installation/index.md @@ -30,6 +30,8 @@ $ curl -L https://nixos.org/nix/install | sh -s -- --daemon > Single-user is not supported on Mac. +> `warning: installing Nix as root is not supported by this script!` + This installation has less requirements than the multi-user install, however it cannot offer equivalent sharing, isolation, or security. From 5a8423720989ba845696d567713c04fdb291c35d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 May 2025 00:32:41 +0200 Subject: [PATCH 318/396] Improve build failure error messages They're now laid out in a more readable way, and they shows the output paths (if known). --- src/libstore/build/derivation-goal.cc | 35 ++++++++++++++++--- .../store/build/derivation-building-misc.hh | 6 ++++ src/libstore/unix/build/derivation-builder.cc | 6 +++- tests/functional/build.sh | 15 ++++++-- 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 344d1c7ea..d1a2dfd41 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -354,6 +354,22 @@ struct value_comparison }; +std::string showKnownOutputs(Store & store, const Derivation & drv) +{ + std::string msg; + StorePathSet expectedOutputPaths; + for (auto & i : drv.outputsAndOptPaths(store)) + if (i.second.second) + expectedOutputPaths.insert(*i.second.second); + if (!expectedOutputPaths.empty()) { + msg += "\nOutput paths:"; + for (auto & p : expectedOutputPaths) + msg += fmt("\n %s", Magenta(store.printStorePath(p))); + } + return msg; +} + + /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ Goal::Co DerivationGoal::gaveUpOnSubstitution() @@ -430,9 +446,14 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - co_return done(BuildResult::DependencyFailed, {}, Error( - "%s dependencies of derivation '%s' failed to build", - nrFailed, worker.store.printStorePath(drvPath))); + auto msg = fmt( + "Cannot build '%s'.\n" + "Reason: " ANSI_RED "%d %s failed" ANSI_NORMAL ".", + Magenta(worker.store.printStorePath(drvPath)), + nrFailed, + nrFailed == 1 ? "dependency" : "dependencies"); + msg += showKnownOutputs(worker.store, *drv); + co_return done(BuildResult::DependencyFailed, {}, Error(msg)); } if (retrySubstitution == RetrySubstitution::YesNeed) { @@ -1033,7 +1054,7 @@ void runPostBuildHook( void DerivationGoal::appendLogTailErrorMsg(std::string & msg) { if (!logger->isVerbose() && !logTail.empty()) { - msg += fmt(";\nlast %d log lines:\n", logTail.size()); + msg += fmt("\nLast %d log lines:\n", logTail.size()); for (auto & line : logTail) { msg += "> "; msg += line; @@ -1090,10 +1111,14 @@ Goal::Co DerivationGoal::hookDone() /* Check the exit status. */ if (!statusOk(status)) { - auto msg = fmt("builder for '%s' %s", + auto msg = fmt( + "Cannot build '%s'.\n" + "Reason: " ANSI_RED "builder %s" ANSI_NORMAL ".", Magenta(worker.store.printStorePath(drvPath)), statusToString(status)); + msg += showKnownOutputs(worker.store, *drv); + appendLogTailErrorMsg(msg); outputLocks.unlock(); diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh index caf94844d..915d891d7 100644 --- a/src/libstore/include/nix/store/build/derivation-building-misc.hh +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -9,6 +9,7 @@ namespace nix { class Store; +struct Derivation; /** * Unless we are repairing, we don't both to test validity and just assume it, @@ -49,4 +50,9 @@ struct InitialOutput void runPostBuildHook(Store & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths); +/** + * Format the known outputs of a derivation for use in error messages. + */ +std::string showKnownOutputs(Store & store, const Derivation & drv); + } diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 4bde9750d..59c0be91c 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -585,10 +585,14 @@ std::variant, SingleDrvOutputs> Derivation diskFull |= cleanupDecideWhetherDiskFull(); - auto msg = fmt("builder for '%s' %s", + auto msg = fmt( + "Cannot build '%s'.\n" + "Reason: " ANSI_RED "builder %s" ANSI_NORMAL ".", Magenta(store.printStorePath(drvPath)), statusToString(status)); + msg += showKnownOutputs(store, drv); + miscMethods->appendLogTailErrorMsg(msg); if (diskFull) diff --git a/tests/functional/build.sh b/tests/functional/build.sh index 3f65a7c2c..0a19ff7da 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -179,12 +179,23 @@ test "$(<<<"$out" grep -cE '^error:')" = 4 out="$(nix build -f fod-failing.nix -L x4 2>&1)" && status=0 || status=$? test "$status" = 1 test "$(<<<"$out" grep -cE '^error:')" = 2 -<<<"$out" grepQuiet -E "error: 1 dependencies of derivation '.*-x4\\.drv' failed to build" + +if isDaemonNewer "2.29pre"; then + <<<"$out" grepQuiet -E "error: Cannot build '.*-x4\\.drv'" + <<<"$out" grepQuiet -E "Reason: 1 dependency failed." +else + <<<"$out" grepQuiet -E "error: 1 dependencies of derivation '.*-x4\\.drv' failed to build" +fi <<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x2\\.drv'" out="$(nix build -f fod-failing.nix -L x4 --keep-going 2>&1)" && status=0 || status=$? test "$status" = 1 test "$(<<<"$out" grep -cE '^error:')" = 3 -<<<"$out" grepQuiet -E "error: 2 dependencies of derivation '.*-x4\\.drv' failed to build" +if isDaemonNewer "2.29pre"; then + <<<"$out" grepQuiet -E "error: Cannot build '.*-x4\\.drv'" + <<<"$out" grepQuiet -E "Reason: 2 dependencies failed." +else + <<<"$out" grepQuiet -E "error: 2 dependencies of derivation '.*-x4\\.drv' failed to build" +fi <<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'" <<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'" From bdb3f613dd0b01be076f219461223e34c20356fe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 May 2025 22:08:17 +0200 Subject: [PATCH 319/396] Improve 'cannot read file from tarball' error It now says e.g. error: cannot read file from tarball: Truncated tar archive detected while reading data --- src/libutil/tarfile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 75373e3eb..0e317457e 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -215,7 +215,7 @@ time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & std::vector buf(128 * 1024); auto n = archive_read_data(archive.archive, buf.data(), buf.size()); if (n < 0) - throw Error("cannot read file '%s' from tarball", path); + checkLibArchive(archive.archive, n, "cannot read file from tarball: %s"); if (n == 0) break; crf(std::string_view{ From 824e0d51fe233ad93109852c3ad84349e518e12c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 May 2025 17:54:41 +0200 Subject: [PATCH 320/396] Test lock file contents more precisely --- tests/functional/flakes/flakes.sh | 1 + tests/functional/flakes/relative-paths.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 72fe79838..e8b051198 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -160,6 +160,7 @@ expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file [[ -e "$flake2Dir/flake.lock" ]] [[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] +[[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] # Rerunning the build should not change the lockfile. nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 4648ba98c..9d31da0ad 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -69,6 +69,8 @@ git -C "$rootFlake" add flake.nix sub2/flake.nix git -C "$rootFlake" add sub2/flake.lock [[ $(nix eval "$subflake2#y") = 15 ]] +[[ $(jq --indent 0 . < "$subflake2/flake.lock") =~ ^'{"nodes":{"root":{"inputs":{"root":"root_2","sub1":"sub1"}},"root_2":{"inputs":{"sub0":"sub0"},"locked":{"path":"..","type":"path"},"original":{"path":"..","type":"path"},"parent":[]},"root_3":{"inputs":{"sub0":"sub0_2"},"locked":{"path":"../","type":"path"},"original":{"path":"../","type":"path"},"parent":["sub1"]},"sub0":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["root"]},"sub0_2":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["sub1","root"]},"sub1":{"inputs":{"root":"root_3"},"locked":{"path":"../sub1","type":"path"},"original":{"path":"../sub1","type":"path"},"parent":[]}},"root":"root","version":7}'$ ]] + # Make sure there are no content locks for relative path flakes. (! grep "$TEST_ROOT" "$subflake2/flake.lock") if ! isTestOnNixOS; then From 7628155d2be3a7b95c79c989abdb031b4741f063 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 13 May 2025 10:50:46 +0000 Subject: [PATCH 321/396] libutil/tarfile: Create the scratch `std::vector` only once I can't find a good way to benchmark in isolation from the git cache, but common sense dictates that creating (and destroying) a 131KiB std::vector for each regular file from the archive imposes quite a significant overhead regardless of the IO bound git cache. AFAICT there is no reason to keep a copy of the data since it always gets fed into the sink and there are no coroutines/threads in sight. --- src/libutil/tarfile.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 75373e3eb..66a7fef02 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -178,6 +178,10 @@ time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & { time_t lastModified = 0; + /* Only allocate the buffer once. Use the heap because 131 KiB is a bit too + much for the stack. */ + std::vector buf(128 * 1024); + for (;;) { // FIXME: merge with extract_archive struct archive_entry * entry; @@ -212,7 +216,6 @@ time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & crf.isExecutable(); while (true) { - std::vector buf(128 * 1024); auto n = archive_read_data(archive.archive, buf.data(), buf.size()); if (n < 0) throw Error("cannot read file '%s' from tarball", path); From 934918ba1667904c6b746e1b57bbc482c615938b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 7 May 2025 17:07:23 -0400 Subject: [PATCH 322/396] Stores no longer inherit from their configs Fix #10766 See that ticket for details. Progress (I hope!) towards #11139. Co-Authored-By: Sergei Zimmerman --- doc/manual/generate-store-info.nix | 1 + src/build-remote/build-remote.cc | 4 +- src/libcmd/include/nix/cmd/command.hh | 2 +- src/libstore-c/nix_api_store.cc | 2 +- src/libstore/binary-cache-store.cc | 43 ++--- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/substitution-goal.cc | 4 +- src/libstore/common-ssh-store-config.cc | 2 +- src/libstore/derivation-options.cc | 2 +- src/libstore/dummy-store.cc | 32 ++-- src/libstore/filetransfer.cc | 2 +- src/libstore/gc.cc | 24 +-- src/libstore/http-binary-cache-store.cc | 58 ++++--- .../include/nix/store/binary-cache-store.hh | 15 +- .../nix/store/common-ssh-store-config.hh | 2 +- .../nix/store/http-binary-cache-store.hh | 22 ++- .../include/nix/store/legacy-ssh-store.hh | 40 ++--- .../nix/store/local-binary-cache-store.hh | 14 +- .../include/nix/store/local-fs-store.hh | 25 +-- .../include/nix/store/local-overlay-store.hh | 40 +++-- src/libstore/include/nix/store/local-store.hh | 26 +-- src/libstore/include/nix/store/profiles.hh | 2 +- .../include/nix/store/remote-fs-accessor.hh | 2 +- .../include/nix/store/remote-store.hh | 8 +- .../include/nix/store/restricted-store.hh | 2 +- .../nix/store/s3-binary-cache-store.hh | 16 +- src/libstore/include/nix/store/ssh-store.hh | 22 +-- src/libstore/include/nix/store/store-api.hh | 163 ++++++++++++------ .../include/nix/store/store-dir-config.hh | 62 +++++-- .../include/nix/store/uds-remote-store.hh | 40 ++--- src/libstore/legacy-ssh-store.cc | 53 +++--- src/libstore/local-binary-cache-store.cc | 60 ++++--- src/libstore/local-fs-store.cc | 11 +- src/libstore/local-overlay-store.cc | 53 +++--- src/libstore/local-store.cc | 90 +++++----- src/libstore/meson.build | 1 + src/libstore/optimise-store.cc | 8 +- src/libstore/path.cc | 12 +- src/libstore/remote-store.cc | 12 +- src/libstore/restricted-store.cc | 29 ++-- src/libstore/s3-binary-cache-store.cc | 78 +++++---- src/libstore/ssh-store.cc | 86 +++++---- src/libstore/store-api.cc | 63 +++---- src/libstore/store-dir-config.cc | 13 ++ src/libstore/uds-remote-store.cc | 43 ++--- src/libstore/unix/build/derivation-builder.cc | 32 ++-- src/nix/main.cc | 11 +- src/nix/unix/daemon.cc | 2 +- 48 files changed, 743 insertions(+), 593 deletions(-) create mode 100644 src/libstore/store-dir-config.cc diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index e8b7377da..e66611aff 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -33,6 +33,7 @@ let { settings, doc, + uri-schemes, experimentalFeature, }: let diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index d967548f5..7329bee5c 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -44,7 +44,7 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) static bool allSupportedLocally(Store & store, const StringSet& requiredFeatures) { for (auto & feature : requiredFeatures) - if (!store.systemFeatures.get().count(feature)) return false; + if (!store.config.systemFeatures.get().count(feature)) return false; return true; } @@ -85,7 +85,7 @@ static int main_build_remote(int argc, char * * argv) that gets cleared on reboot, but it wouldn't work on macOS. */ auto currentLoadName = "/current-load"; if (auto localStore = store.dynamic_pointer_cast()) - currentLoad = std::string { localStore->stateDir } + currentLoadName; + currentLoad = std::string { localStore->config.stateDir } + currentLoadName; else currentLoad = settings.nixStateDir + currentLoadName; diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index 9139594ac..cb436cfdb 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -18,7 +18,7 @@ extern char ** savedArgv; class EvalState; struct Pos; class Store; -class LocalFSStore; +struct LocalFSStore; static constexpr Command::Category catHelp = -1; static constexpr Command::Category catSecondary = 100; diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 92aed9187..d49e7722b 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -42,7 +42,7 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char *** if (!params) return new Store{nix::openStore(uri_str)}; - nix::Store::Params params_map; + nix::Store::Config::Params params_map; for (size_t i = 0; params[i] != nullptr; i++) { params_map[params[i][0]] = params[i][1]; } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index bdc281044..4df9651f0 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -24,16 +24,15 @@ namespace nix { -BinaryCacheStore::BinaryCacheStore(const Params & params) - : BinaryCacheStoreConfig(params) - , Store(params) +BinaryCacheStore::BinaryCacheStore(Config & config) + : config{config} { - if (secretKeyFile != "") + if (config.secretKeyFile != "") signers.push_back(std::make_unique( - SecretKey { readFile(secretKeyFile) })); + SecretKey { readFile(config.secretKeyFile) })); - if (secretKeyFiles != "") { - std::stringstream ss(secretKeyFiles); + if (config.secretKeyFiles != "") { + std::stringstream ss(config.secretKeyFiles); Path keyPath; while (std::getline(ss, keyPath, ',')) { signers.push_back(std::make_unique( @@ -62,9 +61,9 @@ void BinaryCacheStore::init() throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", getUri(), value, storeDir); } else if (name == "WantMassQuery") { - wantMassQuery.setDefault(value == "1"); + config.wantMassQuery.setDefault(value == "1"); } else if (name == "Priority") { - priority.setDefault(std::stoi(value)); + config.priority.setDefault(std::stoi(value)); } } } @@ -156,7 +155,11 @@ ref BinaryCacheStore::addToStoreCommon( { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed { fileSink, fileHashSink }; - auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel); + auto compressionSink = makeCompressionSink( + config.compression, + teeSinkCompressed, + config.parallelCompression, + config.compressionLevel); TeeSink teeSinkUncompressed { *compressionSink, narHashSink }; TeeSource teeSource { narSource, teeSinkUncompressed }; narAccessor = makeNarAccessor(teeSource); @@ -168,17 +171,17 @@ ref BinaryCacheStore::addToStoreCommon( auto info = mkInfo(narHashSink.finish()); auto narInfo = make_ref(info); - narInfo->compression = compression; + narInfo->compression = config.compression; auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "zstd" ? ".zst" : - compression == "lzip" ? ".lzip" : - compression == "lz4" ? ".lz4" : - compression == "br" ? ".br" : + + (config.compression == "xz" ? ".xz" : + config.compression == "bzip2" ? ".bz2" : + config.compression == "zstd" ? ".zst" : + config.compression == "lzip" ? ".lzip" : + config.compression == "lz4" ? ".lz4" : + config.compression == "br" ? ".br" : ""); auto duration = std::chrono::duration_cast(now2 - now1).count(); @@ -200,7 +203,7 @@ ref BinaryCacheStore::addToStoreCommon( /* Optionally write a JSON file containing a listing of the contents of the NAR. */ - if (writeNARListing) { + if (config.writeNARListing) { nlohmann::json j = { {"version", 1}, {"root", listNar(ref(narAccessor), CanonPath::root, true)}, @@ -212,7 +215,7 @@ ref BinaryCacheStore::addToStoreCommon( /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/' that specify the NAR file and member containing the debug info. */ - if (writeDebugInfo) { + if (config.writeDebugInfo) { CanonPath buildIdDir("lib/debug/.build-id"); @@ -524,7 +527,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) { ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), requireValidPath, localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, config.localNarCache); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d1a2dfd41..4d4371128 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1284,7 +1284,7 @@ Path DerivationGoal::openLogFile() /* Create a log file. */ Path logDir; if (auto localStore = dynamic_cast(&worker.store)) - logDir = localStore->logDir; + logDir = localStore->config->logDir; else logDir = settings.nixLogDir; Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2)); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index b1313808e..e24dd7e64 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -121,7 +121,7 @@ Goal::Co PathSubstitutionGoal::init() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) + if (!sub->config.isTrusted && worker.store.pathInfoIsUntrusted(*info)) { warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); @@ -215,7 +215,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, PushActivity pact(act.id); copyStorePath(*sub, worker.store, - subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 7cfbc5f98..bcaa11a96 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -28,7 +28,7 @@ CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_ { } -SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) +SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const { return { host, diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 6625a6be9..e031f8447 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -265,7 +265,7 @@ bool DerivationOptions::canBuildLocally(Store & localStore, const BasicDerivatio return false; for (auto & feature : getRequiredSystemFeatures(drv)) - if (!localStore.systemFeatures.get().count(feature)) + if (!localStore.config.systemFeatures.get().count(feature)) return false; return true; diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 3e1d168a1..68786b31b 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -3,7 +3,7 @@ namespace nix { -struct DummyStoreConfig : virtual StoreConfig { +struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig { using StoreConfig::StoreConfig; DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) @@ -13,9 +13,9 @@ struct DummyStoreConfig : virtual StoreConfig { throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); } - const std::string name() override { return "Dummy Store"; } + static const std::string name() { return "Dummy Store"; } - std::string doc() override + static std::string doc() { return #include "dummy-store.md" @@ -25,23 +25,24 @@ struct DummyStoreConfig : virtual StoreConfig { static StringSet uriSchemes() { return {"dummy"}; } + + ref openStore() const override; }; -struct DummyStore : public virtual DummyStoreConfig, public virtual Store +struct DummyStore : virtual Store { - DummyStore(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , DummyStoreConfig(scheme, authority, params) - , Store(params) - { } + using Config = DummyStoreConfig; - DummyStore(const Params & params) - : DummyStore("dummy", "", params) + ref config; + + DummyStore(ref config) + : Store{*config} + , config(config) { } std::string getUri() override { - return *uriSchemes().begin(); + return *Config::uriSchemes().begin(); } void queryPathInfoUncached(const StorePath & path, @@ -88,6 +89,11 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store } }; -static RegisterStoreImplementation regDummyStore; +ref DummyStore::Config::openStore() const +{ + return make_ref(ref{shared_from_this()}); +} + +static RegisterStoreImplementation regDummyStore; } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index f77881cf7..8080fcfdd 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -771,7 +771,7 @@ struct curlFileTransfer : public FileTransfer } #if NIX_WITH_S3_SUPPORT - std::tuple parseS3Uri(std::string uri) + std::tuple parseS3Uri(std::string uri) { auto [path, params] = splitUriAndParams(uri); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f6a4124ff..8fad9661c 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -43,7 +43,7 @@ static std::string gcRootsDir = "gcroots"; void LocalStore::addIndirectRoot(const Path & path) { std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false); - Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); + Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", config->stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } @@ -82,7 +82,7 @@ void LocalStore::createTempRootsFile() void LocalStore::addTempRoot(const StorePath & path) { - if (readOnly) { + if (config->readOnly) { debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways."); return; } @@ -109,7 +109,7 @@ void LocalStore::addTempRoot(const StorePath & path) auto fdRootsSocket(_fdRootsSocket.lock()); if (!*fdRootsSocket) { - auto socketPath = stateDir.get() + gcSocketPath; + auto socketPath = config->stateDir.get() + gcSocketPath; debug("connecting to '%s'", socketPath); *fdRootsSocket = createUnixDomainSocket(); try { @@ -247,7 +247,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R else { target = absPath(target, dirOf(path)); if (!pathExists(target)) { - if (isInDir(path, std::filesystem::path{stateDir.get()} / gcRootsDir / "auto")) { + if (isInDir(path, std::filesystem::path{config->stateDir.get()} / gcRootsDir / "auto")) { printInfo("removing stale link from '%1%' to '%2%'", path, target); unlink(path.c_str()); } @@ -288,8 +288,8 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { /* Process direct roots in {gcroots,profiles}. */ - findRoots(stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots); - findRoots(stateDir + "/profiles", std::filesystem::file_type::unknown, roots); + findRoots(config->stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots); + findRoots(config->stateDir + "/profiles", std::filesystem::file_type::unknown, roots); /* Add additional roots returned by different platforms-specific heuristics. This is typically used to add running programs to @@ -498,7 +498,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) readFile(*p); /* Start the server for receiving new roots. */ - auto socketPath = stateDir.get() + gcSocketPath; + auto socketPath = config->stateDir.get() + gcSocketPath; createDirs(dirOf(socketPath)); auto fdServer = createUnixDomainSocket(socketPath, 0666); @@ -635,7 +635,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto deleteFromStore = [&](std::string_view baseName) { Path path = storeDir + "/" + std::string(baseName); - Path realPath = realStoreDir + "/" + std::string(baseName); + Path realPath = config->realStoreDir + "/" + std::string(baseName); /* There may be temp directories in the store that are still in use by another process. We need to be sure that we can acquire an @@ -804,8 +804,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printInfo("determining live/dead paths..."); try { - AutoCloseDir dir(opendir(realStoreDir.get().c_str())); - if (!dir) throw SysError("opening directory '%1%'", realStoreDir); + AutoCloseDir dir(opendir(config->realStoreDir.get().c_str())); + if (!dir) throw SysError("opening directory '%1%'", config->realStoreDir); /* Read the store and delete all paths that are invalid or unreachable. We don't use readDirectory() here so that @@ -907,8 +907,8 @@ void LocalStore::autoGC(bool sync) return std::stoll(readFile(*fakeFreeSpaceFile)); struct statvfs st; - if (statvfs(realStoreDir.get().c_str(), &st)) - throw SysError("getting filesystem info about '%s'", realStoreDir); + if (statvfs(config->realStoreDir.get().c_str(), &st)) + throw SysError("getting filesystem info about '%s'", config->realStoreDir); return (uint64_t) st.f_bavail * st.f_frsize; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 4c13d5c73..da8b9e18e 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -9,6 +9,15 @@ namespace nix { MakeError(UploadToHTTP, Error); +StringSet HttpBinaryCacheStoreConfig::uriSchemes() +{ + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = StringSet{"http", "https"}; + if (forceHttp) + ret.insert("file"); + return ret; +} + HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view _cacheUri, @@ -35,10 +44,9 @@ std::string HttpBinaryCacheStoreConfig::doc() } -class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore +class HttpBinaryCacheStore : + public virtual BinaryCacheStore { -private: - struct State { bool enabled = true; @@ -49,37 +57,37 @@ private: public: - HttpBinaryCacheStore( - std::string_view scheme, - PathView cacheUri, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , HttpBinaryCacheStoreConfig(scheme, cacheUri, params) - , Store(params) - , BinaryCacheStore(params) + using Config = HttpBinaryCacheStoreConfig; + + ref config; + + HttpBinaryCacheStore(ref config) + : Store{*config} + // TODO it will actually mutate the configuration + , BinaryCacheStore{*config} + , config{config} { diskCache = getNarInfoDiskCache(); } std::string getUri() override { - return cacheUri; + return config->cacheUri; } void init() override { // FIXME: do this lazily? - if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { - wantMassQuery.setDefault(cacheInfo->wantMassQuery); - priority.setDefault(cacheInfo->priority); + if (auto cacheInfo = diskCache->upToDateCacheExists(config->cacheUri)) { + config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); + config->priority.setDefault(cacheInfo->priority); } else { try { BinaryCacheStore::init(); } catch (UploadToHTTP &) { - throw Error("'%s' does not appear to be a binary cache", cacheUri); + throw Error("'%s' does not appear to be a binary cache", config->cacheUri); } - diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority); + diskCache->createCache(config->cacheUri, config->storeDir, config->wantMassQuery, config->priority); } } @@ -137,7 +145,7 @@ protected: try { getFileTransfer()->upload(req); } catch (FileTransferError & e) { - throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); + throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri, e.msg()); } } @@ -146,7 +154,7 @@ protected: return FileTransferRequest( hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://") ? path - : cacheUri + "/" + path); + : config->cacheUri + "/" + path); } @@ -221,6 +229,14 @@ protected: } }; -static RegisterStoreImplementation regHttpBinaryCacheStore; +ref HttpBinaryCacheStore::Config::openStore() const +{ + return make_ref(ref{ + // FIXME we shouldn't actually need a mutable config + std::const_pointer_cast(shared_from_this()) + }); +} + +static RegisterStoreImplementation regHttpBinaryCacheStore; } diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 445a10328..43f2cf690 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -54,10 +54,17 @@ struct BinaryCacheStoreConfig : virtual StoreConfig * @note subclasses must implement at least one of the two * virtual getFile() methods. */ -class BinaryCacheStore : public virtual BinaryCacheStoreConfig, - public virtual Store, - public virtual LogStore +struct BinaryCacheStore : + virtual Store, + virtual LogStore { + using Config = BinaryCacheStoreConfig; + + /** + * Intentionally mutable because some things we update due to the + * cache's own (remote side) settings. + */ + Config & config; private: std::vector> signers; @@ -69,7 +76,7 @@ protected: const std::string cacheInfoFile = "nix-cache-info"; - BinaryCacheStore(const Params & params); + BinaryCacheStore(Config &); public: diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index f82124c66..82a78f075 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -56,7 +56,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig */ SSHMaster createSSHMaster( bool useMaster, - Descriptor logFD = INVALID_DESCRIPTOR); + Descriptor logFD = INVALID_DESCRIPTOR) const; }; } diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index 592def4f8..66ec5f8d2 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -2,29 +2,27 @@ namespace nix { -struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, + virtual Store::Config, + BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view _cacheUri, const Params & params); + HttpBinaryCacheStoreConfig( + std::string_view scheme, std::string_view cacheUri, const Store::Config::Params & params); Path cacheUri; - const std::string name() override + static const std::string name() { return "HTTP Binary Cache Store"; } - static StringSet uriSchemes() - { - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - auto ret = StringSet({"http", "https"}); - if (forceHttp) - ret.insert("file"); - return ret; - } + static StringSet uriSchemes(); - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; } diff --git a/src/libstore/include/nix/store/legacy-ssh-store.hh b/src/libstore/include/nix/store/legacy-ssh-store.hh index 7859abbc9..65f29d649 100644 --- a/src/libstore/include/nix/store/legacy-ssh-store.hh +++ b/src/libstore/include/nix/store/legacy-ssh-store.hh @@ -10,7 +10,7 @@ namespace nix { -struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig +struct LegacySSHStoreConfig : std::enable_shared_from_this, virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; @@ -19,6 +19,15 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig std::string_view authority, const Params & params); +#ifndef _WIN32 + // Hack for getting remote build log output. + // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in + // the documentation + const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; +#else + Descriptor logFD = INVALID_DESCRIPTOR; +#endif + const Setting remoteProgram{this, {"nix-store"}, "remote-program", "Path to the `nix-store` executable on the remote machine."}; @@ -35,23 +44,20 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig */ std::optional connPipeSize; - const std::string name() override { return "SSH Store"; } + static const std::string name() { return "SSH Store"; } static StringSet uriSchemes() { return {"ssh"}; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; -struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +struct LegacySSHStore : public virtual Store { -#ifndef _WIN32 - // Hack for getting remote build log output. - // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; -#else - Descriptor logFD = INVALID_DESCRIPTOR; -#endif + using Config = LegacySSHStoreConfig; + + ref config; struct Connection; @@ -59,10 +65,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor SSHMaster master; - LegacySSHStore( - std::string_view scheme, - std::string_view host, - const Params & params); + LegacySSHStore(ref); ref openConnection(); @@ -187,10 +190,7 @@ public: * The legacy ssh protocol doesn't support checking for trusted-user. * Try using ssh-ng:// instead if you want to know. */ - std::optional isTrustedClient() override - { - return std::nullopt; - } + std::optional isTrustedClient() override; void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index a46ce07d2..780eaf480 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -2,22 +2,30 @@ namespace nix { -struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this, + virtual Store::Config, + BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + /** + * @param binaryCacheDir `file://` is a short-hand for `file:///` + * for now. + */ LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); Path binaryCacheDir; - const std::string name() override + static const std::string name() { return "Local Binary Cache Store"; } static StringSet uriSchemes(); - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; } diff --git a/src/libstore/include/nix/store/local-fs-store.hh b/src/libstore/include/nix/store/local-fs-store.hh index 6d5afcb08..f9421b7fe 100644 --- a/src/libstore/include/nix/store/local-fs-store.hh +++ b/src/libstore/include/nix/store/local-fs-store.hh @@ -20,36 +20,39 @@ struct LocalFSStoreConfig : virtual StoreConfig */ LocalFSStoreConfig(PathView path, const Params & params); - const OptionalPathSetting rootDir{this, std::nullopt, + OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; - const PathSetting stateDir{this, + PathSetting stateDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, "state", "Directory where Nix will store state."}; - const PathSetting logDir{this, + PathSetting logDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store log files."}; - const PathSetting realStoreDir{this, + PathSetting realStoreDir{this, rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; }; -class LocalFSStore : public virtual LocalFSStoreConfig, - public virtual Store, - public virtual GcStore, - public virtual LogStore +struct LocalFSStore : + virtual Store, + virtual GcStore, + virtual LogStore { -public: + using Config = LocalFSStoreConfig; + + const Config & config; + inline static std::string operationName = "Local Filesystem Store"; const static std::string drvsLogDir; - LocalFSStore(const Params & params); + LocalFSStore(const Config & params); void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor(bool requireValidPath = true) override; @@ -70,7 +73,7 @@ public: */ virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0; - virtual Path getRealStoreDir() { return realStoreDir; } + virtual Path getRealStoreDir() { return config.realStoreDir; } Path toRealPath(const Path & storePath) override { diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index f3658d603..6077d9e53 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -56,9 +56,9 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig The store directory is passed as an argument to the invoked executable. )"}; - const std::string name() override { return "Experimental Local Overlay Store"; } + static const std::string name() { return "Experimental Local Overlay Store"; } - std::optional experimentalFeature() const override + static std::optional experimentalFeature() { return ExperimentalFeature::LocalOverlayStore; } @@ -68,7 +68,9 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig return { "local-overlay" }; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; protected: /** @@ -79,7 +81,9 @@ protected: * at that file path. It might be stored in the lower layer instead, * or it might not be part of this store at all. */ - Path toUpperPath(const StorePath & path); + Path toUpperPath(const StorePath & path) const; + + friend struct LocalOverlayStore; }; /** @@ -88,8 +92,20 @@ protected: * Documentation on overridden methods states how they differ from their * `LocalStore` counterparts. */ -class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore +struct LocalOverlayStore : virtual LocalStore { + using Config = LocalOverlayStoreConfig; + + ref config; + + LocalOverlayStore(ref); + + std::string getUri() override + { + return "local-overlay://"; + } + +private: /** * The store beneath us. * @@ -99,20 +115,6 @@ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual */ ref lowerStore; -public: - LocalOverlayStore(const Params & params) - : LocalOverlayStore("local-overlay", "", params) - { - } - - LocalOverlayStore(std::string_view scheme, PathView path, const Params & params); - - std::string getUri() override - { - return "local-overlay://"; - } - -private: /** * First copy up any lower store realisation with the same key, so we * merge rather than mask it. diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index baa82e7f6..efc59dc8c 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -34,7 +34,7 @@ struct OptimiseStats uint64_t bytesFreed = 0; }; -struct LocalStoreConfig : virtual LocalFSStoreConfig +struct LocalStoreConfig : std::enable_shared_from_this, virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; @@ -65,18 +65,26 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. )"}; - const std::string name() override { return "Local Store"; } + static const std::string name() { return "Local Store"; } static StringSet uriSchemes() { return {"local"}; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; -class LocalStore : public virtual LocalStoreConfig - , public virtual IndirectRootStore - , public virtual GcStore +class LocalStore : + public virtual IndirectRootStore, + public virtual GcStore { +public: + + using Config = LocalStoreConfig; + + ref config; + private: /** @@ -144,11 +152,7 @@ public: * Initialise the local store, upgrading the schema if * necessary. */ - LocalStore(const Params & params); - LocalStore( - std::string_view scheme, - PathView path, - const Params & params); + LocalStore(ref params); ~LocalStore(); diff --git a/src/libstore/include/nix/store/profiles.hh b/src/libstore/include/nix/store/profiles.hh index 804c6e2b7..e20e1198e 100644 --- a/src/libstore/include/nix/store/profiles.hh +++ b/src/libstore/include/nix/store/profiles.hh @@ -86,7 +86,7 @@ typedef std::list Generations; */ std::pair> findGenerations(Path profile); -class LocalFSStore; +struct LocalFSStore; /** * Create a new generation of the given profile diff --git a/src/libstore/include/nix/store/remote-fs-accessor.hh b/src/libstore/include/nix/store/remote-fs-accessor.hh index 75a840fb0..75bb40dfb 100644 --- a/src/libstore/include/nix/store/remote-fs-accessor.hh +++ b/src/libstore/include/nix/store/remote-fs-accessor.hh @@ -19,7 +19,7 @@ class RemoteFSAccessor : public SourceAccessor std::pair, CanonPath> fetch(const CanonPath & path); - friend class BinaryCacheStore; + friend struct BinaryCacheStore; Path makeCacheFile(std::string_view hashPart, const std::string & ext); diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index ecf18bd76..dd2396fe3 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -35,14 +35,16 @@ struct RemoteStoreConfig : virtual StoreConfig * \todo RemoteStore is a misnomer - should be something like * DaemonStore. */ -class RemoteStore : public virtual RemoteStoreConfig, +struct RemoteStore : public virtual Store, public virtual GcStore, public virtual LogStore { -public: + using Config = RemoteStoreConfig; - RemoteStore(const Params & params); + const Config & config; + + RemoteStore(const Config & config); /* Implementations of abstract store API methods. */ diff --git a/src/libstore/include/nix/store/restricted-store.hh b/src/libstore/include/nix/store/restricted-store.hh index 67c26c88b..6f2122c7b 100644 --- a/src/libstore/include/nix/store/restricted-store.hh +++ b/src/libstore/include/nix/store/restricted-store.hh @@ -55,6 +55,6 @@ struct RestrictionContext /** * Create a shared pointer to a restricted store. */ -ref makeRestrictedStore(const Store::Params & params, ref next, RestrictionContext & context); +ref makeRestrictedStore(ref config, ref next, RestrictionContext & context); } diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index e3c95ce78..9a123602e 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -11,7 +11,7 @@ namespace nix { -struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +struct S3BinaryCacheStoreConfig : std::enable_shared_from_this, virtual BinaryCacheStoreConfig { std::string bucketName; @@ -93,7 +93,7 @@ public: const Setting bufferSize{ this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; - const std::string name() override + static const std::string name() { return "S3 Binary Cache Store"; } @@ -103,16 +103,18 @@ public: return {"s3"}; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; -class S3BinaryCacheStore : public virtual BinaryCacheStore +struct S3BinaryCacheStore : virtual BinaryCacheStore { -protected: + using Config = S3BinaryCacheStoreConfig; - S3BinaryCacheStore(const Params & params); + ref config; -public: + S3BinaryCacheStore(ref); struct Stats { diff --git a/src/libstore/include/nix/store/ssh-store.hh b/src/libstore/include/nix/store/ssh-store.hh index a0f73cc8e..fde165445 100644 --- a/src/libstore/include/nix/store/ssh-store.hh +++ b/src/libstore/include/nix/store/ssh-store.hh @@ -8,7 +8,9 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +struct SSHStoreConfig : std::enable_shared_from_this, + virtual RemoteStoreConfig, + virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; using RemoteStoreConfig::RemoteStoreConfig; @@ -18,7 +20,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig const Setting remoteProgram{ this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; - const std::string name() override + static const std::string name() { return "Experimental SSH Store"; } @@ -28,19 +30,17 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig return {"ssh-ng"}; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig { - using LocalFSStoreConfig::LocalFSStoreConfig; - using SSHStoreConfig::SSHStoreConfig; - MountedSSHStoreConfig(StringMap params); - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); - const std::string name() override + static const std::string name() { return "Experimental SSH Store with filesystem mounted"; } @@ -50,12 +50,14 @@ struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfi return {"mounted-ssh-ng"}; } - std::string doc() override; + static std::string doc(); - std::optional experimentalFeature() const override + static std::optional experimentalFeature() { return ExperimentalFeature::MountedSSHStore; } + + ref openStore() const override; }; } diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index fa1873502..fb93c7178 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -26,32 +26,6 @@ namespace nix { -/** - * About the class hierarchy of the store types: - * - * Each store type `Foo` consists of two classes: - * - * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration - * for the store - * - * It should only contain members of type `const Setting` (or subclasses - * of it) and inherit the constructors of `StoreConfig` - * (`using StoreConfig::StoreConfig`). - * - * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the - * implementation of the store. - * - * This class is expected to have a constructor `Foo(const Params & params)` - * that calls `StoreConfig(params)` (otherwise you're gonna encounter an - * `assertion failure` when trying to instantiate it). - * - * You can then register the new store using: - * - * ``` - * cpp static RegisterStoreImplementation regStore; - * ``` - */ - MakeError(SubstError, Error); /** * denotes a permanent build failure @@ -97,27 +71,48 @@ struct KeyedBuildResult; typedef std::map> StorePathCAMap; + +/** + * About the class hierarchy of the store types: + * + * Each store type `Foo` consists of two classes: + * + * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration + * for the store + * + * It should only contain members of type `Setting` (or subclasses + * of it) and inherit the constructors of `StoreConfig` + * (`using StoreConfig::StoreConfig`). + * + * 2. A class `Foo : virtual Store` that contains the + * implementation of the store. + * + * This class is expected to have: + * + * 1. an alias `using Config = FooConfig;` + * + * 2. a constructor `Foo(ref params)`. + * + * You can then register the new store using: + * + * ``` + * cpp static RegisterStoreImplementation regStore; + * ``` + */ struct StoreConfig : public StoreDirConfig { - using Params = StoreReference::Params; - using StoreDirConfig::StoreDirConfig; StoreConfig() = delete; - static StringSet getDefaultSystemFeatures(); - virtual ~StoreConfig() { } - /** - * The name of this type of store. - */ - virtual const std::string name() = 0; + static StringSet getDefaultSystemFeatures(); /** * Documentation for this type of store. */ - virtual std::string doc() + static std::string doc() { return ""; } @@ -126,15 +121,15 @@ struct StoreConfig : public StoreDirConfig * An experimental feature this type store is gated, if it is to be * experimental. */ - virtual std::optional experimentalFeature() const + static std::optional experimentalFeature() { return std::nullopt; } - const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", + Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "Size of the in-memory store path metadata cache."}; - const Setting isTrusted{this, false, "trusted", + Setting isTrusted{this, false, "trusted", R"( Whether paths from this store can be used as substitutes even if they are not signed by a key listed in the @@ -163,10 +158,38 @@ struct StoreConfig : public StoreDirConfig {}, // Don't document the machine-specific default value false}; + + /** + * Open a store of the type corresponding to this configuration + * type. + */ + virtual ref openStore() const = 0; }; -class Store : public std::enable_shared_from_this, public virtual StoreConfig +/** + * A Store (client) + * + * This is an interface type allowing for create and read operations on + * a collection of store objects, and also building new store objects + * from `Derivation`s. See the manual for further details. + * + * "client" used is because this is just one view/actor onto an + * underlying resource, which could be an external process (daemon + * server), file system state, etc. + */ +class Store : public std::enable_shared_from_this, public MixStoreDirMethods { +public: + + using Config = StoreConfig; + + const Config & config; + + /** + * @note Avoid churn, since we used to inherit from `Config`. + */ + operator const Config &() const { return config; } + protected: struct PathInfoCacheValue { @@ -205,7 +228,7 @@ protected: std::shared_ptr diskCache; - Store(const Params & params); + Store(const Store::Config & config); public: /** @@ -877,7 +900,7 @@ ref openStore(StoreReference && storeURI); */ ref openStore(const std::string & uri = settings.storeUri.get(), - const Store::Params & extraParams = Store::Params()); + const Store::Config::Params & extraParams = Store::Config::Params()); /** @@ -888,46 +911,72 @@ std::list> getDefaultSubstituters(); struct StoreFactory { + /** + * Documentation for this type of store. + */ + std::string doc; + + /** + * URIs with these schemes should be handled by this factory + */ StringSet uriSchemes; + + /** + * An experimental feature this type store is gated, if it is to be + * experimental. + */ + std::optional experimentalFeature; + /** * The `authorityPath` parameter is `/`, or really * whatever comes after `://` and before `?`. */ - std::function ( - std::string_view scheme, - std::string_view authorityPath, - const Store::Params & params)> create; - std::function ()> getConfig; + std::function( + std::string_view scheme, std::string_view authorityPath, const Store::Config::Params & params)> + parseConfig; + + /** + * Just for dumping the defaults. Kind of awkward this exists, + * because it means we cannot require fields to be manually + * specified so easily. + */ + std::function()> getConfig; }; struct Implementations { - static std::vector & registered(); + using Map = std::map; - template + static Map & registered(); + + template static void add() { StoreFactory factory{ + .doc = TConfig::doc(), .uriSchemes = TConfig::uriSchemes(), - .create = + .experimentalFeature = TConfig::experimentalFeature(), + .parseConfig = ([](auto scheme, auto uri, auto & params) - -> std::shared_ptr - { return std::make_shared(scheme, uri, params); }), + -> ref + { return make_ref(scheme, uri, params); }), .getConfig = - ([]() - -> std::shared_ptr - { return std::make_shared(StringMap({})); }) + ([]() -> ref + { return make_ref(Store::Config::Params{}); }), }; - registered().push_back(factory); + auto [it, didInsert] = registered().insert({TConfig::name(), std::move(factory)}); + if (!didInsert) { + throw Error("Already registred store with name '%s'", it->first); + } } }; -template +template struct RegisterStoreImplementation { RegisterStoreImplementation() { - Implementations::add(); + Implementations::add(); } }; diff --git a/src/libstore/include/nix/store/store-dir-config.hh b/src/libstore/include/nix/store/store-dir-config.hh index 845a003f5..40a71e446 100644 --- a/src/libstore/include/nix/store/store-dir-config.hh +++ b/src/libstore/include/nix/store/store-dir-config.hh @@ -18,22 +18,18 @@ struct SourcePath; MakeError(BadStorePath, Error); MakeError(BadStorePathName, BadStorePath); -struct StoreDirConfig : public Config +/** + * @todo This should just be part of `StoreDirConfig`. However, it would + * be a huge amount of churn if `Store` didn't have these methods + * anymore, forcing a bunch of code to go from `store.method(...)` to + * `store.config.method(...)`. + * + * So we instead pull out the methods into their own mix-in, so can put + * them directly on the Store too. + */ +struct MixStoreDirMethods { - using Config::Config; - - StoreDirConfig() = delete; - - virtual ~StoreDirConfig() = default; - - const PathSetting storeDir_{this, settings.nixStore, - "store", - R"( - Logical location of the Nix store, usually - `/nix/store`. Note that you can only copy store paths - between stores if they have the same `store` setting. - )"}; - const Path storeDir = storeDir_; + const Path & storeDir; // pure methods @@ -56,7 +52,7 @@ struct StoreDirConfig : public Config * Display a set of paths in human-readable form (i.e., between quotes * and separated by commas). */ - std::string showPaths(const StorePathSet & paths); + std::string showPaths(const StorePathSet & paths) const; /** * @return true if *path* is in the Nix store (but not the Nix @@ -104,4 +100,38 @@ struct StoreDirConfig : public Config PathFilter & filter = defaultPathFilter) const; }; +/** + * Need to make this a separate class so I can get the right + * initialization order in the constructor for `StoreDirConfig`. + */ +struct StoreDirConfigItself : Config +{ + using Config::Config; + + const PathSetting storeDir_{this, settings.nixStore, + "store", + R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )"}; +}; + +/** + * The order of `StoreDirConfigItself` and then `MixStoreDirMethods` is + * very important. This ensures that `StoreDirConfigItself::storeDir_` + * is initialized before we have our one chance (because references are + * immutable) to initialize `MixStoreDirMethods::storeDir`. + */ +struct StoreDirConfig : StoreDirConfigItself, MixStoreDirMethods +{ + using Params = std::map; + + StoreDirConfig(const Params & params); + + StoreDirConfig() = delete; + + virtual ~StoreDirConfig() = default; +}; + } diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index 9c85fbfb2..e9793a9ee 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -7,7 +7,10 @@ namespace nix { -struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig +struct UDSRemoteStoreConfig : + std::enable_shared_from_this, + virtual LocalFSStoreConfig, + virtual RemoteStoreConfig { // TODO(fzakaria): Delete this constructor once moved over to the factory pattern // outlined in https://github.com/NixOS/nix/issues/10766 @@ -22,9 +25,11 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon std::string_view authority, const Params & params); - const std::string name() override { return "Local Daemon Store"; } + UDSRemoteStoreConfig(const Params & params); - std::string doc() override; + static const std::string name() { return "Local Daemon Store"; } + + static std::string doc(); /** * The path to the unix domain socket. @@ -34,32 +39,21 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon */ Path path; -protected: - static constexpr char const * scheme = "unix"; - -public: static StringSet uriSchemes() - { return {scheme}; } + { return {"unix"}; } + + ref openStore() const override; }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig - , public virtual IndirectRootStore - , public virtual RemoteStore +struct UDSRemoteStore : + virtual IndirectRootStore, + virtual RemoteStore { -public: + using Config = UDSRemoteStoreConfig; - /** - * @deprecated This is the old API to construct the store. - */ - UDSRemoteStore(const Params & params); + ref config; - /** - * @param authority is the socket path. - */ - UDSRemoteStore( - std::string_view scheme, - std::string_view authority, - const Params & params); + UDSRemoteStore(ref); std::string getUri() override; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 1512a7944..1bddc280d 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -38,23 +38,19 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; -LegacySSHStore::LegacySSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , LegacySSHStoreConfig(scheme, host, params) - , Store(params) + +LegacySSHStore::LegacySSHStore(ref config) + : Store{*config} + , config{config} , connections(make_ref>( - std::max(1, (int) maxConnections), + std::max(1, (int) config->maxConnections), [this]() { return openConnection(); }, [](const ref & r) { return r->good; } )) - , master(createSSHMaster( + , master(config->createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - logFD)) + config->logFD)) { } @@ -62,16 +58,16 @@ LegacySSHStore::LegacySSHStore( ref LegacySSHStore::openConnection() { auto conn = make_ref(); - Strings command = remoteProgram.get(); + Strings command = config->remoteProgram.get(); command.push_back("--serve"); command.push_back("--write"); - if (remoteStore.get() != "") { + if (config->remoteStore.get() != "") { command.push_back("--store"); - command.push_back(remoteStore.get()); + command.push_back(config->remoteStore.get()); } - conn->sshConn = master.startCommand(std::move(command), std::list{extraSshArgs}); - if (connPipeSize) { - conn->sshConn->trySetBufferSize(*connPipeSize); + conn->sshConn = master.startCommand(std::move(command), std::list{config->extraSshArgs}); + if (config->connPipeSize) { + conn->sshConn->trySetBufferSize(*config->connPipeSize); } conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); @@ -80,7 +76,7 @@ ref LegacySSHStore::openConnection() TeeSource tee(conn->from, saved); try { conn->remoteVersion = ServeProto::BasicClientConnection::handshake( - conn->to, tee, SERVE_PROTOCOL_VERSION, host); + conn->to, tee, SERVE_PROTOCOL_VERSION, config->host); } catch (SerialisationError & e) { // in.close(): Don't let the remote block on us not writing. conn->sshConn->in.close(); @@ -89,9 +85,9 @@ ref LegacySSHStore::openConnection() tee.drainInto(nullSink); } throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", - host, chomp(saved.s)); + config->host, chomp(saved.s)); } catch (EndOfFile & e) { - throw Error("cannot connect to '%1%'", host); + throw Error("cannot connect to '%1%'", config->host); } return conn; @@ -100,7 +96,7 @@ ref LegacySSHStore::openConnection() std::string LegacySSHStore::getUri() { - return *uriSchemes().begin() + "://" + host; + return *Config::uriSchemes().begin() + "://" + config->host; } std::map LegacySSHStore::queryPathInfosUncached( @@ -111,7 +107,7 @@ std::map LegacySSHStore::queryPathInfosUncached /* No longer support missing NAR hash */ assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); - debug("querying remote host '%s' for info on '%s'", host, concatStringsSep(", ", printStorePathSet(paths))); + debug("querying remote host '%s' for info on '%s'", config->host, concatStringsSep(", ", printStorePathSet(paths))); auto infos = conn->queryPathInfos(*this, paths); @@ -151,7 +147,7 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path, void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { - debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); + debug("adding path '%s' to remote host '%s'", printStorePath(info.path), config->host); auto conn(connections->get()); @@ -178,7 +174,7 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, conn->to.flush(); if (readInt(conn->from) != 1) - throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), config->host); } else { @@ -390,12 +386,17 @@ LegacySSHStore::ConnectionStats LegacySSHStore::getConnectionStats() * The legacy ssh protocol doesn't support checking for trusted-user. * Try using ssh-ng:// instead if you want to know. */ -std::optional isTrustedClient() +std::optional LegacySSHStore::isTrustedClient() { return std::nullopt; } -static RegisterStoreImplementation regLegacySSHStore; +ref LegacySSHStore::Config::openStore() const { + return make_ref(ref{shared_from_this()}); +} + + +static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 30396d7b6..645cca5f2 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -10,9 +10,9 @@ namespace nix { LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( std::string_view scheme, PathView binaryCacheDir, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) + const StoreReference::Params & params) + : Store::Config{params} + , BinaryCacheStoreConfig{params} , binaryCacheDir(binaryCacheDir) { } @@ -26,29 +26,26 @@ std::string LocalBinaryCacheStoreConfig::doc() } -struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore +struct LocalBinaryCacheStore : + virtual BinaryCacheStore { - /** - * @param binaryCacheDir `file://` is a short-hand for `file:///` - * for now. - */ - LocalBinaryCacheStore( - std::string_view scheme, - PathView binaryCacheDir, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , LocalBinaryCacheStoreConfig(scheme, binaryCacheDir, params) - , Store(params) - , BinaryCacheStore(params) + using Config = LocalBinaryCacheStoreConfig; + + ref config; + + LocalBinaryCacheStore(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , config{config} { + init(); } void init() override; std::string getUri() override { - return "file://" + binaryCacheDir; + return "file://" + config->binaryCacheDir; } protected: @@ -59,7 +56,7 @@ protected: std::shared_ptr> istream, const std::string & mimeType) override { - auto path2 = binaryCacheDir + "/" + path; + auto path2 = config->binaryCacheDir + "/" + path; static std::atomic counter{0}; Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); AutoDelete del(tmp, false); @@ -72,7 +69,7 @@ protected: void getFile(const std::string & path, Sink & sink) override { try { - readFile(binaryCacheDir + "/" + path, sink); + readFile(config->binaryCacheDir + "/" + path, sink); } catch (SysError & e) { if (e.errNo == ENOENT) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); @@ -84,7 +81,7 @@ protected: { StorePathSet paths; - for (auto & entry : DirectoryIterator{binaryCacheDir}) { + for (auto & entry : DirectoryIterator{config->binaryCacheDir}) { checkInterrupt(); auto name = entry.path().filename().string(); if (name.size() != 40 || @@ -106,17 +103,17 @@ protected: void LocalBinaryCacheStore::init() { - createDirs(binaryCacheDir + "/nar"); - createDirs(binaryCacheDir + "/" + realisationsPrefix); - if (writeDebugInfo) - createDirs(binaryCacheDir + "/debuginfo"); - createDirs(binaryCacheDir + "/log"); + createDirs(config->binaryCacheDir + "/nar"); + createDirs(config->binaryCacheDir + "/" + realisationsPrefix); + if (config->writeDebugInfo) + createDirs(config->binaryCacheDir + "/debuginfo"); + createDirs(config->binaryCacheDir + "/log"); BinaryCacheStore::init(); } bool LocalBinaryCacheStore::fileExists(const std::string & path) { - return pathExists(binaryCacheDir + "/" + path); + return pathExists(config->binaryCacheDir + "/" + path); } StringSet LocalBinaryCacheStoreConfig::uriSchemes() @@ -127,6 +124,13 @@ StringSet LocalBinaryCacheStoreConfig::uriSchemes() return {"file"}; } -static RegisterStoreImplementation regLocalBinaryCacheStore; +ref LocalBinaryCacheStoreConfig::openStore() const { + return make_ref(ref{ + // FIXME we shouldn't actually need a mutable config + std::const_pointer_cast(shared_from_this()) + }); +} + +static RegisterStoreImplementation regLocalBinaryCacheStore; } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index c6c5d53c9..add3b04d2 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -22,8 +22,9 @@ LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) { } -LocalFSStore::LocalFSStore(const Params & params) - : Store(params) +LocalFSStore::LocalFSStore(const Config & config) + : Store{static_cast(*this)} + , config{config} { } @@ -33,7 +34,7 @@ struct LocalStoreAccessor : PosixSourceAccessor bool requireValidPath; LocalStoreAccessor(ref store, bool requireValidPath) - : PosixSourceAccessor(std::filesystem::path{store->realStoreDir.get()}) + : PosixSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()}) , store(store) , requireValidPath(requireValidPath) { @@ -104,8 +105,8 @@ std::optional LocalFSStore::getBuildLogExact(const StorePath & path Path logPath = j == 0 - ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) - : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); + ? fmt("%s/%s/%s/%s", config.logDir.get(), drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) + : fmt("%s/%s/%s", config.logDir.get(), drvsLogDir, baseName); Path logBz2Path = logPath + ".bz2"; if (pathExists(logPath)) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 38fa634ca..e1a4d40de 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -14,25 +14,32 @@ std::string LocalOverlayStoreConfig::doc() ; } -Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { +ref LocalOverlayStoreConfig::openStore() const +{ + return make_ref(ref{ + std::dynamic_pointer_cast(shared_from_this()) + }); +} + + +Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) const +{ return upperLayer + "/" + path.to_string(); } -LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(params) - , LocalOverlayStoreConfig(scheme, path, params) - , Store(params) - , LocalFSStore(params) - , LocalStore(params) - , lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast()) + +LocalOverlayStore::LocalOverlayStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , LocalStore{static_cast>(config)} + , config{config} + , lowerStore(openStore(percentDecode(config->lowerStoreUri.get())).dynamic_pointer_cast()) { - if (checkMount.get()) { + if (config->checkMount.get()) { std::smatch match; std::string mountInfo; auto mounts = readFile(std::filesystem::path{"/proc/self/mounts"}); - auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))"); + auto regex = std::regex(R"((^|\n)overlay )" + config->realStoreDir.get() + R"( .*(\n|$))"); // Mount points can be stacked, so there might be multiple matching entries. // Loop until the last match, which will be the current state of the mount point. @@ -45,13 +52,13 @@ LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, con return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)")); }; - auto expectedLowerDir = lowerStore->realStoreDir.get(); - if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) { + auto expectedLowerDir = lowerStore->config.realStoreDir.get(); + if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", config->upperLayer)) { debug("expected lowerdir: %s", expectedLowerDir); - debug("expected upperdir: %s", upperLayer); + debug("expected upperdir: %s", config->upperLayer); debug("actual mount: %s", mountInfo); throw Error("overlay filesystem '%s' mounted incorrectly", - realStoreDir.get()); + config->realStoreDir.get()); } } } @@ -201,14 +208,14 @@ void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & re void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) { - auto mergedDir = realStoreDir.get() + "/"; + auto mergedDir = config->realStoreDir.get() + "/"; if (path.substr(0, mergedDir.length()) != mergedDir) { warn("local-overlay: unexpected gc path '%s' ", path); return; } StorePath storePath = {path.substr(mergedDir.length())}; - auto upperPath = toUpperPath(storePath); + auto upperPath = config->toUpperPath(storePath); if (pathExists(upperPath)) { debug("upper exists: %s", path); @@ -257,7 +264,7 @@ LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag StorePathSet done; auto existsInStoreDir = [&](const StorePath & storePath) { - return pathExists(realStoreDir.get() + "/" + storePath.to_string()); + return pathExists(config->realStoreDir.get() + "/" + storePath.to_string()); }; bool errors = false; @@ -277,16 +284,16 @@ void LocalOverlayStore::remountIfNecessary() { if (!_remountRequired) return; - if (remountHook.get().empty()) { - warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get()); + if (config->remountHook.get().empty()) { + warn("'%s' needs remounting, set remount-hook to do this automatically", config->realStoreDir.get()); } else { - runProgram(remountHook, false, {realStoreDir}); + runProgram(config->remountHook, false, {config->realStoreDir}); } _remountRequired = false; } -static RegisterStoreImplementation regLocalOverlayStore; +static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f1cf466a6..0293b9af2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -75,6 +75,11 @@ std::string LocalStoreConfig::doc() ; } +ref LocalStore::Config::openStore() const +{ + return make_ref(ref{shared_from_this()}); +} + struct LocalStore::State::Stmts { /* Some precompiled SQLite statements. */ SQLiteStmt RegisterValidPath; @@ -97,38 +102,33 @@ struct LocalStore::State::Stmts { SQLiteStmt AddRealisationReference; }; -LocalStore::LocalStore( - std::string_view scheme, - PathView path, - const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(scheme, path, params) - , Store(params) - , LocalFSStore(params) - , dbDir(stateDir + "/db") - , linksDir(realStoreDir + "/.links") +LocalStore::LocalStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , config{config} + , dbDir(config->stateDir + "/db") + , linksDir(config->realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") - , tempRootsDir(stateDir + "/temproots") + , tempRootsDir(config->stateDir + "/temproots") , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) { auto state(_state.lock()); state->stmts = std::make_unique(); /* Create missing state directories if they don't already exist. */ - createDirs(realStoreDir.get()); - if (readOnly) { + createDirs(config->realStoreDir.get()); + if (config->readOnly) { experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); } else { makeStoreWritable(); } createDirs(linksDir); - Path profilesDir = stateDir + "/profiles"; + Path profilesDir = config->stateDir + "/profiles"; createDirs(profilesDir); createDirs(tempRootsDir); createDirs(dbDir); - Path gcRootsDir = stateDir + "/gcroots"; + Path gcRootsDir = config->stateDir + "/gcroots"; if (!pathExists(gcRootsDir)) { createDirs(gcRootsDir); createSymlink(profilesDir, gcRootsDir + "/profiles"); @@ -136,7 +136,7 @@ LocalStore::LocalStore( for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); - if (!readOnly) { + if (!config->readOnly) { // Skip chmod call if the directory already has the correct permissions (0755). // This is to avoid failing when the executing user lacks permissions to change the directory's permissions // even if it would be no-op. @@ -153,16 +153,16 @@ LocalStore::LocalStore( struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); - else if (!readOnly) { + else if (!config->readOnly) { struct stat st; - if (stat(realStoreDir.get().c_str(), &st)) - throw SysError("getting attributes of path '%1%'", realStoreDir); + if (stat(config->realStoreDir.get().c_str(), &st)) + throw SysError("getting attributes of path '%1%'", config->realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { - if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) - throw SysError("changing ownership of path '%1%'", realStoreDir); - if (chmod(realStoreDir.get().c_str(), perm) == -1) - throw SysError("changing permissions on path '%1%'", realStoreDir); + if (chown(config->realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) + throw SysError("changing ownership of path '%1%'", config->realStoreDir); + if (chmod(config->realStoreDir.get().c_str(), perm) == -1) + throw SysError("changing permissions on path '%1%'", config->realStoreDir); } } } @@ -170,7 +170,7 @@ LocalStore::LocalStore( /* Ensure that the store and its parents are not symlinks. */ if (!settings.allowSymlinkedStore) { - std::filesystem::path path = realStoreDir.get(); + std::filesystem::path path = config->realStoreDir.get(); std::filesystem::path root = path.root_path(); while (path != root) { if (std::filesystem::is_symlink(path)) @@ -217,12 +217,12 @@ LocalStore::LocalStore( /* Acquire the big fat lock in shared mode to make sure that no schema upgrade is in progress. */ - if (!readOnly) { + if (!config->readOnly) { Path globalLockPath = dbDir + "/big-lock"; globalLock = openLockFile(globalLockPath.c_str(), true); } - if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) { + if (!config->readOnly && !lockFile(globalLock.get(), ltRead, false)) { printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -230,7 +230,7 @@ LocalStore::LocalStore( /* Check the current database schema and if necessary do an upgrade. */ int curSchema = getSchema(); - if (readOnly && curSchema < nixSchemaVersion) { + if (config->readOnly && curSchema < nixSchemaVersion) { debug("current schema version: %d", curSchema); debug("supported schema version: %d", nixSchemaVersion); throw Error(curSchema == 0 ? @@ -378,15 +378,9 @@ LocalStore::LocalStore( } -LocalStore::LocalStore(const Params & params) - : LocalStore("local", "", params) -{ -} - - AutoCloseFD LocalStore::openGCLock() { - Path fnGCLock = stateDir + "/gc.lock"; + Path fnGCLock = config->stateDir + "/gc.lock"; auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT #ifndef _WIN32 | O_CLOEXEC @@ -452,17 +446,17 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { - if (create && readOnly) { + if (create && config->readOnly) { throw Error("cannot create database while in read-only mode"); } - if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) + if (access(dbDir.c_str(), R_OK | (config->readOnly ? 0 : W_OK))) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - auto openMode = readOnly ? SQLiteOpenMode::Immutable + auto openMode = config->readOnly ? SQLiteOpenMode::Immutable : create ? SQLiteOpenMode::Normal : SQLiteOpenMode::NoCreate; state.db = SQLite(dbPath, openMode); @@ -575,12 +569,12 @@ void LocalStore::makeStoreWritable() if (!isRootUser()) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; - if (statvfs(realStoreDir.get().c_str(), &stat) != 0) + if (statvfs(config->realStoreDir.get().c_str(), &stat) != 0) throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError("remounting %1% writable", realStoreDir); + if (mount(0, config->realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + throw SysError("remounting %1% writable", config->realStoreDir); } #endif } @@ -920,7 +914,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) for (auto & sub : getDefaultSubstituters()) { if (remaining.empty()) break; if (sub->storeDir != storeDir) continue; - if (!sub->wantMassQuery) continue; + if (!sub->config.wantMassQuery) continue; auto valid = sub->queryValidPaths(remaining); @@ -1032,12 +1026,12 @@ const PublicKeys & LocalStore::getPublicKeys() bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info) { - return requireSigs && !info.checkSignatures(*this, getPublicKeys()); + return config->requireSigs && !info.checkSignatures(*this, getPublicKeys()); } bool LocalStore::realisationIsUntrusted(const Realisation & realisation) { - return requireSigs && !realisation.checkSignatures(getPublicKeys()); + return config->requireSigs && !realisation.checkSignatures(getPublicKeys()); } void LocalStore::addToStore(const ValidPathInfo & info, Source & source, @@ -1334,7 +1328,7 @@ std::pair LocalStore::createTempDirInStore() /* There is a slight possibility that `tmpDir' gets deleted by the GC between createTempDir() and when we acquire a lock on it. We'll repeat until 'tmpDir' exists and we've locked it. */ - tmpDirFn = createTempDir(realStoreDir, "tmp"); + tmpDirFn = createTempDir(config->realStoreDir, "tmp"); tmpDirFd = openDirectory(tmpDirFn); if (!tmpDirFd) { continue; @@ -1475,7 +1469,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair database and the filesystem) in the loop below, in order to catch invalid states. */ - for (auto & i : DirectoryIterator{realStoreDir.to_string()}) { + for (auto & i : DirectoryIterator{config->realStoreDir.get()}) { checkInterrupt(); try { storePathsInStoreDir.insert({i.path().filename().string()}); @@ -1664,7 +1658,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) auto baseName = drvPath.to_string(); - auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); + auto logPath = fmt("%s/%s/%s/%s.bz2", config->logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); if (pathExists(logPath)) return; @@ -1682,6 +1676,6 @@ std::optional LocalStore::getVersion() return nixVersion; } -static RegisterStoreImplementation regLocalStore; +static RegisterStoreImplementation regLocalStore; } // namespace nix diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 255f83f74..f3eac04fc 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -313,6 +313,7 @@ sources = files( 'ssh-store.cc', 'ssh.cc', 'store-api.cc', + 'store-dir-config.cc', 'store-reference.cc', 'uds-remote-store.cc', 'worker-protocol-connection.cc', diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 277795053..e47c0707c 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -101,7 +101,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* HFS/macOS has some undocumented security feature disabling hardlinking for special files within .app dirs. Known affected paths include *.app/Contents/{PkgInfo,Resources/\*.lproj,_CodeSignature} and .DS_Store. - See https://github.com/NixOS/nix/issues/1443 and + See https://github.com/NixOS/nix/issues/1443 and https://github.com/NixOS/nix/pull/2230 for more discussion. */ if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) @@ -216,14 +216,14 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, the store itself (we don't want or need to mess with its permissions). */ const Path dirOfPath(dirOf(path)); - bool mustToggle = dirOfPath != realStoreDir.get(); + bool mustToggle = dirOfPath != config->realStoreDir.get(); if (mustToggle) makeWritable(dirOfPath); /* When we're done, make the directory read-only again and reset its timestamp back to 0. */ MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); - std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), rand()); + std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", config->realStoreDir, getpid(), rand()); try { std::filesystem::create_hard_link(linkPath, tempLink); @@ -285,7 +285,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) if (!isValidPath(i)) continue; /* path was GC'ed, probably */ { Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i))); - optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair); + optimisePath_(&act, stats, config->realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair); } done++; act.progress(done, paths.size()); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 5dd1a1699..d989b1caa 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -75,7 +75,7 @@ StorePath StorePath::random(std::string_view name) return StorePath(Hash::random(HashAlgorithm::SHA1), name); } -StorePath StoreDirConfig::parseStorePath(std::string_view path) const +StorePath MixStoreDirMethods::parseStorePath(std::string_view path) const { // On Windows, `/nix/store` is not a canonical path. More broadly it // is unclear whether this function should be using the native @@ -94,7 +94,7 @@ StorePath StoreDirConfig::parseStorePath(std::string_view path) const return StorePath(baseNameOf(p)); } -std::optional StoreDirConfig::maybeParseStorePath(std::string_view path) const +std::optional MixStoreDirMethods::maybeParseStorePath(std::string_view path) const { try { return parseStorePath(path); @@ -103,24 +103,24 @@ std::optional StoreDirConfig::maybeParseStorePath(std::string_view pa } } -bool StoreDirConfig::isStorePath(std::string_view path) const +bool MixStoreDirMethods::isStorePath(std::string_view path) const { return (bool) maybeParseStorePath(path); } -StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const +StorePathSet MixStoreDirMethods::parseStorePathSet(const PathSet & paths) const { StorePathSet res; for (auto & i : paths) res.insert(parseStorePath(i)); return res; } -std::string StoreDirConfig::printStorePath(const StorePath & path) const +std::string MixStoreDirMethods::printStorePath(const StorePath & path) const { return (storeDir + "/").append(path.to_string()); } -PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const +PathSet MixStoreDirMethods::printStorePathSet(const StorePathSet & paths) const { PathSet res; for (auto & i : paths) res.insert(printStorePath(i)); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8f110ce7c..3151f319c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -24,11 +24,11 @@ namespace nix { /* TODO: Separate these store types into different files, give them better names */ -RemoteStore::RemoteStore(const Params & params) - : RemoteStoreConfig(params) - , Store(params) +RemoteStore::RemoteStore(const Config & config) + : Store{config} + , config{config} , connections(make_ref>( - std::max(1, maxConnections.get()), + std::max(1, config.maxConnections.get()), [this]() { auto conn = openConnectionWrapper(); try { @@ -44,7 +44,7 @@ RemoteStore::RemoteStore(const Params & params) r->to.good() && r->from.good() && std::chrono::duration_cast( - std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; + std::chrono::steady_clock::now() - r->startTime).count() < this->config.maxConnectionAge; } )) { @@ -122,7 +122,7 @@ void RemoteStore::setOptions(Connection & conn) << settings.useSubstitutes; if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { - std::map overrides; + std::map overrides; settings.getSettings(overrides, true); // libstore settings fileTransferSettings.getSettings(overrides, true); overrides.erase(settings.keepFailed.name); diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 3b4c45d3d..0485f5584 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -30,32 +30,23 @@ bool RestrictionContext::isAllowed(const DerivedPath & req) return isAllowed(pathPartOfReq(req)); } -struct RestrictedStoreConfig : virtual LocalFSStoreConfig -{ - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() override - { - return "Restricted Store"; - } -}; - /** * A wrapper around LocalStore that only allows building/querying of * paths that are in the input closures of the build or were added via * recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore +struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStore { + ref config; + ref next; RestrictionContext & goal; - RestrictedStore(const Params & params, ref next, RestrictionContext & goal) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RestrictedStoreConfig(params) - , Store(params) - , LocalFSStore(params) + RestrictedStore(ref config, ref next, RestrictionContext & goal) + : Store{*config} + , LocalFSStore{*config} + , config{config} , next(next) , goal(goal) { @@ -63,7 +54,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In Path getRealStoreDir() override { - return next->realStoreDir; + return next->config->realStoreDir; } std::string getUri() override @@ -176,9 +167,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In } }; -ref makeRestrictedStore(const Store::Params & params, ref next, RestrictionContext & context) +ref makeRestrictedStore(ref config, ref next, RestrictionContext & context) { - return make_ref(params, next, context); + return make_ref(config, next, context); } StorePathSet RestrictedStore::queryAllValidPaths() diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 26c21de44..8d8d82465 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -235,11 +235,6 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } -S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) - : BinaryCacheStoreConfig(params) - , BinaryCacheStore(params) -{ } - S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( std::string_view uriScheme, @@ -258,6 +253,12 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); } + +S3BinaryCacheStore::S3BinaryCacheStore(ref config) + : BinaryCacheStore(*config) + , config{config} +{ } + std::string S3BinaryCacheStoreConfig::doc() { return @@ -266,40 +267,37 @@ std::string S3BinaryCacheStoreConfig::doc() } -struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore +struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore { Stats stats; S3Helper s3Helper; - S3BinaryCacheStoreImpl( - std::string_view uriScheme, - std::string_view bucketName, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , S3BinaryCacheStoreConfig(uriScheme, bucketName, params) - , Store(params) - , BinaryCacheStore(params) - , S3BinaryCacheStore(params) - , s3Helper(profile, region, scheme, endpoint) + S3BinaryCacheStoreImpl(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , S3BinaryCacheStore{config} + , s3Helper(config->profile, config->region, config->scheme, config->endpoint) { diskCache = getNarInfoDiskCache(); + + init(); } std::string getUri() override { - return "s3://" + bucketName; + return "s3://" + config->bucketName; } void init() override { if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { - wantMassQuery.setDefault(cacheInfo->wantMassQuery); - priority.setDefault(cacheInfo->priority); + config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); + config->priority.setDefault(cacheInfo->priority); } else { BinaryCacheStore::init(); - diskCache->createCache(getUri(), storeDir, wantMassQuery, priority); + diskCache->createCache( + getUri(), config->storeDir, config->wantMassQuery, config->priority); } } @@ -328,7 +326,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto res = s3Helper.client->HeadObject( Aws::S3::Model::HeadObjectRequest() - .WithBucket(bucketName) + .WithBucket(config->bucketName) .WithKey(path)); if (!res.IsSuccess()) { @@ -372,7 +370,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual const std::string & mimeType, const std::string & contentEncoding) { - std::string uri = "s3://" + bucketName + "/" + path; + std::string uri = "s3://" + config->bucketName + "/" + path; Activity act(*logger, lvlTalkative, actFileTransfer, fmt("uploading '%s'", uri), Logger::Fields{uri}, getCurActivity()); @@ -387,11 +385,11 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual std::call_once(transferManagerCreated, [&]() { - if (multipartUpload) { + if (config->multipartUpload) { TransferManagerConfiguration transferConfig(executor.get()); transferConfig.s3Client = s3Helper.client; - transferConfig.bufferSize = bufferSize; + transferConfig.bufferSize = config->bufferSize; transferConfig.uploadProgressCallback = [](const TransferManager * transferManager, @@ -421,6 +419,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto now1 = std::chrono::steady_clock::now(); + auto & bucketName = config->bucketName; + if (transferManager) { if (contentEncoding != "") @@ -508,12 +508,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual return std::make_shared(std::move(compressed)); }; - if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) - uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression); - else if (lsCompression != "" && hasSuffix(path, ".ls")) - uploadFile(path, compress(lsCompression), mimeType, lsCompression); - else if (logCompression != "" && hasPrefix(path, "log/")) - uploadFile(path, compress(logCompression), mimeType, logCompression); + if (config->narinfoCompression != "" && hasSuffix(path, ".narinfo")) + uploadFile(path, compress(config->narinfoCompression), mimeType, config->narinfoCompression); + else if (config->lsCompression != "" && hasSuffix(path, ".ls")) + uploadFile(path, compress(config->lsCompression), mimeType, config->lsCompression); + else if (config->logCompression != "" && hasPrefix(path, "log/")) + uploadFile(path, compress(config->logCompression), mimeType, config->logCompression); else uploadFile(path, istream, mimeType, ""); } @@ -523,14 +523,14 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual stats.get++; // FIXME: stream output to sink. - auto res = s3Helper.getObject(bucketName, path); + auto res = s3Helper.getObject(config->bucketName, path); stats.getBytes += res.data ? res.data->size() : 0; stats.getTimeMs += res.durationMs; if (res.data) { printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", - bucketName, path, res.data->size(), res.durationMs); + config->bucketName, path, res.data->size(), res.durationMs); sink(*res.data); } else @@ -542,6 +542,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual StorePathSet paths; std::string marker; + auto & bucketName = config->bucketName; + do { debug("listing bucket 's3://%s' from key '%s'...", bucketName, marker); @@ -580,7 +582,15 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual } }; -static RegisterStoreImplementation regS3BinaryCacheStore; +ref S3BinaryCacheStoreImpl::Config::openStore() const +{ + return make_ref(ref{ + // FIXME we shouldn't actually need a mutable config + std::const_pointer_cast(shared_from_this()) + }); +} + +static RegisterStoreImplementation regS3BinaryCacheStore; } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 45ea05ffc..759a537b9 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -14,12 +14,13 @@ SSHStoreConfig::SSHStoreConfig( std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, authority, params) + : Store::Config{params} + , RemoteStore::Config{params} + , CommonSSHStoreConfig{scheme, authority, params} { } + std::string SSHStoreConfig::doc() { return @@ -27,21 +28,18 @@ std::string SSHStoreConfig::doc() ; } -class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore -{ -public: - SSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(scheme, host, params) - , Store(params) - , RemoteStore(params) - , master(createSSHMaster( +struct SSHStore : virtual RemoteStore +{ + using Config = SSHStoreConfig; + + ref config; + + SSHStore(ref config) + : Store{*config} + , RemoteStore{*config} + , config{config} + , master(config->createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1)) { @@ -49,7 +47,7 @@ public: std::string getUri() override { - return *uriSchemes().begin() + "://" + host; + return *Config::uriSchemes().begin() + "://" + host; } // FIXME extend daemon protocol, move implementation to RemoteStore @@ -101,7 +99,7 @@ MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::strin : StoreConfig(params) , RemoteStoreConfig(params) , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) + , SSHStoreConfig(scheme, host, params) , LocalFSStoreConfig(params) { } @@ -128,35 +126,21 @@ std::string MountedSSHStoreConfig::doc() * The difference lies in how they manage GC roots. See addPermRoot * below for details. */ -class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore +struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore { -public: + using Config = MountedSSHStoreConfig; - MountedSSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) - , MountedSSHStoreConfig(params) - , Store(params) - , RemoteStore(params) - , SSHStore(scheme, host, params) - , LocalFSStore(params) + MountedSSHStore(ref config) + : Store{*config} + , RemoteStore{*config} + , SSHStore{config} + , LocalFSStore{*config} { extraRemoteProgramArgs = { "--process-ops", }; } - std::string getUri() override - { - return *uriSchemes().begin() + "://" + host; - } - void narFromPath(const StorePath & path, Sink & sink) override { return LocalFSStore::narFromPath(path, sink); @@ -198,14 +182,26 @@ public: } }; + +ref SSHStore::Config::openStore() const { + return make_ref(ref{shared_from_this()}); +} + +ref MountedSSHStore::Config::openStore() const { + return make_ref(ref{ + std::dynamic_pointer_cast(shared_from_this()) + }); +} + + ref SSHStore::openConnection() { auto conn = make_ref(); - Strings command = remoteProgram.get(); + Strings command = config->remoteProgram.get(); command.push_back("--stdio"); - if (remoteStore.get() != "") { + if (config->remoteStore.get() != "") { command.push_back("--store"); - command.push_back(remoteStore.get()); + command.push_back(config->remoteStore.get()); } command.insert(command.end(), extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end()); @@ -215,7 +211,7 @@ ref SSHStore::openConnection() return conn; } -static RegisterStoreImplementation regSSHStore; -static RegisterStoreImplementation regMountedSSHStore; +static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7e5ce6e2e..08c542041 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -29,13 +29,13 @@ using json = nlohmann::json; namespace nix { -bool StoreDirConfig::isInStore(PathView path) const +bool MixStoreDirMethods::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair StoreDirConfig::toStorePath(PathView path) const +std::pair MixStoreDirMethods::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); @@ -77,7 +77,7 @@ to match. */ -StorePath StoreDirConfig::makeStorePath(std::string_view type, +StorePath MixStoreDirMethods::makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ @@ -88,14 +88,14 @@ StorePath StoreDirConfig::makeStorePath(std::string_view type, } -StorePath StoreDirConfig::makeStorePath(std::string_view type, +StorePath MixStoreDirMethods::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } -StorePath StoreDirConfig::makeOutputPath(std::string_view id, +StorePath MixStoreDirMethods::makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const { return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id)); @@ -106,7 +106,7 @@ StorePath StoreDirConfig::makeOutputPath(std::string_view id, hacky, but we can't put them in, say, (per the grammar above) since that would be ambiguous. */ static std::string makeType( - const StoreDirConfig & store, + const MixStoreDirMethods & store, std::string && type, const StoreReferences & references) { @@ -119,7 +119,7 @@ static std::string makeType( } -StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const +StorePath MixStoreDirMethods::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1) throw Error("Git file ingestion must use SHA-1 hash"); @@ -141,7 +141,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed } -StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath MixStoreDirMethods::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { @@ -162,7 +162,7 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const } -std::pair StoreDirConfig::computeStorePath( +std::pair MixStoreDirMethods::computeStorePath( std::string_view name, const SourcePath & path, ContentAddressMethod method, @@ -420,7 +420,7 @@ ValidPathInfo Store::addToStoreSlow( return info; } -StringSet StoreConfig::getDefaultSystemFeatures() +StringSet Store::Config::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); @@ -433,9 +433,10 @@ StringSet StoreConfig::getDefaultSystemFeatures() return res; } -Store::Store(const Params & params) - : StoreConfig(params) - , state({(size_t) pathInfoCacheSize}) +Store::Store(const Store::Config & config) + : MixStoreDirMethods{config} + , config{config} + , state({(size_t) config.pathInfoCacheSize}) { assertLibStoreInitialized(); } @@ -1205,7 +1206,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre } -std::string StoreDirConfig::showPaths(const StorePathSet & paths) +std::string MixStoreDirMethods::showPaths(const StorePathSet & paths) const { std::string s; for (auto & i : paths) { @@ -1312,7 +1313,7 @@ void Store::signRealisation(Realisation & realisation) namespace nix { ref openStore(const std::string & uri, - const Store::Params & extraParams) + const Store::Config::Params & extraParams) { return openStore(StoreReference::parse(uri, extraParams)); } @@ -1321,13 +1322,13 @@ ref openStore(StoreReference && storeURI) { auto & params = storeURI.params; - auto store = std::visit(overloaded { - [&](const StoreReference::Auto &) -> std::shared_ptr { + auto storeConfig = std::visit(overloaded { + [&](const StoreReference::Auto &) -> ref { auto stateDir = getOr(params, "state", settings.nixStateDir); if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return std::make_shared(params); + return make_ref(params); else if (pathExists(settings.nixDaemonSocketFile)) - return std::make_shared(params); + return make_ref(params); #ifdef __linux__ else if (!pathExists(stateDir) && params.empty() @@ -1343,31 +1344,33 @@ ref openStore(StoreReference && storeURI) try { createDirs(chrootStore); } catch (SystemError & e) { - return std::make_shared(params); + return make_ref(params); } warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); } else debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - return std::make_shared("local", chrootStore, params); + return make_ref("local", chrootStore, params); } #endif else - return std::make_shared(params); + return make_ref(params); }, [&](const StoreReference::Specified & g) { - for (const auto & implem : Implementations::registered()) + for (const auto & [storeName, implem] : Implementations::registered()) if (implem.uriSchemes.count(g.scheme)) - return implem.create(g.scheme, g.authority, params); + return implem.parseConfig(g.scheme, g.authority, params); throw Error("don't know how to open Nix store with scheme '%s'", g.scheme); }, }, storeURI.variant); - experimentalFeatureSettings.require(store->experimentalFeature()); - store->warnUnknownSettings(); + experimentalFeatureSettings.require(storeConfig->experimentalFeature()); + storeConfig->warnUnknownSettings(); + + auto store = storeConfig->openStore(); store->init(); - return ref { store }; + return store; } std::list> getDefaultSubstituters() @@ -1390,7 +1393,7 @@ std::list> getDefaultSubstituters() addStore(uri); stores.sort([](ref & a, ref & b) { - return a->priority < b->priority; + return a->config.priority < b->config.priority; }); return stores; @@ -1399,9 +1402,9 @@ std::list> getDefaultSubstituters() return stores; } -std::vector & Implementations::registered() +Implementations::Map & Implementations::registered() { - static std::vector registered; + static Map registered; return registered; } diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc new file mode 100644 index 000000000..191926be6 --- /dev/null +++ b/src/libstore/store-dir-config.cc @@ -0,0 +1,13 @@ +#include "nix/store/store-dir-config.hh" +#include "nix/util/util.hh" +#include "nix/store/globals.hh" + +namespace nix { + +StoreDirConfig::StoreDirConfig(const Params & params) + : StoreDirConfigItself(params) + , MixStoreDirMethods{storeDir_} +{ +} + +} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 3c1657d15..480bc5c8a 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -20,13 +20,13 @@ namespace nix { UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) + const StoreReference::Params & params) + : Store::Config{params} + , LocalFSStore::Config{params} + , RemoteStore::Config{params} , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { - if (scheme != UDSRemoteStoreConfig::scheme) { + if (uriSchemes().count(scheme) == 0) { throw UsageError("Scheme must be 'unix'"); } } @@ -44,32 +44,30 @@ std::string UDSRemoteStoreConfig::doc() // empty string will later default to the same nixDaemonSocketFile. Why // don't we just wire it all through? I believe there are cases where it // will live reload so we want to continue to account for that. -UDSRemoteStore::UDSRemoteStore(const Params & params) - : UDSRemoteStore(scheme, "", params) -{} +UDSRemoteStoreConfig::UDSRemoteStoreConfig(const Params & params) + : UDSRemoteStoreConfig(*uriSchemes().begin(), "", params) +{ +} -UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - , UDSRemoteStoreConfig(scheme, authority, params) - , Store(params) - , LocalFSStore(params) - , RemoteStore(params) +UDSRemoteStore::UDSRemoteStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , RemoteStore{*config} + , config{config} { } std::string UDSRemoteStore::getUri() { - return path == settings.nixDaemonSocketFile + return config->path == settings.nixDaemonSocketFile ? // FIXME: Not clear why we return daemon here and not default // to settings.nixDaemonSocketFile // // unix:// with no path also works. Change what we return? "daemon" - : std::string(scheme) + "://" + path; + : std::string(*Config::uriSchemes().begin()) + "://" + config->path; } @@ -86,7 +84,7 @@ ref UDSRemoteStore::openConnection() /* Connect to a daemon that does the privileged work for us. */ conn->fd = createUnixDomainSocket(); - nix::connect(toSocket(conn->fd.get()), path); + nix::connect(toSocket(conn->fd.get()), config->path); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); @@ -106,6 +104,11 @@ void UDSRemoteStore::addIndirectRoot(const Path & path) } -static RegisterStoreImplementation regUDSRemoteStore; +ref UDSRemoteStore::Config::openStore() const { + return make_ref(ref{shared_from_this()}); +} + + +static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index d8a539fe2..58e8d8ba6 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -36,7 +36,7 @@ #include "store-config-private.hh" #if HAVE_STATVFS -#include +# include #endif /* Includes required for chroot support. */ @@ -60,9 +60,9 @@ #endif #ifdef __APPLE__ -#include -#include -#include +# include +# include +# include /* This definition is undocumented but depended upon by all major browsers. */ extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf); @@ -494,7 +494,7 @@ bool DerivationBuilderImpl::prepareBuild() } auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir.get()) { + if (localStore.storeDir != localStore.config->realStoreDir.get()) { #ifdef __linux__ useChroot = true; #else @@ -707,7 +707,7 @@ bool DerivationBuilderImpl::cleanupDecideWhetherDiskFull() auto & localStore = getLocalStore(); uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; - if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && + if (statvfs(localStore.config->realStoreDir.get().c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && @@ -871,7 +871,7 @@ void DerivationBuilderImpl::startBuilder() concatStringsSep(", ", drvOptions.getRequiredSystemFeatures(drv)), store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep(", ", store.systemFeatures)); + concatStringsSep(", ", store.config.systemFeatures)); } } @@ -1594,14 +1594,14 @@ void DerivationBuilderImpl::startDaemon() { experimentalFeatureSettings.require(Xp::RecursiveNix); - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = store.storeDir; - if (auto & optRoot = getLocalStore().rootDir.get()) - params["root"] = *optRoot; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = makeRestrictedStore(params, + auto store = makeRestrictedStore( + [&]{ + auto config = make_ref(*getLocalStore().config); + config->pathInfoCacheSize = 0; + config->stateDir = "/no-such-path"; + config->logDir = "/no-such-path"; + return config; + }(), ref(std::dynamic_pointer_cast(this->store.shared_from_this())), *this); @@ -1946,7 +1946,7 @@ void DerivationBuilderImpl::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + if (store.config.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); ss.push_back("/dev/null"); ss.push_back("/dev/random"); diff --git a/src/nix/main.cc b/src/nix/main.cc index 351fcd1b6..4a0fa6632 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -193,13 +193,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs res["args"] = toJSON(); auto stores = nlohmann::json::object(); - for (auto & implem : Implementations::registered()) { - auto storeConfig = implem.getConfig(); - auto storeName = storeConfig->name(); + for (auto & [storeName, implem] : Implementations::registered()) { auto & j = stores[storeName]; - j["doc"] = storeConfig->doc(); - j["settings"] = storeConfig->toJSON(); - j["experimentalFeature"] = storeConfig->experimentalFeature(); + j["doc"] = implem.doc; + j["uri-schemes"] = implem.uriSchemes; + j["settings"] = implem.getConfig()->toJSON(); + j["experimentalFeature"] = implem.experimentalFeature; } res["stores"] = std::move(stores); res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 607a7bb01..95e84ee72 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -244,7 +244,7 @@ static PeerInfo getPeerInfo(int remote) */ static ref openUncachedStore() { - Store::Params params; // FIXME: get params from somewhere + Store::Config::Params params; // FIXME: get params from somewhere // Disable caching since the client already does that. params["path-info-cache-size"] = "0"; return openStore(settings.storeUri, params); From 20a724d131cb6f53b13ba00ab280d14066a1a4c9 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 13 May 2025 22:20:11 +0000 Subject: [PATCH 323/396] docs: Fix miscellaneous typos and formatting issues --- .../source/advanced-topics/distributed-builds.md | 2 +- doc/manual/source/architecture/architecture.md | 6 +++--- doc/manual/source/language/advanced-attributes.md | 10 +++++----- doc/manual/source/release-notes/rl-2.6.md | 2 +- doc/manual/source/store/derivation/outputs/index.md | 2 +- src/libstore/local-overlay-store.md | 2 +- src/nix/flake-lock.md | 2 +- src/nix/flake.md | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/manual/source/advanced-topics/distributed-builds.md b/doc/manual/source/advanced-topics/distributed-builds.md index 464b87d6e..08a980643 100644 --- a/doc/manual/source/advanced-topics/distributed-builds.md +++ b/doc/manual/source/advanced-topics/distributed-builds.md @@ -27,7 +27,7 @@ nix store info --store ssh://username@mac ``` To specify an SSH identity file as part of the remote store URI add a -query paramater, e.g. +query parameter, e.g. ```console nix store info --store ssh://username@mac?ssh-key=/home/alice/my-key diff --git a/doc/manual/source/architecture/architecture.md b/doc/manual/source/architecture/architecture.md index cbc469355..0c8677a6a 100644 --- a/doc/manual/source/architecture/architecture.md +++ b/doc/manual/source/architecture/architecture.md @@ -22,9 +22,9 @@ The following [concept map] shows its main components (rectangles), the objects | | +----------|-------------------|--------------------------------+ | Nix | V | -| | +-------------------------+ | -| | | commmand line interface |------. | -| | +-------------------------+ | | +| | +------------------------+ | +| | | command line interface |------. | +| | +------------------------+ | | | | | | | | evaluated by calls manages | | | | | | diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index bf196e0b8..dc00e3080 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -280,7 +280,7 @@ All other combinations are invalid. Significant changes should add the following header, which moves them to the top. diff --git a/doc/manual/substitute.py b/doc/manual/substitute.py index a8b11d932..6e27c3388 100644 --- a/doc/manual/substitute.py +++ b/doc/manual/substitute.py @@ -57,6 +57,9 @@ def recursive_replace(data: dict[str, t.Any], book_root: Path, search_path: Path ).replace( '@docroot@', ("../" * len(path_to_chapter.parent.parts) or "./")[:-1] + ).replace( + '@_at_', + '@' ), sub_items = [ recursive_replace(sub_item, book_root, search_path) From f50117ba4c2f2441c526c69c19119c2bcfe5e922 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 May 2025 10:01:57 +0200 Subject: [PATCH 371/396] Revert storeFS to use makeFSSourceAccessor() Need to investigate why store->getFSAccessor() breaks a test. --- src/libexpr/eval.cc | 2 +- tests/functional/flakes/follow-paths.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 531a932bd..868933b95 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -270,7 +270,7 @@ EvalState::EvalState( exception, and make union source accessor catch it, so we don't need to do this hack. */ - {CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)}, + {CanonPath(store->storeDir), makeFSSourceAccessor(dirOf(store->toRealPath(StorePath::dummy)))} })) , rootFS( ({ diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 8abbf3233..25f26137b 100755 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -131,7 +131,7 @@ EOF git -C $flakeFollowsA add flake.nix expect 1 nix flake lock $flakeFollowsA 2>&1 | grep '/flakeB.*is forbidden in pure evaluation mode' -#expect 1 nix flake lock --impure $flakeFollowsA 2>&1 | grep '/flakeB.*does not exist' # FIXME +expect 1 nix flake lock --impure $flakeFollowsA 2>&1 | grep '/flakeB.*does not exist' # FIXME # Test relative non-flake inputs. cat > $flakeFollowsA/flake.nix < Date: Sun, 18 May 2025 13:10:08 +0200 Subject: [PATCH 372/396] Restore the hash mismatch activity --- src/libstore/build/derivation-goal.cc | 1 + src/libstore/unix/build/derivation-builder.cc | 2 -- .../unix/include/nix/store/build/derivation-builder.hh | 9 ++++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 81215eacf..850d21bca 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -872,6 +872,7 @@ Goal::Co DerivationGoal::tryToBuild() *drvOptions, inputPaths, initialOutputs, + act }); } diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index abfe9b2b1..688f4311e 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -2709,14 +2709,12 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() store.printStorePath(drvPath), wanted.to_string(HashFormat::SRI, true), got.to_string(HashFormat::SRI, true))); -#if 0 // FIXME act->result(resHashMismatch, { {"storePath", store.printStorePath(drvPath)}, {"wanted", wanted}, {"got", got}, }); -#endif } if (!newInfo0.references.empty()) { auto numViolations = newInfo.references.size(); diff --git a/src/libstore/unix/include/nix/store/build/derivation-builder.hh b/src/libstore/unix/include/nix/store/build/derivation-builder.hh index d6c40060a..81a574fd0 100644 --- a/src/libstore/unix/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/unix/include/nix/store/build/derivation-builder.hh @@ -58,6 +58,11 @@ struct DerivationBuilderParams const BuildMode & buildMode; + /** + * The activity corresponding to the build. + */ + std::unique_ptr & act; + DerivationBuilderParams( const StorePath & drvPath, const BuildMode & buildMode, @@ -66,7 +71,8 @@ struct DerivationBuilderParams const StructuredAttrs * parsedDrv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths, - std::map & initialOutputs) + std::map & initialOutputs, + std::unique_ptr & act) : drvPath{drvPath} , buildResult{buildResult} , drv{drv} @@ -75,6 +81,7 @@ struct DerivationBuilderParams , inputPaths{inputPaths} , initialOutputs{initialOutputs} , buildMode{buildMode} + , act{act} { } DerivationBuilderParams(DerivationBuilderParams &&) = default; From b33fd1e4fb9c28d0b67a2a80819d69e88c442d8f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 6 May 2025 21:58:52 +0000 Subject: [PATCH 373/396] libstore: Use `boost::regex` for GC root discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As it turns out using `std::regex` is actually the bottleneck for root discovery. Just substituting `std::` -> `boost::` makes root discovery twice as fast (3x if counting only userspace time). Some rather ad-hoc measurements to motivate the switch: (On master) ``` nix build github:nixos/nix/1e822bd4149a8bce1da81ee2ad9404986b07914c#nix-cli --out-link result-1e822bd4149a8bce1da81ee2ad9404986b07914c taskset -c 2,3 hyperfine "result-1e822bd4149a8bce1da81ee2ad9404986b07914c/bin/nix store gc --dry-run --max 0" Benchmark 1: result-1e822bd4149a8bce1da81ee2ad9404986b07914c/bin/nix store gc --dry-run --max 0 Time (mean ± σ): 481.6 ms ± 3.9 ms [User: 336.2 ms, System: 142.0 ms] Range (min … max): 474.6 ms … 487.7 ms 10 runs ``` (After this patch) ``` taskset -c 2,3 hyperfine "result/bin/nix store gc --dry-run --max 0" Benchmark 1: result/bin/nix store gc --dry-run --max 0 Time (mean ± σ): 254.7 ms ± 9.7 ms [User: 111.1 ms, System: 141.3 ms] Range (min … max): 246.5 ms … 281.3 ms 10 runs ``` `boost::regex` is a drop-in replacement for `std::regex`, but much faster. Doing a simple before/after comparison doesn't surface any change in behavior: ``` result/bin/nix store gc --dry-run -vvvvv --max 0 |& grep "got additional" | wc -l result-1e822bd4149a8bce1da81ee2ad9404986b07914c/bin/nix store gc --dry-run -vvvvv --max 0 |& grep "got additional" | wc -l ``` (cherry picked from commit 3a1301cd6db698a212a0c036e40ad402bd8a2a12) --- src/libstore/gc.cc | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 8fad9661c..1469db3ec 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -13,10 +13,11 @@ # include "nix/util/processes.hh" #endif +#include + #include #include #include -#include #include #include @@ -331,8 +332,8 @@ static void readProcLink(const std::filesystem::path & file, UncheckedRoots & ro static std::string quoteRegexChars(const std::string & raw) { - static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])"); - return std::regex_replace(raw, specialRegex, R"(\$&)"); + static auto specialRegex = boost::regex(R"([.^$\\*+?()\[\]{}|])"); + return boost::regex_replace(raw, specialRegex, R"(\$&)"); } #ifdef __linux__ @@ -354,12 +355,12 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) auto procDir = AutoCloseDir{opendir("/proc")}; if (procDir) { struct dirent * ent; - auto digitsRegex = std::regex(R"(^\d+$)"); - auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); - auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); + static const auto digitsRegex = boost::regex(R"(^\d+$)"); + static const auto mapRegex = boost::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); + auto storePathRegex = boost::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); - if (std::regex_match(ent->d_name, digitsRegex)) { + if (boost::regex_match(ent->d_name, digitsRegex)) { try { readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); @@ -386,15 +387,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) std::filesystem::path mapFile = fmt("/proc/%s/maps", ent->d_name); auto mapLines = tokenizeString>(readFile(mapFile.string()), "\n"); for (const auto & line : mapLines) { - auto match = std::smatch{}; - if (std::regex_match(line, match, mapRegex)) + auto match = boost::smatch{}; + if (boost::regex_match(line, match, mapRegex)) unchecked[match[1]].emplace(mapFile.string()); } auto envFile = fmt("/proc/%s/environ", ent->d_name); auto envString = readFile(envFile); - auto env_end = std::sregex_iterator{}; - for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) + auto env_end = boost::sregex_iterator{}; + for (auto i = boost::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) unchecked[i->str()].emplace(envFile); } catch (SystemError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) @@ -413,12 +414,12 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) // Because of this we disable lsof when running the tests. if (getEnv("_NIX_TEST_NO_LSOF") != "1") { try { - std::regex lsofRegex(R"(^n(/.*)$)"); + boost::regex lsofRegex(R"(^n(/.*)$)"); auto lsofLines = tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); for (const auto & line : lsofLines) { - std::smatch match; - if (std::regex_match(line, match, lsofRegex)) + boost::smatch match; + if (boost::regex_match(line, match, lsofRegex)) unchecked[match[1].str()].emplace("{lsof}"); } } catch (ExecError & e) { From 91dc6e7fa0fba0b8b875b135c7904ecc3423ad9a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 13 May 2025 08:47:24 +0000 Subject: [PATCH 374/396] packaging/dependencies: Use boost without enableIcu This reduces the closure size on master by 40MiB. ``` $ nix build github:nixos/nix/1e822bd4149a8bce1da81ee2ad9404986b07914c#nix-store --out-link closure-on-master $ nix build .#nix-store -L --out-link closure-without-icu $ nix path-info --closure-size -h ./closure-on-master /nix/store/8gwr38m5h6p7245ji9jv28a2a11w1isx-nix-store-2.29.0pre 124.4 MiB $ nix path-info --closure-size -h ./closure-without-icu /nix/store/k0gwfykjqpnmaqbwh23nk55lhanc9g24-nix-store-2.29.0pre 86.6 MiB ``` (cherry picked from commit f3090ef7033c9bdc04beacfbb128c688cfa40fee) --- packaging/dependencies.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index a90ef1b4a..7ce3bf125 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -63,6 +63,7 @@ scope: { "--with-coroutine" "--with-iostreams" ]; + enableIcu = false; }).overrideAttrs (old: { # Need to remove `--with-*` to use `--with-libraries=...` From 29d98da6363aa8c6a796550ed27618b1a25dcf75 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 13 May 2025 08:51:46 +0000 Subject: [PATCH 375/396] libstore: Depend on boost_regex explicitly (cherry picked from commit 18a5589f9a6d710fe1f70e694cee513589c1c11c) --- src/libstore/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 9681a38ab..672993bf0 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -94,7 +94,7 @@ subdir('nix-meson-build-support/libatomic') boost = dependency( 'boost', - modules : ['container'], + modules : ['container', 'regex'], include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we From 90eb2f759c76ce538b2eed676a5648edeba751c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 19 May 2025 09:25:34 +0200 Subject: [PATCH 376/396] libutil-tests/json-utils: fix -Werror=sign-compare error I am on a newer different nixpkgs branch, so I am getting this error (cherry picked from commit 1290b7e53d03cc8b084aaa8e58baff177711ccb0) --- src/libutil-tests/json-utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil-tests/json-utils.cc b/src/libutil-tests/json-utils.cc index eae67b4b3..211f8bf1e 100644 --- a/src/libutil-tests/json-utils.cc +++ b/src/libutil-tests/json-utils.cc @@ -131,7 +131,7 @@ TEST(getString, wrongAssertions) { TEST(getIntegralNumber, rightAssertions) { auto simple = R"({ "int": 0, "signed": -1 })"_json; - ASSERT_EQ(getUnsigned(valueAt(getObject(simple), "int")), 0); + ASSERT_EQ(getUnsigned(valueAt(getObject(simple), "int")), 0u); ASSERT_EQ(getInteger(valueAt(getObject(simple), "int")), 0); ASSERT_EQ(getInteger(valueAt(getObject(simple), "signed")), -1); } From 8825cd56b5ed294091b6ed4abe94d44df2fe7f5d Mon Sep 17 00:00:00 2001 From: gustavderdrache Date: Tue, 20 May 2025 13:46:19 -0400 Subject: [PATCH 377/396] Log warnings on IFD with new option --- src/libexpr/include/nix/expr/eval-settings.hh | 10 ++++++++++ src/libexpr/primops.cc | 18 +++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index 6e5bbca20..3ad2e9d2d 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -151,6 +151,16 @@ struct EvalSettings : Config )" }; + Setting traceImportFromDerivation{ + this, false, "trace-import-from-derivation", + R"( + By default, Nix allows [Import from Derivation](@docroot@/language/import-from-derivation.md). + + When this setting is `true`, Nix will log a warning indicating that it performed such an import. + The `allow-import-from-derivation` setting takes precedence, and no warnings will be logged if that setting is also enabled. + )" + }; + Setting enableImportFromDerivation{ this, true, "allow-import-from-derivation", R"( diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 44f7833e0..586952386 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -97,11 +97,19 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (drvs.empty()) return {}; - if (isIFD && !settings.enableImportFromDerivation) - error( - "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", - drvs.begin()->to_string(*store) - ).debugThrow(); + if (isIFD) { + if (!settings.enableImportFromDerivation) + error( + "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", + drvs.begin()->to_string(*store) + ).debugThrow(); + + if (settings.traceImportFromDerivation) + warn( + "built '%1%' during evaluation due to an import from derivation", + drvs.begin()->to_string(*store) + ); + } /* Build/substitute the context. */ std::vector buildReqs; From 4355b7cbd5664364433abc64607b784fbe8c7979 Mon Sep 17 00:00:00 2001 From: gustavderdrache Date: Wed, 21 May 2025 11:11:09 -0400 Subject: [PATCH 378/396] Add test for output warning to ensure stability --- tests/functional/flakes/meson.build | 1 + tests/functional/flakes/trace-ifd.sh | 33 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/functional/flakes/trace-ifd.sh diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build index 213c388a6..801fefc6f 100644 --- a/tests/functional/flakes/meson.build +++ b/tests/functional/flakes/meson.build @@ -33,6 +33,7 @@ suites += { 'debugger.sh', 'source-paths.sh', 'old-lockfiles.sh', + 'trace-ifd.sh', ], 'workdir': meson.current_source_dir(), } diff --git a/tests/functional/flakes/trace-ifd.sh b/tests/functional/flakes/trace-ifd.sh new file mode 100644 index 000000000..f5c54f651 --- /dev/null +++ b/tests/functional/flakes/trace-ifd.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +source ./common.sh + +requireGit + +flake1Dir="$TEST_ROOT/flake" + +createGitRepo "$flake1Dir" +createSimpleGitFlake "$flake1Dir" + +cat > "$flake1Dir/flake.nix" <<'EOF' +{ + outputs = { self }: let inherit (import ./config.nix) mkDerivation; in { + drv = mkDerivation { + name = "drv"; + buildCommand = '' + echo drv >$out + ''; + }; + + ifd = mkDerivation { + name = "ifd"; + buildCommand = '' + echo ${builtins.readFile self.drv} >$out + ''; + }; + }; +} +EOF + +nix build "$flake1Dir#ifd" --option trace-import-from-derivation true 2>&1 \ + | grepQuiet 'warning: built .* during evaluation due to an import from derivation' From 0b66fd3c34f7a5f07629b7c4bc68d8bae9f0ad06 Mon Sep 17 00:00:00 2001 From: gustavderdrache Date: Thu, 22 May 2025 15:28:02 -0400 Subject: [PATCH 379/396] Update src/libexpr/include/nix/expr/eval-settings.hh Co-authored-by: Eelco Dolstra --- src/libexpr/include/nix/expr/eval-settings.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index 3ad2e9d2d..6a58377e1 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -157,7 +157,7 @@ struct EvalSettings : Config By default, Nix allows [Import from Derivation](@docroot@/language/import-from-derivation.md). When this setting is `true`, Nix will log a warning indicating that it performed such an import. - The `allow-import-from-derivation` setting takes precedence, and no warnings will be logged if that setting is also enabled. + This option has no effect if `allow-import-from-derivation` is disabled. )" }; From 90cb816511d7f358bdf6acb83d8911b7a4e1d1cf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 May 2025 23:28:56 +0000 Subject: [PATCH 380/396] Prepare release v3.6.0 From a43997cce4078f919f26d619291aee37cf9cb0b1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 May 2025 23:28:59 +0000 Subject: [PATCH 381/396] Set .version-determinate to 3.6.0 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index 87ce49290..40c341bdc 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.5.2 +3.6.0 From e5e7c2797c03732164ac84869b0c1aa1ccd77862 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 May 2025 23:29:04 +0000 Subject: [PATCH 382/396] Generare release notes for 3.6.0 --- doc/manual/source/SUMMARY.md.in | 1 + .../source/release-notes-determinate/changes.md | 13 +++++++++++-- .../source/release-notes-determinate/rl-3.6.0.md | 12 ++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.0.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 45b56438f..4a792c5df 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -129,6 +129,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.6.0 (2025-05-22)](release-notes-determinate/rl-3.6.0.md) - [Release 3.5.2 (2025-05-12)](release-notes-determinate/rl-3.5.2.md) - [Release 3.5.1 (2025-05-09)](release-notes-determinate/rl-3.5.1.md) - [~~Release 3.5.0 (2025-05-09)~~](release-notes-determinate/rl-3.5.0.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 757fcbbb0..5a6d51833 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.28 and Determinate Nix 3.5.2. +This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.0. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -28,4 +28,13 @@ This section lists the differences between upstream Nix 2.28 and Determinate Nix -* Tell users a source is corrupted ("cannot read file from tarball: Truncated tar archive detected while reading data"), improving over the previous 'cannot read file from tarball' error by @edolstra in [DeterminateSystems/nix-src#64](https://github.com/DeterminateSystems/nix-src/pull/64) \ No newline at end of file +* Tell users a source is corrupted ("cannot read file from tarball: Truncated tar archive detected while reading data"), improving over the previous 'cannot read file from tarball' error by @edolstra in [DeterminateSystems/nix-src#64](https://github.com/DeterminateSystems/nix-src/pull/64) + + +* Switch to determinate-nix-action by @lucperkins in [DeterminateSystems/nix-src#68](https://github.com/DeterminateSystems/nix-src/pull/68) + +* Install 'nix profile add' manpage by @edolstra in [DeterminateSystems/nix-src#69](https://github.com/DeterminateSystems/nix-src/pull/69) + +* Sync with upstream 2.29.0 by @edolstra in [DeterminateSystems/nix-src#67](https://github.com/DeterminateSystems/nix-src/pull/67) + +* Emit warnings when using import-from-derivation by setting the `trace-import-from-derivation` option to `true` by @gustavderdrache in [DeterminateSystems/nix-src#70](https://github.com/DeterminateSystems/nix-src/pull/70) \ No newline at end of file diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.0.md b/doc/manual/source/release-notes-determinate/rl-3.6.0.md new file mode 100644 index 000000000..61cd0232c --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.0.md @@ -0,0 +1,12 @@ +# Release 3.6.0 (2025-05-22) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Switch to determinate-nix-action by @lucperkins in [DeterminateSystems/nix-src#68](https://github.com/DeterminateSystems/nix-src/pull/68) +* Install 'nix profile add' manpage by @edolstra in [DeterminateSystems/nix-src#69](https://github.com/DeterminateSystems/nix-src/pull/69) +* Sync with upstream 2.29.0 by @edolstra in [DeterminateSystems/nix-src#67](https://github.com/DeterminateSystems/nix-src/pull/67) +* Emit warnings when using import-from-derivation by setting the `trace-import-from-derivation` option to `true` by @gustavderdrache in [DeterminateSystems/nix-src#70](https://github.com/DeterminateSystems/nix-src/pull/70) + + +**Full Changelog**: [v3.5.2...v3.6.0](https://github.com/DeterminateSystems/nix-src/compare/v3.5.2...v3.6.0) From 486fca34bcc97cf2f3070772e4f381c03d6782b8 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 22 May 2025 19:36:08 -0400 Subject: [PATCH 383/396] Apply suggestions from code review --- doc/manual/source/release-notes-determinate/changes.md | 6 ------ doc/manual/source/release-notes-determinate/rl-3.6.0.md | 1 - 2 files changed, 7 deletions(-) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 5a6d51833..26538effb 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -31,10 +31,4 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Tell users a source is corrupted ("cannot read file from tarball: Truncated tar archive detected while reading data"), improving over the previous 'cannot read file from tarball' error by @edolstra in [DeterminateSystems/nix-src#64](https://github.com/DeterminateSystems/nix-src/pull/64) -* Switch to determinate-nix-action by @lucperkins in [DeterminateSystems/nix-src#68](https://github.com/DeterminateSystems/nix-src/pull/68) - -* Install 'nix profile add' manpage by @edolstra in [DeterminateSystems/nix-src#69](https://github.com/DeterminateSystems/nix-src/pull/69) - -* Sync with upstream 2.29.0 by @edolstra in [DeterminateSystems/nix-src#67](https://github.com/DeterminateSystems/nix-src/pull/67) - * Emit warnings when using import-from-derivation by setting the `trace-import-from-derivation` option to `true` by @gustavderdrache in [DeterminateSystems/nix-src#70](https://github.com/DeterminateSystems/nix-src/pull/70) \ No newline at end of file diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.0.md b/doc/manual/source/release-notes-determinate/rl-3.6.0.md index 61cd0232c..453ab6c30 100644 --- a/doc/manual/source/release-notes-determinate/rl-3.6.0.md +++ b/doc/manual/source/release-notes-determinate/rl-3.6.0.md @@ -3,7 +3,6 @@ * Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). ## What's Changed -* Switch to determinate-nix-action by @lucperkins in [DeterminateSystems/nix-src#68](https://github.com/DeterminateSystems/nix-src/pull/68) * Install 'nix profile add' manpage by @edolstra in [DeterminateSystems/nix-src#69](https://github.com/DeterminateSystems/nix-src/pull/69) * Sync with upstream 2.29.0 by @edolstra in [DeterminateSystems/nix-src#67](https://github.com/DeterminateSystems/nix-src/pull/67) * Emit warnings when using import-from-derivation by setting the `trace-import-from-derivation` option to `true` by @gustavderdrache in [DeterminateSystems/nix-src#70](https://github.com/DeterminateSystems/nix-src/pull/70) From 61c3efb4f44bee31e4acdecd8168e034da069995 Mon Sep 17 00:00:00 2001 From: gustavderdrache Date: Fri, 23 May 2025 16:49:23 -0400 Subject: [PATCH 384/396] Make platform checks throw BuildError like other failures --- src/libstore/unix/build/derivation-builder.cc | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 688f4311e..d4862108c 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -862,17 +862,22 @@ void DerivationBuilderImpl::startBuilder() /* Right platform? */ if (!drvOptions.canBuildLocally(store, drv)) { + auto msg = fmt( + "Cannot build '%s'.\n" + "Reason: " ANSI_RED "unmet system or feature dependency" ANSI_NORMAL "\n" + "Required system: '%s' with features {%s}\n" + "Current system: '%s' with features {%s}", + Magenta(store.printStorePath(drvPath)), + Magenta(drv.platform), + concatStringsSep(", ", drvOptions.getRequiredSystemFeatures(drv)), + Magenta(settings.thisSystem), + concatStringsSep(", ", store.config.systemFeatures)); + // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - we should tell them to run the command to install Darwin 2 - if (drv.platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") { - throw Error("run `/usr/sbin/softwareupdate --install-rosetta` to enable your %s to run programs for %s", settings.thisSystem, drv.platform); - } else { - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv.platform, - concatStringsSep(", ", drvOptions.getRequiredSystemFeatures(drv)), - store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", store.config.systemFeatures)); - } + if (drv.platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") + msg += fmt("\nNote: run `%s` to run programs for x86_64-darwin", Magenta("/usr/sbin/softwareupdate --install-rosetta")); + + throw BuildError(msg); } /* Create a temporary directory where the build will take From 09d46ad93a197030c56d5793cde907100f4cbd81 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 23 May 2025 23:33:59 +0200 Subject: [PATCH 385/396] Don't use 'callback' object that we may have moved out of --- src/libstore/http-binary-cache-store.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 2b591dda9..e44d146b9 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -176,13 +176,13 @@ protected: void getFile(const std::string & path, Callback> callback) noexcept override { + auto callbackPtr = std::make_shared(std::move(callback)); + try { checkEnabled(); auto request(makeRequest(path)); - auto callbackPtr = std::make_shared(std::move(callback)); - getFileTransfer()->enqueueFileTransfer(request, {[callbackPtr, this](std::future result) { try { @@ -198,7 +198,7 @@ protected: }}); } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); return; } } From af7bfe7827da0467b1432b76d9b93f5c50149f6d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 24 May 2025 00:14:32 +0200 Subject: [PATCH 386/396] fromStructuredAttrs(): Don't crash if exportReferencesGraph is a string Fixes error: [json.exception.type_error.302] type must be array, but is string and other crashes. Fixes #13254. --- src/libstore/derivation-options.cc | 9 +++++++-- src/libstore/misc.cc | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index e031f8447..f6bac2868 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -211,8 +211,13 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt auto e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph"); if (!e || !e->is_object()) return ret; - for (auto & [key, storePathsJson] : getObject(*e)) { - ret.insert_or_assign(key, storePathsJson); + for (auto & [key, value] : getObject(*e)) { + if (value.is_array()) + ret.insert_or_assign(key, value); + else if (value.is_string()) + ret.insert_or_assign(key, StringSet{value}); + else + throw Error("'exportReferencesGraph' value is not an array or a string"); } } else { auto s = getOr(env, "exportReferencesGraph", ""); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 967c91d72..dabae647f 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -225,6 +225,8 @@ void Store::queryMissing(const std::vector & targets, auto parsedDrv = StructuredAttrs::tryParse(drv->env); DerivationOptions drvOptions; try { + // FIXME: this is a lot of work just to get the value + // of `allowSubstitutes`. drvOptions = DerivationOptions::fromStructuredAttrs( drv->env, parsedDrv ? &*parsedDrv : nullptr); From 4bc1043ae466ab40cad6cbe89a6fe0f8e45e0bba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 24 May 2025 00:40:06 +0200 Subject: [PATCH 387/396] Add test --- tests/functional/structured-attrs-shell.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/structured-attrs-shell.nix b/tests/functional/structured-attrs-shell.nix index a819e39cd..e9b9f1e39 100644 --- a/tests/functional/structured-attrs-shell.nix +++ b/tests/functional/structured-attrs-shell.nix @@ -21,7 +21,7 @@ mkDerivation { "b" "c" ]; - exportReferencesGraph.refs = [ dep ]; + exportReferencesGraph.refs = dep; buildCommand = '' touch ''${outputs[out]}; touch ''${outputs[dev]} ''; From 562ed80bb7f4619adb640e2195ab1271c4542cb4 Mon Sep 17 00:00:00 2001 From: gustavderdrache Date: Fri, 23 May 2025 18:58:37 -0400 Subject: [PATCH 388/396] Update src/libstore/unix/build/derivation-builder.cc Co-authored-by: Cole Helbling --- src/libstore/unix/build/derivation-builder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index d4862108c..0ef18966c 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -864,7 +864,7 @@ void DerivationBuilderImpl::startBuilder() if (!drvOptions.canBuildLocally(store, drv)) { auto msg = fmt( "Cannot build '%s'.\n" - "Reason: " ANSI_RED "unmet system or feature dependency" ANSI_NORMAL "\n" + "Reason: " ANSI_RED "required system or feature not available" ANSI_NORMAL "\n" "Required system: '%s' with features {%s}\n" "Current system: '%s' with features {%s}", Magenta(store.printStorePath(drvPath)), From 8e4f7984d196265cdc5513a2b02d31a4cfd78e8f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 24 May 2025 02:11:02 +0000 Subject: [PATCH 389/396] Prepare release v3.6.1 From 20a79d9a73ae55c15eae37e2735a875ad422ce67 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 24 May 2025 02:11:05 +0000 Subject: [PATCH 390/396] Set .version-determinate to 3.6.1 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index 40c341bdc..9575d51ba 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.6.0 +3.6.1 From 3e0433b65dd674d4f30b1ecbe89d012db87eafc4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 24 May 2025 02:11:10 +0000 Subject: [PATCH 391/396] Generare release notes for 3.6.1 --- doc/manual/source/SUMMARY.md.in | 1 + doc/manual/source/release-notes-determinate/changes.md | 7 +++++-- doc/manual/source/release-notes-determinate/rl-3.6.1.md | 9 +++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.1.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 4a792c5df..addcd106b 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -129,6 +129,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.6.1 (2025-05-24)](release-notes-determinate/rl-3.6.1.md) - [Release 3.6.0 (2025-05-22)](release-notes-determinate/rl-3.6.0.md) - [Release 3.5.2 (2025-05-12)](release-notes-determinate/rl-3.5.2.md) - [Release 3.5.1 (2025-05-09)](release-notes-determinate/rl-3.5.1.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 26538effb..5323b3150 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.0. +This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.1. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -31,4 +31,7 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Tell users a source is corrupted ("cannot read file from tarball: Truncated tar archive detected while reading data"), improving over the previous 'cannot read file from tarball' error by @edolstra in [DeterminateSystems/nix-src#64](https://github.com/DeterminateSystems/nix-src/pull/64) -* Emit warnings when using import-from-derivation by setting the `trace-import-from-derivation` option to `true` by @gustavderdrache in [DeterminateSystems/nix-src#70](https://github.com/DeterminateSystems/nix-src/pull/70) \ No newline at end of file +* Emit warnings when using import-from-derivation by setting the `trace-import-from-derivation` option to `true` by @gustavderdrache in [DeterminateSystems/nix-src#70](https://github.com/DeterminateSystems/nix-src/pull/70) + + +* Fix nlohmann error in fromStructuredAttrs() by @edolstra in [DeterminateSystems/nix-src#73](https://github.com/DeterminateSystems/nix-src/pull/73) \ No newline at end of file diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.1.md b/doc/manual/source/release-notes-determinate/rl-3.6.1.md new file mode 100644 index 000000000..12505afee --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.1.md @@ -0,0 +1,9 @@ +# Release 3.6.1 (2025-05-24) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Fix nlohmann error in fromStructuredAttrs() by @edolstra in [DeterminateSystems/nix-src#73](https://github.com/DeterminateSystems/nix-src/pull/73) + + +**Full Changelog**: [v3.6.0...v3.6.1](https://github.com/DeterminateSystems/nix-src/compare/v3.6.0...v3.6.1) From 5f13d13f78e74f8cf70a95b5e2dabfde0a3b8906 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 May 2025 13:56:38 +0200 Subject: [PATCH 392/396] Fix trace-ifd test failure in dev shell Fixes error: cannot create symlink '/home/eelco/Dev/nix/tests/functional/flakes/result'; already exists running the test multiple times in a dev shell. --- tests/functional/flakes/trace-ifd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/flakes/trace-ifd.sh b/tests/functional/flakes/trace-ifd.sh index f5c54f651..4879b9732 100644 --- a/tests/functional/flakes/trace-ifd.sh +++ b/tests/functional/flakes/trace-ifd.sh @@ -29,5 +29,5 @@ cat > "$flake1Dir/flake.nix" <<'EOF' } EOF -nix build "$flake1Dir#ifd" --option trace-import-from-derivation true 2>&1 \ +nix build --no-link "$flake1Dir#ifd" --option trace-import-from-derivation true 2>&1 \ | grepQuiet 'warning: built .* during evaluation due to an import from derivation' From 0278b9e1801f64b2586fbb857d6ad2da4f6b7d09 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Jun 2025 11:41:02 +0200 Subject: [PATCH 393/396] nix store copy-sigs: Use http-connections setting to control parallelism Previously it used the `ThreadPool` default, i.e. `std::thread::hardware_concurrency()`. But copying signatures is not primarily CPU-bound so it makes more sense to use the `http-connections` setting (since we're typically copying from/to a binary cache). --- src/nix/sigs.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index fb868baa1..802c093cb 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -3,6 +3,7 @@ #include "nix/main/shared.hh" #include "nix/store/store-open.hh" #include "nix/util/thread-pool.hh" +#include "nix/store/filetransfer.hh" #include @@ -38,7 +39,7 @@ struct CmdCopySigs : StorePathsCommand for (auto & s : substituterUris) substituters.push_back(openStore(s)); - ThreadPool pool; + ThreadPool pool{fileTransferSettings.httpConnections}; std::atomic added{0}; From b16fa06ff1dc8a2bac101a3daf1839b65f09bfbd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Jun 2025 12:06:21 +0200 Subject: [PATCH 394/396] nix store copy-sigs: Add docs --- src/nix/sigs.cc | 7 +++++++ src/nix/store-copy-sigs.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 src/nix/store-copy-sigs.md diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 802c093cb..89ed7b91d 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -29,6 +29,13 @@ struct CmdCopySigs : StorePathsCommand return "copy store path signatures from substituters"; } + std::string doc() override + { + return + #include "store-copy-sigs.md" + ; + } + void run(ref store, StorePaths && storePaths) override { if (substituterUris.empty()) diff --git a/src/nix/store-copy-sigs.md b/src/nix/store-copy-sigs.md new file mode 100644 index 000000000..678756221 --- /dev/null +++ b/src/nix/store-copy-sigs.md @@ -0,0 +1,30 @@ +R""( + +# Examples + +* To copy signatures from a binary cache to the local store: + + ```console + # nix store copy-sigs --substituter https://cache.nixos.org \ + --recursive /nix/store/y1x7ng5bmc9s8lqrf98brcpk1a7lbcl5-hello-2.12.1 + ``` + +* To copy signatures from one binary cache to another: + + ```console + # nix store copy-sigs --substituter https://cache.nixos.org \ + --store file:///tmp/binary-cache \ + --recursive -v \ + /nix/store/y1x7ng5bmc9s8lqrf98brcpk1a7lbcl5-hello-2.12.1 + imported 2 signatures + ``` + +# Description + +`nix store copy-sigs` copies store path signatures from one store to another. + +It is not advised to copy signatures to binary cache stores. Binary cache signatures are stored in `.narinfo` files. Since these are cached aggressively, clients may not see the new signatures quickly. It is therefore better to set any required signatures when the paths are first uploaded to the binary cache. + +Store paths are processed in parallel. The amount of parallelism is controlled by the [`http-connections`](@docroot@/command-ref/conf-file.md#conf-http-connections) settings. + +)"" From 1647cb56c18850d61d0b12bd7c90e77facc27ebf Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 2 Jun 2025 10:37:57 -0400 Subject: [PATCH 395/396] Document how to replicate nix-store --query --deriver with the nix command --- doc/manual/source/command-ref/nix-store/query.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/manual/source/command-ref/nix-store/query.md b/doc/manual/source/command-ref/nix-store/query.md index b5ba63ada..94eee05b8 100644 --- a/doc/manual/source/command-ref/nix-store/query.md +++ b/doc/manual/source/command-ref/nix-store/query.md @@ -103,6 +103,13 @@ symlink. example when *paths* were substituted from a binary cache. Use `--valid-derivers` instead to obtain valid paths only. + > **Note** + > + > `nix-store --query --deriver` is replaced with the following `nix` command: + > + > nix path-info --json ... | jq -r '.[].deriver' + + [deriver]: @docroot@/glossary.md#gloss-deriver - `--valid-derivers` From 665e76f2e5a02c18f9d54bc2e0867e2890fac2a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 30 May 2025 14:24:59 +0200 Subject: [PATCH 396/396] deletePath(): Keep going when encountering an undeletable file This should reduce the impact of #5207. --- src/libutil/file-system.cc | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 90ec5eda5..f63a5a4c3 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -414,7 +414,7 @@ void recursiveSync(const Path & path) } -static void _deletePath(Descriptor parentfd, const std::filesystem::path & path, uint64_t & bytesFreed) +static void _deletePath(Descriptor parentfd, const std::filesystem::path & path, uint64_t & bytesFreed, std::exception_ptr & ex) { #ifndef _WIN32 checkInterrupt(); @@ -472,7 +472,7 @@ static void _deletePath(Descriptor parentfd, const std::filesystem::path & path, checkInterrupt(); std::string childName = dirent->d_name; if (childName == "." || childName == "..") continue; - _deletePath(dirfd(dir.get()), path + "/" + childName, bytesFreed); + _deletePath(dirfd(dir.get()), path + "/" + childName, bytesFreed, ex); } if (errno) throw SysError("reading directory %1%", path); } @@ -480,7 +480,14 @@ static void _deletePath(Descriptor parentfd, const std::filesystem::path & path, int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; if (unlinkat(parentfd, name.c_str(), flags) == -1) { if (errno == ENOENT) return; - throw SysError("cannot unlink %1%", path); + try { + throw SysError("cannot unlink %1%", path); + } catch (...) { + if (!ex) + ex = std::current_exception(); + else + ignoreExceptionExceptInterrupt(); + } } #else // TODO implement @@ -500,7 +507,12 @@ static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFree throw SysError("opening directory '%1%'", path); } - _deletePath(dirfd.get(), path, bytesFreed); + std::exception_ptr ex; + + _deletePath(dirfd.get(), path, bytesFreed, ex); + + if (ex) + std::rethrow_exception(ex); }