diff --git a/.mergify.yml b/.mergify.yml index e134b0f46..36ffe6e8b 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -128,3 +128,14 @@ pull_request_rules: labels: - automatic backport - merge-queue + + - name: backport patches to 2.28 + conditions: + - label=backport 2.28-maintenance + actions: + backport: + branches: + - "2.28-maintenance" + labels: + - automatic backport + - merge-queue diff --git a/.version b/.version index 1eb56ea3a..f01356823 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.28.3 +2.29.0 diff --git a/README.md b/README.md index 54a6fcc39..02498944c 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Today, a world-wide developer community contributes to Nix and the ecosystem tha - [Nixpkgs](https://github.com/NixOS/nixpkgs) is [the largest, most up-to-date free software repository in the world](https://repology.org/repositories/graphs) - [NixOS](https://github.com/NixOS/nixpkgs/tree/master/nixos) is a Linux distribution that can be configured fully declaratively - [Discourse](https://discourse.nixos.org/) -- [Matrix](https://matrix.to/#/#nix:nixos.org) +- Matrix: [#users:nixos.org](https://matrix.to/#/#users:nixos.org) for user support and [#nix-dev:nixos.org](https://matrix.to/#/#nix-dev:nixos.org) for development ## License diff --git a/doc/manual/generate-deps.py b/doc/manual/generate-deps.py index 297bd3939..8d6b5a267 100755 --- a/doc/manual/generate-deps.py +++ b/doc/manual/generate-deps.py @@ -14,7 +14,7 @@ import sys # literally. since the rules for these aren't even the same for # all three we will just fail when we encounter any of them (if # asserts are off for some reason the depfile will likely point -# to nonexistant paths, making everything phony and thus fine.) +# to nonexistent paths, making everything phony and thus fine.) for path in glob.glob(sys.argv[1] + '/**', recursive=True): assert '\\' not in path assert ' ' not in path diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index e8b7377da..e66611aff 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -33,6 +33,7 @@ let { settings, doc, + uri-schemes, experimentalFeature, }: let diff --git a/doc/manual/meson.build b/doc/manual/meson.build index f7d3f44c5..1ddd26c56 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -69,7 +69,7 @@ subdir('source/release-notes') subdir('source') # Hacky way to figure out if `nix` is an `ExternalProgram` or -# `Exectuable`. Only the latter can occur in custom target input lists. +# `Executable`. Only the latter can occur in custom target input lists. if nix.full_path().startswith(meson.build_root()) nix_input = nix else diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 778440ac2..3953cb861 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -11,6 +11,8 @@ python3, rsync, nix-cli, + changelog-d, + officialRelease, # Configuration Options @@ -45,16 +47,24 @@ mkMesonDerivation (finalAttrs: { ]; # Hack for sake of the dev shell - passthru.externalNativeBuildInputs = [ - meson - ninja - (lib.getBin lowdown-unsandboxed) - mdbook - mdbook-linkcheck - jq - python3 - rsync - ]; + passthru.externalNativeBuildInputs = + [ + meson + ninja + (lib.getBin lowdown-unsandboxed) + mdbook + mdbook-linkcheck + jq + python3 + rsync + changelog-d + ] + ++ lib.optionals (!officialRelease) [ + # When not an official release, we likely have changelog entries that have + # yet to be rendered. + # When released, these are rendered into a committed file to save a dependency. + changelog-d + ]; nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [ nix-cli diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 3648ad898..b2295cf4f 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -370,6 +370,7 @@ const redirects = { }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", + "package-attribute-set": "#package", "gloss-chroot-store": "store/types/local-store.html", "gloss-content-addressed-derivation": "#gloss-content-addressing-derivation", }, diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index a7ed52a23..45b56438f 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -28,6 +28,7 @@ - [Data Types](language/types.md) - [String context](language/string-context.md) - [Syntax and semantics](language/syntax.md) + - [Evaluation](language/evaluation.md) - [Variables](language/variables.md) - [String literals](language/string-literals.md) - [Identifiers](language/identifiers.md) @@ -138,6 +139,7 @@ - [Release 3.0.0 (2025-03-04)](release-notes-determinate/rl-3.0.0.md) - [Nix Release Notes](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.29 (2025-05-14)](release-notes/rl-2.29.md) - [Release 2.28 (2025-04-02)](release-notes/rl-2.28.md) - [Release 2.27 (2025-03-03)](release-notes/rl-2.27.md) - [Release 2.26 (2025-01-22)](release-notes/rl-2.26.md) diff --git a/doc/manual/source/advanced-topics/distributed-builds.md b/doc/manual/source/advanced-topics/distributed-builds.md index 464b87d6e..08a980643 100644 --- a/doc/manual/source/advanced-topics/distributed-builds.md +++ b/doc/manual/source/advanced-topics/distributed-builds.md @@ -27,7 +27,7 @@ nix store info --store ssh://username@mac ``` To specify an SSH identity file as part of the remote store URI add a -query paramater, e.g. +query parameter, e.g. ```console nix store info --store ssh://username@mac?ssh-key=/home/alice/my-key diff --git a/doc/manual/source/architecture/architecture.md b/doc/manual/source/architecture/architecture.md index cbc469355..0c8677a6a 100644 --- a/doc/manual/source/architecture/architecture.md +++ b/doc/manual/source/architecture/architecture.md @@ -22,9 +22,9 @@ The following [concept map] shows its main components (rectangles), the objects | | +----------|-------------------|--------------------------------+ | Nix | V | -| | +-------------------------+ | -| | | commmand line interface |------. | -| | +-------------------------+ | | +| | +------------------------+ | +| | | command line interface |------. | +| | +------------------------+ | | | | | | | | evaluated by calls manages | | | | | | diff --git a/doc/manual/source/command-ref/nix-env/delete-generations.md b/doc/manual/source/command-ref/nix-env/delete-generations.md index b1ff0bb69..b769790fd 100644 --- a/doc/manual/source/command-ref/nix-env/delete-generations.md +++ b/doc/manual/source/command-ref/nix-env/delete-generations.md @@ -27,7 +27,7 @@ This operation deletes the specified generations of the current profile. > > Older *and newer* generations will be deleted by this operation. > - > One might expect this to just delete older generations than the curent one, but that is only true if the current generation is also the latest. + > One might expect this to just delete older generations than the current one, but that is only true if the current generation is also the latest. > Because one can roll back to a previous generation, it is possible to have generations newer than the current one. > They will also be deleted. diff --git a/doc/manual/source/command-ref/nix-shell.md b/doc/manual/source/command-ref/nix-shell.md index e95db9bea..f2e2e3593 100644 --- a/doc/manual/source/command-ref/nix-shell.md +++ b/doc/manual/source/command-ref/nix-shell.md @@ -242,16 +242,21 @@ print(t) ``` Similarly, the following is a Perl script that specifies that it -requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages: +requires Perl and the `HTML::TokeParser::Simple`, `LWP` and +`LWP::Protocol::Https` packages: ```perl #! /usr/bin/env nix-shell -#! nix-shell -i perl --packages perl perlPackages.HTMLTokeParserSimple perlPackages.LWP +#! nix-shell -i perl +#! nix-shell --packages perl +#! nix-shell --packages perlPackages.HTMLTokeParserSimple +#! nix-shell --packages perlPackages.LWP +#! nix-shell --packages perlPackages.LWPProtocolHttps use HTML::TokeParser::Simple; # Fetch nixos.org and print all hrefs. -my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/'); +my $p = HTML::TokeParser::Simple->new(url => 'https://nixos.org/'); while (my $token = $p->get_tag("a")) { my $href = $token->get_attr("href"); @@ -316,7 +321,7 @@ contains: ```nix with import {}; -runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" +runCommand "dummy" { buildInputs = [ python3 python3Packages.prettytable ]; } "" ``` The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. diff --git a/doc/manual/source/command-ref/nix-store/query.md b/doc/manual/source/command-ref/nix-store/query.md index 601f46af6..b5ba63ada 100644 --- a/doc/manual/source/command-ref/nix-store/query.md +++ b/doc/manual/source/command-ref/nix-store/query.md @@ -45,10 +45,19 @@ symlink. [output paths]: @docroot@/glossary.md#gloss-output-path +- `--references` + + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) + + [references]: @docroot@/glossary.md#gloss-reference + - `--requisites` / `-R` - Prints out the [closure] of the store path *paths*. + Prints out the set of [*requisites*][requisite] (better known as the [closure]) of the store path *paths*. + [requisite]: @docroot@/glossary.md#gloss-requisite [closure]: @docroot@/glossary.md#gloss-closure This query has one option: @@ -65,29 +74,25 @@ symlink. dependencies) is obtained by distributing the closure of a store derivation and specifying the option `--include-outputs`. -- `--references` - - Prints the set of [references] of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) - - [references]: @docroot@/glossary.md#gloss-reference - - `--referrers` - Prints the set of *referrers* of the store paths *paths*, that is, + Prints the set of [*referrers*][referrer] of the store paths *paths*, that is, the store paths currently existing in the Nix store that refer to one of *paths*. Note that contrary to the references, the set of referrers is not constant; it can change as store paths are added or removed. + [referrer]: @docroot@/glossary.md#gloss-referrer + - `--referrers-closure` Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or + [referrers relation][referrer]; that is, all store paths that directly or indirectly refer to one of *paths*. These are all the path currently in the Nix store that are dependent on *paths*. + [referrer]: @docroot@/glossary.md#gloss-referrer + - `--deriver` / `-d` Prints the [deriver] that was used to build the store paths *paths*. If diff --git a/doc/manual/source/development/building.md b/doc/manual/source/development/building.md index 49cc9d568..bf2f73c10 100644 --- a/doc/manual/source/development/building.md +++ b/doc/manual/source/development/building.md @@ -121,19 +121,25 @@ Nix uses a string with the following format to identify the *system type* or *pl -[-] ``` -It is set when Nix is compiled for the given system, and based on the output of [`config.guess`](https://github.com/nixos/nix/blob/master/config/config.guess) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess)): +It is set when Nix is compiled for the given system, and based on the output of Meson's [`host_machine` information](https://mesonbuild.com/Reference-manual_builtin_host_machine.html)> ``` --[][-] ``` -When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://github.com/nixos/nix/blob/master/config/config.sub) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub)): +When cross-compiling Nix with Meson for local development, you need to specify a [cross-file](https://mesonbuild.com/Cross-compilation.html) using the `--cross-file` option. Cross-files define the target architecture and toolchain. When cross-compiling Nix with Nix, Nixpkgs takes care of this for you. + +In the nix flake we also have some cross-compilation targets available: ``` --[-]- +nix build .#nix-everything-riscv64-unknown-linux-gnu +nix build .#nix-everything-armv7l-unknown-linux-gnueabihf +nix build .#nix-everything-armv7l-unknown-linux-gnueabihf +nix build .#nix-everything-x86_64-unknown-freebsd +nix build .#nix-everything-x86_64-w64-mingw32 ``` -For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/configure.ac) as follows: +For historic reasons and backward-compatibility, some CPU and OS identifiers are translated as follows: | `config.guess` | Nix | |----------------------------|---------------------| @@ -156,10 +162,10 @@ Nix can be compiled using multiple environments: To build with one of those environments, you can use ```console -$ nix build .#nix-ccacheStdenv +$ nix build .#nix-cli-ccacheStdenv ``` -You can use any of the other supported environments in place of `nix-ccacheStdenv`. +You can use any of the other supported environments in place of `nix-cli-ccacheStdenv`. ## Editor integration diff --git a/doc/manual/source/development/cli-guideline.md b/doc/manual/source/development/cli-guideline.md index 23df844ec..052b4aae8 100644 --- a/doc/manual/source/development/cli-guideline.md +++ b/doc/manual/source/development/cli-guideline.md @@ -170,9 +170,9 @@ sensitive. ```shell -$ nix init --template=template#pyton +$ nix init --template=template#python ------------------------------------------------------------------------ - Error! Template `template#pyton` not found. + Error! Template `template#python` not found. ------------------------------------------------------------------------ Initializing Nix project at `/path/to/here`. Select a template for you new project: diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index 6a7501200..94a6b5825 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -38,6 +38,13 @@ [store derivation]: #gloss-store-derivation +- [directed acyclic graph]{#gloss-directed-acyclic-graph} + + A [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) (DAG) is graph whose edges are given a direction ("a to b" is not the same edge as "b to a"), and for which no possible path (created by joining together edges) forms a cycle. + + DAGs are very important to Nix. + In particular, the non-self-[references][reference] of [store object][store object] form a cycle. + - [derivation path]{#gloss-derivation-path} A [store path] which uniquely identifies a [store derivation]. @@ -71,9 +78,8 @@ This can be achieved by: - Fetching a pre-built [store object] from a [substituter] - - Running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [store derivation] + - [Building](@docroot@/store/building.md) the corresponding [store derivation] - Delegating to a [remote machine](@docroot@/command-ref/conf-file.md#conf-builders) and retrieving the outputs - See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. @@ -156,6 +162,8 @@ non-[fixed-output](#gloss-fixed-output-derivation) derivation. + See [input-addressing derivation outputs](store/derivation/outputs/input-address.md) for details. + - [content-addressed store object]{#gloss-content-addressed-store-object} A [store object] which is [content-addressed](#gloss-content-address), @@ -215,23 +223,25 @@ > **Example** > - > Building and deploying software using Nix entails writing Nix expressions as a high-level description of packages and compositions thereof. + > Building and deploying software using Nix entails writing Nix expressions to describe [packages][package] and compositions thereof. - [reference]{#gloss-reference} - A [store object] `O` is said to have a *reference* to a store object `P` if a [store path] to `P` appears in the contents of `O`. + An edge from one [store object] to another. - Store objects can refer to both other store objects and themselves. - References from a store object to itself are called *self-references*. - References other than a self-reference must not form a cycle. + See [References](@docroot@/store/store-object.md#references) for details. [reference]: #gloss-reference + See [References](@docroot@/store/store-object.md#references) for details. + - [reachable]{#gloss-reachable} A store path `Q` is reachable from another store path `P` if `Q` is in the *closure* of the *references* relation. + See [References](@docroot@/store/store-object.md#references) for details. + - [closure]{#gloss-closure} The closure of a store path is the set of store paths that are @@ -248,8 +258,21 @@ to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` references `R` then `R` is also in the closure of `P`. + See [References](@docroot@/store/store-object.md#references) for details. + [closure]: #gloss-closure +- [requisite]{#gloss-requisite} + + A store object [reachable] by a path (chain of references) from a given [store object]. + The [closure] is the set of requisites. + + See [References](@docroot@/store/store-object.md#references) for details. + +- [referrer]{#gloss-reference} + + A reversed edge from one [store object] to another. + - [output]{#gloss-output} A [store object] produced by a [store derivation]. @@ -320,7 +343,7 @@ See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. -- [`∅`]{#gloss-emtpy-set} +- [`∅`]{#gloss-empty-set} The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. @@ -330,18 +353,17 @@ - [package]{#package} - 1. A software package; a collection of files and other data. + A software package; files that belong together for a particular purpose, and metadata. - 2. A [package attribute set]. + Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. -- [package attribute set]{#package-attribute-set} + The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#attribute-set) containing: + - attributes that refer to the files of a package, typically in the form of [derivation outputs](#output), + - attributes with metadata, such as information about how the package is supposed to be used. - An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as - - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), - - attributes that declare something about how the package is supposed to be installed or used, - - other metadata or arbitrary attributes. + The exact shape of these attribute sets is up to convention. - [package attribute set]: #package-attribute-set + [package]: #package - [string interpolation]{#gloss-string-interpolation} diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index a95e82740..fedb5595a 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -15,8 +15,9 @@ curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix ## How Nix works -Nix treats packages like values in purely functional programming languages -such as Haskell — they are built by functions that don’t have +Nix is a _purely functional package manager_. This means that it +treats packages like values in a purely functional programming language +— packages are built by functions that don’t have side-effects, and they never change after they have been built. Nix stores packages in the _Nix store_, usually the directory `/nix/store`, where each package has its own unique subdirectory such diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index bf196e0b8..4031f763a 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -79,6 +79,8 @@ Derivations can declare some infrequently used optional attributes. ## Output checks +See the [corresponding section in the derivation output page](@docroot@/store/derivation/outputs/index.md). + - [`allowedReferences`]{#adv-attr-allowedReferences}\ The optional attribute `allowedReferences` specifies a list of legal references (dependencies) of the output of the builder. For example, @@ -280,7 +282,7 @@ All other combinations are invalid. Call by need is commonly known as laziness in functional programming, as it is a specific implementation of the concept where evaluation is deferred until the result is required, aiming to only evaluate the parts of an expression that are needed to produce the final result. + +Furthermore, the result of evaluation is preserved, in values, in `let` bindings, in function _parameters_, which behave a lot like `let` bindings, but with the notable exception of function _calls_. Results of function calls rely on being put into `let` bindings, etc to be reused. + +When discussing the process of evaluation in lower level terms, we may define values not as a subset of expressions, but separately, where each "value" is either a data constructor, a function or a _thunk_. A thunk is a delayed computation, represented by an expression reference and a "closure" – the values for the lexical scope around the delayed expression. + +As a user of the language, you generally don't have to think about thunks, as they are not part of the language semantics, but you may encounter them in the repl, in the [C API] or in discussions. + +## Strictness + +Instead of thinking about thunks, it is often more productive to think in terms of _strictness_. +This term is used in functional programming to refer to the opposite of laziness, i.e. not just for something like error propagation. It refers to the need to evaluate certain expressions before evaluation can produce any result. + +Statements about strictness usually implicitly refer to weak head normal form. +For example, we can say that the following function is strict in its argument: + +```nix +x: isAttrs x || isFunction x +``` + +The above function must be strict in its argument `x` because determining its type requires evaluating `x` to at least some degree. + +The following function is not strict in its argument: + +```nix +x: { isOk = isAttrs x || isFunction x; } +``` + +It is not strict, because it can return the attribute set before evaluating `x`. +The attribute value for `isOk` _is_ strict in `x`. + +A function with a _set pattern_ is always strict in its argument, as a consequence of checking the argument's type and/or attribute names: + +```nix +let f = { ... }: "ok"; +in f (throw "kablam") +=> error: kablam +``` + +However, a set pattern does not add any strictness beyond WHNF of the attribute set argument. + +```nix +let f = orig@{ x, ... }: "ok"; +in f { x = throw "error"; y = throw "error"; } +=> "ok" +``` + +[C API]: @docroot@/c-api.md diff --git a/doc/manual/source/language/index.md b/doc/manual/source/language/index.md index 2bfdbb8a0..5bb939e18 100644 --- a/doc/manual/source/language/index.md +++ b/doc/manual/source/language/index.md @@ -11,7 +11,14 @@ The language is: - *domain-specific* - It comes with [built-in functions](@docroot@/language/builtins.md) to integrate with the Nix store, which manages files and performs the derivations declared in the Nix language. + The Nix language is purpose-built for working with text files. + Its most characteristic features are: + + - [File system path primitives](@docroot@/language/types.md#type-path), for accessing source files + - [Indented strings](@docroot@/language/string-literals.md) and [string interpolation](@docroot@/language/string-interpolation.md), for creating file contents + - [Strings with contexts](@docroot@/language/string-context.md), for transparently linking files + + It comes with [built-in functions](@docroot@/language/builtins.md) to integrate with the [Nix store](@docroot@/store/index.md), which manages files and enables [realising](@docroot@/glossary.md#gloss-realise) derivations declared in the Nix language. - *declarative* diff --git a/doc/manual/source/language/string-context.md b/doc/manual/source/language/string-context.md index 979bbf371..0d8fcdefa 100644 --- a/doc/manual/source/language/string-context.md +++ b/doc/manual/source/language/string-context.md @@ -13,8 +13,8 @@ The purpose of string contexts is to collect non-string values attached to strin [string concatenation](./operators.md#string-concatenation), [string interpolation](./string-interpolation.md), and similar operations. -The idea is that a user can combine together values to create a build instructions for derivations without manually keeping track of where they come from. -Then the Nix language implicitly does that bookkeeping to efficiently obtain the closure of derivation inputs. +The idea is that a user can reference other files when creating text files through Nix expressions, without manually keeping track of the exact paths. +Nix will ensure that the all referenced files are accessible – that all [store paths](@docroot@/glossary.md#gloss-store-path) are [valid](@docroot@/glossary.md#gloss-validity). > **Note** > diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index 414a862e8..92956091a 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -18,7 +18,7 @@ is a JSON object with the following fields: * `method`: - For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + For an output which will be [content addressed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. Valid method strings are: - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) @@ -29,7 +29,7 @@ is a JSON object with the following fields: Otherwise, `null`. * `hashAlgo`: - For an output which will be [content addresed], the name of the hash algorithm used. + For an output which will be [content addressed], the name of the hash algorithm used. Valid algorithm strings are: - `blake3` diff --git a/doc/manual/source/protocols/tarball-fetcher.md b/doc/manual/source/protocols/tarball-fetcher.md index 5cff05d66..e8cda784c 100644 --- a/doc/manual/source/protocols/tarball-fetcher.md +++ b/doc/manual/source/protocols/tarball-fetcher.md @@ -46,7 +46,7 @@ defined as the timestamp of the newest file inside the tarball. This protocol is supported by Gitea since v1.22.1 and by Forgejo since v7.0.4/v8.0.0 and can be used with the following flake URL schema: ``` -https://///archive/.tar.gz +https://///archive/.tar.gz ``` > **Example** diff --git a/doc/manual/source/release-notes/rl-2.19.md b/doc/manual/source/release-notes/rl-2.19.md index 13e573c1d..47a0dd3db 100644 --- a/doc/manual/source/release-notes/rl-2.19.md +++ b/doc/manual/source/release-notes/rl-2.19.md @@ -31,7 +31,7 @@ - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`. - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. - They are superceded by `nix flake update`. + They are superseded by `nix flake update`. - Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). diff --git a/doc/manual/source/release-notes/rl-2.23.md b/doc/manual/source/release-notes/rl-2.23.md index 249c183e6..e6b0e9ffc 100644 --- a/doc/manual/source/release-notes/rl-2.23.md +++ b/doc/manual/source/release-notes/rl-2.23.md @@ -15,7 +15,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). - In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, + In particular, the hash algorithm and content addressing method of content-addressed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. This JSON format is only used by the experimental `nix derivation` family of commands, at this time. diff --git a/doc/manual/source/release-notes/rl-2.24.md b/doc/manual/source/release-notes/rl-2.24.md index 31e843eb6..0d6823a68 100644 --- a/doc/manual/source/release-notes/rl-2.24.md +++ b/doc/manual/source/release-notes/rl-2.24.md @@ -173,7 +173,7 @@ **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` - parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + parameters of the `nix hash convert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value remains as a deprecated alias for `"nix32"`. Please convert your code from: ```nix diff --git a/doc/manual/source/release-notes/rl-2.29.md b/doc/manual/source/release-notes/rl-2.29.md new file mode 100644 index 000000000..ad63fff2f --- /dev/null +++ b/doc/manual/source/release-notes/rl-2.29.md @@ -0,0 +1,160 @@ +# Release 2.29.0 (2025-05-14) + +After the special backport-based release of Nix 2.28 (timed to coincide with Nixpkgs 25.05), the release process is back to normal with 2.29. +As such, we have slightly more weeks of work from `master` (since 2.28 was branched from 2.27) than usual. +This fact is counterbalanced by the fact that most of those changes are bug fixes rather than larger new features. + +- Prettified JSON output on the terminal [#12555](https://github.com/NixOS/nix/issues/12555) [#12652](https://github.com/NixOS/nix/pull/12652) + + This makes the output easier to read. + + Scripts are mostly unaffected because for those, stdout will be a file or a pipe, not a terminal, and for those, the old single-line behavior applies. + + `--json --pretty` can be passed to enable it even if the output is not a terminal. + If your script creates a pseudoterminal for Nix's stdout, you can pass `--no-pretty` to disable the new behavior. + +- Repl: improve continuation prompt for incomplete expressions [#12846](https://github.com/NixOS/nix/pull/12846) + + Improved REPL user experience by updating the continuation prompt from invisible blank spaces to a visible `" > "`, enhancing clarity when entering multi-line expressions. + +- REPL `:load-flake` and `:reload` now work together [#8753](https://github.com/NixOS/nix/issues/8753) [#13180](https://github.com/NixOS/nix/pull/13180) + + Previously, `:reload` only reloaded the files specified with `:load` (or on the command line). + Now, it also works with the flakes specified with `:load-flake` (or on the command line). + This makes it correctly reload everything that was previously loaded, regardless of what sort of thing (plain file or flake) each item is. + +- Increase retry delays on HTTP 429 Too Many Requests [#13052](https://github.com/NixOS/nix/pull/13052) + + When downloading Nix, the retry delay was previously set to 0.25 seconds. It has now been increased to 1 minute to better handle transient CI errors, particularly on GitHub. + +- S3: opt-in the STSProfileCredentialsProvider [#12646](https://github.com/NixOS/nix/pull/12646) + + Added support for STS-based authentication for S3-based binary caches, i.e. enabling seamless integration with `aws sso login`. + +- Reduce connect timeout for http substituter [#12876](https://github.com/NixOS/nix/pull/12876) + + Previously, the Nix setting `connect-timeout` had no limit. It is now set to `5s`, offering a more practical default for users self-hosting binary caches, which may occasionally become unavailable, such as during updates. + + +- C API: functions for locking and loading a flake [#10435](https://github.com/NixOS/nix/issues/10435) [#12877](https://github.com/NixOS/nix/pull/12877) [#13098](https://github.com/NixOS/nix/pull/13098) + + This release adds functions to the C API for handling the loading of flakes. Previously, this had to be worked around by using `builtins.getFlake`. + C API consumers and language bindings now have access to basic locking functionality. + + It does not expose the full locking API, so that the implementation can evolve more freely. + Locking is controlled with the functions, which cover the common use cases for consuming a flake: + - `nix_flake_lock_flags_set_mode_check` + - `nix_flake_lock_flags_set_mode_virtual` + - `nix_flake_lock_flags_set_mode_write_as_needed` + - `nix_flake_lock_flags_add_input_override`, which also enables `virtual` + + This change also introduces the new `nix-fetchers-c` library, whose single purpose for now is to manage the (`nix.conf`) settings for the built-in fetchers. + + More details can be found in the [C API documentation](@docroot@/c-api.md). + +- No longer copy flakes that are in the nix store [#10435](https://github.com/NixOS/nix/issues/10435) [#12877](https://github.com/NixOS/nix/pull/12877) [#13098](https://github.com/NixOS/nix/pull/13098) + + Previously, we would duplicate entries like `path:/nix/store/*` back into the Nix store. + This was prominently visible for pinned system flake registry entries in NixOS, e.g., when running `nix run nixpkgs#hello`. + +- Consistently preserve error messages from cached evaluation [#12762](https://github.com/NixOS/nix/issues/12762) [#12809](https://github.com/NixOS/nix/pull/12809) + + In one code path, we are not returning the errors cached from prior evaluation, but instead throwing generic errors stemming from the lack of value (due to the error). + These generic error messages were far less informative. + Now we consistently return the original error message. + +- Faster blake3 hashing [#12676](https://github.com/NixOS/nix/pull/12676) + + The implementation for blake3 hashing is now multi-threaded and used memory-mapped IO. + Benchmark results can be found the [pull request](https://github.com/NixOS/nix/pull/12676). + +- Fix progress bar for S3 binary caches and make file transfers interruptible [#12877](https://github.com/NixOS/nix/issues/12877) [#13098](https://github.com/NixOS/nix/issues/13098) [#12538](https://github.com/NixOS/nix/pull/12538) + + The progress bar now correctly display upload/download progress for S3 up/downloads. S3 uploads are now interruptible. + +- Add host attribute of github/gitlab flakerefs to URL serialization [#12580](https://github.com/NixOS/nix/pull/12580) + + Resolved an issue where `github:` or `gitlab:` URLs lost their `host` attribute when written to a lockfile, resulting in invalid URLs. + +- Multiple signatures support in store urls [#12976](https://github.com/NixOS/nix/pull/12976) + + Added support for a `secretKeyFiles` URI parameter in Nix store URIs, allowing multiple signing key files to be specified as a comma-separated list. + This enables signing paths with multiple keys. This helps with [RFC #149](https://github.com/NixOS/rfcs/pull/149) to enable binary cache key rotation in the NixOS infra. + + Example usage: + + ```bash + nix copy --to "file:///tmp/store?secret-keys=/tmp/key1,/tmp/key2" \ + "$(nix build --print-out-paths nixpkgs#hello)" + ``` + +- nix flake show now skips over import-from-derivation [#4265](https://github.com/NixOS/nix/issues/4265) [#12583](https://github.com/NixOS/nix/pull/12583) + + Previously, if a flake contained outputs relying on [import from derivation](@docroot@/language/import-from-derivation.md) during evaluation, `nix flake show` would fail to display the rest of the flake. The updated behavior skips such outputs, allowing the rest of the flake to be shown. + +- Add `nix formatter build` and `nix formatter run` commands [#13063](https://github.com/NixOS/nix/pull/13063) + + `nix formatter run` is an alias for `nix fmt`. Nothing new there. + + `nix formatter build` is sort of like `nix build`: it builds, links, and prints a path to the formatter program: + + ``` + $ nix formatter build + /nix/store/cb9w44vkhk2x4adfxwgdkkf5gjmm856j-treefmt/bin/treefmt + ``` + + Note that unlike `nix build`, this prints the full path to the program, not just the store path (in the example above that would be `/nix/store/cb9w44vkhk2x4adfxwgdkkf5gjmm856j-treefmt`). + +- Amend OSC 8 escape stripping for xterm-style separator [#13109](https://github.com/NixOS/nix/pull/13109) + + Improve terminal escape code filtering to understand a second type of hyperlink escape codes. + This in particular prevents parts of GCC 14's diagnostics from being improperly filtered away. + + +# Contributors + + +This release was made possible by the following 40 contributors: + +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- The Tumultuous Unicorn Of Darkness [**(@TheTumultuousUnicornOfDarkness)**](https://github.com/TheTumultuousUnicornOfDarkness) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Félix [**(@picnoir)**](https://github.com/picnoir) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Vincent Breitmoser [**(@Valodim)**](https://github.com/Valodim) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- ulucs [**(@ulucs)**](https://github.com/ulucs) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Andrey Butirsky [**(@bam80)**](https://github.com/bam80) +- Dean De Leo [**(@whatsthecraic)**](https://github.com/whatsthecraic) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium) +- Shahar "Dawn" Or [**(@mightyiam)**](https://github.com/mightyiam) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Rodney Lorrimar [**(@rvl)**](https://github.com/rvl) +- Erik Nygren [**(@Kirens)**](https://github.com/Kirens) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Martin Fischer [**(@not-my-profile)**](https://github.com/not-my-profile) +- Graham Christensen [**(@grahamc)**](https://github.com/grahamc) +- Vit Gottwald [**(@VitGottwald)**](https://github.com/VitGottwald) +- silvanshade [**(@silvanshade)**](https://github.com/silvanshade) +- Illia Bobyr [**(@ilya-bobyr)**](https://github.com/ilya-bobyr) +- Jeremy Fleischman [**(@jfly)**](https://github.com/jfly) +- Ruby Rose [**(@oldshensheep)**](https://github.com/oldshensheep) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Tim [**(@Jaculabilis)**](https://github.com/Jaculabilis) +- Anthony Wang [**(@anthowan)**](https://github.com/anthowan) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Sandro [**(@SuperSandro2000)**](https://github.com/SuperSandro2000) +- tomberek [**(@tomberek)**](https://github.com/tomberek) +- Dmitry Bogatov [**(@KAction)**](https://github.com/KAction) +- Sizhe Zhao [**(@Prince213)**](https://github.com/Prince213) +- jade [**(@lf-)**](https://github.com/lf-) +- Pierre-Etienne Meunier [**(@P-E-Meunier)**](https://github.com/P-E-Meunier) +- Alexander Romanov [**(@ajlekcahdp4)**](https://github.com/ajlekcahdp4) +- Domagoj Mišković [**(@allrealmsoflife)**](https://github.com/allrealmsoflife) +- Thomas Miedema [**(@thomie)**](https://github.com/thomie) +- Yannik Sander [**(@ysndr)**](https://github.com/ysndr) +- Philipp Otterbein +- Dmitry Bogatov diff --git a/doc/manual/source/release-notes/rl-2.6.md b/doc/manual/source/release-notes/rl-2.6.md index 280faead1..f99ce4b4e 100644 --- a/doc/manual/source/release-notes/rl-2.6.md +++ b/doc/manual/source/release-notes/rl-2.6.md @@ -13,7 +13,7 @@ * New command `nix store copy-log` to copy build logs from one store to another. * The `commit-lockfile-summary` option can be set to a non-empty - string to override the commit summary used when commiting an updated + string to override the commit summary used when committing an updated lockfile. This may be used in conjunction with the `nixConfig` attribute in `flake.nix` to better conform to repository conventions. diff --git a/doc/manual/source/store/derivation/outputs/content-address.md b/doc/manual/source/store/derivation/outputs/content-address.md index 4539a5eba..7fc689fb3 100644 --- a/doc/manual/source/store/derivation/outputs/content-address.md +++ b/doc/manual/source/store/derivation/outputs/content-address.md @@ -110,18 +110,18 @@ Because the derivation output is not fixed (just like with [input addressing]), > > Strictly speaking, the extent to which sandboxing and deprivilaging is possible varies with the environment Nix is running in. > Nix's configuration settings indicate what level of sandboxing is required or enabled. -> Builds of derivations will fail if they request an absense of sandboxing which is not allowed. -> Builds of derivations will also fail if the level of sandboxing specified in the configure exceeds what is possible in teh given environment. +> Builds of derivations will fail if they request an absence of sandboxing which is not allowed. +> Builds of derivations will also fail if the level of sandboxing specified in the configure exceeds what is possible in the given environment. > -> (The "environment", in this case, consists of attributes such as the Operating System Nix runs atop, along with the operating-system-specific privilages that Nix has been granted. -> Because of how conventional operating systems like macos, Linux, etc. work, granting builders *fewer* privilages may ironically require that Nix be run with *more* privilages.) +> (The "environment", in this case, consists of attributes such as the Operating System Nix runs atop, along with the operating-system-specific privileges that Nix has been granted. +> Because of how conventional operating systems like macos, Linux, etc. work, granting builders *fewer* privileges may ironically require that Nix be run with *more* privileges.) That said, derivations producing floating content-addressed outputs may declare their builders as impure (like the builders of derivations producing producing fixed outputs). This is provisionally supported as part of the [`impure-derivations`][xp-feature-impure-derivations] experimental feature. ### Compatibility negotiation -Any derivation producing a floating content-addresssed output implicitly requires the `ca-derivations` [system feature](@docroot@/command-ref/conf-file.md#conf-system-features). +Any derivation producing a floating content-addressed output implicitly requires the `ca-derivations` [system feature](@docroot@/command-ref/conf-file.md#conf-system-features). This prevents scheduling the building of the derivation on a machine without the experimental feature enabled. Even once the experimental feature is stabilized, this is still useful in order to be allow using remote builder running odler versions of Nix, or alternative implementations that do not support floating content addressing. @@ -132,7 +132,7 @@ For store objects produced by manually inserting into the store to create a stor But for store objects produced by derivation, the "method is quite formal" --- the whole point of derivations is to be a formal notion of building, after all. In this case, we can elevate this informal property to a formal one. -A *determinstic* content-addressing derivation should produce outputs with the same content addresses: +A *deterministic* content-addressing derivation should produce outputs with the same content addresses: 1. Every time the builder is run diff --git a/doc/manual/source/store/derivation/outputs/index.md b/doc/manual/source/store/derivation/outputs/index.md index b02e6eca0..3f87004a8 100644 --- a/doc/manual/source/store/derivation/outputs/index.md +++ b/doc/manual/source/store/derivation/outputs/index.md @@ -2,7 +2,7 @@ As stated on the [main pages on derivations](../index.md#store-derivation), a derivation produces [store objects](@docroot@/store/store-object.md), which are known as the *outputs* of the derivation. -Indeed, the entire point of derivations is to produce these outputs, and to reliably and reproducably produce these derivations each time the derivation is run. +Indeed, the entire point of derivations is to produce these outputs, and to reliably and reproducibly produce these derivations each time the derivation is run. One of the parts of a derivation is its *outputs specification*, which specifies certain information about the outputs the derivation produces when run. The outputs specification is a map, from names to specifications for individual outputs. @@ -87,7 +87,7 @@ The rules for this are fairly concise: - If it is pure, it must be floating. - - Pure, fixed content-addressing derivations are not suppported + - Pure, fixed content-addressing derivations are not supported > There is no use for this forth combination. > The sole purpose of an output's store path being fixed is to support the derivation being impure. diff --git a/doc/manual/source/store/store-object.md b/doc/manual/source/store/store-object.md index caf5657d1..115e107fb 100644 --- a/doc/manual/source/store/store-object.md +++ b/doc/manual/source/store/store-object.md @@ -4,7 +4,64 @@ A Nix store is a collection of *store objects* with *references* between them. A store object consists of - A [file system object](./file-system-object.md) as data - - A set of [store paths](./store-path.md) as references to other store objects + + - A set of [store paths](./store-path.md) as references to store objects + +### References + +Store objects can refer to both other store objects and themselves. +References from a store object to itself are called *self-references*. + +Store objects and their references form a directed graph, where the store objects are the vertices, and the references are the edges. +In particular, the edge corresponding to a reference is from the store object that contains the reference, and to the store object that the store path (which is the reference) refers to. + +References other than a self-reference must not form a cycle. +The graph of references excluding self-references thus forms a [directed acyclic graph]. + +[directed acyclic graph]: @docroot@/glossary.md#gloss-directed acyclic graph + +We can take the [transitive closure] of the references graph, which any pair of store objects have an edge not if there is a single reference from the first to the second, but a path of one or more references from the first to the second. +The *requisites* of a store object are all store objects reachable by paths of references which start with given store object's references. + +[transitive closure]: https://en.wikipedia.org/wiki/Transitive_closure + +We can also take the [transpose graph] ofthe references graph, where we reverse the orientation of all edges. +The *referrers* of a store object are the store objects that reference it. + +[transpose graph]: https://en.wikipedia.org/wiki/Transpose_graph + +One can also combine both concepts: taking the transitive closure of the transposed references graph. +The *referrers closure* of a store object are the store objects that can reach the given store object via paths of references. + +> **Note** +> +> Care must be taken to distinguish between the intrinsic and extrinsic properties of store objects. +> We can create graphs from the store objects in a store, but the contents of the store is not, in general fixed, and may instead change over time. +> +> - The references of a store object --- the set of store paths called the references --- is a field of a store object, and thus intrinsic by definition. + Regardless of what store contains the store object in question, and what else that store may or may not contain, the references are the same. +> +> - The requisites of a store object are almost intrinsic --- some store paths due not precisely refer to a unique single store object. +> Exactly what store object is being referenced, and what in turn *its* references are, depends on the store in question. +> Different stores that disagree. +> +> - The referrers of a store object are completely extrinsic, and depends solely on the store which contains that store object, not the store object itself. +> Other store objects which refer to the store object in question may be added or removed from the store. + +### Immutability Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object): -Once created, they do not change until they are deleted. +Once created, they do not change nor can any store object they reference be changed. + +> **Note** +> +> Stores which support atomically deleting multiple store objects allow more flexibility while still upholding this property. + +### Closure property + +A store can only contain a store object if it also contains all the store objects it refers to. + +> **Note** +> +> The "closure property" isn't meant to prohibit, for example, [lazy loading](https://en.wikipedia.org/wiki/Lazy_loading) of store objects. +> However, the "closure property" and immutability in conjunction imply that any such lazy loading ought to be deterministic. diff --git a/doc/manual/source/store/store-object/content-address.md b/doc/manual/source/store/store-object/content-address.md index 5742b9fe1..36e841fa3 100644 --- a/doc/manual/source/store/store-object/content-address.md +++ b/doc/manual/source/store/store-object/content-address.md @@ -45,7 +45,7 @@ Self-references however cannot be referred to by their path, because we are in t > As far as we know, this is equivalent to finding a hash collision. Instead we have a "has self-reference" boolean, which ends up affecting the digest: -In all currently-supported store object content-addressing methods, when hashing the file system object data, any occurence of store object's own store path in the digested data is replaced with a [sentinel value](https://en.wikipedia.org/wiki/Sentinel_value). +In all currently-supported store object content-addressing methods, when hashing the file system object data, any occurrence of store object's own store path in the digested data is replaced with a [sentinel value](https://en.wikipedia.org/wiki/Sentinel_value). The hashes of these modified input streams are used instead. When validating the content address of a store object after the fact, the above process works as written. diff --git a/docker.nix b/docker.nix index 8522f432f..6679fc8d9 100644 --- a/docker.nix +++ b/docker.nix @@ -7,7 +7,7 @@ channelName ? "nixpkgs", channelURL ? "https://nixos.org/channels/nixpkgs-unstable", extraPkgs ? [ ], - maxLayers ? 100, + maxLayers ? 70, nixConf ? { }, flake-registry ? null, uid ? 0, diff --git a/flake.lock b/flake.lock index 47cab9510..653af0e2e 100644 --- a/flake.lock +++ b/flake.lock @@ -63,16 +63,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1734359947, - "narHash": "sha256-1Noao/H+N8nFB4Beoy8fgwrcOQLVm9o4zKW1ODaqK9E=", + "lastModified": 1746141548, + "narHash": "sha256-IgBWhX7A2oJmZFIrpRuMnw5RAufVnfvOgHWgIdds+hc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "48d12d5e70ee91fe8481378e540433a7303dbf6a", + "rev": "f02fddb8acef29a8b32f10a335d44828d7825b78", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-24.11", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 6511df5f2..aa540a654 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-24.11"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; @@ -384,6 +384,7 @@ "nix-store-tests" = { }; "nix-fetchers" = { }; + "nix-fetchers-c" = { }; "nix-fetchers-tests" = { }; "nix-expr" = { }; @@ -392,6 +393,7 @@ "nix-expr-tests" = { }; "nix-flake" = { }; + "nix-flake-c" = { }; "nix-flake-tests" = { }; "nix-main" = { }; diff --git a/maintainers/README.md b/maintainers/README.md index fd9bbed8e..6553cd048 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -37,7 +37,7 @@ The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/n The team meets twice a week (times are denoted in the [Europe/Amsterdam](https://en.m.wikipedia.org/wiki/Time_in_the_Netherlands) time zone): -- Discussion meeting: [Wednesday 21:00-22:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=ZG5rZzNyajRjajducGV2NGY5aGkzYWIwdnJfMjAyNDA1MDhUMTkwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Discussion meeting: Wednesday 21:00-22:00 Europe/Amsterdam see [calendar](https://calendar.google.com/calendar/u/0/embed?src=b9o52fobqjak8oq8lfkhg3t0qg@group.calendar.google.com). 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min). @@ -46,7 +46,7 @@ The team meets twice a week (times are denoted in the [Europe/Amsterdam](https:/ - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: Mondays 14:00-16:00 Europe/Amsterdam see [calendar](https://calendar.google.com/calendar/u/0/embed?src=b9o52fobqjak8oq8lfkhg3t0qg@group.calendar.google.com). 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json index 977555278..49d33cfdd 100644 --- a/maintainers/data/release-credits-email-to-handle.json +++ b/maintainers/data/release-credits-email-to-handle.json @@ -152,5 +152,19 @@ "kaction@disroot.org": "KAction", "serenity@kaction.cc": null, "dev@erik.work": "Kirens", - "felix@alternativebit.fr": "picnoir" -} \ No newline at end of file + "felix@alternativebit.fr": "picnoir", + "butirsky@gmail.com": "bam80", + "look@my.amazin.horse": "Valodim", + "jeremyfleischman@gmail.com": "jfly", + "vit.gottwald@gmail.com": "VitGottwald", + "a@unnamed.website": "anthowan", + "hello@whatsthecraic.net": "whatsthecraic", + "alex.rom23@mail.ru": "ajlekcahdp4", + "domagoj@tuta.com": "allrealmsoflife", + "uluc.sengil@gmail.com": "ulucs", + "prc.zhao@outlook.com": "Prince213", + "the-tumultuous-unicorn-of-darkness@gmx.com": "TheTumultuousUnicornOfDarkness", + "dev@rodney.id.au": "rvl", + "pe@pijul.org": "P-E-Meunier", + "yannik@floxdev.com": "ysndr" +} diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json index a03a811d4..968c51f58 100644 --- a/maintainers/data/release-credits-handle-to-name.json +++ b/maintainers/data/release-credits-handle-to-name.json @@ -133,5 +133,18 @@ "oldshensheep": "Ruby Rose", "KAction": "Dmitry Bogatov", "thomie": "Thomas Miedema", - "Kirens": "Erik Nygren" -} \ No newline at end of file + "Kirens": "Erik Nygren", + "Prince213": "Sizhe Zhao", + "anthowan": "Anthony Wang", + "jfly": "Jeremy Fleischman", + "VitGottwald": "Vit Gottwald", + "bam80": "Andrey Butirsky", + "ulucs": null, + "P-E-Meunier": "Pierre-Etienne Meunier", + "ysndr": "Yannik Sander", + "TheTumultuousUnicornOfDarkness": "The Tumultuous Unicorn Of Darkness", + "ajlekcahdp4": "Alexander Romanov", + "Valodim": "Vincent Breitmoser", + "rvl": "Rodney Lorrimar", + "whatsthecraic": "Dean De Leo" +} diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index ff40b09d1..6497b17c1 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -284,8 +284,8 @@ ''^src/libstore/build/goal\.cc$'' ''^src/libstore/include/nix/store/build/goal\.hh$'' ''^src/libstore/unix/build/hook-instance\.cc$'' - ''^src/libstore/unix/build/local-derivation-goal\.cc$'' - ''^src/libstore/unix/include/nix/store/build/local-derivation-goal\.hh$'' + ''^src/libstore/unix/build/derivation-builder\.cc$'' + ''^src/libstore/unix/include/nix/store/build/derivation-builder\.hh$'' ''^src/libstore/build/substitution-goal\.cc$'' ''^src/libstore/include/nix/store/build/substitution-goal\.hh$'' ''^src/libstore/build/worker\.cc$'' @@ -360,7 +360,6 @@ ''^src/libutil/linux/namespaces\.cc$'' ''^src/libutil/logging\.cc$'' ''^src/libutil/include/nix/util/logging\.hh$'' - ''^src/libutil/include/nix/util/lru-cache\.hh$'' ''^src/libutil/memory-source-accessor\.cc$'' ''^src/libutil/include/nix/util/memory-source-accessor\.hh$'' ''^src/libutil/include/nix/util/pool\.hh$'' @@ -605,8 +604,8 @@ ''^tests/functional/flakes/prefetch\.sh$'' ''^tests/functional/flakes/run\.sh$'' ''^tests/functional/flakes/show\.sh$'' - ''^tests/functional/fmt\.sh$'' - ''^tests/functional/fmt\.simple\.sh$'' + ''^tests/functional/formatter\.sh$'' + ''^tests/functional/formatter\.simple\.sh$'' ''^tests/functional/gc-auto\.sh$'' ''^tests/functional/gc-concurrent\.builder\.sh$'' ''^tests/functional/gc-concurrent\.sh$'' diff --git a/meson.build b/meson.build index 49adf9832..9f7471143 100644 --- a/meson.build +++ b/meson.build @@ -6,7 +6,8 @@ project('nix-dev-shell', 'cpp', subproject_dir : 'src', default_options : [ 'localstatedir=/nix/var', - ] + ], + meson_version : '>= 1.1' ) # Internal Libraries @@ -33,6 +34,7 @@ endif # External C wrapper libraries subproject('libutil-c') subproject('libstore-c') +subproject('libfetchers-c') subproject('libexpr-c') subproject('libflake-c') subproject('libmain-c') diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index 9d77831b3..b9140256a 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -12,6 +12,7 @@ add_project_arguments( '-Werror=switch-enum', '-Werror=undef', '-Werror=unused-result', + '-Werror=sign-compare', '-Wignored-qualifiers', '-Wimplicit-fallthrough', '-Wno-deprecated-declarations', diff --git a/packaging/components.nix b/packaging/components.nix index c9886aa41..46e2d5851 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -151,13 +151,6 @@ let ]; separateDebugInfo = !stdenv.hostPlatform.isStatic; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - env = - prevAttrs.env or { } - // lib.optionalAttrs ( - stdenv.isLinux - && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") - && !(stdenv.hostPlatform.useLLVM or false) - ) { LDFLAGS = "-fuse-ld=gold"; }; }; mesonLibraryLayer = finalAttrs: prevAttrs: { @@ -209,6 +202,7 @@ in { version = baseVersion + versionSuffix; inherit versionSuffix; + inherit officialRelease; inherit maintainers; inherit filesetToSource; @@ -331,6 +325,7 @@ in nix-store-tests = callPackage ../src/libstore-tests/package.nix { }; nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + nix-fetchers-c = callPackage ../src/libfetchers-c/package.nix { }; nix-fetchers-tests = callPackage ../src/libfetchers-tests/package.nix { }; nix-expr = callPackage ../src/libexpr/package.nix { }; diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index ed05843c7..a90ef1b4a 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -38,6 +38,7 @@ scope: { aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { apis = [ + "identity-management" "s3" "transfer" ]; @@ -60,6 +61,7 @@ scope: { "--with-container" "--with-context" "--with-coroutine" + "--with-iostreams" ]; }).overrideAttrs (old: { diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index fa83fbb29..de1243900 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -71,12 +71,17 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( # We use this shell with the local checkout, not unpackPhase. src = null; - env = { - # For `make format`, to work without installing pre-commit - _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" - modular.pre-commit.settings.rawConfig - }"; - }; + env = + { + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" + modular.pre-commit.settings.rawConfig + }"; + } + // lib.optionalAttrs stdenv.hostPlatform.isLinux { + CC_LD = "mold"; + CXX_LD = "mold"; + }; mesonFlags = map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags) @@ -119,7 +124,8 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) ( lib.hiPrio pkgs.buildPackages.clang-tools - ); + ) + ++ lib.optional stdenv.hostPlatform.isLinux pkgs.buildPackages.mold-wrapped; buildInputs = attrs.buildInputs or [ ] diff --git a/packaging/everything.nix b/packaging/everything.nix index f11b29163..c6229271f 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -15,6 +15,7 @@ nix-store-tests, nix-fetchers, + nix-fetchers-c, nix-fetchers-tests, nix-expr, @@ -54,6 +55,7 @@ let nix-store nix-store-c nix-fetchers + nix-fetchers-c nix-expr nix-expr-c nix-flake @@ -230,6 +232,7 @@ stdenv.mkDerivation (finalAttrs: { "nix-expr" "nix-expr-c" "nix-fetchers" + "nix-fetchers-c" "nix-flake" "nix-flake-c" "nix-main" diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 3bb1bf19e..664ee18ca 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -48,6 +48,7 @@ let "nix-store-test-support" "nix-store-tests" "nix-fetchers" + "nix-fetchers-c" "nix-fetchers-tests" "nix-expr" "nix-expr-c" diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 2ecab0077..3fb727151 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -62,6 +62,13 @@ else end end +# Only use MANPATH if it is already set. In general `man` will just simply +# pick up `.nix-profile/share/man` because is it close to `.nix-profile/bin` +# which is in the $PATH. For more info, run `manpath -d`. +if set --query MANPATH + set --export --prepend --path MANPATH "$NIX_LINK/share/man" +end + add_path "@localstatedir@/nix/profiles/default/bin" add_path "$NIX_LINK/bin" diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index 05d9a187d..3fb727151 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -1,5 +1,5 @@ # Only execute this file once per shell. -if test -z "$HOME" || test -z "$USER" || \ +if test -z "$HOME" || \ test -n "$__ETC_PROFILE_NIX_SOURCED" exit end @@ -53,6 +53,13 @@ else if test -e "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" # fall back to cacert in set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" +else + # Fall back to what is in the nix profiles, favouring whatever is defined last. + for i in (string split ' ' $NIX_PROFILES) + if test -e "$i/etc/ssl/certs/ca-bundle.crt" + set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" + end + end end # Only use MANPATH if it is already set. In general `man` will just simply @@ -62,6 +69,7 @@ if set --query MANPATH set --export --prepend --path MANPATH "$NIX_LINK/share/man" end +add_path "@localstatedir@/nix/profiles/default/bin" add_path "$NIX_LINK/bin" # Cleanup diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 60247b735..a5268bce6 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -16,7 +16,7 @@ #include "nix/store/globals.hh" #include "nix/util/serialise.hh" #include "nix/store/build-result.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/util/strings.hh" #include "nix/store/derivations.hh" #include "nix/store/local-store.hh" @@ -42,9 +42,9 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } -static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { +static bool allSupportedLocally(Store & store, const StringSet& requiredFeatures) { for (auto & feature : requiredFeatures) - if (!store.systemFeatures.get().count(feature)) return false; + if (!store.config.systemFeatures.get().count(feature)) return false; return true; } @@ -85,7 +85,7 @@ static int main_build_remote(int argc, char * * argv) that gets cleared on reboot, but it wouldn't work on macOS. */ auto currentLoadName = "/current-load"; if (auto localStore = store.dynamic_pointer_cast()) - currentLoad = std::string { localStore->stateDir } + currentLoadName; + currentLoad = std::string { localStore->config.stateDir } + currentLoadName; else currentLoad = settings.nixStateDir + currentLoadName; @@ -113,7 +113,7 @@ static int main_build_remote(int argc, char * * argv) auto amWilling = readInt(source); auto neededSystem = readString(source); drvPath = store->parseStorePath(readString(source)); - auto requiredFeatures = readStrings>(source); + auto requiredFeatures = readStrings(source); /* It would be possible to build locally after some builds clear out, so don't show the warning now: */ diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 56541fa57..31f64fd5a 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -3,7 +3,7 @@ #include "nix/cmd/command.hh" #include "nix/cmd/markdown.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/local-fs-store.hh" #include "nix/store/derivations.hh" #include "nix/expr/nixexpr.hh" @@ -14,12 +14,10 @@ namespace nix { -RegisterCommand::Commands * RegisterCommand::commands = nullptr; - nix::Commands RegisterCommand::getCommandsFor(const std::vector & prefix) { nix::Commands res; - for (auto & [name, command] : *RegisterCommand::commands) + for (auto & [name, command] : RegisterCommand::commands()) if (name.size() == prefix.size() + 1) { bool equal = true; for (size_t i = 0; i < prefix.size(); ++i) @@ -40,7 +38,7 @@ nlohmann::json NixMultiCommand::toJSON() void NixMultiCommand::run() { if (!command) { - std::set subCommandTextLines; + StringSet subCommandTextLines; for (auto & [name, _] : commands) subCommandTextLines.insert(fmt("- `%s`", name)); std::string markdownError = @@ -397,4 +395,11 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu } } +void MixOutLinkBase::createOutLinksMaybe(const std::vector & buildables, ref & store) +{ + if (outLink != "") + if (auto store2 = store.dynamic_pointer_cast()) + createOutLinks(outLink, toBuiltPaths(buildables), *store2); +} + } diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 21584b74a..01123a772 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -9,7 +9,7 @@ #include "nix/fetchers/registry.hh" #include "nix/flake/flakeref.hh" #include "nix/flake/settings.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/cmd/command.hh" #include "nix/fetchers/tarball.hh" #include "nix/fetchers/fetch-to-store.hh" @@ -18,7 +18,6 @@ namespace nix { -namespace fs { using namespace std::filesystem; } fetchers::Settings fetchSettings; @@ -122,8 +121,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(fetchSettings, _from, fs::current_path().string()); - auto to = parseFlakeRef(fetchSettings, _to, fs::current_path().string()); + auto from = parseFlakeRef(fetchSettings, _from, std::filesystem::current_path().string()); + auto to = parseFlakeRef(fetchSettings, _to, std::filesystem::current_path().string()); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index 11981a769..c14ed9bde 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -18,7 +18,7 @@ extern char ** savedArgv; class EvalState; struct Pos; class Store; -class LocalFSStore; +struct LocalFSStore; static constexpr Command::Category catHelp = -1; static constexpr Command::Category catSecondary = 100; @@ -287,13 +287,16 @@ struct StorePathCommand : public StorePathsCommand struct RegisterCommand { typedef std::map, std::function()>> Commands; - static Commands * commands; + + static Commands & commands() + { + static Commands commands; + return commands; + } RegisterCommand(std::vector && name, std::function()> command) { - if (!commands) - commands = new Commands; - commands->emplace(name, command); + commands().emplace(name, command); } static nix::Commands getCommandsFor(const std::vector & prefix); @@ -365,7 +368,7 @@ void completeFlakeRefWithFragment( const Strings & defaultFlakeAttrPaths, std::string_view prefix); -std::string showVersions(const std::set & versions); +std::string showVersions(const StringSet & versions); void printClosureDiff( ref store, const StorePath & beforePath, const StorePath & afterPath, std::string_view indent); @@ -376,4 +379,41 @@ void printClosureDiff( */ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store); -} +/** `outLink` parameter, `createOutLinksMaybe` method. See `MixOutLinkByDefault`. */ +struct MixOutLinkBase : virtual Args +{ + /** Prefix for any output symlinks. Empty means do not write an output symlink. */ + Path outLink; + + MixOutLinkBase(const std::string & defaultOutLink) + : outLink(defaultOutLink) + { + } + + void createOutLinksMaybe(const std::vector & buildables, ref & store); +}; + +/** `--out-link`, `--no-link`, `createOutLinksMaybe` */ +struct MixOutLinkByDefault : MixOutLinkBase, virtual Args +{ + MixOutLinkByDefault() + : MixOutLinkBase("result") + { + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.", + .labels = {"path"}, + .handler = {&outLink}, + .completer = completePath, + }); + + addFlag({ + .longName = "no-link", + .description = "Do not create symlinks to the build results.", + .handler = {&outLink, Path("")}, + }); + } +}; + +} // namespace nix diff --git a/src/libcmd/include/nix/cmd/installable-value.hh b/src/libcmd/include/nix/cmd/installable-value.hh index 9c8f1a9fb..e65c199a5 100644 --- a/src/libcmd/include/nix/cmd/installable-value.hh +++ b/src/libcmd/include/nix/cmd/installable-value.hh @@ -21,6 +21,7 @@ struct App struct UnresolvedApp { App unresolved; + std::vector build(ref evalStore, ref store); App resolve(ref evalStore, ref store); }; diff --git a/src/libcmd/include/nix/cmd/legacy.hh b/src/libcmd/include/nix/cmd/legacy.hh index 357500a4d..0c375a7d2 100644 --- a/src/libcmd/include/nix/cmd/legacy.hh +++ b/src/libcmd/include/nix/cmd/legacy.hh @@ -12,12 +12,15 @@ typedef std::function MainFunction; struct RegisterLegacyCommand { typedef std::map Commands; - static Commands * commands; + + static Commands & commands() { + static Commands commands; + return commands; + } RegisterLegacyCommand(const std::string & name, MainFunction fun) { - if (!commands) commands = new Commands; - (*commands)[name] = fun; + commands()[name] = fun; } }; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index fcbfe1482..7783b4f40 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -72,7 +72,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() auto newOutputs = std::visit(overloaded { [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { - std::set outputsToInstall; + StringSet outputsToInstall; for (auto & output : packageInfo.queryOutputs(false, true)) outputsToInstall.insert(output.first); if (outputsToInstall.empty()) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 83285b739..85a4188a7 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -117,7 +117,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() .drvPath = makeConstantStorePathRef(std::move(drvPath)), .outputs = std::visit(overloaded { [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { - std::set outputsToInstall; + StringSet outputsToInstall; if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { if (aOutputSpecified->getBool()) { if (auto aOutputName = attr->maybeGetAttr("outputName")) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 13a9dc70f..85fb3eabd 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -31,8 +31,6 @@ namespace nix { -namespace fs { using namespace std::filesystem; } - void completeFlakeInputAttrPath( AddCompletions & completions, ref evalState, @@ -343,7 +341,7 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), fs::current_path().string()); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), std::filesystem::current_path().string()); auto evalCache = openEvalCache(*evalState, std::make_shared(lockFlake( @@ -488,7 +486,11 @@ Installables SourceExprCommand::parseInstallables( throw UsageError("'--file' and '--expr' are exclusive"); // FIXME: backward compatibility hack - if (file) evalSettings.pureEval = false; + if (file) { + if (evalSettings.pureEval && evalSettings.pureEval.overridden) + throw UsageError("'--file' is not compatible with '--pure-eval'"); + evalSettings.pureEval = false; + } auto state = getEvalState(); auto vFile = state->allocValue(); diff --git a/src/libcmd/legacy.cc b/src/libcmd/legacy.cc deleted file mode 100644 index 69b066831..000000000 --- a/src/libcmd/legacy.cc +++ /dev/null @@ -1,7 +0,0 @@ -#include "nix/cmd/legacy.hh" - -namespace nix { - -RegisterLegacyCommand::Commands * RegisterLegacyCommand::commands = 0; - -} diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 32f44697d..216d4df9c 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -71,7 +71,6 @@ sources = files( 'installable-flake.cc', 'installable-value.cc', 'installables.cc', - 'legacy.cc', 'markdown.cc', 'misc-store-flags.cc', 'network-proxy.cc', diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 0da2cc615..769935efa 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -135,7 +135,7 @@ static constexpr const char * promptForType(ReplPromptType promptType) case ReplPromptType::ReplPrompt: return "nix-repl> "; case ReplPromptType::ContinuationPrompt: - return " "; + return " > "; // 9 spaces + > } assert(false); } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index c5a95268b..f9ac59d36 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -12,7 +12,7 @@ #include "nix/expr/eval-settings.hh" #include "nix/expr/attr-path.hh" #include "nix/util/signals.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/log-store.hh" #include "nix/cmd/common-eval-args.hh" #include "nix/expr/get-drvs.hh" @@ -61,7 +61,10 @@ struct NixRepl { size_t debugTraceIndex; + // Arguments passed to :load, saved so they can be reloaded with :reload Strings loadedFiles; + // Arguments passed to :load-flake, saved so they can be reloaded with :reload + Strings loadedFlakes; std::function getValues; const static int envSize = 32768; @@ -90,7 +93,8 @@ struct NixRepl void loadFile(const Path & path); void loadFlake(const std::string & flakeRef); void loadFiles(); - void reloadFiles(); + void loadFlakes(); + void reloadFilesAndFlakes(); void addAttrsToScope(Value & attrs); void addVarToScope(const Symbol name, Value & v); Expr * parseString(std::string s); @@ -244,14 +248,13 @@ StringSet NixRepl::completePrefix(const std::string & prefix) try { auto dir = std::string(cur, 0, slash); auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) { + for (auto & entry : DirectoryIterator{dir == "" ? "/" : dir}) { checkInterrupt(); auto name = entry.path().filename().string(); if (name[0] != '.' && hasPrefix(name, prefix2)) completions.insert(prev + entry.path().string()); } } catch (Error &) { - } catch (std::filesystem::filesystem_error &) { } } else if ((dot = cur.rfind('.')) == std::string::npos) { /* This is a variable name; look it up in the current scope. */ @@ -467,7 +470,7 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":r" || command == ":reload") { state->resetFileCache(); - reloadFiles(); + reloadFilesAndFlakes(); } else if (command == ":e" || command == ":edit") { @@ -502,7 +505,7 @@ ProcessLineResult NixRepl::processLine(std::string line) // Reload right after exiting the editor state->resetFileCache(); - reloadFiles(); + reloadFilesAndFlakes(); } else if (command == ":t") { @@ -717,6 +720,9 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); + loadedFlakes.remove(flakeRefS); + loadedFlakes.push_back(flakeRefS); + std::filesystem::path cwd; try { cwd = std::filesystem::current_path(); @@ -755,11 +761,12 @@ void NixRepl::initEnv() } -void NixRepl::reloadFiles() +void NixRepl::reloadFilesAndFlakes() { initEnv(); loadFiles(); + loadFlakes(); } @@ -780,6 +787,18 @@ void NixRepl::loadFiles() } +void NixRepl::loadFlakes() +{ + Strings old = loadedFlakes; + loadedFlakes.clear(); + + for (auto & i : old) { + notice("Loading flake '%1%'...", i); + loadFlake(i); + } +} + + void NixRepl::addAttrsToScope(Value & attrs) { state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 7c11ca9cb..ed4582e40 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -37,13 +37,11 @@ include_dirs = [include_directories('.')] headers = files( 'nix_api_expr.h', + 'nix_api_expr_internal.h', 'nix_api_external.h', 'nix_api_value.h', ) -# TODO move this header to libexpr, maybe don't use it in tests? -headers += files('nix_api_expr_internal.h') - subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index f34b1b77f..efaebf0e7 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -199,7 +199,9 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c != NIX_OK) return nullptr; - return nix_eval_state_build(context, builder); + auto *state = nix_eval_state_build(context, builder); + nix_eval_state_builder_free(builder); + return state; } void nix_state_free(EvalState * state) diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index f8d181452..2be739955 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -286,6 +286,11 @@ nix_err nix_gc_incref(nix_c_context * context, const void * object); /** * @brief Decrement the garbage collector reference counter for the given object * + * We also provide typed `nix_*_decref` functions, which are + * - safer to use + * - easier to integrate when deriving bindings + * - allow more flexibility + * * @param[out] context Optional, stores error information * @param[in] object The object to stop referencing */ diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index d0ccd970a..a7522278d 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -33,7 +33,7 @@ namespace nix { ASSERT_EQ(PrintToString(e.info().msg), PrintToString(HintFmt("puppy"))); auto trace = e.info().traces.rbegin(); - ASSERT_EQ(e.info().traces.size(), 2); + ASSERT_EQ(e.info().traces.size(), 2u); ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("doggy"))); trace++; @@ -54,8 +54,8 @@ namespace nix { } catch (Error & e2) { e.addTrace(state.positions[noPos], "beans2"); //e2.addTrace(state.positions[noPos], "Something", ""); - ASSERT_TRUE(e.info().traces.size() == 2); - ASSERT_TRUE(e2.info().traces.size() == 0); + ASSERT_TRUE(e.info().traces.size() == 2u); + ASSERT_TRUE(e2.info().traces.size() == 0u); ASSERT_FALSE(&e.info() == &e2.info()); } } @@ -71,7 +71,7 @@ namespace nix { } catch (BaseError & e) { \ ASSERT_EQ(PrintToString(e.info().msg), \ PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 1) << "while testing " args << std::endl << e.what(); \ + ASSERT_EQ(e.info().traces.size(), 1u) << "while testing " args << std::endl << e.what(); \ auto trace = e.info().traces.rbegin(); \ ASSERT_EQ(PrintToString(trace->hint), \ PrintToString(HintFmt("while calling the '%s' builtin", name))); \ @@ -90,7 +90,7 @@ namespace nix { } catch (BaseError & e) { \ ASSERT_EQ(PrintToString(e.info().msg), \ PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 2) << "while testing " args << std::endl << e.what(); \ + ASSERT_EQ(e.info().traces.size(), 2u) << "while testing " args << std::endl << e.what(); \ auto trace = e.info().traces.rbegin(); \ ASSERT_EQ(PrintToString(trace->hint), \ PrintToString(context)); \ @@ -112,7 +112,7 @@ namespace nix { } catch (BaseError & e) { \ ASSERT_EQ(PrintToString(e.info().msg), \ PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 3) << "while testing " args << std::endl << e.what(); \ + ASSERT_EQ(e.info().traces.size(), 3u) << "while testing " args << std::endl << e.what(); \ auto trace = e.info().traces.rbegin(); \ ASSERT_EQ(PrintToString(trace->hint), \ PrintToString(context1)); \ @@ -137,7 +137,7 @@ namespace nix { } catch (BaseError & e) { \ ASSERT_EQ(PrintToString(e.info().msg), \ PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 4) << "while testing " args << std::endl << e.what(); \ + ASSERT_EQ(e.info().traces.size(), 4u) << "while testing " args << std::endl << e.what(); \ auto trace = e.info().traces.rbegin(); \ ASSERT_EQ(PrintToString(trace->hint), \ PrintToString(context1)); \ diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index e2eeace6c..f3b6fed0e 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -222,7 +222,7 @@ TEST_F(nix_api_expr_test, nix_expr_realise_context) names.push_back(name); } std::sort(names.begin(), names.end()); - ASSERT_EQ(3, names.size()); + ASSERT_EQ(3u, names.size()); EXPECT_THAT(names[0], testing::StrEq("just-a-file")); EXPECT_THAT(names[1], testing::StrEq("letsbuild")); EXPECT_THAT(names[2], testing::StrEq("not-actually-built-yet.drv")); diff --git a/src/libexpr-tests/nix_api_external.cc b/src/libexpr-tests/nix_api_external.cc index b32326f9e..c1deabad6 100644 --- a/src/libexpr-tests/nix_api_external.cc +++ b/src/libexpr-tests/nix_api_external.cc @@ -63,6 +63,9 @@ TEST_F(nix_api_expr_test, nix_expr_eval_external) std::string string_value; nix_get_string(nullptr, valueResult, OBSERVE_STRING(string_value)); ASSERT_STREQ("nix-external", string_value.c_str()); + + nix_state_free(stateResult); + nix_state_free(stateFn); } } diff --git a/src/libexpr-tests/nix_api_value.cc b/src/libexpr-tests/nix_api_value.cc index 14f8bd0b0..1da980ab8 100644 --- a/src/libexpr-tests/nix_api_value.cc +++ b/src/libexpr-tests/nix_api_value.cc @@ -134,12 +134,12 @@ TEST_F(nix_api_expr_test, nix_build_and_init_list_invalid) { ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, nullptr, state, 0)); assert_ctx_err(); - ASSERT_EQ(0, nix_get_list_size(ctx, nullptr)); + ASSERT_EQ(0u, nix_get_list_size(ctx, nullptr)); assert_ctx_err(); ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, 0)); assert_ctx_err(); - ASSERT_EQ(0, nix_get_list_size(ctx, value)); + ASSERT_EQ(0u, nix_get_list_size(ctx, value)); assert_ctx_err(); } @@ -163,7 +163,7 @@ TEST_F(nix_api_expr_test, nix_build_and_init_list) ASSERT_EQ(42, nix_get_int(ctx, nix_get_list_byidx(ctx, value, state, 0))); ASSERT_EQ(43, nix_get_int(ctx, nix_get_list_byidx(ctx, value, state, 1))); ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, 2)); - ASSERT_EQ(10, nix_get_list_size(ctx, value)); + ASSERT_EQ(10u, nix_get_list_size(ctx, value)); ASSERT_STREQ("a list", nix_get_typename(ctx, value)); ASSERT_EQ(NIX_TYPE_LIST, nix_get_type(ctx, value)); @@ -180,7 +180,7 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr_invalid) assert_ctx_err(); ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, nullptr, state, 0)); assert_ctx_err(); - ASSERT_EQ(0, nix_get_attrs_size(ctx, nullptr)); + ASSERT_EQ(0u, nix_get_attrs_size(ctx, nullptr)); assert_ctx_err(); ASSERT_EQ(false, nix_has_attr_byname(ctx, nullptr, state, "no-value")); assert_ctx_err(); @@ -191,7 +191,7 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr_invalid) assert_ctx_err(); ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, 0)); assert_ctx_err(); - ASSERT_EQ(0, nix_get_attrs_size(ctx, value)); + ASSERT_EQ(0u, nix_get_attrs_size(ctx, value)); assert_ctx_err(); ASSERT_EQ(false, nix_has_attr_byname(ctx, value, state, "no-value")); assert_ctx_err(); @@ -215,7 +215,7 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr) nix_make_attrs(ctx, value, builder); nix_bindings_builder_free(builder); - ASSERT_EQ(2, nix_get_attrs_size(ctx, value)); + ASSERT_EQ(2u, nix_get_attrs_size(ctx, value)); nix_value * out_value = nix_get_attr_byname(ctx, value, state, "a"); ASSERT_EQ(42, nix_get_int(ctx, out_value)); @@ -374,7 +374,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) auto n = nix_get_attrs_size(ctx, r); assert_ctx_ok(); - ASSERT_EQ(1, n); + ASSERT_EQ(1u, n); // nix_get_attr_byname isn't lazy (it could have been) so it will throw the exception nix_value * foo = nix_get_attr_byname(ctx, r, state, "foo"); diff --git a/src/libexpr-tests/primops.cc b/src/libexpr-tests/primops.cc index 66850d78b..7695a587a 100644 --- a/src/libexpr-tests/primops.cc +++ b/src/libexpr-tests/primops.cc @@ -56,11 +56,31 @@ namespace nix { TEST_F(PrimOpTest, ceil) { auto v = eval("builtins.ceil 1.9"); ASSERT_THAT(v, IsIntEq(2)); + auto intMin = eval("builtins.ceil (-4611686018427387904 - 4611686018427387904)"); + ASSERT_THAT(intMin, IsIntEq(std::numeric_limits::min())); + ASSERT_THROW(eval("builtins.ceil 1.0e200"), EvalError); + ASSERT_THROW(eval("builtins.ceil -1.0e200"), EvalError); + ASSERT_THROW(eval("builtins.ceil (1.0e200 * 1.0e200)"), EvalError); // inf + ASSERT_THROW(eval("builtins.ceil (-1.0e200 * 1.0e200)"), EvalError); // -inf + ASSERT_THROW(eval("builtins.ceil (1.0e200 * 1.0e200 - 1.0e200 * 1.0e200)"), EvalError); // nan + // bugs in previous Nix versions + ASSERT_THROW(eval("builtins.ceil (4611686018427387904 + 4611686018427387903)"), EvalError); + ASSERT_THROW(eval("builtins.ceil (-4611686018427387904 - 4611686018427387903)"), EvalError); } TEST_F(PrimOpTest, floor) { auto v = eval("builtins.floor 1.9"); ASSERT_THAT(v, IsIntEq(1)); + auto intMin = eval("builtins.ceil (-4611686018427387904 - 4611686018427387904)"); + ASSERT_THAT(intMin, IsIntEq(std::numeric_limits::min())); + ASSERT_THROW(eval("builtins.ceil 1.0e200"), EvalError); + ASSERT_THROW(eval("builtins.ceil -1.0e200"), EvalError); + ASSERT_THROW(eval("builtins.ceil (1.0e200 * 1.0e200)"), EvalError); // inf + ASSERT_THROW(eval("builtins.ceil (-1.0e200 * 1.0e200)"), EvalError); // -inf + ASSERT_THROW(eval("builtins.ceil (1.0e200 * 1.0e200 - 1.0e200 * 1.0e200)"), EvalError); // nan + // bugs in previous Nix versions + ASSERT_THROW(eval("builtins.ceil (4611686018427387904 + 4611686018427387903)"), EvalError); + ASSERT_THROW(eval("builtins.ceil (-4611686018427387904 - 4611686018427387903)"), EvalError); } TEST_F(PrimOpTest, tryEvalFailure) { @@ -204,7 +224,7 @@ namespace nix { auto v = eval("builtins.listToAttrs []"); ASSERT_THAT(v, IsAttrsOfSize(0)); ASSERT_EQ(v.type(), nAttrs); - ASSERT_EQ(v.attrs()->size(), 0); + ASSERT_EQ(v.attrs()->size(), 0u); } TEST_F(PrimOpTest, listToAttrsNotFieldName) { @@ -383,7 +403,7 @@ namespace nix { TEST_F(PrimOpTest, genList) { auto v = eval("builtins.genList (x: x + 1) 3"); ASSERT_EQ(v.type(), nList); - ASSERT_EQ(v.listSize(), 3); + ASSERT_EQ(v.listSize(), 3u); for (const auto [i, elem] : enumerate(v.listItems())) { ASSERT_THAT(*elem, IsThunk()); state.forceValue(*elem, noPos); @@ -394,7 +414,7 @@ namespace nix { TEST_F(PrimOpTest, sortLessThan) { auto v = eval("builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]"); ASSERT_EQ(v.type(), nList); - ASSERT_EQ(v.listSize(), 6); + ASSERT_EQ(v.listSize(), 6u); const std::vector numbers = { 42, 77, 147, 249, 483, 526 }; for (const auto [n, elem] : enumerate(v.listItems())) @@ -414,7 +434,7 @@ namespace nix { auto wrong = v.attrs()->get(createSymbol("wrong")); ASSERT_NE(wrong, nullptr); ASSERT_EQ(wrong->value->type(), nList); - ASSERT_EQ(wrong->value->listSize(), 3); + ASSERT_EQ(wrong->value->listSize(), 3u); ASSERT_THAT(*wrong->value, IsListOfSize(3)); ASSERT_THAT(*wrong->value->listElems()[0], IsIntEq(1)); ASSERT_THAT(*wrong->value->listElems()[1], IsIntEq(9)); @@ -424,7 +444,7 @@ namespace nix { TEST_F(PrimOpTest, concatMap) { auto v = eval("builtins.concatMap (x: x ++ [0]) [ [1 2] [3 4] ]"); ASSERT_EQ(v.type(), nList); - ASSERT_EQ(v.listSize(), 6); + ASSERT_EQ(v.listSize(), 6u); const std::vector numbers = { 1, 2, 0, 3, 4, 0 }; for (const auto [n, elem] : enumerate(v.listItems())) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index cee805d14..722b57bbf 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -74,7 +74,7 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin auto a = v->attrs()->get(state.symbols.create(attr)); if (!a) { - std::set attrNames; + StringSet attrNames; for (auto & attr : *v->attrs()) attrNames.insert(std::string(state.symbols[attr.name])); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 4e44e68cf..72a6b60ea 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -492,7 +492,7 @@ Value & AttrCursor::forceValue() Suggestions AttrCursor::getSuggestionsForAttr(Symbol name) { auto attrNames = getAttrs(); - std::set strAttrNames; + StringSet strAttrNames; for (auto & name : attrNames) strAttrNames.insert(std::string(root->state.symbols[name])); diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 2c8b6e325..eac135008 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -110,5 +110,6 @@ template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; +template class EvalErrorBuilder; } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 85c044c2f..531a932bd 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -252,7 +252,25 @@ EvalState::EvalState( makeMountedSourceAccessor( { {CanonPath::root, makeEmptySourceAccessor()}, - {CanonPath(store->storeDir), makeFSSourceAccessor(dirOf(store->toRealPath(StorePath::dummy)))} + /* In the pure eval case, we can simply require + valid paths. However, in the *impure* eval + case this gets in the way of the union + mechanism, because an invalid access in the + upper layer will *not* be caught by the union + source accessor, but instead abort the entire + lookup. + + This happens when the store dir in the + ambient file system has a path (e.g. because + another Nix store there), but the relocated + store does not. + + TODO make the various source accessors doing + access control all throw the same type of + exception, and make union source accessor + catch it, so we don't need to do this hack. + */ + {CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)}, })) , rootFS( ({ @@ -1435,7 +1453,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } else { state.forceAttrs(*vAttrs, pos, "while selecting an attribute"); if (!(j = vAttrs->attrs()->get(name))) { - std::set allAttrNames; + StringSet allAttrNames; for (auto & attr : *vAttrs->attrs()) allAttrNames.insert(std::string(state.symbols[attr.name])); auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); @@ -1592,7 +1610,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, user. */ for (auto & i : *args[0]->attrs()) if (!lambda.formals->has(i.name)) { - std::set formalNames; + StringSet formalNames; for (auto & formal : lambda.formals->formals) formalNames.insert(std::string(symbols[formal.name])); auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]); diff --git a/src/libexpr/include/nix/expr/eval-error.hh b/src/libexpr/include/nix/expr/eval-error.hh index ae4f40689..6f4c37f90 100644 --- a/src/libexpr/include/nix/expr/eval-error.hh +++ b/src/libexpr/include/nix/expr/eval-error.hh @@ -54,6 +54,7 @@ MakeError(TypeError, EvalError); MakeError(UndefinedVarError, EvalError); MakeError(MissingArgumentError, EvalError); MakeError(InfiniteRecursionError, EvalError); +MakeError(IFDError, EvalBaseError); struct InvalidPathError : public EvalError { diff --git a/src/libexpr/include/nix/expr/primops.hh b/src/libexpr/include/nix/expr/primops.hh index f0742a138..0b4ecdd50 100644 --- a/src/libexpr/include/nix/expr/primops.hh +++ b/src/libexpr/include/nix/expr/primops.hh @@ -27,7 +27,12 @@ constexpr size_t conservativeStackReservation = 16; struct RegisterPrimOp { typedef std::vector PrimOps; - static PrimOps * primOps; + + static PrimOps & primOps() + { + static PrimOps primOps; + return primOps; + } /** * You can register a constant by passing an arity of 0. fun diff --git a/src/libexpr/include/nix/expr/symbol-table.hh b/src/libexpr/include/nix/expr/symbol-table.hh index 018465bf5..c04cc041b 100644 --- a/src/libexpr/include/nix/expr/symbol-table.hh +++ b/src/libexpr/include/nix/expr/symbol-table.hh @@ -81,26 +81,29 @@ public: class SymbolTable { private: - std::unordered_map> symbols; + /** + * Map from string view (backed by ChunkedVector) -> offset into the store. + * ChunkedVector references are never invalidated. + */ + std::unordered_map symbols; ChunkedVector store{16}; public: /** - * converts a string into a symbol. + * Converts a string into a symbol. */ Symbol create(std::string_view s) { // Most symbols are looked up more than once, so we trade off insertion performance // for lookup performance. - // TODO: could probably be done more efficiently with transparent Hash and Equals - // on the original implementation using unordered_set // FIXME: make this thread-safe. auto it = symbols.find(s); - if (it != symbols.end()) return Symbol(it->second.second + 1); + if (it != symbols.end()) + return Symbol(it->second + 1); - const auto & [rawSym, idx] = store.add(std::string(s)); - symbols.emplace(rawSym, std::make_pair(&rawSym, idx)); + const auto & [rawSym, idx] = store.add(s); + symbols.emplace(rawSym, idx); return Symbol(idx + 1); } diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 1e196741d..1005f9f7e 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -1,3 +1,4 @@ +%option 8bit %option reentrant bison-bridge bison-locations %option align %option noyywrap diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 2e773938d..2b465b85a 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -112,6 +112,7 @@ lexer_tab = custom_target( ], command : [ 'flex', + '-Cf', # Use full scanner tables '--outfile', '@OUTPUT0@', '--header-file=' + '@OUTPUT1@', diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 99cc687cc..8878b86c2 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -179,7 +179,12 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { %% -start: expr { state->result = $1; }; +start: expr { + state->result = $1; + + // This parser does not use yynerrs; suppress the warning. + (void) yynerrs; +}; expr: expr_function; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c6a97fdae..44f7833e0 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -98,7 +98,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (drvs.empty()) return {}; if (isIFD && !settings.enableImportFromDerivation) - error( + error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", drvs.begin()->to_string(*store) ).debugThrow(); @@ -895,18 +895,40 @@ static void prim_ceil(EvalState & state, const PosIdx pos, Value * * args, Value { auto value = state.forceFloat(*args[0], args[0]->determinePos(pos), "while evaluating the first argument passed to builtins.ceil"); - v.mkInt(ceil(value)); + auto ceilValue = ceil(value); + bool isInt = args[0]->type() == nInt; + constexpr NixFloat int_min = std::numeric_limits::min(); // power of 2, so that no rounding occurs + if (ceilValue >= int_min && ceilValue < -int_min) { + v.mkInt(ceilValue); + } else if (isInt) { + // a NixInt, e.g. INT64_MAX, can be rounded to -int_min due to the cast to NixFloat + state.error("Due to a bug (see https://github.com/NixOS/nix/issues/12899) the NixInt argument %1% caused undefined behavior in previous Nix versions.\n\tFuture Nix versions might implement the correct behavior.", args[0]->integer().value).atPos(pos).debugThrow(); + } else { + state.error("NixFloat argument %1% is not in the range of NixInt", args[0]->fpoint()).atPos(pos).debugThrow(); + } + // `forceFloat` casts NixInt to NixFloat, but instead NixInt args shall be returned unmodified + if (isInt) { + auto arg = args[0]->integer(); + auto res = v.integer(); + if (arg != res) { + state.error("Due to a bug (see https://github.com/NixOS/nix/issues/12899) a loss of precision occured in previous Nix versions because the NixInt argument %1% was rounded to %2%.\n\tFuture Nix versions might implement the correct behavior.", arg, res).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_ceil({ .name = "__ceil", - .args = {"double"}, + .args = {"number"}, .doc = R"( - Converts an IEEE-754 double-precision floating-point number (*double*) to - the next higher integer. + Rounds and converts *number* to the next higher NixInt value if possible, i.e. `ceil *number* >= *number*` and + `ceil *number* - *number* < 1`. - If the datatype is neither an integer nor a "float", an evaluation error will be - thrown. + An evaluation error is thrown, if there exists no such NixInt value `ceil *number*`. + Due to bugs in previous Nix versions an evaluation error might be thrown, if the datatype of *number* is + a NixInt and if `*number* < -9007199254740992` or `*number* > 9007199254740992`. + + If the datatype of *number* is neither a NixInt (signed 64-bit integer) nor a NixFloat + (IEEE-754 double-precision floating-point number), an evaluation error will be thrown. )", .fun = prim_ceil, }); @@ -914,18 +936,40 @@ static RegisterPrimOp primop_ceil({ static void prim_floor(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto value = state.forceFloat(*args[0], args[0]->determinePos(pos), "while evaluating the first argument passed to builtins.floor"); - v.mkInt(floor(value)); + auto floorValue = floor(value); + bool isInt = args[0]->type() == nInt; + constexpr NixFloat int_min = std::numeric_limits::min(); // power of 2, so that no rounding occurs + if (floorValue >= int_min && floorValue < -int_min) { + v.mkInt(floorValue); + } else if (isInt) { + // a NixInt, e.g. INT64_MAX, can be rounded to -int_min due to the cast to NixFloat + state.error("Due to a bug (see https://github.com/NixOS/nix/issues/12899) the NixInt argument %1% caused undefined behavior in previous Nix versions.\n\tFuture Nix versions might implement the correct behavior.", args[0]->integer().value).atPos(pos).debugThrow(); + } else { + state.error("NixFloat argument %1% is not in the range of NixInt", args[0]->fpoint()).atPos(pos).debugThrow(); + } + // `forceFloat` casts NixInt to NixFloat, but instead NixInt args shall be returned unmodified + if (isInt) { + auto arg = args[0]->integer(); + auto res = v.integer(); + if (arg != res) { + state.error("Due to a bug (see https://github.com/NixOS/nix/issues/12899) a loss of precision occured in previous Nix versions because the NixInt argument %1% was rounded to %2%.\n\tFuture Nix versions might implement the correct behavior.", arg, res).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_floor({ .name = "__floor", - .args = {"double"}, + .args = {"number"}, .doc = R"( - Converts an IEEE-754 double-precision floating-point number (*double*) to - the next lower integer. + Rounds and converts *number* to the next lower NixInt value if possible, i.e. `floor *number* <= *number*` and + `*number* - floor *number* < 1`. - If the datatype is neither an integer nor a "float", an evaluation error will be - thrown. + An evaluation error is thrown, if there exists no such NixInt value `floor *number*`. + Due to bugs in previous Nix versions an evaluation error might be thrown, if the datatype of *number* is + a NixInt and if `*number* < -9007199254740992` or `*number* > 9007199254740992`. + + If the datatype of *number* is neither a NixInt (signed 64-bit integer) nor a NixFloat + (IEEE-754 double-precision floating-point number), an evaluation error will be thrown. )", .fun = prim_floor, }); @@ -2813,7 +2857,13 @@ static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value * * static RegisterPrimOp primop_unsafeGetAttrPos(PrimOp { .name = "__unsafeGetAttrPos", + .args = {"s", "set"}, .arity = 2, + .doc = R"( + `unsafeGetAttrPos` returns the position of the attribute named *s* + from *set*. This is used by Nixpkgs to provide location information + in error messages. + )", .fun = prim_unsafeGetAttrPos, }); @@ -4299,9 +4349,7 @@ struct RegexCache { struct State { - // TODO use C++20 transparent comparison when available - std::unordered_map cache; - std::list keys; + std::unordered_map> cache; }; Sync state_; @@ -4312,8 +4360,14 @@ struct RegexCache auto it = state->cache.find(re); if (it != state->cache.end()) return it->second; - state->keys.emplace_back(re); - return state->cache.emplace(state->keys.back(), std::regex(state->keys.back(), std::regex::extended)).first->second; + /* No std::regex constructor overload from std::string_view, but can be constructed + from a pointer + size or an iterator range. */ + return state->cache + .emplace( + std::piecewise_construct, + std::forward_as_tuple(re), + std::forward_as_tuple(/*s=*/re.data(), /*count=*/re.size(), std::regex::extended)) + .first->second; } }; @@ -4697,13 +4751,9 @@ static RegisterPrimOp primop_splitVersion({ *************************************************************/ -RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; - - RegisterPrimOp::RegisterPrimOp(PrimOp && primOp) { - if (!primOps) primOps = new PrimOps; - primOps->push_back(std::move(primOp)); + primOps().push_back(std::move(primOp)); } @@ -4957,14 +5007,18 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) )", }); - if (RegisterPrimOp::primOps) - for (auto & primOp : *RegisterPrimOp::primOps) - if (experimentalFeatureSettings.isEnabled(primOp.experimentalFeature)) - { - auto primOpAdjusted = primOp; - primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity); - addPrimOp(std::move(primOpAdjusted)); - } + for (auto & primOp : RegisterPrimOp::primOps()) + if (experimentalFeatureSettings.isEnabled(primOp.experimentalFeature)) { + auto primOpAdjusted = primOp; + primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity); + addPrimOp(std::move(primOpAdjusted)); + } + + for (auto & primOp : evalSettings.extraPrimOps) { + auto primOpAdjusted = primOp; + primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity); + addPrimOp(std::move(primOpAdjusted)); + } for (auto & primOp : evalSettings.extraPrimOps) { auto primOpAdjusted = primOp; diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index d28680ae5..4dd8b2606 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -1,5 +1,5 @@ #include "nix/expr/primops.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/realisation.hh" #include "nix/store/make-content-addressed.hh" #include "nix/util/url.hh" diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index a50687f37..f51108459 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -11,6 +11,7 @@ namespace nix { using json = nlohmann::json; +// TODO: rename. It doesn't print. json printValueAsJSON(EvalState & state, bool strict, Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore) { diff --git a/src/libfetchers-c/.version b/src/libfetchers-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libfetchers-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libfetchers-c/meson.build b/src/libfetchers-c/meson.build new file mode 100644 index 000000000..e34997f09 --- /dev/null +++ b/src/libfetchers-c/meson.build @@ -0,0 +1,65 @@ +project('nix-fetchers-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('nix-meson-build-support/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('nix-meson-build-support/subprojects') + +add_project_arguments( + language : 'cpp', +) + +subdir('nix-meson-build-support/common') + +sources = files( + 'nix_api_fetchers.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'nix_api_fetchers.h', + 'nix_api_fetchers_internal.hh', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_fetchers.h') + +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') + +this_library = library( + 'nixfetchersc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, preserve_path : true) + +libraries_private = [] + +subdir('nix-meson-build-support/export') diff --git a/src/libfetchers-c/nix-meson-build-support b/src/libfetchers-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libfetchers-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libfetchers-c/nix_api_fetchers.cc b/src/libfetchers-c/nix_api_fetchers.cc new file mode 100644 index 000000000..4e8037a5e --- /dev/null +++ b/src/libfetchers-c/nix_api_fetchers.cc @@ -0,0 +1,19 @@ +#include "nix_api_fetchers.h" +#include "nix_api_fetchers_internal.hh" +#include "nix_api_util_internal.h" + +nix_fetchers_settings * nix_fetchers_settings_new(nix_c_context * context) +{ + try { + auto fetchersSettings = nix::make_ref(nix::fetchers::Settings{}); + return new nix_fetchers_settings{ + .settings = fetchersSettings, + }; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_fetchers_settings_free(nix_fetchers_settings * settings) +{ + delete settings; +} diff --git a/src/libfetchers-c/nix_api_fetchers.h b/src/libfetchers-c/nix_api_fetchers.h new file mode 100644 index 000000000..19da112a6 --- /dev/null +++ b/src/libfetchers-c/nix_api_fetchers.h @@ -0,0 +1,32 @@ +#ifndef NIX_API_FETCHERS_H +#define NIX_API_FETCHERS_H +/** @defgroup libfetchers libfetchers + * @brief Bindings to the Nix fetchers library + * @{ + */ +/** @file + * @brief Main entry for the libfetchers C bindings + */ + +#include "nix_api_util.h" + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +// Type definitions +/** + * @brief Shared settings object + */ +typedef struct nix_fetchers_settings nix_fetchers_settings; + +nix_fetchers_settings * nix_fetchers_settings_new(nix_c_context * context); + +void nix_fetchers_settings_free(nix_fetchers_settings * settings); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // NIX_API_FETCHERS_H \ No newline at end of file diff --git a/src/libfetchers-c/nix_api_fetchers_internal.hh b/src/libfetchers-c/nix_api_fetchers_internal.hh new file mode 100644 index 000000000..b0dea5754 --- /dev/null +++ b/src/libfetchers-c/nix_api_fetchers_internal.hh @@ -0,0 +1,12 @@ +#pragma once +#include "nix/fetchers/fetch-settings.hh" +#include "nix/util/ref.hh" + +/** + * A shared reference to `nix::fetchers::Settings` + * @see nix::fetchers::Settings + */ +struct nix_fetchers_settings +{ + nix::ref settings; +}; diff --git a/src/libfetchers-c/package.nix b/src/libfetchers-c/package.nix new file mode 100644 index 000000000..9a601d704 --- /dev/null +++ b/src/libfetchers-c/package.nix @@ -0,0 +1,50 @@ +{ + lib, + mkMesonLibrary, + + nix-store-c, + nix-expr-c, + nix-util-c, + nix-fetchers, + + # Configuration Options + + version, +}: + +let + inherit (lib) fileset; +in + +mkMesonLibrary (finalAttrs: { + pname = "nix-fetchers-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../nix-meson-build-support + ./nix-meson-build-support + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + propagatedBuildInputs = [ + nix-util-c + nix-expr-c + nix-store-c + nix-fetchers + ]; + + mesonFlags = [ + ]; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index ceac809de..c2c7f9da0 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -11,14 +11,10 @@ namespace nix { -namespace fs { -using namespace std::filesystem; -} - class GitUtilsTest : public ::testing::Test { // We use a single repository for all tests. - fs::path tmpDir; + std::filesystem::path tmpDir; std::unique_ptr delTmpDir; public: @@ -90,11 +86,11 @@ TEST_F(GitUtilsTest, sink_basic) auto result = repo->dereferenceSingletonDirectory(sink->flush()); auto accessor = repo->getAccessor(result, false, getRepoName()); auto entries = accessor->readDirectory(CanonPath::root); - ASSERT_EQ(entries.size(), 5); + ASSERT_EQ(entries.size(), 5u); ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world"); ASSERT_EQ(accessor->readFile(CanonPath("bye")), "thanks for all the fish"); ASSERT_EQ(accessor->readLink(CanonPath("bye-link")), "bye"); - ASSERT_EQ(accessor->readDirectory(CanonPath("empty")).size(), 0); + ASSERT_EQ(accessor->readDirectory(CanonPath("empty")).size(), 0u); ASSERT_EQ(accessor->readFile(CanonPath("links/foo")), "hello world"); }; diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index 12b748e65..33bc7f30e 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -17,6 +17,7 @@ subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-store-test-support'), dependency('nix-fetchers'), + dependency('nix-fetchers-c'), ] deps_public_maybe_subproject = [ ] @@ -39,6 +40,7 @@ subdir('nix-meson-build-support/common') sources = files( 'access-tokens.cc', 'git-utils.cc', + 'nix_api_fetchers.cc', 'public-key.cc', ) diff --git a/src/libfetchers-tests/nix_api_fetchers.cc b/src/libfetchers-tests/nix_api_fetchers.cc new file mode 100644 index 000000000..8f3e6e3c5 --- /dev/null +++ b/src/libfetchers-tests/nix_api_fetchers.cc @@ -0,0 +1,18 @@ +#include "gmock/gmock.h" +#include + +#include "nix_api_fetchers.h" +#include "nix/store/tests/nix_api_store.hh" + +namespace nixC { + +TEST_F(nix_api_store_test, nix_api_fetchers_new_free) +{ + nix_fetchers_settings * settings = nix_fetchers_settings_new(ctx); + assert_ctx_ok(); + ASSERT_NE(nullptr, settings); + + nix_fetchers_settings_free(settings); +} + +} // namespace nixC diff --git a/src/libfetchers-tests/package.nix b/src/libfetchers-tests/package.nix index 6e3581183..48c1a07d8 100644 --- a/src/libfetchers-tests/package.nix +++ b/src/libfetchers-tests/package.nix @@ -5,6 +5,7 @@ mkMesonExecutable, nix-fetchers, + nix-fetchers-c, nix-store-test-support, libgit2, @@ -40,6 +41,7 @@ mkMesonExecutable (finalAttrs: { buildInputs = [ nix-fetchers + nix-fetchers-c nix-store-test-support rapidcheck gtest diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 33301933c..614b3c90e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -12,24 +12,26 @@ namespace nix::fetchers { using InputSchemeMap = std::map>; -std::unique_ptr inputSchemes = nullptr; +static InputSchemeMap & inputSchemes() +{ + static InputSchemeMap inputSchemeMap; + return inputSchemeMap; +} void registerInputScheme(std::shared_ptr && inputScheme) { - if (!inputSchemes) - inputSchemes = std::make_unique(); auto schemeName = inputScheme->schemeName(); - if (inputSchemes->count(schemeName) > 0) + if (!inputSchemes().emplace(schemeName, std::move(inputScheme)).second) throw Error("Input scheme with name %s already registered", schemeName); - inputSchemes->insert_or_assign(schemeName, std::move(inputScheme)); } -nlohmann::json dumpRegisterInputSchemeInfo() { +nlohmann::json dumpRegisterInputSchemeInfo() +{ using nlohmann::json; auto res = json::object(); - for (auto & [name, scheme] : *inputSchemes) { + for (auto & [name, scheme] : inputSchemes()) { auto & r = res[name] = json::object(); r["allowedAttrs"] = scheme->allowedAttrs(); } @@ -57,7 +59,7 @@ Input Input::fromURL( const Settings & settings, const ParsedURL & url, bool requireTree) { - for (auto & [_, inputScheme] : *inputSchemes) { + for (auto & [_, inputScheme] : inputSchemes()) { auto res = inputScheme->inputFromURL(settings, url, requireTree); if (res) { experimentalFeatureSettings.require(inputScheme->experimentalFeature()); @@ -91,8 +93,8 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs) }; std::shared_ptr inputScheme = ({ - auto i = inputSchemes->find(schemeName); - i == inputSchemes->end() ? nullptr : i->second; + auto i = get(inputSchemes(), schemeName); + i ? *i : nullptr; }); if (!inputScheme) return raw(); diff --git a/src/libfetchers/git-lfs-fetch.cc b/src/libfetchers/git-lfs-fetch.cc index dbf4b1eb9..97f10f0c6 100644 --- a/src/libfetchers/git-lfs-fetch.cc +++ b/src/libfetchers/git-lfs-fetch.cc @@ -44,10 +44,11 @@ static void downloadToSink( static std::string getLfsApiToken(const ParsedURL & url) { - auto [status, output] = runProgram(RunOptions{ - .program = "ssh", - .args = {*url.authority, "git-lfs-authenticate", url.path, "download"}, - }); + auto [status, output] = runProgram( + RunOptions{ + .program = "ssh", + .args = {*url.authority, "git-lfs-authenticate", url.path, "download"}, + }); if (output.empty()) throw Error( diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 7e1f085f5..935d328d6 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1244,7 +1244,12 @@ std::vector> GitRepoImpl::getSubmodules auto configS = accessor->readFile(modulesFile); auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules"); - writeFull(fdTemp.get(), configS); + try { + writeFull(fdTemp.get(), configS); + } catch (SysError & e) { + e.addTrace({}, "while writing .gitmodules file to temporary file"); + throw; + } std::vector> result; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index ef74397ff..7730e0db4 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -39,7 +39,7 @@ const std::string gitInitialBranch = "__nix_dummy_branch"; bool isCacheFileWithinTtl(time_t now, const struct stat & st) { - return st.st_mtime + settings.tarballTtl > now; + return st.st_mtime + static_cast(settings.tarballTtl) > now; } Path getCachePath(std::string_view key, bool shallow) @@ -84,10 +84,9 @@ std::optional readHead(const Path & path) } // Persist the HEAD ref from the remote repo in the local cached repo. -bool storeCachedHead(const std::string & actualUrl, const std::string & headRef) +bool storeCachedHead(const std::string & actualUrl, bool shallow, const std::string & headRef) { - // set shallow=false as HEAD will never be queried for a shallow repo - Path cacheDir = getCachePath(actualUrl, false); + Path cacheDir = getCachePath(actualUrl, shallow); try { runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef }); } catch (ExecError &e) { @@ -106,12 +105,11 @@ bool storeCachedHead(const std::string & actualUrl, const std::string & headRef) return true; } -std::optional readHeadCached(const std::string & actualUrl) +std::optional readHeadCached(const std::string & actualUrl, bool shallow) { // Create a cache path to store the branch of the HEAD ref. Append something // in front of the URL to prevent collision with the repository itself. - // set shallow=false as HEAD will never be queried for a shallow repo - Path cacheDir = getCachePath(actualUrl, false); + Path cacheDir = getCachePath(actualUrl, shallow); Path headRefFile = cacheDir + "/HEAD"; time_t now = time(0); @@ -517,14 +515,14 @@ struct GitInputScheme : InputScheme return revCount; } - std::string getDefaultRef(const RepoInfo & repoInfo) const + std::string getDefaultRef(const RepoInfo & repoInfo, bool shallow) const { auto head = std::visit( overloaded { [&](const std::filesystem::path & path) { return GitRepo::openRepo(path)->getWorkdirRef(); }, [&](const ParsedURL & url) - { return readHeadCached(url.to_string()); } + { return readHeadCached(url.to_string(), shallow); } }, repoInfo.location); if (!head) { warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.locationToArg()); @@ -536,7 +534,7 @@ struct GitInputScheme : InputScheme static MakeNotAllowedError makeNotAllowedError(std::filesystem::path repoPath) { return [repoPath{std::move(repoPath)}](const CanonPath & path) -> RestrictedPathError { - if (fs::symlink_exists(repoPath / path.rel())) + if (pathExists(repoPath / path.rel())) return RestrictedPathError( "Path '%1%' in the repository %2% is not tracked by Git.\n" "\n" @@ -573,7 +571,8 @@ struct GitInputScheme : InputScheme auto origRev = input.getRev(); auto originalRef = input.getRef(); - auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo); + bool shallow = getShallowAttr(input); + auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow); input.attrs.insert_or_assign("ref", ref); std::filesystem::path repoDir; @@ -584,7 +583,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev()); } else { auto repoUrl = std::get(repoInfo.location); - std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), getShallowAttr(input)); + std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow); repoDir = cacheDir; repoInfo.gitDir = "."; @@ -621,6 +620,7 @@ struct GitInputScheme : InputScheme } if (doFetch) { + bool shallow = getShallowAttr(input); try { auto fetchRef = getAllRefsAttr(input) @@ -633,7 +633,7 @@ struct GitInputScheme : InputScheme ? ref : fmt("%1%:%1%", "refs/heads/" + ref); - repo->fetch(repoUrl.to_string(), fetchRef, getShallowAttr(input)); + repo->fetch(repoUrl.to_string(), fetchRef, shallow); } catch (Error & e) { if (!std::filesystem::exists(localRefFile)) throw; logError(e.info()); @@ -644,9 +644,9 @@ struct GitInputScheme : InputScheme if (!input.getRev()) setWriteTime(localRefFile, now, now); } catch (Error & e) { - warn("could not update mtime for file '%s': %s", localRefFile, e.info().msg); + warn("could not update mtime for file %s: %s", localRefFile, e.info().msg); } - if (!originalRef && !storeCachedHead(repoUrl.to_string(), ref)) + if (!originalRef && !storeCachedHead(repoUrl.to_string(), shallow, ref)) warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg()); } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index bb82f751f..fcddb13ed 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -149,6 +149,9 @@ struct GitArchiveInputScheme : InputScheme }; if (auto narHash = input.getNarHash()) url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); + auto host = maybeGetStrAttr(input.attrs, "host"); + if (host) + url.query.insert_or_assign("host", *host); return url; } diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index c3b99fa5a..1a90fe9ef 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -4,8 +4,6 @@ #include -#include - namespace nix { /** diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 1f43e951a..1a4bb28a3 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -16,7 +16,7 @@ InputCache::getAccessor(ref store, const Input & originalInput, UseRegist auto [accessor, lockedInput] = originalInput.getAccessor(store); fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor}); } else { - if (useRegistries != fetchers::UseRegistries::No) { + if (useRegistries != UseRegistries::No) { auto [res, extraAttrs] = lookupInRegistries(store, originalInput, useRegistries); resolvedInput = std::move(res); fetched = lookup(resolvedInput); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index eb6bdd1eb..74e9fd089 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -194,7 +194,7 @@ struct MercurialInputScheme : InputScheme input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl }))); - auto files = tokenizeString>( + auto files = tokenizeString( runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); Path actualPath(absPath(actualUrl)); diff --git a/src/libfetchers/store-path-accessor.cc b/src/libfetchers/store-path-accessor.cc index bed51541e..f389d0327 100644 --- a/src/libfetchers/store-path-accessor.cc +++ b/src/libfetchers/store-path-accessor.cc @@ -5,11 +5,7 @@ namespace nix { ref makeStorePathAccessor(ref store, const StorePath & storePath) { - // FIXME: should use `store->getFSAccessor()` - auto root = std::filesystem::path{store->toRealPath(storePath)}; - auto accessor = makeFSSourceAccessor(root); - accessor->setPathDisplay(root.string()); - return accessor; + return projectSubdirSourceAccessor(store->getFSAccessor(), storePath.to_string()); } } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ef91d6b25..1bd7e3e59 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -224,7 +224,7 @@ ref downloadTarball( // An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { - const std::set transportUrlSchemes = {"file", "http", "https"}; + const StringSet transportUrlSchemes = {"file", "http", "https"}; bool hasTarballExtension(std::string_view path) const { @@ -236,7 +236,7 @@ struct CurlInputScheme : InputScheme virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0; - static const std::set specialParams; + static const StringSet specialParams; std::optional inputFromURL( const Settings & settings, diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index fd3cdd01b..5a81618c8 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -17,12 +17,14 @@ subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), + dependency('nix-fetchers'), dependency('nix-expr'), dependency('nix-flake'), ] deps_public_maybe_subproject = [ dependency('nix-util-c'), dependency('nix-store-c'), + dependency('nix-fetchers-c'), dependency('nix-expr-c'), ] subdir('nix-meson-build-support/subprojects') @@ -37,6 +39,7 @@ include_dirs = [include_directories('.')] headers = files( 'nix_api_flake.h', + 'nix_api_flake_internal.hh', ) # TODO move this header to libexpr, maybe don't use it in tests? diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc index a1b586e82..ad8f0bf4e 100644 --- a/src/libflake-c/nix_api_flake.cc +++ b/src/libflake-c/nix_api_flake.cc @@ -1,12 +1,18 @@ +#include + #include "nix_api_flake.h" #include "nix_api_flake_internal.hh" +#include "nix_api_util.h" #include "nix_api_util_internal.h" #include "nix_api_expr_internal.h" +#include "nix_api_fetchers_internal.hh" +#include "nix_api_fetchers.h" #include "nix/flake/flake.hh" nix_flake_settings * nix_flake_settings_new(nix_c_context * context) { + nix_clear_err(context); try { auto settings = nix::make_ref(); return new nix_flake_settings{settings}; @@ -22,8 +28,178 @@ void nix_flake_settings_free(nix_flake_settings * settings) nix_err nix_flake_settings_add_to_eval_state_builder( nix_c_context * context, nix_flake_settings * settings, nix_eval_state_builder * builder) { + nix_clear_err(context); try { settings->settings->configureEvalSettings(builder->settings); } NIXC_CATCH_ERRS } + +nix_flake_reference_parse_flags * +nix_flake_reference_parse_flags_new(nix_c_context * context, nix_flake_settings * settings) +{ + nix_clear_err(context); + try { + return new nix_flake_reference_parse_flags{ + .baseDirectory = std::nullopt, + }; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_flake_reference_parse_flags_free(nix_flake_reference_parse_flags * flags) +{ + delete flags; +} + +nix_err nix_flake_reference_parse_flags_set_base_directory( + nix_c_context * context, + nix_flake_reference_parse_flags * flags, + const char * baseDirectory, + size_t baseDirectoryLen) +{ + nix_clear_err(context); + try { + flags->baseDirectory.emplace(nix::Path{std::string(baseDirectory, baseDirectoryLen)}); + return NIX_OK; + } + NIXC_CATCH_ERRS +} + +nix_err nix_flake_reference_and_fragment_from_string( + nix_c_context * context, + nix_fetchers_settings * fetchSettings, + nix_flake_settings * flakeSettings, + nix_flake_reference_parse_flags * parseFlags, + const char * strData, + size_t strSize, + nix_flake_reference ** flakeReferenceOut, + nix_get_string_callback fragmentCallback, + void * fragmentCallbackUserData) +{ + nix_clear_err(context); + *flakeReferenceOut = nullptr; + try { + std::string str(strData, strSize); + + auto [flakeRef, fragment] = + nix::parseFlakeRefWithFragment(*fetchSettings->settings, str, parseFlags->baseDirectory, true); + *flakeReferenceOut = new nix_flake_reference{nix::make_ref(flakeRef)}; + return call_nix_get_string_callback(fragment, fragmentCallback, fragmentCallbackUserData); + } + NIXC_CATCH_ERRS +} + +void nix_flake_reference_free(nix_flake_reference * flakeReference) +{ + delete flakeReference; +} + +nix_flake_lock_flags * nix_flake_lock_flags_new(nix_c_context * context, nix_flake_settings * settings) +{ + nix_clear_err(context); + try { + auto lockSettings = nix::make_ref(nix::flake::LockFlags{ + .recreateLockFile = false, + .updateLockFile = true, // == `nix_flake_lock_flags_set_mode_write_as_needed` + .writeLockFile = true, // == `nix_flake_lock_flags_set_mode_write_as_needed` + .failOnUnlocked = false, // == `nix_flake_lock_flags_set_mode_write_as_needed` + .useRegistries = false, + .allowUnlocked = false, // == `nix_flake_lock_flags_set_mode_write_as_needed` + .commitLockFile = false, + + }); + return new nix_flake_lock_flags{lockSettings}; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_flake_lock_flags_free(nix_flake_lock_flags * flags) +{ + delete flags; +} + +nix_err nix_flake_lock_flags_set_mode_virtual(nix_c_context * context, nix_flake_lock_flags * flags) +{ + nix_clear_err(context); + try { + flags->lockFlags->updateLockFile = true; + flags->lockFlags->writeLockFile = false; + flags->lockFlags->failOnUnlocked = false; + flags->lockFlags->allowUnlocked = true; + } + NIXC_CATCH_ERRS +} + +nix_err nix_flake_lock_flags_set_mode_write_as_needed(nix_c_context * context, nix_flake_lock_flags * flags) +{ + nix_clear_err(context); + try { + flags->lockFlags->updateLockFile = true; + flags->lockFlags->writeLockFile = true; + flags->lockFlags->failOnUnlocked = false; + flags->lockFlags->allowUnlocked = true; + } + NIXC_CATCH_ERRS +} + +nix_err nix_flake_lock_flags_set_mode_check(nix_c_context * context, nix_flake_lock_flags * flags) +{ + nix_clear_err(context); + try { + flags->lockFlags->updateLockFile = false; + flags->lockFlags->writeLockFile = false; + flags->lockFlags->failOnUnlocked = true; + flags->lockFlags->allowUnlocked = false; + } + NIXC_CATCH_ERRS +} + +nix_err nix_flake_lock_flags_add_input_override( + nix_c_context * context, nix_flake_lock_flags * flags, const char * inputPath, nix_flake_reference * flakeRef) +{ + nix_clear_err(context); + try { + auto path = nix::flake::parseInputAttrPath(inputPath); + flags->lockFlags->inputOverrides.emplace(path, *flakeRef->flakeRef); + if (flags->lockFlags->writeLockFile) { + return nix_flake_lock_flags_set_mode_virtual(context, flags); + } + } + NIXC_CATCH_ERRS +} + +nix_locked_flake * nix_flake_lock( + nix_c_context * context, + nix_fetchers_settings * fetchSettings, + nix_flake_settings * flakeSettings, + EvalState * eval_state, + nix_flake_lock_flags * flags, + nix_flake_reference * flakeReference) +{ + nix_clear_err(context); + try { + eval_state->state.resetFileCache(); + auto lockedFlake = nix::make_ref(nix::flake::lockFlake( + *flakeSettings->settings, eval_state->state, *flakeReference->flakeRef, *flags->lockFlags)); + return new nix_locked_flake{lockedFlake}; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_locked_flake_free(nix_locked_flake * lockedFlake) +{ + delete lockedFlake; +} + +nix_value * nix_locked_flake_get_output_attrs( + nix_c_context * context, nix_flake_settings * settings, EvalState * evalState, nix_locked_flake * lockedFlake) +{ + nix_clear_err(context); + try { + auto v = nix_alloc_value(context, evalState); + nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, v->value); + return v; + } + NIXC_CATCH_ERRS_NULL +} diff --git a/src/libflake-c/nix_api_flake.h b/src/libflake-c/nix_api_flake.h index 75675835e..f5b9dc542 100644 --- a/src/libflake-c/nix_api_flake.h +++ b/src/libflake-c/nix_api_flake.h @@ -9,6 +9,7 @@ * @brief Main entry for the libflake C bindings */ +#include "nix_api_fetchers.h" #include "nix_api_store.h" #include "nix_api_util.h" #include "nix_api_expr.h" @@ -18,8 +19,46 @@ extern "C" { #endif // cffi start +/** + * @brief A settings object for configuring the behavior of the nix-flake-c library. + * @see nix_flake_settings_new + * @see nix_flake_settings_free + */ typedef struct nix_flake_settings nix_flake_settings; +/** + * @brief Context and paramaters for parsing a flake reference + * @see nix_flake_reference_parse_flags_free + * @see nix_flake_reference_parse_string + */ +typedef struct nix_flake_reference_parse_flags nix_flake_reference_parse_flags; + +/** + * @brief A reference to a flake + * + * A flake reference specifies how to fetch a flake. + * + * @see nix_flake_reference_from_string + * @see nix_flake_reference_free + */ +typedef struct nix_flake_reference nix_flake_reference; + +/** + * @brief Parameters for locking a flake + * @see nix_flake_lock_flags_new + * @see nix_flake_lock_flags_free + * @see nix_flake_lock + */ +typedef struct nix_flake_lock_flags nix_flake_lock_flags; + +/** + * @brief A flake with a suitable lock (file or otherwise) + * @see nix_flake_lock + * @see nix_locked_flake_free + * @see nix_locked_flake_get_output_attrs + */ +typedef struct nix_locked_flake nix_locked_flake; + // Function prototypes /** * Create a nix_flake_settings initialized with default values. @@ -38,6 +77,8 @@ void nix_flake_settings_free(nix_flake_settings * settings); * @brief Initialize a `nix_flake_settings` to contain `builtins.getFlake` and * potentially more. * + * @warning This does not put the eval state in pure mode! + * * @param[out] context Optional, stores error information * @param[in] settings The settings to use for e.g. `builtins.getFlake` * @param[in] builder The builder to modify @@ -45,6 +86,158 @@ void nix_flake_settings_free(nix_flake_settings * settings); nix_err nix_flake_settings_add_to_eval_state_builder( nix_c_context * context, nix_flake_settings * settings, nix_eval_state_builder * builder); +/** + * @brief A new `nix_flake_reference_parse_flags` with defaults + */ +nix_flake_reference_parse_flags * +nix_flake_reference_parse_flags_new(nix_c_context * context, nix_flake_settings * settings); + +/** + * @brief Deallocate and release the resources associated with a `nix_flake_reference_parse_flags`. + * Does not fail. + * @param[in] flags the `nix_flake_reference_parse_flags *` to free + */ +void nix_flake_reference_parse_flags_free(nix_flake_reference_parse_flags * flags); + +/** + * @brief Provide a base directory for parsing relative flake references + * @param[out] context Optional, stores error information + * @param[in] flags The flags to modify + * @param[in] baseDirectory The base directory to add + * @param[in] baseDirectoryLen The length of baseDirectory + * @return NIX_OK on success, NIX_ERR on failure + */ +nix_err nix_flake_reference_parse_flags_set_base_directory( + nix_c_context * context, + nix_flake_reference_parse_flags * flags, + const char * baseDirectory, + size_t baseDirectoryLen); + +/** + * @brief A new `nix_flake_lock_flags` with defaults + * @param[in] settings Flake settings that may affect the defaults + */ +nix_flake_lock_flags * nix_flake_lock_flags_new(nix_c_context * context, nix_flake_settings * settings); + +/** + * @brief Deallocate and release the resources associated with a `nix_flake_lock_flags`. + * Does not fail. + * @param[in] settings the `nix_flake_lock_flags *` to free + */ +void nix_flake_lock_flags_free(nix_flake_lock_flags * settings); + +/** + * @brief Put the lock flags in a mode that checks whether the lock is up to date. + * @param[out] context Optional, stores error information + * @param[in] flags The flags to modify + * @return NIX_OK on success, NIX_ERR on failure + * + * This causes `nix_flake_lock` to fail if the lock needs to be updated. + */ +nix_err nix_flake_lock_flags_set_mode_check(nix_c_context * context, nix_flake_lock_flags * flags); + +/** + * @brief Put the lock flags in a mode that updates the lock file in memory, if needed. + * @param[out] context Optional, stores error information + * @param[in] flags The flags to modify + * @param[in] update Whether to allow updates + * + * This will cause `nix_flake_lock` to update the lock file in memory, if needed. + */ +nix_err nix_flake_lock_flags_set_mode_virtual(nix_c_context * context, nix_flake_lock_flags * flags); + +/** + * @brief Put the lock flags in a mode that updates the lock file on disk, if needed. + * @param[out] context Optional, stores error information + * @param[in] flags The flags to modify + * @param[in] update Whether to allow updates + * + * This will cause `nix_flake_lock` to update the lock file on disk, if needed. + */ +nix_err nix_flake_lock_flags_set_mode_write_as_needed(nix_c_context * context, nix_flake_lock_flags * flags); + +/** + * @brief Add input overrides to the lock flags + * @param[out] context Optional, stores error information + * @param[in] flags The flags to modify + * @param[in] inputPath The input path to override + * @param[in] flakeRef The flake reference to use as the override + * + * This switches the `flags` to `nix_flake_lock_flags_set_mode_virtual` if not in mode + * `nix_flake_lock_flags_set_mode_check`. + */ +nix_err nix_flake_lock_flags_add_input_override( + nix_c_context * context, nix_flake_lock_flags * flags, const char * inputPath, nix_flake_reference * flakeRef); + +/** + * @brief Lock a flake, if not already locked. + * @param[out] context Optional, stores error information + * @param[in] settings The flake (and fetch) settings to use + * @param[in] flags The locking flags to use + * @param[in] flake The flake to lock + */ +nix_locked_flake * nix_flake_lock( + nix_c_context * context, + nix_fetchers_settings * fetchSettings, + nix_flake_settings * settings, + EvalState * eval_state, + nix_flake_lock_flags * flags, + nix_flake_reference * flake); + +/** + * @brief Deallocate and release the resources associated with a `nix_locked_flake`. + * Does not fail. + * @param[in] locked_flake the `nix_locked_flake *` to free + */ +void nix_locked_flake_free(nix_locked_flake * locked_flake); + +/** + * @brief Parse a URL-like string into a `nix_flake_reference`. + * + * @param[out] context **context** – Optional, stores error information + * @param[in] fetchSettings **context** – The fetch settings to use + * @param[in] flakeSettings **context** – The flake settings to use + * @param[in] parseFlags **context** – Specific context and parameters such as base directory + * + * @param[in] str **input** – The URI-like string to parse + * @param[in] strLen **input** – The length of `str` + * + * @param[out] flakeReferenceOut **result** – The resulting flake reference + * @param[in] fragmentCallback **result** – A callback to call with the fragment part of the URL + * @param[in] fragmentCallbackUserData **result** – User data to pass to the fragment callback + * + * @return NIX_OK on success, NIX_ERR on failure + */ +nix_err nix_flake_reference_and_fragment_from_string( + nix_c_context * context, + nix_fetchers_settings * fetchSettings, + nix_flake_settings * flakeSettings, + nix_flake_reference_parse_flags * parseFlags, + const char * str, + size_t strLen, + nix_flake_reference ** flakeReferenceOut, + nix_get_string_callback fragmentCallback, + void * fragmentCallbackUserData); + +/** + * @brief Deallocate and release the resources associated with a `nix_flake_reference`. + * + * Does not fail. + * + * @param[in] store the `nix_flake_reference *` to free + */ +void nix_flake_reference_free(nix_flake_reference * store); + +/** + * @brief Get the output attributes of a flake. + * @param[out] context Optional, stores error information + * @param[in] settings The settings to use + * @param[in] locked_flake the flake to get the output attributes from + * @return A new nix_value or NULL on failure. Release the `nix_value` with `nix_value_decref`. + */ +nix_value * nix_locked_flake_get_output_attrs( + nix_c_context * context, nix_flake_settings * settings, EvalState * evalState, nix_locked_flake * lockedFlake); + #ifdef __cplusplus } // extern "C" #endif diff --git a/src/libflake-c/nix_api_flake_internal.hh b/src/libflake-c/nix_api_flake_internal.hh index f7c5e7838..fbc6574d6 100644 --- a/src/libflake-c/nix_api_flake_internal.hh +++ b/src/libflake-c/nix_api_flake_internal.hh @@ -1,9 +1,32 @@ #pragma once +#include #include "nix/util/ref.hh" +#include "nix/flake/flake.hh" +#include "nix/flake/flakeref.hh" #include "nix/flake/settings.hh" struct nix_flake_settings { nix::ref settings; }; + +struct nix_flake_reference_parse_flags +{ + std::optional baseDirectory; +}; + +struct nix_flake_reference +{ + nix::ref flakeRef; +}; + +struct nix_flake_lock_flags +{ + nix::ref lockFlags; +}; + +struct nix_locked_flake +{ + nix::ref lockedFlake; +}; diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index 958cf233e..9ae3ec695 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -4,6 +4,7 @@ nix-store-c, nix-expr-c, + nix-fetchers-c, nix-flake, # Configuration Options @@ -35,6 +36,7 @@ mkMesonLibrary (finalAttrs: { propagatedBuildInputs = [ nix-expr-c nix-store-c + nix-fetchers-c nix-flake ]; diff --git a/src/libflake-tests/nix_api_flake.cc b/src/libflake-tests/nix_api_flake.cc index b72342e4d..f7e0cb719 100644 --- a/src/libflake-tests/nix_api_flake.cc +++ b/src/libflake-tests/nix_api_flake.cc @@ -1,7 +1,6 @@ +#include "nix/util/file-system.hh" #include "nix_api_store.h" -#include "nix_api_store_internal.h" #include "nix_api_util.h" -#include "nix_api_util_internal.h" #include "nix_api_expr.h" #include "nix_api_value.h" #include "nix_api_flake.h" @@ -14,7 +13,7 @@ namespace nixC { -TEST_F(nix_api_store_test, nix_api_init_global_getFlake_exists) +TEST_F(nix_api_store_test, nix_api_init_getFlake_exists) { nix_libstore_init(ctx); assert_ctx_ok(); @@ -43,9 +42,342 @@ TEST_F(nix_api_store_test, nix_api_init_global_getFlake_exists) ASSERT_NE(nullptr, value); nix_err err = nix_expr_eval_from_string(ctx, state, "builtins.getFlake", ".", value); + + nix_state_free(state); + assert_ctx_ok(); ASSERT_EQ(NIX_OK, err); ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(ctx, value)); } +TEST_F(nix_api_store_test, nix_api_flake_reference_not_absolute_no_basedir_fail) +{ + nix_libstore_init(ctx); + assert_ctx_ok(); + nix_libexpr_init(ctx); + assert_ctx_ok(); + + auto settings = nix_flake_settings_new(ctx); + assert_ctx_ok(); + ASSERT_NE(nullptr, settings); + + auto fetchSettings = nix_fetchers_settings_new(ctx); + assert_ctx_ok(); + ASSERT_NE(nullptr, fetchSettings); + + auto parseFlags = nix_flake_reference_parse_flags_new(ctx, settings); + + std::string str(".#legacyPackages.aarch127-unknown...orion"); + std::string fragment; + nix_flake_reference * flakeReference = nullptr; + auto r = nix_flake_reference_and_fragment_from_string( + ctx, fetchSettings, settings, parseFlags, str.data(), str.size(), &flakeReference, OBSERVE_STRING(fragment)); + + ASSERT_NE(NIX_OK, r); + ASSERT_EQ(nullptr, flakeReference); + + nix_flake_reference_parse_flags_free(parseFlags); +} + +TEST_F(nix_api_store_test, nix_api_load_flake) +{ + auto tmpDir = nix::createTempDir(); + nix::AutoDelete delTmpDir(tmpDir, true); + + nix::writeFile(tmpDir + "/flake.nix", R"( + { + outputs = { ... }: { + hello = "potato"; + }; + } + )"); + + nix_libstore_init(ctx); + assert_ctx_ok(); + nix_libexpr_init(ctx); + assert_ctx_ok(); + + auto fetchSettings = nix_fetchers_settings_new(ctx); + assert_ctx_ok(); + ASSERT_NE(nullptr, fetchSettings); + + auto settings = nix_flake_settings_new(ctx); + assert_ctx_ok(); + ASSERT_NE(nullptr, settings); + + nix_eval_state_builder * builder = nix_eval_state_builder_new(ctx, store); + ASSERT_NE(nullptr, builder); + assert_ctx_ok(); + + auto state = nix_eval_state_build(ctx, builder); + assert_ctx_ok(); + ASSERT_NE(nullptr, state); + + nix_eval_state_builder_free(builder); + + auto parseFlags = nix_flake_reference_parse_flags_new(ctx, settings); + assert_ctx_ok(); + ASSERT_NE(nullptr, parseFlags); + + auto r0 = nix_flake_reference_parse_flags_set_base_directory(ctx, parseFlags, tmpDir.c_str(), tmpDir.size()); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, r0); + + std::string fragment; + const std::string ref = ".#legacyPackages.aarch127-unknown...orion"; + nix_flake_reference * flakeReference = nullptr; + auto r = nix_flake_reference_and_fragment_from_string( + ctx, fetchSettings, settings, parseFlags, ref.data(), ref.size(), &flakeReference, OBSERVE_STRING(fragment)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, r); + ASSERT_NE(nullptr, flakeReference); + ASSERT_EQ(fragment, "legacyPackages.aarch127-unknown...orion"); + + nix_flake_reference_parse_flags_free(parseFlags); + + auto lockFlags = nix_flake_lock_flags_new(ctx, settings); + assert_ctx_ok(); + ASSERT_NE(nullptr, lockFlags); + + auto lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference); + assert_ctx_ok(); + ASSERT_NE(nullptr, lockedFlake); + + nix_flake_lock_flags_free(lockFlags); + + auto value = nix_locked_flake_get_output_attrs(ctx, settings, state, lockedFlake); + assert_ctx_ok(); + ASSERT_NE(nullptr, value); + + auto helloAttr = nix_get_attr_byname(ctx, value, state, "hello"); + assert_ctx_ok(); + ASSERT_NE(nullptr, helloAttr); + + std::string helloStr; + nix_get_string(ctx, helloAttr, OBSERVE_STRING(helloStr)); + assert_ctx_ok(); + ASSERT_EQ("potato", helloStr); + + nix_value_decref(ctx, value); + nix_locked_flake_free(lockedFlake); + nix_flake_reference_free(flakeReference); + nix_state_free(state); + nix_flake_settings_free(settings); +} + +TEST_F(nix_api_store_test, nix_api_load_flake_with_flags) +{ + nix_libstore_init(ctx); + assert_ctx_ok(); + nix_libexpr_init(ctx); + assert_ctx_ok(); + + auto tmpDir = nix::createTempDir(); + nix::AutoDelete delTmpDir(tmpDir, true); + + nix::createDirs(tmpDir + "/b"); + nix::writeFile(tmpDir + "/b/flake.nix", R"( + { + outputs = { ... }: { + hello = "BOB"; + }; + } + )"); + + nix::createDirs(tmpDir + "/a"); + nix::writeFile(tmpDir + "/a/flake.nix", R"( + { + inputs.b.url = ")" + tmpDir + R"(/b"; + outputs = { b, ... }: { + hello = b.hello; + }; + } + )"); + + nix::createDirs(tmpDir + "/c"); + nix::writeFile(tmpDir + "/c/flake.nix", R"( + { + outputs = { ... }: { + hello = "Claire"; + }; + } + )"); + + nix_libstore_init(ctx); + assert_ctx_ok(); + + auto fetchSettings = nix_fetchers_settings_new(ctx); + assert_ctx_ok(); + ASSERT_NE(nullptr, fetchSettings); + + auto settings = nix_flake_settings_new(ctx); + assert_ctx_ok(); + ASSERT_NE(nullptr, settings); + + nix_eval_state_builder * builder = nix_eval_state_builder_new(ctx, store); + ASSERT_NE(nullptr, builder); + assert_ctx_ok(); + + auto state = nix_eval_state_build(ctx, builder); + assert_ctx_ok(); + ASSERT_NE(nullptr, state); + + nix_eval_state_builder_free(builder); + + auto parseFlags = nix_flake_reference_parse_flags_new(ctx, settings); + assert_ctx_ok(); + ASSERT_NE(nullptr, parseFlags); + + auto r0 = nix_flake_reference_parse_flags_set_base_directory(ctx, parseFlags, tmpDir.c_str(), tmpDir.size()); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, r0); + + std::string fragment; + const std::string ref = "./a"; + nix_flake_reference * flakeReference = nullptr; + auto r = nix_flake_reference_and_fragment_from_string( + ctx, fetchSettings, settings, parseFlags, ref.data(), ref.size(), &flakeReference, OBSERVE_STRING(fragment)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, r); + ASSERT_NE(nullptr, flakeReference); + ASSERT_EQ(fragment, ""); + + // Step 1: Do not update, fails + + auto lockFlags = nix_flake_lock_flags_new(ctx, settings); + assert_ctx_ok(); + ASSERT_NE(nullptr, lockFlags); + + nix_flake_lock_flags_set_mode_check(ctx, lockFlags); + assert_ctx_ok(); + + // Step 2: Update but do not write, succeeds + + auto lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference); + assert_ctx_err(); + ASSERT_EQ(nullptr, lockedFlake); + + nix_flake_lock_flags_set_mode_virtual(ctx, lockFlags); + assert_ctx_ok(); + + lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference); + assert_ctx_ok(); + ASSERT_NE(nullptr, lockedFlake); + + // Get the output attrs + auto value = nix_locked_flake_get_output_attrs(ctx, settings, state, lockedFlake); + assert_ctx_ok(); + ASSERT_NE(nullptr, value); + + auto helloAttr = nix_get_attr_byname(ctx, value, state, "hello"); + assert_ctx_ok(); + ASSERT_NE(nullptr, helloAttr); + + std::string helloStr; + nix_get_string(ctx, helloAttr, OBSERVE_STRING(helloStr)); + assert_ctx_ok(); + ASSERT_EQ("BOB", helloStr); + + nix_value_decref(ctx, value); + nix_locked_flake_free(lockedFlake); + + // Step 3: Lock was not written, so Step 1 would fail again + + nix_flake_lock_flags_set_mode_check(ctx, lockFlags); + + lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference); + assert_ctx_err(); + ASSERT_EQ(nullptr, lockedFlake); + + // Step 4: Update and write, succeeds + + nix_flake_lock_flags_set_mode_write_as_needed(ctx, lockFlags); + assert_ctx_ok(); + + lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference); + assert_ctx_ok(); + ASSERT_NE(nullptr, lockedFlake); + + // Get the output attrs + value = nix_locked_flake_get_output_attrs(ctx, settings, state, lockedFlake); + assert_ctx_ok(); + ASSERT_NE(nullptr, value); + + helloAttr = nix_get_attr_byname(ctx, value, state, "hello"); + assert_ctx_ok(); + ASSERT_NE(nullptr, helloAttr); + + helloStr.clear(); + nix_get_string(ctx, helloAttr, OBSERVE_STRING(helloStr)); + assert_ctx_ok(); + ASSERT_EQ("BOB", helloStr); + + nix_value_decref(ctx, value); + nix_locked_flake_free(lockedFlake); + + // Step 5: Lock was written, so Step 1 would succeed + + nix_flake_lock_flags_set_mode_check(ctx, lockFlags); + assert_ctx_ok(); + + lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference); + assert_ctx_ok(); + ASSERT_NE(nullptr, lockedFlake); + + // Get the output attrs + value = nix_locked_flake_get_output_attrs(ctx, settings, state, lockedFlake); + assert_ctx_ok(); + ASSERT_NE(nullptr, value); + + helloAttr = nix_get_attr_byname(ctx, value, state, "hello"); + assert_ctx_ok(); + ASSERT_NE(nullptr, helloAttr); + + helloStr.clear(); + nix_get_string(ctx, helloAttr, OBSERVE_STRING(helloStr)); + assert_ctx_ok(); + ASSERT_EQ("BOB", helloStr); + + nix_value_decref(ctx, value); + nix_locked_flake_free(lockedFlake); + + // Step 6: Lock with override, do not write + + nix_flake_lock_flags_set_mode_write_as_needed(ctx, lockFlags); + assert_ctx_ok(); + + nix_flake_reference * overrideFlakeReference = nullptr; + nix_flake_reference_and_fragment_from_string( + ctx, fetchSettings, settings, parseFlags, "./c", 3, &overrideFlakeReference, OBSERVE_STRING(fragment)); + assert_ctx_ok(); + ASSERT_NE(nullptr, overrideFlakeReference); + + nix_flake_lock_flags_add_input_override(ctx, lockFlags, "b", overrideFlakeReference); + assert_ctx_ok(); + + lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference); + assert_ctx_ok(); + ASSERT_NE(nullptr, lockedFlake); + + // Get the output attrs + value = nix_locked_flake_get_output_attrs(ctx, settings, state, lockedFlake); + assert_ctx_ok(); + ASSERT_NE(nullptr, value); + + helloAttr = nix_get_attr_byname(ctx, value, state, "hello"); + assert_ctx_ok(); + ASSERT_NE(nullptr, helloAttr); + + helloStr.clear(); + nix_get_string(ctx, helloAttr, OBSERVE_STRING(helloStr)); + assert_ctx_ok(); + ASSERT_EQ("Claire", helloStr); + + nix_flake_reference_parse_flags_free(parseFlags); + nix_flake_lock_flags_free(lockFlags); + nix_flake_reference_free(flakeReference); + nix_state_free(state); + nix_flake_settings_free(settings); +} + } // namespace nixC diff --git a/src/libflake/config.cc b/src/libflake/config.cc index a67f7884c..030104e7f 100644 --- a/src/libflake/config.cc +++ b/src/libflake/config.cc @@ -32,7 +32,7 @@ static void writeTrustedList(const TrustedList & trustedList) void ConfigFile::apply(const Settings & flakeSettings) { - std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"}; + StringSet whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"}; for (auto & [name, value] : settings) { diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index a85acf4b2..06c81325b 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -307,7 +307,7 @@ static FlakeRef applySelfAttrs( { auto newRef(ref); - std::set allowedAttrs{"submodules", "lfs"}; + StringSet allowedAttrs{"submodules", "lfs"}; for (auto & attr : flake.selfAttrs) { if (!allowedAttrs.contains(attr.first)) @@ -821,16 +821,16 @@ LockedFlake lockFlake( auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; auto outputLockFilePath = *sourcePath / relPath; - bool lockFileExists = fs::symlink_exists(outputLockFilePath); + bool lockFileExists = pathExists(outputLockFilePath); auto s = chomp(diff); if (lockFileExists) { if (s.empty()) - warn("updating lock file '%s'", outputLockFilePath); + warn("updating lock file %s", outputLockFilePath); else - warn("updating lock file '%s':\n%s", outputLockFilePath, s); + warn("updating lock file %s:\n%s", outputLockFilePath, s); } else - warn("creating lock file '%s': \n%s", outputLockFilePath, s); + warn("creating lock file %s: \n%s", outputLockFilePath, s); std::optional commitMessage = std::nullopt; diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index fdac4397f..50fd826af 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -63,7 +63,7 @@ struct ConfigFile }; /** - * The contents of a flake.nix file. + * A flake in context */ struct Flake { diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 13b85e544..dcf252a4f 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,3 +1,5 @@ +#include + #include "nix/main/common-args.hh" #include "nix/util/args/root.hh" #include "nix/util/config-global.hh" @@ -93,5 +95,18 @@ void MixCommonArgs::initialFlagsProcessed() pluginsInited(); } - +template +void MixPrintJSON::printJSON(const T /* nlohmann::json */ & json) +{ + auto suspension = logger->suspend(); + if (outputPretty) { + logger->writeToStdout(json.dump(2)); + } else { + logger->writeToStdout(json.dump()); + } } + +template void MixPrintJSON::printJSON(const nlohmann::json & json); + + +} // namespace nix diff --git a/src/libmain/include/nix/main/common-args.hh b/src/libmain/include/nix/main/common-args.hh index ae0f3c6c5..cc6d3d3f0 100644 --- a/src/libmain/include/nix/main/common-args.hh +++ b/src/libmain/include/nix/main/common-args.hh @@ -29,13 +29,69 @@ struct MixDryRun : virtual Args addFlag({ .longName = "dry-run", .description = "Show what this command would do without doing it.", - //.category = commonArgsCategory, .handler = {&dryRun, true}, }); } }; -struct MixJSON : virtual Args +/** + * Commands that can print JSON according to the + * `--pretty`/`--no-pretty` flag. + * + * This is distinct from MixJSON, because for some commands, + * JSON outputs is not optional. + */ +struct MixPrintJSON : virtual Args +{ + bool outputPretty = isatty(STDOUT_FILENO); + + MixPrintJSON() + { + addFlag({ + .longName = "pretty", + .description = + R"( + Print multi-line, indented JSON output for readability. + + Default: indent if output is to a terminal. + + This option is only effective when `--json` is also specified. + )", + .handler = {&outputPretty, true}, + }); + addFlag({ + .longName = "no-pretty", + .description = + R"( + Print compact JSON output on a single line, even when the output is a terminal. + Some commands may print multiple JSON objects on separate lines. + + See `--pretty`. + )", + .handler = {&outputPretty, false}, + }); + }; + + /** + * Print an `nlohmann::json` to stdout + * + * - respecting `--pretty` / `--no-pretty`. + * - suspending the progress bar + * + * This is a template to avoid accidental coercions from `string` to `json` in the caller, + * to avoid mistakenly passing an already serialized JSON to this function. + * + * It is not recommended to print a JSON string - see the JSON guidelines + * about extensibility, https://nix.dev/manual/nix/development/development/json-guideline.html - + * but you _can_ print a sole JSON string by explicitly coercing it to + * `nlohmann::json` first. + */ + template >> + void printJSON(const T & json); +}; + +/** Optional JSON support via `--json` flag */ +struct MixJSON : virtual Args, virtual MixPrintJSON { bool json = false; @@ -44,7 +100,6 @@ struct MixJSON : virtual Args addFlag({ .longName = "json", .description = "Produce output in JSON format, suitable for consumption by another program.", - //.category = commonArgsCategory, .handler = {&json, true}, }); } diff --git a/src/libmain/plugin.cc b/src/libmain/plugin.cc index 63ed650a7..db686a251 100644 --- a/src/libmain/plugin.cc +++ b/src/libmain/plugin.cc @@ -6,6 +6,7 @@ #include "nix/util/config-global.hh" #include "nix/util/signals.hh" +#include "nix/util/file-system.hh" namespace nix { @@ -18,7 +19,7 @@ struct PluginFilesSetting : public BaseSetting const Paths & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) + const StringSet & aliases = {}) : BaseSetting(def, true, name, description, aliases) { options->addSetting(this); @@ -77,13 +78,13 @@ void initPlugins() for (const auto & pluginFile : pluginSettings.pluginFiles.get()) { std::vector pluginFiles; try { - auto ents = std::filesystem::directory_iterator{pluginFile}; + auto ents = DirectoryIterator{pluginFile}; for (const auto & ent : ents) { checkInterrupt(); pluginFiles.emplace_back(ent.path()); } - } catch (std::filesystem::filesystem_error & e) { - if (e.code() != std::errc::not_a_directory) + } catch (SysError & e) { + if (e.errNo != ENOTDIR) throw; pluginFiles.emplace_back(pluginFile); } diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 92aed9187..b7b437e9c 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -5,6 +5,7 @@ #include "nix/store/path.hh" #include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/build-result.hh" #include "nix/store/globals.hh" @@ -42,7 +43,7 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char *** if (!params) return new Store{nix::openStore(uri_str)}; - nix::Store::Params params_map; + nix::Store::Config::Params params_map; for (size_t i = 0; params[i] != nullptr; i++) { params_map[params[i][0]] = params[i][1]; } diff --git a/src/libstore-test-support/include/nix/store/tests/libstore.hh b/src/libstore-test-support/include/nix/store/tests/libstore.hh index 466b6f9b1..822ec3aa8 100644 --- a/src/libstore-test-support/include/nix/store/tests/libstore.hh +++ b/src/libstore-test-support/include/nix/store/tests/libstore.hh @@ -5,6 +5,7 @@ #include #include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" namespace nix { diff --git a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh index bc0f31d05..63f80cf91 100644 --- a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh +++ b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh @@ -11,8 +11,6 @@ #include #include -namespace fs { using namespace std::filesystem; } - namespace nixC { class nix_api_store_test : public nix_api_util_context { @@ -27,10 +25,10 @@ public: { nix_store_free(store); - for (auto & path : fs::recursive_directory_iterator(nixDir)) { - fs::permissions(path, fs::perms::owner_all); + for (auto & path : std::filesystem::recursive_directory_iterator(nixDir)) { + std::filesystem::permissions(path, std::filesystem::perms::owner_all); } - fs::remove_all(nixDir); + std::filesystem::remove_all(nixDir); } Store * store; @@ -45,7 +43,7 @@ protected: auto tmpl = nix::defaultTempDir() + "/tests_nix-store."; for (size_t i = 0; true; ++i) { nixDir = tmpl + std::string { i }; - if (fs::create_directory(nixDir)) break; + if (std::filesystem::create_directory(nixDir)) break; } #else // resolve any symlinks in i.e. on macOS /tmp -> /private/tmp diff --git a/src/libstore-test-support/outputs-spec.cc b/src/libstore-test-support/outputs-spec.cc index e186ad8ae..5b5251361 100644 --- a/src/libstore-test-support/outputs-spec.cc +++ b/src/libstore-test-support/outputs-spec.cc @@ -14,8 +14,9 @@ Gen Arbitrary::arbitrary() return gen::just((OutputsSpec) OutputsSpec::All{}); case 1: return gen::map( - gen::nonEmpty(gen::container( - gen::map(gen::arbitrary(), [](StorePathName n) { return n.name; }))), + gen::nonEmpty( + gen::container( + gen::map(gen::arbitrary(), [](StorePathName n) { return n.name; }))), [](StringSet names) { return (OutputsSpec) OutputsSpec::Names{names}; }); default: assert(false); diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 6bfb8bd80..5164f154a 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -154,7 +154,7 @@ CHARACTERIZATION_TEST( CHARACTERIZATION_TEST( set, "set", - (std::tuple, std::set, std::set, std::set>> { + (std::tuple> { { }, { "" }, { "", "foo", "bar" }, diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json index f6cdc1f16..a421efea7 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -5,7 +5,7 @@ ], "builder": "/bin/bash", "env": { - "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"refs2\":[\"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", "bin": "/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m", "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" @@ -26,7 +26,9 @@ ] } }, - "inputSrcs": [], + "inputSrcs": [ + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.json b/src/libstore-tests/data/derivation/ca/advanced-attributes.json index 2105c6256..0ac0a9c5c 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.json @@ -15,6 +15,7 @@ "builder": "/bin/bash", "disallowedReferences": "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g", "disallowedRequisites": "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8", + "exportReferencesGraph": "refs1 /164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9 refs2 /nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv", "impureEnvVars": "UNICORN", "name": "advanced-attributes", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9", @@ -40,7 +41,9 @@ ] } }, - "inputSrcs": [], + "inputSrcs": [ + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ], "name": "advanced-attributes", "outputs": { "out": { diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json index b45a0d624..d68502d56 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json @@ -5,10 +5,10 @@ ], "builder": "/bin/bash", "env": { - "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", - "bin": "/nix/store/qjjj3zrlimpjbkk686m052b3ks9iz2sl-advanced-attributes-structured-attrs-bin", - "dev": "/nix/store/lpz5grl48v93pdadavyg5is1rqvfdipf-advanced-attributes-structured-attrs-dev", - "out": "/nix/store/nzvz1bmh1g89a5dkpqcqan0av7q3hgv3-advanced-attributes-structured-attrs" + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"refs2\":[\"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "bin": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin", + "dev": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev", + "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" }, "inputDrvs": { "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { @@ -26,17 +26,19 @@ ] } }, - "inputSrcs": [], + "inputSrcs": [ + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { - "path": "/nix/store/qjjj3zrlimpjbkk686m052b3ks9iz2sl-advanced-attributes-structured-attrs-bin" + "path": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin" }, "dev": { - "path": "/nix/store/lpz5grl48v93pdadavyg5is1rqvfdipf-advanced-attributes-structured-attrs-dev" + "path": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev" }, "out": { - "path": "/nix/store/nzvz1bmh1g89a5dkpqcqan0av7q3hgv3-advanced-attributes-structured-attrs" + "path": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" } }, "system": "my-system" diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.json b/src/libstore-tests/data/derivation/ia/advanced-attributes.json index 1eb8de86e..20ce5e1c2 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.json @@ -15,9 +15,10 @@ "builder": "/bin/bash", "disallowedReferences": "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar", "disallowedRequisites": "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev", + "exportReferencesGraph": "refs1 /nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo refs2 /nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv", "impureEnvVars": "UNICORN", "name": "advanced-attributes", - "out": "/nix/store/swkj0mrq0cq3dfli95v4am0427mi2hxf-advanced-attributes", + "out": "/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes", "preferLocalBuild": "1", "requiredSystemFeatures": "rainbow uid-range", "system": "my-system" @@ -38,11 +39,13 @@ ] } }, - "inputSrcs": [], + "inputSrcs": [ + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ], "name": "advanced-attributes", "outputs": { "out": { - "path": "/nix/store/swkj0mrq0cq3dfli95v4am0427mi2hxf-advanced-attributes" + "path": "/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes" } }, "system": "my-system" diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index f82cea026..b68134cd1 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -99,6 +99,8 @@ TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attribute #undef TEST_ATERM_JSON +using ExportReferencesMap = decltype(DerivationOptions::exportReferencesGraph); + TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) { this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { @@ -106,16 +108,17 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); - EXPECT_TRUE(!parsedDrv.hasStructuredAttrs()); + EXPECT_TRUE(!parsedDrv); EXPECT_EQ(options.additionalSandboxProfile, ""); EXPECT_EQ(options.noChroot, false); EXPECT_EQ(options.impureHostDeps, StringSet{}); EXPECT_EQ(options.impureEnvVars, StringSet{}); EXPECT_EQ(options.allowLocalNetworking, false); + EXPECT_EQ(options.exportReferencesGraph, ExportReferencesMap{}); { auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); ASSERT_TRUE(checksForAllOutputs_ != nullptr); @@ -140,8 +143,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_defaults) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); }); @@ -154,8 +157,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); }); @@ -168,10 +171,10 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); - EXPECT_TRUE(!parsedDrv.hasStructuredAttrs()); + EXPECT_TRUE(!parsedDrv); EXPECT_EQ(options.additionalSandboxProfile, "sandcastle"); EXPECT_EQ(options.noChroot, true); @@ -192,8 +195,25 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); + + EXPECT_EQ( + options.exportReferencesGraph, + (ExportReferencesMap{ + { + "refs1", + { + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo", + }, + }, + { + "refs2", + { + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv", + }, + }, + })); { auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); @@ -225,8 +245,25 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); + + EXPECT_EQ( + options.exportReferencesGraph, + (ExportReferencesMap{ + { + "refs1", + { + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9", + }, + }, + { + "refs2", + { + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv", + }, + }, + })); { auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); @@ -261,23 +298,24 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); - EXPECT_TRUE(parsedDrv.hasStructuredAttrs()); + EXPECT_TRUE(parsedDrv); EXPECT_EQ(options.additionalSandboxProfile, ""); EXPECT_EQ(options.noChroot, false); EXPECT_EQ(options.impureHostDeps, StringSet{}); EXPECT_EQ(options.impureEnvVars, StringSet{}); EXPECT_EQ(options.allowLocalNetworking, false); + EXPECT_EQ(options.exportReferencesGraph, ExportReferencesMap{}); { auto * checksPerOutput_ = std::get_if<1>(&options.outputChecks); ASSERT_TRUE(checksPerOutput_ != nullptr); auto & checksPerOutput = *checksPerOutput_; - EXPECT_EQ(checksPerOutput.size(), 0); + EXPECT_EQ(checksPerOutput.size(), 0u); } EXPECT_EQ(options.canBuildLocally(*this->store, got), false); @@ -294,8 +332,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); }); @@ -308,8 +346,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_default auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); }); @@ -322,10 +360,10 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); - EXPECT_TRUE(parsedDrv.hasStructuredAttrs()); + EXPECT_TRUE(parsedDrv); EXPECT_EQ(options.additionalSandboxProfile, "sandcastle"); EXPECT_EQ(options.noChroot, true); @@ -356,8 +394,25 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); + + EXPECT_EQ( + options.exportReferencesGraph, + (ExportReferencesMap{ + { + "refs1", + { + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo", + }, + }, + { + "refs2", + { + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv", + }, + }, + })); { { @@ -393,8 +448,25 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - ParsedDerivation parsedDrv(drvPath, got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(got.env); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr); + + EXPECT_EQ( + options.exportReferencesGraph, + (ExportReferencesMap{ + { + "refs1", + { + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9", + }, + }, + { + "refs2", + { + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv", + }, + }, + })); { { diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index 084807130..8873ff183 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -163,8 +163,8 @@ TEST(machines, getMachinesWithIncorrectFormat) { } TEST(machines, getMachinesWithCorrectFileReference) { - auto path = fs::weakly_canonical(getUnitTestData() / "machines/valid"); - ASSERT_TRUE(fs::exists(path)); + auto path = std::filesystem::weakly_canonical(getUnitTestData() / "machines/valid"); + ASSERT_TRUE(std::filesystem::exists(path)); auto actual = Machine::parseConfig({}, "@" + path.string()); ASSERT_THAT(actual, SizeIs(3)); @@ -174,22 +174,22 @@ TEST(machines, getMachinesWithCorrectFileReference) { } TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { - fs::path path = "/dev/null"; - ASSERT_TRUE(fs::exists(path)); + std::filesystem::path path = "/dev/null"; + ASSERT_TRUE(std::filesystem::exists(path)); auto actual = Machine::parseConfig({}, "@" + path.string()); ASSERT_THAT(actual, SizeIs(0)); } TEST(machines, getMachinesWithIncorrectFileReference) { - auto path = fs::weakly_canonical("/not/a/file"); - ASSERT_TRUE(!fs::exists(path)); + auto path = std::filesystem::weakly_canonical("/not/a/file"); + ASSERT_TRUE(!std::filesystem::exists(path)); auto actual = Machine::parseConfig({}, "@" + path.string()); ASSERT_THAT(actual, SizeIs(0)); } TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { EXPECT_THROW( - Machine::parseConfig({}, "@" + fs::weakly_canonical(getUnitTestData() / "machines" / "bad_format").string()), + Machine::parseConfig({}, "@" + std::filesystem::weakly_canonical(getUnitTestData() / "machines" / "bad_format").string()), FormatError); } diff --git a/src/libstore-tests/outputs-spec.cc b/src/libstore-tests/outputs-spec.cc index a17922c46..a1c13d2f8 100644 --- a/src/libstore-tests/outputs-spec.cc +++ b/src/libstore-tests/outputs-spec.cc @@ -7,7 +7,7 @@ namespace nix { TEST(OutputsSpec, no_empty_names) { - ASSERT_DEATH(OutputsSpec::Names { std::set { } }, ""); + ASSERT_DEATH(OutputsSpec::Names { StringSet { } }, ""); } #define TEST_DONT_PARSE(NAME, STR) \ diff --git a/src/libstore-tests/path-info.cc b/src/libstore-tests/path-info.cc index 9cd98a3d9..a7699f7ad 100644 --- a/src/libstore-tests/path-info.cc +++ b/src/libstore-tests/path-info.cc @@ -93,7 +93,7 @@ TEST_F(PathInfoTest, PathInfo_full_shortRefs) { ValidPathInfo it = makeFullKeyed(*store, true); // it.references = unkeyed.references; auto refs = it.shortRefs(); - ASSERT_EQ(refs.size(), 2); + ASSERT_EQ(refs.size(), 2u); ASSERT_EQ(*refs.begin(), "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"); ASSERT_EQ(*++refs.begin(), "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"); } diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 9297d46ea..69dab5488 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -374,7 +374,7 @@ VERSIONED_CHARACTERIZATION_TEST( set, "set", defaultVersion, - (std::tuple, std::set, std::set, std::set>> { + (std::tuple> { { }, { "" }, { "", "foo", "bar" }, diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index 091cf8a0e..4baf8a325 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -574,7 +574,7 @@ VERSIONED_CHARACTERIZATION_TEST( set, "set", defaultVersion, - (std::tuple, std::set, std::set, std::set>> { + (std::tuple> { { }, { "" }, { "", "foo", "bar" }, @@ -685,7 +685,7 @@ TEST_F(WorkerProtoTest, handshake_features) toClient.create(); toServer.create(); - std::tuple> clientResult; + std::tuple clientResult; auto clientThread = std::thread([&]() { FdSink out { toServer.writeSide.get() }; @@ -702,8 +702,8 @@ TEST_F(WorkerProtoTest, handshake_features) clientThread.join(); EXPECT_EQ(clientResult, daemonResult); - EXPECT_EQ(std::get<0>(clientResult), 123); - EXPECT_EQ(std::get<1>(clientResult), std::set({"bar", "xyzzy"})); + EXPECT_EQ(std::get<0>(clientResult), 123u); + EXPECT_EQ(std::get<1>(clientResult), WorkerProto::FeatureSet({"bar", "xyzzy"})); } /// Has to be a `BufferedSink` for handshake. diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 60bd68026..4df9651f0 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -24,13 +24,21 @@ namespace nix { -BinaryCacheStore::BinaryCacheStore(const Params & params) - : BinaryCacheStoreConfig(params) - , Store(params) +BinaryCacheStore::BinaryCacheStore(Config & config) + : config{config} { - if (secretKeyFile != "") - signer = std::make_unique( - SecretKey { readFile(secretKeyFile) }); + if (config.secretKeyFile != "") + signers.push_back(std::make_unique( + SecretKey { readFile(config.secretKeyFile) })); + + if (config.secretKeyFiles != "") { + std::stringstream ss(config.secretKeyFiles); + Path keyPath; + while (std::getline(ss, keyPath, ',')) { + signers.push_back(std::make_unique( + SecretKey { readFile(keyPath) })); + } + } StringSink sink; sink << narVersionMagic1; @@ -53,9 +61,9 @@ void BinaryCacheStore::init() throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", getUri(), value, storeDir); } else if (name == "WantMassQuery") { - wantMassQuery.setDefault(value == "1"); + config.wantMassQuery.setDefault(value == "1"); } else if (name == "Priority") { - priority.setDefault(std::stoi(value)); + config.priority.setDefault(std::stoi(value)); } } } @@ -147,7 +155,11 @@ ref BinaryCacheStore::addToStoreCommon( { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed { fileSink, fileHashSink }; - auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel); + auto compressionSink = makeCompressionSink( + config.compression, + teeSinkCompressed, + config.parallelCompression, + config.compressionLevel); TeeSink teeSinkUncompressed { *compressionSink, narHashSink }; TeeSource teeSource { narSource, teeSinkUncompressed }; narAccessor = makeNarAccessor(teeSource); @@ -159,17 +171,17 @@ ref BinaryCacheStore::addToStoreCommon( auto info = mkInfo(narHashSink.finish()); auto narInfo = make_ref(info); - narInfo->compression = compression; + narInfo->compression = config.compression; auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "zstd" ? ".zst" : - compression == "lzip" ? ".lzip" : - compression == "lz4" ? ".lz4" : - compression == "br" ? ".br" : + + (config.compression == "xz" ? ".xz" : + config.compression == "bzip2" ? ".bz2" : + config.compression == "zstd" ? ".zst" : + config.compression == "lzip" ? ".lzip" : + config.compression == "lz4" ? ".lz4" : + config.compression == "br" ? ".br" : ""); auto duration = std::chrono::duration_cast(now2 - now1).count(); @@ -191,7 +203,7 @@ ref BinaryCacheStore::addToStoreCommon( /* Optionally write a JSON file containing a listing of the contents of the NAR. */ - if (writeNARListing) { + if (config.writeNARListing) { nlohmann::json j = { {"version", 1}, {"root", listNar(ref(narAccessor), CanonPath::root, true)}, @@ -203,7 +215,7 @@ ref BinaryCacheStore::addToStoreCommon( /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/' that specify the NAR file and member containing the debug info. */ - if (writeDebugInfo) { + if (config.writeDebugInfo) { CanonPath buildIdDir("lib/debug/.build-id"); @@ -270,9 +282,9 @@ ref BinaryCacheStore::addToStoreCommon( stats.narWriteCompressedBytes += fileSize; stats.narWriteCompressionTimeMs += duration; - /* Atomically write the NAR info file.*/ - if (signer) narInfo->sign(*this, *signer); + narInfo->sign(*this, signers); + /* Atomically write the NAR info file.*/ writeNarInfo(narInfo); stats.narInfoWrite++; @@ -515,7 +527,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) { ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), requireValidPath, localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, config.localNarCache); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 33a4af7f0..81215eacf 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1,35 +1,22 @@ #include "nix/store/build/derivation-goal.hh" #ifndef _WIN32 // TODO enable build hook on Windows # include "nix/store/build/hook-instance.hh" +# include "nix/store/build/derivation-builder.hh" #endif #include "nix/util/processes.hh" #include "nix/util/config-global.hh" #include "nix/store/build/worker.hh" -#include "nix/store/builtins.hh" -#include "nix/store/builtins/buildenv.hh" -#include "nix/util/references.hh" -#include "nix/util/finally.hh" #include "nix/util/util.hh" -#include "nix/util/archive.hh" #include "nix/util/compression.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" -#include "nix/util/topo-sort.hh" -#include "nix/util/callback.hh" #include "nix/store/local-store.hh" // TODO remove, along with remaining downcasts -#include -#include - #include #include #include #include -#ifndef _WIN32 // TODO abstract over proc exit status -# include -#endif - #include #include "nix/util/strings.hh" @@ -38,7 +25,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) + : Goal(worker) , useDerivation(true) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -56,7 +43,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) + : Goal(worker) , useDerivation(false) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -82,6 +69,13 @@ DerivationGoal::~DerivationGoal() { /* Careful: we should never ever throw an exception from a destructor. */ + try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } +#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows + if (builder) { + try { builder->stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } + try { builder->deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } + } +#endif try { closeLogFile(); } catch (...) { ignoreExceptionInDestructor(); } } @@ -101,6 +95,22 @@ void DerivationGoal::killChild() #ifndef _WIN32 // TODO enable build hook on Windows hook.reset(); #endif +#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows + if (builder && builder->pid != -1) { + worker.childTerminated(this); + + /* If we're using a build user, then there is a tricky race + condition: if we kill the build user before the child has + done its setuid() to the build user uid, then it won't be + killed, and we'll potentially lock up in pid.wait(). So + also send a conventional kill to the child. */ + ::kill(-builder->pid, SIGKILL); /* ignore the result */ + + builder->killSandbox(true); + + builder->pid.wait(); + } +#endif } @@ -142,8 +152,8 @@ Goal::Co DerivationGoal::init() { substitute. */ if (buildMode != bmNormal || !worker.evalStore.isValidPath(drvPath)) { - addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); - co_await Suspend{}; + Goals waitees{upcast_goal(worker.makePathSubstitutionGoal(drvPath))}; + co_await await(std::move(waitees)); } trace("loading derivation"); @@ -180,8 +190,16 @@ Goal::Co DerivationGoal::haveDerivation() { trace("have derivation"); - parsedDrv = std::make_unique(drvPath, *drv); - drvOptions = std::make_unique(DerivationOptions::fromParsedDerivation(*parsedDrv)); + if (auto parsedOpt = StructuredAttrs::tryParse(drv->env)) { + parsedDrv = std::make_unique(*parsedOpt); + } + try { + drvOptions = std::make_unique( + DerivationOptions::fromStructuredAttrs(drv->env, parsedDrv.get())); + } catch (Error & e) { + e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath)); + throw; + } if (!drv->type().hasKnownOutputPaths()) experimentalFeatureSettings.require(Xp::CaDerivations); @@ -235,6 +253,8 @@ Goal::Co DerivationGoal::haveDerivation() } } + Goals waitees; + /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ @@ -242,7 +262,7 @@ Goal::Co DerivationGoal::haveDerivation() for (auto & [outputName, status] : initialOutputs) { if (!status.wanted) continue; if (!status.known) - addWaitee( + waitees.insert( upcast_goal( worker.makeDrvOutputSubstitutionGoal( DrvOutput{status.outputHash, outputName}, @@ -252,14 +272,14 @@ Goal::Co DerivationGoal::haveDerivation() ); else { auto * cap = getDerivationCA(*drv); - addWaitee(upcast_goal(worker.makePathSubstitutionGoal( + waitees.insert(upcast_goal(worker.makePathSubstitutionGoal( status.known->path, buildMode == bmRepair ? Repair : NoRepair, cap ? std::optional { *cap } : std::nullopt))); } } - if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */ + co_await await(std::move(waitees)); trace("all outputs substituted (maybe)"); @@ -322,7 +342,19 @@ Goal::Co DerivationGoal::haveDerivation() } -static std::string showKnownOutputs(Store & store, const Derivation & drv) +/** + * Used for `inputGoals` local variable below + */ +struct value_comparison +{ + template + bool operator()(const ref & lhs, const ref & rhs) const { + return *lhs < *rhs; + } +}; + + +std::string showKnownOutputs(Store & store, const Derivation & drv) { std::string msg; StorePathSet expectedOutputPaths; @@ -346,19 +378,24 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() is no need to restart. */ needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed; - /* The inputs must be built before we can build this goal. */ - inputDrvOutputs.clear(); - if (useDerivation) { - std::function, const DerivedPathMap::ChildNode &)> addWaiteeDerivedPath; + Goals waitees; - addWaiteeDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { - if (!inputNode.value.empty()) - addWaitee(worker.makeGoal( + std::map, GoalPtr, value_comparison> inputGoals; + + if (useDerivation) { + std::function, const DerivedPathMap::ChildNode &)> addWaiteeDerivedPath; + + addWaiteeDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) { + auto g = worker.makeGoal( DerivedPath::Built { .drvPath = inputDrv, .outputs = inputNode.value, }, - buildMode == bmRepair ? bmRepair : bmNormal)); + buildMode == bmRepair ? bmRepair : bmNormal); + inputGoals.insert_or_assign(inputDrv, g); + waitees.insert(std::move(g)); + } for (const auto & [outputName, childNode] : inputNode.childMap) addWaiteeDerivedPath( make_ref(SingleDerivedPath::Built { inputDrv, outputName }), @@ -398,10 +435,11 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() if (!settings.useSubstitutes) throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i))); + waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(i))); } - if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */ + co_await await(std::move(waitees)); + trace("all inputs realised"); @@ -451,7 +489,12 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() [&](const DerivationType::Impure &) { return true; } - }, drvType.raw); + }, drvType.raw) + /* no inputs are outputs of dynamic derivations */ + || std::ranges::any_of( + fullDrv.inputDrvs.map.begin(), + fullDrv.inputDrvs.map.end(), + [](auto & pair) { return !pair.second.childMap.empty(); }); if (resolveDrv && !fullDrv.inputDrvs.map.empty()) { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -459,7 +502,19 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal. */ - std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs); + std::optional attempt = fullDrv.tryResolve(worker.store, + [&](ref drvPath, const std::string & outputName) -> std::optional { + auto mEntry = get(inputGoals, drvPath); + if (!mEntry) return std::nullopt; + + auto buildResult = (*mEntry)->getBuildResult(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}}); + if (!buildResult.success()) return std::nullopt; + + auto i = get(buildResult.builtOutputs, outputName); + if (!i) return std::nullopt; + + return i->outPath; + }); if (!attempt) { /* TODO (impure derivations-induced tech debt) (see below): The above attempt should have found it, but because we manage @@ -482,62 +537,106 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() worker.store.printStorePath(pathResolved), }); - resolvedDrvGoal = worker.makeDerivationGoal( + auto resolvedDrvGoal = worker.makeDerivationGoal( pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedDrvGoal); + { + Goals waitees{resolvedDrvGoal}; + co_await await(std::move(waitees)); + } - co_await Suspend{}; - co_return resolvedFinished(); - } + trace("resolved derivation finished"); - std::function::ChildNode &)> accumInputPaths; + auto resolvedDrv = *resolvedDrvGoal->drv; + auto & resolvedResult = resolvedDrvGoal->buildResult; - accumInputPaths = [&](const StorePath & depDrvPath, const DerivedPathMap::ChildNode & inputNode) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - auto getOutput = [&](const std::string & outputName) { - /* TODO (impure derivations-induced tech debt): - Tracking input derivation outputs statefully through the - goals is error prone and has led to bugs. - For a robust nix, we need to move towards the `else` branch, - which does not rely on goal state to match up with the - reality of the store, which is our real source of truth. - However, the impure derivations feature still relies on this - fragile way of doing things, because its builds do not have - a representation in the store, which is a usability problem - in itself. When implementing this logic entirely with lookups - make sure that they're cached. */ - if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) { - return *outPath; - } - else { - auto outMap = [&]{ - for (auto * drvStore : { &worker.evalStore, &worker.store }) - if (drvStore->isValidPath(depDrvPath)) - return worker.store.queryDerivationOutputMap(depDrvPath, drvStore); - assert(false); + SingleDrvOutputs builtOutputs; + + if (resolvedResult.success()) { + auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); + + StorePathSet outputPaths; + + for (auto & outputName : resolvedDrv.outputNames()) { + auto initialOutput = get(initialOutputs, outputName); + auto resolvedHash = get(resolvedHashes, outputName); + if ((!initialOutput) || (!resolvedHash)) + throw Error( + "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", + worker.store.printStorePath(drvPath), outputName); + + auto realisation = [&]{ + auto take1 = get(resolvedResult.builtOutputs, outputName); + if (take1) return *take1; + + /* The above `get` should work. But sateful tracking of + outputs in resolvedResult, this can get out of sync with the + store, which is our actual source of truth. For now we just + check the store directly if it fails. */ + auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, outputName }); + if (take2) return *take2; + + throw Error( + "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/realisation)", + worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName); }(); - auto outMapPath = outMap.find(outputName); - if (outMapPath == outMap.end()) { - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath)); + if (!drv->type().isImpure()) { + auto newRealisation = realisation; + newRealisation.id = DrvOutput { initialOutput->outputHash, outputName }; + newRealisation.signatures.clear(); + if (!drv->type().isFixed()) { + auto & drvStore = worker.evalStore.isValidPath(drvPath) + ? worker.evalStore + : worker.store; + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore); + } + worker.store.signRealisation(newRealisation); + worker.store.registerDrvOutput(newRealisation); } - return outMapPath->second; + outputPaths.insert(realisation.outPath); + builtOutputs.emplace(outputName, realisation); } - }; - for (auto & outputName : inputNode.value) - worker.store.computeFSClosure(getOutput(outputName), inputPaths); + runPostBuildHook( + worker.store, + *logger, + drvPath, + outputPaths + ); + } - for (auto & [outputName, childNode] : inputNode.childMap) - accumInputPaths(getOutput(outputName), childNode); - }; + auto status = resolvedResult.status; + if (status == BuildResult::AlreadyValid) + status = BuildResult::ResolvesToAlreadyValid; - for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map) - accumInputPaths(depDrvPath, depNode); + co_return done(status, std::move(builtOutputs)); + } + + /* If we get this far, we know no dynamic drvs inputs */ + + for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map) { + for (auto & outputName : depNode.value) { + /* Don't need to worry about `inputGoals`, because + impure derivations are always resolved above. Can + just use DB. This case only happens in the (older) + input addressed and fixed output derivation cases. */ + auto outMap = [&]{ + for (auto * drvStore : { &worker.evalStore, &worker.store }) + if (drvStore->isValidPath(depDrvPath)) + return worker.store.queryDerivationOutputMap(depDrvPath, drvStore); + assert(false); + }(); + + auto outMapPath = outMap.find(outputName); + if (outMapPath == outMap.end()) { + throw Error( + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath)); + } + + worker.store.computeFSClosure(outMapPath->second, inputPaths); + } + } } /* Second, the input sources. */ @@ -545,14 +644,10 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution() debug("added input paths %s", worker.store.showPaths(inputPaths)); - /* What type of derivation are we building? */ - derivationType = drv->type(); - /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ - worker.wakeUp(shared_from_this()); - co_await Suspend{}; + co_await yield(); co_return tryToBuild(); } @@ -609,16 +704,17 @@ Goal::Co DerivationGoal::tryToBuild() } } - if (!outputLocks.lockPaths(lockFiles, "", false)) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + if (!outputLocks.lockPaths(lockFiles, "", false)) + { + Activity act(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); - worker.waitForAWhile(shared_from_this()); - co_await Suspend{}; - co_return tryToBuild(); - } - actLock.reset(); + /* Wait then try locking again, repeat until success (returned + boolean is true). */ + do { + co_await waitForAWhile(); + } while (!outputLocks.lockPaths(lockFiles, "", false)); + } /* Now check again whether the outputs are valid. This is because another process may have started building in parallel. After @@ -661,16 +757,15 @@ Goal::Co DerivationGoal::tryToBuild() buildResult.startTime = time(0); // inexact started(); co_await Suspend{}; - co_return buildDone(); + co_return hookDone(); case rpPostpone: /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); - co_await Suspend{}; + co_await waitForAWhile(); co_return tryToBuild(); case rpDecline: /* We should do it ourselves. */ @@ -680,20 +775,153 @@ Goal::Co DerivationGoal::tryToBuild() actLock.reset(); - worker.wakeUp(shared_from_this()); + co_await yield(); + + if (!dynamic_cast(&worker.store)) { + throw Error( + R"( + Unable to build with a primary store that isn't a local store; + either pass a different '--store' or enable remote builds. + + For more information check 'man nix.conf' and search for '/machines'. + )" + ); + } + +#ifdef _WIN32 // TODO enable `DerivationBuilder` on Windows + throw UnimplementedError("building derivations is not yet implemented on Windows"); +#else + + // Will continue here while waiting for a build user below + while (true) { + + assert(!hook); + + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= settings.maxBuildJobs) { + outputLocks.unlock(); + co_await waitForBuildSlot(); + co_return tryToBuild(); + } + + if (!builder) { + /** + * Local implementation of these virtual methods, consider + * this just a record of lambdas. + */ + struct DerivationGoalCallbacks : DerivationBuilderCallbacks + { + DerivationGoal & goal; + + DerivationGoalCallbacks(DerivationGoal & goal, std::unique_ptr & builder) + : goal{goal} + {} + + ~DerivationGoalCallbacks() override = default; + + void childStarted(Descriptor builderOut) override + { + goal.worker.childStarted(goal.shared_from_this(), {builderOut}, true, true); + } + + void childTerminated() override + { + goal.worker.childTerminated(&goal); + } + + void noteHashMismatch() override + { + goal.worker.hashMismatch = true; + } + + void noteCheckMismatch() override + { + goal.worker.checkMismatch = true; + } + + void markContentsGood(const StorePath & path) override + { + goal.worker.markContentsGood(path); + } + + Path openLogFile() override { + return goal.openLogFile(); + } + void closeLogFile() override { + goal.closeLogFile(); + } + SingleDrvOutputs assertPathValidity() override { + return goal.assertPathValidity(); + } + void appendLogTailErrorMsg(std::string & msg) override { + goal.appendLogTailErrorMsg(msg); + } + }; + + /* If we have to wait and retry (see below), then `builder` will + already be created, so we don't need to create it again. */ + builder = makeDerivationBuilder( + worker.store, + std::make_unique(*this, builder), + DerivationBuilderParams { + drvPath, + buildMode, + buildResult, + *drv, + parsedDrv.get(), + *drvOptions, + inputPaths, + initialOutputs, + }); + } + + if (!builder->prepareBuild()) { + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); + co_await waitForAWhile(); + continue; + } + + break; + } + + actLock.reset(); + + try { + + /* Okay, we have to build. */ + builder->startBuilder(); + + } catch (BuildError & e) { + outputLocks.unlock(); + builder->buildUser.reset(); + worker.permanentFailure = true; + co_return done(BuildResult::InputRejected, {}, std::move(e)); + } + + started(); co_await Suspend{}; - co_return tryLocalBuild(); -} -Goal::Co DerivationGoal::tryLocalBuild() { - throw Error( - R"( - Unable to build with a primary store that isn't a local store; - either pass a different '--store' or enable remote builds. + trace("build done"); - For more information check 'man nix.conf' and search for '/machines'. - )" - ); + auto res = builder->unprepareBuild(); + // N.B. cannot use `std::visit` with co-routine return + if (auto * ste = std::get_if<0>(&res)) { + outputLocks.unlock(); + co_return done(std::move(ste->first), {}, std::move(ste->second)); + } else if (auto * builtOutputs = std::get_if<1>(&res)) { + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + co_return done(BuildResult::Built, std::move(*builtOutputs)); + } else { + unreachable(); + } +#endif } @@ -732,6 +960,8 @@ Goal::Co DerivationGoal::repairClosure() outputsToDrv.insert_or_assign(*j.second, i); } + Goals waitees; + /* Check each path (slow!). */ for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; @@ -740,9 +970,9 @@ Goal::Co DerivationGoal::repairClosure() worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) - addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); + waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); else - addWaitee(worker.makeGoal( + waitees.insert(worker.makeGoal( DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath2->second), .outputs = OutputsSpec::All { }, @@ -750,127 +980,18 @@ Goal::Co DerivationGoal::repairClosure() bmRepair)); } - if (waitees.empty()) { - co_return done(BuildResult::AlreadyValid, assertPathValidity()); - } else { - co_await Suspend{}; + co_await await(std::move(waitees)); + if (!waitees.empty()) { trace("closure repaired"); if (nrFailed > 0) throw Error("some paths in the output closure of derivation '%s' could not be repaired", worker.store.printStorePath(drvPath)); - co_return done(BuildResult::AlreadyValid, assertPathValidity()); } + co_return done(BuildResult::AlreadyValid, assertPathValidity()); } -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = ( -#ifndef _WIN32 - geteuid() -#else - !isRootUser() -#endif - && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - std::filesystem::rename(src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - -void replaceValidPath(const Path & storePath, const Path & tmpPath) -{ - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), rand()); - if (pathExists(storePath)) - movePath(storePath, oldPath); - - try { - movePath(tmpPath, storePath); - } catch (...) { - try { - // attempt to recover - movePath(oldPath, storePath); - } catch (...) { - ignoreExceptionExceptInterrupt(); - } - throw; - } - - deletePath(oldPath); -} - - -int DerivationGoal::getChildStatus() -{ -#ifndef _WIN32 // TODO enable build hook on Windows - return hook->pid.kill(); -#else - return 0; -#endif -} - - -void DerivationGoal::closeReadPipes() -{ -#ifndef _WIN32 // TODO enable build hook on Windows - hook->builderOut.readSide.close(); - hook->fromHook.readSide.close(); -#endif -} - - -void DerivationGoal::cleanupHookFinally() -{ -} - - -void DerivationGoal::cleanupPreChildKill() -{ -} - - -void DerivationGoal::cleanupPostChildKill() -{ -} - - -bool DerivationGoal::cleanupDecideWhetherDiskFull() -{ - return false; -} - - -void DerivationGoal::cleanupPostOutputsRegisteredModeCheck() -{ -} - - -void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() -{ -} - void runPostBuildHook( Store & store, Logger & logger, @@ -929,21 +1050,47 @@ void runPostBuildHook( }); } -Goal::Co DerivationGoal::buildDone() + +void DerivationGoal::appendLogTailErrorMsg(std::string & msg) { - trace("build done"); + if (!logger->isVerbose() && !logTail.empty()) { + msg += fmt("\nLast %d log lines:\n", logTail.size()); + for (auto & line : logTail) { + msg += "> "; + msg += line; + msg += "\n"; + } + auto nixLogCommand = "nix log"; + // The command is on a separate line for easy copying, such as with triple click. + // This message will be indented elsewhere, so removing the indentation before the + // command will not put it at the start of the line unfortunately. + msg += fmt("For full logs, run:\n " ANSI_BOLD "%s %s" ANSI_NORMAL, + nixLogCommand, + worker.store.printStorePath(drvPath)); + } +} - Finally releaseBuildUser([&](){ this->cleanupHookFinally(); }); - cleanupPreChildKill(); +Goal::Co DerivationGoal::hookDone() +{ +#ifndef _WIN32 + assert(hook); +#endif + + trace("hook build done"); /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have simply have closed its end of the pipe, so just to be sure, kill it. */ - int status = getChildStatus(); + int status = +#ifndef _WIN32 // TODO enable build hook on Windows + hook->pid.kill(); +#else + 0; +#endif - debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); + debug("build hook for '%s' finished", worker.store.printStorePath(drvPath)); buildResult.timesBuilt++; buildResult.stopTime = time(0); @@ -952,181 +1099,63 @@ Goal::Co DerivationGoal::buildDone() worker.childTerminated(this); /* Close the read side of the logger pipe. */ - closeReadPipes(); +#ifndef _WIN32 // TODO enable build hook on Windows + hook->builderOut.readSide.close(); + hook->fromHook.readSide.close(); +#endif /* Close the log file. */ closeLogFile(); - cleanupPostChildKill(); + /* Check the exit status. */ + if (!statusOk(status)) { + auto msg = fmt( + "Cannot build '%s'.\n" + "Reason: " ANSI_RED "builder %s" ANSI_NORMAL ".", + Magenta(worker.store.printStorePath(drvPath)), + statusToString(status)); - if (buildResult.cpuUser && buildResult.cpuSystem) { - debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs", - worker.store.printStorePath(drvPath), - status, - ((double) buildResult.cpuUser->count()) / 1000000, - ((double) buildResult.cpuSystem->count()) / 1000000); - } + msg += showKnownOutputs(worker.store, *drv); - bool diskFull = false; + appendLogTailErrorMsg(msg); - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - diskFull |= cleanupDecideWhetherDiskFull(); - - auto msg = fmt( - "Cannot build '%s'.\n" - "Reason: " ANSI_RED "builder %s" ANSI_NORMAL ".", - Magenta(worker.store.printStorePath(drvPath)), - statusToString(status)); - - msg += showKnownOutputs(worker.store, *drv); - - if (!logger->isVerbose() && !logTail.empty()) { - msg += fmt("\nLast %d log lines:\n", logTail.size()); - for (auto & line : logTail) { - msg += "> "; - msg += line; - msg += "\n"; - } - auto nixLogCommand = "nix log"; - // The command is on a separate line for easy copying, such as with triple click. - // This message will be indented elsewhere, so removing the indentation before the - // command will not put it at the start of the line unfortunately. - msg += fmt("For full logs, run:\n " ANSI_BOLD "%s %s" ANSI_NORMAL, - nixLogCommand, - worker.store.printStorePath(drvPath)); - } - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - auto builtOutputs = registerOutputs(); - - StorePathSet outputPaths; - for (auto & [_, output] : builtOutputs) - outputPaths.insert(output.outPath); - runPostBuildHook( - worker.store, - *logger, - drvPath, - outputPaths - ); - - cleanupPostOutputsRegisteredModeNonCheck(); - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); outputLocks.unlock(); - co_return done(BuildResult::Built, std::move(builtOutputs)); + /* TODO (once again) support fine-grained error codes, see issue #12641. */ - } catch (BuildError & e) { - outputLocks.unlock(); - - BuildResult::Status st = BuildResult::MiscFailure; - -#ifndef _WIN32 // TODO abstract over proc exit status - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) - st = BuildResult::TimedOut; - - else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - } - - else -#endif - { - assert(derivationType); - st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - !derivationType->isSandboxed() || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - } - - co_return done(st, {}, std::move(e)); - } -} - -Goal::Co DerivationGoal::resolvedFinished() -{ - trace("resolved derivation finished"); - - assert(resolvedDrvGoal); - auto resolvedDrv = *resolvedDrvGoal->drv; - auto & resolvedResult = resolvedDrvGoal->buildResult; - - SingleDrvOutputs builtOutputs; - - if (resolvedResult.success()) { - auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); - - StorePathSet outputPaths; - - for (auto & outputName : resolvedDrv.outputNames()) { - auto initialOutput = get(initialOutputs, outputName); - auto resolvedHash = get(resolvedHashes, outputName); - if ((!initialOutput) || (!resolvedHash)) - throw Error( - "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)", - worker.store.printStorePath(drvPath), outputName); - - auto realisation = [&]{ - auto take1 = get(resolvedResult.builtOutputs, outputName); - if (take1) return *take1; - - /* The above `get` should work. But sateful tracking of - outputs in resolvedResult, this can get out of sync with the - store, which is our actual source of truth. For now we just - check the store directly if it fails. */ - auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, outputName }); - if (take2) return *take2; - - throw Error( - "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", - worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName); - }(); - - if (!drv->type().isImpure()) { - auto newRealisation = realisation; - newRealisation.id = DrvOutput { initialOutput->outputHash, outputName }; - newRealisation.signatures.clear(); - if (!drv->type().isFixed()) { - auto & drvStore = worker.evalStore.isValidPath(drvPath) - ? worker.evalStore - : worker.store; - newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore); - } - signRealisation(newRealisation); - worker.store.registerDrvOutput(newRealisation); - } - outputPaths.insert(realisation.outPath); - builtOutputs.emplace(outputName, realisation); - } - - runPostBuildHook( - worker.store, - *logger, - drvPath, - outputPaths - ); + co_return done(BuildResult::MiscFailure, {}, BuildError(msg)); } - auto status = resolvedResult.status; - if (status == BuildResult::AlreadyValid) - status = BuildResult::ResolvesToAlreadyValid; + /* Compute the FS closure of the outputs and register them as + being valid. */ + auto builtOutputs = + /* When using a build hook, the build hook can register the output + as valid (by doing `nix-store --import'). If so we don't have + to do anything here. - co_return done(status, std::move(builtOutputs)); + We can only early return when the outputs are known a priori. For + floating content-addressing derivations this isn't the case. + */ + assertPathValidity(); + + StorePathSet outputPaths; + for (auto & [_, output] : builtOutputs) + outputPaths.insert(output.outPath); + runPostBuildHook( + worker.store, + *logger, + drvPath, + outputPaths + ); + + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + + co_return done(BuildResult::Built, std::move(builtOutputs)); } HookReply DerivationGoal::tryBuildHook() @@ -1242,18 +1271,6 @@ HookReply DerivationGoal::tryBuildHook() } -SingleDrvOutputs DerivationGoal::registerOutputs() -{ - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. - - We can only early return when the outputs are known a priori. For - floating content-addressing derivations this isn't the case. - */ - return assertPathValidity(); -} - Path DerivationGoal::openLogFile() { logSize = 0; @@ -1265,7 +1282,7 @@ Path DerivationGoal::openLogFile() /* Create a log file. */ Path logDir; if (auto localStore = dynamic_cast(&worker.store)) - logDir = localStore->logDir; + logDir = localStore->config->logDir; else logDir = settings.nixLogDir; Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2)); @@ -1307,7 +1324,10 @@ bool DerivationGoal::isReadDesc(Descriptor fd) #ifdef _WIN32 // TODO enable build hook on Windows return false; #else - return fd == hook->builderOut.readSide.get(); + return + (hook && fd == hook->builderOut.readSide.get()) + || + (builder && fd == builder->builderOut.get()); #endif } @@ -1576,34 +1596,4 @@ Goal::Done DerivationGoal::done( return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } - -void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - Goal::waiteeDone(waitee, result); - - if (!useDerivation || !drv) return; - auto & fullDrv = *dynamic_cast(drv.get()); - - auto * dg = dynamic_cast(&*waitee); - if (!dg) return; - - auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath }); - if (!nodeP) return; - auto & outputs = nodeP->value; - - for (auto & outputName : outputs) { - auto buildResult = dg->getBuildResult(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(dg->drvPath), - .outputs = OutputsSpec::Names { outputName }, - }); - if (buildResult.success()) { - auto i = buildResult.builtOutputs.find(outputName); - if (i != buildResult.builtOutputs.end()) - inputDrvOutputs.insert_or_assign( - { dg->drvPath, outputName }, - i->second.outPath); - } - } -} - } diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index bc2030fa5..c553eeedb 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -3,6 +3,7 @@ #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/util/callback.hh" +#include "nix/store/store-open.hh" namespace nix { @@ -11,7 +12,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( Worker & worker, RepairFlag repair, std::optional ca) - : Goal(worker, DerivedPath::Opaque { StorePath::dummy }) + : Goal(worker) , id(id) { name = fmt("substitution of '%s'", id.to_string()); @@ -87,6 +88,8 @@ Goal::Co DrvOutputSubstitutionGoal::init() bool failed = false; + Goals waitees; + for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { if (depId != id) { if (auto localOutputInfo = worker.store.queryRealisation(depId); @@ -103,13 +106,13 @@ Goal::Co DrvOutputSubstitutionGoal::init() failed = true; break; } - addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); + waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId)); } } if (failed) continue; - co_return realisationFetched(outputInfo, sub); + co_return realisationFetched(std::move(waitees), outputInfo, sub); } /* None left. Terminate this goal and let someone else deal @@ -127,10 +130,10 @@ Goal::Co DrvOutputSubstitutionGoal::init() co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters); } -Goal::Co DrvOutputSubstitutionGoal::realisationFetched(std::shared_ptr outputInfo, nix::ref sub) { - addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); +Goal::Co DrvOutputSubstitutionGoal::realisationFetched(Goals waitees, std::shared_ptr outputInfo, nix::ref sub) { + waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath)); - if (!waitees.empty()) co_await Suspend{}; + co_await await(std::move(waitees)); trace("output path substituted"); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index aaa426793..d2feb34c7 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -132,38 +132,18 @@ void addToWeakGoals(WeakGoals & goals, GoalPtr p) goals.insert(p); } - -void Goal::addWaitee(GoalPtr waitee) +Co Goal::await(Goals new_waitees) { - waitees.insert(waitee); - addToWeakGoals(waitee->waiters, shared_from_this()); -} - - -void Goal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - assert(waitees.count(waitee)); - waitees.erase(waitee); - - trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); - - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; - - if (result == ecNoSubstituters) ++nrNoSubstituters; - - if (result == ecIncompleteClosure) ++nrIncompleteClosure; - - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { - - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & goal : waitees) { - goal->waiters.extract(shared_from_this()); + assert(waitees.empty()); + if (!new_waitees.empty()) { + waitees = std::move(new_waitees); + for (auto waitee : waitees) { + addToWeakGoals(waitee->waiters, shared_from_this()); } - waitees.clear(); - - worker.wakeUp(shared_from_this()); + co_await Suspend{}; + assert(waitees.empty()); } + co_return Return{}; } Goal::Done Goal::amDone(ExitCode result, std::optional ex) @@ -183,7 +163,32 @@ Goal::Done Goal::amDone(ExitCode result, std::optional ex) for (auto & i : waiters) { GoalPtr goal = i.lock(); - if (goal) goal->waiteeDone(shared_from_this(), result); + if (goal) { + auto me = shared_from_this(); + assert(goal->waitees.count(me)); + goal->waitees.erase(me); + + goal->trace(fmt("waitee '%s' done; %d left", name, goal->waitees.size())); + + if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++goal->nrFailed; + + if (result == ecNoSubstituters) ++goal->nrNoSubstituters; + + if (result == ecIncompleteClosure) ++goal->nrIncompleteClosure; + + if (goal->waitees.empty()) { + worker.wakeUp(goal); + } else if (result == ecFailed && !settings.keepGoing) { + /* If we failed and keepGoing is not set, we remove all + remaining waitees. */ + for (auto & g : goal->waitees) { + g->waiters.extract(goal); + } + goal->waitees.clear(); + + worker.wakeUp(goal); + } + } } waiters.clear(); worker.removeGoal(shared_from_this()); @@ -215,5 +220,22 @@ void Goal::work() assert(top_co || exitCode != ecBusy); } +Goal::Co Goal::yield() { + worker.wakeUp(shared_from_this()); + co_await Suspend{}; + co_return Return{}; +} + +Goal::Co Goal::waitForAWhile() { + worker.waitForAWhile(shared_from_this()); + co_await Suspend{}; + co_return Return{}; +} + +Goal::Co Goal::waitForBuildSlot() { + worker.waitForBuildSlot(shared_from_this()); + co_await Suspend{}; + co_return Return{}; +} } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index fc1261935..c07f309e4 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -1,4 +1,5 @@ #include "nix/store/build/worker.hh" +#include "nix/store/store-open.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/nar-info.hh" #include "nix/util/finally.hh" @@ -11,7 +12,7 @@ namespace nix { PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) - : Goal(worker, DerivedPath::Opaque { storePath }) + : Goal(worker) , storePath(storePath) , repair(repair) , ca(ca) @@ -133,20 +134,22 @@ Goal::Co PathSubstitutionGoal::init() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) + if (!sub->config.isTrusted && worker.store.pathInfoIsUntrusted(*info)) { warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); continue; } + Goals waitees; + /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ for (auto & i : info->references) if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); + waitees.insert(worker.makePathSubstitutionGoal(i)); - if (!waitees.empty()) co_await Suspend{}; + co_await await(std::move(waitees)); // FIXME: consider returning boolean instead of passing in reference bool out = false; // is mutated by tryToRun @@ -184,11 +187,15 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, } for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); + /* ignore self-references */ + if (i != storePath) { + if (!worker.store.isValidPath(i)) { + throw Error("reference '%s' of path '%s' is not a valid path", + worker.store.printStorePath(i), worker.store.printStorePath(storePath)); + } + } - worker.wakeUp(shared_from_this()); - co_await Suspend{}; + co_await yield(); trace("trying to run"); @@ -196,8 +203,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, if maxSubstitutionJobs == 0, we still allow a substituter to run. This prevents infinite waiting. */ while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) { - worker.waitForBuildSlot(shared_from_this()); - co_await Suspend{}; + co_await waitForBuildSlot(); } auto maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); @@ -222,7 +228,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, PushActivity pact(act.id); copyStorePath(*sub, worker.store, - subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 87710e9ee..dd3692f41 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -5,7 +5,6 @@ #include "nix/store/build/drv-output-substitution-goal.hh" #include "nix/store/build/derivation-goal.hh" #ifndef _WIN32 // TODO Enable building on Windows -# include "nix/store/build/local-derivation-goal.hh" # include "nix/store/build/hook-instance.hh" #endif #include "nix/util/signals.hh" @@ -65,13 +64,7 @@ std::shared_ptr Worker::makeDerivationGoal(const StorePath & drv const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { - return -#ifndef _WIN32 // TODO Enable building on Windows - dynamic_cast(&store) - ? std::make_shared(drvPath, wantedOutputs, *this, buildMode) - : -#endif - std::make_shared(drvPath, wantedOutputs, *this, buildMode); + return std::make_shared(drvPath, wantedOutputs, *this, buildMode); }); } @@ -79,13 +72,7 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { - return -#ifndef _WIN32 // TODO Enable building on Windows - dynamic_cast(&store) - ? std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode) - : -#endif - std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); + return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); }); } @@ -524,7 +511,7 @@ bool Worker::pathContentsGood(const StorePath & path) res = false; else { auto current = hashPath( - {store.getFSAccessor(), CanonPath(store.printStorePath(path))}, + {store.getFSAccessor(), CanonPath(path.to_string())}, FileIngestionMethod::NixArchive, info->narHash.algo).first; Hash nullHash(HashAlgorithm::SHA256); res = info->narHash == nullHash || info->narHash == current; @@ -552,4 +539,9 @@ GoalPtr upcast_goal(std::shared_ptr subGoal) return subGoal; } +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ + return subGoal; +} + } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c3b80bb0b..0e99ca0e5 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,5 @@ #include "nix/store/builtins/buildenv.hh" +#include "nix/store/builtins.hh" #include "nix/store/derivations.hh" #include "nix/util/signals.hh" @@ -18,12 +19,12 @@ struct State /* For each activated package, create symlinks */ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority) { - std::filesystem::directory_iterator srcFiles; + DirectoryIterator srcFiles; try { - srcFiles = std::filesystem::directory_iterator{srcDir}; - } catch (std::filesystem::filesystem_error & e) { - if (e.code() == std::errc::not_a_directory) { + srcFiles = DirectoryIterator{srcDir}; + } catch (SysError & e) { + if (e.errNo == ENOTDIR) { warn("not including '%s' in the user environment because it's not a directory", srcDir); return; } @@ -123,7 +124,7 @@ void buildProfile(const Path & out, Packages && pkgs) { State state; - std::set done, postponed; + PathSet done, postponed; auto addPkg = [&](const Path & pkgDir, int priority) { if (!done.insert(pkgDir).second) return; @@ -157,7 +158,7 @@ void buildProfile(const Path & out, Packages && pkgs) */ auto priorityCounter = 1000; while (!postponed.empty()) { - std::set pkgDirs; + PathSet pkgDirs; postponed.swap(pkgDirs); for (const auto & pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++); @@ -166,17 +167,15 @@ void buildProfile(const Path & out, Packages && pkgs) debug("created %d symlinks in user environment", state.symlinks); } -void builtinBuildenv( - const BasicDerivation & drv, - const std::map & outputs) +static void builtinBuildenv(const BuiltinBuilderContext & ctx) { auto getAttr = [&](const std::string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + auto i = ctx.drv.env.find(name); + if (i == ctx.drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; }; - auto out = outputs.at("out"); + auto out = ctx.outputs.at("out"); createDirs(out); /* Convert the stuff we get from the environment back into a @@ -203,4 +202,6 @@ void builtinBuildenv( createSymlink(getAttr("manifest"), out + "/manifest.nix"); } +static RegisterBuiltinBuilder registerBuildenv("buildenv", builtinBuildenv); + } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 82f268d80..18fa75558 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -6,33 +6,29 @@ namespace nix { -void builtinFetchurl( - const BasicDerivation & drv, - const std::map & outputs, - const std::string & netrcData, - const std::string & caFileData) +static void builtinFetchurl(const BuiltinBuilderContext & ctx) { /* Make the host's netrc data available. Too bad curl requires this to be stored in a file. It would be nice if we could just pass a pointer to the data. */ - if (netrcData != "") { + if (ctx.netrcData != "") { settings.netrcFile = "netrc"; - writeFile(settings.netrcFile, netrcData, 0600); + writeFile(settings.netrcFile, ctx.netrcData, 0600); } settings.caFile = "ca-certificates.crt"; - writeFile(settings.caFile, caFileData, 0600); + writeFile(settings.caFile, ctx.caFileData, 0600); - auto out = get(drv.outputs, "out"); + auto out = get(ctx.drv.outputs, "out"); if (!out) throw Error("'builtin:fetchurl' requires an 'out' output"); - if (!(drv.type().isFixed() || drv.type().isImpure())) + if (!(ctx.drv.type().isFixed() || ctx.drv.type().isImpure())) throw Error("'builtin:fetchurl' must be a fixed-output or impure derivation"); - auto storePath = outputs.at("out"); - auto mainUrl = drv.env.at("url"); - bool unpack = getOr(drv.env, "unpack", "") == "1"; + auto storePath = ctx.outputs.at("out"); + auto mainUrl = ctx.drv.env.at("url"); + bool unpack = getOr(ctx.drv.env, "unpack", "") == "1"; /* Note: have to use a fresh fileTransfer here because we're in a forked process. */ @@ -56,8 +52,8 @@ void builtinFetchurl( else writeFile(storePath, *source); - auto executable = drv.env.find("executable"); - if (executable != drv.env.end() && executable->second == "1") { + auto executable = ctx.drv.env.find("executable"); + if (executable != ctx.drv.env.end() && executable->second == "1") { if (chmod(storePath.c_str(), 0755) == -1) throw SysError("making '%1%' executable", storePath); } @@ -79,4 +75,6 @@ void builtinFetchurl( fetch(mainUrl); } +static RegisterBuiltinBuilder registerFetchurl("fetchurl", builtinFetchurl); + } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index f6be21e35..dd6b8bb71 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -3,23 +3,19 @@ namespace nix { -namespace fs { using namespace std::filesystem; } - -void builtinUnpackChannel( - const BasicDerivation & drv, - const std::map & outputs) +static void builtinUnpackChannel(const BuiltinBuilderContext & ctx) { auto getAttr = [&](const std::string & name) -> const std::string & { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + auto i = ctx.drv.env.find(name); + if (i == ctx.drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; }; - fs::path out{outputs.at("out")}; + std::filesystem::path out{ctx.outputs.at("out")}; auto & channelName = getAttr("channelName"); auto & src = getAttr("src"); - if (fs::path{channelName}.filename().string() != channelName) { + if (std::filesystem::path{channelName}.filename().string() != channelName) { throw Error("channelName is not allowed to contain filesystem separators, got %1%", channelName); } @@ -29,23 +25,21 @@ void builtinUnpackChannel( size_t fileCount; std::string fileName; - try { - auto entries = fs::directory_iterator{out}; - fileName = entries->path().string(); - fileCount = std::distance(fs::begin(entries), fs::end(entries)); - } catch (fs::filesystem_error &) { - throw SysError("failed to read directory %1%", out.string()); - } + auto entries = DirectoryIterator{out}; + fileName = entries->path().string(); + fileCount = std::distance(entries.begin(), entries.end()); if (fileCount != 1) throw Error("channel tarball '%s' contains more than one file", src); auto target = out / channelName; try { - fs::rename(fileName, target); - } catch (fs::filesystem_error &) { + std::filesystem::rename(fileName, target); + } catch (std::filesystem::filesystem_error &) { throw SysError("failed to rename %1% to %2%", fileName, target.string()); } } +static RegisterBuiltinBuilder registerUnpackChannel("unpack-channel", builtinUnpackChannel); + } diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 7cfbc5f98..bcaa11a96 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -28,7 +28,7 @@ CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_ { } -SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) +SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const { return { host, diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index af3a319e9..e031f8447 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -1,46 +1,124 @@ #include "nix/store/derivation-options.hh" #include "nix/util/json-utils.hh" #include "nix/store/parsed-derivations.hh" +#include "nix/store/derivations.hh" +#include "nix/store/store-api.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" + #include #include #include +#include namespace nix { +static std::optional +getStringAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name) +{ + if (parsed) { + auto i = parsed->structuredAttrs.find(name); + if (i == parsed->structuredAttrs.end()) + return {}; + else { + if (!i->is_string()) + throw Error("attribute '%s' of must be a string", name); + return i->get(); + } + } else { + auto i = env.find(name); + if (i == env.end()) + return {}; + else + return i->second; + } +} + +static bool getBoolAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name, bool def) +{ + if (parsed) { + auto i = parsed->structuredAttrs.find(name); + if (i == parsed->structuredAttrs.end()) + return def; + else { + if (!i->is_boolean()) + throw Error("attribute '%s' must be a Boolean", name); + return i->get(); + } + } else { + auto i = env.find(name); + if (i == env.end()) + return def; + else + return i->second == "1"; + } +} + +static std::optional +getStringsAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name) +{ + if (parsed) { + auto i = parsed->structuredAttrs.find(name); + if (i == parsed->structuredAttrs.end()) + return {}; + else { + if (!i->is_array()) + throw Error("attribute '%s' must be a list of strings", name); + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' must be a list of strings", name); + res.push_back(j->get()); + } + return res; + } + } else { + auto i = env.find(name); + if (i == env.end()) + return {}; + else + return tokenizeString(i->second); + } +} + +static std::optional +getStringSetAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name) +{ + auto ss = getStringsAttr(env, parsed, name); + return ss ? (std::optional{StringSet{ss->begin(), ss->end()}}) : (std::optional{}); +} + using OutputChecks = DerivationOptions::OutputChecks; using OutputChecksVariant = std::variant>; -DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation & parsed, bool shouldWarn) +DerivationOptions +DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn) { DerivationOptions defaults = {}; - auto structuredAttrs = parsed.structuredAttrs.get(); - - if (shouldWarn && structuredAttrs) { - if (get(*structuredAttrs, "allowedReferences")) { + if (shouldWarn && parsed) { + if (get(parsed->structuredAttrs, "allowedReferences")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead"); } - if (get(*structuredAttrs, "allowedRequisites")) { + if (get(parsed->structuredAttrs, "allowedRequisites")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead"); } - if (get(*structuredAttrs, "disallowedRequisites")) { + if (get(parsed->structuredAttrs, "disallowedRequisites")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead"); } - if (get(*structuredAttrs, "disallowedReferences")) { + if (get(parsed->structuredAttrs, "disallowedReferences")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead"); } - if (get(*structuredAttrs, "maxSize")) { + if (get(parsed->structuredAttrs, "maxSize")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead"); } - if (get(*structuredAttrs, "maxClosureSize")) { + if (get(parsed->structuredAttrs, "maxClosureSize")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead"); } @@ -48,9 +126,9 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation return { .outputChecks = [&]() -> OutputChecksVariant { - if (auto structuredAttrs = parsed.structuredAttrs.get()) { + if (parsed) { std::map res; - if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { + if (auto outputChecks = get(parsed->structuredAttrs, "outputChecks")) { for (auto & [outputName, output] : getObject(*outputChecks)) { OutputChecks checks; @@ -87,10 +165,10 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation return OutputChecks{ // legacy non-structured-attributes case .ignoreSelfRefs = true, - .allowedReferences = parsed.getStringSetAttr("allowedReferences"), - .disallowedReferences = parsed.getStringSetAttr("disallowedReferences").value_or(StringSet{}), - .allowedRequisites = parsed.getStringSetAttr("allowedRequisites"), - .disallowedRequisites = parsed.getStringSetAttr("disallowedRequisites").value_or(StringSet{}), + .allowedReferences = getStringSetAttr(env, parsed, "allowedReferences"), + .disallowedReferences = getStringSetAttr(env, parsed, "disallowedReferences").value_or(StringSet{}), + .allowedRequisites = getStringSetAttr(env, parsed, "allowedRequisites"), + .disallowedRequisites = getStringSetAttr(env, parsed, "disallowedRequisites").value_or(StringSet{}), }; } }(), @@ -98,8 +176,8 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation [&] { std::map res; - if (auto structuredAttrs = parsed.structuredAttrs.get()) { - if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { + if (parsed) { + if (auto udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) { for (auto & [outputName, output] : getObject(*udr)) { if (!output.is_boolean()) throw Error("attribute 'unsafeDiscardReferences.\"%s\"' must be a Boolean", outputName); @@ -113,8 +191,8 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation .passAsFile = [&] { StringSet res; - if (auto * passAsFileString = get(parsed.drv.env, "passAsFile")) { - if (parsed.hasStructuredAttrs()) { + if (auto * passAsFileString = get(env, "passAsFile")) { + if (parsed) { if (shouldWarn) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'passAsFile'; because all JSON is always passed via file"); @@ -125,16 +203,44 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation } return res; }(), + .exportReferencesGraph = + [&] { + std::map ret; + + if (parsed) { + auto e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph"); + if (!e || !e->is_object()) + return ret; + for (auto & [key, storePathsJson] : getObject(*e)) { + ret.insert_or_assign(key, storePathsJson); + } + } else { + auto s = getOr(env, "exportReferencesGraph", ""); + Strings ss = tokenizeString(s); + if (ss.size() % 2 != 0) + throw Error("odd number of tokens in 'exportReferencesGraph': '%1%'", s); + for (Strings::iterator i = ss.begin(); i != ss.end();) { + auto fileName = std::move(*i++); + static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); + if (!std::regex_match(fileName, regex)) + throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); + + auto & storePathS = *i++; + ret.insert_or_assign(std::move(fileName), StringSet{storePathS}); + } + } + return ret; + }(), .additionalSandboxProfile = - parsed.getStringAttr("__sandboxProfile").value_or(defaults.additionalSandboxProfile), - .noChroot = parsed.getBoolAttr("__noChroot", defaults.noChroot), - .impureHostDeps = parsed.getStringSetAttr("__impureHostDeps").value_or(defaults.impureHostDeps), - .impureEnvVars = parsed.getStringSetAttr("impureEnvVars").value_or(defaults.impureEnvVars), - .allowLocalNetworking = parsed.getBoolAttr("__darwinAllowLocalNetworking", defaults.allowLocalNetworking), + getStringAttr(env, parsed, "__sandboxProfile").value_or(defaults.additionalSandboxProfile), + .noChroot = getBoolAttr(env, parsed, "__noChroot", defaults.noChroot), + .impureHostDeps = getStringSetAttr(env, parsed, "__impureHostDeps").value_or(defaults.impureHostDeps), + .impureEnvVars = getStringSetAttr(env, parsed, "impureEnvVars").value_or(defaults.impureEnvVars), + .allowLocalNetworking = getBoolAttr(env, parsed, "__darwinAllowLocalNetworking", defaults.allowLocalNetworking), .requiredSystemFeatures = - parsed.getStringSetAttr("requiredSystemFeatures").value_or(defaults.requiredSystemFeatures), - .preferLocalBuild = parsed.getBoolAttr("preferLocalBuild", defaults.preferLocalBuild), - .allowSubstitutes = parsed.getBoolAttr("allowSubstitutes", defaults.allowSubstitutes), + getStringSetAttr(env, parsed, "requiredSystemFeatures").value_or(defaults.requiredSystemFeatures), + .preferLocalBuild = getBoolAttr(env, parsed, "preferLocalBuild", defaults.preferLocalBuild), + .allowSubstitutes = getBoolAttr(env, parsed, "allowSubstitutes", defaults.allowSubstitutes), }; } @@ -159,7 +265,7 @@ bool DerivationOptions::canBuildLocally(Store & localStore, const BasicDerivatio return false; for (auto & feature : getRequiredSystemFeatures(drv)) - if (!localStore.systemFeatures.get().count(feature)) + if (!localStore.config.systemFeatures.get().count(feature)) return false; return true; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fdfdc37b4..42de5ee0c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1052,49 +1052,36 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String std::optional Derivation::tryResolve(Store & store, Store * evalStore) const { - std::map, StorePath> inputDrvOutputs; - - std::function::ChildNode &)> accum; - accum = [&](auto & inputDrv, auto & node) { - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) { - if (outputPath) { - inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); - if (auto p = get(node.childMap, outputName)) - accum(*outputPath, *p); + return tryResolve( + store, + [&](ref drvPath, const std::string & outputName) -> std::optional { + try { + return resolveDerivedPath(store, SingleDerivedPath::Built{drvPath, outputName}, evalStore); + } catch (Error &) { + return std::nullopt; } - } - }; - - for (auto & [inputDrv, node] : inputDrvs.map) - accum(inputDrv, node); - - return tryResolve(store, inputDrvOutputs); + }); } static bool tryResolveInput( Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites, const DownstreamPlaceholder * placeholderOpt, - const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode, - const std::map, StorePath> & inputDrvOutputs) + ref drvPath, const DerivedPathMap::ChildNode & inputNode, + std::function(ref drvPath, const std::string & outputName)> queryResolutionChain) { - auto getOutput = [&](const std::string & outputName) { - auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName }); - if (!actualPathOpt) - warn("output %s of input %s missing, aborting the resolving", - outputName, - store.printStorePath(inputDrv) - ); - return actualPathOpt; - }; - auto getPlaceholder = [&](const std::string & outputName) { return placeholderOpt ? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName) - : DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName); + : [&]{ + auto * p = std::get_if(&drvPath->raw()); + // otherwise we should have had a placeholder to build-upon already + assert(p); + return DownstreamPlaceholder::unknownCaOutput(p->path, outputName); + }(); }; for (auto & outputName : inputNode.value) { - auto actualPathOpt = getOutput(outputName); + auto actualPathOpt = queryResolutionChain(drvPath, outputName); if (!actualPathOpt) return false; auto actualPath = *actualPathOpt; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { @@ -1106,13 +1093,12 @@ static bool tryResolveInput( } for (auto & [outputName, childNode] : inputNode.childMap) { - auto actualPathOpt = getOutput(outputName); - if (!actualPathOpt) return false; - auto actualPath = *actualPathOpt; auto nextPlaceholder = getPlaceholder(outputName); if (!tryResolveInput(store, inputSrcs, inputRewrites, - &nextPlaceholder, actualPath, childNode, - inputDrvOutputs)) + &nextPlaceholder, + make_ref(SingleDerivedPath::Built{drvPath, outputName}), + childNode, + queryResolutionChain)) return false; } return true; @@ -1120,7 +1106,7 @@ static bool tryResolveInput( std::optional Derivation::tryResolve( Store & store, - const std::map, StorePath> & inputDrvOutputs) const + std::function(ref drvPath, const std::string & outputName)> queryResolutionChain) const { BasicDerivation resolved { *this }; @@ -1129,7 +1115,7 @@ std::optional Derivation::tryResolve( for (auto & [inputDrv, inputNode] : inputDrvs.map) if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, - nullptr, inputDrv, inputNode, inputDrvOutputs)) + nullptr, make_ref(SingleDerivedPath::Opaque{inputDrv}), inputNode, queryResolutionChain)) return std::nullopt; rewriteDerivation(store, resolved, inputRewrites); @@ -1242,13 +1228,13 @@ DerivationOutput DerivationOutput::fromJSON( keys.insert(key); auto methodAlgo = [&]() -> std::pair { - auto & method_ = getString(valueAt(json, "method")); - ContentAddressMethod method = ContentAddressMethod::parse(method_); + ContentAddressMethod method = ContentAddressMethod::parse( + getString(valueAt(json, "method"))); if (method == ContentAddressMethod::Raw::Text) xpSettings.require(Xp::DynamicDerivations); - auto & hashAlgo_ = getString(valueAt(json, "hashAlgo")); - auto hashAlgo = parseHashAlgo(hashAlgo_); + auto hashAlgo = parseHashAlgo( + getString(valueAt(json, "hashAlgo"))); return { std::move(method), std::move(hashAlgo) }; }; @@ -1365,7 +1351,8 @@ Derivation Derivation::fromJSON( res.name = getString(valueAt(json, "name")); try { - for (auto & [outputName, output] : getObject(valueAt(json, "outputs"))) { + auto outputs = getObject(valueAt(json, "outputs")); + for (auto & [outputName, output] : outputs) { res.outputs.insert_or_assign( outputName, DerivationOutput::fromJSON(store, res.name, outputName, output, xpSettings)); @@ -1376,7 +1363,8 @@ Derivation Derivation::fromJSON( } try { - for (auto & input : getArray(valueAt(json, "inputSrcs"))) + auto inputSrcs = getArray(valueAt(json, "inputSrcs")); + for (auto & input : inputSrcs) res.inputSrcs.insert(store.parseStorePath(static_cast(input))); } catch (Error & e) { e.addTrace({}, "while reading key 'inputSrcs'"); @@ -1389,13 +1377,15 @@ Derivation Derivation::fromJSON( auto & json = getObject(_json); DerivedPathMap::ChildNode node; node.value = getStringSet(valueAt(json, "outputs")); - for (auto & [outputId, childNode] : getObject(valueAt(json, "dynamicOutputs"))) { + auto drvs = getObject(valueAt(json, "dynamicOutputs")); + for (auto & [outputId, childNode] : drvs) { xpSettings.require(Xp::DynamicDerivations); node.childMap[outputId] = doInput(childNode); } return node; }; - for (auto & [inputDrvPath, inputOutputs] : getObject(valueAt(json, "inputDrvs"))) + auto drvs = getObject(valueAt(json, "inputDrvs")); + for (auto & [inputDrvPath, inputOutputs] : drvs) res.inputDrvs.map[store.parseStorePath(inputDrvPath)] = doInput(inputOutputs); } catch (Error & e) { diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index d4234d92c..b785dddd9 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -55,17 +55,17 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single namespace nix { template<> -bool DerivedPathMap>::ChildNode::operator == ( - const DerivedPathMap>::ChildNode &) const noexcept = default; +bool DerivedPathMap::ChildNode::operator == ( + const DerivedPathMap::ChildNode &) const noexcept = default; // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. #if 0 template<> -std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( - const DerivedPathMap>::ChildNode &) const noexcept = default; +std::strong_ordering DerivedPathMap::ChildNode::operator <=> ( + const DerivedPathMap::ChildNode &) const noexcept = default; #endif -template struct DerivedPathMap>::ChildNode; -template struct DerivedPathMap>; +template struct DerivedPathMap::ChildNode; +template struct DerivedPathMap; }; diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 950ac1c1a..6186f0582 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -170,7 +170,7 @@ void drvRequireExperiment( } SingleDerivedPath::Built SingleDerivedPath::Built::parse( - const StoreDirConfig & store, ref drv, + const StoreDirConfig & store, ref drv, OutputNameView output, const ExperimentalFeatureSettings & xpSettings) { @@ -182,7 +182,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse( } DerivedPath::Built DerivedPath::Built::parse( - const StoreDirConfig & store, ref drv, + const StoreDirConfig & store, ref drv, OutputNameView outputsS, const ExperimentalFeatureSettings & xpSettings) { @@ -201,7 +201,7 @@ static SingleDerivedPath parseWithSingle( return n == s.npos ? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s) : (SingleDerivedPath) SingleDerivedPath::Built::parse(store, - make_ref(parseWithSingle( + make_ref(parseWithSingle( store, s.substr(0, n), separator, @@ -234,7 +234,7 @@ static DerivedPath parseWith( return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) : (DerivedPath) DerivedPath::Built::parse(store, - make_ref(parseWithSingle( + make_ref(parseWithSingle( store, s.substr(0, n), separator, diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 7252e1d33..819c47bab 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,9 +1,9 @@ -#include "nix/store/store-api.hh" +#include "nix/store/store-registration.hh" #include "nix/util/callback.hh" namespace nix { -struct DummyStoreConfig : virtual StoreConfig { +struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig { using StoreConfig::StoreConfig; DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) @@ -13,35 +13,36 @@ struct DummyStoreConfig : virtual StoreConfig { throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); } - const std::string name() override { return "Dummy Store"; } + static const std::string name() { return "Dummy Store"; } - std::string doc() override + static std::string doc() { return #include "dummy-store.md" ; } - static std::set uriSchemes() { + static StringSet uriSchemes() { return {"dummy"}; } + + ref openStore() const override; }; -struct DummyStore : public virtual DummyStoreConfig, public virtual Store +struct DummyStore : virtual Store { - DummyStore(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , DummyStoreConfig(scheme, authority, params) - , Store(params) - { } + using Config = DummyStoreConfig; - DummyStore(const Params & params) - : DummyStore("dummy", "", params) + ref config; + + DummyStore(ref config) + : Store{*config} + , config(config) { } std::string getUri() override { - return *uriSchemes().begin(); + return *Config::uriSchemes().begin(); } void queryPathInfoUncached(const StorePath & path, @@ -83,9 +84,16 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store { callback(nullptr); } virtual ref getFSAccessor(bool requireValidPath) override - { unsupported("getFSAccessor"); } + { + return makeEmptySourceAccessor(); + } }; -static RegisterStoreImplementation regDummyStore; +ref DummyStore::Config::openStore() const +{ + return make_ref(ref{shared_from_this()}); +} + +static RegisterStoreImplementation regDummyStore; } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index fb7c6c7a2..164cb37a7 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -33,6 +33,9 @@ using namespace std::string_literals; namespace nix { +const unsigned int RETRY_TIME_MS_DEFAULT = 250; +const unsigned int RETRY_TIME_MS_TOO_MANY_REQUESTS = 60000; + FileTransferSettings fileTransferSettings; static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); @@ -309,6 +312,23 @@ struct curlFileTransfer : public FileTransfer } #endif + size_t seekCallback(curl_off_t offset, int origin) + { + if (origin == SEEK_SET) { + readOffset = offset; + } else if (origin == SEEK_CUR) { + readOffset += offset; + } else if (origin == SEEK_END) { + readOffset = request.data->length() + offset; + } + return CURL_SEEKFUNC_OK; + } + + static size_t seekCallbackWrapper(void *clientp, curl_off_t offset, int origin) + { + return ((TransferItem *) clientp)->seekCallback(offset, origin); + } + void init() { if (!req) req = curl_easy_init(); @@ -363,6 +383,8 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); curl_easy_setopt(req, CURLOPT_READDATA, this); curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length()); + curl_easy_setopt(req, CURLOPT_SEEKFUNCTION, seekCallbackWrapper); + curl_easy_setopt(req, CURLOPT_SEEKDATA, this); } if (request.verifyTLS) { @@ -401,6 +423,8 @@ struct curlFileTransfer : public FileTransfer { auto finishTime = std::chrono::steady_clock::now(); + auto retryTimeMs = request.baseRetryTimeMs; + auto httpStatus = getHTTPStatus(); debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", @@ -451,10 +475,12 @@ struct curlFileTransfer : public FileTransfer } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) { // Don't retry on authentication/authorization failures err = Forbidden; - } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) { + } else if (httpStatus == 429) { + // 429 means too many requests, so we retry (with a substantially longer delay) + retryTimeMs = RETRY_TIME_MS_TOO_MANY_REQUESTS; + } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408) { // Most 4xx errors are client errors and are probably not worth retrying: // * 408 means the server timed out waiting for us, so we try again - // * 429 means too many requests, so we retry (with a delay) err = Misc; } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) { // Let's treat most 5xx (server) errors as transient, except for a handful: @@ -520,7 +546,7 @@ struct curlFileTransfer : public FileTransfer || writtenToSink == 0 || (acceptRanges && encoding.empty()))) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); + int ms = retryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); if (writtenToSink) warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); else @@ -747,7 +773,7 @@ struct curlFileTransfer : public FileTransfer } #if NIX_WITH_S3_SUPPORT - std::tuple parseS3Uri(std::string uri) + std::tuple parseS3Uri(std::string uri) { auto [path, params] = splitUriAndParams(uri); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index dabfa4a5f..8fad9661c 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -43,7 +43,7 @@ static std::string gcRootsDir = "gcroots"; void LocalStore::addIndirectRoot(const Path & path) { std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false); - Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); + Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", config->stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } @@ -82,7 +82,7 @@ void LocalStore::createTempRootsFile() void LocalStore::addTempRoot(const StorePath & path) { - if (readOnly) { + if (config->readOnly) { debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways."); return; } @@ -109,7 +109,7 @@ void LocalStore::addTempRoot(const StorePath & path) auto fdRootsSocket(_fdRootsSocket.lock()); if (!*fdRootsSocket) { - auto socketPath = stateDir.get() + gcSocketPath; + auto socketPath = config->stateDir.get() + gcSocketPath; debug("connecting to '%s'", socketPath); *fdRootsSocket = createUnixDomainSocket(); try { @@ -164,7 +164,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) { /* Read the `temproots' directory for per-process temporary root files. */ - for (auto & i : std::filesystem::directory_iterator{tempRootsDir}) { + for (auto & i : DirectoryIterator{tempRootsDir}) { checkInterrupt(); auto name = i.path().filename().string(); if (name[0] == '.') { @@ -232,7 +232,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R type = std::filesystem::symlink_status(path).type(); if (type == std::filesystem::file_type::directory) { - for (auto & i : std::filesystem::directory_iterator{path}) { + for (auto & i : DirectoryIterator{path}) { checkInterrupt(); findRoots(i.path().string(), i.symlink_status().type(), roots); } @@ -247,7 +247,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R else { target = absPath(target, dirOf(path)); if (!pathExists(target)) { - if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { + if (isInDir(path, std::filesystem::path{config->stateDir.get()} / gcRootsDir / "auto")) { printInfo("removing stale link from '%1%' to '%2%'", path, target); unlink(path.c_str()); } @@ -288,8 +288,8 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { /* Process direct roots in {gcroots,profiles}. */ - findRoots(stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots); - findRoots(stateDir + "/profiles", std::filesystem::file_type::unknown, roots); + findRoots(config->stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots); + findRoots(config->stateDir + "/profiles", std::filesystem::file_type::unknown, roots); /* Add additional roots returned by different platforms-specific heuristics. This is typically used to add running programs to @@ -498,7 +498,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) readFile(*p); /* Start the server for receiving new roots. */ - auto socketPath = stateDir.get() + gcSocketPath; + auto socketPath = config->stateDir.get() + gcSocketPath; createDirs(dirOf(socketPath)); auto fdServer = createUnixDomainSocket(socketPath, 0666); @@ -635,7 +635,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto deleteFromStore = [&](std::string_view baseName) { Path path = storeDir + "/" + std::string(baseName); - Path realPath = realStoreDir + "/" + std::string(baseName); + Path realPath = config->realStoreDir + "/" + std::string(baseName); /* There may be temp directories in the store that are still in use by another process. We need to be sure that we can acquire an @@ -804,8 +804,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printInfo("determining live/dead paths..."); try { - AutoCloseDir dir(opendir(realStoreDir.get().c_str())); - if (!dir) throw SysError("opening directory '%1%'", realStoreDir); + AutoCloseDir dir(opendir(config->realStoreDir.get().c_str())); + if (!dir) throw SysError("opening directory '%1%'", config->realStoreDir); /* Read the store and delete all paths that are invalid or unreachable. We don't use readDirectory() here so that @@ -907,8 +907,8 @@ void LocalStore::autoGC(bool sync) return std::stoll(readFile(*fakeFreeSpaceFile)); struct statvfs st; - if (statvfs(realStoreDir.get().c_str(), &st)) - throw SysError("getting filesystem info about '%s'", realStoreDir); + if (statvfs(config->realStoreDir.get().c_str(), &st)) + throw SysError("getting filesystem info about '%s'", config->realStoreDir); return (uint64_t) st.f_bavail * st.f_frsize; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 4c13d5c73..2b591dda9 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -3,12 +3,22 @@ #include "nix/store/globals.hh" #include "nix/store/nar-info-disk-cache.hh" #include "nix/util/callback.hh" +#include "nix/store/store-registration.hh" namespace nix { MakeError(UploadToHTTP, Error); +StringSet HttpBinaryCacheStoreConfig::uriSchemes() +{ + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = StringSet{"http", "https"}; + if (forceHttp) + ret.insert("file"); + return ret; +} + HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view _cacheUri, @@ -35,10 +45,9 @@ std::string HttpBinaryCacheStoreConfig::doc() } -class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore +class HttpBinaryCacheStore : + public virtual BinaryCacheStore { -private: - struct State { bool enabled = true; @@ -49,37 +58,37 @@ private: public: - HttpBinaryCacheStore( - std::string_view scheme, - PathView cacheUri, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , HttpBinaryCacheStoreConfig(scheme, cacheUri, params) - , Store(params) - , BinaryCacheStore(params) + using Config = HttpBinaryCacheStoreConfig; + + ref config; + + HttpBinaryCacheStore(ref config) + : Store{*config} + // TODO it will actually mutate the configuration + , BinaryCacheStore{*config} + , config{config} { diskCache = getNarInfoDiskCache(); } std::string getUri() override { - return cacheUri; + return config->cacheUri; } void init() override { // FIXME: do this lazily? - if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { - wantMassQuery.setDefault(cacheInfo->wantMassQuery); - priority.setDefault(cacheInfo->priority); + if (auto cacheInfo = diskCache->upToDateCacheExists(config->cacheUri)) { + config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); + config->priority.setDefault(cacheInfo->priority); } else { try { BinaryCacheStore::init(); } catch (UploadToHTTP &) { - throw Error("'%s' does not appear to be a binary cache", cacheUri); + throw Error("'%s' does not appear to be a binary cache", config->cacheUri); } - diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority); + diskCache->createCache(config->cacheUri, config->storeDir, config->wantMassQuery, config->priority); } } @@ -137,7 +146,7 @@ protected: try { getFileTransfer()->upload(req); } catch (FileTransferError & e) { - throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); + throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri, e.msg()); } } @@ -146,7 +155,7 @@ protected: return FileTransferRequest( hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://") ? path - : cacheUri + "/" + path); + : config->cacheUri + "/" + path); } @@ -221,6 +230,14 @@ protected: } }; -static RegisterStoreImplementation regHttpBinaryCacheStore; +ref HttpBinaryCacheStore::Config::openStore() const +{ + return make_ref(ref{ + // FIXME we shouldn't actually need a mutable config + std::const_pointer_cast(shared_from_this()) + }); +} + +static RegisterStoreImplementation regHttpBinaryCacheStore; } diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index da4906d3f..43f2cf690 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -32,6 +32,9 @@ struct BinaryCacheStoreConfig : virtual StoreConfig const Setting secretKeyFile{this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; + const Setting secretKeyFiles{this, "", "secret-keys", + "List of comma-separated paths to the secret keys used to sign the binary cache."}; + const Setting localNarCache{this, "", "local-nar-cache", "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; @@ -51,13 +54,20 @@ struct BinaryCacheStoreConfig : virtual StoreConfig * @note subclasses must implement at least one of the two * virtual getFile() methods. */ -class BinaryCacheStore : public virtual BinaryCacheStoreConfig, - public virtual Store, - public virtual LogStore +struct BinaryCacheStore : + virtual Store, + virtual LogStore { + using Config = BinaryCacheStoreConfig; + + /** + * Intentionally mutable because some things we update due to the + * cache's own (remote side) settings. + */ + Config & config; private: - std::unique_ptr signer; + std::vector> signers; protected: @@ -66,7 +76,7 @@ protected: const std::string cacheInfoFile = "nix-cache-info"; - BinaryCacheStore(const Params & params); + BinaryCacheStore(Config &); public: diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh new file mode 100644 index 000000000..915d891d7 --- /dev/null +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -0,0 +1,58 @@ +#pragma once +/** + * @file Misc type defitions for both local building and remote (RPC building) + */ + +#include "nix/util/hash.hh" +#include "nix/store/path.hh" + +namespace nix { + +class Store; +struct Derivation; + +/** + * Unless we are repairing, we don't both to test validity and just assume it, + * so the choices are `Absent` or `Valid`. + */ +enum struct PathStatus { + Corrupt, + Absent, + Valid, +}; + +struct InitialOutputStatus +{ + StorePath path; + PathStatus status; + /** + * Valid in the store, and additionally non-corrupt if we are repairing + */ + bool isValid() const + { + return status == PathStatus::Valid; + } + /** + * Merely present, allowed to be corrupt + */ + bool isPresent() const + { + return status == PathStatus::Corrupt || status == PathStatus::Valid; + } +}; + +struct InitialOutput +{ + bool wanted; + Hash outputHash; + std::optional known; +}; + +void runPostBuildHook(Store & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths); + +/** + * Format the known outputs of a derivation for use in error messages. + */ +std::string showKnownOutputs(Store & store, const Derivation & drv); + +} diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index 8a1c6f33b..485a34ec4 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -2,10 +2,9 @@ ///@file #include "nix/store/parsed-derivations.hh" +#include "nix/store/derivations.hh" #include "nix/store/derivation-options.hh" -#ifndef _WIN32 -# include "nix/store/user-lock.hh" -#endif +#include "nix/store/build/derivation-building-misc.hh" #include "nix/store/outputs-spec.hh" #include "nix/store/store-api.hh" #include "nix/store/pathlocks.hh" @@ -17,43 +16,17 @@ using std::map; #ifndef _WIN32 // TODO enable build hook on Windows struct HookInstance; +struct DerivationBuilder; #endif typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -/** - * Unless we are repairing, we don't both to test validity and just assume it, - * so the choices are `Absent` or `Valid`. - */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /** - * Valid in the store, and additionally non-corrupt if we are repairing - */ - bool isValid() const { - return status == PathStatus::Valid; - } - /** - * Merely present, allowed to be corrupt - */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - Hash outputHash; - std::optional known; -}; +/** Used internally */ +void runPostBuildHook( + Store & store, + Logger & logger, + const StorePath & drvPath, + const StorePathSet & outputPaths); /** * A goal for building some or all of the outputs of a derivation. @@ -68,23 +41,11 @@ struct DerivationGoal : public Goal /** The path of the derivation. */ StorePath drvPath; - /** - * The goal for the corresponding resolved derivation - */ - std::shared_ptr resolvedDrvGoal; - /** * The specific outputs that we need to build. */ OutputsSpec wantedOutputs; - /** - * Mapping from input derivations + output names to actual store - * paths. This is filled in by waiteeDone() as each dependency - * finishes, before `trace("all inputs realised")` is reached. - */ - std::map, StorePath> inputDrvOutputs; - /** * See `needRestart`; just for that field. */ @@ -143,7 +104,7 @@ struct DerivationGoal : public Goal */ std::unique_ptr drv; - std::unique_ptr parsedDrv; + std::unique_ptr parsedDrv; std::unique_ptr drvOptions; /** @@ -189,12 +150,9 @@ struct DerivationGoal : public Goal * The build hook. */ std::unique_ptr hook; -#endif - /** - * The sort of derivation we are building. - */ - std::optional derivationType; + std::unique_ptr builder; +#endif BuildMode buildMode; @@ -220,7 +178,7 @@ struct DerivationGoal : public Goal DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); - virtual ~DerivationGoal(); + ~DerivationGoal(); void timedOut(Error && ex) override; @@ -238,55 +196,24 @@ struct DerivationGoal : public Goal Co haveDerivation(); Co gaveUpOnSubstitution(); Co tryToBuild(); - virtual Co tryLocalBuild(); - Co buildDone(); - - Co resolvedFinished(); + Co hookDone(); /** * Is the build hook willing to perform the build? */ HookReply tryBuildHook(); - virtual int getChildStatus(); - - /** - * Check that the derivation outputs all exist and register them - * as valid. - */ - virtual SingleDrvOutputs registerOutputs(); - /** * Open a log file and a pipe to it. */ Path openLogFile(); - /** - * Sign the newly built realisation if the store allows it - */ - virtual void signRealisation(Realisation&) {} - /** * Close the log file. */ void closeLogFile(); - /** - * Close the read side of the logger pipe. - */ - virtual void closeReadPipes(); - - /** - * Cleanup hooks for buildDone() - */ - virtual void cleanupHookFinally(); - virtual void cleanupPreChildKill(); - virtual void cleanupPostChildKill(); - virtual bool cleanupDecideWhetherDiskFull(); - virtual void cleanupPostOutputsRegisteredModeCheck(); - virtual void cleanupPostOutputsRegisteredModeNonCheck(); - - virtual bool isReadDesc(Descriptor fd); + bool isReadDesc(Descriptor fd); /** * Callback used by the worker to write to the log. @@ -320,7 +247,7 @@ struct DerivationGoal : public Goal /** * Forcibly kill the child process, if any. */ - virtual void killChild(); + void killChild(); Co repairClosure(); @@ -331,7 +258,7 @@ struct DerivationGoal : public Goal SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); - void waiteeDone(GoalPtr waitee, ExitCode result) override; + void appendLogTailErrorMsg(std::string & msg); StorePathSet exportReferences(const StorePathSet & storePaths); @@ -340,6 +267,4 @@ struct DerivationGoal : public Goal }; }; -MakeError(NotDeterministic, BuildError); - } diff --git a/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh b/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh index 81d66fe1e..a00de41ad 100644 --- a/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh +++ b/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh @@ -34,7 +34,7 @@ public: GoalState state; Co init() override; - Co realisationFetched(std::shared_ptr outputInfo, nix::ref sub); + Co realisationFetched(Goals waitees, std::shared_ptr outputInfo, nix::ref sub); void timedOut(Error && ex) override { unreachable(); }; diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 7c3873012..9be27f6b3 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -54,6 +54,13 @@ enum struct JobCategory { struct Goal : public std::enable_shared_from_this { +private: + /** + * Goals that this goal is waiting for. + */ + Goals waitees; + +public: typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; /** @@ -61,11 +68,6 @@ struct Goal : public std::enable_shared_from_this */ Worker & worker; - /** - * Goals that this goal is waiting for. - */ - Goals waitees; - /** * Goals waiting for this one to finish. Must use weak pointers * here to prevent cycles. @@ -104,8 +106,8 @@ protected: * Build result. */ BuildResult buildResult; -public: +public: /** * Suspend our goal and wait until we get `work`-ed again. * `co_await`-able by @ref Co. @@ -332,6 +334,7 @@ public: std::suspend_always await_transform(Suspend) { return {}; }; }; +protected: /** * The coroutine being currently executed. * MUST be updated when switching the coroutine being executed. @@ -359,6 +362,7 @@ public: */ Done amDone(ExitCode result, std::optional ex = {}); +public: virtual void cleanup() { } /** @@ -378,7 +382,7 @@ public: */ std::optional ex; - Goal(Worker & worker, DerivedPath path) + Goal(Worker & worker) : worker(worker), top_co(init_wrapper()) { // top_co shouldn't have a goal already, should be nullptr. @@ -394,10 +398,6 @@ public: void work(); - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - virtual void handleChildOutput(Descriptor fd, std::string_view data) { unreachable(); @@ -429,6 +429,13 @@ public: * @see JobCategory */ virtual JobCategory jobCategory() const = 0; + +protected: + Co await(Goals waitees); + + Co waitForAWhile(); + Co waitForBuildSlot(); + Co yield(); }; void addToWeakGoals(WeakGoals & goals, GoalPtr p); diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index 004e9ef64..096c8af7b 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -5,15 +5,30 @@ namespace nix { -// TODO: make pluggable. -void builtinFetchurl( - const BasicDerivation & drv, - const std::map & outputs, - const std::string & netrcData, - const std::string & caFileData); +struct BuiltinBuilderContext +{ + const BasicDerivation & drv; + std::map outputs; + std::string netrcData; + std::string caFileData; + Path tmpDirInSandbox; +}; -void builtinUnpackChannel( - const BasicDerivation & drv, - const std::map & outputs); +using BuiltinBuilder = std::function; + +struct RegisterBuiltinBuilder +{ + typedef std::map BuiltinBuilders; + + static BuiltinBuilders & builtinBuilders() { + static BuiltinBuilders builders; + return builders; + } + + RegisterBuiltinBuilder(const std::string & name, BuiltinBuilder && fun) + { + builtinBuilders().insert_or_assign(name, std::move(fun)); + } +}; } diff --git a/src/libstore/include/nix/store/builtins/buildenv.hh b/src/libstore/include/nix/store/builtins/buildenv.hh index a0a262037..163666c0b 100644 --- a/src/libstore/include/nix/store/builtins/buildenv.hh +++ b/src/libstore/include/nix/store/builtins/buildenv.hh @@ -45,8 +45,4 @@ typedef std::vector Packages; void buildProfile(const Path & out, Packages && pkgs); -void builtinBuildenv( - const BasicDerivation & drv, - const std::map & outputs); - } diff --git a/src/libstore/include/nix/store/common-protocol-impl.hh b/src/libstore/include/nix/store/common-protocol-impl.hh index 171b4c6a5..18e63ac33 100644 --- a/src/libstore/include/nix/store/common-protocol-impl.hh +++ b/src/libstore/include/nix/store/common-protocol-impl.hh @@ -25,11 +25,11 @@ namespace nix { LengthPrefixedProtoHelper::write(store, conn, t); \ } +#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) -COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) -#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER( template, std::map) diff --git a/src/libstore/include/nix/store/common-protocol.hh b/src/libstore/include/nix/store/common-protocol.hh index b464cda67..7887120b5 100644 --- a/src/libstore/include/nix/store/common-protocol.hh +++ b/src/libstore/include/nix/store/common-protocol.hh @@ -72,14 +72,14 @@ DECLARE_COMMON_SERIALISER(DrvOutput); template<> DECLARE_COMMON_SERIALISER(Realisation); +#define COMMA_ , template DECLARE_COMMON_SERIALISER(std::vector); -template -DECLARE_COMMON_SERIALISER(std::set); +template +DECLARE_COMMON_SERIALISER(std::set); template DECLARE_COMMON_SERIALISER(std::tuple); -#define COMMA_ , template DECLARE_COMMON_SERIALISER(std::map); #undef COMMA_ diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index f82124c66..82a78f075 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -56,7 +56,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig */ SSHMaster createSSHMaster( bool useMaster, - Descriptor logFD = INVALID_DESCRIPTOR); + Descriptor logFD = INVALID_DESCRIPTOR) const; }; } diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index 8f549b737..16730b5c9 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -13,16 +13,16 @@ namespace nix { class Store; struct BasicDerivation; -class ParsedDerivation; +struct StructuredAttrs; /** * This represents all the special options on a `Derivation`. * * Currently, these options are parsed from the environment variables - * with the aid of `ParsedDerivation`. + * with the aid of `StructuredAttrs`. * * The first goal of this data type is to make sure that no other code - * uses `ParsedDerivation` to ad-hoc parse some additional options. That + * uses `StructuredAttrs` to ad-hoc parse some additional options. That * ensures this data type is up to date and fully correct. * * The second goal of this data type is to allow an alternative to @@ -95,6 +95,27 @@ struct DerivationOptions */ StringSet passAsFile; + /** + * The `exportReferencesGraph' feature allows the references graph + * to be passed to a builder + * + * ### Legacy case + * + * Given a `name` `pathSet` key-value pair, the references graph of + * `pathSet` will be stored in a text file `name' in the temporary + * build directory. The text files have the format used by + * `nix-store + * --register-validity'. However, the `deriver` fields are left + * empty. + * + * ### "Structured attributes" case + * + * The same information will be put put in the final structured + * attributes give to the builder. The set of paths in the original JSON + * is replaced with a list of `PathInfo` in JSON format. + */ + std::map exportReferencesGraph; + /** * env: __sandboxProfile * @@ -152,7 +173,8 @@ struct DerivationOptions * (e.g. JSON) but is necessary for supporing old formats (e.g. * ATerm). */ - static DerivationOptions fromParsedDerivation(const ParsedDerivation & parsed, bool shouldWarn = true); + static DerivationOptions + fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn = true); /** * @param drv Must be the same derivation we parsed this from. In diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index df490dc7b..46a9e2d02 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -343,7 +343,7 @@ struct Derivation : BasicDerivation /** * inputs that are sub-derivations */ - DerivedPathMap> inputDrvs; + DerivedPathMap>> inputDrvs; /** * Print a derivation. @@ -369,7 +369,7 @@ struct Derivation : BasicDerivation */ std::optional tryResolve( Store & store, - const std::map, StorePath> & inputDrvOutputs) const; + std::function(ref drvPath, const std::string & outputName)> queryResolutionChain) const; /** * Check that the derivation is valid and does not present any diff --git a/src/libstore/include/nix/store/derived-path-map.hh b/src/libstore/include/nix/store/derived-path-map.hh index 956f8bb0b..cad86d1b4 100644 --- a/src/libstore/include/nix/store/derived-path-map.hh +++ b/src/libstore/include/nix/store/derived-path-map.hh @@ -91,20 +91,20 @@ struct DerivedPathMap { }; template<> -bool DerivedPathMap>::ChildNode::operator == ( - const DerivedPathMap>::ChildNode &) const noexcept; +bool DerivedPathMap::ChildNode::operator == ( + const DerivedPathMap::ChildNode &) const noexcept; // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. #if 0 template<> -std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( - const DerivedPathMap>::ChildNode &) const noexcept; +std::strong_ordering DerivedPathMap::ChildNode::operator <=> ( + const DerivedPathMap::ChildNode &) const noexcept; template<> -inline auto DerivedPathMap>::operator <=> (const DerivedPathMap> &) const noexcept = default; +inline auto DerivedPathMap::operator <=> (const DerivedPathMap &) const noexcept = default; #endif -extern template struct DerivedPathMap>::ChildNode; -extern template struct DerivedPathMap>; +extern template struct DerivedPathMap::ChildNode; +extern template struct DerivedPathMap; } diff --git a/src/libstore/include/nix/store/derived-path.hh b/src/libstore/include/nix/store/derived-path.hh index 2cf06c9b7..64189bd41 100644 --- a/src/libstore/include/nix/store/derived-path.hh +++ b/src/libstore/include/nix/store/derived-path.hh @@ -45,7 +45,7 @@ struct SingleDerivedPath; * path of the given output name. */ struct SingleDerivedPathBuilt { - ref drvPath; + ref drvPath; OutputName output; /** @@ -74,7 +74,7 @@ struct SingleDerivedPathBuilt { * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPathBuilt parse( - const StoreDirConfig & store, ref drvPath, + const StoreDirConfig & store, ref drvPath, OutputNameView outputs, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -172,7 +172,7 @@ static inline ref makeConstantStorePathRef(StorePath drvPath) * output name. */ struct DerivedPathBuilt { - ref drvPath; + ref drvPath; OutputsSpec outputs; /** @@ -201,7 +201,7 @@ struct DerivedPathBuilt { * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPathBuilt parse( - const StoreDirConfig & store, ref, + const StoreDirConfig & store, ref, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index f87f68e7f..10c3ec7ef 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -30,7 +30,7 @@ struct FileTransferSettings : Config {"binary-caches-parallel-connections"}}; Setting connectTimeout{ - this, 0, "connect-timeout", + this, 5, "connect-timeout", R"( The timeout (in seconds) for establishing connections in the binary cache substituter. It corresponds to `curl`’s @@ -58,6 +58,8 @@ struct FileTransferSettings : Config extern FileTransferSettings fileTransferSettings; +extern const unsigned int RETRY_TIME_MS_DEFAULT; + struct FileTransferRequest { std::string uri; @@ -67,7 +69,7 @@ struct FileTransferRequest bool head = false; bool post = false; size_t tries = fileTransferSettings.tries; - unsigned int baseRetryTimeMs = 250; + unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT; ActivityId parentAct; bool decompress = true; std::optional data; diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index 10a7f7ca7..ee7e9e623 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -24,7 +24,7 @@ struct MaxBuildJobsSetting : public BaseSetting unsigned int def, const std::string & name, const std::string & description, - const std::set & aliases = {}) + const StringSet & aliases = {}) : BaseSetting(def, true, name, description, aliases) { options->addSetting(this); diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index aaec3116d..66ec5f8d2 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -2,29 +2,27 @@ namespace nix { -struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, + virtual Store::Config, + BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view _cacheUri, const Params & params); + HttpBinaryCacheStoreConfig( + std::string_view scheme, std::string_view cacheUri, const Store::Config::Params & params); Path cacheUri; - const std::string name() override + static const std::string name() { return "HTTP Binary Cache Store"; } - static std::set uriSchemes() - { - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - auto ret = std::set({"http", "https"}); - if (forceHttp) - ret.insert("file"); - return ret; - } + static StringSet uriSchemes(); - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; } diff --git a/src/libstore/include/nix/store/legacy-ssh-store.hh b/src/libstore/include/nix/store/legacy-ssh-store.hh index a1fbf3f1e..65f29d649 100644 --- a/src/libstore/include/nix/store/legacy-ssh-store.hh +++ b/src/libstore/include/nix/store/legacy-ssh-store.hh @@ -10,7 +10,7 @@ namespace nix { -struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig +struct LegacySSHStoreConfig : std::enable_shared_from_this, virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; @@ -19,6 +19,15 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig std::string_view authority, const Params & params); +#ifndef _WIN32 + // Hack for getting remote build log output. + // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in + // the documentation + const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; +#else + Descriptor logFD = INVALID_DESCRIPTOR; +#endif + const Setting remoteProgram{this, {"nix-store"}, "remote-program", "Path to the `nix-store` executable on the remote machine."}; @@ -35,23 +44,20 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig */ std::optional connPipeSize; - const std::string name() override { return "SSH Store"; } + static const std::string name() { return "SSH Store"; } - static std::set uriSchemes() { return {"ssh"}; } + static StringSet uriSchemes() { return {"ssh"}; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; -struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +struct LegacySSHStore : public virtual Store { -#ifndef _WIN32 - // Hack for getting remote build log output. - // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; -#else - Descriptor logFD = INVALID_DESCRIPTOR; -#endif + using Config = LegacySSHStoreConfig; + + ref config; struct Connection; @@ -59,10 +65,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor SSHMaster master; - LegacySSHStore( - std::string_view scheme, - std::string_view host, - const Params & params); + LegacySSHStore(ref); ref openConnection(); @@ -187,10 +190,7 @@ public: * The legacy ssh protocol doesn't support checking for trusted-user. * Try using ssh-ng:// instead if you want to know. */ - std::optional isTrustedClient() override - { - return std::nullopt; - } + std::optional isTrustedClient() override; void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override diff --git a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh index 664841aae..a83635aa4 100644 --- a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh +++ b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh @@ -52,8 +52,10 @@ struct LengthPrefixedProtoHelper; template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); -template -LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); +#define COMMA_ , +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); +#undef COMMA_ template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); @@ -86,12 +88,11 @@ LengthPrefixedProtoHelper>::write( } } -template -std::set -LengthPrefixedProtoHelper>::read( +template +std::set LengthPrefixedProtoHelper>::read( const StoreDirConfig & store, typename Inner::ReadConn conn) { - std::set resSet; + std::set resSet; auto size = readNum(conn.from); while (size--) { resSet.insert(S::read(store, conn)); @@ -99,10 +100,9 @@ LengthPrefixedProtoHelper>::read( return resSet; } -template -void -LengthPrefixedProtoHelper>::write( - const StoreDirConfig & store, typename Inner::WriteConn conn, const std::set & resSet) +template +void LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::set & resSet) { conn.to << resSet.size(); for (auto & key : resSet) { diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index dde4701da..780eaf480 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -2,22 +2,30 @@ namespace nix { -struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this, + virtual Store::Config, + BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + /** + * @param binaryCacheDir `file://` is a short-hand for `file:///` + * for now. + */ LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); Path binaryCacheDir; - const std::string name() override + static const std::string name() { return "Local Binary Cache Store"; } - static std::set uriSchemes(); + static StringSet uriSchemes(); - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; } diff --git a/src/libstore/include/nix/store/local-fs-store.hh b/src/libstore/include/nix/store/local-fs-store.hh index 6d5afcb08..f9421b7fe 100644 --- a/src/libstore/include/nix/store/local-fs-store.hh +++ b/src/libstore/include/nix/store/local-fs-store.hh @@ -20,36 +20,39 @@ struct LocalFSStoreConfig : virtual StoreConfig */ LocalFSStoreConfig(PathView path, const Params & params); - const OptionalPathSetting rootDir{this, std::nullopt, + OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; - const PathSetting stateDir{this, + PathSetting stateDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, "state", "Directory where Nix will store state."}; - const PathSetting logDir{this, + PathSetting logDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store log files."}; - const PathSetting realStoreDir{this, + PathSetting realStoreDir{this, rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; }; -class LocalFSStore : public virtual LocalFSStoreConfig, - public virtual Store, - public virtual GcStore, - public virtual LogStore +struct LocalFSStore : + virtual Store, + virtual GcStore, + virtual LogStore { -public: + using Config = LocalFSStoreConfig; + + const Config & config; + inline static std::string operationName = "Local Filesystem Store"; const static std::string drvsLogDir; - LocalFSStore(const Params & params); + LocalFSStore(const Config & params); void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor(bool requireValidPath = true) override; @@ -70,7 +73,7 @@ public: */ virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0; - virtual Path getRealStoreDir() { return realStoreDir; } + virtual Path getRealStoreDir() { return config.realStoreDir; } Path toRealPath(const Path & storePath) override { diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index 825214cb6..6077d9e53 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -56,19 +56,21 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig The store directory is passed as an argument to the invoked executable. )"}; - const std::string name() override { return "Experimental Local Overlay Store"; } + static const std::string name() { return "Experimental Local Overlay Store"; } - std::optional experimentalFeature() const override + static std::optional experimentalFeature() { return ExperimentalFeature::LocalOverlayStore; } - static std::set uriSchemes() + static StringSet uriSchemes() { return { "local-overlay" }; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; protected: /** @@ -79,7 +81,9 @@ protected: * at that file path. It might be stored in the lower layer instead, * or it might not be part of this store at all. */ - Path toUpperPath(const StorePath & path); + Path toUpperPath(const StorePath & path) const; + + friend struct LocalOverlayStore; }; /** @@ -88,8 +92,20 @@ protected: * Documentation on overridden methods states how they differ from their * `LocalStore` counterparts. */ -class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore +struct LocalOverlayStore : virtual LocalStore { + using Config = LocalOverlayStoreConfig; + + ref config; + + LocalOverlayStore(ref); + + std::string getUri() override + { + return "local-overlay://"; + } + +private: /** * The store beneath us. * @@ -99,20 +115,6 @@ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual */ ref lowerStore; -public: - LocalOverlayStore(const Params & params) - : LocalOverlayStore("local-overlay", "", params) - { - } - - LocalOverlayStore(std::string_view scheme, PathView path, const Params & params); - - std::string getUri() override - { - return "local-overlay://"; - } - -private: /** * First copy up any lower store realisation with the same key, so we * merge rather than mask it. diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 3691fb4b6..efc59dc8c 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -34,7 +34,7 @@ struct OptimiseStats uint64_t bytesFreed = 0; }; -struct LocalStoreConfig : virtual LocalFSStoreConfig +struct LocalStoreConfig : std::enable_shared_from_this, virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; @@ -65,18 +65,26 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. )"}; - const std::string name() override { return "Local Store"; } + static const std::string name() { return "Local Store"; } - static std::set uriSchemes() + static StringSet uriSchemes() { return {"local"}; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; -class LocalStore : public virtual LocalStoreConfig - , public virtual IndirectRootStore - , public virtual GcStore +class LocalStore : + public virtual IndirectRootStore, + public virtual GcStore { +public: + + using Config = LocalStoreConfig; + + ref config; + private: /** @@ -144,11 +152,7 @@ public: * Initialise the local store, upgrading the schema if * necessary. */ - LocalStore(const Params & params); - LocalStore( - std::string_view scheme, - PathView path, - const Params & params); + LocalStore(ref params); ~LocalStore(); @@ -396,16 +400,8 @@ private: bool isValidPath_(State & state, const StorePath & path); void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); - /** - * Add signatures to a ValidPathInfo or Realisation using the secret keys - * specified by the ‘secret-key-files’ option. - */ - void signPathInfo(ValidPathInfo & info); - void signRealisation(Realisation &); - void addBuildLog(const StorePath & drvPath, std::string_view log) override; - friend struct LocalDerivationGoal; friend struct PathSubstitutionGoal; friend struct SubstitutionGoal; friend struct DerivationGoal; diff --git a/src/libstore/include/nix/store/machines.hh b/src/libstore/include/nix/store/machines.hh index f07d6b63b..2bf7408f6 100644 --- a/src/libstore/include/nix/store/machines.hh +++ b/src/libstore/include/nix/store/machines.hh @@ -15,12 +15,12 @@ typedef std::vector Machines; struct Machine { const StoreReference storeUri; - const std::set systemTypes; + const StringSet systemTypes; const std::string sshKey; const unsigned int maxJobs; const float speedFactor; - const std::set supportedFeatures; - const std::set mandatoryFeatures; + const StringSet supportedFeatures; + const StringSet mandatoryFeatures; const std::string sshPublicHostKey; bool enabled = true; @@ -34,12 +34,12 @@ struct Machine { * @return Whether `features` is a subset of the union of `supportedFeatures` and * `mandatoryFeatures`. */ - bool allSupported(const std::set & features) const; + bool allSupported(const StringSet & features) const; /** * @return Whether `mandatoryFeatures` is a subset of `features`. */ - bool mandatoryMet(const std::set & features) const; + bool mandatoryMet(const StringSet & features) const; Machine( const std::string & storeUri, @@ -75,7 +75,7 @@ struct Machine { * with `@` are interpreted as paths to other configuration files in * the same format. */ - static Machines parseConfig(const std::set & defaultSystems, const std::string & config); + static Machines parseConfig(const StringSet & defaultSystems, const std::string & config); }; /** diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index 312fd5e87..c5aa9b461 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -13,6 +13,7 @@ headers = [config_pub_h] + files( 'binary-cache-store.hh', 'build-result.hh', 'build/derivation-goal.hh', + 'build/derivation-building-misc.hh', 'build/drv-output-substitution-goal.hh', 'build/goal.hh', 'build/substitution-goal.hh', @@ -62,18 +63,21 @@ headers = [config_pub_h] + files( 'remote-fs-accessor.hh', 'remote-store-connection.hh', 'remote-store.hh', + 'restricted-store.hh', 's3-binary-cache-store.hh', 's3.hh', - 'ssh-store.hh', 'serve-protocol-connection.hh', 'serve-protocol-impl.hh', 'serve-protocol.hh', 'sqlite.hh', + 'ssh-store.hh', 'ssh.hh', 'store-api.hh', 'store-cast.hh', 'store-dir-config.hh', + 'store-open.hh', 'store-reference.hh', + 'store-registration.hh', 'uds-remote-store.hh', 'worker-protocol-connection.hh', 'worker-protocol-impl.hh', diff --git a/src/libstore/include/nix/store/outputs-spec.hh b/src/libstore/include/nix/store/outputs-spec.hh index b89f425c2..b47f26542 100644 --- a/src/libstore/include/nix/store/outputs-spec.hh +++ b/src/libstore/include/nix/store/outputs-spec.hh @@ -27,20 +27,24 @@ struct OutputsSpec { /** * A non-empty set of outputs, specified by name */ - struct Names : std::set { - using std::set::set; + struct Names : std::set> { + private: + using BaseType = std::set>; + + public: + using BaseType::BaseType; /* These need to be "inherited manually" */ - Names(const std::set & s) - : std::set(s) + Names(const BaseType & s) + : BaseType(s) { assert(!empty()); } /** * Needs to be "inherited manually" */ - Names(std::set && s) - : std::set(s) + Names(BaseType && s) + : BaseType(std::move(s)) { assert(!empty()); } /* This set should always be non-empty, so we delete this diff --git a/src/libstore/include/nix/store/parsed-derivations.hh b/src/libstore/include/nix/store/parsed-derivations.hh index d65db6133..a7c053a8f 100644 --- a/src/libstore/include/nix/store/parsed-derivations.hh +++ b/src/libstore/include/nix/store/parsed-derivations.hh @@ -1,51 +1,43 @@ #pragma once ///@file -#include "nix/store/derivations.hh" -#include "nix/store/store-api.hh" +#include -#include +#include "nix/util/types.hh" +#include "nix/store/path.hh" namespace nix { +class Store; struct DerivationOptions; +struct DerivationOutput; -class ParsedDerivation +typedef std::map DerivationOutputs; + +struct StructuredAttrs { - StorePath drvPath; - BasicDerivation & drv; - std::unique_ptr structuredAttrs; + nlohmann::json structuredAttrs; - std::optional getStringAttr(const std::string & name) const; + static std::optional tryParse(const StringPairs & env); - bool getBoolAttr(const std::string & name, bool def = false) const; - - std::optional getStringsAttr(const std::string & name) const; - - std::optional getStringSetAttr(const std::string & name) const; + nlohmann::json prepareStructuredAttrs( + Store & store, + const DerivationOptions & drvOptions, + const StorePathSet & inputPaths, + const DerivationOutputs & outputs) const; /** - * Only `DerivationOptions` is allowed to parse individual fields - * from `ParsedDerivation`. This ensure that it includes all - * derivation options, and, the likes of `LocalDerivationGoal` are - * incapable of more ad-hoc options. + * As a convenience to bash scripts, write a shell file that + * maps all attributes that are representable in bash - + * namely, strings, integers, nulls, Booleans, and arrays and + * objects consisting entirely of those values. (So nested + * arrays or objects are not supported.) + * + * @param prepared This should be the result of + * `prepareStructuredAttrs`, *not* the original `structuredAttrs` + * field. */ - friend struct DerivationOptions; - -public: - - ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv); - - ~ParsedDerivation(); - - bool hasStructuredAttrs() const - { - return static_cast(structuredAttrs); - } - - std::optional prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths); + static std::string writeShell(const nlohmann::json & prepared); }; -std::string writeStructuredAttrsShell(const nlohmann::json & json); - } diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index 9bd493422..4691bfa95 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -144,6 +144,7 @@ struct ValidPathInfo : UnkeyedValidPathInfo { std::string fingerprint(const Store & store) const; void sign(const Store & store, const Signer & signer); + void sign(const Store & store, const std::vector> & signers); /** * @return The `ContentAddressWithReferences` that determines the diff --git a/src/libstore/include/nix/store/path-with-outputs.hh b/src/libstore/include/nix/store/path-with-outputs.hh index 76c1f9f8f..368667c47 100644 --- a/src/libstore/include/nix/store/path-with-outputs.hh +++ b/src/libstore/include/nix/store/path-with-outputs.hh @@ -19,7 +19,7 @@ struct StoreDirConfig; struct StorePathWithOutputs { StorePath path; - std::set outputs; + StringSet outputs; std::string to_string(const StoreDirConfig & store) const; diff --git a/src/libstore/include/nix/store/profiles.hh b/src/libstore/include/nix/store/profiles.hh index 804c6e2b7..e20e1198e 100644 --- a/src/libstore/include/nix/store/profiles.hh +++ b/src/libstore/include/nix/store/profiles.hh @@ -86,7 +86,7 @@ typedef std::list Generations; */ std::pair> findGenerations(Path profile); -class LocalFSStore; +struct LocalFSStore; /** * Create a new generation of the given profile diff --git a/src/libstore/include/nix/store/remote-fs-accessor.hh b/src/libstore/include/nix/store/remote-fs-accessor.hh index 75a840fb0..75bb40dfb 100644 --- a/src/libstore/include/nix/store/remote-fs-accessor.hh +++ b/src/libstore/include/nix/store/remote-fs-accessor.hh @@ -19,7 +19,7 @@ class RemoteFSAccessor : public SourceAccessor std::pair, CanonPath> fetch(const CanonPath & path); - friend class BinaryCacheStore; + friend struct BinaryCacheStore; Path makeCacheFile(std::string_view hashPart, const std::string & ext); diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index ecf18bd76..dd2396fe3 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -35,14 +35,16 @@ struct RemoteStoreConfig : virtual StoreConfig * \todo RemoteStore is a misnomer - should be something like * DaemonStore. */ -class RemoteStore : public virtual RemoteStoreConfig, +struct RemoteStore : public virtual Store, public virtual GcStore, public virtual LogStore { -public: + using Config = RemoteStoreConfig; - RemoteStore(const Params & params); + const Config & config; + + RemoteStore(const Config & config); /* Implementations of abstract store API methods. */ diff --git a/src/libstore/include/nix/store/restricted-store.hh b/src/libstore/include/nix/store/restricted-store.hh new file mode 100644 index 000000000..6f2122c7b --- /dev/null +++ b/src/libstore/include/nix/store/restricted-store.hh @@ -0,0 +1,60 @@ +#pragma once +///@file + +#include "nix/store/local-store.hh" + +namespace nix { + +/** + * A restricted store has a pointer to one of these, which manages the + * restrictions that are in place. + * + * This is a separate data type so the whitelists can be mutated before + * the restricted store is created: put differently, someones we don't + * know whether we will in fact create a restricted store, but we need + * to prepare the whitelists just in case. + * + * It is possible there are other ways to solve this problem. This was + * just the easiest place to begin, when this was extracted from + * `LocalDerivationGoal`. + */ +struct RestrictionContext +{ + /** + * Paths that are already allowed to begin with + */ + virtual const StorePathSet & originalPaths() = 0; + + /** + * Paths that were added via recursive Nix calls. + */ + StorePathSet addedPaths; + + /** + * Realisations that were added via recursive Nix calls. + */ + std::set addedDrvOutputs; + + /** + * Recursive Nix calls are only allowed to build or realize paths + * in the original input closure or added via a recursive Nix call + * (so e.g. you can't do 'nix-store -r /nix/store/' where + * /nix/store/ is some arbitrary path in a binary cache). + */ + virtual bool isAllowed(const StorePath &) = 0; + virtual bool isAllowed(const DrvOutput & id) = 0; + bool isAllowed(const DerivedPath & id); + + /** + * Add 'path' to the set of paths that may be referenced by the + * outputs, and make it appear in the sandbox. + */ + virtual void addDependency(const StorePath & path) = 0; +}; + +/** + * Create a shared pointer to a restricted store. + */ +ref makeRestrictedStore(ref config, ref next, RestrictionContext & context); + +} diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index 7bc04aa4a..9a123602e 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -11,7 +11,7 @@ namespace nix { -struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +struct S3BinaryCacheStoreConfig : std::enable_shared_from_this, virtual BinaryCacheStoreConfig { std::string bucketName; @@ -93,26 +93,28 @@ public: const Setting bufferSize{ this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; - const std::string name() override + static const std::string name() { return "S3 Binary Cache Store"; } - static std::set uriSchemes() + static StringSet uriSchemes() { return {"s3"}; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; -class S3BinaryCacheStore : public virtual BinaryCacheStore +struct S3BinaryCacheStore : virtual BinaryCacheStore { -protected: + using Config = S3BinaryCacheStoreConfig; - S3BinaryCacheStore(const Params & params); + ref config; -public: + S3BinaryCacheStore(ref); struct Stats { diff --git a/src/libstore/include/nix/store/serve-protocol-impl.hh b/src/libstore/include/nix/store/serve-protocol-impl.hh index 769b9ae2b..4ab164721 100644 --- a/src/libstore/include/nix/store/serve-protocol-impl.hh +++ b/src/libstore/include/nix/store/serve-protocol-impl.hh @@ -26,7 +26,9 @@ namespace nix { } SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) -SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +#define COMMA_ , +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +#undef COMMA_ SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA , diff --git a/src/libstore/include/nix/store/serve-protocol.hh b/src/libstore/include/nix/store/serve-protocol.hh index 76f0ecd49..6f6bf6b60 100644 --- a/src/libstore/include/nix/store/serve-protocol.hh +++ b/src/libstore/include/nix/store/serve-protocol.hh @@ -180,12 +180,12 @@ DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); template DECLARE_SERVE_SERIALISER(std::vector); -template -DECLARE_SERVE_SERIALISER(std::set); +#define COMMA_ , +template +DECLARE_SERVE_SERIALISER(std::set); template DECLARE_SERVE_SERIALISER(std::tuple); -#define COMMA_ , template DECLARE_SERVE_SERIALISER(std::map); #undef COMMA_ diff --git a/src/libstore/include/nix/store/ssh-store.hh b/src/libstore/include/nix/store/ssh-store.hh index 76e8e33a4..fde165445 100644 --- a/src/libstore/include/nix/store/ssh-store.hh +++ b/src/libstore/include/nix/store/ssh-store.hh @@ -8,7 +8,9 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +struct SSHStoreConfig : std::enable_shared_from_this, + virtual RemoteStoreConfig, + virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; using RemoteStoreConfig::RemoteStoreConfig; @@ -18,44 +20,44 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig const Setting remoteProgram{ this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; - const std::string name() override + static const std::string name() { return "Experimental SSH Store"; } - static std::set uriSchemes() + static StringSet uriSchemes() { return {"ssh-ng"}; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig { - using LocalFSStoreConfig::LocalFSStoreConfig; - using SSHStoreConfig::SSHStoreConfig; - MountedSSHStoreConfig(StringMap params); - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); - const std::string name() override + static const std::string name() { return "Experimental SSH Store with filesystem mounted"; } - static std::set uriSchemes() + static StringSet uriSchemes() { return {"mounted-ssh-ng"}; } - std::string doc() override; + static std::string doc(); - std::optional experimentalFeature() const override + static std::optional experimentalFeature() { return ExperimentalFeature::MountedSSHStore; } + + ref openStore() const override; }; } diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index cee1dba6e..1648b13c1 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -26,32 +26,6 @@ namespace nix { -/** - * About the class hierarchy of the store types: - * - * Each store type `Foo` consists of two classes: - * - * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration - * for the store - * - * It should only contain members of type `const Setting` (or subclasses - * of it) and inherit the constructors of `StoreConfig` - * (`using StoreConfig::StoreConfig`). - * - * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the - * implementation of the store. - * - * This class is expected to have a constructor `Foo(const Params & params)` - * that calls `StoreConfig(params)` (otherwise you're gonna encounter an - * `assertion failure` when trying to instantiate it). - * - * You can then register the new store using: - * - * ``` - * cpp static RegisterStoreImplementation regStore; - * ``` - */ - MakeError(SubstError, Error); /** * denotes a permanent build failure @@ -97,27 +71,48 @@ struct KeyedBuildResult; typedef std::map> StorePathCAMap; + +/** + * About the class hierarchy of the store types: + * + * Each store type `Foo` consists of two classes: + * + * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration + * for the store + * + * It should only contain members of type `Setting` (or subclasses + * of it) and inherit the constructors of `StoreConfig` + * (`using StoreConfig::StoreConfig`). + * + * 2. A class `Foo : virtual Store` that contains the + * implementation of the store. + * + * This class is expected to have: + * + * 1. an alias `using Config = FooConfig;` + * + * 2. a constructor `Foo(ref params)`. + * + * You can then register the new store using: + * + * ``` + * cpp static RegisterStoreImplementation regStore; + * ``` + */ struct StoreConfig : public StoreDirConfig { - using Params = StoreReference::Params; - using StoreDirConfig::StoreDirConfig; StoreConfig() = delete; - static StringSet getDefaultSystemFeatures(); - virtual ~StoreConfig() { } - /** - * The name of this type of store. - */ - virtual const std::string name() = 0; + static StringSet getDefaultSystemFeatures(); /** * Documentation for this type of store. */ - virtual std::string doc() + static std::string doc() { return ""; } @@ -126,15 +121,15 @@ struct StoreConfig : public StoreDirConfig * An experimental feature this type store is gated, if it is to be * experimental. */ - virtual std::optional experimentalFeature() const + static std::optional experimentalFeature() { return std::nullopt; } - const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", + Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "Size of the in-memory store path metadata cache."}; - const Setting isTrusted{this, false, "trusted", + Setting isTrusted{this, false, "trusted", R"( Whether paths from this store can be used as substitutes even if they are not signed by a key listed in the @@ -163,10 +158,38 @@ struct StoreConfig : public StoreDirConfig {}, // Don't document the machine-specific default value false}; + + /** + * Open a store of the type corresponding to this configuration + * type. + */ + virtual ref openStore() const = 0; }; -class Store : public std::enable_shared_from_this, public virtual StoreConfig +/** + * A Store (client) + * + * This is an interface type allowing for create and read operations on + * a collection of store objects, and also building new store objects + * from `Derivation`s. See the manual for further details. + * + * "client" used is because this is just one view/actor onto an + * underlying resource, which could be an external process (daemon + * server), file system state, etc. + */ +class Store : public std::enable_shared_from_this, public MixStoreDirMethods { +public: + + using Config = StoreConfig; + + const Config & config; + + /** + * @note Avoid churn, since we used to inherit from `Config`. + */ + operator const Config &() const { return config; } + protected: struct PathInfoCacheValue { @@ -205,7 +228,7 @@ protected: std::shared_ptr diskCache; - Store(const Params & params); + Store(const Store::Config & config); public: /** @@ -622,6 +645,14 @@ public: virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) { unsupported("addSignatures"); } + /** + * Add signatures to a ValidPathInfo or Realisation using the secret keys + * specified by the ‘secret-key-files’ option. + */ + void signPathInfo(ValidPathInfo & info); + + void signRealisation(Realisation &); + /* Utility functions. */ /** @@ -857,74 +888,6 @@ StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalSto OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); -/** - * @return a Store object to access the Nix store denoted by - * ‘uri’ (slight misnomer...). - */ -ref openStore(StoreReference && storeURI); - - -/** - * Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse` - - */ -ref openStore(const std::string & uri = settings.storeUri.get(), - const Store::Params & extraParams = Store::Params()); - - -/** - * @return the default substituter stores, defined by the - * ‘substituters’ option and various legacy options. - */ -std::list> getDefaultSubstituters(); - -struct StoreFactory -{ - std::set uriSchemes; - /** - * The `authorityPath` parameter is `/`, or really - * whatever comes after `://` and before `?`. - */ - std::function ( - std::string_view scheme, - std::string_view authorityPath, - const Store::Params & params)> create; - std::function ()> getConfig; -}; - -struct Implementations -{ - static std::vector * registered; - - template - static void add() - { - if (!registered) registered = new std::vector(); - StoreFactory factory{ - .uriSchemes = TConfig::uriSchemes(), - .create = - ([](auto scheme, auto uri, auto & params) - -> std::shared_ptr - { return std::make_shared(scheme, uri, params); }), - .getConfig = - ([]() - -> std::shared_ptr - { return std::make_shared(StringMap({})); }) - }; - registered->push_back(factory); - } -}; - -template -struct RegisterStoreImplementation -{ - RegisterStoreImplementation() - { - Implementations::add(); - } -}; - - /** * Display a set of paths in human-readable form (i.e., between quotes * and separated by commas). diff --git a/src/libstore/include/nix/store/store-dir-config.hh b/src/libstore/include/nix/store/store-dir-config.hh index 845a003f5..6bf9ebf14 100644 --- a/src/libstore/include/nix/store/store-dir-config.hh +++ b/src/libstore/include/nix/store/store-dir-config.hh @@ -18,22 +18,18 @@ struct SourcePath; MakeError(BadStorePath, Error); MakeError(BadStorePathName, BadStorePath); -struct StoreDirConfig : public Config +/** + * @todo This should just be part of `StoreDirConfig`. However, it would + * be a huge amount of churn if `Store` didn't have these methods + * anymore, forcing a bunch of code to go from `store.method(...)` to + * `store.config.method(...)`. + * + * So we instead pull out the methods into their own mix-in, so can put + * them directly on the Store too. + */ +struct MixStoreDirMethods { - using Config::Config; - - StoreDirConfig() = delete; - - virtual ~StoreDirConfig() = default; - - const PathSetting storeDir_{this, settings.nixStore, - "store", - R"( - Logical location of the Nix store, usually - `/nix/store`. Note that you can only copy store paths - between stores if they have the same `store` setting. - )"}; - const Path storeDir = storeDir_; + const Path & storeDir; // pure methods @@ -56,7 +52,7 @@ struct StoreDirConfig : public Config * Display a set of paths in human-readable form (i.e., between quotes * and separated by commas). */ - std::string showPaths(const StorePathSet & paths); + std::string showPaths(const StorePathSet & paths) const; /** * @return true if *path* is in the Nix store (but not the Nix @@ -104,4 +100,38 @@ struct StoreDirConfig : public Config PathFilter & filter = defaultPathFilter) const; }; +/** + * Need to make this a separate class so I can get the right + * initialization order in the constructor for `StoreDirConfig`. + */ +struct StoreDirConfigBase : Config +{ + using Config::Config; + + const PathSetting storeDir_{this, settings.nixStore, + "store", + R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )"}; +}; + +/** + * The order of `StoreDirConfigBase` and then `MixStoreDirMethods` is + * very important. This ensures that `StoreDirConfigBase::storeDir_` + * is initialized before we have our one chance (because references are + * immutable) to initialize `MixStoreDirMethods::storeDir`. + */ +struct StoreDirConfig : StoreDirConfigBase, MixStoreDirMethods +{ + using Params = std::map; + + StoreDirConfig(const Params & params); + + StoreDirConfig() = delete; + + virtual ~StoreDirConfig() = default; +}; + } diff --git a/src/libstore/include/nix/store/store-open.hh b/src/libstore/include/nix/store/store-open.hh new file mode 100644 index 000000000..7c1cda5be --- /dev/null +++ b/src/libstore/include/nix/store/store-open.hh @@ -0,0 +1,43 @@ +#pragma once +/** + * @file + * + * For opening a store described by an `StoreReference`, which is an "untyped" + * notion which needs to be decoded against a collection of specific + * implementations. + * + * For consumers of the store registration machinery defined in + * `store-registration.hh`. Not needed by store implementation definitions, or + * usages of a given `Store` which will be passed in. + */ + +#include "nix/store/store-api.hh" + +namespace nix { + +/** + * @return The store config denoted by `storeURI` (slight misnomer...). + */ +ref resolveStoreConfig(StoreReference && storeURI); + +/** + * @return a Store object to access the Nix store denoted by + * ‘uri’ (slight misnomer...). + */ +ref openStore(StoreReference && storeURI); + +/** + * Opens the store at `uri`, where `uri` is in the format expected by + * `StoreReference::parse` + */ +ref openStore( + const std::string & uri = settings.storeUri.get(), + const StoreReference::Params & extraParams = StoreReference::Params()); + +/** + * @return the default substituter stores, defined by the + * ‘substituters’ option and various legacy options. + */ +std::list> getDefaultSubstituters(); + +} diff --git a/src/libstore/include/nix/store/store-registration.hh b/src/libstore/include/nix/store/store-registration.hh new file mode 100644 index 000000000..3f82ff51c --- /dev/null +++ b/src/libstore/include/nix/store/store-registration.hh @@ -0,0 +1,88 @@ +#pragma once +/** + * @file + * + * Infrastructure for "registering" store implementations. Used by the + * store implementation definitions themselves but not by consumers of + * those implementations. + * + * Consumers of an arbitrary store from a URL/JSON configuration instead + * just need the defintions `nix/store/store-open.hh`; those do use this + * but only as an implementation. Consumers of a specific extra type of + * store can skip both these, and just use the definition of the store + * in question directly. + */ + +#include "nix/store/store-api.hh" + +namespace nix { + +struct StoreFactory +{ + /** + * Documentation for this type of store. + */ + std::string doc; + + /** + * URIs with these schemes should be handled by this factory + */ + StringSet uriSchemes; + + /** + * An experimental feature this type store is gated, if it is to be + * experimental. + */ + std::optional experimentalFeature; + + /** + * The `authorityPath` parameter is `/`, or really + * whatever comes after `://` and before `?`. + */ + std::function( + std::string_view scheme, std::string_view authorityPath, const Store::Config::Params & params)> + parseConfig; + + /** + * Just for dumping the defaults. Kind of awkward this exists, + * because it means we cannot require fields to be manually + * specified so easily. + */ + std::function()> getConfig; +}; + +struct Implementations +{ + using Map = std::map; + + static Map & registered(); + + template + static void add() + { + StoreFactory factory{ + .doc = TConfig::doc(), + .uriSchemes = TConfig::uriSchemes(), + .experimentalFeature = TConfig::experimentalFeature(), + .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { + return make_ref(scheme, uri, params); + }), + .getConfig = ([]() -> ref { return make_ref(Store::Config::Params{}); }), + }; + auto [it, didInsert] = registered().insert({TConfig::name(), std::move(factory)}); + if (!didInsert) { + throw Error("Already registred store with name '%s'", it->first); + } + } +}; + +template +struct RegisterStoreImplementation +{ + RegisterStoreImplementation() + { + Implementations::add(); + } +}; + +} diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index f7ef76058..e9793a9ee 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -7,7 +7,10 @@ namespace nix { -struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig +struct UDSRemoteStoreConfig : + std::enable_shared_from_this, + virtual LocalFSStoreConfig, + virtual RemoteStoreConfig { // TODO(fzakaria): Delete this constructor once moved over to the factory pattern // outlined in https://github.com/NixOS/nix/issues/10766 @@ -22,9 +25,11 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon std::string_view authority, const Params & params); - const std::string name() override { return "Local Daemon Store"; } + UDSRemoteStoreConfig(const Params & params); - std::string doc() override; + static const std::string name() { return "Local Daemon Store"; } + + static std::string doc(); /** * The path to the unix domain socket. @@ -34,32 +39,21 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon */ Path path; -protected: - static constexpr char const * scheme = "unix"; + static StringSet uriSchemes() + { return {"unix"}; } -public: - static std::set uriSchemes() - { return {scheme}; } + ref openStore() const override; }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig - , public virtual IndirectRootStore - , public virtual RemoteStore +struct UDSRemoteStore : + virtual IndirectRootStore, + virtual RemoteStore { -public: + using Config = UDSRemoteStoreConfig; - /** - * @deprecated This is the old API to construct the store. - */ - UDSRemoteStore(const Params & params); + ref config; - /** - * @param authority is the socket path. - */ - UDSRemoteStore( - std::string_view scheme, - std::string_view authority, - const Params & params); + UDSRemoteStore(ref); std::string getUri() override; diff --git a/src/libstore/include/nix/store/worker-protocol-connection.hh b/src/libstore/include/nix/store/worker-protocol-connection.hh index df2fe0ec2..11f112a71 100644 --- a/src/libstore/include/nix/store/worker-protocol-connection.hh +++ b/src/libstore/include/nix/store/worker-protocol-connection.hh @@ -26,7 +26,7 @@ struct WorkerProto::BasicConnection /** * The set of features that both sides support. */ - std::set features; + FeatureSet features; /** * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the @@ -92,11 +92,8 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection * @param supportedFeatures The protocol features that we support. */ // FIXME: this should probably be a constructor. - static std::tuple> handshake( - BufferedSink & to, - Source & from, - WorkerProto::Version localVersion, - const std::set & supportedFeatures); + static std::tuple handshake( + BufferedSink & to, Source & from, WorkerProto::Version localVersion, const FeatureSet & supportedFeatures); /** * After calling handshake, must call this to exchange some basic @@ -155,11 +152,8 @@ struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection * @param supportedFeatures The protocol features that we support. */ // FIXME: this should probably be a constructor. - static std::tuple> handshake( - BufferedSink & to, - Source & from, - WorkerProto::Version localVersion, - const std::set & supportedFeatures); + static std::tuple handshake( + BufferedSink & to, Source & from, WorkerProto::Version localVersion, const FeatureSet & supportedFeatures); /** * After calling handshake, must call this to exchange some basic diff --git a/src/libstore/include/nix/store/worker-protocol-impl.hh b/src/libstore/include/nix/store/worker-protocol-impl.hh index 337c245e2..908a9323e 100644 --- a/src/libstore/include/nix/store/worker-protocol-impl.hh +++ b/src/libstore/include/nix/store/worker-protocol-impl.hh @@ -26,7 +26,9 @@ namespace nix { } WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) -WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +#define COMMA_ , +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +#undef COMMA_ WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA , diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 3060681b8..1b188806d 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -135,8 +135,9 @@ struct WorkerProto } using Feature = std::string; + using FeatureSet = std::set>; - static const std::set allFeatures; + static const FeatureSet allFeatures; }; enum struct WorkerProto::Op : uint64_t @@ -272,12 +273,12 @@ DECLARE_WORKER_SERIALISER(WorkerProto::ClientHandshakeInfo); template DECLARE_WORKER_SERIALISER(std::vector); -template -DECLARE_WORKER_SERIALISER(std::set); +#define COMMA_ , +template +DECLARE_WORKER_SERIALISER(std::set); template DECLARE_WORKER_SERIALISER(std::tuple); -#define COMMA_ , template DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 1512a7944..9ec9e6eec 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -12,6 +12,7 @@ #include "nix/store/ssh.hh" #include "nix/store/derivations.hh" #include "nix/util/callback.hh" +#include "nix/store/store-registration.hh" namespace nix { @@ -38,23 +39,19 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; -LegacySSHStore::LegacySSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , LegacySSHStoreConfig(scheme, host, params) - , Store(params) + +LegacySSHStore::LegacySSHStore(ref config) + : Store{*config} + , config{config} , connections(make_ref>( - std::max(1, (int) maxConnections), + std::max(1, (int) config->maxConnections), [this]() { return openConnection(); }, [](const ref & r) { return r->good; } )) - , master(createSSHMaster( + , master(config->createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - logFD)) + config->logFD)) { } @@ -62,16 +59,16 @@ LegacySSHStore::LegacySSHStore( ref LegacySSHStore::openConnection() { auto conn = make_ref(); - Strings command = remoteProgram.get(); + Strings command = config->remoteProgram.get(); command.push_back("--serve"); command.push_back("--write"); - if (remoteStore.get() != "") { + if (config->remoteStore.get() != "") { command.push_back("--store"); - command.push_back(remoteStore.get()); + command.push_back(config->remoteStore.get()); } - conn->sshConn = master.startCommand(std::move(command), std::list{extraSshArgs}); - if (connPipeSize) { - conn->sshConn->trySetBufferSize(*connPipeSize); + conn->sshConn = master.startCommand(std::move(command), std::list{config->extraSshArgs}); + if (config->connPipeSize) { + conn->sshConn->trySetBufferSize(*config->connPipeSize); } conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); @@ -80,7 +77,7 @@ ref LegacySSHStore::openConnection() TeeSource tee(conn->from, saved); try { conn->remoteVersion = ServeProto::BasicClientConnection::handshake( - conn->to, tee, SERVE_PROTOCOL_VERSION, host); + conn->to, tee, SERVE_PROTOCOL_VERSION, config->host); } catch (SerialisationError & e) { // in.close(): Don't let the remote block on us not writing. conn->sshConn->in.close(); @@ -89,9 +86,9 @@ ref LegacySSHStore::openConnection() tee.drainInto(nullSink); } throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", - host, chomp(saved.s)); + config->host, chomp(saved.s)); } catch (EndOfFile & e) { - throw Error("cannot connect to '%1%'", host); + throw Error("cannot connect to '%1%'", config->host); } return conn; @@ -100,7 +97,7 @@ ref LegacySSHStore::openConnection() std::string LegacySSHStore::getUri() { - return *uriSchemes().begin() + "://" + host; + return *Config::uriSchemes().begin() + "://" + config->host; } std::map LegacySSHStore::queryPathInfosUncached( @@ -111,7 +108,7 @@ std::map LegacySSHStore::queryPathInfosUncached /* No longer support missing NAR hash */ assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); - debug("querying remote host '%s' for info on '%s'", host, concatStringsSep(", ", printStorePathSet(paths))); + debug("querying remote host '%s' for info on '%s'", config->host, concatStringsSep(", ", printStorePathSet(paths))); auto infos = conn->queryPathInfos(*this, paths); @@ -151,7 +148,7 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path, void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { - debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); + debug("adding path '%s' to remote host '%s'", printStorePath(info.path), config->host); auto conn(connections->get()); @@ -178,7 +175,7 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, conn->to.flush(); if (readInt(conn->from) != 1) - throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), config->host); } else { @@ -390,12 +387,17 @@ LegacySSHStore::ConnectionStats LegacySSHStore::getConnectionStats() * The legacy ssh protocol doesn't support checking for trusted-user. * Try using ssh-ng:// instead if you want to know. */ -std::optional isTrustedClient() +std::optional LegacySSHStore::isTrustedClient() { return std::nullopt; } -static RegisterStoreImplementation regLegacySSHStore; +ref LegacySSHStore::Config::openStore() const { + return make_ref(ref{shared_from_this()}); +} + + +static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 212eacc8c..2f23135fa 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -2,6 +2,7 @@ #include "nix/store/globals.hh" #include "nix/store/nar-info-disk-cache.hh" #include "nix/util/signals.hh" +#include "nix/store/store-registration.hh" #include @@ -10,9 +11,9 @@ namespace nix { LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( std::string_view scheme, PathView binaryCacheDir, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) + const StoreReference::Params & params) + : Store::Config{params} + , BinaryCacheStoreConfig{params} , binaryCacheDir(binaryCacheDir) { } @@ -26,29 +27,26 @@ std::string LocalBinaryCacheStoreConfig::doc() } -struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore +struct LocalBinaryCacheStore : + virtual BinaryCacheStore { - /** - * @param binaryCacheDir `file://` is a short-hand for `file:///` - * for now. - */ - LocalBinaryCacheStore( - std::string_view scheme, - PathView binaryCacheDir, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , LocalBinaryCacheStoreConfig(scheme, binaryCacheDir, params) - , Store(params) - , BinaryCacheStore(params) + using Config = LocalBinaryCacheStoreConfig; + + ref config; + + LocalBinaryCacheStore(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , config{config} { + init(); } void init() override; std::string getUri() override { - return "file://" + binaryCacheDir; + return "file://" + config->binaryCacheDir; } protected: @@ -59,7 +57,7 @@ protected: std::shared_ptr> istream, const std::string & mimeType) override { - auto path2 = binaryCacheDir + "/" + path; + auto path2 = config->binaryCacheDir + "/" + path; static std::atomic counter{0}; Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); AutoDelete del(tmp, false); @@ -72,7 +70,7 @@ protected: void getFile(const std::string & path, Sink & sink) override { try { - readFile(binaryCacheDir + "/" + path, sink); + readFile(config->binaryCacheDir + "/" + path, sink); } catch (SysError & e) { if (e.errNo == ENOENT) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); @@ -84,7 +82,7 @@ protected: { StorePathSet paths; - for (auto & entry : std::filesystem::directory_iterator{binaryCacheDir}) { + for (auto & entry : DirectoryIterator{config->binaryCacheDir}) { checkInterrupt(); auto name = entry.path().filename().string(); if (name.size() != 40 || @@ -106,20 +104,20 @@ protected: void LocalBinaryCacheStore::init() { - createDirs(binaryCacheDir + "/nar"); - createDirs(binaryCacheDir + "/" + realisationsPrefix); - if (writeDebugInfo) - createDirs(binaryCacheDir + "/debuginfo"); - createDirs(binaryCacheDir + "/log"); + createDirs(config->binaryCacheDir + "/nar"); + createDirs(config->binaryCacheDir + "/" + realisationsPrefix); + if (config->writeDebugInfo) + createDirs(config->binaryCacheDir + "/debuginfo"); + createDirs(config->binaryCacheDir + "/log"); BinaryCacheStore::init(); } bool LocalBinaryCacheStore::fileExists(const std::string & path) { - return pathExists(binaryCacheDir + "/" + path); + return pathExists(config->binaryCacheDir + "/" + path); } -std::set LocalBinaryCacheStoreConfig::uriSchemes() +StringSet LocalBinaryCacheStoreConfig::uriSchemes() { if (getEnv("_NIX_FORCE_HTTP") == "1") return {}; @@ -127,6 +125,13 @@ std::set LocalBinaryCacheStoreConfig::uriSchemes() return {"file"}; } -static RegisterStoreImplementation regLocalBinaryCacheStore; +ref LocalBinaryCacheStoreConfig::openStore() const { + return make_ref(ref{ + // FIXME we shouldn't actually need a mutable config + std::const_pointer_cast(shared_from_this()) + }); +} + +static RegisterStoreImplementation regLocalBinaryCacheStore; } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 599765ced..add3b04d2 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -22,8 +22,9 @@ LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) { } -LocalFSStore::LocalFSStore(const Params & params) - : Store(params) +LocalFSStore::LocalFSStore(const Config & config) + : Store{static_cast(*this)} + , config{config} { } @@ -33,30 +34,35 @@ struct LocalStoreAccessor : PosixSourceAccessor bool requireValidPath; LocalStoreAccessor(ref store, bool requireValidPath) - : store(store) + : PosixSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()}) + , store(store) , requireValidPath(requireValidPath) - { } - - CanonPath toRealPath(const CanonPath & path) { - auto [storePath, rest] = store->toStorePath(path.abs()); + } + + + void requireStoreObject(const CanonPath & path) + { + auto [storePath, rest] = store->toStorePath(store->storeDir + path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return CanonPath(store->getRealStoreDir()) / storePath.to_string() / CanonPath(rest); } std::optional maybeLstat(const CanonPath & path) override { - /* Handle the case where `path` is (a parent of) the store. */ - if (isDirOrInDir(store->storeDir, path.abs())) + /* Also allow `path` to point to the entire store, which is + needed for resolving symlinks. */ + if (path.isRoot()) return Stat{ .type = tDirectory }; - return PosixSourceAccessor::maybeLstat(toRealPath(path)); + requireStoreObject(path); + return PosixSourceAccessor::maybeLstat(path); } DirEntries readDirectory(const CanonPath & path) override { - return PosixSourceAccessor::readDirectory(toRealPath(path)); + requireStoreObject(path); + return PosixSourceAccessor::readDirectory(path); } void readFile( @@ -64,12 +70,14 @@ struct LocalStoreAccessor : PosixSourceAccessor Sink & sink, std::function sizeCallback) override { - return PosixSourceAccessor::readFile(toRealPath(path), sink, sizeCallback); + requireStoreObject(path); + return PosixSourceAccessor::readFile(path, sink, sizeCallback); } std::string readLink(const CanonPath & path) override { - return PosixSourceAccessor::readLink(toRealPath(path)); + requireStoreObject(path); + return PosixSourceAccessor::readLink(path); } }; @@ -97,8 +105,8 @@ std::optional LocalFSStore::getBuildLogExact(const StorePath & path Path logPath = j == 0 - ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) - : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); + ? fmt("%s/%s/%s/%s", config.logDir.get(), drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) + : fmt("%s/%s/%s", config.logDir.get(), drvsLogDir, baseName); Path logBz2Path = logPath + ".bz2"; if (pathExists(logPath)) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 38fa634ca..e40c5fa6e 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -1,9 +1,12 @@ +#include + #include "nix/store/local-overlay-store.hh" #include "nix/util/callback.hh" #include "nix/store/realisation.hh" #include "nix/util/processes.hh" #include "nix/util/url.hh" -#include +#include "nix/store/store-open.hh" +#include "nix/store/store-registration.hh" namespace nix { @@ -14,25 +17,32 @@ std::string LocalOverlayStoreConfig::doc() ; } -Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { +ref LocalOverlayStoreConfig::openStore() const +{ + return make_ref(ref{ + std::dynamic_pointer_cast(shared_from_this()) + }); +} + + +Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) const +{ return upperLayer + "/" + path.to_string(); } -LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(params) - , LocalOverlayStoreConfig(scheme, path, params) - , Store(params) - , LocalFSStore(params) - , LocalStore(params) - , lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast()) + +LocalOverlayStore::LocalOverlayStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , LocalStore{static_cast>(config)} + , config{config} + , lowerStore(openStore(percentDecode(config->lowerStoreUri.get())).dynamic_pointer_cast()) { - if (checkMount.get()) { + if (config->checkMount.get()) { std::smatch match; std::string mountInfo; auto mounts = readFile(std::filesystem::path{"/proc/self/mounts"}); - auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))"); + auto regex = std::regex(R"((^|\n)overlay )" + config->realStoreDir.get() + R"( .*(\n|$))"); // Mount points can be stacked, so there might be multiple matching entries. // Loop until the last match, which will be the current state of the mount point. @@ -45,13 +55,13 @@ LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, con return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)")); }; - auto expectedLowerDir = lowerStore->realStoreDir.get(); - if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) { + auto expectedLowerDir = lowerStore->config.realStoreDir.get(); + if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", config->upperLayer)) { debug("expected lowerdir: %s", expectedLowerDir); - debug("expected upperdir: %s", upperLayer); + debug("expected upperdir: %s", config->upperLayer); debug("actual mount: %s", mountInfo); throw Error("overlay filesystem '%s' mounted incorrectly", - realStoreDir.get()); + config->realStoreDir.get()); } } } @@ -201,14 +211,14 @@ void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & re void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) { - auto mergedDir = realStoreDir.get() + "/"; + auto mergedDir = config->realStoreDir.get() + "/"; if (path.substr(0, mergedDir.length()) != mergedDir) { warn("local-overlay: unexpected gc path '%s' ", path); return; } StorePath storePath = {path.substr(mergedDir.length())}; - auto upperPath = toUpperPath(storePath); + auto upperPath = config->toUpperPath(storePath); if (pathExists(upperPath)) { debug("upper exists: %s", path); @@ -257,7 +267,7 @@ LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag StorePathSet done; auto existsInStoreDir = [&](const StorePath & storePath) { - return pathExists(realStoreDir.get() + "/" + storePath.to_string()); + return pathExists(config->realStoreDir.get() + "/" + storePath.to_string()); }; bool errors = false; @@ -277,16 +287,16 @@ void LocalOverlayStore::remountIfNecessary() { if (!_remountRequired) return; - if (remountHook.get().empty()) { - warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get()); + if (config->remountHook.get().empty()) { + warn("'%s' needs remounting, set remount-hook to do this automatically", config->realStoreDir.get()); } else { - runProgram(remountHook, false, {realStoreDir}); + runProgram(config->remountHook, false, {config->realStoreDir}); } _remountRequired = false; } -static RegisterStoreImplementation regLocalOverlayStore; +static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md index 9434ebfb9..4ac8a476d 100644 --- a/src/libstore/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -85,7 +85,7 @@ The parts of a local overlay store are as follows: > The location of the database instead depends on the [`state`](#store-experimental-local-overlay-store-state) setting. > It is always `${state}/db`. - This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadta. + This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadata. The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects][store object] can be found entirely within the upper layer. (This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way.) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 949f0f74f..76fadba86 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -17,6 +17,8 @@ #include "nix/util/posix-source-accessor.hh" #include "nix/store/keys.hh" #include "nix/util/users.hh" +#include "nix/store/store-open.hh" +#include "nix/store/store-registration.hh" #include #include @@ -75,6 +77,11 @@ std::string LocalStoreConfig::doc() ; } +ref LocalStore::Config::openStore() const +{ + return make_ref(ref{shared_from_this()}); +} + struct LocalStore::State::Stmts { /* Some precompiled SQLite statements. */ SQLiteStmt RegisterValidPath; @@ -97,39 +104,33 @@ struct LocalStore::State::Stmts { SQLiteStmt AddRealisationReference; }; -LocalStore::LocalStore( - std::string_view scheme, - PathView path, - const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(scheme, path, params) - , Store(params) - , LocalFSStore(params) - , dbDir(stateDir + "/db") - , linksDir(realStoreDir + "/.links") +LocalStore::LocalStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , config{config} + , dbDir(config->stateDir + "/db") + , linksDir(config->realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") - , tempRootsDir(stateDir + "/temproots") + , tempRootsDir(config->stateDir + "/temproots") , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) - , locksHeld(tokenizeString(getEnv("NIX_HELD_LOCKS").value_or(""))) { auto state(_state.lock()); state->stmts = std::make_unique(); /* Create missing state directories if they don't already exist. */ - createDirs(realStoreDir.get()); - if (readOnly) { + createDirs(config->realStoreDir.get()); + if (config->readOnly) { experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); } else { makeStoreWritable(); } createDirs(linksDir); - Path profilesDir = stateDir + "/profiles"; + Path profilesDir = config->stateDir + "/profiles"; createDirs(profilesDir); createDirs(tempRootsDir); createDirs(dbDir); - Path gcRootsDir = stateDir + "/gcroots"; + Path gcRootsDir = config->stateDir + "/gcroots"; if (!pathExists(gcRootsDir)) { createDirs(gcRootsDir); createSymlink(profilesDir, gcRootsDir + "/profiles"); @@ -137,14 +138,11 @@ LocalStore::LocalStore( for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); - if (!readOnly) { - auto st = lstat(perUserDir); - + if (!config->readOnly) { // Skip chmod call if the directory already has the correct permissions (0755). // This is to avoid failing when the executing user lacks permissions to change the directory's permissions // even if it would be no-op. - if ((st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) != 0755 && chmod(perUserDir.c_str(), 0755) == -1) - throw SysError("could not set permissions on '%s' to 755", perUserDir); + chmodIfNeeded(perUserDir, 0755, S_IRWXU | S_IRWXG | S_IRWXO); } } @@ -157,16 +155,16 @@ LocalStore::LocalStore( struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); - else if (!readOnly) { + else if (!config->readOnly) { struct stat st; - if (stat(realStoreDir.get().c_str(), &st)) - throw SysError("getting attributes of path '%1%'", realStoreDir); + if (stat(config->realStoreDir.get().c_str(), &st)) + throw SysError("getting attributes of path '%1%'", config->realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { - if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) - throw SysError("changing ownership of path '%1%'", realStoreDir); - if (chmod(realStoreDir.get().c_str(), perm) == -1) - throw SysError("changing permissions on path '%1%'", realStoreDir); + if (chown(config->realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) + throw SysError("changing ownership of path '%1%'", config->realStoreDir); + if (chmod(config->realStoreDir.get().c_str(), perm) == -1) + throw SysError("changing permissions on path '%1%'", config->realStoreDir); } } } @@ -174,7 +172,7 @@ LocalStore::LocalStore( /* Ensure that the store and its parents are not symlinks. */ if (!settings.allowSymlinkedStore) { - std::filesystem::path path = realStoreDir.get(); + std::filesystem::path path = config->realStoreDir.get(); std::filesystem::path root = path.root_path(); while (path != root) { if (std::filesystem::is_symlink(path)) @@ -221,12 +219,12 @@ LocalStore::LocalStore( /* Acquire the big fat lock in shared mode to make sure that no schema upgrade is in progress. */ - if (!readOnly) { + if (!config->readOnly) { Path globalLockPath = dbDir + "/big-lock"; globalLock = openLockFile(globalLockPath.c_str(), true); } - if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) { + if (!config->readOnly && !lockFile(globalLock.get(), ltRead, false)) { printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -234,7 +232,7 @@ LocalStore::LocalStore( /* Check the current database schema and if necessary do an upgrade. */ int curSchema = getSchema(); - if (readOnly && curSchema < nixSchemaVersion) { + if (config->readOnly && curSchema < nixSchemaVersion) { debug("current schema version: %d", curSchema); debug("supported schema version: %d", nixSchemaVersion); throw Error(curSchema == 0 ? @@ -382,15 +380,9 @@ LocalStore::LocalStore( } -LocalStore::LocalStore(const Params & params) - : LocalStore("local", "", params) -{ -} - - AutoCloseFD LocalStore::openGCLock() { - Path fnGCLock = stateDir + "/gc.lock"; + Path fnGCLock = config->stateDir + "/gc.lock"; auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT #ifndef _WIN32 | O_CLOEXEC @@ -456,17 +448,17 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { - if (create && readOnly) { + if (create && config->readOnly) { throw Error("cannot create database while in read-only mode"); } - if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) + if (access(dbDir.c_str(), R_OK | (config->readOnly ? 0 : W_OK))) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - auto openMode = readOnly ? SQLiteOpenMode::Immutable + auto openMode = config->readOnly ? SQLiteOpenMode::Immutable : create ? SQLiteOpenMode::Normal : SQLiteOpenMode::NoCreate; state.db = SQLite(dbPath, openMode); @@ -539,7 +531,7 @@ void LocalStore::upgradeDBSchema(State & state) { state.db.exec("create table if not exists SchemaMigrations (migration text primary key not null);"); - std::set schemaMigrations; + StringSet schemaMigrations; { SQLiteStmt querySchemaMigrations; @@ -579,12 +571,12 @@ void LocalStore::makeStoreWritable() if (!isRootUser()) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; - if (statvfs(realStoreDir.get().c_str(), &stat) != 0) + if (statvfs(config->realStoreDir.get().c_str(), &stat) != 0) throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError("remounting %1% writable", realStoreDir); + if (mount(0, config->realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + throw SysError("remounting %1% writable", config->realStoreDir); } #endif } @@ -924,7 +916,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) for (auto & sub : getDefaultSubstituters()) { if (remaining.empty()) break; if (sub->storeDir != storeDir) continue; - if (!sub->wantMassQuery) continue; + if (!sub->config.wantMassQuery) continue; auto valid = sub->queryValidPaths(remaining); @@ -1036,12 +1028,12 @@ const PublicKeys & LocalStore::getPublicKeys() bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info) { - return requireSigs && !info.checkSignatures(*this, getPublicKeys()); + return config->requireSigs && !info.checkSignatures(*this, getPublicKeys()); } bool LocalStore::realisationIsUntrusted(const Realisation & realisation) { - return requireSigs && !realisation.checkSignatures(getPublicKeys()); + return config->requireSigs && !realisation.checkSignatures(getPublicKeys()); } void LocalStore::addToStore(const ValidPathInfo & info, Source & source, @@ -1106,7 +1098,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto & specified = *info.ca; auto actualHash = ({ auto accessor = getFSAccessor(false); - CanonPath path { printStorePath(info.path) }; + CanonPath path { info.path.to_string() }; Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ auto fim = specified.method.getFileIngestionMethod(); switch (fim) { @@ -1338,7 +1330,7 @@ std::pair LocalStore::createTempDirInStore() /* There is a slight possibility that `tmpDir' gets deleted by the GC between createTempDir() and when we acquire a lock on it. We'll repeat until 'tmpDir' exists and we've locked it. */ - tmpDirFn = createTempDir(realStoreDir, "tmp"); + tmpDirFn = createTempDir(config->realStoreDir, "tmp"); tmpDirFd = openDirectory(tmpDirFn); if (!tmpDirFd) { continue; @@ -1386,7 +1378,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking link hashes..."); - for (auto & link : std::filesystem::directory_iterator{linksDir}) { + for (auto & link : DirectoryIterator{linksDir}) { checkInterrupt(); auto name = link.path().filename(); printMsg(lvlTalkative, "checking contents of '%s'", name); @@ -1479,7 +1471,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair database and the filesystem) in the loop below, in order to catch invalid states. */ - for (auto & i : std::filesystem::directory_iterator{realStoreDir.to_string()}) { + for (auto & i : DirectoryIterator{config->realStoreDir.get()}) { checkInterrupt(); try { storePathsInStoreDir.insert({i.path().filename().string()}); @@ -1588,33 +1580,6 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si } -void LocalStore::signRealisation(Realisation & realisation) -{ - // FIXME: keep secret keys in memory. - - auto secretKeyFiles = settings.secretKeyFiles; - - for (auto & secretKeyFile : secretKeyFiles.get()) { - SecretKey secretKey(readFile(secretKeyFile)); - LocalSigner signer(std::move(secretKey)); - realisation.sign(signer); - } -} - -void LocalStore::signPathInfo(ValidPathInfo & info) -{ - // FIXME: keep secret keys in memory. - - auto secretKeyFiles = settings.secretKeyFiles; - - for (auto & secretKeyFile : secretKeyFiles.get()) { - SecretKey secretKey(readFile(secretKeyFile)); - LocalSigner signer(std::move(secretKey)); - info.sign(*this, signer); - } -} - - std::optional> LocalStore::queryRealisationCore_( LocalStore::State & state, const DrvOutput & id) @@ -1695,7 +1660,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) auto baseName = drvPath.to_string(); - auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); + auto logPath = fmt("%s/%s/%s/%s.bz2", config->logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); if (pathExists(logPath)) return; @@ -1713,6 +1678,6 @@ std::optional LocalStore::getVersion() return nixVersion; } -static RegisterStoreImplementation regLocalStore; +static RegisterStoreImplementation regLocalStore; } // namespace nix diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index d98d06651..483b337bf 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,6 +1,6 @@ #include "nix/store/machines.hh" #include "nix/store/globals.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include @@ -47,7 +47,7 @@ bool Machine::systemSupported(const std::string & system) const return system == "builtin" || (systemTypes.count(system) > 0); } -bool Machine::allSupported(const std::set & features) const +bool Machine::allSupported(const StringSet & features) const { return std::all_of(features.begin(), features.end(), [&](const std::string & feature) { @@ -56,7 +56,7 @@ bool Machine::allSupported(const std::set & features) const }); } -bool Machine::mandatoryMet(const std::set & features) const +bool Machine::mandatoryMet(const StringSet & features) const { return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), [&](const std::string & feature) { @@ -134,7 +134,7 @@ static std::vector expandBuilderLines(const std::string & builders) return result; } -static Machine parseBuilderLine(const std::set & defaultSystems, const std::string & line) +static Machine parseBuilderLine(const StringSet & defaultSystems, const std::string & line) { const auto tokens = tokenizeString>(line); @@ -178,7 +178,7 @@ static Machine parseBuilderLine(const std::set & defaultSystems, co // `storeUri` tokens[0], // `systemTypes` - isSet(1) ? tokenizeString>(tokens[1], ",") : defaultSystems, + isSet(1) ? tokenizeString(tokens[1], ",") : defaultSystems, // `sshKey` isSet(2) ? tokens[2] : "", // `maxJobs` @@ -186,15 +186,15 @@ static Machine parseBuilderLine(const std::set & defaultSystems, co // `speedFactor` isSet(4) ? parseFloatField(4) : 1.0f, // `supportedFeatures` - isSet(5) ? tokenizeString>(tokens[5], ",") : std::set{}, + isSet(5) ? tokenizeString(tokens[5], ",") : StringSet{}, // `mandatoryFeatures` - isSet(6) ? tokenizeString>(tokens[6], ",") : std::set{}, + isSet(6) ? tokenizeString(tokens[6], ",") : StringSet{}, // `sshPublicHostKey` isSet(7) ? ensureBase64(7) : "" }; } -static Machines parseBuilderLines(const std::set & defaultSystems, const std::vector & builders) +static Machines parseBuilderLines(const StringSet & defaultSystems, const std::vector & builders) { Machines result; std::transform( @@ -203,7 +203,7 @@ static Machines parseBuilderLines(const std::set & defaultSystems, return result; } -Machines Machine::parseConfig(const std::set & defaultSystems, const std::string & s) +Machines Machine::parseConfig(const StringSet & defaultSystems, const std::string & s) { const auto builderLines = expandBuilderLines(s); return parseBuilderLines(defaultSystems, builderLines); diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 112a8ef7b..271c8964f 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -139,6 +139,9 @@ if aws_s3.found() '-L' + aws_s3.get_variable('libdir'), '-laws-cpp-sdk-transfer', '-laws-cpp-sdk-s3', + '-laws-cpp-sdk-identity-management', + '-laws-cpp-sdk-cognito-identity', + '-laws-cpp-sdk-sts', '-laws-cpp-sdk-core', '-laws-crt-cpp', ], @@ -304,6 +307,7 @@ sources = files( 'realisation.cc', 'remote-fs-accessor.cc', 'remote-store.cc', + 'restricted-store.cc', 's3-binary-cache-store.cc', 'serve-protocol-connection.cc', 'serve-protocol.cc', @@ -311,6 +315,8 @@ sources = files( 'ssh-store.cc', 'ssh.cc', 'store-api.cc', + 'store-dir-config.cc', + 'store-registration.cc', 'store-reference.cc', 'uds-remote-store.cc', 'worker-protocol-connection.cc', diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 0e2b62db5..967c91d72 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -4,7 +4,7 @@ #include "nix/store/parsed-derivations.hh" #include "nix/store/derivation-options.hh" #include "nix/store/globals.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/util/thread-pool.hh" #include "nix/store/realisation.hh" #include "nix/util/topo-sort.hh" @@ -222,8 +222,16 @@ void Store::queryMissing(const std::vector & targets, if (knownOutputPaths && invalid.empty()) return; auto drv = make_ref(derivationFromPath(drvPath)); - ParsedDerivation parsedDrv(StorePath(drvPath), *drv); - DerivationOptions drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(drv->env); + DerivationOptions drvOptions; + try { + drvOptions = DerivationOptions::fromStructuredAttrs( + drv->env, + parsedDrv ? &*parsedDrv : nullptr); + } catch (Error & e) { + e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath)); + throw; + } if (!knownOutputPaths && settings.useSubstitutes && drvOptions.substitutesAllowed()) { experimentalFeatureSettings.require(Xp::CaDerivations); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ba80652d0..ef7af6126 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -176,7 +176,7 @@ NarInfo NarInfo::fromJSON( std::nullopt); if (json.contains("downloadSize")) - res.fileSize = getInteger(valueAt(json, "downloadSize")); + res.fileSize = getUnsigned(valueAt(json, "downloadSize")); return res; } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 277795053..e47c0707c 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -101,7 +101,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* HFS/macOS has some undocumented security feature disabling hardlinking for special files within .app dirs. Known affected paths include *.app/Contents/{PkgInfo,Resources/\*.lproj,_CodeSignature} and .DS_Store. - See https://github.com/NixOS/nix/issues/1443 and + See https://github.com/NixOS/nix/issues/1443 and https://github.com/NixOS/nix/pull/2230 for more discussion. */ if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) @@ -216,14 +216,14 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, the store itself (we don't want or need to mess with its permissions). */ const Path dirOfPath(dirOf(path)); - bool mustToggle = dirOfPath != realStoreDir.get(); + bool mustToggle = dirOfPath != config->realStoreDir.get(); if (mustToggle) makeWritable(dirOfPath); /* When we're done, make the directory read-only again and reset its timestamp back to 0. */ MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); - std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), rand()); + std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", config->realStoreDir, getpid(), rand()); try { std::filesystem::create_hard_link(linkPath, tempLink); @@ -285,7 +285,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) if (!isValidPath(i)) continue; /* path was GC'ed, probably */ { Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i))); - optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair); + optimisePath_(&act, stats, config->realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair); } done++; act.progress(done, paths.size()); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index cc7203c6b..d6453c6db 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,98 +1,27 @@ #include "nix/store/parsed-derivations.hh" +#include "nix/store/store-api.hh" +#include "nix/store/derivations.hh" +#include "nix/store/derivation-options.hh" #include #include namespace nix { -ParsedDerivation::ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv) - : drvPath(drvPath), drv(drv) +std::optional StructuredAttrs::tryParse(const StringPairs & env) { /* Parse the __json attribute, if any. */ - auto jsonAttr = drv.env.find("__json"); - if (jsonAttr != drv.env.end()) { + auto jsonAttr = env.find("__json"); + if (jsonAttr != env.end()) { try { - structuredAttrs = std::make_unique(nlohmann::json::parse(jsonAttr->second)); + return StructuredAttrs { + .structuredAttrs = nlohmann::json::parse(jsonAttr->second), + }; } catch (std::exception & e) { - throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what()); + throw Error("cannot process __json attribute: %s", e.what()); } } -} - -ParsedDerivation::~ParsedDerivation() { } - -std::optional ParsedDerivation::getStringAttr(const std::string & name) const -{ - if (structuredAttrs) { - auto i = structuredAttrs->find(name); - if (i == structuredAttrs->end()) - return {}; - else { - if (!i->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath.to_string()); - return i->get(); - } - } else { - auto i = drv.env.find(name); - if (i == drv.env.end()) - return {}; - else - return i->second; - } -} - -bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const -{ - if (structuredAttrs) { - auto i = structuredAttrs->find(name); - if (i == structuredAttrs->end()) - return def; - else { - if (!i->is_boolean()) - throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath.to_string()); - return i->get(); - } - } else { - auto i = drv.env.find(name); - if (i == drv.env.end()) - return def; - else - return i->second == "1"; - } -} - -std::optional ParsedDerivation::getStringsAttr(const std::string & name) const -{ - if (structuredAttrs) { - auto i = structuredAttrs->find(name); - if (i == structuredAttrs->end()) - return {}; - else { - if (!i->is_array()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string()); - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string()); - res.push_back(j->get()); - } - return res; - } - } else { - auto i = drv.env.find(name); - if (i == drv.env.end()) - return {}; - else - return tokenizeString(i->second); - } -} - -std::optional ParsedDerivation::getStringSetAttr(const std::string & name) const -{ - auto ss = getStringsAttr(name); - return ss - ? (std::optional{StringSet{ss->begin(), ss->end()}}) - : (std::optional{}); + return {}; } static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); @@ -151,44 +80,39 @@ static nlohmann::json pathInfoToJSON( return jsonList; } -std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) +nlohmann::json StructuredAttrs::prepareStructuredAttrs( + Store & store, + const DerivationOptions & drvOptions, + const StorePathSet & inputPaths, + const DerivationOutputs & outputs) const { - if (!structuredAttrs) return std::nullopt; - - auto json = *structuredAttrs; + /* Copy to then modify */ + auto json = structuredAttrs; /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv.outputs) - outputs[i.first] = hashPlaceholder(i.first); - json["outputs"] = outputs; + nlohmann::json outputsJson; + for (auto & i : outputs) + outputsJson[i.first] = hashPlaceholder(i.first); + json["outputs"] = std::move(outputsJson); /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(store.toStorePath(p.get()).first); - json[i.key()] = pathInfoToJSON(store, - store.exportReferences(storePaths, inputPaths)); - } + for (auto & [key, inputPaths] : drvOptions.exportReferencesGraph) { + StorePathSet storePaths; + for (auto & p : inputPaths) + storePaths.insert(store.toStorePath(p).first); + json[key] = pathInfoToJSON(store, + store.exportReferences(storePaths, storePaths)); } return json; } -/* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ -std::string writeStructuredAttrsShell(const nlohmann::json & json) +std::string StructuredAttrs::writeShell(const nlohmann::json & json) { auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { if (value.is_string()) - return shellEscape(value.get()); + return escapeShellArgAlways(value.get()); if (value.is_number()) { auto f = value.get(); @@ -236,7 +160,7 @@ std::string writeStructuredAttrsShell(const nlohmann::json & json) for (auto & [key2, value2] : value.items()) { auto s3 = handleSimpleType(value2); if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(key2), *s3); + s2 += fmt("[%s]=%s ", escapeShellArgAlways(key2), *s3); } if (good) diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index df20edb3b..175146435 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -40,6 +40,14 @@ void ValidPathInfo::sign(const Store & store, const Signer & signer) sigs.insert(signer.signDetached(fingerprint(store))); } +void ValidPathInfo::sign(const Store & store, const std::vector> & signers) +{ + auto fingerprint = this->fingerprint(store); + for (auto & signer: signers) { + sigs.insert(signer->signDetached(fingerprint)); + } +} + std::optional ValidPathInfo::contentAddressWithReferences() const { if (! ca) @@ -192,7 +200,7 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( auto & json = getObject(_json); res.narHash = Hash::parseAny(getString(valueAt(json, "narHash")), std::nullopt); - res.narSize = getInteger(valueAt(json, "narSize")); + res.narSize = getUnsigned(valueAt(json, "narSize")); try { auto references = getStringList(valueAt(json, "references")); @@ -216,7 +224,7 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( if (json.contains("registrationTime")) if (auto * rawRegistrationTime = getNullable(valueAt(json, "registrationTime"))) - res.registrationTime = getInteger(*rawRegistrationTime); + res.registrationTime = getInteger(*rawRegistrationTime); if (json.contains("ultimate")) res.ultimate = getBoolean(valueAt(json, "ultimate")); diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 9fbbc8f46..f3fc534ef 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -82,9 +82,9 @@ std::pair parsePathWithOutputs(std::string_view s) { size_t n = s.find("!"); return n == s.npos - ? std::make_pair(s, std::set()) + ? std::make_pair(s, StringSet()) : std::make_pair(s.substr(0, n), - tokenizeString>(s.substr(n + 1), ",")); + tokenizeString(s.substr(n + 1), ",")); } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 5dd1a1699..d989b1caa 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -75,7 +75,7 @@ StorePath StorePath::random(std::string_view name) return StorePath(Hash::random(HashAlgorithm::SHA1), name); } -StorePath StoreDirConfig::parseStorePath(std::string_view path) const +StorePath MixStoreDirMethods::parseStorePath(std::string_view path) const { // On Windows, `/nix/store` is not a canonical path. More broadly it // is unclear whether this function should be using the native @@ -94,7 +94,7 @@ StorePath StoreDirConfig::parseStorePath(std::string_view path) const return StorePath(baseNameOf(p)); } -std::optional StoreDirConfig::maybeParseStorePath(std::string_view path) const +std::optional MixStoreDirMethods::maybeParseStorePath(std::string_view path) const { try { return parseStorePath(path); @@ -103,24 +103,24 @@ std::optional StoreDirConfig::maybeParseStorePath(std::string_view pa } } -bool StoreDirConfig::isStorePath(std::string_view path) const +bool MixStoreDirMethods::isStorePath(std::string_view path) const { return (bool) maybeParseStorePath(path); } -StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const +StorePathSet MixStoreDirMethods::parseStorePathSet(const PathSet & paths) const { StorePathSet res; for (auto & i : paths) res.insert(parseStorePath(i)); return res; } -std::string StoreDirConfig::printStorePath(const StorePath & path) const +std::string MixStoreDirMethods::printStorePath(const StorePath & path) const { return (storeDir + "/").append(path.to_string()); } -PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const +PathSet MixStoreDirMethods::printStorePathSet(const StorePathSet & paths) const { PathSet res; for (auto & i : paths) res.insert(printStorePath(i)); diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index aeb35eab5..792fe5c76 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -136,7 +136,7 @@ static void canonicalisePathMetaData_( #endif if (S_ISDIR(st.st_mode)) { - for (auto & i : std::filesystem::directory_iterator{path}) { + for (auto & i : DirectoryIterator{path}) { checkInterrupt(); canonicalisePathMetaData_( i.path().string(), diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index bd24332cb..b5161b79f 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -38,7 +38,7 @@ std::pair> findGenerations(Path pro std::filesystem::path profileDir = dirOf(profile); auto profileName = std::string(baseNameOf(profile)); - for (auto & i : std::filesystem::directory_iterator{profileDir}) { + for (auto & i : DirectoryIterator{profileDir}) { checkInterrupt(); if (auto n = parseName(profileName, i.path().filename().string())) { auto path = i.path().string(); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 340e7ee2e..fdbe12fa9 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -51,7 +51,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std: std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) { - auto [storePath, restPath_] = store->toStorePath(path.abs()); + auto [storePath, restPath_] = store->toStorePath(store->storeDir + path.abs()); auto restPath = CanonPath(restPath_); if (requireValidPath && !store->isValidPath(storePath)) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0533b7c8a..3151f319c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -24,11 +24,11 @@ namespace nix { /* TODO: Separate these store types into different files, give them better names */ -RemoteStore::RemoteStore(const Params & params) - : RemoteStoreConfig(params) - , Store(params) +RemoteStore::RemoteStore(const Config & config) + : Store{config} + , config{config} , connections(make_ref>( - std::max(1, (int) maxConnections), + std::max(1, config.maxConnections.get()), [this]() { auto conn = openConnectionWrapper(); try { @@ -44,7 +44,7 @@ RemoteStore::RemoteStore(const Params & params) r->to.good() && r->from.good() && std::chrono::duration_cast( - std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; + std::chrono::steady_clock::now() - r->startTime).count() < this->config.maxConnectionAge; } )) { @@ -122,7 +122,7 @@ void RemoteStore::setOptions(Connection & conn) << settings.useSubstitutes; if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { - std::map overrides; + std::map overrides; settings.getSettings(overrides, true); // libstore settings fileTransferSettings.getSettings(overrides, true); overrides.erase(settings.keepFailed.name); diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc new file mode 100644 index 000000000..0485f5584 --- /dev/null +++ b/src/libstore/restricted-store.cc @@ -0,0 +1,332 @@ +#include "nix/store/restricted-store.hh" +#include "nix/store/build-result.hh" +#include "nix/util/callback.hh" +#include "nix/store/realisation.hh" + +namespace nix { + +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) { return bo.path; }, + [&](const SingleDerivedPath::Built & bfd) { return pathPartOfReq(*bfd.drvPath); }, + }, + req.raw()); +} + +static StorePath pathPartOfReq(const DerivedPath & req) +{ + return std::visit( + overloaded{ + [&](const DerivedPath::Opaque & bo) { return bo.path; }, + [&](const DerivedPath::Built & bfd) { return pathPartOfReq(*bfd.drvPath); }, + }, + req.raw()); +} + +bool RestrictionContext::isAllowed(const DerivedPath & req) +{ + return isAllowed(pathPartOfReq(req)); +} + +/** + * A wrapper around LocalStore that only allows building/querying of + * paths that are in the input closures of the build or were added via + * recursive Nix calls. + */ +struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStore +{ + ref config; + + ref next; + + RestrictionContext & goal; + + RestrictedStore(ref config, ref next, RestrictionContext & goal) + : Store{*config} + , LocalFSStore{*config} + , config{config} + , next(next) + , goal(goal) + { + } + + Path getRealStoreDir() override + { + return next->config->realStoreDir; + } + + std::string getUri() override + { + return next->getUri(); + } + + StorePathSet queryAllValidPaths() override; + + void queryPathInfoUncached( + const StorePath & path, Callback> callback) noexcept override; + + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; + + std::map> + queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override; + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { + throw Error("queryPathFromHashPart"); + } + + StorePath addToStore( + std::string_view name, + const SourcePath & srcPath, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter, + RepairFlag repair) override + { + throw Error("addToStore"); + } + + void addToStore( + const ValidPathInfo & info, + Source & narSource, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs) override; + + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) override; + + void narFromPath(const StorePath & path, Sink & sink) override; + + void ensurePath(const StorePath & path) override; + + void registerDrvOutput(const Realisation & info) override; + + void queryRealisationUncached( + const DrvOutput & id, Callback> callback) noexcept override; + + void + buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; + + std::vector buildPathsWithResults( + const std::vector & paths, + BuildMode buildMode = bmNormal, + std::shared_ptr evalStore = nullptr) override; + + BuildResult + buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) override + { + unsupported("buildDerivation"); + } + + void addTempRoot(const StorePath & path) override {} + + void addIndirectRoot(const Path & path) override {} + + Roots findRoots(bool censor) override + { + return Roots(); + } + + void collectGarbage(const GCOptions & options, GCResults & results) override {} + + void addSignatures(const StorePath & storePath, const StringSet & sigs) override + { + unsupported("addSignatures"); + } + + void queryMissing( + const std::vector & targets, + StorePathSet & willBuild, + StorePathSet & willSubstitute, + StorePathSet & unknown, + uint64_t & downloadSize, + uint64_t & narSize) override; + + virtual std::optional getBuildLogExact(const StorePath & path) override + { + return std::nullopt; + } + + virtual void addBuildLog(const StorePath & path, std::string_view log) override + { + unsupported("addBuildLog"); + } + + std::optional isTrustedClient() override + { + return NotTrusted; + } +}; + +ref makeRestrictedStore(ref config, ref next, RestrictionContext & context) +{ + return make_ref(config, next, context); +} + +StorePathSet RestrictedStore::queryAllValidPaths() +{ + StorePathSet paths; + for (auto & p : goal.originalPaths()) + paths.insert(p); + for (auto & p : goal.addedPaths) + paths.insert(p); + return paths; +} + +void RestrictedStore::queryPathInfoUncached( + const StorePath & path, Callback> callback) noexcept +{ + if (goal.isAllowed(path)) { + try { + /* Censor impure information. */ + auto info = std::make_shared(*next->queryPathInfo(path)); + info->deriver.reset(); + info->registrationTime = 0; + info->ultimate = false; + info->sigs.clear(); + callback(info); + } catch (InvalidPath &) { + callback(nullptr); + } + } else + callback(nullptr); +}; + +void RestrictedStore::queryReferrers(const StorePath & path, StorePathSet & referrers) {} + +std::map> +RestrictedStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore) +{ + if (!goal.isAllowed(path)) + throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); + return next->queryPartialDerivationOutputMap(path, evalStore); +} + +void RestrictedStore::addToStore( + const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) +{ + next->addToStore(info, narSource, repair, checkSigs); + goal.addDependency(info.path); +} + +StorePath RestrictedStore::addToStoreFromDump( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) +{ + auto path = next->addToStoreFromDump(dump, name, dumpMethod, hashMethod, hashAlgo, references, repair); + goal.addDependency(path); + return path; +} + +void RestrictedStore::narFromPath(const StorePath & path, Sink & sink) +{ + if (!goal.isAllowed(path)) + throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); + LocalFSStore::narFromPath(path, sink); +} + +void RestrictedStore::ensurePath(const StorePath & path) +{ + if (!goal.isAllowed(path)) + throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); + /* Nothing to be done; 'path' must already be valid. */ +} + +void RestrictedStore::registerDrvOutput(const Realisation & info) +// XXX: This should probably be allowed as a no-op if the realisation +// corresponds to an allowed derivation +{ + throw Error("registerDrvOutput"); +} + +void RestrictedStore::queryRealisationUncached( + const DrvOutput & id, Callback> callback) noexcept +// XXX: This should probably be allowed if the realisation corresponds to +// an allowed derivation +{ + if (!goal.isAllowed(id)) + callback(nullptr); + next->queryRealisation(id, std::move(callback)); +} + +void RestrictedStore::buildPaths( + const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) +{ + for (auto & result : buildPathsWithResults(paths, buildMode, evalStore)) + if (!result.success()) + result.rethrow(); +} + +std::vector RestrictedStore::buildPathsWithResults( + const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) +{ + assert(!evalStore); + + if (buildMode != bmNormal) + throw Error("unsupported build mode"); + + StorePathSet newPaths; + std::set newRealisations; + + for (auto & req : paths) { + if (!goal.isAllowed(req)) + throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next)); + } + + auto results = next->buildPathsWithResults(paths, buildMode); + + for (auto & result : results) { + for (auto & [outputName, output] : result.builtOutputs) { + newPaths.insert(output.outPath); + newRealisations.insert(output); + } + } + + StorePathSet closure; + next->computeFSClosure(newPaths, closure); + for (auto & path : closure) + goal.addDependency(path); + for (auto & real : Realisation::closure(*next, newRealisations)) + goal.addedDrvOutputs.insert(real.id); + + return results; +} + +void RestrictedStore::queryMissing( + const std::vector & targets, + StorePathSet & willBuild, + StorePathSet & willSubstitute, + StorePathSet & unknown, + uint64_t & downloadSize, + uint64_t & narSize) +{ + /* This is slightly impure since it leaks information to the + client about what paths will be built/substituted or are + already present. Probably not a big deal. */ + + std::vector allowed; + for (auto & req : targets) { + if (goal.isAllowed(req)) + allowed.emplace_back(req); + else + unknown.insert(pathPartOfReq(req)); + } + + next->queryMissing(allowed, willBuild, willSubstitute, unknown, downloadSize, narSize); +} + +} diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f9e583307..618112d1c 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -11,6 +11,7 @@ #include "nix/util/compression.hh" #include "nix/store/filetransfer.hh" #include "nix/util/signals.hh" +#include "nix/store/store-registration.hh" #include #include @@ -21,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +73,29 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem #endif }; +/* Retrieve the credentials from the list of AWS default providers, with the addition of the STS creds provider. This + last can be used to acquire further permissions with a specific IAM role. + Roughly based on https://github.com/aws/aws-sdk-cpp/issues/150#issuecomment-538548438 +*/ +struct CustomAwsCredentialsProviderChain : public Aws::Auth::AWSCredentialsProviderChain +{ + CustomAwsCredentialsProviderChain(const std::string & profile) + { + if (profile.empty()) { + // Use all the default AWS providers, plus the possibility to acquire a IAM role directly via a profile. + Aws::Auth::DefaultAWSCredentialsProviderChain default_aws_chain; + for (auto provider : default_aws_chain.GetProviders()) + AddProvider(provider); + AddProvider(std::make_shared()); + } else { + // Override the profile name to retrieve from the AWS config and credentials. I believe this option + // comes from the ?profile querystring in nix.conf. + AddProvider(std::make_shared(profile.c_str())); + AddProvider(std::make_shared(profile)); + } + } +}; + static void initAWS() { static std::once_flag flag; @@ -102,13 +127,8 @@ S3Helper::S3Helper( const std::string & endpoint) : config(makeConfig(region, scheme, endpoint)) , client(make_ref( - profile == "" - ? std::dynamic_pointer_cast( - std::make_shared()) - : std::dynamic_pointer_cast( - std::make_shared(profile.c_str())), + std::make_shared(profile), *config, - // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 #if AWS_SDK_VERSION_MAJOR == 1 && AWS_SDK_VERSION_MINOR < 3 false, #else @@ -216,11 +236,6 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } -S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) - : BinaryCacheStoreConfig(params) - , BinaryCacheStore(params) -{ } - S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( std::string_view uriScheme, @@ -239,6 +254,12 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); } + +S3BinaryCacheStore::S3BinaryCacheStore(ref config) + : BinaryCacheStore(*config) + , config{config} +{ } + std::string S3BinaryCacheStoreConfig::doc() { return @@ -247,40 +268,37 @@ std::string S3BinaryCacheStoreConfig::doc() } -struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore +struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore { Stats stats; S3Helper s3Helper; - S3BinaryCacheStoreImpl( - std::string_view uriScheme, - std::string_view bucketName, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , S3BinaryCacheStoreConfig(uriScheme, bucketName, params) - , Store(params) - , BinaryCacheStore(params) - , S3BinaryCacheStore(params) - , s3Helper(profile, region, scheme, endpoint) + S3BinaryCacheStoreImpl(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , S3BinaryCacheStore{config} + , s3Helper(config->profile, config->region, config->scheme, config->endpoint) { diskCache = getNarInfoDiskCache(); + + init(); } std::string getUri() override { - return "s3://" + bucketName; + return "s3://" + config->bucketName; } void init() override { if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { - wantMassQuery.setDefault(cacheInfo->wantMassQuery); - priority.setDefault(cacheInfo->priority); + config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); + config->priority.setDefault(cacheInfo->priority); } else { BinaryCacheStore::init(); - diskCache->createCache(getUri(), storeDir, wantMassQuery, priority); + diskCache->createCache( + getUri(), config->storeDir, config->wantMassQuery, config->priority); } } @@ -309,7 +327,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto res = s3Helper.client->HeadObject( Aws::S3::Model::HeadObjectRequest() - .WithBucket(bucketName) + .WithBucket(config->bucketName) .WithKey(path)); if (!res.IsSuccess()) { @@ -353,7 +371,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual const std::string & mimeType, const std::string & contentEncoding) { - std::string uri = "s3://" + bucketName + "/" + path; + std::string uri = "s3://" + config->bucketName + "/" + path; Activity act(*logger, lvlTalkative, actFileTransfer, fmt("uploading '%s'", uri), Logger::Fields{uri}, getCurActivity()); @@ -368,11 +386,11 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual std::call_once(transferManagerCreated, [&]() { - if (multipartUpload) { + if (config->multipartUpload) { TransferManagerConfiguration transferConfig(executor.get()); transferConfig.s3Client = s3Helper.client; - transferConfig.bufferSize = bufferSize; + transferConfig.bufferSize = config->bufferSize; transferConfig.uploadProgressCallback = [](const TransferManager * transferManager, @@ -402,6 +420,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto now1 = std::chrono::steady_clock::now(); + auto & bucketName = config->bucketName; + if (transferManager) { if (contentEncoding != "") @@ -489,12 +509,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual return std::make_shared(std::move(compressed)); }; - if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) - uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression); - else if (lsCompression != "" && hasSuffix(path, ".ls")) - uploadFile(path, compress(lsCompression), mimeType, lsCompression); - else if (logCompression != "" && hasPrefix(path, "log/")) - uploadFile(path, compress(logCompression), mimeType, logCompression); + if (config->narinfoCompression != "" && hasSuffix(path, ".narinfo")) + uploadFile(path, compress(config->narinfoCompression), mimeType, config->narinfoCompression); + else if (config->lsCompression != "" && hasSuffix(path, ".ls")) + uploadFile(path, compress(config->lsCompression), mimeType, config->lsCompression); + else if (config->logCompression != "" && hasPrefix(path, "log/")) + uploadFile(path, compress(config->logCompression), mimeType, config->logCompression); else uploadFile(path, istream, mimeType, ""); } @@ -504,14 +524,14 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual stats.get++; // FIXME: stream output to sink. - auto res = s3Helper.getObject(bucketName, path); + auto res = s3Helper.getObject(config->bucketName, path); stats.getBytes += res.data ? res.data->size() : 0; stats.getTimeMs += res.durationMs; if (res.data) { printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", - bucketName, path, res.data->size(), res.durationMs); + config->bucketName, path, res.data->size(), res.durationMs); sink(*res.data); } else @@ -523,6 +543,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual StorePathSet paths; std::string marker; + auto & bucketName = config->bucketName; + do { debug("listing bucket 's3://%s' from key '%s'...", bucketName, marker); @@ -561,7 +583,15 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual } }; -static RegisterStoreImplementation regS3BinaryCacheStore; +ref S3BinaryCacheStoreImpl::Config::openStore() const +{ + return make_ref(ref{ + // FIXME we shouldn't actually need a mutable config + std::const_pointer_cast(shared_from_this()) + }); +} + +static RegisterStoreImplementation regS3BinaryCacheStore; } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 45ea05ffc..753256d48 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -7,6 +7,7 @@ #include "nix/store/worker-protocol-impl.hh" #include "nix/util/pool.hh" #include "nix/store/ssh.hh" +#include "nix/store/store-registration.hh" namespace nix { @@ -14,12 +15,13 @@ SSHStoreConfig::SSHStoreConfig( std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, authority, params) + : Store::Config{params} + , RemoteStore::Config{params} + , CommonSSHStoreConfig{scheme, authority, params} { } + std::string SSHStoreConfig::doc() { return @@ -27,21 +29,18 @@ std::string SSHStoreConfig::doc() ; } -class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore -{ -public: - SSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(scheme, host, params) - , Store(params) - , RemoteStore(params) - , master(createSSHMaster( +struct SSHStore : virtual RemoteStore +{ + using Config = SSHStoreConfig; + + ref config; + + SSHStore(ref config) + : Store{*config} + , RemoteStore{*config} + , config{config} + , master(config->createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1)) { @@ -49,7 +48,7 @@ public: std::string getUri() override { - return *uriSchemes().begin() + "://" + host; + return *Config::uriSchemes().begin() + "://" + host; } // FIXME extend daemon protocol, move implementation to RemoteStore @@ -101,7 +100,7 @@ MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::strin : StoreConfig(params) , RemoteStoreConfig(params) , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) + , SSHStoreConfig(scheme, host, params) , LocalFSStoreConfig(params) { } @@ -128,35 +127,21 @@ std::string MountedSSHStoreConfig::doc() * The difference lies in how they manage GC roots. See addPermRoot * below for details. */ -class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore +struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore { -public: + using Config = MountedSSHStoreConfig; - MountedSSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) - , MountedSSHStoreConfig(params) - , Store(params) - , RemoteStore(params) - , SSHStore(scheme, host, params) - , LocalFSStore(params) + MountedSSHStore(ref config) + : Store{*config} + , RemoteStore{*config} + , SSHStore{config} + , LocalFSStore{*config} { extraRemoteProgramArgs = { "--process-ops", }; } - std::string getUri() override - { - return *uriSchemes().begin() + "://" + host; - } - void narFromPath(const StorePath & path, Sink & sink) override { return LocalFSStore::narFromPath(path, sink); @@ -198,14 +183,26 @@ public: } }; + +ref SSHStore::Config::openStore() const { + return make_ref(ref{shared_from_this()}); +} + +ref MountedSSHStore::Config::openStore() const { + return make_ref(ref{ + std::dynamic_pointer_cast(shared_from_this()) + }); +} + + ref SSHStore::openConnection() { auto conn = make_ref(); - Strings command = remoteProgram.get(); + Strings command = config->remoteProgram.get(); command.push_back("--stdio"); - if (remoteStore.get() != "") { + if (config->remoteStore.get() != "") { command.push_back("--store"); - command.push_back(remoteStore.get()); + command.push_back(config->remoteStore.get()); } command.insert(command.end(), extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end()); @@ -215,7 +212,7 @@ ref SSHStore::openConnection() return conn; } -static RegisterStoreImplementation regSSHStore; -static RegisterStoreImplementation regMountedSSHStore; +static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ae1a53e79..c9ccc69fc 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -5,6 +5,7 @@ #include "nix/store/realisation.hh" #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/util/util.hh" #include "nix/store/nar-info-disk-cache.hh" #include "nix/util/thread-pool.hh" @@ -29,13 +30,13 @@ using json = nlohmann::json; namespace nix { -bool StoreDirConfig::isInStore(PathView path) const +bool MixStoreDirMethods::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair StoreDirConfig::toStorePath(PathView path) const +std::pair MixStoreDirMethods::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); @@ -77,7 +78,7 @@ to match. */ -StorePath StoreDirConfig::makeStorePath(std::string_view type, +StorePath MixStoreDirMethods::makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ @@ -88,14 +89,14 @@ StorePath StoreDirConfig::makeStorePath(std::string_view type, } -StorePath StoreDirConfig::makeStorePath(std::string_view type, +StorePath MixStoreDirMethods::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } -StorePath StoreDirConfig::makeOutputPath(std::string_view id, +StorePath MixStoreDirMethods::makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const { return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id)); @@ -106,7 +107,7 @@ StorePath StoreDirConfig::makeOutputPath(std::string_view id, hacky, but we can't put them in, say, (per the grammar above) since that would be ambiguous. */ static std::string makeType( - const StoreDirConfig & store, + const MixStoreDirMethods & store, std::string && type, const StoreReferences & references) { @@ -119,7 +120,7 @@ static std::string makeType( } -StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const +StorePath MixStoreDirMethods::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1) throw Error("Git file ingestion must use SHA-1 hash"); @@ -141,7 +142,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed } -StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath MixStoreDirMethods::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { @@ -162,7 +163,7 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const } -std::pair StoreDirConfig::computeStorePath( +std::pair MixStoreDirMethods::computeStorePath( std::string_view name, const SourcePath & path, ContentAddressMethod method, @@ -424,7 +425,7 @@ ValidPathInfo Store::addToStoreSlow( return info; } -StringSet StoreConfig::getDefaultSystemFeatures() +StringSet Store::Config::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); @@ -437,9 +438,10 @@ StringSet StoreConfig::getDefaultSystemFeatures() return res; } -Store::Store(const Params & params) - : StoreConfig(params) - , state({(size_t) pathInfoCacheSize}) +Store::Store(const Store::Config & config) + : MixStoreDirMethods{config} + , config{config} + , state({(size_t) config.pathInfoCacheSize}) { assertLibStoreInitialized(); } @@ -573,7 +575,7 @@ bool Store::isValidPath(const StorePath & storePath) { { auto state_(state.lock()); - auto res = state_->pathInfoCache.get(std::string(storePath.to_string())); + auto res = state_->pathInfoCache.get(storePath.to_string()); if (res && res->isKnownNow()) { stats.narInfoReadAverted++; return res->didExist(); @@ -585,7 +587,7 @@ bool Store::isValidPath(const StorePath & storePath) if (res.first != NarInfoDiskCache::oUnknown) { stats.narInfoReadAverted++; auto state_(state.lock()); - state_->pathInfoCache.upsert(std::string(storePath.to_string()), + state_->pathInfoCache.upsert(storePath.to_string(), res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second }); return res.first == NarInfoDiskCache::oValid; } @@ -644,7 +646,7 @@ std::optional> Store::queryPathInfoFromClie auto hashPart = std::string(storePath.hashPart()); { - auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string())); + auto res = state.lock()->pathInfoCache.get(storePath.to_string()); if (res && res->isKnownNow()) { stats.narInfoReadAverted++; if (res->didExist()) @@ -660,7 +662,7 @@ std::optional> Store::queryPathInfoFromClie stats.narInfoReadAverted++; { auto state_(state.lock()); - state_->pathInfoCache.upsert(std::string(storePath.to_string()), + state_->pathInfoCache.upsert(storePath.to_string(), res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); if (res.first == NarInfoDiskCache::oInvalid || !goodStorePath(storePath, res.second->path)) @@ -704,7 +706,7 @@ void Store::queryPathInfo(const StorePath & storePath, { auto state_(state.lock()); - state_->pathInfoCache.upsert(std::string(storePath.to_string()), PathInfoCacheValue { .value = info }); + state_->pathInfoCache.upsert(storePath.to_string(), PathInfoCacheValue { .value = info }); } if (!info || !goodStorePath(storePath, info->path)) { @@ -1209,7 +1211,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre } -std::string StoreDirConfig::showPaths(const StorePathSet & paths) +std::string MixStoreDirMethods::showPaths(const StorePathSet & paths) const { std::string s; for (auto & i : paths) { @@ -1237,7 +1239,7 @@ static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, auto accessor = store.getFSAccessor(requireValidPath); try { return parseDerivation(store, - accessor->readFile(CanonPath(store.printStorePath(drvPath))), + accessor->readFile(CanonPath(drvPath.to_string())), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); @@ -1278,103 +1280,32 @@ Derivation Store::readDerivation(const StorePath & drvPath) Derivation Store::readInvalidDerivation(const StorePath & drvPath) { return readDerivationCommon(*this, drvPath, false); } -} - -#include "nix/store/local-store.hh" -#include "nix/store/uds-remote-store.hh" - - -namespace nix { - -ref openStore(const std::string & uri, - const Store::Params & extraParams) +void Store::signPathInfo(ValidPathInfo & info) { - return openStore(StoreReference::parse(uri, extraParams)); + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.secretKeyFiles; + + for (auto & secretKeyFile : secretKeyFiles.get()) { + SecretKey secretKey(readFile(secretKeyFile)); + LocalSigner signer(std::move(secretKey)); + info.sign(*this, signer); + } } -ref openStore(StoreReference && storeURI) + +void Store::signRealisation(Realisation & realisation) { - auto & params = storeURI.params; + // FIXME: keep secret keys in memory. - auto store = std::visit(overloaded { - [&](const StoreReference::Auto &) -> std::shared_ptr { - auto stateDir = getOr(params, "state", settings.nixStateDir); - if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return std::make_shared(params); - else if (pathExists(settings.nixDaemonSocketFile)) - return std::make_shared(params); - #ifdef __linux__ - else if (!pathExists(stateDir) - && params.empty() - && !isRootUser() - && !getEnv("NIX_STORE_DIR").has_value() - && !getEnv("NIX_STATE_DIR").has_value()) - { - /* If /nix doesn't exist, there is no daemon socket, and - we're not root, then automatically set up a chroot - store in ~/.local/share/nix/root. */ - auto chrootStore = getDataDir() + "/root"; - if (!pathExists(chrootStore)) { - try { - createDirs(chrootStore); - } catch (SystemError & e) { - return std::make_shared(params); - } - warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - } else - debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - return std::make_shared("local", chrootStore, params); - } - #endif - else - return std::make_shared(params); - }, - [&](const StoreReference::Specified & g) { - for (const auto & implem : *Implementations::registered) - if (implem.uriSchemes.count(g.scheme)) - return implem.create(g.scheme, g.authority, params); + auto secretKeyFiles = settings.secretKeyFiles; - throw Error("don't know how to open Nix store with scheme '%s'", g.scheme); - }, - }, storeURI.variant); - - experimentalFeatureSettings.require(store->experimentalFeature()); - store->warnUnknownSettings(); - store->init(); - - return ref { store }; + for (auto & secretKeyFile : secretKeyFiles.get()) { + SecretKey secretKey(readFile(secretKeyFile)); + LocalSigner signer(std::move(secretKey)); + realisation.sign(signer); + } } -std::list> getDefaultSubstituters() -{ - static auto stores([]() { - std::list> stores; - - StringSet done; - - auto addStore = [&](const std::string & uri) { - if (!done.insert(uri).second) return; - try { - stores.push_back(openStore(uri)); - } catch (Error & e) { - logWarning(e.info()); - } - }; - - for (const auto & uri : settings.substituters.get()) - addStore(uri); - - stores.sort([](ref & a, ref & b) { - return a->priority < b->priority; - }); - - return stores; - } ()); - - return stores; -} - -std::vector * Implementations::registered = 0; - } diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc new file mode 100644 index 000000000..ec65013ef --- /dev/null +++ b/src/libstore/store-dir-config.cc @@ -0,0 +1,13 @@ +#include "nix/store/store-dir-config.hh" +#include "nix/util/util.hh" +#include "nix/store/globals.hh" + +namespace nix { + +StoreDirConfig::StoreDirConfig(const Params & params) + : StoreDirConfigBase(params) + , MixStoreDirMethods{storeDir_} +{ +} + +} diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc new file mode 100644 index 000000000..6362ac036 --- /dev/null +++ b/src/libstore/store-registration.cc @@ -0,0 +1,105 @@ +#include "nix/store/store-registration.hh" +#include "nix/store/store-open.hh" +#include "nix/store/local-store.hh" +#include "nix/store/uds-remote-store.hh" + +namespace nix { + +ref openStore(const std::string & uri, const Store::Config::Params & extraParams) +{ + return openStore(StoreReference::parse(uri, extraParams)); +} + +ref openStore(StoreReference && storeURI) +{ + auto store = resolveStoreConfig(std::move(storeURI))->openStore(); + store->init(); + return store; +} + +ref resolveStoreConfig(StoreReference && storeURI) +{ + auto & params = storeURI.params; + + auto storeConfig = std::visit( + overloaded{ + [&](const StoreReference::Auto &) -> ref { + auto stateDir = getOr(params, "state", settings.nixStateDir); + if (access(stateDir.c_str(), R_OK | W_OK) == 0) + return make_ref(params); + else if (pathExists(settings.nixDaemonSocketFile)) + return make_ref(params); +#ifdef __linux__ + else if ( + !pathExists(stateDir) && params.empty() && !isRootUser() && !getEnv("NIX_STORE_DIR").has_value() + && !getEnv("NIX_STATE_DIR").has_value()) { + /* If /nix doesn't exist, there is no daemon socket, and + we're not root, then automatically set up a chroot + store in ~/.local/share/nix/root. */ + auto chrootStore = getDataDir() + "/root"; + if (!pathExists(chrootStore)) { + try { + createDirs(chrootStore); + } catch (SystemError & e) { + return make_ref(params); + } + warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + } else + debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + return make_ref("local", chrootStore, params); + } +#endif + else + return make_ref(params); + }, + [&](const StoreReference::Specified & g) { + for (const auto & [storeName, implem] : Implementations::registered()) + if (implem.uriSchemes.count(g.scheme)) + return implem.parseConfig(g.scheme, g.authority, params); + + throw Error("don't know how to open Nix store with scheme '%s'", g.scheme); + }, + }, + storeURI.variant); + + experimentalFeatureSettings.require(storeConfig->experimentalFeature()); + storeConfig->warnUnknownSettings(); + + return storeConfig; +} + +std::list> getDefaultSubstituters() +{ + static auto stores([]() { + std::list> stores; + + StringSet done; + + auto addStore = [&](const std::string & uri) { + if (!done.insert(uri).second) + return; + try { + stores.push_back(openStore(uri)); + } catch (Error & e) { + logWarning(e.info()); + } + }; + + for (const auto & uri : settings.substituters.get()) + addStore(uri); + + stores.sort([](ref & a, ref & b) { return a->config.priority < b->config.priority; }); + + return stores; + }()); + + return stores; +} + +Implementations::Map & Implementations::registered() +{ + static Map registered; + return registered; +} + +} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index b58dd4783..c979b5e47 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,6 +1,7 @@ #include "nix/store/uds-remote-store.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/store/worker-protocol.hh" +#include "nix/store/store-registration.hh" #include #include @@ -20,13 +21,13 @@ namespace nix { UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) + const StoreReference::Params & params) + : Store::Config{params} + , LocalFSStore::Config{params} + , RemoteStore::Config{params} , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { - if (scheme != UDSRemoteStoreConfig::scheme) { + if (uriSchemes().count(scheme) == 0) { throw UsageError("Scheme must be 'unix'"); } } @@ -44,32 +45,30 @@ std::string UDSRemoteStoreConfig::doc() // empty string will later default to the same nixDaemonSocketFile. Why // don't we just wire it all through? I believe there are cases where it // will live reload so we want to continue to account for that. -UDSRemoteStore::UDSRemoteStore(const Params & params) - : UDSRemoteStore(scheme, "", params) -{} +UDSRemoteStoreConfig::UDSRemoteStoreConfig(const Params & params) + : UDSRemoteStoreConfig(*uriSchemes().begin(), "", params) +{ +} -UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - , UDSRemoteStoreConfig(scheme, authority, params) - , Store(params) - , LocalFSStore(params) - , RemoteStore(params) +UDSRemoteStore::UDSRemoteStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , RemoteStore{*config} + , config{config} { } std::string UDSRemoteStore::getUri() { - return path == settings.nixDaemonSocketFile + return config->path == settings.nixDaemonSocketFile ? // FIXME: Not clear why we return daemon here and not default // to settings.nixDaemonSocketFile // // unix:// with no path also works. Change what we return? "daemon" - : std::string(scheme) + "://" + path; + : std::string(*Config::uriSchemes().begin()) + "://" + config->path; } @@ -84,7 +83,7 @@ ref UDSRemoteStore::openConnection() auto conn = make_ref(); /* Connect to a daemon that does the privileged work for us. */ - conn->fd = nix::connect(path); + conn->fd = nix::connect(config->path); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); @@ -104,6 +103,11 @@ void UDSRemoteStore::addIndirectRoot(const Path & path) } -static RegisterStoreImplementation regUDSRemoteStore; +ref UDSRemoteStore::Config::openStore() const { + return make_ref(ref{shared_from_this()}); +} + + +static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/derivation-builder.cc similarity index 78% rename from src/libstore/unix/build/local-derivation-goal.cc rename to src/libstore/unix/build/derivation-builder.cc index 3ba1e823f..abfe9b2b1 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1,4 +1,4 @@ -#include "nix/store/build/local-derivation-goal.hh" +#include "nix/store/build/derivation-builder.hh" #include "nix/store/local-store.hh" #include "nix/util/processes.hh" #include "nix/store/indirect-root-store.hh" @@ -21,9 +21,8 @@ #include "nix/util/unix-domain-socket.hh" #include "nix/store/posix-fs-canonicalise.hh" #include "nix/util/posix-source-accessor.hh" -#include "nix/store/config.hh" +#include "nix/store/restricted-store.hh" -#include #include #include @@ -37,7 +36,7 @@ #include "store-config-private.hh" #if HAVE_STATVFS -#include +# include #endif /* Includes required for chroot support. */ @@ -61,9 +60,9 @@ #endif #ifdef __APPLE__ -#include -#include -#include +# include +# include +# include /* This definition is undocumented but depended upon by all major browsers. */ extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf); @@ -80,6 +79,318 @@ extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, namespace nix { +MakeError(NotDeterministic, BuildError); + +/** + * This class represents the state for building locally. + * + * @todo Ideally, it would not be a class, but a single function. + * However, besides the main entry point, there are a few more methods + * which are externally called, and need to be gotten rid of. There are + * also some virtual methods (either directly here or inherited from + * `DerivationBuilderCallbacks`, a stop-gap) that represent outgoing + * rather than incoming call edges that either should be removed, or + * become (higher order) function parameters. + */ +class DerivationBuilderImpl : public DerivationBuilder, DerivationBuilderParams +{ + Store & store; + + std::unique_ptr miscMethods; + +public: + + DerivationBuilderImpl( + Store & store, + std::unique_ptr miscMethods, + DerivationBuilderParams params) + : DerivationBuilderParams{std::move(params)} + , store{store} + , miscMethods{std::move(miscMethods)} + { } + + LocalStore & getLocalStore(); + +private: + + /** + * The cgroup of the builder, if any. + */ + std::optional cgroup; + + /** + * The temporary directory used for the build. + */ + Path tmpDir; + + /** + * The top-level temporary directory. `tmpDir` is either equal to + * or a child of this directory. + */ + Path topTmpDir; + + /** + * The path of the temporary directory in the sandbox. + */ + Path tmpDirInSandbox; + + /** + * Pipe for synchronising updates to the builder namespaces. + */ + Pipe userNamespaceSync; + + /** + * The mount namespace and user namespace of the builder, used to add additional + * paths to the sandbox as a result of recursive Nix calls. + */ + AutoCloseFD sandboxMountNamespace; + AutoCloseFD sandboxUserNamespace; + + /** + * On Linux, whether we're doing the build in its own user + * namespace. + */ + bool usingUserNamespace = true; + + /** + * Whether we're currently doing a chroot build. + */ + bool useChroot = false; + + /** + * The root of the chroot environment. + */ + Path chrootRootDir; + + /** + * RAII object to delete the chroot directory. + */ + std::shared_ptr autoDelChroot; + + /** + * The sort of derivation we are building. + * + * Just a cached value, can be recomputed from `drv`. + */ + std::optional derivationType; + + /** + * Stuff we need to pass to initChild(). + */ + struct ChrootPath { + Path source; + bool optional; + ChrootPath(Path source = "", bool optional = false) + : source(source), optional(optional) + { } + }; + typedef std::map PathsInChroot; // maps target path to source path + PathsInChroot pathsInChroot; + + typedef std::map Environment; + Environment env; + + /** + * Hash rewriting. + */ + StringMap inputRewrites, outputRewrites; + typedef std::map RedirectedOutputs; + RedirectedOutputs redirectedOutputs; + + /** + * The output paths used during the build. + * + * - Input-addressed derivations or fixed content-addressed outputs are + * sometimes built when some of their outputs already exist, and can not + * be hidden via sandboxing. We use temporary locations instead and + * rewrite after the build. Otherwise the regular predetermined paths are + * put here. + * + * - Floating content-addressing derivations do not know their final build + * output paths until the outputs are hashed, so random locations are + * used, and then renamed. The randomness helps guard against hidden + * self-references. + */ + OutputPathMap scratchOutputs; + + uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } + + const static Path homeDir; + + /** + * The recursive Nix daemon socket. + */ + AutoCloseFD daemonSocket; + + /** + * The daemon main thread. + */ + std::thread daemonThread; + + /** + * The daemon worker threads. + */ + std::vector daemonWorkerThreads; + + const StorePathSet & originalPaths() override + { + return inputPaths; + } + + bool isAllowed(const StorePath & path) override + { + return inputPaths.count(path) || addedPaths.count(path); + } + bool isAllowed(const DrvOutput & id) override + { + return addedDrvOutputs.count(id); + } + + bool isAllowed(const DerivedPath & req); + + friend struct RestrictedStore; + + /** + * Whether we need to perform hash rewriting if there are valid output paths. + */ + bool needsHashRewrite(); + +public: + + /** + * Set up build environment / sandbox, acquiring resources (e.g. + * locks as needed). After this is run, the builder should be + * started. + * + * @returns true if successful, false if we could not acquire a build + * user. In that case, the caller must wait and then try again. + */ + bool prepareBuild() override; + + /** + * Start building a derivation. + */ + void startBuilder() override;; + + /** + * Tear down build environment after the builder exits (either on + * its own or if it is killed). + * + * @returns The first case indicates failure during output + * processing. A status code and exception are returned, providing + * more information. The second case indicates success, and + * realisations for each output of the derivation are returned. + */ + std::variant, SingleDrvOutputs> unprepareBuild() override; + +private: + + /** + * Fill in the environment for the builder. + */ + void initEnv(); + + /** + * Process messages send by the sandbox initialization. + */ + void processSandboxSetupMessages(); + + /** + * Setup tmp dir location. + */ + void initTmpDir(); + + /** + * Write a JSON file containing the derivation attributes. + */ + void writeStructuredAttrs(); + + /** + * Start an in-process nix daemon thread for recursive-nix. + */ + void startDaemon(); + +public: + + /** + * Stop the in-process nix daemon thread. + * @see startDaemon + */ + void stopDaemon() override; + +private: + + void addDependency(const StorePath & path) override; + + /** + * Make a file owned by the builder. + */ + void chownToBuilder(const Path & path); + + /** + * Run the builder's process. + */ + void runChild(); + + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ + SingleDrvOutputs registerOutputs(); + + /** + * Check that an output meets the requirements specified by the + * 'outputChecks' attribute (or the legacy + * '{allowed,disallowed}{References,Requisites}' attributes). + */ + void checkOutputs(const std::map & outputs); + +public: + + /** + * Delete the temporary directory, if we have one. + */ + void deleteTmpDir(bool force) override; + + /** + * Kill any processes running under the build user UID or in the + * cgroup of the build. + */ + void killSandbox(bool getStats) override; + +private: + + bool cleanupDecideWhetherDiskFull(); + + /** + * Create alternative path calculated from but distinct from the + * input, so we can avoid overwriting outputs (or other store paths) + * that already exist. + */ + StorePath makeFallbackPath(const StorePath & path); + + /** + * Make a path to another based on the output name along with the + * derivation hash. + * + * @todo Add option to randomize, so we can audit whether our + * rewrites caught everything + */ + StorePath makeFallbackPath(OutputNameView outputName); +}; + +std::unique_ptr makeDerivationBuilder( + Store & store, + std::unique_ptr miscMethods, + DerivationBuilderParams params) +{ + return std::make_unique( + store, + std::move(miscMethods), + std::move(params)); +} + void handleDiffHook( uid_t uid, uid_t gid, const Path & tryA, const Path & tryB, @@ -114,20 +425,10 @@ void handleDiffHook( } } -const Path LocalDerivationGoal::homeDir = "/homeless-shelter"; +const Path DerivationBuilderImpl::homeDir = "/homeless-shelter"; -LocalDerivationGoal::~LocalDerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } - try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } - try { stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } -} - - -inline bool LocalDerivationGoal::needsHashRewrite() +inline bool DerivationBuilderImpl::needsHashRewrite() { #ifdef __linux__ return !useChroot; @@ -138,36 +439,15 @@ inline bool LocalDerivationGoal::needsHashRewrite() } -LocalStore & LocalDerivationGoal::getLocalStore() +LocalStore & DerivationBuilderImpl::getLocalStore() { - auto p = dynamic_cast(&worker.store); + auto p = dynamic_cast(&store); assert(p); return *p; } -void LocalDerivationGoal::killChild() -{ - if (pid != -1) { - worker.childTerminated(this); - - /* If we're using a build user, then there is a tricky race - condition: if we kill the build user before the child has - done its setuid() to the build user uid, then it won't be - killed, and we'll potentially lock up in pid.wait(). So - also send a conventional kill to the child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - - killSandbox(true); - - pid.wait(); - } - - DerivationGoal::killChild(); -} - - -void LocalDerivationGoal::killSandbox(bool getStats) +void DerivationBuilderImpl::killSandbox(bool getStats) { if (cgroup) { #ifdef __linux__ @@ -189,39 +469,32 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -Goal::Co LocalDerivationGoal::tryLocalBuild() +bool DerivationBuilderImpl::prepareBuild() { - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); - co_await Suspend{}; - co_return tryToBuild(); - } - - assert(derivationType); + /* Cache this */ + derivationType = drv.type(); /* Are we doing a chroot build? */ { if (settings.sandboxMode == smEnabled) { - if (drvOptions->noChroot) + if (drvOptions.noChroot) throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); + "but that's not allowed when 'sandbox' is 'true'", store.printStorePath(drvPath)); #ifdef __APPLE__ - if (drvOptions->additionalSandboxProfile != "") + if (drvOptions.additionalSandboxProfile != "") throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); + "but this is only allowed when 'sandbox' is 'relaxed'", store.printStorePath(drvPath)); #endif useChroot = true; } else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = derivationType->isSandboxed() && !drvOptions->noChroot; + useChroot = derivationType->isSandboxed() && !drvOptions.noChroot; } auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir.get()) { + if (localStore.storeDir != localStore.config->realStoreDir.get()) { #ifdef __linux__ useChroot = true; #else @@ -242,38 +515,130 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() if (useBuildUsers()) { if (!buildUser) - buildUser = acquireUserLock(drvOptions->useUidRange(*drv) ? 65536 : 1, useChroot); + buildUser = acquireUserLock(drvOptions.useUidRange(drv) ? 65536 : 1, useChroot); if (!buildUser) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - co_await Suspend{}; - co_return tryLocalBuild(); + return false; } } - actLock.reset(); + return true; +} + + +std::variant, SingleDrvOutputs> DerivationBuilderImpl::unprepareBuild() +{ + Finally releaseBuildUser([&](){ + /* Release the build user at the end of this function. We don't do + it right away because we don't want another build grabbing this + uid and then messing around with our output. */ + buildUser.reset(); + }); + + sandboxMountNamespace = -1; + sandboxUserNamespace = -1; + + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = pid.kill(); + + debug("builder process for '%s' finished", store.printStorePath(drvPath)); + + buildResult.timesBuilt++; + buildResult.stopTime = time(0); + + /* So the child is gone now. */ + miscMethods->childTerminated(); + + /* Close the read side of the logger pipe. */ + builderOut.close(); + + /* Close the log file. */ + miscMethods->closeLogFile(); + + /* When running under a build user, make sure that all processes + running under that uid are gone. This is to prevent a + malicious user from leaving behind a process that keeps files + open and modifies them after they have been chown'ed to + root. */ + killSandbox(true); + + /* Terminate the recursive Nix daemon. */ + stopDaemon(); + + if (buildResult.cpuUser && buildResult.cpuSystem) { + debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs", + store.printStorePath(drvPath), + status, + ((double) buildResult.cpuUser->count()) / 1000000, + ((double) buildResult.cpuSystem->count()) / 1000000); + } + + bool diskFull = false; try { - /* Okay, we have to build. */ - startBuilder(); + /* Check the exit status. */ + if (!statusOk(status)) { + + diskFull |= cleanupDecideWhetherDiskFull(); + + auto msg = fmt( + "Cannot build '%s'.\n" + "Reason: " ANSI_RED "builder %s" ANSI_NORMAL ".", + Magenta(store.printStorePath(drvPath)), + statusToString(status)); + + msg += showKnownOutputs(store, drv); + + miscMethods->appendLogTailErrorMsg(msg); + + if (diskFull) + msg += "\nnote: build failure may have been caused by lack of free disk space"; + + throw BuildError(msg); + } + + /* Compute the FS closure of the outputs and register them as + being valid. */ + auto builtOutputs = registerOutputs(); + + StorePathSet outputPaths; + for (auto & [_, output] : builtOutputs) + outputPaths.insert(output.outPath); + runPostBuildHook( + store, + *logger, + drvPath, + outputPaths + ); + + /* Delete unused redirected outputs (when doing hash rewriting). */ + for (auto & i : redirectedOutputs) + deletePath(store.Store::toRealPath(i.second)); + + /* Delete the chroot (if we were using one). */ + autoDelChroot.reset(); /* this runs the destructor */ + + deleteTmpDir(true); + + return std::move(builtOutputs); } catch (BuildError & e) { - outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - co_return done(BuildResult::InputRejected, {}, std::move(e)); - } + assert(derivationType); + BuildResult::Status st = + dynamic_cast(&e) ? BuildResult::NotDeterministic : + statusOk(status) ? BuildResult::OutputRejected : + !derivationType->isSandboxed() || diskFull ? BuildResult::TransientFailure : + BuildResult::PermanentFailure; - started(); - co_await Suspend{}; - // after EOF on child - co_return buildDone(); + return std::pair{std::move(st), std::move(e)}; + } } + static void chmod_(const Path & path, mode_t mode) { if (chmod(path.c_str(), mode) == -1) @@ -300,54 +665,35 @@ static void movePath(const Path & src, const Path & dst) } -extern void replaceValidPath(const Path & storePath, const Path & tmpPath); - - -int LocalDerivationGoal::getChildStatus() +static void replaceValidPath(const Path & storePath, const Path & tmpPath) { - return hook ? DerivationGoal::getChildStatus() : pid.kill(); -} + /* We can't atomically replace storePath (the original) with + tmpPath (the replacement), so we have to move it out of the + way first. We'd better not be interrupted here, because if + we're repairing (say) Glibc, we end up with a broken system. */ + Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), rand()); + if (pathExists(storePath)) + movePath(storePath, oldPath); -void LocalDerivationGoal::closeReadPipes() -{ - if (hook) { - DerivationGoal::closeReadPipes(); - } else - builderOut.close(); + try { + movePath(tmpPath, storePath); + } catch (...) { + try { + // attempt to recover + movePath(oldPath, storePath); + } catch (...) { + ignoreExceptionExceptInterrupt(); + } + throw; + } + + deletePath(oldPath); } -void LocalDerivationGoal::cleanupHookFinally() -{ - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - buildUser.reset(); -} -void LocalDerivationGoal::cleanupPreChildKill() -{ - sandboxMountNamespace = -1; - sandboxUserNamespace = -1; -} - - -void LocalDerivationGoal::cleanupPostChildKill() -{ - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - killSandbox(true); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); -} - - -bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() +bool DerivationBuilderImpl::cleanupDecideWhetherDiskFull() { bool diskFull = false; @@ -361,7 +707,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() auto & localStore = getLocalStore(); uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; - if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && + if (statvfs(localStore.config->realStoreDir.get().c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && @@ -378,7 +724,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() for (auto & [_, status] : initialOutputs) { if (!status.known) continue; if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.toRealPath(status.known->path); + auto p = store.toRealPath(status.known->path); if (pathExists(chrootRootDir + p)) std::filesystem::rename((chrootRootDir + p), p); } @@ -387,24 +733,6 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() } -void LocalDerivationGoal::cleanupPostOutputsRegisteredModeCheck() -{ - deleteTmpDir(true); -} - - -void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() -{ - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - cleanupPostOutputsRegisteredModeCheck(); -} - #ifdef __linux__ static void doBind(const Path & source, const Path & target, bool optional = false) { debug("bind mounting '%1%' to '%2%'", source, target); @@ -458,7 +786,7 @@ static void rethrowExceptionAsError() /** * Send the current exception to the parent in the format expected by - * `LocalDerivationGoal::processSandboxSetupMessages()`. + * `DerivationBuilderImpl::processSandboxSetupMessages()`. */ static void handleChildException(bool sendException) { @@ -475,7 +803,7 @@ static void handleChildException(bool sendException) } } -void LocalDerivationGoal::startBuilder() +void DerivationBuilderImpl::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) #ifdef __linux__ @@ -533,17 +861,17 @@ void LocalDerivationGoal::startBuilder() killSandbox(false); /* Right platform? */ - if (!drvOptions->canBuildLocally(worker.store, *drv)) { + if (!drvOptions.canBuildLocally(store, drv)) { // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - we should tell them to run the command to install Darwin 2 - if (drv->platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") { - throw Error("run `/usr/sbin/softwareupdate --install-rosetta` to enable your %s to run programs for %s", settings.thisSystem, drv->platform); + if (drv.platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") { + throw Error("run `/usr/sbin/softwareupdate --install-rosetta` to enable your %s to run programs for %s", settings.thisSystem, drv.platform); } else { throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)), - worker.store.printStorePath(drvPath), + drv.platform, + concatStringsSep(", ", drvOptions.getRequiredSystemFeatures(drv)), + store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); + concatStringsSep(", ", store.config.systemFeatures)); } } @@ -601,7 +929,7 @@ void LocalDerivationGoal::startBuilder() /* Substitute output placeholders with the scratch output paths. We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); + inputRewrites[hashPlaceholder(outputName)] = store.printStorePath(scratchPath); /* Additional tasks if we know the final path a priori. */ if (!status.known) continue; @@ -612,7 +940,7 @@ void LocalDerivationGoal::startBuilder() if (fixedFinalPath == scratchPath) continue; /* Ensure scratch path is ours to use. */ - deletePath(worker.store.printStorePath(scratchPath)); + deletePath(store.printStorePath(scratchPath)); /* Rewrite and unrewrite paths */ { @@ -630,33 +958,18 @@ void LocalDerivationGoal::startBuilder() writeStructuredAttrs(); /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->hasStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - auto s = getOr(drv->env, "exportReferencesGraph", ""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - auto fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - + if (!parsedDrv) { + for (auto & [fileName, ss] : drvOptions.exportReferencesGraph) { + StorePathSet storePathSet; + for (auto & storePathS : ss) { + if (!store.isInStore(storePathS)) + throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); + storePathSet.insert(store.toStorePath(storePathS).first); + } /* Write closure info to . */ writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration( - worker.store.exportReferences({storePath}, inputPaths), false, false)); + store.makeValidityRegistration( + store.exportReferences(storePathSet, inputPaths), false, false)); } } @@ -679,7 +992,7 @@ void LocalDerivationGoal::startBuilder() else pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; } - if (hasPrefix(worker.store.storeDir, tmpDirInSandbox)) + if (hasPrefix(store.storeDir, tmpDirInSandbox)) { throw Error("`sandbox-build-dir` must not contain the storeDir"); } @@ -689,22 +1002,22 @@ void LocalDerivationGoal::startBuilder() StorePathSet closure; for (auto & i : pathsInChroot) try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); + if (store.isInStore(i.second.source)) + store.computeFSClosure(store.toStorePath(i.second.source).first, closure); } catch (InvalidPath & e) { } catch (Error & e) { e.addTrace({}, "while processing 'sandbox-paths'"); throw; } for (auto & i : closure) { - auto p = worker.store.printStorePath(i); + auto p = store.printStorePath(i); pathsInChroot.insert_or_assign(p, p); } PathSet allowedPaths = settings.allowedImpureHostPrefixes; /* This works like the above, except on a per-derivation level */ - auto impurePaths = drvOptions->impureHostDeps; + auto impurePaths = drvOptions.impureHostDeps; for (auto & i : impurePaths) { bool found = false; @@ -715,16 +1028,16 @@ void LocalDerivationGoal::startBuilder() /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ for (auto & a : allowedPaths) { Path canonA = canonPath(a); - if (canonI == canonA || isInDir(canonI, canonA)) { + if (isDirOrInDir(canonI, canonA)) { found = true; break; } } if (!found) throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - worker.store.printStorePath(drvPath), i); + store.printStorePath(drvPath), i); - /* Allow files in drvOptions->impureHostDeps to be missing; e.g. + /* Allow files in drvOptions.impureHostDeps to be missing; e.g. macOS 11+ has no /usr/lib/libSystem*.dylib */ pathsInChroot[i] = {i, true}; } @@ -734,7 +1047,7 @@ void LocalDerivationGoal::startBuilder() environment using bind-mounts. We put it in the Nix store so that the build outputs can be moved efficiently from the chroot to their final location. */ - chrootParentDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; + auto chrootParentDir = store.Store::toRealPath(drvPath) + ".chroot"; deletePath(chrootParentDir); /* Clean up the chroot directory automatically. */ @@ -764,10 +1077,10 @@ void LocalDerivationGoal::startBuilder() nobody account. The latter is kind of a hack to support Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); - if (drvOptions->useUidRange(*drv)) + if (drvOptions.useUidRange(drv)) chownToBuilder(chrootRootDir + "/etc"); - if (drvOptions->useUidRange(*drv) && (!buildUser || buildUser->getUIDCount() < 65536)) + if (drvOptions.useUidRange(drv) && (!buildUser || buildUser->getUIDCount() < 65536)) throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); /* Declare the build user's group so that programs get a consistent @@ -788,7 +1101,7 @@ void LocalDerivationGoal::startBuilder() can be bind-mounted). !!! As an extra security precaution, make the fake Nix store only writable by the build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + Path chrootStoreDir = chrootRootDir + store.storeDir; createDirs(chrootStoreDir); chmod_(chrootStoreDir, 01775); @@ -796,8 +1109,8 @@ void LocalDerivationGoal::startBuilder() throw SysError("cannot change ownership of '%1%'", chrootStoreDir); for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - Path r = worker.store.toRealPath(p); + auto p = store.printStorePath(i); + Path r = store.toRealPath(p); pathsInChroot.insert_or_assign(p, r); } @@ -806,14 +1119,14 @@ void LocalDerivationGoal::startBuilder() rebuilding a path that is in settings.sandbox-paths (typically the dependencies of /bin/sh). Throw them out. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { + for (auto & i : drv.outputsAndOptPaths(store)) { /* If the name isn't known a priori (i.e. floating content-addressing derivation), the temporary location we use should be fresh. Freshness means it is impossible that the path is already in the sandbox, so we don't need to worry about removing it. */ if (i.second.second) - pathsInChroot.erase(worker.store.printStorePath(*i.second.second)); + pathsInChroot.erase(store.printStorePath(*i.second.second)); } if (cgroup) { @@ -826,7 +1139,7 @@ void LocalDerivationGoal::startBuilder() } #else - if (drvOptions->useUidRange(*drv)) + if (drvOptions.useUidRange(drv)) throw Error("feature 'uid-range' is not supported on this platform"); #ifdef __APPLE__ /* We don't really have any parent prep work to do (yet?) @@ -836,17 +1149,17 @@ void LocalDerivationGoal::startBuilder() #endif #endif } else { - if (drvOptions->useUidRange(*drv)) + if (drvOptions.useUidRange(drv)) throw Error("feature 'uid-range' is only supported in sandboxed builds"); } if (needsHashRewrite() && pathExists(homeDir)) throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { + if (useChroot && settings.preBuildHook != "") { printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); - auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : - Strings({ worker.store.printStorePath(drvPath) }); + auto args = useChroot ? Strings({store.printStorePath(drvPath), chrootRootDir}) : + Strings({ store.printStorePath(drvPath) }); enum BuildHookState { stBegin, stExtraChrootDirs @@ -881,17 +1194,17 @@ void LocalDerivationGoal::startBuilder() /* Fire up a Nix daemon to process recursive Nix calls from the builder. */ - if (drvOptions->getRequiredSystemFeatures(*drv).count("recursive-nix")) + if (drvOptions.getRequiredSystemFeatures(drv).count("recursive-nix")) startDaemon(); /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args)); - for (auto & i : drv->env) + printMsg(lvlChatty, "executing builder '%1%'", drv.builder); + printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv.args)); + for (auto & i : drv.env) printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); /* Create the log file. */ - [[maybe_unused]] Path logFile = openLogFile(); + [[maybe_unused]] Path logFile = miscMethods->openLogFile(); /* Create a pseudoterminal to get the output of the builder. */ builderOut = posix_openpt(O_RDWR | O_NOCTTY); @@ -979,9 +1292,6 @@ void LocalDerivationGoal::startBuilder() us. */ - if (derivationType->isSandboxed()) - privateNetwork = true; - userNamespaceSync.create(); usingUserNamespace = userNamespacesSupported(); @@ -1009,7 +1319,7 @@ void LocalDerivationGoal::startBuilder() ProcessOptions options; options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) + if (derivationType->isSandboxed()) options.cloneFlags |= CLONE_NEWNET; if (usingUserNamespace) options.cloneFlags |= CLONE_NEWUSER; @@ -1104,13 +1414,13 @@ void LocalDerivationGoal::startBuilder() /* parent */ pid.setSeparatePG(true); - worker.childStarted(shared_from_this(), {builderOut.get()}, true, true); + miscMethods->childStarted(builderOut.get()); processSandboxSetupMessages(); } -void LocalDerivationGoal::processSandboxSetupMessages() +void DerivationBuilderImpl::processSandboxSetupMessages() { std::vector msgs; while (true) { @@ -1120,7 +1430,7 @@ void LocalDerivationGoal::processSandboxSetupMessages() } catch (Error & e) { auto status = pid.wait(); e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", - worker.store.printStorePath(drvPath), + store.printStorePath(drvPath), statusToString(status), concatStringsSep("|", msgs)); throw; @@ -1139,7 +1449,7 @@ void LocalDerivationGoal::processSandboxSetupMessages() } -void LocalDerivationGoal::initTmpDir() +void DerivationBuilderImpl::initTmpDir() { /* In a sandbox, for determinism, always use the same temporary directory. */ @@ -1152,9 +1462,9 @@ void LocalDerivationGoal::initTmpDir() /* In non-structured mode, set all bindings either directory in the environment or via a file, as specified by `DerivationOptions::passAsFile`. */ - if (!parsedDrv->hasStructuredAttrs()) { - for (auto & i : drv->env) { - if (drvOptions->passAsFile.find(i.first) == drvOptions->passAsFile.end()) { + if (!parsedDrv) { + for (auto & i : drv.env) { + if (drvOptions.passAsFile.find(i.first) == drvOptions.passAsFile.end()) { env[i.first] = i.second; } else { auto hash = hashString(HashAlgorithm::SHA256, i.first); @@ -1183,7 +1493,7 @@ void LocalDerivationGoal::initTmpDir() } -void LocalDerivationGoal::initEnv() +void DerivationBuilderImpl::initEnv() { env.clear(); @@ -1204,7 +1514,7 @@ void LocalDerivationGoal::initEnv() shouldn't care, but this is useful for purity checking (e.g., the compiler or linker might only want to accept paths to files in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; + env["NIX_STORE"] = store.storeDir; /* The maximum number of cores to utilize for parallel building. */ env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores); @@ -1231,7 +1541,7 @@ void LocalDerivationGoal::initEnv() if (!impureEnv.empty()) experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv); - for (auto & i : drvOptions->impureEnvVars){ + for (auto & i : drvOptions.impureEnvVars){ auto envVar = impureEnv.find(i); if (envVar != impureEnv.end()) { env[i] = envVar->second; @@ -1251,10 +1561,14 @@ void LocalDerivationGoal::initEnv() } -void LocalDerivationGoal::writeStructuredAttrs() +void DerivationBuilderImpl::writeStructuredAttrs() { - if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, inputPaths)) { - auto json = structAttrsJson.value(); + if (parsedDrv) { + auto json = parsedDrv->prepareStructuredAttrs( + store, + drvOptions, + inputPaths, + drv.outputs); nlohmann::json rewritten; for (auto & [i, v] : json["outputs"].get()) { /* The placeholder must have a rewrite, so we use it to cover both the @@ -1264,7 +1578,7 @@ void LocalDerivationGoal::writeStructuredAttrs() json["outputs"] = rewritten; - auto jsonSh = writeStructuredAttrsShell(json); + auto jsonSh = StructuredAttrs::writeShell(json); writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); chownToBuilder(tmpDir + "/.attrs.sh"); @@ -1276,276 +1590,19 @@ void LocalDerivationGoal::writeStructuredAttrs() } -static StorePath pathPartOfReq(const SingleDerivedPath & req) -{ - return std::visit(overloaded { - [&](const SingleDerivedPath::Opaque & bo) { - return bo.path; - }, - [&](const SingleDerivedPath::Built & bfd) { - return pathPartOfReq(*bfd.drvPath); - }, - }, req.raw()); -} - - -static StorePath pathPartOfReq(const DerivedPath & req) -{ - return std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) { - return bo.path; - }, - [&](const DerivedPath::Built & bfd) { - return pathPartOfReq(*bfd.drvPath); - }, - }, req.raw()); -} - - -bool LocalDerivationGoal::isAllowed(const DerivedPath & req) -{ - return this->isAllowed(pathPartOfReq(req)); -} - - -struct RestrictedStoreConfig : virtual LocalFSStoreConfig -{ - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() override { return "Restricted Store"; } -}; - -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore -{ - ref next; - - LocalDerivationGoal & goal; - - RestrictedStore(const Params & params, ref next, LocalDerivationGoal & goal) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RestrictedStoreConfig(params) - , Store(params) - , LocalFSStore(params) - , next(next), goal(goal) - { } - - Path getRealStoreDir() override - { return next->realStoreDir; } - - std::string getUri() override - { return next->getUri(); } - - StorePathSet queryAllValidPaths() override - { - StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); - return paths; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - if (goal.isAllowed(path)) { - try { - /* Censor impure information. */ - auto info = std::make_shared(*next->queryPathInfo(path)); - info->deriver.reset(); - info->registrationTime = 0; - info->ultimate = false; - info->sigs.clear(); - callback(info); - } catch (InvalidPath &) { - callback(nullptr); - } - } else - callback(nullptr); - }; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override - { } - - std::map> queryPartialDerivationOutputMap( - const StorePath & path, - Store * evalStore = nullptr) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path, evalStore); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { throw Error("queryPathFromHashPart"); } - - StorePath addToStore( - std::string_view name, - const SourcePath & srcPath, - ContentAddressMethod method, - HashAlgorithm hashAlgo, - const StorePathSet & references, - PathFilter & filter, - RepairFlag repair) override - { throw Error("addToStore"); } - - void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override - { - next->addToStore(info, narSource, repair, checkSigs); - goal.addDependency(info.path); - } - - StorePath addToStoreFromDump( - Source & dump, - std::string_view name, - FileSerialisationMethod dumpMethod, - ContentAddressMethod hashMethod, - HashAlgorithm hashAlgo, - const StorePathSet & references, - RepairFlag repair) override - { - auto path = next->addToStoreFromDump(dump, name, dumpMethod, hashMethod, hashAlgo, references, repair); - goal.addDependency(path); - return path; - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); - } - - void ensurePath(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); - /* Nothing to be done; 'path' must already be valid. */ - } - - void registerDrvOutput(const Realisation & info) override - // XXX: This should probably be allowed as a no-op if the realisation - // corresponds to an allowed derivation - { throw Error("registerDrvOutput"); } - - void queryRealisationUncached(const DrvOutput & id, - Callback> callback) noexcept override - // XXX: This should probably be allowed if the realisation corresponds to - // an allowed derivation - { - if (!goal.isAllowed(id)) - callback(nullptr); - next->queryRealisation(id, std::move(callback)); - } - - void buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override - { - for (auto & result : buildPathsWithResults(paths, buildMode, evalStore)) - if (!result.success()) - result.rethrow(); - } - - std::vector buildPathsWithResults( - const std::vector & paths, - BuildMode buildMode = bmNormal, - std::shared_ptr evalStore = nullptr) override - { - assert(!evalStore); - - if (buildMode != bmNormal) throw Error("unsupported build mode"); - - StorePathSet newPaths; - std::set newRealisations; - - for (auto & req : paths) { - if (!goal.isAllowed(req)) - throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next)); - } - - auto results = next->buildPathsWithResults(paths, buildMode); - - for (auto & result : results) { - for (auto & [outputName, output] : result.builtOutputs) { - newPaths.insert(output.outPath); - newRealisations.insert(output); - } - } - - StorePathSet closure; - next->computeFSClosure(newPaths, closure); - for (auto & path : closure) - goal.addDependency(path); - for (auto & real : Realisation::closure(*next, newRealisations)) - goal.addedDrvOutputs.insert(real.id); - - return results; - } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override - { unsupported("buildDerivation"); } - - void addTempRoot(const StorePath & path) override - { } - - void addIndirectRoot(const Path & path) override - { } - - Roots findRoots(bool censor) override - { return Roots(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { } - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override - { unsupported("addSignatures"); } - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override - { - /* This is slightly impure since it leaks information to the - client about what paths will be built/substituted or are - already present. Probably not a big deal. */ - - std::vector allowed; - for (auto & req : targets) { - if (goal.isAllowed(req)) - allowed.emplace_back(req); - else - unknown.insert(pathPartOfReq(req)); - } - - next->queryMissing(allowed, willBuild, willSubstitute, - unknown, downloadSize, narSize); - } - - virtual std::optional getBuildLogExact(const StorePath & path) override - { return std::nullopt; } - - virtual void addBuildLog(const StorePath & path, std::string_view log) override - { unsupported("addBuildLog"); } - - std::optional isTrustedClient() override - { return NotTrusted; } -}; - - -void LocalDerivationGoal::startDaemon() +void DerivationBuilderImpl::startDaemon() { experimentalFeatureSettings.require(Xp::RecursiveNix); - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - if (auto & optRoot = getLocalStore().rootDir.get()) - params["root"] = *optRoot; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = make_ref(params, - ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), + auto store = makeRestrictedStore( + [&]{ + auto config = make_ref(*getLocalStore().config); + config->pathInfoCacheSize = 0; + config->stateDir = "/no-such-path"; + config->logDir = "/no-such-path"; + return config; + }(), + ref(std::dynamic_pointer_cast(this->store.shared_from_this())), *this); addedPaths.clear(); @@ -1601,7 +1658,7 @@ void LocalDerivationGoal::startDaemon() } -void LocalDerivationGoal::stopDaemon() +void DerivationBuilderImpl::stopDaemon() { if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) { // According to the POSIX standard, the 'shutdown' function should @@ -1634,7 +1691,7 @@ void LocalDerivationGoal::stopDaemon() } -void LocalDerivationGoal::addDependency(const StorePath & path) +void DerivationBuilderImpl::addDependency(const StorePath & path) { if (isAllowed(path)) return; @@ -1644,17 +1701,17 @@ void LocalDerivationGoal::addDependency(const StorePath & path) appear in the sandbox. */ if (useChroot) { - debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); + debug("materialising '%s' in the sandbox", store.printStorePath(path)); #ifdef __linux__ - Path source = worker.store.Store::toRealPath(path); - Path target = chrootRootDir + worker.store.printStorePath(path); + Path source = store.Store::toRealPath(path); + Path target = chrootRootDir + store.printStorePath(path); if (pathExists(target)) { // There is a similar debug message in doBind, so only run it in this block to not have double messages. debug("bind-mounting %s -> %s", target, source); - throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); + throw Error("store path '%s' already exists in the sandbox", store.printStorePath(path)); } /* Bind-mount the path into the sandbox. This requires @@ -1676,17 +1733,17 @@ void LocalDerivationGoal::addDependency(const StorePath & path) int status = child.wait(); if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); + throw Error("could not add path '%s' to sandbox", store.printStorePath(path)); #else throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - worker.store.printStorePath(path)); + store.printStorePath(path)); #endif } } -void LocalDerivationGoal::chownToBuilder(const Path & path) +void DerivationBuilderImpl::chownToBuilder(const Path & path) { if (!buildUser) return; if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) @@ -1782,7 +1839,7 @@ void setupSeccomp() } -void LocalDerivationGoal::runChild() +void DerivationBuilderImpl::runChild() { /* Warning: in the child we should absolutely not make any SQLite calls! */ @@ -1804,15 +1861,18 @@ void LocalDerivationGoal::runChild() /* Make the contents of netrc and the CA certificate bundle available to builtin:fetchurl (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - std::string caFileData; - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") { + BuiltinBuilderContext ctx{ + .drv = drv, + .tmpDirInSandbox = tmpDirInSandbox, + }; + + if (drv.isBuiltin() && drv.builder == "builtin:fetchurl") { try { - netrcData = readFile(settings.netrcFile); + ctx.netrcData = readFile(settings.netrcFile); } catch (SystemError &) { } try { - caFileData = readFile(settings.caFile); + ctx.caFileData = readFile(settings.caFile); } catch (SystemError &) { } } @@ -1826,7 +1886,7 @@ void LocalDerivationGoal::runChild() userNamespaceSync.readSide = -1; - if (privateNetwork) { + if (derivationType->isSandboxed()) { /* Initialise the loopback interface. */ AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); @@ -1871,7 +1931,7 @@ void LocalDerivationGoal::runChild() Marking chrootRootDir as MS_SHARED causes pivot_root() to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + Path chrootStoreDir = chrootRootDir + store.storeDir; if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) throw SysError("unable to bind mount the Nix store", chrootStoreDir); @@ -1886,7 +1946,7 @@ void LocalDerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + if (store.config.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); ss.push_back("/dev/null"); ss.push_back("/dev/random"); @@ -1916,9 +1976,10 @@ void LocalDerivationGoal::runChild() if (pathExists(path)) ss.push_back(path); - if (settings.caFile != "" && pathExists(settings.caFile)) { + if (settings.caFile != "") { Path caFile = settings.caFile; - pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true); + if (pathExists(caFile)) + pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true); } } @@ -1991,7 +2052,7 @@ void LocalDerivationGoal::runChild() } /* Make /etc unwritable */ - if (!drvOptions->useUidRange(*drv)) + if (!drvOptions.useUidRange(drv)) chmod_(chrootRootDir + "/etc", 0555); /* Unshare this mount namespace. This is necessary because @@ -2051,7 +2112,7 @@ void LocalDerivationGoal::runChild() unix::closeExtraFDs(); #ifdef __linux__ - linux::setPersonality(drv->platform); + linux::setPersonality(drv.platform); #endif /* Disable core dumps by default. */ @@ -2111,7 +2172,7 @@ void LocalDerivationGoal::runChild() /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; + Path cur = store.storeDir; while (cur.compare("/") != 0) { ancestry.insert(cur); cur = dirOf(cur); @@ -2119,7 +2180,7 @@ void LocalDerivationGoal::runChild() /* Add all our input paths to the chroot */ for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); + auto p = store.printStorePath(i); pathsInChroot[p] = p; } @@ -2142,7 +2203,7 @@ void LocalDerivationGoal::runChild() /* Add the output paths we'll use at build-time to the chroot */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); + sandboxProfile += fmt("\t(subpath \"%s\")\n", store.printStorePath(path)); sandboxProfile += ")\n"; @@ -2155,7 +2216,7 @@ void LocalDerivationGoal::runChild() // We create multiple allow lists, to avoid exceeding a limit in the darwin sandbox interpreter. // See https://github.com/NixOS/nix/issues/4119 // We split our allow groups approximately at half the actual limit, 1 << 16 - const int breakpoint = sandboxProfile.length() + (1 << 14); + const size_t breakpoint = sandboxProfile.length() + (1 << 14); for (auto & i : pathsInChroot) { if (sandboxProfile.length() >= breakpoint) { @@ -2189,7 +2250,7 @@ void LocalDerivationGoal::runChild() } sandboxProfile += ")\n"; - sandboxProfile += drvOptions->additionalSandboxProfile; + sandboxProfile += drvOptions.additionalSandboxProfile; } else sandboxProfile += #include "sandbox-minimal.sb" @@ -2210,7 +2271,7 @@ void LocalDerivationGoal::runChild() Strings sandboxArgs; sandboxArgs.push_back("_GLOBAL_TMP_DIR"); sandboxArgs.push_back(globalTmpDir); - if (drvOptions->allowLocalNetworking) { + if (drvOptions.allowLocalNetworking) { sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); sandboxArgs.push_back("1"); } @@ -2228,23 +2289,20 @@ void LocalDerivationGoal::runChild() sendException = false; /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { + if (drv.isBuiltin()) { try { logger = makeJSONLogger(getStandardError()); - std::map outputs; - for (auto & e : drv->outputs) - outputs.insert_or_assign(e.first, - worker.store.printStorePath(scratchOutputs.at(e.first))); + for (auto & e : drv.outputs) + ctx.outputs.insert_or_assign(e.first, + store.printStorePath(scratchOutputs.at(e.first))); - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(*drv, outputs, netrcData, caFileData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(*drv, outputs); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(*drv, outputs); + std::string builtinName = drv.builder.substr(8); + assert(RegisterBuiltinBuilder::builtinBuilders); + if (auto builtin = get(RegisterBuiltinBuilder::builtinBuilders(), builtinName)) + (*builtin)(ctx); else - throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8)); + throw Error("unsupported builtin builder '%1%'", builtinName); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, e.what() + std::string("\n")); @@ -2255,9 +2313,9 @@ void LocalDerivationGoal::runChild() // Now builder is not builtin Strings args; - args.push_back(std::string(baseNameOf(drv->builder))); + args.push_back(std::string(baseNameOf(drv.builder))); - for (auto & i : drv->args) + for (auto & i : drv.args) args.push_back(rewriteStrings(i, inputRewrites)); #ifdef __APPLE__ @@ -2269,24 +2327,24 @@ void LocalDerivationGoal::runChild() if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) throw SysError("failed to initialize builder"); - if (drv->platform == "aarch64-darwin") { + if (drv.platform == "aarch64-darwin") { // Unset kern.curproc_arch_affinity so we can escape Rosetta int affinity = 0; sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); cpu_type_t cpu = CPU_TYPE_ARM64; posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); - } else if (drv->platform == "x86_64-darwin") { + } else if (drv.platform == "x86_64-darwin") { cpu_type_t cpu = CPU_TYPE_X86_64; posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); } - posix_spawn(NULL, drv->builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); + posix_spawn(NULL, drv.builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); #else - execve(drv->builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); + execve(drv.builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); #endif - throw SysError("executing '%1%'", drv->builder); + throw SysError("executing '%1%'", drv.builder); } catch (...) { handleChildException(sendException); @@ -2295,18 +2353,8 @@ void LocalDerivationGoal::runChild() } -SingleDrvOutputs LocalDerivationGoal::registerOutputs() +SingleDrvOutputs DerivationBuilderImpl::registerOutputs() { - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. - - We can only early return when the outputs are known a priori. For - floating content-addressing derivations this isn't the case. - */ - if (hook) - return DerivationGoal::registerOutputs(); - std::map infos; /* Set of inodes seen during calls to canonicalisePathMetaData() @@ -2331,7 +2379,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto toRealPathChroot = [&](const Path & p) -> Path { return useChroot && !needsHashRewrite() ? chrootRootDir + p - : worker.store.toRealPath(p); + : store.toRealPath(p); }; /* Check whether the output paths were created, and make all @@ -2344,13 +2392,13 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() struct PerhapsNeedToRegister { StorePathSet refs; }; std::map> outputReferencesIfUnregistered; std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { + for (auto & [outputName, _] : drv.outputs) { auto scratchOutput = get(scratchOutputs, outputName); if (!scratchOutput) throw BuildError( "builder for '%s' has no scratch output for '%s'", - worker.store.printStorePath(drvPath), outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchOutput)); + store.printStorePath(drvPath), outputName); + auto actualPath = toRealPathChroot(store.printStorePath(*scratchOutput)); outputsToSort.insert(outputName); @@ -2359,7 +2407,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (!initialOutput) throw BuildError( "builder for '%s' has no initial output for '%s'", - worker.store.printStorePath(drvPath), outputName); + store.printStorePath(drvPath), outputName); auto & initialInfo = *initialOutput; /* Don't register if already valid, and not checking */ @@ -2376,7 +2424,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (!optSt) throw BuildError( "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); + store.printStorePath(drvPath), outputName, actualPath); struct stat & st = *optSt; #ifndef __CYGWIN__ @@ -2400,7 +2448,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() inodesSeen); bool discardReferences = false; - if (auto udr = get(drvOptions->unsafeDiscardReferences, outputName)) { + if (auto udr = get(drvOptions.unsafeDiscardReferences, outputName)) { discardReferences = *udr; } @@ -2427,7 +2475,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (!orifu) throw BuildError( "no output reference for '%s' in build of '%s'", - name, worker.store.printStorePath(drvPath)); + name, store.printStorePath(drvPath)); return std::visit(overloaded { /* Since we'll use the already installed versions of these, we can treat them as leaves and ignore any references they @@ -2448,7 +2496,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() // TODO with more -vvvv also show the temporary paths for manual inspection. return BuildError( "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - worker.store.printStorePath(drvPath), path, parent); + store.printStorePath(drvPath), path, parent); }}); std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); @@ -2456,10 +2504,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() OutputPathMap finalOutputs; for (auto & outputName : sortedOutputNames) { - auto output = get(drv->outputs, outputName); + auto output = get(drv.outputs, outputName); auto scratchPath = get(scratchOutputs, outputName); assert(output && scratchPath); - auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchPath)); + auto actualPath = toRealPathChroot(store.printStorePath(*scratchPath)); auto finish = [&](StorePath finalStorePath) { /* Store the final path */ @@ -2577,8 +2625,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() }(); ValidPathInfo newInfo0 { - worker.store, - outputPathName(drv->name, outputName), + store, + outputPathName(drv.name, outputName), ContentAddressWithReferences::fromParts( outputHash.method, std::move(got), @@ -2655,26 +2703,28 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (wanted != got) { /* Throw an error after registering the path as valid. */ - worker.hashMismatch = true; + miscMethods->noteHashMismatch(); delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", - worker.store.printStorePath(drvPath), + store.printStorePath(drvPath), wanted.to_string(HashFormat::SRI, true), got.to_string(HashFormat::SRI, true))); +#if 0 // FIXME act->result(resHashMismatch, { - {"storePath", worker.store.printStorePath(drvPath)}, + {"storePath", store.printStorePath(drvPath)}, {"wanted", wanted}, {"got", got}, }); +#endif } if (!newInfo0.references.empty()) { auto numViolations = newInfo.references.size(); delayedException = std::make_exception_ptr( BuildError("fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", - worker.store.printStorePath(drvPath), + store.printStorePath(drvPath), numViolations, - worker.store.printStorePath(*newInfo.references.begin()))); + store.printStorePath(*newInfo.references.begin()))); } return newInfo0; @@ -2706,36 +2756,36 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() /* Calculate where we'll move the output files. In the checking case we will leave leave them where they are, for now, rather than move to their usual "final destination" */ - auto finalDestPath = worker.store.printStorePath(newInfo.path); + auto finalDestPath = store.printStorePath(newInfo.path); /* Lock final output path, if not already locked. This happens with floating CA derivations and hash-mismatching fixed-output derivations. */ PathLocks dynamicOutputLock; dynamicOutputLock.setDeletion(true); - auto optFixedPath = output->path(worker.store, drv->name, outputName); + auto optFixedPath = output->path(store, drv.name, outputName); if (!optFixedPath || - worker.store.printStorePath(*optFixedPath) != finalDestPath) + store.printStorePath(*optFixedPath) != finalDestPath) { assert(newInfo.ca); - dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); + dynamicOutputLock.lockPaths({store.toRealPath(finalDestPath)}); } /* Move files, if needed */ - if (worker.store.toRealPath(finalDestPath) != actualPath) { + if (store.toRealPath(finalDestPath) != actualPath) { if (buildMode == bmRepair) { /* Path already exists, need to replace it */ - replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); - actualPath = worker.store.toRealPath(finalDestPath); + replaceValidPath(store.toRealPath(finalDestPath), actualPath); + actualPath = store.toRealPath(finalDestPath); } else if (buildMode == bmCheck) { /* Path already exists, and we want to compare, so we leave out new path in place. */ - } else if (worker.store.isValidPath(newInfo.path)) { + } else if (store.isValidPath(newInfo.path)) { /* Path already exists because CA path produced by something else. No moving needed. */ assert(newInfo.ca); } else { - auto destPath = worker.store.toRealPath(finalDestPath); + auto destPath = store.toRealPath(finalDestPath); deletePath(destPath); movePath(actualPath, destPath); actualPath = destPath; @@ -2746,25 +2796,25 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (buildMode == bmCheck) { - if (!worker.store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); + if (!store.isValidPath(newInfo.path)) continue; + ValidPathInfo oldInfo(*store.queryPathInfo(newInfo.path)); if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; + miscMethods->noteCheckMismatch(); if (settings.runDiffHook || settings.keepFailed) { - auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); + auto dst = store.toRealPath(finalDestPath + checkSuffix); deletePath(dst); movePath(actualPath, dst); handleDiffHook( buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); + finalDestPath, dst, store.printStorePath(drvPath), tmpDir); throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); + store.printStorePath(drvPath), store.toRealPath(finalDestPath), dst); } else throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); + store.printStorePath(drvPath), store.toRealPath(finalDestPath)); } /* Since we verified the build, it's now ultimately trusted. */ @@ -2780,13 +2830,13 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() /* For debugging, print out the referenced and unreferenced paths. */ for (auto & i : inputPaths) { if (references.count(i)) - debug("referenced input: '%1%'", worker.store.printStorePath(i)); + debug("referenced input: '%1%'", store.printStorePath(i)); else - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); + debug("unreferenced input: '%1%'", store.printStorePath(i)); } localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); + miscMethods->markContentsGood(newInfo.path); newInfo.deriver = drvPath; newInfo.ultimate = true; @@ -2809,7 +2859,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() also a source for non-determinism. */ if (delayedException) std::rethrow_exception(delayedException); - return assertPathValidity(); + return miscMethods->assertPathValidity(); } /* Apply output checks. */ @@ -2851,10 +2901,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() .outPath = newInfo.path }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) - && !drv->type().isImpure()) + && !drv.type().isImpure()) { - signRealisation(thisRealisation); - worker.store.registerDrvOutput(thisRealisation); + store.signRealisation(thisRealisation); + store.registerDrvOutput(thisRealisation); } builtOutputs.emplace(outputName, thisRealisation); } @@ -2862,17 +2912,12 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() return builtOutputs; } -void LocalDerivationGoal::signRealisation(Realisation & realisation) -{ - getLocalStore().signRealisation(realisation); -} - -void LocalDerivationGoal::checkOutputs(const std::map & outputs) +void DerivationBuilderImpl::checkOutputs(const std::map & outputs) { std::map outputsByPath; for (auto & output : outputs) - outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); + outputsByPath.emplace(store.printStorePath(output.second.path), output.second); for (auto & output : outputs) { auto & outputName = output.first; @@ -2893,13 +2938,13 @@ void LocalDerivationGoal::checkOutputs(const std::mapsecond.narSize; for (auto & ref : i->second.references) pathsLeft.push(ref); } else { - auto info = worker.store.queryPathInfo(path); + auto info = store.queryPathInfo(path); closureSize += info->narSize; for (auto & ref : info->references) pathsLeft.push(ref); @@ -2913,13 +2958,13 @@ void LocalDerivationGoal::checkOutputs(const std::map *checks.maxSize) throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); + store.printStorePath(info.path), info.narSize, *checks.maxSize); if (checks.maxClosureSize) { uint64_t closureSize = getClosure(info.path).second; if (closureSize > *checks.maxClosureSize) throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); + store.printStorePath(info.path), closureSize, *checks.maxClosureSize); } auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive) @@ -2929,15 +2974,15 @@ void LocalDerivationGoal::checkOutputs(const std::mappath); else { std::string outputsListing = concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; }); throw BuildError("derivation '%s' output check for '%s' contains an illegal reference specifier '%s'," " expected store path or output name (one of [%s])", - worker.store.printStorePath(drvPath), outputName, i, outputsListing); + store.printStorePath(drvPath), outputName, i, outputsListing); } } @@ -2963,10 +3008,10 @@ void LocalDerivationGoal::checkOutputs(const std::mapoutputChecks); + }, drvOptions.outputChecks); } } -void LocalDerivationGoal::deleteTmpDir(bool force) +void DerivationBuilderImpl::deleteTmpDir(bool force) { if (topTmpDir != "") { /* Don't keep temporary directories for builtins because they might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { + if (settings.keepFailed && !force && !drv.isBuiltin()) { printError("note: keeping build directory '%s'", tmpDir); chmod(topTmpDir.c_str(), 0755); chmod(tmpDir.c_str(), 0755); @@ -3021,32 +3066,25 @@ void LocalDerivationGoal::deleteTmpDir(bool force) } -bool LocalDerivationGoal::isReadDesc(int fd) -{ - return (hook && DerivationGoal::isReadDesc(fd)) || - (!hook && fd == builderOut.get()); -} - - -StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) +StorePath DerivationBuilderImpl::makeFallbackPath(OutputNameView outputName) { // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path // See doc/manual/source/protocols/store-path.md for details // TODO: We may want to separate the responsibilities of constructing the path fingerprint and of actually doing the hashing auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName); - return worker.store.makeStorePath( + return store.makeStorePath( pathType, // pass an all-zeroes hash - Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName)); + Hash(HashAlgorithm::SHA256), outputPathName(drv.name, outputName)); } -StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) +StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path) { // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path // See doc/manual/source/protocols/store-path.md for details auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()); - return worker.store.makeStorePath( + return store.makeStorePath( pathType, // pass an all-zeroes hash Hash(HashAlgorithm::SHA256), path.name()); diff --git a/src/libstore/unix/include/nix/store/build/derivation-builder.hh b/src/libstore/unix/include/nix/store/build/derivation-builder.hh new file mode 100644 index 000000000..d6c40060a --- /dev/null +++ b/src/libstore/unix/include/nix/store/build/derivation-builder.hh @@ -0,0 +1,209 @@ +#pragma once +///@file + +#include "nix/store/build-result.hh" +#include "nix/store/derivation-options.hh" +#include "nix/store/build/derivation-building-misc.hh" +#include "nix/store/derivations.hh" +#include "nix/store/parsed-derivations.hh" +#include "nix/util/processes.hh" +#include "nix/store/restricted-store.hh" +#include "nix/store/user-lock.hh" + +namespace nix { + +/** + * Parameters by (mostly) `const` reference for `DerivationBuilder`. + */ +struct DerivationBuilderParams +{ + /** The path of the derivation. */ + const StorePath & drvPath; + + BuildResult & buildResult; + + /** + * The derivation stored at drvPath. + */ + const Derivation & drv; + + /** + * The "structured attrs" of `drv`, if it has them. + * + * @todo this should be part of `Derivation`. + * + * @todo this should be renamed from `parsedDrv`. + */ + const StructuredAttrs * parsedDrv; + + /** + * The derivation options of `drv`. + * + * @todo this should be part of `Derivation`. + */ + const DerivationOptions & drvOptions; + + // The remainder is state held during the build. + + /** + * All input paths (that is, the union of FS closures of the + * immediate input paths). + */ + const StorePathSet & inputPaths; + + /** + * @note we do in fact mutate this + */ + std::map & initialOutputs; + + const BuildMode & buildMode; + + DerivationBuilderParams( + const StorePath & drvPath, + const BuildMode & buildMode, + BuildResult & buildResult, + const Derivation & drv, + const StructuredAttrs * parsedDrv, + const DerivationOptions & drvOptions, + const StorePathSet & inputPaths, + std::map & initialOutputs) + : drvPath{drvPath} + , buildResult{buildResult} + , drv{drv} + , parsedDrv{parsedDrv} + , drvOptions{drvOptions} + , inputPaths{inputPaths} + , initialOutputs{initialOutputs} + , buildMode{buildMode} + { } + + DerivationBuilderParams(DerivationBuilderParams &&) = default; +}; + +/** + * Callbacks that `DerivationBuilder` needs. + */ +struct DerivationBuilderCallbacks +{ + virtual ~DerivationBuilderCallbacks() = default; + + /** + * Open a log file and a pipe to it. + */ + virtual Path openLogFile() = 0; + + /** + * Close the log file. + */ + virtual void closeLogFile() = 0; + + /** + * Aborts if any output is not valid or corrupt, and otherwise + * returns a 'SingleDrvOutputs' structure containing all outputs. + * + * @todo Probably should just be in `DerivationGoal`. + */ + virtual SingleDrvOutputs assertPathValidity() = 0; + + virtual void appendLogTailErrorMsg(std::string & msg) = 0; + + /** + * Hook up `builderOut` to some mechanism to ingest the log + * + * @todo this should be reworked + */ + virtual void childStarted(Descriptor builderOut) = 0; + + /** + * @todo this should be reworked + */ + virtual void childTerminated() = 0; + + virtual void noteHashMismatch(void) = 0; + virtual void noteCheckMismatch(void) = 0; + + virtual void markContentsGood(const StorePath & path) = 0; +}; + +/** + * This class represents the state for building locally. + * + * @todo Ideally, it would not be a class, but a single function. + * However, besides the main entry point, there are a few more methods + * which are externally called, and need to be gotten rid of. There are + * also some virtual methods (either directly here or inherited from + * `DerivationBuilderCallbacks`, a stop-gap) that represent outgoing + * rather than incoming call edges that either should be removed, or + * become (higher order) function parameters. + */ +struct DerivationBuilder : RestrictionContext +{ + /** + * User selected for running the builder. + */ + std::unique_ptr buildUser; + + /** + * The process ID of the builder. + */ + Pid pid; + + DerivationBuilder() = default; + virtual ~DerivationBuilder() = default; + + /** + * Master side of the pseudoterminal used for the builder's + * standard output/error. + */ + AutoCloseFD builderOut; + + /** + * Set up build environment / sandbox, acquiring resources (e.g. + * locks as needed). After this is run, the builder should be + * started. + * + * @returns true if successful, false if we could not acquire a build + * user. In that case, the caller must wait and then try again. + */ + virtual bool prepareBuild() = 0; + + /** + * Start building a derivation. + */ + virtual void startBuilder() = 0; + + /** + * Tear down build environment after the builder exits (either on + * its own or if it is killed). + * + * @returns The first case indicates failure during output + * processing. A status code and exception are returned, providing + * more information. The second case indicates success, and + * realisations for each output of the derivation are returned. + */ + virtual std::variant, SingleDrvOutputs> unprepareBuild() = 0; + + /** + * Stop the in-process nix daemon thread. + * @see startDaemon + */ + virtual void stopDaemon() = 0; + + /** + * Delete the temporary directory, if we have one. + */ + virtual void deleteTmpDir(bool force) = 0; + + /** + * Kill any processes running under the build user UID or in the + * cgroup of the build. + */ + virtual void killSandbox(bool getStats) = 0; +}; + +std::unique_ptr makeDerivationBuilder( + Store & store, + std::unique_ptr miscMethods, + DerivationBuilderParams params); + +} diff --git a/src/libstore/unix/include/nix/store/build/local-derivation-goal.hh b/src/libstore/unix/include/nix/store/build/local-derivation-goal.hh deleted file mode 100644 index 795286a01..000000000 --- a/src/libstore/unix/include/nix/store/build/local-derivation-goal.hh +++ /dev/null @@ -1,319 +0,0 @@ -#pragma once -///@file - -#include "nix/store/build/derivation-goal.hh" -#include "nix/store/local-store.hh" -#include "nix/util/processes.hh" - -namespace nix { - -struct LocalDerivationGoal : public DerivationGoal -{ - LocalStore & getLocalStore(); - - /** - * User selected for running the builder. - */ - std::unique_ptr buildUser; - - /** - * The process ID of the builder. - */ - Pid pid; - - /** - * The cgroup of the builder, if any. - */ - std::optional cgroup; - - /** - * The temporary directory used for the build. - */ - Path tmpDir; - - /** - * The top-level temporary directory. `tmpDir` is either equal to - * or a child of this directory. - */ - Path topTmpDir; - - /** - * The path of the temporary directory in the sandbox. - */ - Path tmpDirInSandbox; - - /** - * Master side of the pseudoterminal used for the builder's - * standard output/error. - */ - AutoCloseFD builderOut; - - /** - * Pipe for synchronising updates to the builder namespaces. - */ - Pipe userNamespaceSync; - - /** - * The mount namespace and user namespace of the builder, used to add additional - * paths to the sandbox as a result of recursive Nix calls. - */ - AutoCloseFD sandboxMountNamespace; - AutoCloseFD sandboxUserNamespace; - - /** - * On Linux, whether we're doing the build in its own user - * namespace. - */ - bool usingUserNamespace = true; - - /** - * Whether we're currently doing a chroot build. - */ - bool useChroot = false; - - /** - * The parent directory of `chrootRootDir`. It has permission 700 - * and is owned by root to ensure other users cannot mess with - * `chrootRootDir`. - */ - Path chrootParentDir; - - /** - * The root of the chroot environment. - */ - Path chrootRootDir; - - /** - * RAII object to delete the chroot directory. - */ - std::shared_ptr autoDelChroot; - - /** - * Whether to run the build in a private network namespace. - */ - bool privateNetwork = false; - - /** - * Stuff we need to pass to initChild(). - */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map PathsInChroot; // maps target path to source path - PathsInChroot pathsInChroot; - - typedef map Environment; - Environment env; - - /** - * Hash rewriting. - */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /** - * The output paths used during the build. - * - * - Input-addressed derivations or fixed content-addressed outputs are - * sometimes built when some of their outputs already exist, and can not - * be hidden via sandboxing. We use temporary locations instead and - * rewrite after the build. Otherwise the regular predetermined paths are - * put here. - * - * - Floating content-addressing derivations do not know their final build - * output paths until the outputs are hashed, so random locations are - * used, and then renamed. The randomness helps guard against hidden - * self-references. - */ - OutputPathMap scratchOutputs; - - uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } - - const static Path homeDir; - - /** - * The recursive Nix daemon socket. - */ - AutoCloseFD daemonSocket; - - /** - * The daemon main thread. - */ - std::thread daemonThread; - - /** - * The daemon worker threads. - */ - std::vector daemonWorkerThreads; - - /** - * Paths that were added via recursive Nix calls. - */ - StorePathSet addedPaths; - - /** - * Realisations that were added via recursive Nix calls. - */ - std::set addedDrvOutputs; - - /** - * Recursive Nix calls are only allowed to build or realize paths - * in the original input closure or added via a recursive Nix call - * (so e.g. you can't do 'nix-store -r /nix/store/' where - * /nix/store/ is some arbitrary path in a binary cache). - */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - bool isAllowed(const DrvOutput & id) - { - return addedDrvOutputs.count(id); - } - - bool isAllowed(const DerivedPath & req); - - friend struct RestrictedStore; - - using DerivationGoal::DerivationGoal; - - virtual ~LocalDerivationGoal() override; - - /** - * Whether we need to perform hash rewriting if there are valid output paths. - */ - bool needsHashRewrite(); - - /** - * The additional states. - */ - Goal::Co tryLocalBuild() override; - - /** - * Start building a derivation. - */ - void startBuilder(); - - /** - * Fill in the environment for the builder. - */ - void initEnv(); - - /** - * Process messages send by the sandbox initialization. - */ - void processSandboxSetupMessages(); - - /** - * Setup tmp dir location. - */ - void initTmpDir(); - - /** - * Write a JSON file containing the derivation attributes. - */ - void writeStructuredAttrs(); - - /** - * Start an in-process nix daemon thread for recursive-nix. - */ - void startDaemon(); - - /** - * Stop the in-process nix daemon thread. - * @see startDaemon - */ - void stopDaemon(); - - /** - * Add 'path' to the set of paths that may be referenced by the - * outputs, and make it appear in the sandbox. - */ - void addDependency(const StorePath & path); - - /** - * Make a file owned by the builder. - */ - void chownToBuilder(const Path & path); - - int getChildStatus() override; - - /** - * Run the builder's process. - */ - void runChild(); - - /** - * Check that the derivation outputs all exist and register them - * as valid. - */ - SingleDrvOutputs registerOutputs() override; - - void signRealisation(Realisation &) override; - - /** - * Check that an output meets the requirements specified by the - * 'outputChecks' attribute (or the legacy - * '{allowed,disallowed}{References,Requisites}' attributes). - */ - void checkOutputs(const std::map & outputs); - - /** - * Close the read side of the logger pipe. - */ - void closeReadPipes() override; - - /** - * Cleanup hooks for buildDone() - */ - void cleanupHookFinally() override; - void cleanupPreChildKill() override; - void cleanupPostChildKill() override; - bool cleanupDecideWhetherDiskFull() override; - void cleanupPostOutputsRegisteredModeCheck() override; - void cleanupPostOutputsRegisteredModeNonCheck() override; - - bool isReadDesc(int fd) override; - - /** - * Delete the temporary directory, if we have one. - */ - void deleteTmpDir(bool force); - - /** - * Forcibly kill the child process, if any. - * - * Called by destructor, can't be overridden - */ - void killChild() override final; - - /** - * Kill any processes running under the build user UID or in the - * cgroup of the build. - */ - void killSandbox(bool getStats); - - /** - * Create alternative path calculated from but distinct from the - * input, so we can avoid overwriting outputs (or other store paths) - * that already exist. - */ - StorePath makeFallbackPath(const StorePath & path); - - /** - * Make a path to another based on the output name along with the - * derivation hash. - * - * @todo Add option to randomize, so we can audit whether our - * rewrites caught everything - */ - StorePath makeFallbackPath(OutputNameView outputName); -}; - -} diff --git a/src/libstore/unix/include/nix/store/meson.build b/src/libstore/unix/include/nix/store/meson.build index 9f12440cd..7cf973223 100644 --- a/src/libstore/unix/include/nix/store/meson.build +++ b/src/libstore/unix/include/nix/store/meson.build @@ -2,7 +2,7 @@ include_dirs += include_directories('../..') headers += files( 'build/child.hh', + 'build/derivation-builder.hh', 'build/hook-instance.hh', - 'build/local-derivation-goal.hh', 'user-lock.hh', ) diff --git a/src/libstore/unix/meson.build b/src/libstore/unix/meson.build index f06c9aa95..4b8a6b105 100644 --- a/src/libstore/unix/meson.build +++ b/src/libstore/unix/meson.build @@ -1,7 +1,7 @@ sources += files( 'build/child.cc', + 'build/derivation-builder.cc', 'build/hook-instance.cc', - 'build/local-derivation-goal.cc', 'pathlocks.cc', 'user-lock.cc', ) diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index d83be10e6..d07dc8163 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -5,7 +5,7 @@ namespace nix { -const std::set WorkerProto::allFeatures{}; +const WorkerProto::FeatureSet WorkerProto::allFeatures{}; WorkerProto::BasicClientConnection::~BasicClientConnection() { @@ -146,21 +146,20 @@ void WorkerProto::BasicClientConnection::processStderr( } } -static std::set -intersectFeatures(const std::set & a, const std::set & b) +static WorkerProto::FeatureSet intersectFeatures(const WorkerProto::FeatureSet & a, const WorkerProto::FeatureSet & b) { - std::set res; + WorkerProto::FeatureSet res; for (auto & x : a) if (b.contains(x)) res.insert(x); return res; } -std::tuple> WorkerProto::BasicClientConnection::handshake( +std::tuple WorkerProto::BasicClientConnection::handshake( BufferedSink & to, Source & from, WorkerProto::Version localVersion, - const std::set & supportedFeatures) + const WorkerProto::FeatureSet & supportedFeatures) { to << WORKER_MAGIC_1 << localVersion; to.flush(); @@ -178,21 +177,21 @@ std::tuple> WorkerProto::Ba auto protoVersion = std::min(daemonVersion, localVersion); /* Exchange features. */ - std::set daemonFeatures; + WorkerProto::FeatureSet daemonFeatures; if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { to << supportedFeatures; to.flush(); - daemonFeatures = readStrings>(from); + daemonFeatures = readStrings(from); } return {protoVersion, intersectFeatures(daemonFeatures, supportedFeatures)}; } -std::tuple> WorkerProto::BasicServerConnection::handshake( +std::tuple WorkerProto::BasicServerConnection::handshake( BufferedSink & to, Source & from, WorkerProto::Version localVersion, - const std::set & supportedFeatures) + const WorkerProto::FeatureSet & supportedFeatures) { unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_1) @@ -204,9 +203,9 @@ std::tuple> WorkerProto::Ba auto protoVersion = std::min(clientVersion, localVersion); /* Exchange features. */ - std::set clientFeatures; + WorkerProto::FeatureSet clientFeatures; if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { - clientFeatures = readStrings>(from); + clientFeatures = readStrings(from); to << supportedFeatures; to.flush(); } diff --git a/src/libutil-test-support/include/nix/util/tests/nix_api_util.hh b/src/libutil-test-support/include/nix/util/tests/nix_api_util.hh index 006dc497c..382c7b292 100644 --- a/src/libutil-test-support/include/nix/util/tests/nix_api_util.hh +++ b/src/libutil-test-support/include/nix/util/tests/nix_api_util.hh @@ -3,6 +3,7 @@ #include "nix_api_util.h" #include +#include namespace nixC { @@ -24,7 +25,12 @@ protected: nix_c_context * ctx; - inline void assert_ctx_ok() + inline std::string loc(const char * file, int line) + { + return std::string(file) + ":" + std::to_string(line); + } + + inline void assert_ctx_ok(const char * file, int line) { if (nix_err_code(ctx) == NIX_OK) { return; @@ -32,16 +38,18 @@ protected: unsigned int n; const char * p = nix_err_msg(nullptr, ctx, &n); std::string msg(p, n); - throw std::runtime_error(std::string("nix_err_code(ctx) != NIX_OK, message: ") + msg); + throw std::runtime_error(loc(file, line) + ": nix_err_code(ctx) != NIX_OK, message: " + msg); } +#define assert_ctx_ok() assert_ctx_ok(__FILE__, __LINE__) - inline void assert_ctx_err() + inline void assert_ctx_err(const char * file, int line) { if (nix_err_code(ctx) != NIX_OK) { return; } - throw std::runtime_error("Got NIX_OK, but expected an error!"); + throw std::runtime_error(loc(file, line) + ": Got NIX_OK, but expected an error!"); } +#define assert_ctx_err() assert_ctx_err(__FILE__, __LINE__) }; } diff --git a/src/libutil-tests/args.cc b/src/libutil-tests/args.cc index 2cc1a3438..f5ad43a55 100644 --- a/src/libutil-tests/args.cc +++ b/src/libutil-tests/args.cc @@ -9,7 +9,7 @@ namespace nix { TEST(parseShebangContent, basic) { std::list r = parseShebangContent("hi there"); - ASSERT_EQ(r.size(), 2); + ASSERT_EQ(r.size(), 2u); auto i = r.begin(); ASSERT_EQ(*i++, "hi"); ASSERT_EQ(*i++, "there"); @@ -17,26 +17,26 @@ namespace nix { TEST(parseShebangContent, empty) { std::list r = parseShebangContent(""); - ASSERT_EQ(r.size(), 0); + ASSERT_EQ(r.size(), 0u); } TEST(parseShebangContent, doubleBacktick) { std::list r = parseShebangContent("``\"ain't that nice\"``"); - ASSERT_EQ(r.size(), 1); + ASSERT_EQ(r.size(), 1u); auto i = r.begin(); ASSERT_EQ(*i++, "\"ain't that nice\""); } TEST(parseShebangContent, doubleBacktickEmpty) { std::list r = parseShebangContent("````"); - ASSERT_EQ(r.size(), 1); + ASSERT_EQ(r.size(), 1u); auto i = r.begin(); ASSERT_EQ(*i++, ""); } TEST(parseShebangContent, doubleBacktickMarkdownInlineCode) { std::list r = parseShebangContent("``# I'm markdown section about `coolFunction` ``"); - ASSERT_EQ(r.size(), 1); + ASSERT_EQ(r.size(), 1u); auto i = r.begin(); ASSERT_EQ(*i++, "# I'm markdown section about `coolFunction`"); } @@ -44,49 +44,49 @@ namespace nix { TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockNaive) { std::list r = parseShebangContent("``Example 1\n```nix\na: a\n``` ``"); auto i = r.begin(); - ASSERT_EQ(r.size(), 1); + ASSERT_EQ(r.size(), 1u); ASSERT_EQ(*i++, "Example 1\n``nix\na: a\n``"); } TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockCorrect) { std::list r = parseShebangContent("``Example 1\n````nix\na: a\n```` ``"); auto i = r.begin(); - ASSERT_EQ(r.size(), 1); + ASSERT_EQ(r.size(), 1u); ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```"); } TEST(parseShebangContent, doubleBacktickMarkdownCodeBlock2) { std::list r = parseShebangContent("``Example 1\n````nix\na: a\n````\nExample 2\n````nix\na: a\n```` ``"); auto i = r.begin(); - ASSERT_EQ(r.size(), 1); + ASSERT_EQ(r.size(), 1u); ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```\nExample 2\n```nix\na: a\n```"); } TEST(parseShebangContent, singleBacktickInDoubleBacktickQuotes) { std::list r = parseShebangContent("``` ``"); auto i = r.begin(); - ASSERT_EQ(r.size(), 1); + ASSERT_EQ(r.size(), 1u); ASSERT_EQ(*i++, "`"); } TEST(parseShebangContent, singleBacktickAndSpaceInDoubleBacktickQuotes) { std::list r = parseShebangContent("``` ``"); auto i = r.begin(); - ASSERT_EQ(r.size(), 1); + ASSERT_EQ(r.size(), 1u); ASSERT_EQ(*i++, "` "); } TEST(parseShebangContent, doubleBacktickInDoubleBacktickQuotes) { std::list r = parseShebangContent("````` ``"); auto i = r.begin(); - ASSERT_EQ(r.size(), 1); + ASSERT_EQ(r.size(), 1u); ASSERT_EQ(*i++, "``"); } TEST(parseShebangContent, increasingQuotes) { std::list r = parseShebangContent("```` ``` `` ````` `` `````` ``"); auto i = r.begin(); - ASSERT_EQ(r.size(), 4); + ASSERT_EQ(r.size(), 4u); ASSERT_EQ(*i++, ""); ASSERT_EQ(*i++, "`"); ASSERT_EQ(*i++, "``"); @@ -146,7 +146,7 @@ RC_GTEST_PROP( auto escaped = escape(orig); // RC_LOG() << "escaped: <[[" << escaped << "]]>" << std::endl; auto ss = parseShebangContent(escaped); - RC_ASSERT(ss.size() == 1); + RC_ASSERT(ss.size() == 1u); RC_ASSERT(*ss.begin() == orig); } @@ -156,7 +156,7 @@ RC_GTEST_PROP( (const std::string & one, const std::string & two)) { auto ss = parseShebangContent(escape(one) + " " + escape(two)); - RC_ASSERT(ss.size() == 2); + RC_ASSERT(ss.size() == 2u); auto i = ss.begin(); RC_ASSERT(*i++ == one); RC_ASSERT(*i++ == two); diff --git a/src/libutil-tests/chunked-vector.cc b/src/libutil-tests/chunked-vector.cc index 658581c2a..c4f1d3858 100644 --- a/src/libutil-tests/chunked-vector.cc +++ b/src/libutil-tests/chunked-vector.cc @@ -5,12 +5,12 @@ namespace nix { TEST(ChunkedVector, InitEmpty) { auto v = ChunkedVector(100); - ASSERT_EQ(v.size(), 0); + ASSERT_EQ(v.size(), 0u); } TEST(ChunkedVector, GrowsCorrectly) { auto v = ChunkedVector(100); - for (auto i = 1; i < 20; i++) { + for (uint32_t i = 1; i < 20; i++) { v.add(i); ASSERT_EQ(v.size(), i); } @@ -31,7 +31,7 @@ namespace nix { for (auto i = 1; i < 20; i++) { v.add(i); } - int count = 0; + uint32_t count = 0; v.forEach([&count](int elt) { count++; }); diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 8c9eccc11..2d1058c4f 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -203,12 +203,10 @@ TEST(isInDir, notInDir) ASSERT_EQ(p1, false); } -// XXX: hm, bug or feature? :) Looking at the implementation -// this might be problematic. TEST(isInDir, emptyDir) { auto p1 = isInDir("/zes/foo/bar", ""); - ASSERT_EQ(p1, true); + ASSERT_EQ(p1, false); } /* ---------------------------------------------------------------------------- @@ -233,14 +231,12 @@ TEST(isDirOrInDir, falseForDisjunctPaths) TEST(isDirOrInDir, relativePaths) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), false); } -// XXX: while it is possible to use "." or ".." in the -// first argument this doesn't seem to work in the second. -TEST(isDirOrInDir, DISABLED_shouldWork) +TEST(isDirOrInDir, relativePathsTwice) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), false); } /* ---------------------------------------------------------------------------- @@ -275,4 +271,51 @@ TEST(makeParentCanonical, root) { ASSERT_EQ(makeParentCanonical("/"), "/"); } + +/* ---------------------------------------------------------------------------- + * chmodIfNeeded + * --------------------------------------------------------------------------*/ + +TEST(chmodIfNeeded, works) +{ + auto [autoClose_, tmpFile] = nix::createTempFile(); + auto deleteTmpFile = AutoDelete(tmpFile); + + auto modes = std::vector{0755, 0644, 0422, 0600, 0777}; + for (mode_t oldMode : modes) { + for (mode_t newMode : modes) { + chmod(tmpFile.c_str(), oldMode); + bool permissionsChanged = false; + ASSERT_NO_THROW(permissionsChanged = chmodIfNeeded(tmpFile, newMode)); + ASSERT_EQ(permissionsChanged, oldMode != newMode); + } + } +} + +TEST(chmodIfNeeded, nonexistent) +{ + ASSERT_THROW(chmodIfNeeded("/schnitzel/darmstadt/pommes", 0755), SysError); +} + +/* ---------------------------------------------------------------------------- + * DirectoryIterator + * --------------------------------------------------------------------------*/ + +TEST(DirectoryIterator, works) +{ + auto tmpDir = nix::createTempDir(); + nix::AutoDelete delTmpDir(tmpDir, true); + + nix::writeFile(tmpDir + "/somefile", ""); + + for (auto path : DirectoryIterator(tmpDir)) { + ASSERT_EQ(path.path().string(), tmpDir + "/somefile"); + } +} + +TEST(DirectoryIterator, nonexistent) +{ + ASSERT_THROW(DirectoryIterator("/schnitzel/darmstadt/pommes"), SysError); +} + } diff --git a/src/libutil-tests/json-utils.cc b/src/libutil-tests/json-utils.cc index 051d86ec7..eae67b4b3 100644 --- a/src/libutil-tests/json-utils.cc +++ b/src/libutil-tests/json-utils.cc @@ -50,7 +50,7 @@ TEST(from_json, optionalInt) { TEST(from_json, vectorOfOptionalInts) { nlohmann::json json = { 420, nullptr }; std::vector> vals = json; - ASSERT_EQ(vals.size(), 2); + ASSERT_EQ(vals.size(), 2u); ASSERT_TRUE(vals.at(0).has_value()); ASSERT_EQ(*vals.at(0), 420); ASSERT_FALSE(vals.at(1).has_value()); @@ -63,9 +63,7 @@ TEST(valueAt, simpleObject) { auto nested = R"({ "hello": { "world": "" } })"_json; - auto & nestedObject = valueAt(getObject(nested), "hello"); - - ASSERT_EQ(valueAt(nestedObject, "world"), ""); + ASSERT_EQ(valueAt(valueAt(getObject(nested), "hello"), "world"), ""); } TEST(valueAt, missingKey) { @@ -83,7 +81,7 @@ TEST(getObject, rightAssertions) { auto nested = R"({ "object": { "object": {} } })"_json; - auto & nestedObject = getObject(valueAt(getObject(nested), "object")); + auto nestedObject = getObject(valueAt(getObject(nested), "object")); ASSERT_EQ(nestedObject, getObject(nlohmann::json::parse(R"({ "object": {} })"))); ASSERT_EQ(getObject(valueAt(getObject(nestedObject), "object")), (nlohmann::json::object_t {})); @@ -130,19 +128,29 @@ TEST(getString, wrongAssertions) { ASSERT_THROW(getString(valueAt(json, "boolean")), Error); } -TEST(getInteger, rightAssertions) { - auto simple = R"({ "int": 0 })"_json; +TEST(getIntegralNumber, rightAssertions) { + auto simple = R"({ "int": 0, "signed": -1 })"_json; - ASSERT_EQ(getInteger(valueAt(getObject(simple), "int")), 0); + ASSERT_EQ(getUnsigned(valueAt(getObject(simple), "int")), 0); + ASSERT_EQ(getInteger(valueAt(getObject(simple), "int")), 0); + ASSERT_EQ(getInteger(valueAt(getObject(simple), "signed")), -1); } -TEST(getInteger, wrongAssertions) { - auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; +TEST(getIntegralNumber, wrongAssertions) { + auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "signed": -256, "large": 128, "boolean": false })"_json; - ASSERT_THROW(getInteger(valueAt(json, "object")), Error); - ASSERT_THROW(getInteger(valueAt(json, "array")), Error); - ASSERT_THROW(getInteger(valueAt(json, "string")), Error); - ASSERT_THROW(getInteger(valueAt(json, "boolean")), Error); + ASSERT_THROW(getUnsigned(valueAt(json, "object")), Error); + ASSERT_THROW(getUnsigned(valueAt(json, "array")), Error); + ASSERT_THROW(getUnsigned(valueAt(json, "string")), Error); + ASSERT_THROW(getUnsigned(valueAt(json, "boolean")), Error); + ASSERT_THROW(getUnsigned(valueAt(json, "signed")), Error); + + ASSERT_THROW(getInteger(valueAt(json, "object")), Error); + ASSERT_THROW(getInteger(valueAt(json, "array")), Error); + ASSERT_THROW(getInteger(valueAt(json, "string")), Error); + ASSERT_THROW(getInteger(valueAt(json, "boolean")), Error); + ASSERT_THROW(getInteger(valueAt(json, "large")), Error); + ASSERT_THROW(getInteger(valueAt(json, "signed")), Error); } TEST(getBoolean, rightAssertions) { diff --git a/src/libutil-tests/lru-cache.cc b/src/libutil-tests/lru-cache.cc index daa2a91fe..a6a27cd3e 100644 --- a/src/libutil-tests/lru-cache.cc +++ b/src/libutil-tests/lru-cache.cc @@ -9,13 +9,13 @@ namespace nix { TEST(LRUCache, sizeOfEmptyCacheIsZero) { LRUCache c(10); - ASSERT_EQ(c.size(), 0); + ASSERT_EQ(c.size(), 0u); } TEST(LRUCache, sizeOfSingleElementCacheIsOne) { LRUCache c(10); c.upsert("foo", "bar"); - ASSERT_EQ(c.size(), 1); + ASSERT_EQ(c.size(), 1u); } /* ---------------------------------------------------------------------------- @@ -55,12 +55,12 @@ namespace nix { auto val = c.get("foo"); ASSERT_EQ(val.value_or("error"), "bar"); - ASSERT_EQ(c.size(), 1); + ASSERT_EQ(c.size(), 1u); c.upsert("foo", "changed"); val = c.get("foo"); ASSERT_EQ(val.value_or("error"), "changed"); - ASSERT_EQ(c.size(), 1); + ASSERT_EQ(c.size(), 1u); } TEST(LRUCache, overwriteOldestWhenCapacityIsReached) { @@ -69,13 +69,13 @@ namespace nix { c.upsert("two", "zwei"); c.upsert("three", "drei"); - ASSERT_EQ(c.size(), 3); + ASSERT_EQ(c.size(), 3u); ASSERT_EQ(c.get("one").value_or("error"), "eins"); // exceed capacity c.upsert("another", "whatever"); - ASSERT_EQ(c.size(), 3); + ASSERT_EQ(c.size(), 3u); // Retrieving "one" makes it the most recent element thus // two will be the oldest one and thus replaced. ASSERT_EQ(c.get("two").has_value(), false); @@ -89,7 +89,7 @@ namespace nix { TEST(LRUCache, clearEmptyCache) { LRUCache c(10); c.clear(); - ASSERT_EQ(c.size(), 0); + ASSERT_EQ(c.size(), 0u); } TEST(LRUCache, clearNonEmptyCache) { @@ -97,9 +97,9 @@ namespace nix { c.upsert("one", "eins"); c.upsert("two", "zwei"); c.upsert("three", "drei"); - ASSERT_EQ(c.size(), 3); + ASSERT_EQ(c.size(), 3u); c.clear(); - ASSERT_EQ(c.size(), 0); + ASSERT_EQ(c.size(), 0u); } /* ---------------------------------------------------------------------------- @@ -109,14 +109,14 @@ namespace nix { TEST(LRUCache, eraseFromEmptyCache) { LRUCache c(10); ASSERT_EQ(c.erase("foo"), false); - ASSERT_EQ(c.size(), 0); + ASSERT_EQ(c.size(), 0u); } TEST(LRUCache, eraseMissingFromNonEmptyCache) { LRUCache c(10); c.upsert("one", "eins"); ASSERT_EQ(c.erase("foo"), false); - ASSERT_EQ(c.size(), 1); + ASSERT_EQ(c.size(), 1u); ASSERT_EQ(c.get("one").value_or("error"), "eins"); } @@ -124,7 +124,7 @@ namespace nix { LRUCache c(10); c.upsert("one", "eins"); ASSERT_EQ(c.erase("one"), true); - ASSERT_EQ(c.size(), 0); + ASSERT_EQ(c.size(), 0u); ASSERT_EQ(c.get("one").value_or("empty"), "empty"); } } diff --git a/src/libutil-tests/pool.cc b/src/libutil-tests/pool.cc index c9f31f9a0..d41bab8ed 100644 --- a/src/libutil-tests/pool.cc +++ b/src/libutil-tests/pool.cc @@ -26,8 +26,8 @@ namespace nix { Pool pool = Pool((size_t)1, createResource, isGood); - ASSERT_EQ(pool.count(), 0); - ASSERT_EQ(pool.capacity(), 1); + ASSERT_EQ(pool.count(), 0u); + ASSERT_EQ(pool.capacity(), 1u); } TEST(Pool, freshPoolCanGetAResource) { @@ -35,12 +35,12 @@ namespace nix { auto createResource = []() { return make_ref(); }; Pool pool = Pool((size_t)1, createResource, isGood); - ASSERT_EQ(pool.count(), 0); + ASSERT_EQ(pool.count(), 0u); TestResource r = *(pool.get()); - ASSERT_EQ(pool.count(), 1); - ASSERT_EQ(pool.capacity(), 1); + ASSERT_EQ(pool.count(), 1u); + ASSERT_EQ(pool.capacity(), 1u); ASSERT_EQ(r.dummyValue, 1); ASSERT_EQ(r.good, true); } @@ -50,9 +50,9 @@ namespace nix { auto createResource = []() { return make_ref(); }; Pool pool = Pool((size_t)1, createResource, isGood); - ASSERT_EQ(pool.capacity(), 1); + ASSERT_EQ(pool.capacity(), 1u); pool.incCapacity(); - ASSERT_EQ(pool.capacity(), 2); + ASSERT_EQ(pool.capacity(), 2u); } TEST(Pool, capacityCanBeDecremented) { @@ -60,9 +60,9 @@ namespace nix { auto createResource = []() { return make_ref(); }; Pool pool = Pool((size_t)1, createResource, isGood); - ASSERT_EQ(pool.capacity(), 1); + ASSERT_EQ(pool.capacity(), 1u); pool.decCapacity(); - ASSERT_EQ(pool.capacity(), 0); + ASSERT_EQ(pool.capacity(), 0u); } TEST(Pool, flushBadDropsOutOfScopeResources) { @@ -73,11 +73,11 @@ namespace nix { { auto _r = pool.get(); - ASSERT_EQ(pool.count(), 1); + ASSERT_EQ(pool.count(), 1u); } pool.flushBad(); - ASSERT_EQ(pool.count(), 0); + ASSERT_EQ(pool.count(), 0u); } // Test that the resources we allocate are being reused when they are still good. diff --git a/src/libutil-tests/suggestions.cc b/src/libutil-tests/suggestions.cc index c58f033da..d21b286c8 100644 --- a/src/libutil-tests/suggestions.cc +++ b/src/libutil-tests/suggestions.cc @@ -34,10 +34,10 @@ namespace nix { TEST(Suggestions, Trim) { auto suggestions = Suggestions::bestMatches({"foooo", "bar", "fo", "gao"}, "foo"); auto onlyOne = suggestions.trim(1); - ASSERT_EQ(onlyOne.suggestions.size(), 1); + ASSERT_EQ(onlyOne.suggestions.size(), 1u); ASSERT_TRUE(onlyOne.suggestions.begin()->suggestion == "fo"); auto closest = suggestions.trim(999, 2); - ASSERT_EQ(closest.suggestions.size(), 3); + ASSERT_EQ(closest.suggestions.size(), 3u); } } diff --git a/src/libutil-tests/terminal.cc b/src/libutil-tests/terminal.cc index 329c1a186..198b20951 100644 --- a/src/libutil-tests/terminal.cc +++ b/src/libutil-tests/terminal.cc @@ -66,4 +66,12 @@ TEST(filterANSIEscapes, osc8) ASSERT_EQ(filterANSIEscapes("\e]8;;http://example.com\e\\This is a link\e]8;;\e\\"), "This is a link"); } +TEST(filterANSIEscapes, osc8_bell_as_sep) +{ + // gcc-14 uses \a as a separator, xterm style: + // https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda + ASSERT_EQ(filterANSIEscapes("\e]8;;http://example.com\aThis is a link\e]8;;\a"), "This is a link"); + ASSERT_EQ(filterANSIEscapes("\e]8;;http://example.com\a\\This is a link\e]8;;\a"), "\\This is a link"); +} + } // namespace nix diff --git a/src/libutil-tests/util.cc b/src/libutil-tests/util.cc index 954867be8..534731c6c 100644 --- a/src/libutil-tests/util.cc +++ b/src/libutil-tests/util.cc @@ -79,7 +79,7 @@ TEST(base64Encode, encodeAndDecodeNonPrintable) auto encoded = base64Encode(s); auto decoded = base64Decode(encoded); - EXPECT_EQ(decoded.length(), 255); + EXPECT_EQ(decoded.length(), 255u); ASSERT_EQ(decoded, s); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 0541291ad..80cd02969 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -593,7 +593,7 @@ MultiCommand::MultiCommand(std::string_view commandName, const Commands & comman assert(!command); auto i = commands.find(s); if (i == commands.end()) { - std::set commandNames; + StringSet commandNames; for (auto & [name, _] : commands) commandNames.insert(name); auto suggestions = Suggestions::bestMatches(commandNames, s); diff --git a/src/libutil/compute-levels.cc b/src/libutil/compute-levels.cc index c80b99404..dd221bd70 100644 --- a/src/libutil/compute-levels.cc +++ b/src/libutil/compute-levels.cc @@ -3,7 +3,8 @@ #include "util-config-private.hh" #if HAVE_LIBCPUID -#include +# include +# include #endif namespace nix { @@ -12,61 +13,21 @@ namespace nix { StringSet computeLevels() { StringSet levels; + struct cpu_id_t data; - if (!cpuid_present()) + const std::map feature_strings = { + { FEATURE_LEVEL_X86_64_V1, "x86_64-v1" }, + { FEATURE_LEVEL_X86_64_V2, "x86_64-v2" }, + { FEATURE_LEVEL_X86_64_V3, "x86_64-v3" }, + { FEATURE_LEVEL_X86_64_V4, "x86_64-v4" }, + }; + + if (cpu_identify(NULL, &data) < 0) return levels; - cpu_raw_data_t raw; - cpu_id_t data; - - if (cpuid_get_raw_data(&raw) < 0) - return levels; - - if (cpu_identify(&raw, &data) < 0) - return levels; - - if (!(data.flags[CPU_FEATURE_CMOV] && - data.flags[CPU_FEATURE_CX8] && - data.flags[CPU_FEATURE_FPU] && - data.flags[CPU_FEATURE_FXSR] && - data.flags[CPU_FEATURE_MMX] && - data.flags[CPU_FEATURE_SSE] && - data.flags[CPU_FEATURE_SSE2])) - return levels; - - levels.insert("x86_64-v1"); - - if (!(data.flags[CPU_FEATURE_CX16] && - data.flags[CPU_FEATURE_LAHF_LM] && - data.flags[CPU_FEATURE_POPCNT] && - // SSE3 - data.flags[CPU_FEATURE_PNI] && - data.flags[CPU_FEATURE_SSSE3] && - data.flags[CPU_FEATURE_SSE4_1] && - data.flags[CPU_FEATURE_SSE4_2])) - return levels; - - levels.insert("x86_64-v2"); - - if (!(data.flags[CPU_FEATURE_AVX] && - data.flags[CPU_FEATURE_AVX2] && - data.flags[CPU_FEATURE_F16C] && - data.flags[CPU_FEATURE_FMA3] && - // LZCNT - data.flags[CPU_FEATURE_ABM] && - data.flags[CPU_FEATURE_MOVBE])) - return levels; - - levels.insert("x86_64-v3"); - - if (!(data.flags[CPU_FEATURE_AVX512F] && - data.flags[CPU_FEATURE_AVX512BW] && - data.flags[CPU_FEATURE_AVX512CD] && - data.flags[CPU_FEATURE_AVX512DQ] && - data.flags[CPU_FEATURE_AVX512VL])) - return levels; - - levels.insert("x86_64-v4"); + for (auto & [level, levelString] : feature_strings) + if (data.feature_level >= level) + levels.insert(levelString); return levels; } diff --git a/src/libutil/config-global.cc b/src/libutil/config-global.cc index 10d176c51..94d715443 100644 --- a/src/libutil/config-global.cc +++ b/src/libutil/config-global.cc @@ -6,7 +6,7 @@ namespace nix { bool GlobalConfig::set(const std::string & name, const std::string & value) { - for (auto & config : *configRegistrations) + for (auto & config : configRegistrations()) if (config->set(name, value)) return true; @@ -17,20 +17,20 @@ bool GlobalConfig::set(const std::string & name, const std::string & value) void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) { - for (auto & config : *configRegistrations) + for (auto & config : configRegistrations()) config->getSettings(res, overriddenOnly); } void GlobalConfig::resetOverridden() { - for (auto & config : *configRegistrations) + for (auto & config : configRegistrations()) config->resetOverridden(); } nlohmann::json GlobalConfig::toJSON() { auto res = nlohmann::json::object(); - for (const auto & config : *configRegistrations) + for (const auto & config : configRegistrations()) res.update(config->toJSON()); return res; } @@ -47,19 +47,15 @@ std::string GlobalConfig::toKeyValue() void GlobalConfig::convertToArgs(Args & args, const std::string & category) { - for (auto & config : *configRegistrations) + for (auto & config : configRegistrations()) config->convertToArgs(args, category); } GlobalConfig globalConfig; -GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; - GlobalConfig::Register::Register(Config * config) { - if (!configRegistrations) - configRegistrations = new ConfigRegistrations; - configRegistrations->emplace_back(config); + configRegistrations().emplace_back(config); } ExperimentalFeatureSettings experimentalFeatureSettings; diff --git a/src/libutil/configuration.cc b/src/libutil/configuration.cc index 08e5919e0..c3c2325dc 100644 --- a/src/libutil/configuration.cc +++ b/src/libutil/configuration.cc @@ -220,7 +220,7 @@ void Config::convertToArgs(Args & args, const std::string & category) AbstractSetting::AbstractSetting( const std::string & name, const std::string & description, - const std::set & aliases, + const StringSet & aliases, std::optional experimentalFeature) : name(name) , description(stripIndentation(description)) @@ -428,7 +428,7 @@ PathSetting::PathSetting(Config * options, const Path & def, const std::string & name, const std::string & description, - const std::set & aliases) + const StringSet & aliases) : BaseSetting(def, true, name, description, aliases) { options->addSetting(this); @@ -444,7 +444,7 @@ OptionalPathSetting::OptionalPathSetting(Config * options, const std::optional & def, const std::string & name, const std::string & description, - const std::set & aliases) + const StringSet & aliases) : BaseSetting>(def, true, name, description, aliases) { options->addSetting(this); diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 926714ae8..4cc5a4218 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -57,7 +57,7 @@ size_t savedStackSize = 0; void setStackSize(size_t stackSize) { struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + if (getrlimit(RLIMIT_STACK, &limit) == 0 && static_cast(limit.rlim_cur) < stackSize) { savedStackSize = limit.rlim_cur; limit.rlim_cur = std::min(static_cast(stackSize), limit.rlim_max); if (setrlimit(RLIMIT_STACK, &limit) != 0) { diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index ed1ac49ce..75ab91f3a 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -6,10 +6,6 @@ namespace nix { -namespace fs { -using namespace std::filesystem; -} - constexpr static const OsStringView path_var_separator{ &ExecutablePath::separator, 1, @@ -28,7 +24,7 @@ ExecutablePath ExecutablePath::parse(const OsString & path) auto strings = path.empty() ? (std::list{}) : basicSplitString, OsChar>(path, path_var_separator); - std::vector ret; + std::vector ret; ret.reserve(strings.size()); std::transform( @@ -36,7 +32,7 @@ ExecutablePath ExecutablePath::parse(const OsString & path) std::make_move_iterator(strings.end()), std::back_inserter(ret), [](OsString && str) { - return fs::path{ + return std::filesystem::path{ str.empty() // "A zero-length prefix is a legacy feature that // indicates the current working directory. It @@ -62,13 +58,13 @@ OsString ExecutablePath::render() const return basicConcatStringsSep(path_var_separator, path2); } -std::optional -ExecutablePath::findName(const OsString & exe, std::function isExecutable) const +std::optional +ExecutablePath::findName(const OsString & exe, std::function isExecutable) const { // "If the pathname being sought contains a , the search // through the path prefixes shall not be performed." // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 - assert(OsPathTrait::rfindPathSep(exe) == exe.npos); + assert(OsPathTrait::rfindPathSep(exe) == exe.npos); for (auto & dir : directories) { auto candidate = dir / exe; @@ -79,7 +75,8 @@ ExecutablePath::findName(const OsString & exe, std::function isExecutable) const +std::filesystem::path ExecutablePath::findPath( + const std::filesystem::path & exe, std::function isExecutable) const { // "If the pathname being sought contains a , the search // through the path prefixes shall not be performed." diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 14116aa0c..be829b92f 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -348,7 +348,7 @@ nlohmann::json documentExperimentalFeatures() return (nlohmann::json) res; } -std::set parseFeatures(const std::set & rawFeatures) +std::set parseFeatures(const StringSet & rawFeatures) { std::set res; for (auto & rawFeature : rawFeatures) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 6fb797103..90ec5eda5 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -21,6 +21,8 @@ #include #include +#include + #ifdef _WIN32 # include #endif @@ -31,21 +33,37 @@ namespace nix { -namespace fs { - using namespace std::filesystem; +DirectoryIterator::DirectoryIterator(const std::filesystem::path& p) { + try { + // **Attempt to create the underlying directory_iterator** + it_ = std::filesystem::directory_iterator(p); + } catch (const std::filesystem::filesystem_error& e) { + // **Catch filesystem_error and throw SysError** + // Adapt the error message as needed for SysError + throw SysError("cannot read directory %s", p); + } +} - bool symlink_exists(const std::filesystem::path & path) { - try { - return std::filesystem::exists(std::filesystem::symlink_status(path)); - } catch (const std::filesystem::filesystem_error & e) { - throw SysError("cannot check existence of %1%", path); - } - } +DirectoryIterator& DirectoryIterator::operator++() { + // **Attempt to increment the underlying iterator** + std::error_code ec; + it_.increment(ec); + if (ec) { + // Try to get path info if possible, might fail if iterator is bad + try { + if (it_ != std::filesystem::directory_iterator{}) { + throw SysError("cannot read directory past %s: %s", it_->path(), ec.message()); + } + } catch (...) { + throw SysError("cannot read directory"); + } + } + return *this; } bool isAbsolute(PathView path) { - return fs::path { path }.is_absolute(); + return std::filesystem::path { path }.is_absolute(); } @@ -94,7 +112,7 @@ Path canonPath(PathView path, bool resolveSymlinks) throw Error("not an absolute path: '%1%'", path); // For Windows - auto rootName = fs::path { path }.root_name(); + auto rootName = std::filesystem::path { path }.root_name(); /* This just exists because we cannot set the target of `remaining` (the callback parameter) directly to a newly-constructed string, @@ -109,7 +127,7 @@ Path canonPath(PathView path, bool resolveSymlinks) path, [&followCount, &temp, maxFollow, resolveSymlinks] (std::string & result, std::string_view & remaining) { - if (resolveSymlinks && fs::is_symlink(result)) { + if (resolveSymlinks && std::filesystem::is_symlink(result)) { if (++followCount >= maxFollow) throw Error("infinite symlink recursion in path '%1%'", remaining); remaining = (temp = concatStrings(readLink(result), remaining)); @@ -138,7 +156,7 @@ Path dirOf(const PathView path) Path::size_type pos = OsPathTrait::rfindPathSep(path); if (pos == path.npos) return "."; - return fs::path{path}.parent_path().string(); + return std::filesystem::path{path}.parent_path().string(); } @@ -161,16 +179,18 @@ std::string_view baseNameOf(std::string_view path) } -bool isInDir(std::string_view path, std::string_view dir) +bool isInDir(const std::filesystem::path & path, const std::filesystem::path & dir) { - return path.substr(0, 1) == "/" - && path.substr(0, dir.size()) == dir - && path.size() >= dir.size() + 2 - && path[dir.size()] == '/'; + /* Note that while the standard doesn't guarantee this, the + `lexically_*` functions should do no IO and not throw. */ + auto rel = path.lexically_relative(dir); + /* Method from + https://stackoverflow.com/questions/62503197/check-if-path-contains-another-in-c++ */ + return !rel.empty() && rel.native()[0] != OS_STR('.'); } -bool isDirOrInDir(std::string_view path, std::string_view dir) +bool isDirOrInDir(const std::filesystem::path & path, const std::filesystem::path & dir) { return path == dir || isInDir(path, dir); } @@ -213,9 +233,9 @@ std::optional maybeLstat(const Path & path) } -bool pathExists(const Path & path) +bool pathExists(const std::filesystem::path & path) { - return maybeLstat(path).has_value(); + return maybeLstat(path.string()).has_value(); } bool pathAccessible(const std::filesystem::path & path) @@ -233,7 +253,7 @@ bool pathAccessible(const std::filesystem::path & path) Path readLink(const Path & path) { checkInterrupt(); - return fs::read_symlink(path).string(); + return std::filesystem::read_symlink(path).string(); } @@ -255,9 +275,22 @@ std::string readFile(const std::filesystem::path & path) return readFile(os_string_to_string(PathViewNG { path })); } - -void readFile(const Path & path, Sink & sink) +void readFile(const Path & path, Sink & sink, bool memory_map) { + // Memory-map the file for faster processing where possible. + if (memory_map) { + try { + boost::iostreams::mapped_file_source mmap(path); + if (mmap.is_open()) { + sink({mmap.data(), mmap.size()}); + return; + } + } catch (const boost::exception & e) { + } + debug("memory-mapping failed for path: %s", path); + } + + // Stream the file instead if memory-mapping fails or is disabled. AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY // TODO #ifndef _WIN32 @@ -351,17 +384,17 @@ void recursiveSync(const Path & path) /* Otherwise, perform a depth-first traversal of the directory and fsync all the files. */ - std::deque dirsToEnumerate; + std::deque dirsToEnumerate; dirsToEnumerate.push_back(path); - std::vector dirsToFsync; + std::vector dirsToFsync; while (!dirsToEnumerate.empty()) { auto currentDir = dirsToEnumerate.back(); dirsToEnumerate.pop_back(); - for (auto & entry : std::filesystem::directory_iterator(currentDir)) { + for (auto & entry : DirectoryIterator(currentDir)) { auto st = entry.symlink_status(); - if (fs::is_directory(st)) { + if (std::filesystem::is_directory(st)) { dirsToEnumerate.emplace_back(entry.path()); - } else if (fs::is_regular_file(st)) { + } else if (std::filesystem::is_regular_file(st)) { AutoCloseFD fd = toDescriptor(open(entry.path().string().c_str(), O_RDONLY, 0)); if (!fd) throw SysError("opening file '%1%'", entry.path()); @@ -381,7 +414,7 @@ void recursiveSync(const Path & path) } -static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & bytesFreed) +static void _deletePath(Descriptor parentfd, const std::filesystem::path & path, uint64_t & bytesFreed) { #ifndef _WIN32 checkInterrupt(); @@ -455,7 +488,7 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b #endif } -static void _deletePath(const fs::path & path, uint64_t & bytesFreed) +static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFreed) { Path dir = dirOf(path.string()); if (dir == "") @@ -471,7 +504,7 @@ static void _deletePath(const fs::path & path, uint64_t & bytesFreed) } -void deletePath(const fs::path & path) +void deletePath(const std::filesystem::path & path) { uint64_t dummy; deletePath(path, dummy); @@ -487,17 +520,17 @@ void createDir(const Path & path, mode_t mode) throw SysError("creating directory '%1%'", path); } -void createDirs(const fs::path & path) +void createDirs(const std::filesystem::path & path) { try { - fs::create_directories(path); - } catch (fs::filesystem_error & e) { + std::filesystem::create_directories(path); + } catch (std::filesystem::filesystem_error & e) { throw SysError("creating directory '%1%'", path.string()); } } -void deletePath(const fs::path & path, uint64_t & bytesFreed) +void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed) { //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); bytesFreed = 0; @@ -522,7 +555,7 @@ AutoDelete::~AutoDelete() if (recursive) deletePath(_path); else { - fs::remove(_path); + std::filesystem::remove(_path); } } } catch (...) { @@ -535,7 +568,7 @@ void AutoDelete::cancel() del = false; } -void AutoDelete::reset(const fs::path & p, bool recursive) { +void AutoDelete::reset(const std::filesystem::path & p, bool recursive) { _path = p; this->recursive = recursive; del = true; @@ -611,30 +644,30 @@ std::pair createTempFile(const Path & prefix) void createSymlink(const Path & target, const Path & link) { try { - fs::create_symlink(target, link); - } catch (fs::filesystem_error & e) { + std::filesystem::create_symlink(target, link); + } catch (std::filesystem::filesystem_error & e) { throw SysError("creating symlink '%1%' -> '%2%'", link, target); } } -void replaceSymlink(const fs::path & target, const fs::path & link) +void replaceSymlink(const std::filesystem::path & target, const std::filesystem::path & link) { for (unsigned int n = 0; true; n++) { - auto tmp = link.parent_path() / fs::path{fmt(".%d_%s", n, link.filename().string())}; + auto tmp = link.parent_path() /std::filesystem::path{fmt(".%d_%s", n, link.filename().string())}; tmp = tmp.lexically_normal(); try { - fs::create_symlink(target, tmp); - } catch (fs::filesystem_error & e) { + std::filesystem::create_symlink(target, tmp); + } catch (std::filesystem::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; - throw SysError("creating symlink '%1%' -> '%2%'", tmp, target); + throw SysError("creating symlink %1% -> %2%", tmp, target); } try { - fs::rename(tmp, link); - } catch (fs::filesystem_error & e) { + std::filesystem::rename(tmp, link); + } catch (std::filesystem::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; - throw SysError("renaming '%1%' to '%2%'", tmp, link); + throw SysError("renaming %1% to %2%", tmp, link); } @@ -642,26 +675,25 @@ void replaceSymlink(const fs::path & target, const fs::path & link) } } -void setWriteTime(const fs::path & path, const struct stat & st) +void setWriteTime(const std::filesystem::path & path, const struct stat & st) { setWriteTime(path, st.st_atime, st.st_mtime, S_ISLNK(st.st_mode)); } -void copyFile(const fs::path & from, const fs::path & to, bool andDelete) +void copyFile(const std::filesystem::path & from, const std::filesystem::path & to, bool andDelete) { - auto fromStatus = fs::symlink_status(from); + auto fromStatus =std::filesystem::symlink_status(from); // Mark the directory as writable so that we can delete its children - if (andDelete && fs::is_directory(fromStatus)) { - fs::permissions(from, fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + if (andDelete &&std::filesystem::is_directory(fromStatus)) { + std::filesystem::permissions(from, std::filesystem::perms::owner_write, std::filesystem::perm_options::add | std::filesystem::perm_options::nofollow); } - - if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { - fs::copy(from, to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); - } else if (fs::is_directory(fromStatus)) { - fs::create_directory(to); - for (auto & entry : fs::directory_iterator(from)) { + if (std::filesystem::is_symlink(fromStatus) ||std::filesystem::is_regular_file(fromStatus)) { + std::filesystem::copy(from, to, std::filesystem::copy_options::copy_symlinks | std::filesystem::copy_options::overwrite_existing); + } else if (std::filesystem::is_directory(fromStatus)) { + std::filesystem::create_directory(to); + for (auto & entry : DirectoryIterator(from)) { copyFile(entry, to / entry.path().filename(), andDelete); } } else { @@ -670,9 +702,9 @@ void copyFile(const fs::path & from, const fs::path & to, bool andDelete) setWriteTime(to, lstat(from.string().c_str())); if (andDelete) { - if (!fs::is_symlink(fromStatus)) - fs::permissions(from, fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - fs::remove(from); + if (!std::filesystem::is_symlink(fromStatus)) + std::filesystem::permissions(from, std::filesystem::perms::owner_write, std::filesystem::perm_options::add | std::filesystem::perm_options::nofollow); + std::filesystem::remove(from); } } @@ -680,18 +712,18 @@ void moveFile(const Path & oldName, const Path & newName) { try { std::filesystem::rename(oldName, newName); - } catch (fs::filesystem_error & e) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); + } catch (std::filesystem::filesystem_error & e) { + auto oldPath = std::filesystem::path(oldName); + auto newPath = std::filesystem::path(newName); // For the move to be as atomic as possible, copy to a temporary // directory - fs::path temp = createTempDir( + std::filesystem::path temp = createTempDir( os_string_to_string(PathViewNG { newPath.parent_path() }), "rename-tmp"); - Finally removeTemp = [&]() { fs::remove(temp); }; + Finally removeTemp = [&]() { std::filesystem::remove(temp); }; auto tempCopyTarget = temp / "copy-target"; if (e.code().value() == EXDEV) { - fs::remove(newPath); + std::filesystem::remove(newPath); warn("can’t rename %s as %s, copying instead", oldName, newName); copyFile(oldPath, tempCopyTarget, true); std::filesystem::rename( @@ -703,7 +735,7 @@ void moveFile(const Path & oldName, const Path & newName) ////////////////////////////////////////////////////////////////////// -bool isExecutableFileAmbient(const fs::path & exe) { +bool isExecutableFileAmbient(const std::filesystem::path & exe) { // Check file type, because directory being executable means // something completely different. // `is_regular_file` follows symlinks before checking. @@ -727,9 +759,23 @@ std::filesystem::path makeParentCanonical(const std::filesystem::path & rawPath) return parent; } return std::filesystem::canonical(parent) / path.filename(); - } catch (fs::filesystem_error & e) { + } catch (std::filesystem::filesystem_error & e) { throw SysError("canonicalising parent path of '%1%'", path); } } +bool chmodIfNeeded(const std::filesystem::path & path, mode_t mode, mode_t mask) +{ + auto pathString = path.string(); + auto prevMode = lstat(pathString).st_mode; + + if (((prevMode ^ mode) & mask) == 0) + return false; + + if (chmod(pathString.c_str(), mode) != 0) + throw SysError("could not set permissions on '%s' to %o", pathString, mode); + + return true; +} + } // namespace nix diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index b0f09583b..53942c956 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -34,9 +34,9 @@ static size_t regularHashSize(HashAlgorithm type) { } -const std::set hashAlgorithms = {"blake3", "md5", "sha1", "sha256", "sha512" }; +const StringSet hashAlgorithms = {"blake3", "md5", "sha1", "sha256", "sha512" }; -const std::set hashFormats = {"base64", "nix32", "base16", "sri" }; +const StringSet hashFormats = {"base64", "nix32", "base16", "sri" }; Hash::Hash(HashAlgorithm algo, const ExperimentalFeatureSettings & xpSettings) : algo(algo) { @@ -309,11 +309,31 @@ static void start(HashAlgorithm ha, Ctx & ctx) else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512); } +// BLAKE3 data size threshold beyond which parallel hashing with TBB is likely faster. +// +// NOTE: This threshold is based on the recommended rule-of-thumb from the official BLAKE3 documentation for typical +// x86_64 hardware as of 2025. In the future it may make sense to allow the user to tune this through nix.conf. +const size_t blake3TbbThreshold = 128000; + +// Decide which BLAKE3 update strategy to use based on some heuristics. Currently this just checks the data size but in +// the future it might also take into consideration available system resources or the presence of a shared-memory +// capable GPU for a heterogenous compute implementation. +void blake3_hasher_update_with_heuristics(blake3_hasher * blake3, std::string_view data) +{ +#ifdef BLAKE3_USE_TBB + if (data.size() >= blake3TbbThreshold) { + blake3_hasher_update_tbb(blake3, data.data(), data.size()); + } else +#endif + { + blake3_hasher_update(blake3, data.data(), data.size()); + } +} static void update(HashAlgorithm ha, Ctx & ctx, std::string_view data) { - if (ha == HashAlgorithm::BLAKE3) blake3_hasher_update(&ctx.blake3, data.data(), data.size()); + if (ha == HashAlgorithm::BLAKE3) blake3_hasher_update_with_heuristics(&ctx.blake3, data); else if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size()); else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); diff --git a/src/libutil/include/nix/util/args.hh b/src/libutil/include/nix/util/args.hh index 463270374..f1eb96675 100644 --- a/src/libutil/include/nix/util/args.hh +++ b/src/libutil/include/nix/util/args.hh @@ -179,7 +179,7 @@ public: using ptr = std::shared_ptr; std::string longName; - std::set aliases; + StringSet aliases; char shortName = 0; std::string description; std::string category; @@ -263,7 +263,7 @@ protected: virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) { return pos; } - std::set hiddenCategories; + StringSet hiddenCategories; /** * Called after all command line flags before the first non-flag diff --git a/src/libutil/include/nix/util/chunked-vector.hh b/src/libutil/include/nix/util/chunked-vector.hh index 96a717556..2c21183ac 100644 --- a/src/libutil/include/nix/util/chunked-vector.hh +++ b/src/libutil/include/nix/util/chunked-vector.hh @@ -45,9 +45,10 @@ public: addChunk(); } - uint32_t size() const { return size_; } + uint32_t size() const noexcept { return size_; } - std::pair add(T value) + template + std::pair add(Args &&... args) { const auto idx = size_++; auto & chunk = [&] () -> auto & { @@ -55,11 +56,16 @@ public: return back; return addChunk(); }(); - auto & result = chunk.emplace_back(std::move(value)); + auto & result = chunk.emplace_back(std::forward(args)...); return {result, idx}; } - const T & operator[](uint32_t idx) const + /** + * Unchecked subscript operator. + * @pre add must have been called at least idx + 1 times. + * @throws nothing + */ + const T & operator[](uint32_t idx) const noexcept { return chunks[idx / ChunkSize][idx % ChunkSize]; } diff --git a/src/libutil/include/nix/util/config-global.hh b/src/libutil/include/nix/util/config-global.hh index b47ee0ad1..44f89e06d 100644 --- a/src/libutil/include/nix/util/config-global.hh +++ b/src/libutil/include/nix/util/config-global.hh @@ -8,7 +8,12 @@ namespace nix { struct GlobalConfig : public AbstractConfig { typedef std::vector ConfigRegistrations; - static ConfigRegistrations * configRegistrations; + + static ConfigRegistrations & configRegistrations() + { + static ConfigRegistrations configRegistrations; + return configRegistrations; + } bool set(const std::string & name, const std::string & value) override; diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index cf51d4818..fb0325fdc 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -179,7 +179,7 @@ public: const std::string name; const std::string description; - const std::set aliases; + const StringSet aliases; int created = 123; @@ -192,7 +192,7 @@ protected: AbstractSetting( const std::string & name, const std::string & description, - const std::set & aliases, + const StringSet & aliases, std::optional experimentalFeature = std::nullopt); virtual ~AbstractSetting(); @@ -251,7 +251,7 @@ public: const bool documentDefault, const std::string & name, const std::string & description, - const std::set & aliases = {}, + const StringSet & aliases = {}, std::optional experimentalFeature = std::nullopt) : AbstractSetting(name, description, aliases, experimentalFeature) , value(def) @@ -323,7 +323,7 @@ public: const T & def, const std::string & name, const std::string & description, - const std::set & aliases = {}, + const StringSet & aliases = {}, const bool documentDefault = true, std::optional experimentalFeature = std::nullopt) : BaseSetting(def, documentDefault, name, description, aliases, std::move(experimentalFeature)) @@ -349,7 +349,7 @@ public: const Path & def, const std::string & name, const std::string & description, - const std::set & aliases = {}); + const StringSet & aliases = {}); Path parse(const std::string & str) const override; @@ -371,7 +371,7 @@ public: const std::optional & def, const std::string & name, const std::string & description, - const std::set & aliases = {}); + const StringSet & aliases = {}); std::optional parse(const std::string & str) const override; diff --git a/src/libutil/include/nix/util/experimental-features.hh b/src/libutil/include/nix/util/experimental-features.hh index 24cee610b..d7bc56f27 100644 --- a/src/libutil/include/nix/util/experimental-features.hh +++ b/src/libutil/include/nix/util/experimental-features.hh @@ -76,7 +76,7 @@ std::ostream & operator<<( * Parse a set of strings to the corresponding set of experimental * features, ignoring (but warning for) any unknown feature. */ -std::set parseFeatures(const std::set &); +std::set parseFeatures(const StringSet &); /** * An experimental feature was required for some (experimental) diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index acae88306..b8fa4cfa0 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -2,7 +2,7 @@ /** * @file * - * Utiltities for working with the file sytem and file paths. + * Utilities for working with the file system and file paths. */ #include "nix/util/types.hh" @@ -105,13 +105,13 @@ std::string_view baseNameOf(std::string_view path); * Check whether 'path' is a descendant of 'dir'. Both paths must be * canonicalized. */ -bool isInDir(std::string_view path, std::string_view dir); +bool isInDir(const std::filesystem::path & path, const std::filesystem::path & dir); /** * Check whether 'path' is equal to 'dir' or a descendant of * 'dir'. Both paths must be canonicalized. */ -bool isDirOrInDir(std::string_view path, std::string_view dir); +bool isDirOrInDir(const std::filesystem::path & path, const std::filesystem::path & dir); /** * Get status of `path`. @@ -126,26 +126,8 @@ std::optional maybeLstat(const Path & path); /** * @return true iff the given path exists. - * - * In the process of being deprecated for `fs::symlink_exists`. */ -bool pathExists(const Path & path); - -namespace fs { - -/** - * TODO: we may actually want to use pathExists instead of this function - * ``` - * symlink_exists(p) = std::filesystem::exists(std::filesystem::symlink_status(p)) - * ``` - * Missing convenience analogous to - * ``` - * std::filesystem::exists(p) = std::filesystem::exists(std::filesystem::status(p)) - * ``` - */ -bool symlink_exists(const std::filesystem::path & path); - -} // namespace fs +bool pathExists(const std::filesystem::path & path); /** * Canonicalize a path except for the last component. @@ -191,7 +173,7 @@ Descriptor openDirectory(const std::filesystem::path & path); */ std::string readFile(const Path & path); std::string readFile(const std::filesystem::path & path); -void readFile(const Path & path, Sink & sink); +void readFile(const Path & path, Sink & sink, bool memory_map = true); /** * Write a string to a file. @@ -360,4 +342,80 @@ typedef std::function PathFilter; extern PathFilter defaultPathFilter; +/** + * Change permissions of a file only if necessary. + * + * @details + * Skip chmod call if the directory already has the requested permissions. + * This is to avoid failing when the executing user lacks permissions to change the + * directory's permissions even if it would be no-op. + * + * @param path Path to the file to change the permissions for. + * @param mode New file mode. + * @param mask Used for checking if the file already has requested permissions. + * + * @return true if permissions changed, false otherwise. + */ +bool chmodIfNeeded(const std::filesystem::path & path, mode_t mode, mode_t mask = S_IRWXU | S_IRWXG | S_IRWXO); + +/** + * @brief A directory iterator that can be used to iterate over the + * contents of a directory. It is similar to std::filesystem::directory_iterator + * but throws NixError on failure instead of std::filesystem::filesystem_error. + */ +class DirectoryIterator { +public: + // --- Iterator Traits --- + using iterator_category = std::input_iterator_tag; + using value_type = std::filesystem::directory_entry; + using difference_type = std::ptrdiff_t; + using pointer = const std::filesystem::directory_entry*; + using reference = const std::filesystem::directory_entry&; + + // Default constructor (represents end iterator) + DirectoryIterator() noexcept = default; + + // Constructor taking a path + explicit DirectoryIterator(const std::filesystem::path& p); + + reference operator*() const { + // Accessing the value itself doesn't typically throw filesystem_error + // after successful construction/increment, but underlying operations might. + // If directory_entry methods called via -> could throw, add try-catch there. + return *it_; + } + + pointer operator->() const { + return &(*it_); + } + + + DirectoryIterator& operator++(); + + // Postfix increment operator + DirectoryIterator operator++(int) { + DirectoryIterator temp = *this; + ++(*this); // Uses the prefix increment's try-catch logic + return temp; + } + + // Equality comparison + friend bool operator==(const DirectoryIterator& a, const DirectoryIterator& b) noexcept { + return a.it_ == b.it_; + } + + // Inequality comparison + friend bool operator!=(const DirectoryIterator& a, const DirectoryIterator& b) noexcept { + return !(a == b); + } + + // Allow direct use in range-based for loops if iterating over an instance + DirectoryIterator begin() const { return *this; } + DirectoryIterator end() const { return DirectoryIterator{}; } + + +private: + std::filesystem::directory_iterator it_; +}; + } diff --git a/src/libutil/include/nix/util/hash.hh b/src/libutil/include/nix/util/hash.hh index 12de4a6ac..5dc3d1017 100644 --- a/src/libutil/include/nix/util/hash.hh +++ b/src/libutil/include/nix/util/hash.hh @@ -22,7 +22,7 @@ const int sha1HashSize = 20; const int sha256HashSize = 32; const int sha512HashSize = 64; -extern const std::set hashAlgorithms; +extern const StringSet hashAlgorithms; extern const std::string nix32Chars; @@ -42,7 +42,7 @@ enum struct HashFormat : int { SRI }; -extern const std::set hashFormats; +extern const StringSet hashFormats; struct Hash { diff --git a/src/libutil/include/nix/util/json-utils.hh b/src/libutil/include/nix/util/json-utils.hh index 9308d4392..37f4d58f8 100644 --- a/src/libutil/include/nix/util/json-utils.hh +++ b/src/libutil/include/nix/util/json-utils.hh @@ -4,6 +4,7 @@ #include #include +#include "nix/util/error.hh" #include "nix/util/types.hh" namespace nix { @@ -35,7 +36,26 @@ const nlohmann::json * getNullable(const nlohmann::json & value); const nlohmann::json::object_t & getObject(const nlohmann::json & value); const nlohmann::json::array_t & getArray(const nlohmann::json & value); const nlohmann::json::string_t & getString(const nlohmann::json & value); -const nlohmann::json::number_integer_t & getInteger(const nlohmann::json & value); +const nlohmann::json::number_unsigned_t & getUnsigned(const nlohmann::json & value); + +template +auto getInteger(const nlohmann::json & value) -> std::enable_if_t && std::is_integral_v, T> +{ + if (auto ptr = value.get_ptr()) { + if (*ptr <= std::make_unsigned_t(std::numeric_limits::max())) { + return *ptr; + } + } else if (auto ptr = value.get_ptr()) { + if (*ptr >= std::numeric_limits::min() && *ptr <= std::numeric_limits::max()) { + return *ptr; + } + } else { + auto typeName = value.is_number_float() ? "floating point number" : value.type_name(); + throw Error("Expected JSON value to be an integral number but it is of type '%s': %s", typeName, value.dump()); + } + throw Error("Out of range: JSON value '%s' cannot be casted to %d-bit integer", value.dump(), 8 * sizeof(T)); +} + const nlohmann::json::boolean_t & getBoolean(const nlohmann::json & value); Strings getStringList(const nlohmann::json & value); StringMap getStringMap(const nlohmann::json & value); @@ -70,8 +90,8 @@ struct json_avoids_null> : std::true_type {}; template struct json_avoids_null> : std::true_type {}; -template -struct json_avoids_null> : std::true_type {}; +template +struct json_avoids_null> : std::true_type {}; template struct json_avoids_null> : std::true_type {}; diff --git a/src/libutil/include/nix/util/lru-cache.hh b/src/libutil/include/nix/util/lru-cache.hh index 6e14cac35..c9bcd7ee0 100644 --- a/src/libutil/include/nix/util/lru-cache.hh +++ b/src/libutil/include/nix/util/lru-cache.hh @@ -11,7 +11,7 @@ namespace nix { /** * A simple least-recently used cache. Not thread-safe. */ -template +template> class LRUCache { private: @@ -22,24 +22,32 @@ private: // and LRU. struct LRUIterator; - using Data = std::map>; + using Data = std::map, Compare>; using LRU = std::list; - struct LRUIterator { typename LRU::iterator it; }; + struct LRUIterator + { + typename LRU::iterator it; + }; Data data; LRU lru; public: - LRUCache(size_t capacity) : capacity(capacity) { } + LRUCache(size_t capacity) + : capacity(capacity) + { + } /** * Insert or upsert an item in the cache. */ - void upsert(const Key & key, const Value & value) + template + void upsert(const K & key, const Value & value) { - if (capacity == 0) return; + if (capacity == 0) + return; erase(key); @@ -61,10 +69,12 @@ public: i->second.first.it = j; } - bool erase(const Key & key) + template + bool erase(const K & key) { auto i = data.find(key); - if (i == data.end()) return false; + if (i == data.end()) + return false; lru.erase(i->second.first.it); data.erase(i); return true; @@ -74,27 +84,33 @@ public: * Look up an item in the cache. If it exists, it becomes the most * recently used item. * */ - std::optional get(const Key & key) + template + std::optional get(const K & key) { auto i = data.find(key); - if (i == data.end()) return {}; + if (i == data.end()) + return {}; /** * Move this item to the back of the LRU list. + * + * Think of std::list iterators as stable pointers to the list node, + * which never get invalidated. Thus, we can reuse the same lru list + * element and just splice it to the back of the list without the need + * to update its value in the key -> list iterator map. */ - lru.erase(i->second.first.it); - auto j = lru.insert(lru.end(), i); - i->second.first.it = j; + auto & [it, value] = i->second; + lru.splice(/*pos=*/lru.end(), /*other=*/lru, it.it); - return i->second.second; + return value; } - size_t size() const + size_t size() const noexcept { return data.size(); } - void clear() + void clear() noexcept { data.clear(); lru.clear(); diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 90e39207b..f5ec04646 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -220,4 +220,10 @@ ref makeFSSourceAccessor(std::filesystem::path root); */ ref makeUnionSourceAccessor(std::vector> && accessors); +/** + * Creates a new source accessor which is confined to the subdirectory + * of the given source accessor. + */ +ref projectSubdirSourceAccessor(ref, CanonPath subdirectory); + } diff --git a/src/libutil/include/nix/util/strings.hh b/src/libutil/include/nix/util/strings.hh index 521e3425f..4c77516a3 100644 --- a/src/libutil/include/nix/util/strings.hh +++ b/src/libutil/include/nix/util/strings.hh @@ -1,5 +1,7 @@ #pragma once +#include "nix/util/types.hh" + #include #include #include @@ -30,7 +32,7 @@ template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); extern template std::list tokenizeString(std::string_view s, std::string_view separators); -extern template std::set tokenizeString(std::string_view s, std::string_view separators); +extern template StringSet tokenizeString(std::string_view s, std::string_view separators); extern template std::vector tokenizeString(std::string_view s, std::string_view separators); /** @@ -44,7 +46,7 @@ template C splitString(std::string_view s, std::string_view separators); extern template std::list splitString(std::string_view s, std::string_view separators); -extern template std::set splitString(std::string_view s, std::string_view separators); +extern template StringSet splitString(std::string_view s, std::string_view separators); extern template std::vector splitString(std::string_view s, std::string_view separators); /** @@ -54,7 +56,7 @@ template std::string concatStringsSep(const std::string_view sep, const C & ss); extern template std::string concatStringsSep(std::string_view, const std::list &); -extern template std::string concatStringsSep(std::string_view, const std::set &); +extern template std::string concatStringsSep(std::string_view, const StringSet &); extern template std::string concatStringsSep(std::string_view, const std::vector &); extern template std::string concatStringsSep(std::string_view, const boost::container::small_vector &); @@ -85,7 +87,7 @@ template dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss); extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); -extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const StringSet &); extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); /** @@ -95,4 +97,39 @@ extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, * Arguments that need to be passed to ssh with spaces in them. */ std::list shellSplitString(std::string_view s); + +/** + * Hash implementation that can be used for zero-copy heterogenous lookup from + * P1690R1[1] in unordered containers. + * + * [1]: https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1690r1.html + */ +struct StringViewHash +{ +private: + using HashType = std::hash; + +public: + using is_transparent = void; + + auto operator()(const char * str) const + { + /* This has a slight overhead due to an implicit strlen, but there isn't + a good way around it because the hash value of all overloads must be + consistent. Delegating to string_view is the solution initially proposed + in P0919R3. */ + return HashType{}(std::string_view{str}); + } + + auto operator()(std::string_view str) const + { + return HashType{}(str); + } + + auto operator()(const std::string & str) const + { + return HashType{}(std::string_view{str}); + } +}; + } diff --git a/src/libutil/include/nix/util/suggestions.hh b/src/libutil/include/nix/util/suggestions.hh index 16496379c..6a76eb9d9 100644 --- a/src/libutil/include/nix/util/suggestions.hh +++ b/src/libutil/include/nix/util/suggestions.hh @@ -35,7 +35,7 @@ public: ) const; static Suggestions bestMatches ( - const std::set & allMatches, + const StringSet & allMatches, std::string_view query ); diff --git a/src/libutil/include/nix/util/topo-sort.hh b/src/libutil/include/nix/util/topo-sort.hh index 77a9ce421..6ba6fda71 100644 --- a/src/libutil/include/nix/util/topo-sort.hh +++ b/src/libutil/include/nix/util/topo-sort.hh @@ -5,13 +5,13 @@ namespace nix { -template -std::vector topoSort(std::set items, - std::function(const T &)> getChildren, +template +std::vector topoSort(std::set items, + std::function(const T &)> getChildren, std::function makeCycleError) { std::vector sorted; - std::set visited, parents; + decltype(items) visited, parents; std::function dfsVisit; @@ -21,7 +21,7 @@ std::vector topoSort(std::set items, if (!visited.insert(path).second) return; parents.insert(path); - std::set references = getChildren(path); + auto references = getChildren(path); for (auto & i : references) /* Don't traverse into items that don't exist in our starting set. */ diff --git a/src/libutil/include/nix/util/types.hh b/src/libutil/include/nix/util/types.hh index 9f5c75827..5139256ca 100644 --- a/src/libutil/include/nix/util/types.hh +++ b/src/libutil/include/nix/util/types.hh @@ -12,17 +12,35 @@ namespace nix { typedef std::list Strings; -typedef std::set StringSet; typedef std::map StringMap; typedef std::map StringPairs; +/** + * Alias to ordered set container with transparent comparator. + * + * Used instead of std::set to use C++14 N3657 [1] + * heterogenous lookup consistently across the whole codebase. + * Transparent comparators get rid of creation of unnecessary + * temporary variables when looking up keys by `std::string_view` + * or C-style `const char *` strings. + * + * [1]: https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2013/n3657.htm + */ +using StringSet = std::set>; + /** * Paths are just strings. */ typedef std::string Path; typedef std::string_view PathView; typedef std::list Paths; -typedef std::set PathSet; + +/** + * Alias to an ordered set of `Path`s. Uses transparent comparator. + * + * @see StringSet + */ +using PathSet = std::set>; typedef std::vector> Headers; diff --git a/src/libutil/include/nix/util/util.hh b/src/libutil/include/nix/util/util.hh index 5a4530798..2361bf2e7 100644 --- a/src/libutil/include/nix/util/util.hh +++ b/src/libutil/include/nix/util/util.hh @@ -152,8 +152,13 @@ std::string toLower(std::string s); /** * Escape a string as a shell word. + * + * This always adds single quotes, even if escaping is not strictly necessary. + * So both + * - `"hello world"` -> `"'hello world'"`, which needs escaping because of the space + * - `"echo"` -> `"'echo'"`, which doesn't need escaping */ -std::string shellEscape(const std::string_view s); +std::string escapeShellArgAlways(const std::string_view s); /** diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index 2c8edfce8..34da83a2c 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -92,9 +92,18 @@ const nlohmann::json::string_t & getString(const nlohmann::json & value) return ensureType(value, nlohmann::json::value_t::string).get_ref(); } -const nlohmann::json::number_integer_t & getInteger(const nlohmann::json & value) +const nlohmann::json::number_unsigned_t & getUnsigned(const nlohmann::json & value) { - return ensureType(value, nlohmann::json::value_t::number_integer).get_ref(); + if (auto ptr = value.get()) { + return *ptr; + } + const char * typeName = value.type_name(); + if (typeName == nlohmann::json(0).type_name()) { + typeName = value.is_number_float() ? "floating point number" : "signed integral number"; + } + throw Error( + "Expected JSON value to be an unsigned integral number but it is of type '%s': %s", + typeName, value.dump()); } const nlohmann::json::boolean_t & getBoolean(const nlohmann::json & value) diff --git a/src/libutil/linux/cgroup.cc b/src/libutil/linux/cgroup.cc index 890797c91..4acfe82f1 100644 --- a/src/libutil/linux/cgroup.cc +++ b/src/libutil/linux/cgroup.cc @@ -65,7 +65,7 @@ static CgroupStats destroyCgroup(const std::filesystem::path & cgroup, bool retu /* Otherwise, manually kill every process in the subcgroups and this cgroup. */ - for (auto & entry : std::filesystem::directory_iterator{cgroup}) { + for (auto & entry : DirectoryIterator{cgroup}) { checkInterrupt(); if (entry.symlink_status().type() != std::filesystem::file_type::directory) continue; destroyCgroup(cgroup / entry.path().filename(), false); @@ -134,7 +134,7 @@ static CgroupStats destroyCgroup(const std::filesystem::path & cgroup, bool retu } if (rmdir(cgroup.c_str()) == -1) - throw SysError("deleting cgroup '%s'", cgroup); + throw SysError("deleting cgroup %s", cgroup); return stats; } diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 9ecbdcd06..04ca06eee 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -50,13 +50,14 @@ endif blake3 = dependency( 'libblake3', - version: '>= 1.5.5', + version: '>= 1.8.2', + method : 'pkg-config', ) deps_private += blake3 boost = dependency( 'boost', - modules : ['context', 'coroutine'], + modules : ['context', 'coroutine', 'iostreams'], include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we @@ -91,7 +92,7 @@ cpuid_required = get_option('cpuid') if host_machine.cpu_family() != 'x86_64' and cpuid_required.enabled() warning('Force-enabling seccomp on non-x86_64 does not make sense') endif -cpuid = dependency('libcpuid', 'cpuid', required : cpuid_required) +cpuid = dependency('libcpuid', 'cpuid', version : '>= 0.7.0', required : cpuid_required) configdata.set('HAVE_LIBCPUID', cpuid.found().to_int()) deps_private += cpuid @@ -142,6 +143,7 @@ sources = [config_priv_h] + files( 'signature/signer.cc', 'source-accessor.cc', 'source-path.cc', + 'subdir-source-accessor.cc', 'strings.cc', 'suggestions.cc', 'tarfile.cc', diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 5c7b4654b..773540e6a 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -138,42 +138,38 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & { assertNoSymlinks(path); DirEntries res; - try { - for (auto & entry : std::filesystem::directory_iterator{makeAbsPath(path)}) { - checkInterrupt(); - auto type = [&]() -> std::optional { - std::filesystem::file_type nativeType; - try { - nativeType = entry.symlink_status().type(); - } catch (std::filesystem::filesystem_error & e) { - // We cannot always stat the child. (Ideally there is no - // stat because the native directory entry has the type - // already, but this isn't always the case.) - if (e.code() == std::errc::permission_denied || e.code() == std::errc::operation_not_permitted) - return std::nullopt; - else throw; - } + for (auto & entry : DirectoryIterator{makeAbsPath(path)}) { + checkInterrupt(); + auto type = [&]() -> std::optional { + std::filesystem::file_type nativeType; + try { + nativeType = entry.symlink_status().type(); + } catch (std::filesystem::filesystem_error & e) { + // We cannot always stat the child. (Ideally there is no + // stat because the native directory entry has the type + // already, but this isn't always the case.) + if (e.code() == std::errc::permission_denied || e.code() == std::errc::operation_not_permitted) + return std::nullopt; + else throw; + } - // cannot exhaustively enumerate because implementation-specific - // additional file types are allowed. + // cannot exhaustively enumerate because implementation-specific + // additional file types are allowed. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wswitch-enum" - switch (nativeType) { - case std::filesystem::file_type::regular: return Type::tRegular; break; - case std::filesystem::file_type::symlink: return Type::tSymlink; break; - case std::filesystem::file_type::directory: return Type::tDirectory; break; - case std::filesystem::file_type::character: return Type::tChar; break; - case std::filesystem::file_type::block: return Type::tBlock; break; - case std::filesystem::file_type::fifo: return Type::tFifo; break; - case std::filesystem::file_type::socket: return Type::tSocket; break; - default: return tUnknown; - } -#pragma GCC diagnostic pop - }(); - res.emplace(entry.path().filename().string(), type); + switch (nativeType) { + case std::filesystem::file_type::regular: return Type::tRegular; break; + case std::filesystem::file_type::symlink: return Type::tSymlink; break; + case std::filesystem::file_type::directory: return Type::tDirectory; break; + case std::filesystem::file_type::character: return Type::tChar; break; + case std::filesystem::file_type::block: return Type::tBlock; break; + case std::filesystem::file_type::fifo: return Type::tFifo; break; + case std::filesystem::file_type::socket: return Type::tSocket; break; + default: return tUnknown; } - } catch (std::filesystem::filesystem_error & e) { - throw SysError("reading directory %1%", showPath(path)); +#pragma GCC diagnostic pop + }(); + res.emplace(entry.path().filename().string(), type); } return res; } diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index fc0d6cff1..b9ebc82b6 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -114,9 +114,11 @@ CanonPath SourceAccessor::resolveSymlinks( if (!linksAllowed--) throw Error("infinite symlink recursion in path '%s'", showPath(path)); auto target = readLink(res); - res.pop(); - if (isAbsolute(target)) + if (isAbsolute(target)) { res = CanonPath::root; + } else { + res.pop(); + } todo.splice(todo.begin(), tokenizeString>(target, "/")); } } diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index 7ce37d73c..a95390089 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -26,18 +26,18 @@ __attribute__((no_sanitize("undefined"))) std::string_view toView(const std::ost } template std::list tokenizeString(std::string_view s, std::string_view separators); -template std::set tokenizeString(std::string_view s, std::string_view separators); +template StringSet tokenizeString(std::string_view s, std::string_view separators); template std::vector tokenizeString(std::string_view s, std::string_view separators); template std::list splitString(std::string_view s, std::string_view separators); -template std::set splitString(std::string_view s, std::string_view separators); +template StringSet splitString(std::string_view s, std::string_view separators); template std::vector splitString(std::string_view s, std::string_view separators); template std::list basicSplitString(std::basic_string_view s, std::basic_string_view separators); template std::string concatStringsSep(std::string_view, const std::list &); -template std::string concatStringsSep(std::string_view, const std::set &); +template std::string concatStringsSep(std::string_view, const StringSet &); template std::string concatStringsSep(std::string_view, const std::vector &); template std::string concatStringsSep(std::string_view, const boost::container::small_vector &); @@ -49,7 +49,7 @@ typedef std::string_view strings_4[4]; template std::string concatStringsSep(std::string_view, const strings_4 &); template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); -template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const StringSet &); template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); /** diff --git a/src/libutil/subdir-source-accessor.cc b/src/libutil/subdir-source-accessor.cc new file mode 100644 index 000000000..265836118 --- /dev/null +++ b/src/libutil/subdir-source-accessor.cc @@ -0,0 +1,59 @@ +#include "nix/util/source-accessor.hh" + +namespace nix { + +struct SubdirSourceAccessor : SourceAccessor +{ + ref parent; + + CanonPath subdirectory; + + SubdirSourceAccessor(ref && parent, CanonPath && subdirectory) + : parent(std::move(parent)) + , subdirectory(std::move(subdirectory)) + { + displayPrefix.clear(); + } + + std::string readFile(const CanonPath & path) override + { + return parent->readFile(subdirectory / path); + } + + void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override + { + return parent->readFile(subdirectory / path, sink, sizeCallback); + } + + bool pathExists(const CanonPath & path) override + { + return parent->pathExists(subdirectory / path); + } + + std::optional maybeLstat(const CanonPath & path) override + { + return parent->maybeLstat(subdirectory / path); + } + + DirEntries readDirectory(const CanonPath & path) override + { + return parent->readDirectory(subdirectory / path); + } + + std::string readLink(const CanonPath & path) override + { + return parent->readLink(subdirectory / path); + } + + std::string showPath(const CanonPath & path) override + { + return displayPrefix + parent->showPath(subdirectory / path) + displaySuffix; + } +}; + +ref projectSubdirSourceAccessor(ref parent, CanonPath subdirectory) +{ + return make_ref(std::move(parent), std::move(subdirectory)); +} + +} diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 0105c30e7..aee23d45e 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -38,7 +38,7 @@ int levenshteinDistance(std::string_view first, std::string_view second) } Suggestions Suggestions::bestMatches ( - const std::set & allMatches, + const StringSet & allMatches, std::string_view query) { std::set res; diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 5f21bc0d5..299847850 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -8,10 +8,6 @@ namespace nix { -namespace fs { -using namespace std::filesystem; -} - namespace { int callback_open(struct archive *, void * self) @@ -127,7 +123,7 @@ TarArchive::~TarArchive() archive_read_free(this->archive); } -static void extract_archive(TarArchive & archive, const fs::path & destDir) +static void extract_archive(TarArchive & archive, const std::filesystem::path & destDir) { int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS | ARCHIVE_EXTRACT_SECURE_NODOTDOT; @@ -162,7 +158,7 @@ static void extract_archive(TarArchive & archive, const fs::path & destDir) archive.close(); } -void unpackTarfile(Source & source, const fs::path & destDir) +void unpackTarfile(Source & source, const std::filesystem::path & destDir) { auto archive = TarArchive(source); @@ -170,7 +166,7 @@ void unpackTarfile(Source & source, const fs::path & destDir) extract_archive(archive, destDir); } -void unpackTarfile(const fs::path & tarFile, const fs::path & destDir) +void unpackTarfile(const std::filesystem::path & tarFile, const std::filesystem::path & destDir) { auto archive = TarArchive(tarFile); @@ -182,6 +178,10 @@ time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & { time_t lastModified = 0; + /* Only allocate the buffer once. Use the heap because 131 KiB is a bit too + much for the stack. */ + std::vector buf(128 * 1024); + for (;;) { // FIXME: merge with extract_archive struct archive_entry * entry; @@ -216,7 +216,6 @@ time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & crf.isExecutable(); while (true) { - std::vector buf(128 * 1024); auto n = archive_read_data(archive.archive, buf.data(), buf.size()); if (n < 0) checkLibArchive(archive.archive, n, "cannot read file from tarball: %s"); diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index fa0f7e871..63473d1a9 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -95,10 +95,19 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w } else if (i != s.end() && *i == ']') { // OSC e += *i++; - // eat ESC - while (i != s.end() && *i != '\e') e += *i++; - // eat backslash - if (i != s.end() && *i == '\\') e += last = *i++; + // https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda defines + // two forms of a URI separator: + // 1. ESC '\' (standard) + // 2. BEL ('\a') (xterm-style, used by gcc) + + // eat ESC or BEL + while (i != s.end() && *i != '\e' && *i != '\a') e += *i++; + if (i != s.end()) { + char v = *i; + e += *i++; + // eat backslash after ESC + if (i != s.end() && v == '\e' && *i == '\\') e += last = *i++; + } } else { if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; } diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index 73ee49982..e6d0c255d 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -191,7 +191,7 @@ void unix::closeExtraFDs() #ifdef __linux__ try { - for (auto & s : std::filesystem::directory_iterator{"/proc/self/fd"}) { + for (auto & s : DirectoryIterator{"/proc/self/fd"}) { checkInterrupt(); auto fd = std::stoi(s.path().filename()); if (fd > MAX_KEPT_FD) { @@ -201,7 +201,6 @@ void unix::closeExtraFDs() } return; } catch (SysError &) { - } catch (std::filesystem::filesystem_error &) { } #endif diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index e62b7d1c2..a1941db05 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -23,7 +23,8 @@ Descriptor openDirectory(const std::filesystem::path & path) return open(path.c_str(), O_RDONLY | O_DIRECTORY); } -void setWriteTime(const fs::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) +void setWriteTime( + const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) { // Would be nice to use std::filesystem unconditionally, but // doesn't support access time just modification time. @@ -57,7 +58,7 @@ void setWriteTime(const fs::path & path, time_t accessedTime, time_t modificatio if (lutimes(path.c_str(), times) == -1) throw SysError("changing modification time of %s", path); # else - bool isSymlink = optIsSymlink ? *optIsSymlink : fs::is_symlink(path); + bool isSymlink = optIsSymlink ? *optIsSymlink : std::filesystem::is_symlink(path); if (!isSymlink) { if (utimes(path.c_str(), times) == -1) diff --git a/src/libutil/unix/users.cc b/src/libutil/unix/users.cc index 18df7fdf2..5ac851e95 100644 --- a/src/libutil/unix/users.cc +++ b/src/libutil/unix/users.cc @@ -9,8 +9,6 @@ namespace nix { -namespace fs { using namespace std::filesystem; } - std::string getUserName() { auto pw = getpwuid(geteuid()); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ffd85ffbb..c9cc80fef 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -171,7 +171,7 @@ std::string toLower(std::string s) } -std::string shellEscape(const std::string_view s) +std::string escapeShellArgAlways(const std::string_view s) { std::string r; r.reserve(s.size() + 2); diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index f9f384a5b..d7cc7b488 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -13,8 +13,10 @@ std::optional getEnvOs(const OsString & key) return std::nullopt; } - // Allocate a buffer to hold the environment variable value - std::wstring value{bufferSize, L'\0'}; + /* Allocate a buffer to hold the environment variable value. + WARNING: Do not even think about using uniform initialization here, + we DONT want to call the initializer list ctor accidentally. */ + std::wstring value(bufferSize, L'\0'); // Retrieve the environment variable value DWORD resultSize = GetEnvironmentVariableW(key.c_str(), &value[0], bufferSize); diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 1dac7e754..a73fa223a 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -3,13 +3,10 @@ #ifdef _WIN32 namespace nix { -namespace fs { -using namespace std::filesystem; -} - -void setWriteTime(const fs::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) +void setWriteTime( + const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) { - // FIXME use `fs::last_write_time`. + // FIXME use `std::filesystem::last_write_time`. // // Would be nice to use std::filesystem unconditionally, but // doesn't support access time just modification time. diff --git a/src/libutil/windows/include/nix/util/signals-impl.hh b/src/libutil/windows/include/nix/util/signals-impl.hh index f716ffd1a..55606debd 100644 --- a/src/libutil/windows/include/nix/util/signals-impl.hh +++ b/src/libutil/windows/include/nix/util/signals-impl.hh @@ -25,6 +25,7 @@ inline void setInterruptThrown() static inline bool isInterrupted() { /* Do nothing for now */ + return false; } inline void checkInterrupt() diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 45f891808..80ebf6bfa 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -12,7 +12,7 @@ #include "nix/util/current-process.hh" #include "nix/store/parsed-derivations.hh" #include "nix/store/derivation-options.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/local-fs-store.hh" #include "nix/store/globals.hh" #include "nix/store/realisation.hh" @@ -147,7 +147,7 @@ static void main_nix_build(int argc, char * * argv) std::string outLink = "./result"; // List of environment variables kept for --pure - std::set keepVars{ + StringSet keepVars{ "HOME", "XDG_RUNTIME_DIR", "USER", "LOGNAME", "DISPLAY", "WAYLAND_DISPLAY", "WAYLAND_SOCKET", "PATH", "TERM", "IN_NIX_SHELL", "NIX_SHELL_PRESERVE_PROMPT", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL", @@ -255,16 +255,16 @@ static void main_nix_build(int argc, char * * argv) std::ostringstream joined; for (const auto & i : savedArgs) - joined << shellEscape(i) << ' '; + joined << escapeShellArgAlways(i) << ' '; if (std::regex_search(interpreter, std::regex("ruby"))) { // Hack for Ruby. Ruby also examines the shebang. It tries to // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, shellEscape(script), toView(joined)); + envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, escapeShellArgAlways(script), toView(joined)); } else { - envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, shellEscape(script), toView(joined)); + envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, escapeShellArgAlways(script), toView(joined)); } } @@ -544,8 +544,16 @@ static void main_nix_build(int argc, char * * argv) env["NIX_STORE"] = store->storeDir; env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); - ParsedDerivation parsedDrv(packageInfo.requireDrvPath(), drv); - DerivationOptions drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); + auto parsedDrv = StructuredAttrs::tryParse(drv.env); + DerivationOptions drvOptions; + try { + drvOptions = DerivationOptions::fromStructuredAttrs( + drv.env, + parsedDrv ? &*parsedDrv : nullptr); + } catch (Error & e) { + e.addTrace({}, "while parsing derivation '%s'", store->printStorePath(packageInfo.requireDrvPath())); + throw; + } int fileNr = 0; @@ -560,7 +568,7 @@ static void main_nix_build(int argc, char * * argv) std::string structuredAttrsRC; - if (parsedDrv.hasStructuredAttrs()) { + if (parsedDrv) { StorePathSet inputs; std::function::ChildNode &)> accumInputClosure; @@ -578,19 +586,22 @@ static void main_nix_build(int argc, char * * argv) for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) accumInputClosure(inputDrv, inputNode); - if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, inputs)) { - auto json = structAttrs.value(); - structuredAttrsRC = writeStructuredAttrsShell(json); + auto json = parsedDrv->prepareStructuredAttrs( + *store, + drvOptions, + inputs, + drv.outputs); - auto attrsJSON = (tmpDir.path() / ".attrs.json").string(); - writeFile(attrsJSON, json.dump()); + structuredAttrsRC = StructuredAttrs::writeShell(json); - auto attrsSH = (tmpDir.path() / ".attrs.sh").string(); - writeFile(attrsSH, structuredAttrsRC); + auto attrsJSON = (tmpDir.path() / ".attrs.json").string(); + writeFile(attrsJSON, json.dump()); - env["NIX_ATTRS_SH_FILE"] = attrsSH; - env["NIX_ATTRS_JSON_FILE"] = attrsJSON; - } + auto attrsSH = (tmpDir.path() / ".attrs.sh").string(); + writeFile(attrsSH, structuredAttrsRC); + + env["NIX_ATTRS_SH_FILE"] = attrsSH; + env["NIX_ATTRS_JSON_FILE"] = attrsJSON; } /* Run a shell using the derivation's environment. For @@ -626,12 +637,12 @@ static void main_nix_build(int argc, char * * argv) "unset TZ; %6%" "shopt -s execfail;" "%7%", - shellEscape(tmpDir.path().string()), + escapeShellArgAlways(tmpDir.path().string()), (pure ? "" : "p=$PATH; "), (pure ? "" : "PATH=$PATH:$p; unset p; "), - shellEscape(dirOf(*shell)), - shellEscape(*shell), - (getenv("TZ") ? (std::string("export TZ=") + shellEscape(getenv("TZ")) + "; ") : ""), + escapeShellArgAlways(dirOf(*shell)), + escapeShellArgAlways(*shell), + (getenv("TZ") ? (std::string("export TZ=") + escapeShellArgAlways(getenv("TZ")) + "; ") : ""), envCommand); vomit("Sourcing nix-shell with file %s and contents:\n%s", rcfile, rc); writeFile(rcfile, rc); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 2ad88dbbe..6699a2ac9 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -2,7 +2,7 @@ #include "nix/main/shared.hh" #include "nix/store/globals.hh" #include "nix/store/filetransfer.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/cmd/legacy.hh" #include "nix/expr/eval-settings.hh" // for defexpr #include "nix/util/users.hh" diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 3a84d97aa..7f86b2b5c 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,6 +1,6 @@ #include "nix/util/file-system.hh" #include "nix/util/signals.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/store-cast.hh" #include "nix/store/gc-store.hh" #include "nix/store/profiles.hh" @@ -24,23 +24,23 @@ bool dryRun = false; * Of course, this makes rollbacks to before this point in time * impossible. */ -void removeOldGenerations(fs::path dir) +void removeOldGenerations(std::filesystem::path dir) { if (access(dir.string().c_str(), R_OK) != 0) return; bool canWrite = access(dir.string().c_str(), W_OK) == 0; - for (auto & i : fs::directory_iterator{dir}) { + for (auto & i : DirectoryIterator{dir}) { checkInterrupt(); auto path = i.path().string(); auto type = i.symlink_status().type(); - if (type == fs::file_type::symlink && canWrite) { + if (type == std::filesystem::file_type::symlink && canWrite) { std::string link; try { link = readLink(path); - } catch (fs::filesystem_error & e) { + } catch (std::filesystem::filesystem_error & e) { if (e.code() == std::errc::no_such_file_or_directory) continue; throw; } @@ -52,7 +52,7 @@ void removeOldGenerations(fs::path dir) } else deleteOldGenerations(path, dryRun); } - } else if (type == fs::file_type::directory) { + } else if (type == std::filesystem::file_type::directory) { removeOldGenerations(path); } } @@ -84,10 +84,10 @@ static int main_nix_collect_garbage(int argc, char * * argv) }); if (removeOld) { - std::set dirsToClean = { + std::set dirsToClean = { profilesDir(), - fs::path{settings.nixStateDir} / "profiles", - fs::path{getDefaultProfile()}.parent_path(), + std::filesystem::path{settings.nixStateDir} / "profiles", + std::filesystem::path{getDefaultProfile()}.parent_path(), }; for (auto & dir : dirsToClean) removeOldGenerations(dir); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 6d0db1008..87d0f6590 100644 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,6 +1,6 @@ #include "nix/main/shared.hh" #include "nix/store/realisation.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/cmd/legacy.hh" #include "man-pages.hh" diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 021619ada..25ff39e38 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -9,7 +9,7 @@ #include "nix/store/profiles.hh" #include "nix/store/path-with-outputs.hh" #include "nix/main/shared.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/local-fs-store.hh" #include "user-env.hh" #include "nix/expr/value-to-json.hh" @@ -238,9 +238,9 @@ static void checkSelectorUse(DrvNames & selectors) namespace { -std::set searchByPrefix(const PackageInfos & allElems, std::string_view prefix) { +StringSet searchByPrefix(const PackageInfos & allElems, std::string_view prefix) { constexpr std::size_t maxResults = 3; - std::set result; + StringSet result; for (const auto & packageInfo : allElems) { const auto drvName = DrvName { packageInfo.queryName() }; if (hasPrefix(drvName.name, prefix)) { diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 4ae82b2bf..89a8505bb 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -8,7 +8,7 @@ #include "nix/util/signals.hh" #include "nix/expr/value-to-xml.hh" #include "nix/expr/value-to-json.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/local-fs-store.hh" #include "nix/cmd/common-eval-args.hh" #include "nix/cmd/legacy.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index fbbb57f43..9acdf4554 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -2,6 +2,7 @@ #include "nix/store/derivations.hh" #include "dotgraph.hh" #include "nix/store/globals.hh" +#include "nix/store/store-open.hh" #include "nix/store/store-cast.hh" #include "nix/store/local-fs-store.hh" #include "nix/store/log-store.hh" @@ -497,7 +498,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) /* Print each environment variable in the derivation in a format * that can be sourced by the shell. */ for (auto & i : drv.env) - logger->cout("export %1%; %1%=%2%\n", i.first, shellEscape(i.second)); + logger->cout("export %1%; %1%=%2%\n", i.first, escapeShellArgAlways(i.second)); /* Also output the arguments. This doesn't preserve whitespace in arguments. */ @@ -506,7 +507,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) for (auto & i : drv.args) { if (!first) cout << ' '; first = false; - cout << shellEscape(i); + cout << escapeShellArgAlways(i); } cout << "'\n"; } @@ -563,7 +564,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) #endif if (!hashGiven) { HashResult hash = hashPath( - {store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) }}, + {store->getFSAccessor(false), CanonPath { info->path.to_string() }}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); info->narHash = hash.first; info->narSize = hash.second; diff --git a/src/nix/app.cc b/src/nix/app.cc index 0ba231c41..d3c14c062 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -132,18 +132,23 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) throw Error("attribute '%s' has unsupported type '%s'", cursor->getAttrPathStr(), type); } -// FIXME: move to libcmd -App UnresolvedApp::resolve(ref evalStore, ref store) +std::vector UnresolvedApp::build(ref evalStore, ref store) { - auto res = unresolved; - Installables installableContext; for (auto & ctxElt : unresolved.context) installableContext.push_back( make_ref(store, DerivedPath { ctxElt })); - auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); + return Installable::build(evalStore, store, Realise::Outputs, installableContext); +} + +// FIXME: move to libcmd +App UnresolvedApp::resolve(ref evalStore, ref store) +{ + auto res = unresolved; + + auto builtContext = build(evalStore, store); res.program = resolveString(*store, unresolved.program, builtContext); if (!store->isInStore(res.program)) throw Error("app program '%s' is not in the Nix store", res.program); diff --git a/src/nix/build.cc b/src/nix/build.cc index 8db831240..bd0c8862b 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -41,29 +41,13 @@ static nlohmann::json builtPathsWithResultToJSON(const std::vectorcout("%s", derivedPathsToJSON(pathsToBuild, *store).dump()); + printJSON(derivedPathsToJSON(pathsToBuild, *store)); return; } @@ -114,9 +98,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, *store).dump()); - if (outLink != "") - if (auto store2 = store.dynamic_pointer_cast()) - createOutLinks(outLink, toBuiltPaths(buildables), *store2); + createOutLinksMaybe(buildables, store); if (printOutputPaths) { logger->stop(); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index c334469b5..50d7bf6a3 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -80,7 +80,7 @@ struct CmdBundle : InstallableValueCommand auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec( - fetchSettings, bundler, fs::current_path().string()); + fetchSettings, bundler, std::filesystem::current_path().string()); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, evalState, std::move(bundlerFlakeRef), bundlerName, std::move(extendedOutputsSpec), diff --git a/src/nix/cat.cc b/src/nix/cat.cc index a790c0301..aa27446d2 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -6,21 +6,21 @@ using namespace nix; struct MixCat : virtual Args { - std::string path; - - void cat(ref accessor) + void cat(ref accessor, CanonPath path) { - auto st = accessor->lstat(CanonPath(path)); + auto st = accessor->lstat(path); if (st.type != SourceAccessor::Type::tRegular) - throw Error("path '%1%' is not a regular file", path); + throw Error("path '%1%' is not a regular file", path.abs()); logger->stop(); - writeFull(getStandardOutput(), accessor->readFile(CanonPath(path))); + writeFull(getStandardOutput(), accessor->readFile(path)); } }; struct CmdCatStore : StoreCommand, MixCat { + std::string path; + CmdCatStore() { expectArgs({ @@ -44,7 +44,8 @@ struct CmdCatStore : StoreCommand, MixCat void run(ref store) override { - cat(store->getFSAccessor()); + auto [storePath, rest] = store->toStorePath(path); + cat(store->getFSAccessor(), CanonPath{storePath.to_string()} / CanonPath{rest}); } }; @@ -52,6 +53,8 @@ struct CmdCatNar : StoreCommand, MixCat { Path narPath; + std::string path; + CmdCatNar() { expectArgs({ @@ -76,7 +79,7 @@ struct CmdCatNar : StoreCommand, MixCat void run(ref store) override { - cat(makeNarAccessor(readFile(narPath))); + cat(makeNarAccessor(readFile(narPath)), CanonPath{path}); } }; diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index deac8e560..27d053b9f 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -78,12 +78,12 @@ struct CmdConfigCheck : StoreCommand bool checkNixInPath() { - std::set dirs; + std::set dirs; for (auto & dir : ExecutablePath::load().directories) { auto candidate = dir / "nix-env"; - if (fs::exists(candidate)) - dirs.insert(fs::canonical(candidate).parent_path() ); + if (std::filesystem::exists(candidate)) + dirs.insert(std::filesystem::canonical(candidate).parent_path() ); } if (dirs.size() != 1) { @@ -99,12 +99,12 @@ struct CmdConfigCheck : StoreCommand bool checkProfileRoots(ref store) { - std::set dirs; + std::set dirs; for (auto & dir : ExecutablePath::load().directories) { auto profileDir = dir.parent_path(); try { - auto userEnv = fs::weakly_canonical(profileDir); + auto userEnv = std::filesystem::weakly_canonical(profileDir); auto noContainsProfiles = [&]{ for (auto && part : profileDir) @@ -114,8 +114,8 @@ struct CmdConfigCheck : StoreCommand if (store->isStorePath(userEnv.string()) && hasSuffix(userEnv.string(), "user-environment")) { while (noContainsProfiles() && std::filesystem::is_symlink(profileDir)) - profileDir = fs::weakly_canonical( - profileDir.parent_path() / fs::read_symlink(profileDir)); + profileDir = std::filesystem::weakly_canonical( + profileDir.parent_path() / std::filesystem::read_symlink(profileDir)); if (noContainsProfiles()) dirs.insert(dir); diff --git a/src/nix/config.cc b/src/nix/config.cc index 1dc2bed20..cd82b08a6 100644 --- a/src/nix/config.cc +++ b/src/nix/config.cc @@ -63,7 +63,7 @@ struct CmdConfigShow : Command, MixJSON if (json) { // FIXME: use appropriate JSON types (bool, ints, etc). - logger->cout("%s", globalConfig.toJSON().dump()); + printJSON(globalConfig.toJSON()); } else { logger->cout("%s", globalConfig.toKeyValue()); } diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc index 86755c3e8..26108b8b8 100644 --- a/src/nix/derivation-show.cc +++ b/src/nix/derivation-show.cc @@ -11,7 +11,7 @@ using namespace nix; using json = nlohmann::json; -struct CmdShowDerivation : InstallablesCommand +struct CmdShowDerivation : InstallablesCommand, MixPrintJSON { bool recursive = false; @@ -57,7 +57,7 @@ struct CmdShowDerivation : InstallablesCommand jsonRoot[store->printStorePath(drvPath)] = store->readDerivation(drvPath).toJSON(*store); } - logger->cout(jsonRoot.dump(2)); + printJSON(jsonRoot); } }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 02947ff41..ec23d3212 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -64,13 +64,11 @@ struct BuildEnvironment std::map bashFunctions; std::optional> structuredAttrs; - static BuildEnvironment fromJSON(std::string_view in) + static BuildEnvironment fromJSON(const nlohmann::json & json) { BuildEnvironment res; - std::set exported; - - auto json = nlohmann::json::parse(in); + StringSet exported; for (auto & [name, info] : json["variables"].items()) { std::string type = info["type"]; @@ -93,7 +91,14 @@ struct BuildEnvironment return res; } - std::string toJSON() const + static BuildEnvironment parseJSON(std::string_view in) + { + auto json = nlohmann::json::parse(in); + + return fromJSON(json); + } + + nlohmann::json toJSON() const { auto res = nlohmann::json::object(); @@ -125,11 +130,9 @@ struct BuildEnvironment res["structuredAttrs"] = std::move(contents); } - auto json = res.dump(); + assert(BuildEnvironment::fromJSON(res) == *this); - assert(BuildEnvironment::fromJSON(json) == *this); - - return json; + return res; } bool providesStructuredAttrs() const @@ -149,25 +152,25 @@ struct BuildEnvironment return structuredAttrs->second; } - void toBash(std::ostream & out, const std::set & ignoreVars) const + void toBash(std::ostream & out, const StringSet & ignoreVars) const { for (auto & [name, value] : vars) { if (!ignoreVars.count(name)) { if (auto str = std::get_if(&value)) { - out << fmt("%s=%s\n", name, shellEscape(str->value)); + out << fmt("%s=%s\n", name, escapeShellArgAlways(str->value)); if (str->exported) out << fmt("export %s\n", name); } else if (auto arr = std::get_if(&value)) { out << "declare -a " << name << "=("; for (auto & s : *arr) - out << shellEscape(s) << " "; + out << escapeShellArgAlways(s) << " "; out << ")\n"; } else if (auto arr = std::get_if(&value)) { out << "declare -A " << name << "=("; for (auto & [n, v] : *arr) - out << "[" << shellEscape(n) << "]=" << shellEscape(v) << " "; + out << "[" << escapeShellArgAlways(n) << "]=" << escapeShellArgAlways(v) << " "; out << ")\n"; } } @@ -306,7 +309,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore struct Common : InstallableCommand, MixProfile { - std::set ignoreVars{ + StringSet ignoreVars{ "BASHOPTS", "HOME", // FIXME: don't ignore in pure mode? "NIX_BUILD_TOP", @@ -343,7 +346,7 @@ struct Common : InstallableCommand, MixProfile ref store, const BuildEnvironment & buildEnvironment, const std::filesystem::path & tmpDir, - const std::filesystem::path & outputsDir = fs::path { fs::current_path() } / "outputs") + const std::filesystem::path & outputsDir = std::filesystem::path { std::filesystem::current_path() } / "outputs") { // A list of colon-separated environment variables that should be // prepended to, rather than overwritten, in order to keep the shell usable. @@ -506,7 +509,7 @@ struct Common : InstallableCommand, MixProfile debug("reading environment file '%s'", strPath); - return {BuildEnvironment::fromJSON(readFile(store->toRealPath(shellOutPath))), strPath}; + return {BuildEnvironment::parseJSON(readFile(store->toRealPath(shellOutPath))), strPath}; } }; @@ -617,7 +620,7 @@ struct CmdDevelop : Common, MixEnvironment std::vector args; args.reserve(command.size()); for (const auto & s : command) - args.push_back(shellEscape(s)); + args.push_back(escapeShellArgAlways(s)); script += fmt("exec %s\n", concatStringsSep(" ", args)); } @@ -625,13 +628,13 @@ struct CmdDevelop : Common, MixEnvironment script = "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\nshopt -u expand_aliases\n" + script + "\nshopt -s expand_aliases\n"; if (developSettings.bashPrompt != "") script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", - shellEscape(developSettings.bashPrompt.get())); + escapeShellArgAlways(developSettings.bashPrompt.get())); if (developSettings.bashPromptPrefix != "") script += fmt("[ -n \"$PS1\" ] && PS1=%s\"$PS1\";\n", - shellEscape(developSettings.bashPromptPrefix.get())); + escapeShellArgAlways(developSettings.bashPromptPrefix.get())); if (developSettings.bashPromptSuffix != "") script += fmt("[ -n \"$PS1\" ] && PS1+=%s;\n", - shellEscape(developSettings.bashPromptSuffix.get())); + escapeShellArgAlways(developSettings.bashPromptSuffix.get())); } writeFull(rcFileFd.get(), script); @@ -739,7 +742,7 @@ struct CmdPrintDevEnv : Common, MixJSON logger->stop(); if (json) { - logger->writeToStdout(buildEnvironment.toJSON()); + printJSON(buildEnvironment.toJSON()); } else { AutoDelete tmpDir(createTempDir("", "nix-dev-env"), true); logger->writeToStdout(makeRcScript(store, buildEnvironment, tmpDir)); diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index c4d21db6f..ff9f9db40 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -47,10 +47,10 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) return groupedPaths; } -std::string showVersions(const std::set & versions) +std::string showVersions(const StringSet & versions) { if (versions.empty()) return "∅"; - std::set versions2; + StringSet versions2; for (auto & version : versions) versions2.insert(version.empty() ? "ε" : version); return concatStringsSep(", ", versions2); @@ -65,7 +65,7 @@ void printClosureDiff( auto beforeClosure = getClosureInfo(store, beforePath); auto afterClosure = getClosureInfo(store, afterPath); - std::set allNames; + StringSet allNames; for (auto & [name, _] : beforeClosure) allNames.insert(name); for (auto & [name, _] : afterClosure) allNames.insert(name); @@ -87,11 +87,11 @@ void printClosureDiff( auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize; auto showDelta = std::abs(sizeDelta) >= 8 * 1024; - std::set removed, unchanged; + StringSet removed, unchanged; for (auto & [version, _] : beforeVersions) if (!afterVersions.count(version)) removed.insert(version); else unchanged.insert(version); - std::set added; + StringSet added; for (auto & [version, _] : afterVersions) if (!beforeVersions.count(version)) added.insert(version); diff --git a/src/nix/env.cc b/src/nix/env.cc index f6b12f21c..a0b0e976b 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -6,6 +6,7 @@ #include "run.hh" #include "nix/util/strings.hh" #include "nix/util/executable-path.hh" +#include "nix/util/mounted-source-accessor.hh" using namespace nix; @@ -65,11 +66,11 @@ struct CmdShell : InstallablesCommand, MixEnvironment void run(ref store, Installables && installables) override { + auto state = getEvalState(); + auto outPaths = Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); - auto accessor = store->getFSAccessor(); - std::unordered_set done; std::queue todo; for (auto & path : outPaths) @@ -85,13 +86,16 @@ struct CmdShell : InstallablesCommand, MixEnvironment if (!done.insert(path).second) continue; - if (true) - pathAdditions.push_back(store->printStorePath(path) + "/bin"); + auto binDir = state->storeFS->resolveSymlinks(CanonPath(store->printStorePath(path)) / "bin"); + if (!store->isInStore(binDir.abs())) + throw Error("path '%s' is not in the Nix store", binDir); - auto propPath = accessor->resolveSymlinks( + pathAdditions.push_back(binDir.abs()); + + auto propPath = state->storeFS->resolveSymlinks( CanonPath(store->printStorePath(path)) / "nix-support" / "propagated-user-env-packages"); - if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { - for (auto & p : tokenizeString(accessor->readFile(propPath))) + if (auto st = state->storeFS->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { + for (auto & p : tokenizeString(state->storeFS->readFile(propPath))) todo.push(store->parseStorePath(p)); } } @@ -108,7 +112,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment // Release our references to eval caches to ensure they are persisted to disk, because // we are about to exec out of this process without running C++ destructors. - getEvalState()->evalCaches.clear(); + state->evalCaches.clear(); execProgramInStore(store, UseLookupPath::Use, *command.begin(), args); } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index bd58ba010..e5b0aa968 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -16,7 +16,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption { bool raw = false; std::optional apply; - std::optional writeTo; + std::optional writeTo; CmdEval() : InstallableValueCommand() { @@ -76,19 +76,19 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption if (writeTo) { logger->stop(); - if (fs::symlink_exists(*writeTo)) + if (pathExists(*writeTo)) throw Error("path '%s' already exists", writeTo->string()); - std::function recurse; + std::function recurse; - recurse = [&](Value & v, const PosIdx pos, const fs::path & path) + recurse = [&](Value & v, const PosIdx pos, const std::filesystem::path & path) { state->forceValue(v, pos); if (v.type() == nString) // FIXME: disallow strings with contexts? writeFile(path.string(), v.string_view()); else if (v.type() == nAttrs) { - [[maybe_unused]] bool directoryCreated = fs::create_directory(path); + [[maybe_unused]] bool directoryCreated = std::filesystem::create_directory(path); // Directory should not already exist assert(directoryCreated); for (auto & attr : *v.attrs()) { @@ -122,9 +122,11 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption } else if (json) { + // FIXME: use printJSON + auto j = printValueAsJSON(*state, true, *v, pos, context, false); logger->cout("%s", state->devirtualize( - printValueAsJSON(*state, true, *v, pos, context, false).dump(), + outputPretty ? j.dump(2) : j.dump(), context)); } diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md index d13666a4c..1148f5ef2 100644 --- a/src/nix/flake-lock.md +++ b/src/nix/flake-lock.md @@ -32,7 +32,7 @@ R""( This command updates the lock file of a flake (`flake.lock`) so that it contains an up-to-date lock for every flake input specified in -`flake.nix`. Lock file entries are aready up-to-date are not modified. +`flake.nix`. Lock file entries are already up-to-date are not modified. If you want to update existing lock entries, use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 4782cbb29..48e2ae392 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -8,7 +8,7 @@ #include "nix/flake/flake.hh" #include "nix/expr/get-drvs.hh" #include "nix/util/signals.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/derivations.hh" #include "nix/store/outputs-spec.hh" #include "nix/expr/attr-path.hh" @@ -54,7 +54,7 @@ public: FlakeRef getFlakeRef() { - return parseFlakeRef(fetchSettings, flakeUrl, fs::current_path().string()); //FIXME + return parseFlakeRef(fetchSettings, flakeUrl, std::filesystem::current_path().string()); //FIXME } LockedFlake lockFlake() @@ -66,7 +66,7 @@ public: { return { // Like getFlakeRef but with expandTilde calld first - parseFlakeRef(fetchSettings, expandTilde(flakeUrl), fs::current_path().string()) + parseFlakeRef(fetchSettings, expandTilde(flakeUrl), std::filesystem::current_path().string()) }; } }; @@ -251,7 +251,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["locks"] = lockedFlake.lockFile.toJSON().first; if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings)) j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); - logger->cout("%s", j.dump()); + printJSON(j); } else { logger->cout( ANSI_BOLD "Resolved URL:" ANSI_NORMAL " %s", @@ -396,7 +396,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::set omittedSystems; + StringSet omittedSystems; // FIXME: rewrite to use EvalCache. @@ -895,7 +895,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto evalState = getEvalState(); auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment( - fetchSettings, templateUrl, fs::current_path().string()); + fetchSettings, templateUrl, std::filesystem::current_path().string()); auto installable = InstallableFlake(nullptr, evalState, std::move(templateFlakeRef), templateName, ExtendedOutputsSpec::Default(), @@ -909,11 +909,11 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand NixStringContext context; auto templateDir = evalState->coerceToPath(noPos, templateDirAttr, context, ""); - std::vector changedFiles; - std::vector conflictedFiles; + std::vector changedFiles; + std::vector conflictedFiles; - std::function copyDir; - copyDir = [&](const SourcePath & from, const fs::path & to) + std::function copyDir; + copyDir = [&](const SourcePath & from, const std::filesystem::path & to) { createDirs(to); @@ -922,12 +922,12 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto from2 = from / name; auto to2 = to / name; auto st = from2.lstat(); - auto to_st = fs::symlink_status(to2); + auto to_st = std::filesystem::symlink_status(to2); if (st.type == SourceAccessor::tDirectory) copyDir(from2, to2); else if (st.type == SourceAccessor::tRegular) { auto contents = from2.readFile(); - if (fs::exists(to_st)) { + if (std::filesystem::exists(to_st)) { auto contents2 = readFile(to2.string()); if (contents != contents2) { printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2); @@ -941,8 +941,8 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } else if (st.type == SourceAccessor::tSymlink) { auto target = from2.readLink(); - if (fs::exists(to_st)) { - if (fs::read_symlink(to2) != target) { + if (std::filesystem::exists(to_st)) { + if (std::filesystem::read_symlink(to2) != target) { printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2); conflictedFiles.push_back(to2); } else { @@ -961,7 +961,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand copyDir(templateDir, flakeDir); - if (!changedFiles.empty() && fs::exists(std::filesystem::path{flakeDir} / ".git")) { + if (!changedFiles.empty() && std::filesystem::exists(std::filesystem::path{flakeDir} / ".git")) { Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" }; for (auto & s : changedFiles) args.emplace_back(s.string()); runProgram("git", true, args); @@ -1128,7 +1128,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun {"path", store->printStorePath(storePath)}, {"inputs", traverse(*flake.lockFile.root)}, }; - logger->cout("%s", jsonRoot); + printJSON(jsonRoot); } else { traverse(*flake.lockFile.root); } @@ -1342,18 +1342,34 @@ struct CmdFlakeShow : FlakeCommand, MixJSON logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); } } else { - if (visitor.isDerivation()) - showDerivation(); - else - throw Error("expected a derivation"); + try { + if (visitor.isDerivation()) + showDerivation(); + else + throw Error("expected a derivation"); + } catch (IFDError & e) { + if (!json) { + logger->cout(fmt("%s " ANSI_WARNING "omitted due to use of import from derivation" ANSI_NORMAL, headerPrefix)); + } else { + logger->warn(fmt("%s omitted due to use of import from derivation", concatStringsSep(".", attrPathS))); + } + } } } else if (attrPath.size() > 0 && attrPathS[0] == "hydraJobs") { - if (visitor.isDerivation()) - showDerivation(); - else - recurse(); + try { + if (visitor.isDerivation()) + showDerivation(); + else + recurse(); + } catch (IFDError & e) { + if (!json) { + logger->cout(fmt("%s " ANSI_WARNING "omitted due to use of import from derivation" ANSI_NORMAL, headerPrefix)); + } else { + logger->warn(fmt("%s omitted due to use of import from derivation", concatStringsSep(".", attrPathS))); + } + } } else if (attrPath.size() > 0 && attrPathS[0] == "legacyPackages") { @@ -1372,11 +1388,19 @@ struct CmdFlakeShow : FlakeCommand, MixJSON logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); } } else { - if (visitor.isDerivation()) - showDerivation(); - else if (attrPath.size() <= 2) - // FIXME: handle recurseIntoAttrs - recurse(); + try { + if (visitor.isDerivation()) + showDerivation(); + else if (attrPath.size() <= 2) + // FIXME: handle recurseIntoAttrs + recurse(); + } catch (IFDError & e) { + if (!json) { + logger->cout(fmt("%s " ANSI_WARNING "omitted due to use of import from derivation" ANSI_NORMAL, headerPrefix)); + } else { + logger->warn(fmt("%s omitted due to use of import from derivation", concatStringsSep(".", attrPathS))); + } + } } } @@ -1440,7 +1464,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto j = visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); if (json) - logger->cout("%s", j.dump()); + printJSON(j); } }; @@ -1486,7 +1510,8 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON res["hash"] = hash.to_string(HashFormat::SRI, true); res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); - logger->cout(res.dump()); + res["locked"].erase("__final"); // internal for now + printJSON(res); } else { notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), diff --git a/src/nix/flake.md b/src/nix/flake.md index 364302b61..6cb39fd5f 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -695,7 +695,7 @@ following fields: The attributes in `locked` are considered "final", meaning that they are the only ones that are passed via the arguments of the `outputs` function of a flake. For instance, if `locked` contains a `lastModified` attribute while the fetcher does not return a `lastModified` attribute, then the `lastModified` attribute will be passed to the `outputs` function. Conversely, if `locked` does *not* contain a `lastModified` attribute while the fetcher *does* return a `lastModified` attribute, then no `lastModified` attribute will be passed. - If `locked` contains a `lastModifed` attribute and the fetcher returns a `lastModified` attribute, then they must have the same value. + If `locked` contains a `lastModified` attribute and the fetcher returns a `lastModified` attribute, then they must have the same value. * `flake`: A Boolean denoting whether this is a flake or non-flake dependency. Corresponds to the `flake` attribute in the `inputs` diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc deleted file mode 100644 index dc270fb8c..000000000 --- a/src/nix/fmt.cc +++ /dev/null @@ -1,55 +0,0 @@ -#include "nix/cmd/command.hh" -#include "nix/cmd/installable-value.hh" -#include "nix/expr/eval.hh" -#include "run.hh" - -using namespace nix; - -struct CmdFmt : SourceExprCommand { - std::vector args; - - CmdFmt() { expectArgs({.label = "args", .handler = {&args}}); } - - std::string description() override { - return "reformat your code in the standard style"; - } - - std::string doc() override { - return - #include "fmt.md" - ; - } - - Category category() override { return catSecondary; } - - Strings getDefaultFlakeAttrPaths() override { - return Strings{"formatter." + settings.thisSystem.get()}; - } - - Strings getDefaultFlakeAttrPathPrefixes() override { return Strings{}; } - - void run(ref store) override - { - auto evalState = getEvalState(); - auto evalStore = getEvalStore(); - - auto installable_ = parseInstallable(store, "."); - auto & installable = InstallableValue::require(*installable_); - auto app = installable.toApp(*evalState).resolve(evalStore, store); - - Strings programArgs{app.program}; - - // Propagate arguments from the CLI - for (auto &i : args) { - programArgs.push_back(i); - } - - // Release our references to eval caches to ensure they are persisted to disk, because - // we are about to exec out of this process without running C++ destructors. - evalState->evalCaches.clear(); - - execProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); - }; -}; - -static auto r2 = registerCommand("fmt"); diff --git a/src/nix/fmt.md b/src/nix/fmt.md deleted file mode 100644 index b4693eb65..000000000 --- a/src/nix/fmt.md +++ /dev/null @@ -1,47 +0,0 @@ -R""( - -# Description - -`nix fmt` calls the formatter specified in the flake. - -Flags can be forwarded to the formatter by using `--` followed by the flags. - -Any arguments will be forwarded to the formatter. Typically these are the files to format. - - -# Examples - -With [nixpkgs-fmt](https://github.com/nix-community/nixpkgs-fmt): - -```nix -# flake.nix -{ - outputs = { nixpkgs, self }: { - formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixpkgs-fmt; - }; -} -``` - -With [nixfmt](https://github.com/NixOS/nixfmt): - -```nix -# flake.nix -{ - outputs = { nixpkgs, self }: { - formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt-rfc-style; - }; -} -``` - -With [Alejandra](https://github.com/kamadorueda/alejandra): - -```nix -# flake.nix -{ - outputs = { nixpkgs, self }: { - formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.alejandra; - }; -} -``` - -)"" diff --git a/src/nix/formatter-build.md b/src/nix/formatter-build.md new file mode 100644 index 000000000..fbb6adb1c --- /dev/null +++ b/src/nix/formatter-build.md @@ -0,0 +1,23 @@ +R""( + +# Description + +`nix formatter build` builds the formatter specified in the flake. + +Similar to [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md), +unless `--no-link` is specified, after a successful +build, it creates a symlink to the store path of the formatter. This symlink is +named `./result` by default; this can be overridden using the +`--out-link` option. + +It always prints the command to standard output. + +# Examples + +* Build the formatter: + + ```console + # nix formatter build + /nix/store/cb9w44vkhk2x4adfxwgdkkf5gjmm856j-treefmt/bin/treefmt + ``` +)"" diff --git a/src/nix/formatter-run.md b/src/nix/formatter-run.md new file mode 100644 index 000000000..db8583c95 --- /dev/null +++ b/src/nix/formatter-run.md @@ -0,0 +1,25 @@ +R""( + +# Description + +`nix fmt` (an alias for `nix formatter run`) calls the formatter specified in the flake. + +Flags can be forwarded to the formatter by using `--` followed by the flags. + +Any arguments will be forwarded to the formatter. Typically these are the files to format. + + +# Example + +To use the [official Nix formatter](https://github.com/NixOS/nixfmt): + +```nix +# flake.nix +{ + outputs = { nixpkgs, self }: { + formatter.x86_64-linux = nixpkgs.legacyPackages.${system}.nixfmt-tree; + }; +} +``` + +)"" diff --git a/src/nix/formatter.cc b/src/nix/formatter.cc new file mode 100644 index 000000000..8b171b244 --- /dev/null +++ b/src/nix/formatter.cc @@ -0,0 +1,143 @@ +#include "nix/cmd/command.hh" +#include "nix/cmd/installable-value.hh" +#include "nix/expr/eval.hh" +#include "nix/store/local-fs-store.hh" +#include "nix/cmd/installable-derived-path.hh" +#include "run.hh" + +using namespace nix; + +struct CmdFormatter : NixMultiCommand +{ + CmdFormatter() + : NixMultiCommand("formatter", RegisterCommand::getCommandsFor({"formatter"})) + { + } + + std::string description() override + { + return "build or run the formatter"; + } + + Category category() override + { + return catSecondary; + } +}; + +static auto rCmdFormatter = registerCommand("formatter"); + +/** Common implementation bits for the `nix formatter` subcommands. */ +struct MixFormatter : SourceExprCommand +{ + Strings getDefaultFlakeAttrPaths() override + { + return Strings{"formatter." + settings.thisSystem.get()}; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + return Strings{}; + } +}; + +struct CmdFormatterRun : MixFormatter, MixJSON +{ + std::vector args; + + CmdFormatterRun() + { + expectArgs({.label = "args", .handler = {&args}}); + } + + std::string description() override + { + return "reformat your code in the standard style"; + } + + std::string doc() override + { + return +#include "formatter-run.md" + ; + } + + Category category() override + { + return catSecondary; + } + + void run(ref store) override + { + auto evalState = getEvalState(); + auto evalStore = getEvalStore(); + + auto installable_ = parseInstallable(store, "."); + auto & installable = InstallableValue::require(*installable_); + auto app = installable.toApp(*evalState).resolve(evalStore, store); + + Strings programArgs{app.program}; + + // Propagate arguments from the CLI + for (auto & i : args) { + programArgs.push_back(i); + } + + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + evalState->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); + }; +}; + +static auto rFormatterRun = registerCommand2({"formatter", "run"}); + +struct CmdFormatterBuild : MixFormatter, MixOutLinkByDefault +{ + CmdFormatterBuild() {} + + std::string description() override + { + return "build the current flake's formatter"; + } + + std::string doc() override + { + return +#include "formatter-build.md" + ; + } + + Category category() override + { + return catSecondary; + } + + void run(ref store) override + { + auto evalState = getEvalState(); + auto evalStore = getEvalStore(); + + auto installable_ = parseInstallable(store, "."); + auto & installable = InstallableValue::require(*installable_); + auto unresolvedApp = installable.toApp(*evalState); + auto app = unresolvedApp.resolve(evalStore, store); + auto buildables = unresolvedApp.build(evalStore, store); + createOutLinksMaybe(buildables, store); + + logger->cout("%s", app.program); + }; +}; + +static auto rFormatterBuild = registerCommand2({"formatter", "build"}); + +struct CmdFmt : CmdFormatterRun +{ + void run(ref store) override + { + CmdFormatterRun::run(store); + } +}; + +static auto rFmt = registerCommand("fmt"); diff --git a/src/nix/log.cc b/src/nix/log.cc index 00ab74ea6..78f1dd570 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -1,7 +1,7 @@ #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" #include "nix/main/shared.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/log-store.hh" using namespace nix; @@ -35,7 +35,7 @@ struct CmdLog : InstallableCommand // For compat with CLI today, TODO revisit auto oneUp = std::visit(overloaded { [&](const DerivedPath::Opaque & bo) { - return make_ref(bo); + return make_ref(bo); }, [&](const DerivedPath::Built & bfd) { return bfd.drvPath; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 1a90ed074..4b282bc43 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -8,8 +8,6 @@ using namespace nix; struct MixLs : virtual Args, MixJSON { - std::string path; - bool recursive = false; bool verbose = false; bool showDirectory = false; @@ -38,7 +36,7 @@ struct MixLs : virtual Args, MixJSON }); } - void listText(ref accessor) + void listText(ref accessor, CanonPath path) { std::function doPath; @@ -77,26 +75,27 @@ struct MixLs : virtual Args, MixJSON showFile(curPath, relPath); }; - auto path2 = CanonPath(path); - auto st = accessor->lstat(path2); - doPath(st, path2, - st.type == SourceAccessor::Type::tDirectory ? "." : path2.baseName().value_or(""), + auto st = accessor->lstat(path); + doPath(st, path, + st.type == SourceAccessor::Type::tDirectory ? "." : path.baseName().value_or(""), showDirectory); } - void list(ref accessor) + void list(ref accessor, CanonPath path) { if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - logger->cout("%s", listNar(accessor, CanonPath(path), recursive)); + logger->cout("%s", listNar(accessor, path, recursive)); } else - listText(accessor); + listText(accessor, std::move(path)); } }; struct CmdLsStore : StoreCommand, MixLs { + std::string path; + CmdLsStore() { expectArgs({ @@ -120,7 +119,8 @@ struct CmdLsStore : StoreCommand, MixLs void run(ref store) override { - list(store->getFSAccessor()); + auto [storePath, rest] = store->toStorePath(path); + list(store->getFSAccessor(), CanonPath{storePath.to_string()} / CanonPath{rest}); } }; @@ -128,6 +128,8 @@ struct CmdLsNar : Command, MixLs { Path narPath; + std::string path; + CmdLsNar() { expectArgs({ @@ -152,7 +154,7 @@ struct CmdLsNar : Command, MixLs void run() override { - list(makeNarAccessor(readFile(narPath))); + list(makeNarAccessor(readFile(narPath)), CanonPath{path}); } }; diff --git a/src/nix/main.cc b/src/nix/main.cc index 098d461a3..b000b9916 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -5,10 +5,10 @@ #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" #include "nix/store/globals.hh" -#include "nix/util/config-global.hh" #include "nix/cmd/legacy.hh" #include "nix/main/shared.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" +#include "nix/store/store-registration.hh" #include "nix/store/filetransfer.hh" #include "nix/util/finally.hh" #include "nix/main/loggers.hh" @@ -191,13 +191,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs res["args"] = toJSON(); auto stores = nlohmann::json::object(); - for (auto & implem : *Implementations::registered) { - auto storeConfig = implem.getConfig(); - auto storeName = storeConfig->name(); + for (auto & [storeName, implem] : Implementations::registered()) { auto & j = stores[storeName]; - j["doc"] = storeConfig->doc(); - j["settings"] = storeConfig->toJSON(); - j["experimentalFeature"] = storeConfig->experimentalFeature(); + j["doc"] = implem.doc; + j["uri-schemes"] = implem.uriSchemes; + j["settings"] = implem.getConfig()->toJSON(); + j["experimentalFeature"] = implem.experimentalFeature; } res["stores"] = std::move(stores); res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); @@ -371,7 +370,7 @@ void mainWrapped(int argc, char * * argv) } { - auto legacy = (*RegisterLegacyCommand::commands)[programName]; + auto legacy = RegisterLegacyCommand::commands()[programName]; if (legacy) return legacy(argc, argv); } diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index f8f588ae9..5523ae279 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -1,5 +1,5 @@ #include "nix/cmd/command.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/make-content-addressed.hh" #include "nix/main/common-args.hh" @@ -44,7 +44,7 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, } auto json = json::object(); json["rewrites"] = jsonRewrites; - logger->cout("%s", json); + printJSON(json); } else { for (auto & path : storePaths) { auto i = remappings.find(path); diff --git a/src/nix/meson.build b/src/nix/meson.build index 901021330..11c30914b 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -78,7 +78,7 @@ nix_sources = [config_priv_h] + files( 'env.cc', 'eval.cc', 'flake.cc', - 'fmt.cc', + 'formatter.cc', 'hash.cc', 'log.cc', 'ls.cc', diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 329e15830..04af72646 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -154,11 +154,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON pathLen = std::max(pathLen, store->printStorePath(storePath).size()); if (json) { - logger->cout(pathInfoToJSON( + printJSON(pathInfoToJSON( *store, // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - showClosureSize).dump()); + showClosureSize)); } else { diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 4495a1489..9e5e3c09a 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -1,7 +1,7 @@ #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" #include "nix/main/shared.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/filetransfer.hh" #include "nix/util/finally.hh" #include "nix/main/loggers.hh" @@ -116,11 +116,11 @@ std::tuple prefetchFile( createDirs(unpacked); unpackTarfile(tmpFile.string(), unpacked); - auto entries = std::filesystem::directory_iterator{unpacked}; + auto entries = DirectoryIterator{unpacked}; /* If the archive unpacks to a single file/directory, then use that as the top-level. */ tmpFile = entries->path(); - auto fileCount = std::distance(entries, std::filesystem::directory_iterator{}); + auto fileCount = std::distance(entries, DirectoryIterator{}); if (fileCount != 1) { /* otherwise, use the directory itself */ tmpFile = unpacked; @@ -327,7 +327,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(storePath); res["hash"] = hash.to_string(HashFormat::SRI, true); - logger->cout(res.dump()); + printJSON(res); } else { notice("Downloaded '%s' to '%s' (hash '%s').", url, diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 13ab0f659..2c593729f 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -67,7 +67,7 @@ struct ProfileElement * Return a string representing an installable corresponding to the current * element, either a flakeref or a plain store path */ - std::set toInstallables(Store & store) + StringSet toInstallables(Store & store) { if (source) return {source->to_string()}; @@ -600,7 +600,7 @@ public: }); } - std::set getMatchingElementNames(ProfileManifest & manifest) { + StringSet getMatchingElementNames(ProfileManifest & manifest) { if (_matchers.empty()) { throw UsageError("No packages specified."); } @@ -614,7 +614,7 @@ public: return {}; } - std::set result; + StringSet result; for (auto & matcher : _matchers) { bool foundMatch = false; for (auto & [name, element] : manifest.elements) { @@ -800,7 +800,7 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro ProfileManifest manifest(*getEvalState(), *profile); if (json) { - std::cout << manifest.toJSON(*store).dump() << "\n"; + printJSON(manifest.toJSON(*store)); } else { for (const auto & [i, e] : enumerate(manifest.elements)) { auto & [name, element] = e; diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index 77465e0b7..f21567639 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -57,7 +57,7 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON res.push_back(currentPath); } - logger->cout("%s", res); + printJSON(res); } else { for (auto & path : realisations) { diff --git a/src/nix/repl.cc b/src/nix/repl.cc index fcce43b8f..ca470e99b 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -2,6 +2,7 @@ #include "nix/expr/eval-settings.hh" #include "nix/util/config-global.hh" #include "nix/store/globals.hh" +#include "nix/store/store-open.hh" #include "nix/cmd/command.hh" #include "nix/cmd/installable-value.hh" #include "nix/cmd/repl.hh" diff --git a/src/nix/run.cc b/src/nix/run.cc index 146ae9ec9..0473c99b7 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -172,25 +172,25 @@ void chrootHelper(int argc, char * * argv) if (!pathExists(storeDir)) { // FIXME: Use overlayfs? - fs::path tmpDir = createTempDir(); + std::filesystem::path tmpDir = createTempDir(); createDirs(tmpDir + storeDir); if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - for (const auto & entry : fs::directory_iterator{"/"}) { + for (const auto & entry : DirectoryIterator{"/"}) { checkInterrupt(); const auto & src = entry.path(); - fs::path dst = tmpDir / entry.path().filename(); + std::filesystem::path dst = tmpDir / entry.path().filename(); if (pathExists(dst)) continue; auto st = entry.symlink_status(); - if (fs::is_directory(st)) { + if (std::filesystem::is_directory(st)) { if (mkdir(dst.c_str(), 0700) == -1) throw SysError("creating directory '%s'", dst); if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) throw SysError("mounting '%s' on '%s'", src, dst); - } else if (fs::is_symlink(st)) + } else if (std::filesystem::is_symlink(st)) createSymlink(readLink(src), dst); } @@ -208,9 +208,9 @@ void chrootHelper(int argc, char * * argv) if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - writeFile(fs::path{"/proc/self/setgroups"}, "deny"); - writeFile(fs::path{"/proc/self/uid_map"}, fmt("%d %d %d", uid, uid, 1)); - writeFile(fs::path{"/proc/self/gid_map"}, fmt("%d %d %d", gid, gid, 1)); + writeFile(std::filesystem::path{"/proc/self/setgroups"}, "deny"); + writeFile(std::filesystem::path{"/proc/self/uid_map"}, fmt("%d %d %d", uid, uid, 1)); + writeFile(std::filesystem::path{"/proc/self/gid_map"}, fmt("%d %d %d", gid, gid, 1)); #ifdef __linux__ if (system != "") diff --git a/src/nix/search.cc b/src/nix/search.cc index a27891c93..306a80594 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -198,7 +198,7 @@ struct CmdSearch : InstallableValueCommand, MixJSON visit(*cursor, cursor->getAttrPath(), true); if (json) - logger->cout("%s", *jsonOut); + printJSON(*jsonOut); if (!json && !results) throw Error("no results for the given search term(s)!"); diff --git a/src/nix/self-exe.cc b/src/nix/self-exe.cc index 5cc2326be..b5eb1190d 100644 --- a/src/nix/self-exe.cc +++ b/src/nix/self-exe.cc @@ -7,32 +7,28 @@ namespace nix { -namespace fs { -using namespace std::filesystem; -} - -fs::path getNixBin(std::optional binaryNameOpt) +std::filesystem::path getNixBin(std::optional binaryNameOpt) { auto getBinaryName = [&] { return binaryNameOpt ? *binaryNameOpt : "nix"; }; // If the environment variable is set, use it unconditionally. if (auto envOpt = getEnvNonEmpty("NIX_BIN_DIR")) - return fs::path{*envOpt} / std::string{getBinaryName()}; + return std::filesystem::path{*envOpt} / std::string{getBinaryName()}; // Try OS tricks, if available, to get to the path of this Nix, and // see if we can find the right executable next to that. if (auto selfOpt = getSelfExe()) { - fs::path path{*selfOpt}; + std::filesystem::path path{*selfOpt}; if (binaryNameOpt) path = path.parent_path() / std::string{*binaryNameOpt}; - if (fs::exists(path)) + if (std::filesystem::exists(path)) return path; } // If `nix` exists at the hardcoded fallback path, use it. { - auto path = fs::path{NIX_BIN_DIR} / std::string{getBinaryName()}; - if (fs::exists(path)) + auto path = std::filesystem::path{NIX_BIN_DIR} / std::string{getBinaryName()}; + if (std::filesystem::exists(path)) return path; } diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 9ef54a414..fb868baa1 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -1,7 +1,7 @@ #include "nix/util/signals.hh" #include "nix/cmd/command.hh" #include "nix/main/shared.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/util/thread-pool.hh" #include diff --git a/src/nix/store-info.cc b/src/nix/store-info.cc index 9402e8228..c4c63ae3a 100644 --- a/src/nix/store-info.cc +++ b/src/nix/store-info.cc @@ -33,7 +33,7 @@ struct CmdInfoStore : StoreCommand, MixJSON } else { nlohmann::json res; Finally printRes([&]() { - logger->cout("%s", res); + printJSON(res); }); res["url"] = store->getUri(); diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 607a7bb01..301f8aa50 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -7,6 +7,7 @@ #include "nix/store/local-store.hh" #include "nix/store/remote-store.hh" #include "nix/store/remote-store-connection.hh" +#include "nix/store/store-open.hh" #include "nix/util/serialise.hh" #include "nix/util/archive.hh" #include "nix/store/globals.hh" @@ -244,7 +245,7 @@ static PeerInfo getPeerInfo(int remote) */ static ref openUncachedStore() { - Store::Params params; // FIXME: get params from somewhere + Store::Config::Params params; // FIXME: get params from somewhere // Disable caching since the client already does that. params["path-info-cache-size"] = "0"; return openStore(settings.storeUri, params); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index ff81d78b6..eb2cde93c 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -1,6 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/main/shared.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/util/thread-pool.hh" #include "nix/util/signals.hh" #include "nix/store/keys.hh" diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 8dfd8343f..3aac45d34 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -172,7 +172,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions struct BailOut { }; printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) { - CanonPath pathS(store->printStorePath(node.path)); + CanonPath pathS(node.path.to_string()); assert(node.dist != inf); if (precise) { @@ -193,7 +193,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions /* Sort the references by distance to `dependency` to ensure that the shortest path is printed first. */ std::multimap refs; - std::set hashes; + StringSet hashes; for (auto & ref : node.refs) { if (ref == node.path && packagePath != dependencyPath) continue; diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index 34ed8b5f0..edcb6d72a 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -9,7 +9,7 @@ #include "nix/store/derivations.hh" #include "nix/store/realisation.hh" #include "nix/store/globals.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/util/posix-source-accessor.hh" #include diff --git a/tests/functional/build.sh b/tests/functional/build.sh index d65ac6854..0a19ff7da 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -179,14 +179,23 @@ test "$(<<<"$out" grep -cE '^error:')" = 4 out="$(nix build -f fod-failing.nix -L x4 2>&1)" && status=0 || status=$? test "$status" = 1 test "$(<<<"$out" grep -cE '^error:')" = 2 -<<<"$out" grepQuiet -E "error: Cannot build '.*-x4\\.drv'" -<<<"$out" grepQuiet -E "Reason: 1 dependency failed." + +if isDaemonNewer "2.29pre"; then + <<<"$out" grepQuiet -E "error: Cannot build '.*-x4\\.drv'" + <<<"$out" grepQuiet -E "Reason: 1 dependency failed." +else + <<<"$out" grepQuiet -E "error: 1 dependencies of derivation '.*-x4\\.drv' failed to build" +fi <<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x2\\.drv'" out="$(nix build -f fod-failing.nix -L x4 --keep-going 2>&1)" && status=0 || status=$? test "$status" = 1 test "$(<<<"$out" grep -cE '^error:')" = 3 -<<<"$out" grepQuiet -E "error: Cannot build '.*-x4\\.drv'" -<<<"$out" grepQuiet -E "Reason: 2 dependencies failed." +if isDaemonNewer "2.29pre"; then + <<<"$out" grepQuiet -E "error: Cannot build '.*-x4\\.drv'" + <<<"$out" grepQuiet -E "Reason: 2 dependencies failed." +else + <<<"$out" grepQuiet -E "error: 2 dependencies of derivation '.*-x4\\.drv' failed to build" +fi <<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'" <<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'" diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.nix b/tests/functional/derivation/advanced-attributes-structured-attrs.nix index 27d9e7cf9..46f619272 100644 --- a/tests/functional/derivation/advanced-attributes-structured-attrs.nix +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.nix @@ -83,4 +83,6 @@ derivation' { ]; preferLocalBuild = true; allowSubstitutes = false; + exportReferencesGraph.refs1 = [ foo ]; + exportReferencesGraph.refs2 = [ bar.drvPath ]; } diff --git a/tests/functional/derivation/advanced-attributes.nix b/tests/functional/derivation/advanced-attributes.nix index e988e0a70..dd0c09e22 100644 --- a/tests/functional/derivation/advanced-attributes.nix +++ b/tests/functional/derivation/advanced-attributes.nix @@ -67,4 +67,10 @@ derivation' { ]; preferLocalBuild = true; allowSubstitutes = false; + exportReferencesGraph = [ + "refs1" + foo + "refs2" + bar.drvPath + ]; } diff --git a/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv index a81e74d41..cd02c2f86 100644 --- a/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv +++ b/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv @@ -1 +1 @@ -Derive([("bin","","r:sha256",""),("dev","","r:sha256",""),("out","","r:sha256","")],[("/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",["dev","out"]),("/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",["dev","out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m"),("dev","/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9")]) \ No newline at end of file +Derive([("bin","","r:sha256",""),("dev","","r:sha256",""),("out","","r:sha256","")],[("/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",["dev","out"]),("/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",["dev","out"])],["/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"refs2\":[\"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m"),("dev","/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9")]) \ No newline at end of file diff --git a/tests/functional/derivation/ca/advanced-attributes.drv b/tests/functional/derivation/ca/advanced-attributes.drv index dded6c620..068cb593e 100644 --- a/tests/functional/derivation/ca/advanced-attributes.drv +++ b/tests/functional/derivation/ca/advanced-attributes.drv @@ -1 +1 @@ -Derive([("out","","r:sha256","")],[("/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",["dev","out"]),("/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",["dev","out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"),("allowedRequisites","/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"),("builder","/bin/bash"),("disallowedReferences","/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"),("disallowedRequisites","/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"),("outputHashAlgo","sha256"),("outputHashMode","recursive"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file +Derive([("out","","r:sha256","")],[("/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",["dev","out"]),("/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",["dev","out"])],["/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"),("allowedRequisites","/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"),("builder","/bin/bash"),("disallowedReferences","/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"),("disallowedRequisites","/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"),("exportReferencesGraph","refs1 /164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9 refs2 /nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"),("outputHashAlgo","sha256"),("outputHashMode","recursive"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv index 1560bca66..1dfcac42d 100644 --- a/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv +++ b/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv @@ -1 +1 @@ -Derive([("bin","/nix/store/qjjj3zrlimpjbkk686m052b3ks9iz2sl-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/lpz5grl48v93pdadavyg5is1rqvfdipf-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/nzvz1bmh1g89a5dkpqcqan0av7q3hgv3-advanced-attributes-structured-attrs","","")],[("/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv",["dev","out"]),("/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",["dev","out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/qjjj3zrlimpjbkk686m052b3ks9iz2sl-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/lpz5grl48v93pdadavyg5is1rqvfdipf-advanced-attributes-structured-attrs-dev"),("out","/nix/store/nzvz1bmh1g89a5dkpqcqan0av7q3hgv3-advanced-attributes-structured-attrs")]) \ No newline at end of file +Derive([("bin","/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs","","")],[("/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv",["dev","out"]),("/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",["dev","out"])],["/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"refs2\":[\"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev"),("out","/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs")]) \ No newline at end of file diff --git a/tests/functional/derivation/ia/advanced-attributes.drv b/tests/functional/derivation/ia/advanced-attributes.drv index 2c5d5a692..c71a88886 100644 --- a/tests/functional/derivation/ia/advanced-attributes.drv +++ b/tests/functional/derivation/ia/advanced-attributes.drv @@ -1 +1 @@ -Derive([("out","/nix/store/swkj0mrq0cq3dfli95v4am0427mi2hxf-advanced-attributes","","")],[("/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv",["dev","out"]),("/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",["dev","out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"),("allowedRequisites","/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"),("builder","/bin/bash"),("disallowedReferences","/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"),("disallowedRequisites","/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/swkj0mrq0cq3dfli95v4am0427mi2hxf-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file +Derive([("out","/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes","","")],[("/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv",["dev","out"]),("/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",["dev","out"])],["/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"),("allowedRequisites","/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"),("builder","/bin/bash"),("disallowedReferences","/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"),("disallowedRequisites","/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"),("exportReferencesGraph","refs1 /nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo refs2 /nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index ed9c214f5..f876f5ac4 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -39,6 +39,9 @@ nix-instantiate --eval -E 'assert 1 + 2 == 3; true' ln -sfn cycle.nix "$TEST_ROOT/cycle.nix" (! nix eval --file "$TEST_ROOT/cycle.nix") +# --file and --pure-eval don't mix. +expectStderr 1 nix eval --pure-eval --file "$TEST_ROOT/cycle.nix" | grepQuiet "not compatible" + # Check that relative symlinks are resolved correctly. mkdir -p "$TEST_ROOT/xyzzy" "$TEST_ROOT/foo" ln -sfn ../xyzzy "$TEST_ROOT/foo/bar" diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 06e414e9d..422cab96c 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -88,6 +88,19 @@ writeDependentFlake() { EOF } +writeIfdFlake() { + local flakeDir="$1" + cat > "$flakeDir/flake.nix" < "$flakeDir/flake.nix" <&1 | grep '/flakeB.*is forbidden in pure evaluation mode' -expect 1 nix flake lock --impure $flakeFollowsA 2>&1 | grep '/flakeB.*does not exist' +#expect 1 nix flake lock --impure $flakeFollowsA 2>&1 | grep '/flakeB.*does not exist' # FIXME # Test relative non-flake inputs. cat > $flakeFollowsA/flake.nix < show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.packages.${builtins.currentSystem}.default == { }; +true +' diff --git a/tests/functional/fmt.sh b/tests/functional/fmt.sh deleted file mode 100755 index e9bff50d5..000000000 --- a/tests/functional/fmt.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -source common.sh - -TODO_NixOS # Provide a `shell` variable. Try not to `export` it, perhaps. - -clearStoreIfPossible -rm -rf "$TEST_HOME"/.cache "$TEST_HOME"/.config "$TEST_HOME"/.local - -cp ./simple.nix ./simple.builder.sh ./fmt.simple.sh "${config_nix}" "$TEST_HOME" - -cd "$TEST_HOME" - -nix fmt --help | grep "forward" - -cat << EOF > flake.nix -{ - outputs = _: { - formatter.$system = - with import ./config.nix; - mkDerivation { - name = "formatter"; - buildCommand = '' - mkdir -p \$out/bin - echo "#! ${shell}" > \$out/bin/formatter - cat \${./fmt.simple.sh} >> \$out/bin/formatter - chmod +x \$out/bin/formatter - ''; - }; - }; -} -EOF -# No arguments check -[[ "$(nix fmt)" = "Formatting(0):" ]] -# Argument forwarding check -nix fmt ./file ./folder | grep 'Formatting(2): ./file ./folder' -nix flake check -nix flake show | grep -P "package 'formatter'" diff --git a/tests/functional/formatter.sh b/tests/functional/formatter.sh new file mode 100755 index 000000000..ea6f9e1ce --- /dev/null +++ b/tests/functional/formatter.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS # Provide a `shell` variable. Try not to `export` it, perhaps. + +clearStoreIfPossible +rm -rf "$TEST_HOME"/.cache "$TEST_HOME"/.config "$TEST_HOME"/.local + +cp ./simple.nix ./simple.builder.sh ./formatter.simple.sh "${config_nix}" "$TEST_HOME" + +cd "$TEST_HOME" + +nix formatter --help | grep "build or run the formatter" +nix fmt --help | grep "reformat your code" +nix fmt run --help | grep "reformat your code" +nix fmt build --help | grep "build" + +cat << EOF > flake.nix +{ + outputs = _: { + formatter.$system = + with import ./config.nix; + mkDerivation { + name = "formatter"; + buildCommand = '' + mkdir -p \$out/bin + echo "#! ${shell}" > \$out/bin/formatter + cat \${./formatter.simple.sh} >> \$out/bin/formatter + chmod +x \$out/bin/formatter + ''; + }; + }; +} +EOF + +# No arguments check +[[ "$(nix fmt)" = "Formatting(0):" ]] +[[ "$(nix formatter run)" = "Formatting(0):" ]] + +# Argument forwarding check +nix fmt ./file ./folder | grep 'Formatting(2): ./file ./folder' +nix formatter run ./file ./folder | grep 'Formatting(2): ./file ./folder' + +# Build checks +## Defaults to a ./result. +nix formatter build | grep ".\+/bin/formatter" +[[ -L ./result ]] +rm result + +## Can prevent the symlink. +nix formatter build --no-link +[[ ! -e ./result ]] + +## Can change the symlink name. +nix formatter build --out-link my-result | grep ".\+/bin/formatter" +[[ -L ./my-result ]] +rm ./my-result + +# Flake outputs check. +nix flake check +nix flake show | grep -P "package 'formatter'" diff --git a/tests/functional/fmt.simple.sh b/tests/functional/formatter.simple.sh similarity index 100% rename from tests/functional/fmt.simple.sh rename to tests/functional/formatter.simple.sh diff --git a/tests/functional/json.sh b/tests/functional/json.sh new file mode 100644 index 000000000..49992e0d9 --- /dev/null +++ b/tests/functional/json.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +source common.sh + +# Meson would split the output into two buffers, ruining the coherence of the log. +exec 1>&2 + +cat > "$TEST_HOME/expected-machine.json" < "$TEST_HOME/expected-pretty.json" < "$TEST_HOME/actual.json" +diff -U3 "$TEST_HOME/expected-machine.json" "$TEST_HOME/actual.json" + +nix eval --json --pretty --expr \ + '{ a.b.c = true; }' > "$TEST_HOME/actual.json" +diff -U3 "$TEST_HOME/expected-pretty.json" "$TEST_HOME/actual.json" + +if type script &>/dev/null; then + acceptsCommandFlag=0 + # The macOS version just accepts multiple arguments, but util-linux and its `-c` flag only accept a single argument, which is then split on whitespace. We thus have to quote in that case. + if script -c true /dev/null 2>/dev/null; then + acceptsCommandFlag=1 + fi + + runScript() { + if [[ $acceptsCommandFlag -eq 0 ]]; then + script -e -q /dev/null "$@" + else + script -e -q /dev/null -c "$(shellEscapeArray "$@")" + fi + } + runScript nix eval --json --expr "{ a.b.c = true; }" > "$TEST_HOME/actual.json" + cat "$TEST_HOME/actual.json" + # script isn't perfectly accurate? Let's grep for a pretty good indication, as the pretty output has a space between the key and the value. + # diff -U3 "$TEST_HOME/expected-pretty.json" "$TEST_HOME/actual.json" + grep -F '"a": {' "$TEST_HOME/actual.json" + + runScript nix eval --json --pretty --expr "{ a.b.c = true; }" > "$TEST_HOME/actual.json" + cat "$TEST_HOME/actual.json" + grep -F '"a": {' "$TEST_HOME/actual.json" + + runScript nix eval --json --no-pretty --expr "{ a.b.c = true; }" > "$TEST_HOME/actual.json" + cat "$TEST_HOME/actual.json" + grep -F '"a":{' "$TEST_HOME/actual.json" + +fi diff --git a/tests/functional/meson.build b/tests/functional/meson.build index af95879fb..b2005d9d9 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -132,7 +132,7 @@ suites = [ 'nix-copy-ssh-ng.sh', 'post-hook.sh', 'function-trace.sh', - 'fmt.sh', + 'formatter.sh', 'eval-store.sh', 'why-depends.sh', 'derivation-json.sh', @@ -157,6 +157,7 @@ suites = [ 'impure-derivations.sh', 'path-from-hash-part.sh', 'path-info.sh', + 'json.sh', 'toString-path.sh', 'read-only-store.sh', 'nested-sandboxing.sh', diff --git a/tests/functional/package.nix b/tests/functional/package.nix index a84ad1791..43f2f25a2 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -11,6 +11,7 @@ git, mercurial, util-linux, + unixtools, nix-store, nix-expr, @@ -54,6 +55,7 @@ mkMesonDerivation ( jq git mercurial + unixtools.script ] ++ lib.optionals stdenv.hostPlatform.isLinux [ # For various sandboxing tests that needs a statically-linked shell, diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh index 8ec093a48..2893efec7 100755 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -110,3 +110,13 @@ nix store verify --store "$TEST_ROOT"/store0 -r "$outPath2" --trusted-public-key # Content-addressed stuff can be copied without signatures. nix copy --to "$TEST_ROOT"/store0 "$outPathCA" + +# Test multiple signing keys +nix copy --to "file://$TEST_ROOT/storemultisig?secret-keys=$TEST_ROOT/sk1,$TEST_ROOT/sk2" "$outPath" +for file in "$TEST_ROOT/storemultisig/"*.narinfo; do + if [[ "$(grep -cE '^Sig: cache[1,2]\.example.org' "$file")" -ne 2 ]]; then + echo "ERROR: Cannot find cache1.example.org and cache2.example.org signatures in ${file}" + cat "${file}" + exit 1 + fi +done diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh index f080031f5..dfab2ddf4 100755 --- a/tests/functional/store-info.sh +++ b/tests/functional/store-info.sh @@ -3,9 +3,11 @@ source common.sh STORE_INFO=$(nix store info 2>&1) +LEGACY_STORE_INFO=$(nix store ping 2>&1) # alias to nix store info STORE_INFO_JSON=$(nix store info --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" +echo "$LEGACY_STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then DAEMON_VERSION=$("$NIX_DAEMON_PACKAGE"/bin/nix daemon --version | sed 's/.*) //') @@ -13,6 +15,7 @@ if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi + expect 127 NIX_REMOTE=unix:"$PWD"/store nix store info || \ fail "nix store info on a non-existent store should fail" diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index 53807603c..6b09cf6a5 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -73,7 +73,8 @@ test_tarball .gz gzip # All entries in tree.tar.gz refer to the same file, and all have the same inode when unpacked by GNU tar. # We don't preserve the hard links, because that's an optimization we think is not worth the complexity, # so we only make sure that the contents are copied correctly. -nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" --out-link "$TEST_ROOT/result" +json=$(nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" --out-link "$TEST_ROOT/result") +[[ $json =~ ^'{"hash":"sha256-'.*'","locked":{"lastModified":'.*',"narHash":"sha256-'.*'","type":"tarball","url":"file:///'.*'/tree.tar.gz"},"original":{"type":"tarball","url":"file:///'.*'/tree.tar.gz"},"storePath":"'.*'/store/'.*'-source"}'$ ]] [[ $(cat "$TEST_ROOT/result/a/b/foo") = bar ]] [[ $(cat "$TEST_ROOT/result/a/b/xyzzy") = bar ]] [[ $(cat "$TEST_ROOT/result/a/yyy") = bar ]] diff --git a/tests/functional/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc index 2c0402094..0dc5a5a46 100644 --- a/tests/functional/test-libstoreconsumer/main.cc +++ b/tests/functional/test-libstoreconsumer/main.cc @@ -1,5 +1,5 @@ #include "nix/store/globals.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/build-result.hh" #include diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 3e2d20a71..f0b1a8865 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -94,13 +94,6 @@ let ); }; - otherNixes.nix_2_18.setNixPackage = - { lib, pkgs, ... }: - { - imports = [ checkOverrideNixVersion ]; - nix.package = lib.mkForce pkgs.nixVersions.nix_2_18; - }; - in { diff --git a/tests/nixos/git-submodules.nix b/tests/nixos/git-submodules.nix index e56851790..9105eb79b 100644 --- a/tests/nixos/git-submodules.nix +++ b/tests/nixos/git-submodules.nix @@ -44,14 +44,14 @@ client.succeed("chmod 600 /root/.ssh/id_ed25519") # Install the SSH key on the builders. - client.wait_for_unit("network-online.target") + client.wait_for_unit("network-addresses-eth1.service") remote.succeed("mkdir -p -m 700 /root/.ssh") remote.copy_from_host("key.pub", "/root/.ssh/authorized_keys") remote.wait_for_unit("sshd") remote.wait_for_unit("multi-user.target") - remote.wait_for_unit("network-online.target") - client.wait_for_unit("network-online.target") + remote.wait_for_unit("network-addresses-eth1.service") + client.wait_for_unit("network-addresses-eth1.service") client.succeed(f"ssh -o StrictHostKeyChecking=no {remote.name} 'echo hello world'") remote.succeed(""" diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index eb7266b8c..ac1fb93a7 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -186,9 +186,9 @@ in github.succeed("cat /var/log/httpd/*.log >&2") github.wait_for_unit("httpd.service") - github.wait_for_unit("network-online.target") + github.wait_for_unit("network-addresses-eth1.service") - client.wait_for_unit("network-online.target") + client.wait_for_unit("network-addresses-eth1.service") client.succeed("curl -v https://github.com/ >&2") out = client.succeed("nix registry list") print(out) diff --git a/tests/nixos/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix index b6ec856e0..34e3a2c7d 100644 --- a/tests/nixos/nix-copy-closure.nix +++ b/tests/nixos/nix-copy-closure.nix @@ -70,9 +70,9 @@ in server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") server.wait_for_unit("sshd") server.wait_for_unit("multi-user.target") - server.wait_for_unit("network-online.target") + server.wait_for_unit("network-addresses-eth1.service") - client.wait_for_unit("network-online.target") + client.wait_for_unit("network-addresses-eth1.service") client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") # Copy the closure of package A from the client to the server. diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index 9fe1e8513..a7f0a6a32 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -78,9 +78,9 @@ in server.wait_for_unit("sshd") server.wait_for_unit("multi-user.target") - server.wait_for_unit("network-online.target") + server.wait_for_unit("network-addresses-eth1.service") - client.wait_for_unit("network-online.target") + client.wait_for_unit("network-addresses-eth1.service") client.wait_for_unit("getty@tty1.service") # Either the prompt: ]# # or an OCR misreading of it: 1# diff --git a/tests/nixos/nix-docker.nix b/tests/nixos/nix-docker.nix index bd77b25c8..c58a00cdd 100644 --- a/tests/nixos/nix-docker.nix +++ b/tests/nixos/nix-docker.nix @@ -61,7 +61,7 @@ in { nodes }: '' cache.wait_for_unit("harmonia.service") - cache.wait_for_unit("network-online.target") + cache.wait_for_unit("network-addresses-eth1.service") machine.succeed("mkdir -p /etc/containers") machine.succeed("""echo '{"default":[{"type":"insecureAcceptAnything"}]}' > /etc/containers/policy.json""") diff --git a/tests/nixos/nss-preload.nix b/tests/nixos/nss-preload.nix index 29cd5e6a2..d99f22208 100644 --- a/tests/nixos/nss-preload.nix +++ b/tests/nixos/nss-preload.nix @@ -145,7 +145,7 @@ in testScript = { nodes, ... }: '' - http_dns.wait_for_unit("network-online.target") + http_dns.wait_for_unit("network-addresses-eth1.service") http_dns.wait_for_unit("nginx") http_dns.wait_for_open_port(80) http_dns.wait_for_unit("unbound") @@ -153,7 +153,7 @@ in client.start() client.wait_for_unit('multi-user.target') - client.wait_for_unit('network-online.target') + client.wait_for_unit('network-addresses-eth1.service') with subtest("can fetch data from a remote server outside sandbox"): client.succeed("nix --version >&2") diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix index 726522029..c298ab92d 100644 --- a/tests/nixos/remote-builds-ssh-ng.nix +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -102,12 +102,12 @@ in client.succeed("chmod 600 /root/.ssh/id_ed25519") # Install the SSH key on the builder. - client.wait_for_unit("network-online.target") + client.wait_for_unit("network-addresses-eth1.service") builder.succeed("mkdir -p -m 700 /root/.ssh") builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") builder.wait_for_unit("sshd") builder.wait_for_unit("multi-user.target") - builder.wait_for_unit("network-online.target") + builder.wait_for_unit("network-addresses-eth1.service") client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 3251984db..fbfff9a7d 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -123,12 +123,12 @@ in client.succeed("chmod 600 /root/.ssh/id_ed25519") # Install the SSH key on the builders. - client.wait_for_unit("network-online.target") + client.wait_for_unit("network-addresses-eth1.service") for builder in [builder1, builder2]: builder.succeed("mkdir -p -m 700 /root/.ssh") builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") builder.wait_for_unit("sshd") - builder.wait_for_unit("network-online.target") + builder.wait_for_unit("network-addresses-eth1.service") # Make sure the builder can handle our login correctly builder.wait_for_unit("multi-user.target") # Make sure there's no funny business on the client either diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 8389281b6..136193c11 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -65,14 +65,14 @@ in # Create a binary cache. server.wait_for_unit("minio") - server.wait_for_unit("network-online.target") + server.wait_for_unit("network-addresses-eth1.service") server.succeed("mc config host add minio http://localhost:9000 ${accessKey} ${secretKey} --api s3v4") server.succeed("mc mb minio/my-cache") server.succeed("${env} nix copy --to '${storeUrl}' ${pkgA}") - client.wait_for_unit("network-online.target") + client.wait_for_unit("network-addresses-eth1.service") # Test fetchurl on s3:// URLs while we're at it. client.succeed("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"s3://my-cache/nix-cache-info?endpoint=http://server:9000®ion=eu-west-1\"; }'") diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix index 5a7912d30..ff9a42dcd 100644 --- a/tests/nixos/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -138,8 +138,8 @@ in start_all() sourcehut.wait_for_unit("httpd.service") - sourcehut.wait_for_unit("network-online.target") - client.wait_for_unit("network-online.target") + sourcehut.wait_for_unit("network-addresses-eth1.service") + client.wait_for_unit("network-addresses-eth1.service") client.succeed("curl -v https://git.sr.ht/ >&2") client.succeed("nix registry list | grep nixpkgs")