Merge remote-tracking branch 'origin/2.30-maintenance' into sync-2.30.0

.gitignore

@@ -47,3 +47,6 @@ result-*
 .DS_Store
 
 flake-regressions
+
+# direnv
+.direnv/

.mergify.yml

@@ -139,3 +139,14 @@ pull_request_rules:
         labels:
           - automatic backport
           - merge-queue
+
+  - name: backport patches to 2.29
+    conditions:
+      - label=backport 2.29-maintenance
+    actions:
+      backport:
+        branches:
+          - "2.29-maintenance"
+        labels:
+          - automatic backport
+          - merge-queue

.version

@@ -1 +1 @@
-2.29.1
+2.30.0

doc/manual/source/SUMMARY.md.in

@@ -52,6 +52,7 @@
   - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md)
   - [Verifying Build Reproducibility](advanced-topics/diff-hook.md)
   - [Using the `post-build-hook`](advanced-topics/post-build-hook.md)
+  - [Evaluation profiler](advanced-topics/eval-profiler.md)
 - [Command Reference](command-ref/index.md)
   - [Common Options](command-ref/opt-common.md)
   - [Common Environment Variables](command-ref/env-common.md)
@@ -147,6 +148,7 @@
   - [Release 3.0.0 (2025-03-04)](release-notes-determinate/rl-3.0.0.md)
 - [Nix Release Notes](release-notes/index.md)
 {{#include ./SUMMARY-rl-next.md}}
+  - [Release 2.30 (2025-07-07)](release-notes/rl-2.30.md)
   - [Release 2.29 (2025-05-14)](release-notes/rl-2.29.md)
   - [Release 2.28 (2025-04-02)](release-notes/rl-2.28.md)
   - [Release 2.27 (2025-03-03)](release-notes/rl-2.27.md)

doc/manual/source/advanced-topics/eval-profiler.md

@@ -0,0 +1,33 @@
+# Using the `eval-profiler`
+
+Nix evaluator supports [evaluation](@docroot@/language/evaluation.md)
+[profiling](<https://en.wikipedia.org/wiki/Profiling_(computer_programming)>)
+compatible with `flamegraph.pl`. The profiler samples the nix
+function call stack at regular intervals. It can be enabled with the
+[`eval-profiler`](@docroot@/command-ref/conf-file.md#conf-eval-profiler)
+setting:
+
+```console
+$ nix-instantiate "<nixpkgs>" -A hello --eval-profiler flamegraph
+```
+
+Stack sampling frequency and the output file path can be configured with
+[`eval-profile-file`](@docroot@/command-ref/conf-file.md#conf-eval-profile-file)
+and [`eval-profiler-frequency`](@docroot@/command-ref/conf-file.md#conf-eval-profiler-frequency).
+By default the collected profile is saved to `nix.profile` file in the current working directory.
+
+The collected profile can be directly consumed by `flamegraph.pl`:
+
+```console
+$ flamegraph.pl nix.profile > flamegraph.svg
+```
+
+The line information in the profile contains the location of the [call
+site](https://en.wikipedia.org/wiki/Call_site) position and the name of the
+function being called (when available). For example:
+
+```
+/nix/store/x9wnkly3k1gkq580m90jjn32q9f05q2v-source/pkgs/top-level/default.nix:167:5:primop import
+```
+
+Here `import` primop is called at `/nix/store/x9wnkly3k1gkq580m90jjn32q9f05q2v-source/pkgs/top-level/default.nix:167:5`.

doc/manual/source/command-ref/nix-channel.md

@@ -59,6 +59,11 @@ This command has the following operations:
   Download the Nix expressions of subscribed channels and create a new generation.
   Update all channels if none is specified, and only those included in *names* otherwise.
 
+  > **Note**
+  >
+  > Downloaded channel contents are cached.
+  > Use `--tarball-ttl` or the [`tarball-ttl` configuration option](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) to change the validity period of cached downloads.
+
 - `--list-generations`
 
   Prints a list of all the current existing generations for the

doc/manual/source/glossary.md

@@ -31,9 +31,22 @@
 
   The industry term for storage and retrieval systems using [content addressing](#gloss-content-address). A Nix store also has [input addressing](#gloss-input-addressed-store-object), and metadata.
 
+- [derivation]{#gloss-derivation}
+
+  A derivation can be thought of as a [pure function](https://en.wikipedia.org/wiki/Pure_function) that produces new [store objects][store object] from existing store objects.
+
+  Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md#builder-execution).
+  This sandbox by default only allows reading from store objects specified as inputs, and only allows writing to designated [outputs][output] to be [captured as store objects](@docroot@/store/building.md#processing-outputs).
+
+  A derivation is typically specified as a [derivation expression] in the [Nix language], and [instantiated][instantiate] to a [store derivation].
+  There are multiple ways of obtaining store objects from store derivatons, collectively called [realisation][realise].
+
+  [derivation]: #gloss-derivation
+
 - [store derivation]{#gloss-store-derivation}
 
-  A single build task.
+  A [derivation] represented as a [store object].
+
   See [Store Derivation](@docroot@/store/derivation/index.md#store-derivation) for details.
 
   [store derivation]: #gloss-store-derivation
@@ -57,10 +70,7 @@
 
 - [derivation expression]{#gloss-derivation-expression}
 
-  A description of a [store derivation] in the Nix language.
+  A description of a [store derivation] using the [`derivation` primitive](./language/derivations.md) in the [Nix language].
-  The output(s) of a derivation are store objects.
-  Derivations are typically specified in Nix expressions using the [`derivation` primitive](./language/derivations.md).
-  These are translated into store layer *derivations* (implicitly by `nix-env` and `nix-build`, or explicitly by `nix-instantiate`).
 
   [derivation expression]: #gloss-derivation-expression
 

doc/manual/source/language/advanced-attributes.md

@@ -53,23 +53,13 @@ Derivations can declare some infrequently used optional attributes.
 
   - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\
     If the special attribute `__structuredAttrs` is set to `true`, the other derivation
-    attributes are serialised into a file in JSON format. The environment variable
+    attributes are serialised into a file in JSON format.
-    `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build
-    and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for
-    [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions,
-    unlike process environments.
 
-    It also makes it possible to tweak derivation settings in a structured way; see
+    This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, unlike process environments.
-    [`outputChecks`](#adv-attr-outputChecks) for example.
+    It also makes it possible to tweak derivation settings in a structured way;
+    see [`outputChecks`](#adv-attr-outputChecks) for example.
 
-    As a convenience to Bash builders,
+    See the [corresponding section in the derivation page](@docroot@/store/derivation/index.md#structured-attrs) for further details.
-    Nix writes a script that initialises shell variables
-    corresponding to all attributes that are representable in Bash. The
-    environment variable `NIX_ATTRS_SH_FILE` points to the exact
-    location of the script, both in a build and a
-    [`nix-shell`](../command-ref/nix-shell.md). This includes non-nested
-    (associative) arrays. For example, the attribute `hardening.format = true`
-    ends up as the Bash associative array element `${hardening[format]}`.
 
     > **Warning**
     >

doc/manual/source/language/index.md

@@ -1,6 +1,6 @@
 # Nix Language
 
-The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files.
+The Nix language is designed for conveniently creating and composing [derivations](@docroot@/glossary.md#gloss-derivation) – precise descriptions of how contents of existing files are used to derive new files.
 
 > **Tip**
 >

doc/manual/source/language/operators.md

@@ -196,7 +196,7 @@ All comparison operators are implemented in terms of `<`, and the following equi
 
 ## Logical implication
 
-Equivalent to `!`*b1* `||` *b2*.
+Equivalent to `!`*b1* `||` *b2*  (or `if` *b1* `then` *b2* `else true`)
 
 [Logical implication]: #logical-implication
 

doc/manual/source/language/syntax.md

@@ -225,8 +225,8 @@ passed in first , e.g.,
 
 ```nix
 let add = { __functor = self: x: x + self.x; };
-    inc = add // { x = 1; };
+    inc = add // { x = 1; }; # inc is { x = 1; __functor = (...) }
-in inc 1
+in inc 1 # equivalent of `add.__functor add 1` i.e. `1 + self.x`
 ```
 
 evaluates to `2`. This can be used to attach metadata to a function

doc/manual/source/protocols/json/derivation.md

@@ -85,3 +85,7 @@ is a JSON object with the following fields:
 
 * `env`:
   The environment passed to the `builder`.
+
+* `structuredAttrs`:
+  [Strucutured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them.
+  Structured attributes are JSON, and thus embedded as-is.

doc/manual/source/release-notes/rl-2.24.md

@@ -284,7 +284,7 @@
 
   `<nix/fetchurl.nix>` is also known as the builtin derivation builder `builtin:fetchurl`. It's not to be confused with the evaluation-time function `builtins.fetchurl`, which was not affected by this issue.
 
-# Contributors
+## Contributors
 
 This release was made possible by the following 43 contributors:
 

doc/manual/source/release-notes/rl-2.25.md

@@ -77,7 +77,7 @@
   `<nix/fetchurl.nix>` is also known as the builtin derivation builder `builtin:fetchurl`. It's not to be confused with the evaluation-time function `builtins.fetchurl`, which was not affected by this issue.
 
 
-# Contributors
+## Contributors
 
 This release was made possible by the following 58 contributors:
 

doc/manual/source/release-notes/rl-2.26.md

@@ -76,7 +76,7 @@
 
 - Evaluation caching now works for dirty Git workdirs [#11992](https://github.com/NixOS/nix/pull/11992)
 
-# Contributors
+## Contributors
 
 This release was made possible by the following 45 contributors:
 

doc/manual/source/release-notes/rl-2.27.md

@@ -47,7 +47,7 @@
   blake3-34P4p+iZXcbbyB1i4uoF7eWCGcZHjmaRn6Y7QdynLwU=
   ```
 
-# Contributors
+## Contributors
 
 This release was made possible by the following 21 contributors:
 

doc/manual/source/release-notes/rl-2.28.md

@@ -82,7 +82,7 @@ This completes the infrastructure overhaul for the [RFC 132](https://github.com/
   Although this change is not as critical, we figured it would be good to do this API change at the same time, also.
   Also note that we try to keep the C API compatible, but we decided to break this function because it was young and likely not in widespread use yet. This frees up time to make important progress on the rest of the C API.
 
-# Contributors
+## Contributors
 
 This earlier-than-usual release was made possible by the following 16 contributors:
 

doc/manual/source/release-notes/rl-2.29.md

@@ -111,7 +111,7 @@ This fact is counterbalanced by the fact that most of those changes are bug fixe
   This in particular prevents parts of GCC 14's diagnostics from being improperly filtered away.
 
 
-# Contributors
+## Contributors
 
 
 This release was made possible by the following 40 contributors:

doc/manual/source/release-notes/rl-2.30.md

@@ -0,0 +1,153 @@
+# Release 2.30.0 (2025-07-07)
+
+## Backward-incompatible changes and deprecations
+
+- [`build-dir`] no longer defaults to `$TMPDIR`
+
+  The directory in which temporary build directories are created no longer defaults
+  to `TMPDIR` or `/tmp`, to avoid builders making their directories
+  world-accessible. This behavior allowed escaping the build sandbox and can
+  cause build impurities even when not used maliciously. We now default to `builds`
+  in `NIX_STATE_DIR` (which is `/nix/var/nix/builds` in the default configuration).
+
+- Deprecate manually making structured attrs using the `__json` attribute [#13220](https://github.com/NixOS/nix/pull/13220)
+
+  The proper way to create a derivation using [structured attrs] in the Nix language is by using `__structuredAttrs = true` with [`builtins.derivation`].
+  However, by exploiting how structured attrs are implementated, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string.
+  This sneaky alternative method is now deprecated, and may be disallowed in future versions of Nix.
+
+  [structured attrs]: @docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs
+  [`builtins.derivation`]: @docroot@/language/builtins.html#builtins-derivation
+
+- Rename `nix profile install` to [`nix profile add`] [#13224](https://github.com/NixOS/nix/pull/13224)
+
+  The command `nix profile install` has been renamed to [`nix profile add`] (though the former is still available as an alias). This is because the verb "add" is a better antonym for the verb "remove" (i.e. `nix profile remove`). Nix also does not have install hooks or general behavior often associated with "installing".
+
+## Performance improvements
+
+This release has a number performance improvements, in particular:
+
+- Reduce the size of value from 24 to 16 bytes [#13407](https://github.com/NixOS/nix/pull/13407)
+
+  This shaves off a very significant amount of memory used for evaluation (~20% percent reduction in maximum heap size and ~17% in total bytes).
+
+## Features
+
+- Add [stack sampling evaluation profiler] [#13220](https://github.com/NixOS/nix/pull/13220)
+
+  The Nix evaluator now supports [stack sampling evaluation profiling](@docroot@/advanced-topics/eval-profiler.md) via the [`--eval-profiler flamegraph`] setting.
+  It outputs collapsed call stack information to the file specified by
+  [`--eval-profile-file`] (`nix.profile` by default) in a format directly consumable
+  by `flamegraph.pl` and compatible tools like [speedscope](https://speedscope.app/).
+  Sampling frequency can be configured via [`--eval-profiler-frequency`] (99 Hz by default).
+
+  Unlike the existing [`--trace-function-calls`], this profiler includes the name of the function
+  being called when it's available.
+
+- [`nix repl`] prints which variables were loaded [#11406](https://github.com/NixOS/nix/pull/11406)
+
+  Instead of `Added <n> variables` it now prints the first 10 variables that were added to the global scope.
+
+- `nix flake archive`: Add [`--no-check-sigs`] option [#13277](https://github.com/NixOS/nix/pull/13277)
+
+  This is useful when using [`nix flake archive`] with the destination set to a remote store.
+
+- Emit warnings for IFDs with [`trace-import-from-derivation`] option [#13279](https://github.com/NixOS/nix/pull/13279)
+
+  While we have the setting [`allow-import-from-derivation`] to deny import-from-derivation (IFD), sometimes users would like to observe IFDs during CI processes to gradually phase out the idiom. The new setting `trace-import-from-derivation`, when set, logs a simple warning to the console.
+
+- `json-log-path` setting [#13003](https://github.com/NixOS/nix/pull/13003)
+
+  New setting [`json-log-path`] that sends a copy of all Nix log messages (in JSON format) to a file or Unix domain socket.
+
+- Non-flake inputs now contain a `sourceInfo` attribute [#13164](https://github.com/NixOS/nix/issues/13164) [#13170](https://github.com/NixOS/nix/pull/13170)
+
+  Flakes have always had a `sourceInfo` attribute which describes the source of the flake.
+  The `sourceInfo.outPath` is often identical to the flake's `outPath`. However, it can differ when the flake is located in a subdirectory of its source.
+
+  Non-flake inputs (i.e. inputs with [`flake = false`]) can also be located at some path _within_ a wider source.
+  This usually happens when defining a relative path input within the same source as the parent flake, e.g. `inputs.foo.url = ./some-file.nix`.
+  Such relative inputs will now inherit their parent's `sourceInfo`.
+
+  This also means it is now possible to use `?dir=subdir` on non-flake inputs.
+
+  This iterates on the work done in 2.26 to improve relative path support ([#10089](https://github.com/NixOS/nix/pull/10089)),
+  and resolves a regression introduced in 2.28 relating to nested relative path inputs ([#13164](https://github.com/NixOS/nix/issues/13164)).
+
+## Miscellaneous changes
+
+- [`builtins.sort`] uses PeekSort [#12623](https://github.com/NixOS/nix/pull/12623)
+
+  Previously it used libstdc++'s `std::stable_sort()`. However, that implementation is not reliable if the user-supplied comparison function is not a strict weak ordering.
+
+- Revert incomplete closure mixed download and build feature [#77](https://github.com/NixOS/nix/issues/77) [#12628](https://github.com/NixOS/nix/issues/12628) [#13176](https://github.com/NixOS/nix/pull/13176)
+
+  Since Nix 1.3 ([commit `299141e`] in 2013) Nix has attempted to mix together upstream fresh builds and downstream substitutions when remote substuters contain an "incomplete closure" (have some store objects, but not the store objects they reference).
+  This feature is now removed.
+
+  In the worst case, removing this feature could cause more building downstream, but it should not cause outright failures, since this is not happening for opaque store objects that we don't know how to build if we decide not to substitute.
+  In practice, however, we doubt even more building is very likely to happen.
+  Remote stores that are missing dependencies in arbitrary ways (e.g. corruption) don't seem to be very common.
+
+  On the contrary, when remote stores fail to implement the [closure property](@docroot@/store/store-object.md#closure-property), it is usually an *intentional* choice on the part of the remote store, because it wishes to serve as an "overlay" store over another store, such as `https://cache.nixos.org`.
+  If an "incomplete closure" is encountered in that situation, the right fix is not to do some sort of "franken-building" as this feature implemented, but instead to make sure both substituters are enabled in the settings.
+
+  (In the future, we should make it easier for remote stores to indicate this to clients, to catch settings that won't work in general before a missing dependency is actually encountered.)
+
+## Contributors
+
+This release was made possible by the following 32 contributors:
+
+- Cole Helbling [**(@cole-h)**](https://github.com/cole-h)
+- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra)
+- Egor Konovalov [**(@egorkonovalov)**](https://github.com/egorkonovalov)
+- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria)
+- Graham Christensen [**(@grahamc)**](https://github.com/grahamc)
+- gustavderdrache [**(@gustavderdrache)**](https://github.com/gustavderdrache)
+- Gwenn Le Bihan [**(@gwennlbh)**](https://github.com/gwennlbh)
+- h0nIg [**(@h0nIg)**](https://github.com/h0nIg)
+- Jade Masker [**(@donottellmetonottellyou)**](https://github.com/donottellmetonottellyou)
+- jayeshv [**(@jayeshv)**](https://github.com/jayeshv)
+- Jeremy Fleischman [**(@jfly)**](https://github.com/jfly)
+- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314)
+- Jonas Chevalier [**(@zimbatm)**](https://github.com/zimbatm)
+- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92)
+- kstrafe [**(@kstrafe)**](https://github.com/kstrafe)
+- Luc Perkins [**(@lucperkins)**](https://github.com/lucperkins)
+- Matt Sturgeon [**(@MattSturgeon)**](https://github.com/MattSturgeon)
+- Nikita Krasnov [**(@synalice)**](https://github.com/synalice)
+- Peder Bergebakken Sundt [**(@pbsds)**](https://github.com/pbsds)
+- pennae [**(@pennae)**](https://github.com/pennae)
+- Philipp Otterbein
+- Pol Dellaiera [**(@drupol)**](https://github.com/drupol)
+- PopeRigby [**(@poperigby)**](https://github.com/poperigby)
+- Raito Bezarius
+- Robert Hensing [**(@roberth)**](https://github.com/roberth)
+- Samuli Thomasson [**(@SimSaladin)**](https://github.com/SimSaladin)
+- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium)
+- Seth Flynn [**(@getchoo)**](https://github.com/getchoo)
+- Stefan Boca [**(@stefanboca)**](https://github.com/stefanboca)
+- tomberek [**(@tomberek)**](https://github.com/tomberek)
+- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy)
+- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk)
+- Vladimír Čunát [**(@vcunat)**](https://github.com/vcunat)
+- Wolfgang Walther [**(@wolfgangwalther)**](https://github.com/wolfgangwalther)
+
+<!-- markdown links -->
+[stack sampling evaluation profiler]: @docroot@/advanced-topics/eval-profiler.md
+[`--eval-profiler`]: @docroot@/command-ref/conf-file.md#conf-eval-profiler
+[`--eval-profiler flamegraph`]: @docroot@/command-ref/conf-file.md#conf-eval-profiler
+[`--trace-function-calls`]: @docroot@/command-ref/conf-file.md#conf-trace-function-calls
+[`--eval-profile-file`]: @docroot@/command-ref/conf-file.md#conf-eval-profile-file
+[`--eval-profiler-frequency`]: @docroot@/command-ref/conf-file.md#conf-eval-profiler-frequency
+[`build-dir`]: @docroot@/command-ref/conf-file.md#conf-build-dir
+[`nix profile add`]: @docroot@/command-ref/new-cli/nix3-profile-add.md
+[`nix repl`]: @docroot@/command-ref/new-cli/nix3-repl.md
+[`nix flake archive`]: @docroot@/command-ref/new-cli/nix3-flake-archive.md
+[`json-log-path`]: @docroot@/command-ref/conf-file.md#conf-json-log-path
+[`trace-import-from-derivation`]: @docroot@/command-ref/conf-file.md#conf-trace-import-from-derivation
+[`allow-import-from-derivation`]: @docroot@/command-ref/conf-file.md#conf-allow-import-from-derivation
+[`builtins.sort`]: @docroot@/language/builtins.md#builtins-sort
+[`flake = false`]: @docroot@/command-ref/new-cli/nix3-flake.md?highlight=false#flake-inputs
+[`--no-check-sigs`]: @docroot@/command-ref/new-cli/nix3-flake-archive.md#opt-no-check-sigs
+[commit `299141e`]: https://github.com/NixOS/nix/commit/299141ecbd08bae17013226dbeae71e842b4fdd7

doc/manual/source/store/derivation/index.md

@@ -138,6 +138,17 @@ See [Wikipedia](https://en.wikipedia.org/wiki/Argv) for details.
 
 Environment variables which will be passed to the [builder](#builder) executable.
 
+#### Structured Attributes {#structured-attrs}
+
+Nix also has special support for embedding JSON in the derivations.
+
+The environment variable `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build and a [`nix-shell`](@docroot@/command-ref/nix-shell.md).
+
+As a convenience to Bash builders, Nix writes a script that initialises shell variables corresponding to all attributes that are representable in Bash.
+The environment variable `NIX_ATTRS_SH_FILE` points to the exact location of the script, both in a build and a [`nix-shell`](@docroot@/command-ref/nix-shell.md).
+This includes non-nested (associative) arrays.
+For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`.
+
 ### Placeholders
 
 Placeholders are opaque values used within the [process creation fields] to [store objects] for which we don't yet know [store path]s.
@@ -162,7 +173,7 @@ There are two types of placeholder, corresponding to the two cases where this pr
 
 > **Explanation**
 >
-> In general, we need to realise [realise] a [store object] in order to be sure to have a store object for it.
+> In general, we need to [realise] a [store object] in order to be sure to have a store object for it.
 > But for these two cases this is either impossible or impractical:
 >
 > - In the output case this is impossible:
@@ -189,7 +200,7 @@ This ensures that there is a canonical [store path] used to refer to the derivat
 > **Note**
 >
 > Currently, the canonical encoding for every derivation is the "ATerm" format,
-> but this is subject to change for types derivations which are not yet stable.
+> but this is subject to change for the types of derivations which are not yet stable.
 
 Regardless of the format used, when serializing a derivation to a store object, that store object will be content-addressed.
 
@@ -282,7 +293,7 @@ type DerivingPath = ConstantPath | OutputPath;
 
 Under this extended model, `DerivingPath`s are thus inductively built up from a root `ConstantPath`, wrapped with zero or more outer `OutputPath`s.
 
-### Encoding {#deriving-path-encoding}
+### Encoding {#deriving-path-encoding-higher-order}
 
 The encoding is adjusted in the natural way, encoding the `drv` field recursively using the same deriving path encoding.
 The result of this is that it is possible to have a chain of `^<output-name>` at the end of the final string, as opposed to just a single one.

doc/manual/source/store/store-object.md

@@ -18,14 +18,14 @@ In particular, the edge corresponding to a reference is from the store object th
 References other than a self-reference must not form a cycle.
 The graph of references excluding self-references thus forms a [directed acyclic graph].
 
-[directed acyclic graph]: @docroot@/glossary.md#gloss-directed acyclic graph
+[directed acyclic graph]: @docroot@/glossary.md#gloss-directed-acyclic-graph
 
 We can take the [transitive closure] of the references graph, which any pair of store objects have an edge not if there is a single reference from the first to the second, but a path of one or more references from the first to the second.
 The *requisites* of a store object are all store objects reachable by paths of references which start with given store object's references.
 
 [transitive closure]: https://en.wikipedia.org/wiki/Transitive_closure
 
-We can also take the [transpose graph] ofthe references graph, where we reverse the orientation of all edges.
+We can also take the [transpose graph] of the references graph, where we reverse the orientation of all edges.
 The *referrers* of a store object are the store objects that reference it.
 
 [transpose graph]: https://en.wikipedia.org/wiki/Transpose_graph

docker.nix

@@ -1,6 +1,11 @@
 {
-  pkgs ? import <nixpkgs> { },
+  # Core dependencies
-  lib ? pkgs.lib,
+  pkgs,
+  lib,
+  dockerTools,
+  runCommand,
+  buildPackages,
+  # Image configuration
   name ? "nix",
   tag ? "latest",
   bundleNixpkgs ? true,
@@ -14,36 +19,60 @@
   gid ? 0,
   uname ? "root",
   gname ? "root",
+  Labels ? {
+    "org.opencontainers.image.title" = "Nix";
+    "org.opencontainers.image.source" = "https://github.com/NixOS/nix";
+    "org.opencontainers.image.vendor" = "Nix project";
+    "org.opencontainers.image.version" = nix.version;
+    "org.opencontainers.image.description" = "Nix container image";
+  },
+  Cmd ? [ (lib.getExe bashInteractive) ],
+  # Default Packages
+  nix,
+  bashInteractive,
+  coreutils-full,
+  gnutar,
+  gzip,
+  gnugrep,
+  which,
+  curl,
+  less,
+  wget,
+  man,
+  cacert,
+  findutils,
+  iana-etc,
+  gitMinimal,
+  openssh,
+  # Other dependencies
+  shadow,
 }:
 let
-  defaultPkgs =
+  defaultPkgs = [
-    with pkgs;
+    nix
-    [
+    bashInteractive
-      nix
+    coreutils-full
-      bashInteractive
+    gnutar
-      coreutils-full
+    gzip
-      gnutar
+    gnugrep
-      gzip
+    which
-      gnugrep
+    curl
-      which
+    less
-      curl
+    wget
-      less
+    man
-      wget
+    cacert.out
-      man
+    findutils
-      cacert.out
+    iana-etc
-      findutils
+    gitMinimal
-      iana-etc
+    openssh
-      git
+  ] ++ extraPkgs;
-      openssh
-    ]
-    ++ extraPkgs;
 
   users =
     {
 
       root = {
         uid = 0;
-        shell = "${pkgs.bashInteractive}/bin/bash";
+        shell = lib.getExe bashInteractive;
         home = "/root";
         gid = 0;
         groups = [ "root" ];
@@ -52,7 +81,7 @@ let
 
       nobody = {
         uid = 65534;
-        shell = "${pkgs.shadow}/bin/nologin";
+        shell = lib.getExe' shadow "nologin";
         home = "/var/empty";
         gid = 65534;
         groups = [ "nobody" ];
@@ -63,7 +92,7 @@ let
     // lib.optionalAttrs (uid != 0) {
       "${uname}" = {
         uid = uid;
-        shell = "${pkgs.bashInteractive}/bin/bash";
+        shell = lib.getExe bashInteractive;
         home = "/home/${uname}";
         gid = gid;
         groups = [ "${gname}" ];
@@ -147,41 +176,39 @@ let
     "${k}:x:${toString gid}:${lib.concatStringsSep "," members}";
   groupContents = (lib.concatStringsSep "\n" (lib.attrValues (lib.mapAttrs groupToGroup groups)));
 
-  defaultNixConf = {
+  toConf =
-    sandbox = "false";
+    with pkgs.lib.generators;
+    toKeyValue {
+      mkKeyValue = mkKeyValueDefault {
+        mkValueString = v: if lib.isList v then lib.concatStringsSep " " v else mkValueStringDefault { } v;
+      } " = ";
+    };
+
+  nixConfContents = toConf {
+    sandbox = false;
     build-users-group = "nixbld";
     trusted-public-keys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ];
   };
 
-  nixConfContents =
-    (lib.concatStringsSep "\n" (
-      lib.mapAttrsFlatten (
-        n: v:
-        let
-          vStr = if builtins.isList v then lib.concatStringsSep " " v else v;
-        in
-        "${n} = ${vStr}"
-      ) (defaultNixConf // nixConf)
-    ))
-    + "\n";
-
   userHome = if uid == 0 then "/root" else "/home/${uname}";
 
   baseSystem =
     let
       nixpkgs = pkgs.path;
-      channel = pkgs.runCommand "channel-nixos" { inherit bundleNixpkgs; } ''
+      channel = runCommand "channel-nixos" { inherit bundleNixpkgs; } ''
         mkdir $out
         if [ "$bundleNixpkgs" ]; then
-          ln -s ${nixpkgs} $out/nixpkgs
+          ln -s ${
+            builtins.path {
+              path = nixpkgs;
+              name = "source";
+            }
+          } $out/nixpkgs
           echo "[]" > $out/manifest.nix
         fi
       '';
-      rootEnv = pkgs.buildPackages.buildEnv {
+      # doc/manual/source/command-ref/files/manifest.nix.md
-        name = "root-profile-env";
+      manifest = buildPackages.runCommand "manifest.nix" { } ''
-        paths = defaultPkgs;
-      };
-      manifest = pkgs.buildPackages.runCommand "manifest.nix" { } ''
         cat > $out <<EOF
         [
         ${lib.concatStringsSep "\n" (
@@ -210,11 +237,15 @@ let
         ]
         EOF
       '';
-      profile = pkgs.buildPackages.runCommand "user-environment" { } ''
+      profile = buildPackages.buildEnv {
-        mkdir $out
+        name = "root-profile-env";
-        cp -a ${rootEnv}/* $out/
+        paths = defaultPkgs;
-        ln -s ${manifest} $out/manifest.nix
+
-      '';
+        postBuild = ''
+          mv $out/manifest $out/manifest.nix
+        '';
+        inherit manifest;
+      };
       flake-registry-path =
         if (flake-registry == null) then
           null
@@ -223,7 +254,7 @@ let
         else
           flake-registry;
     in
-    pkgs.runCommand "base-system"
+    runCommand "base-system"
       {
         inherit
           passwdContents
@@ -246,6 +277,7 @@ let
           set -x
           mkdir -p $out/etc
 
+          # may get replaced by pkgs.dockerTools.caCertificates
           mkdir -p $out/etc/ssl/certs
           ln -s /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt $out/etc/ssl/certs
 
@@ -273,20 +305,23 @@ let
           mkdir -p $out${userHome}
           mkdir -p $out/nix/var/nix/profiles/per-user/${uname}
 
+          # see doc/manual/source/command-ref/files/profiles.md
           ln -s ${profile} $out/nix/var/nix/profiles/default-1-link
           ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
-          ln -s /nix/var/nix/profiles/default $out${userHome}/.nix-profile
 
+          # see doc/manual/source/command-ref/files/channels.md
           ln -s ${channel} $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link
           ln -s /nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels
 
+          # see doc/manual/source/command-ref/files/default-nix-expression.md
           mkdir -p $out${userHome}/.nix-defexpr
           ln -s /nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels
           echo "${channelURL} ${channelName}" > $out${userHome}/.nix-channels
 
+          # may get replaced by pkgs.dockerTools.binSh & pkgs.dockerTools.usrBinEnv
           mkdir -p $out/bin $out/usr/bin
-          ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env
+          ln -s ${lib.getExe' coreutils-full "env"} $out/usr/bin/env
-          ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh
+          ln -s ${lib.getExe bashInteractive} $out/bin/sh
 
         ''
         + (lib.optionalString (flake-registry-path != null) ''
@@ -295,13 +330,13 @@ let
           globalFlakeRegistryPath="$nixCacheDir/flake-registry.json"
           ln -s ${flake-registry-path} $out$globalFlakeRegistryPath
           mkdir -p $out/nix/var/nix/gcroots/auto
-          rootName=$(${pkgs.nix}/bin/nix hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath))
+          rootName=$(${lib.getExe' nix "nix"} hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath))
           ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName
         '')
       );
 
 in
-pkgs.dockerTools.buildLayeredImageWithNixDb {
+dockerTools.buildLayeredImageWithNixDb {
 
   inherit
     name
@@ -327,7 +362,7 @@ pkgs.dockerTools.buildLayeredImageWithNixDb {
   '';
 
   config = {
-    Cmd = [ "${userHome}/.nix-profile/bin/bash" ];
+    inherit Cmd Labels;
     User = "${toString uid}:${toString gid}";
     Env = [
       "USER=${uname}"

flake.nix

@@ -172,19 +172,6 @@
               };
 
           nix = final.nixComponents2.nix-cli;
-
-          # See https://github.com/NixOS/nixpkgs/pull/214409
-          # Remove when fixed in this flake's nixpkgs
-          pre-commit =
-            if prev.stdenv.hostPlatform.system == "i686-linux" then
-              (prev.pre-commit.override (o: {
-                dotnet-sdk = "";
-              })).overridePythonAttrs
-                (o: {
-                  doCheck = false;
-                })
-            else
-              prev.pre-commit;
         };
 
     in
@@ -230,28 +217,24 @@
             This shouldn't build anything significant; just check that things
             (including derivations) are _set up_ correctly.
           */
-          # Disabled due to a bug in `testEqualContents` (see
+          packaging-overriding =
-          # https://github.com/NixOS/nix/issues/12690).
+            let
-          /*
+              pkgs = nixpkgsFor.${system}.native;
-            packaging-overriding =
+              nix = self.packages.${system}.nix;
-              let
+            in
-                pkgs = nixpkgsFor.${system}.native;
+            assert (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src.patches == [ pkgs.emptyFile ];
-                nix = self.packages.${system}.nix;
+            if pkgs.stdenv.buildPlatform.isDarwin then
-              in
+              lib.warn "packaging-overriding check currently disabled because of a permissions issue on macOS" pkgs.emptyFile
-              assert (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src.patches == [ pkgs.emptyFile ];
+            else
-              if pkgs.stdenv.buildPlatform.isDarwin then
+              # If this fails, something might be wrong with how we've wired the scope,
-                lib.warn "packaging-overriding check currently disabled because of a permissions issue on macOS" pkgs.emptyFile
+              # or something could be broken in Nixpkgs.
-              else
+              pkgs.testers.testEqualContents {
-                # If this fails, something might be wrong with how we've wired the scope,
+                assertion = "trivial patch does not change source contents";
-                # or something could be broken in Nixpkgs.
+                expected = "${./.}";
-                pkgs.testers.testEqualContents {
+                actual =
-                  assertion = "trivial patch does not change source contents";
+                  # Same for all components; nix-util is an arbitrary pick
-                  expected = "${./.}";
+                  (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src;
-                  actual =
+              };
-                    # Same for all components; nix-util is an arbitrary pick
-                    (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src;
-                };
-          */
         }
         // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
           dockerImage = self.hydraJobs.dockerImage.${system};
@@ -450,8 +433,7 @@
           dockerImage =
             let
               pkgs = nixpkgsFor.${system}.native;
-              image = import ./docker.nix {
+              image = pkgs.callPackage ./docker.nix {
-                inherit pkgs;
                 tag = pkgs.nix.version;
               };
             in

maintainers/data/release-credits-email-to-handle.json

@@ -166,5 +166,24 @@
     "the-tumultuous-unicorn-of-darkness@gmx.com": "TheTumultuousUnicornOfDarkness",
     "dev@rodney.id.au": "rvl",
     "pe@pijul.org": "P-E-Meunier",
-    "yannik@floxdev.com": "ysndr"
+    "yannik@floxdev.com": "ysndr",
+    "73017521+egorkonovalov@users.noreply.github.com": "egorkonovalov",
+    "raito@lix.systems": null,
+    "nikita.nikita.krasnov@gmail.com": "synalice",
+    "lucperkins@gmail.com": "lucperkins",
+    "vladimir.cunat@nic.cz": "vcunat",
+    "walther@technowledgy.de": "wolfgangwalther",
+    "jayesh.mail@gmail.com": "jayeshv",
+    "samuli.thomasson@pm.me": "SimSaladin",
+    "kevin@stravers.net": "kstrafe",
+    "poperigby@mailbox.org": "poperigby",
+    "cole.helbling@determinate.systems": "cole-h",
+    "donottellmetonottellyou@gmail.com": "donottellmetonottellyou",
+    "getchoo@tuta.io": "getchoo",
+    "alex.ford@determinate.systems": "gustavderdrache",
+    "stefan.r.boca@gmail.com": "stefanboca",
+    "gwenn.lebihan7@gmail.com": "gwennlbh",
+    "hey@ewen.works": "gwennlbh",
+    "matt@sturgeon.me.uk": "MattSturgeon",
+    "pbsds@hotmail.com": "pbsds"
 }

maintainers/data/release-credits-handle-to-name.json

@@ -146,5 +146,21 @@
     "ajlekcahdp4": "Alexander Romanov",
     "Valodim": "Vincent Breitmoser",
     "rvl": "Rodney Lorrimar",
-    "whatsthecraic": "Dean De Leo"
+    "whatsthecraic": "Dean De Leo",
+    "gwennlbh": "Gwenn Le Bihan",
+    "donottellmetonottellyou": "Jade Masker",
+    "kstrafe": null,
+    "synalice": "Nikita Krasnov",
+    "poperigby": "PopeRigby",
+    "MattSturgeon": "Matt Sturgeon",
+    "lucperkins": "Luc Perkins",
+    "gustavderdrache": null,
+    "SimSaladin": "Samuli Thomasson",
+    "getchoo": "Seth Flynn",
+    "stefanboca": "Stefan Boca",
+    "wolfgangwalther": "Wolfgang Walther",
+    "pbsds": "Peder Bergebakken Sundt",
+    "egorkonovalov": "Egor Konovalov",
+    "jayeshv": "jayeshv",
+    "vcunat": "Vladim\u00edr \u010cun\u00e1t"
 }

maintainers/flake-module.nix

@@ -37,6 +37,118 @@
               fi
             ''}";
           };
+          meson-format = {
+            enable = true;
+            files = "(meson.build|meson.options)$";
+            entry = "${pkgs.writeScript "format-meson" ''
+              #!${pkgs.runtimeShell}
+              for file in "$@"; do
+                ${lib.getExe pkgs.meson} format -ic ${../meson.format} "$file"
+              done
+            ''}";
+            excludes = [
+              # We haven't applied formatting to these files yet
+              ''^doc/manual/meson.build$''
+              ''^doc/manual/source/command-ref/meson.build$''
+              ''^doc/manual/source/development/meson.build$''
+              ''^doc/manual/source/language/meson.build$''
+              ''^doc/manual/source/meson.build$''
+              ''^doc/manual/source/release-notes/meson.build$''
+              ''^doc/manual/source/store/meson.build$''
+              ''^misc/bash/meson.build$''
+              ''^misc/fish/meson.build$''
+              ''^misc/launchd/meson.build$''
+              ''^misc/meson.build$''
+              ''^misc/systemd/meson.build$''
+              ''^misc/zsh/meson.build$''
+              ''^nix-meson-build-support/$''
+              ''^nix-meson-build-support/big-objs/meson.build$''
+              ''^nix-meson-build-support/common/meson.build$''
+              ''^nix-meson-build-support/deps-lists/meson.build$''
+              ''^nix-meson-build-support/export/meson.build$''
+              ''^nix-meson-build-support/export-all-symbols/meson.build$''
+              ''^nix-meson-build-support/generate-header/meson.build$''
+              ''^nix-meson-build-support/libatomic/meson.build$''
+              ''^nix-meson-build-support/subprojects/meson.build$''
+              ''^scripts/meson.build$''
+              ''^src/external-api-docs/meson.build$''
+              ''^src/internal-api-docs/meson.build$''
+              ''^src/libcmd/include/nix/cmd/meson.build$''
+              ''^src/libcmd/meson.build$''
+              ''^src/libcmd/nix-meson-build-support$''
+              ''^src/libexpr/include/nix/expr/meson.build$''
+              ''^src/libexpr/meson.build$''
+              ''^src/libexpr/nix-meson-build-support$''
+              ''^src/libexpr-c/meson.build$''
+              ''^src/libexpr-c/nix-meson-build-support$''
+              ''^src/libexpr-test-support/meson.build$''
+              ''^src/libexpr-test-support/nix-meson-build-support$''
+              ''^src/libexpr-tests/meson.build$''
+              ''^src/libexpr-tests/nix-meson-build-support$''
+              ''^src/libfetchers/include/nix/fetchers/meson.build$''
+              ''^src/libfetchers/meson.build$''
+              ''^src/libfetchers/nix-meson-build-support$''
+              ''^src/libfetchers-c/meson.build$''
+              ''^src/libfetchers-c/nix-meson-build-support$''
+              ''^src/libfetchers-tests/meson.build$''
+              ''^src/libfetchers-tests/nix-meson-build-support$''
+              ''^src/libflake/include/nix/flake/meson.build$''
+              ''^src/libflake/meson.build$''
+              ''^src/libflake/nix-meson-build-support$''
+              ''^src/libflake-c/meson.build$''
+              ''^src/libflake-c/nix-meson-build-support$''
+              ''^src/libflake-tests/meson.build$''
+              ''^src/libflake-tests/nix-meson-build-support$''
+              ''^src/libmain/include/nix/main/meson.build$''
+              ''^src/libmain/meson.build$''
+              ''^src/libmain/nix-meson-build-support$''
+              ''^src/libmain-c/meson.build$''
+              ''^src/libmain-c/nix-meson-build-support$''
+              ''^src/libstore/include/nix/store/meson.build$''
+              ''^src/libstore/meson.build$''
+              ''^src/libstore/nix-meson-build-support$''
+              ''^src/libstore/unix/include/nix/store/meson.build$''
+              ''^src/libstore/unix/meson.build$''
+              ''^src/libstore/windows/meson.build$''
+              ''^src/libstore-c/meson.build$''
+              ''^src/libstore-c/nix-meson-build-support$''
+              ''^src/libstore-test-support/include/nix/store/tests/meson.build$''
+              ''^src/libstore-test-support/meson.build$''
+              ''^src/libstore-test-support/nix-meson-build-support$''
+              ''^src/libstore-tests/meson.build$''
+              ''^src/libstore-tests/nix-meson-build-support$''
+              ''^src/libutil/meson.build$''
+              ''^src/libutil/nix-meson-build-support$''
+              ''^src/libutil/unix/include/nix/util/meson.build$''
+              ''^src/libutil/unix/meson.build$''
+              ''^src/libutil/windows/meson.build$''
+              ''^src/libutil-c/meson.build$''
+              ''^src/libutil-c/nix-meson-build-support$''
+              ''^src/libutil-test-support/include/nix/util/tests/meson.build$''
+              ''^src/libutil-test-support/meson.build$''
+              ''^src/libutil-test-support/nix-meson-build-support$''
+              ''^src/libutil-tests/meson.build$''
+              ''^src/libutil-tests/nix-meson-build-support$''
+              ''^src/nix/meson.build$''
+              ''^src/nix/nix-meson-build-support$''
+              ''^src/perl/lib/Nix/meson.build$''
+              ''^src/perl/meson.build$''
+              ''^tests/functional/ca/meson.build$''
+              ''^tests/functional/common/meson.build$''
+              ''^tests/functional/dyn-drv/meson.build$''
+              ''^tests/functional/flakes/meson.build$''
+              ''^tests/functional/git-hashing/meson.build$''
+              ''^tests/functional/local-overlay-store/meson.build$''
+              ''^tests/functional/meson.build$''
+              ''^src/libcmd/meson.options$''
+              ''^src/libexpr/meson.options$''
+              ''^src/libstore/meson.options$''
+              ''^src/libutil/meson.options$''
+              ''^src/libutil-c/meson.options$''
+              ''^src/nix/meson.options$''
+              ''^src/perl/meson.options$''
+            ];
+          };
           nixfmt-rfc-style = {
             enable = true;
             excludes = [
@@ -81,7 +193,6 @@
               # We haven't applied formatting to these files yet
               ''^doc/manual/redirects\.js$''
               ''^doc/manual/theme/highlight\.js$''
-              ''^precompiled-headers\.h$''
               ''^src/build-remote/build-remote\.cc$''
               ''^src/libcmd/built-path\.cc$''
               ''^src/libcmd/include/nix/cmd/built-path\.hh$''
@@ -145,7 +256,6 @@
               ''^src/libexpr/include/nix/expr/value-to-json\.hh$''
               ''^src/libexpr/value-to-xml\.cc$''
               ''^src/libexpr/include/nix/expr/value-to-xml\.hh$''
-              ''^src/libexpr/include/nix/expr/value\.hh$''
               ''^src/libexpr/value/context\.cc$''
               ''^src/libexpr/include/nix/expr/value/context\.hh$''
               ''^src/libfetchers/attrs\.cc$''
@@ -276,6 +386,8 @@
               ''^src/libstore/store-api\.cc$''
               ''^src/libstore/include/nix/store/store-api\.hh$''
               ''^src/libstore/include/nix/store/store-dir-config\.hh$''
+              ''^src/libstore/build/derivation-building-goal\.cc$''
+              ''^src/libstore/include/nix/store/build/derivation-building-goal\.hh$''
               ''^src/libstore/build/derivation-goal\.cc$''
               ''^src/libstore/include/nix/store/build/derivation-goal\.hh$''
               ''^src/libstore/build/drv-output-substitution-goal\.cc$''
@@ -357,7 +469,7 @@
               ''^src/libutil/json-utils\.cc$''
               ''^src/libutil/include/nix/util/json-utils\.hh$''
               ''^src/libutil/linux/cgroup\.cc$''
-              ''^src/libutil/linux/namespaces\.cc$''
+              ''^src/libutil/linux/linux-namespaces\.cc$''
               ''^src/libutil/logging\.cc$''
               ''^src/libutil/include/nix/util/logging\.hh$''
               ''^src/libutil/memory-source-accessor\.cc$''

maintainers/release-notes

@@ -157,7 +157,7 @@ section_title="Release $version_full ($DATE)"
 
   if ! $IS_PATCH; then
     echo
-    echo "# Contributors"
+    echo "## Contributors"
     echo
     VERSION=$version_full ./maintainers/release-credits
   fi

meson.build

@@ -1,13 +1,16 @@
 # This is just a stub project to include all the others as subprojects
 # for development shell purposes
 
-project('nix-dev-shell', 'cpp',
+project(
+  'nix-dev-shell',
+  'cpp',
   version : files('.version'),
   subproject_dir : 'src',
   default_options : [
     'localstatedir=/nix/var',
+    # hack for trailing newline
   ],
-  meson_version : '>= 1.1'
+  meson_version : '>= 1.1',
 )
 
 # Internal Libraries

meson.format

@@ -0,0 +1,7 @@
+indent_by = '  '
+space_array = true
+kwargs_force_multiline = false
+wide_colon = true
+group_arg_value = true
+indent_before_comments = ' '
+use_editor_config = true

meson.options

@@ -1,13 +1,22 @@
 # vim: filetype=meson
 
-option('doc-gen', type : 'boolean', value : false,
+option(
+  'doc-gen',
+  type : 'boolean',
+  value : false,
   description : 'Generate documentation',
 )
 
-option('unit-tests', type : 'boolean', value : true,
+option(
+  'unit-tests',
+  type : 'boolean',
+  value : true,
   description : 'Build unit tests',
 )
 
-option('bindings', type : 'boolean', value : true,
+option(
+  'bindings',
+  type : 'boolean',
+  value : true,
   description : 'Build language bindings (e.g. Perl)',
 )

misc/systemd/nix-daemon.conf.in

@@ -1 +1,2 @@
-d @localstatedir@/nix/daemon-socket 0755 root root - -
+d @localstatedir@/nix/daemon-socket 0755 root root -  -
+d @localstatedir@/nix/builds        0755 root root 7d -

nix-meson-build-support/export/meson.build

@@ -11,12 +11,18 @@ endforeach
 requires_public += deps_public
 
 extra_pkg_config_variables = get_variable('extra_pkg_config_variables', {})
+
+extra_cflags = []
+if not meson.project_name().endswith('-c')
+    extra_cflags += ['-std=c++2a']
+endif
+
 import('pkgconfig').generate(
   this_library,
   filebase : meson.project_name(),
   name : 'Nix',
   description : 'Nix Package Manager',
-  extra_cflags : ['-std=c++2a'],
+  extra_cflags : extra_cflags,
   requires : requires_public,
   requires_private : requires_private,
   libraries_private : libraries_private,

nix-meson-build-support/windows-version/meson.build

@@ -2,5 +2,5 @@ if host_machine.system() == 'windows'
   # https://learn.microsoft.com/en-us/cpp/porting/modifying-winver-and-win32-winnt?view=msvc-170
   # #define _WIN32_WINNT_WIN8 0x0602
   # We currently don't use any API which requires higher than this.
-  add_project_arguments([ '-D_WIN32_WINNT=0x0602' ], language: 'cpp')
+  add_project_arguments([ '-D_WIN32_WINNT=0x0602' ], language : 'cpp')
 endif

packaging/components.nix

@@ -157,6 +157,17 @@ let
     outputs = prevAttrs.outputs or [ "out" ] ++ [ "dev" ];
   };
 
+  fixupStaticLayer = finalAttrs: prevAttrs: {
+    postFixup =
+      prevAttrs.postFixup or ""
+      + lib.optionalString (stdenv.hostPlatform.isStatic) ''
+        # HACK: Otherwise the result will have the entire buildInputs closure
+        # injected by the pkgsStatic stdenv
+        # <https://github.com/NixOS/nixpkgs/issues/83667>
+        rm -f $out/nix-support/propagated-build-inputs
+      '';
+  };
+
   # Work around weird `--as-needed` linker behavior with BSD, see
   # https://github.com/mesonbuild/meson/issues/3593
   bsdNoLinkAsNeeded =
@@ -292,6 +303,7 @@ in
     scope.sourceLayer
     setVersionLayer
     mesonLayer
+    fixupStaticLayer
     scope.mesonComponentOverrides
   ];
   mkMesonExecutable = mkPackageBuilder [
@@ -301,6 +313,7 @@ in
     setVersionLayer
     mesonLayer
     mesonBuildLayer
+    fixupStaticLayer
     scope.mesonComponentOverrides
   ];
   mkMesonLibrary = mkPackageBuilder [
@@ -311,6 +324,7 @@ in
     setVersionLayer
     mesonBuildLayer
     mesonLibraryLayer
+    fixupStaticLayer
     scope.mesonComponentOverrides
   ];
 

precompiled-headers.h

@@ -1,63 +0,0 @@
-#include <algorithm>
-#include <array>
-#include <atomic>
-#include <cassert>
-#include <cctype>
-#include <chrono>
-#include <climits>
-#include <cmath>
-#include <condition_variable>
-#include <cstddef>
-#include <cstdint>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <exception>
-#include <functional>
-#include <future>
-#include <iostream>
-#include <limits>
-#include <list>
-#include <locale>
-#include <map>
-#include <memory>
-#include <mutex>
-#include <numeric>
-#include <optional>
-#include <queue>
-#include <random>
-#include <regex>
-#include <set>
-#include <sstream>
-#include <stack>
-#include <stdexcept>
-#include <string>
-#include <thread>
-#include <unordered_map>
-#include <unordered_set>
-#include <vector>
-
-#include <boost/format.hpp>
-
-#include <dirent.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <signal.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#ifndef _WIN32
-# include <grp.h>
-# include <netdb.h>
-# include <pwd.h>
-# include <sys/resource.h>
-# include <sys/select.h>
-# include <sys/socket.h>
-# include <sys/utsname.h>
-# include <sys/wait.h>
-# include <termios.h>
-#endif
-
-#include <nlohmann/json.hpp>

scripts/nix-profile-daemon.fish.in

@@ -1,17 +1,16 @@
 # Only execute this file once per shell.
-if test -z "$HOME" || \
+if test -z "$HOME" || test -n "$__ETC_PROFILE_NIX_SOURCED"
-    test -n "$__ETC_PROFILE_NIX_SOURCED"
     exit
 end
 
-set --global __ETC_PROFILE_NIX_SOURCED 1
+set --global --export __ETC_PROFILE_NIX_SOURCED 1
 
 # Local helpers
 
 function add_path --argument-names new_path
     if type -q fish_add_path
         # fish 3.2.0 or newer
-	fish_add_path --prepend --global $new_path
+        fish_add_path --prepend --global $new_path
     else
         # older versions of fish
         if not contains $new_path $fish_user_paths
@@ -24,7 +23,33 @@ end
 
 # Set up the per-user profile.
 
-set --local NIX_LINK $HOME/.nix-profile
+set --local NIX_LINK "$HOME/.nix-profile"
+set --local NIX_LINK_NEW
+if test -n "$XDG_STATE_HOME"
+    set NIX_LINK_NEW "$XDG_STATE_HOME/nix/profile"
+else
+    set NIX_LINK_NEW "$HOME/.local/state/nix/profile"
+end
+if test -e "$NIX_LINK_NEW"
+    if test -t 2; and test -e "$NIX_LINK"
+        set --local warning "\033[1;35mwarning:\033[0m "
+        printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
+
+        if test (realpath "$NIX_LINK") = (realpath "$NIX_LINK_NEW")
+            printf "         Since the profiles match, you can safely delete either of them.\n" 1>&2
+        else
+            # This should be an exceptionally rare occasion: the only way to get it would be to
+            # 1. Update to newer Nix;
+            # 2. Remove .nix-profile;
+            # 3. Set the $NIX_LINK_NEW to something other than the default user profile;
+            # 4. Roll back to older Nix.
+            # If someone did all that, they can probably figure out how to migrate the profile.
+            printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2
+        end
+    end
+
+    set NIX_LINK "$NIX_LINK_NEW"
+end
 
 # Set up environment.
 # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix

scripts/nix-profile.fish.in

@@ -1,17 +1,16 @@
 # Only execute this file once per shell.
-if test -z "$HOME" || \
+if test -z "$HOME" || test -n "$__ETC_PROFILE_NIX_SOURCED"
-    test -n "$__ETC_PROFILE_NIX_SOURCED"
     exit
 end
 
-set --global __ETC_PROFILE_NIX_SOURCED 1
+set --global --export __ETC_PROFILE_NIX_SOURCED 1
 
 # Local helpers
 
 function add_path --argument-names new_path
     if type -q fish_add_path
         # fish 3.2.0 or newer
-	fish_add_path --prepend --global $new_path
+        fish_add_path --prepend --global $new_path
     else
         # older versions of fish
         if not contains $new_path $fish_user_paths
@@ -24,7 +23,38 @@ end
 
 # Set up the per-user profile.
 
-set --local NIX_LINK $HOME/.nix-profile
+set --local NIX_LINK
+if test -n "$NIX_STATE_HOME"
+    set NIX_LINK "$NIX_STATE_HOME/.nix-profile"
+else
+    set NIX_LINK "$HOME/.nix-profile"
+    set --local NIX_LINK_NEW
+    if test -n "$XDG_STATE_HOME"
+        set NIX_LINK_NEW "$XDG_STATE_HOME/nix/profile"
+    else
+        set NIX_LINK_NEW "$HOME/.local/state/nix/profile"
+    end
+    if test -e "$NIX_LINK_NEW"
+        if test -t 2; and test -e "$NIX_LINK"
+            set --local warning "\033[1;35mwarning:\033[0m "
+            printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
+
+            if test (realpath "$NIX_LINK") = (realpath "$NIX_LINK_NEW")
+                printf "         Since the profiles match, you can safely delete either of them.\n" 1>&2
+            else
+                # This should be an exceptionally rare occasion: the only way to get it would be to
+                # 1. Update to newer Nix;
+                # 2. Remove .nix-profile;
+                # 3. Set the $NIX_LINK_NEW to something other than the default user profile;
+                # 4. Roll back to older Nix.
+                # If someone did all that, they can probably figure out how to migrate the profile.
+                printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2
+            end
+        end
+
+        set NIX_LINK "$NIX_LINK_NEW"
+    end
+end
 
 # Set up environment.
 # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix

src/libcmd/common-eval-args.cc

@@ -33,7 +33,12 @@ EvalSettings evalSettings {
                 auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
                 debug("fetching flake search path element '%s''", rest);
                 auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
-                auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
+                auto storePath = nix::fetchToStore(
+                    state.fetchSettings,
+                    *state.store,
+                    SourcePath(accessor),
+                    FetchMode::Copy,
+                    lockedRef.input.getName());
                 state.allowPath(storePath);
                 return state.storePath(storePath);
             },
@@ -176,14 +181,23 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
             state.store,
             state.fetchSettings,
             EvalSettings::resolvePseudoUrl(s));
-        auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
+        auto storePath = fetchToStore(
+            state.fetchSettings,
+            *state.store,
+            SourcePath(accessor),
+            FetchMode::Copy);
         return state.storePath(storePath);
     }
 
     else if (hasPrefix(s, "flake:")) {
         auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
         auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
-        auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
+        auto storePath = nix::fetchToStore(
+            state.fetchSettings,
+            *state.store,
+            SourcePath(accessor),
+            FetchMode::Copy,
+            lockedRef.input.getName());
         state.allowPath(storePath);
         return state.storePath(storePath);
     }

src/libcmd/include/nix/cmd/command.hh

@@ -338,7 +338,7 @@ struct MixEnvironment : virtual Args
 
     StringSet keepVars;
     StringSet unsetVars;
-    std::map<std::string, std::string> setVars;
+    StringMap setVars;
     bool ignoreEnvironment;
 
     MixEnvironment();

src/libcmd/include/nix/cmd/common-eval-args.hh

@@ -22,17 +22,17 @@ class Bindings;
 namespace flake { struct Settings; }
 
 /**
- * @todo Get rid of global setttings variables
+ * @todo Get rid of global settings variables
  */
 extern fetchers::Settings fetchSettings;
 
 /**
- * @todo Get rid of global setttings variables
+ * @todo Get rid of global settings variables
  */
 extern EvalSettings evalSettings;
 
 /**
- * @todo Get rid of global setttings variables
+ * @todo Get rid of global settings variables
  */
 extern flake::Settings flakeSettings;
 

src/libcmd/installable-value.cc

@@ -45,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
 std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     if (v.type() == nPath) {
-        auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy);
+        auto storePath = fetchToStore(state->fetchSettings, *state->store, v.path(), FetchMode::Copy);
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),

src/libcmd/repl-interacter.cc

@@ -2,6 +2,8 @@
 
 #include <cstdio>
 
+#include <signal.h>
+
 #if USE_READLINE
 #include <readline/history.h>
 #include <readline/readline.h>

src/libcmd/repl.cc

@@ -69,6 +69,7 @@ struct NixRepl
 
     const static int envSize = 32768;
     std::shared_ptr<StaticEnv> staticEnv;
+    Value lastLoaded;
     Env * env;
     int displ;
     StringSet varNames;
@@ -95,6 +96,7 @@ struct NixRepl
     void loadFiles();
     void loadFlakes();
     void reloadFilesAndFlakes();
+    void showLastLoaded();
     void addAttrsToScope(Value & attrs);
     void addVarToScope(const Symbol name, Value & v);
     Expr * parseString(std::string s);
@@ -158,6 +160,8 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
     return out;
 }
 
+MakeError(IncompleteReplExpr, ParseError);
+
 static bool isFirstRepl = true;
 
 ReplExitStatus NixRepl::mainLoop()
@@ -205,16 +209,8 @@ ReplExitStatus NixRepl::mainLoop()
                 default:
                     unreachable();
             }
-        } catch (ParseError & e) {
+        } catch (IncompleteReplExpr &) {
-            if (e.msg().find("unexpected end of file") != std::string::npos) {
+            continue;
-                // For parse errors on incomplete input, we continue waiting for the next line of
-                // input without clearing the input so far.
-                continue;
-            } else {
-              printMsg(lvlError, e.msg());
-            }
-        } catch (EvalError & e) {
-            printMsg(lvlError, e.msg());
         } catch (Error & e) {
             printMsg(lvlError, e.msg());
         } catch (Interrupted & e) {
@@ -294,7 +290,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
         } catch (BadURL & e) {
             // Quietly ignore BadURL flake-related errors.
         } catch (FileNotFound & e) {
-            // Quietly ignore non-existent file beeing `import`-ed.
+            // Quietly ignore non-existent file being `import`-ed.
         }
     }
 
@@ -378,6 +374,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
              << "                               current profile\n"
              << "  :l, :load <path>             Load Nix expression and add it to scope\n"
              << "  :lf, :load-flake <ref>       Load Nix flake and add it to scope\n"
+             << "  :ll, :last-loaded            Show most recently loaded variables added to scope\n"
              << "  :p, :print <expr>            Evaluate and print expression recursively\n"
              << "                               Strings are printed directly, without escaping.\n"
              << "  :q, :quit                    Exit nix-repl\n"
@@ -468,6 +465,10 @@ ProcessLineResult NixRepl::processLine(std::string line)
         loadFlake(arg);
     }
 
+    else if (command == ":ll" || command == ":last-loaded") {
+        showLastLoaded();
+    }
+
     else if (command == ":r" || command == ":reload") {
         state->resetFileCache();
         reloadFilesAndFlakes();
@@ -483,7 +484,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
                 auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit");
                 return {path, 0};
             } else if (v.isLambda()) {
-                auto pos = state->positions[v.payload.lambda.fun->pos];
+                auto pos = state->positions[v.lambda().fun->pos];
                 if (auto path = std::get_if<SourcePath>(&pos.origin))
                     return {*path, pos.line};
                 else
@@ -760,6 +761,16 @@ void NixRepl::initEnv()
         varNames.emplace(state->symbols[i.first]);
 }
 
+void NixRepl::showLastLoaded()
+{
+    RunPager pager;
+
+    for (auto & i : *lastLoaded.attrs()) {
+        std::string_view name = state->symbols[i.name];
+        logger->cout(name);
+    }
+}
+
 
 void NixRepl::reloadFilesAndFlakes()
 {
@@ -813,6 +824,27 @@ void NixRepl::addAttrsToScope(Value & attrs)
     staticEnv->sort();
     staticEnv->deduplicate();
     notice("Added %1% variables.", attrs.attrs()->size());
+
+    lastLoaded = attrs;
+
+    const int max_print = 20;
+    int counter = 0;
+    std::ostringstream loaded;
+    for (auto & i : attrs.attrs()->lexicographicOrder(state->symbols)) {
+        if (counter >= max_print)
+            break;
+
+        if (counter > 0)
+            loaded << ", ";
+
+        printIdentifier(loaded, state->symbols[i->name]);
+        counter += 1;
+    }
+
+    notice("%1%", loaded.str());
+
+    if (attrs.attrs()->size() > max_print)
+        notice("... and %1% more; view with :ll", attrs.attrs()->size() - max_print);
 }
 
 
@@ -837,7 +869,17 @@ Expr * NixRepl::parseString(std::string s)
 
 void NixRepl::evalString(std::string s, Value & v)
 {
-    Expr * e = parseString(s);
+    Expr * e;
+    try {
+        e = parseString(s);
+    } catch (ParseError & e) {
+        if (e.msg().find("unexpected end of file") != std::string::npos)
+            // For parse errors on incomplete input, we continue waiting for the next line of
+            // input without clearing the input so far.
+            throw IncompleteReplExpr(e.msg());
+        else
+            throw;
+    }
     e->eval(*state, *env, v);
     state->forceValue(v, v.determinePos(noPos));
 }

src/libexpr-c/nix_api_value.cc

@@ -252,7 +252,7 @@ const char * nix_get_path_string(nix_c_context * context, const nix_value * valu
         // We could use v.path().to_string().c_str(), but I'm concerned this
         // crashes. Looks like .path() allocates a CanonPath with a copy of the
         // string, then it gets the underlying data from that.
-        return v.payload.path.path;
+        return v.pathStr();
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -324,7 +324,7 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
     try {
         auto & v = check_value_in(value);
         assert(v.type() == nix::nList);
-        auto * p = v.listElems()[ix];
+        auto * p = v.listView()[ix];
         nix_gc_incref(nullptr, p);
         if (p != nullptr)
             state->state.forceValue(*p, nix::noPos);

src/libexpr-test-support/include/nix/expr/tests/meson.build

@@ -1,9 +1,10 @@
 # Public headers directory
 
-include_dirs = [include_directories('../../..')]
+include_dirs = [ include_directories('../../..') ]
 
 headers = files(
   'libexpr.hh',
   'nix_api_expr.hh',
   'value/context.hh',
+  # hack for trailing newline
 )

src/libexpr-tests/error_traces.cc

@@ -458,7 +458,7 @@ namespace nix {
                       HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")),
                       HintFmt("while evaluating the first argument passed to builtins.filterSource"));
 
-        // Usupported by store "dummy"
+        // Unsupported by store "dummy"
 
         // ASSERT_TRACE2("filterSource (_: 1) ./.",
         //               TypeError,
@@ -636,7 +636,7 @@ namespace nix {
                       HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")),
                       HintFmt("while evaluating the second argument passed to builtins.mapAttrs"));
 
-        // XXX: defered
+        // XXX: deferred
         // ASSERT_TRACE2("mapAttrs \"\" { foo.bar = 1; }",
         //               TypeError,
         //               HintFmt("attempt to call something which is not a function but %s", "a string"),
@@ -666,9 +666,9 @@ namespace nix {
                       HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)),
                       HintFmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith"));
 
-        // XXX: How to properly tell that the fucntion takes two arguments ?
+        // XXX: How to properly tell that the function takes two arguments ?
         // The same question also applies to sort, and maybe others.
-        // Due to lazyness, we only create a thunk, and it fails later on.
+        // Due to laziness, we only create a thunk, and it fails later on.
         // ASSERT_TRACE2("zipAttrsWith (_: 1) [ { foo = 1; } ]",
         //               TypeError,
         //               HintFmt("attempt to call something which is not a function but %s", "an integer"),
@@ -877,7 +877,7 @@ namespace nix {
                       HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)),
                       HintFmt("while evaluating the first argument passed to builtins.genList"));
 
-        // XXX: defered
+        // XXX: deferred
         // ASSERT_TRACE2("genList (x: x + \"foo\") 2 #TODO",
         //               TypeError,
         //               HintFmt("cannot add %s to an integer", "a string"),

src/libexpr-tests/meson.build

@@ -32,8 +32,8 @@ deps_private += rapidcheck
 gtest = dependency('gtest')
 deps_private += gtest
 
-gtest = dependency('gmock')
+gmock = dependency('gmock')
-deps_private += gtest
+deps_private += gmock
 
 configdata = configuration_data()
 configdata.set_quoted('PACKAGE_VERSION', meson.project_version())

src/libexpr-tests/primops.cc

@@ -150,8 +150,8 @@ namespace nix {
     TEST_F(PrimOpTest, attrValues) {
         auto v = eval("builtins.attrValues { x = \"foo\";  a = 1; }");
         ASSERT_THAT(v, IsListOfSize(2));
-        ASSERT_THAT(*v.listElems()[0], IsIntEq(1));
+        ASSERT_THAT(*v.listView()[0], IsIntEq(1));
-        ASSERT_THAT(*v.listElems()[1], IsStringEq("foo"));
+        ASSERT_THAT(*v.listView()[1], IsStringEq("foo"));
     }
 
     TEST_F(PrimOpTest, getAttr) {
@@ -250,8 +250,8 @@ namespace nix {
     TEST_F(PrimOpTest, catAttrs) {
         auto v = eval("builtins.catAttrs \"a\" [{a = 1;} {b = 0;} {a = 2;}]");
         ASSERT_THAT(v, IsListOfSize(2));
-        ASSERT_THAT(*v.listElems()[0], IsIntEq(1));
+        ASSERT_THAT(*v.listView()[0], IsIntEq(1));
-        ASSERT_THAT(*v.listElems()[1], IsIntEq(2));
+        ASSERT_THAT(*v.listView()[1], IsIntEq(2));
     }
 
     TEST_F(PrimOpTest, functionArgs) {
@@ -301,6 +301,7 @@ namespace nix {
 
     TEST_F(PrimOpTest, elemtAtOutOfBounds) {
         ASSERT_THROW(eval("builtins.elemAt [0 1 2 3] 5"), Error);
+        ASSERT_THROW(eval("builtins.elemAt [0] 4294967296"), Error);
     }
 
     TEST_F(PrimOpTest, head) {
@@ -319,7 +320,8 @@ namespace nix {
     TEST_F(PrimOpTest, tail) {
         auto v = eval("builtins.tail [ 3 2 1 0 ]");
         ASSERT_THAT(v, IsListOfSize(3));
-        for (const auto [n, elem] : enumerate(v.listItems()))
+        auto listView = v.listView();
+        for (const auto [n, elem] : enumerate(listView))
             ASSERT_THAT(*elem, IsIntEq(2 - static_cast<int>(n)));
     }
 
@@ -330,17 +332,17 @@ namespace nix {
     TEST_F(PrimOpTest, map) {
         auto v = eval("map (x: \"foo\" + x) [ \"bar\" \"bla\" \"abc\" ]");
         ASSERT_THAT(v, IsListOfSize(3));
-        auto elem = v.listElems()[0];
+        auto elem = v.listView()[0];
         ASSERT_THAT(*elem, IsThunk());
         state.forceValue(*elem, noPos);
         ASSERT_THAT(*elem, IsStringEq("foobar"));
 
-        elem = v.listElems()[1];
+        elem = v.listView()[1];
         ASSERT_THAT(*elem, IsThunk());
         state.forceValue(*elem, noPos);
         ASSERT_THAT(*elem, IsStringEq("foobla"));
 
-        elem = v.listElems()[2];
+        elem = v.listView()[2];
         ASSERT_THAT(*elem, IsThunk());
         state.forceValue(*elem, noPos);
         ASSERT_THAT(*elem, IsStringEq("fooabc"));
@@ -349,7 +351,7 @@ namespace nix {
     TEST_F(PrimOpTest, filter) {
         auto v = eval("builtins.filter (x: x == 2) [ 3 2 3 2 3 2 ]");
         ASSERT_THAT(v, IsListOfSize(3));
-        for (const auto elem : v.listItems())
+        for (const auto elem : v.listView())
             ASSERT_THAT(*elem, IsIntEq(2));
     }
 
@@ -366,7 +368,8 @@ namespace nix {
     TEST_F(PrimOpTest, concatLists) {
         auto v = eval("builtins.concatLists [[1 2] [3 4]]");
         ASSERT_THAT(v, IsListOfSize(4));
-        for (const auto [i, elem] : enumerate(v.listItems()))
+        auto listView = v.listView();
+        for (const auto [i, elem] : enumerate(listView))
             ASSERT_THAT(*elem, IsIntEq(static_cast<int>(i)+1));
     }
 
@@ -404,7 +407,8 @@ namespace nix {
         auto v = eval("builtins.genList (x: x + 1) 3");
         ASSERT_EQ(v.type(), nList);
         ASSERT_EQ(v.listSize(), 3u);
-        for (const auto [i, elem] : enumerate(v.listItems())) {
+        auto listView = v.listView();
+        for (const auto [i, elem] : enumerate(listView)) {
             ASSERT_THAT(*elem, IsThunk());
             state.forceValue(*elem, noPos);
             ASSERT_THAT(*elem, IsIntEq(static_cast<int>(i)+1));
@@ -417,7 +421,8 @@ namespace nix {
         ASSERT_EQ(v.listSize(), 6u);
 
         const std::vector<int> numbers = { 42, 77, 147, 249, 483, 526 };
-        for (const auto [n, elem] : enumerate(v.listItems()))
+        auto listView = v.listView();
+        for (const auto [n, elem] : enumerate(listView))
             ASSERT_THAT(*elem, IsIntEq(numbers[n]));
     }
 
@@ -428,17 +433,17 @@ namespace nix {
         auto right = v.attrs()->get(createSymbol("right"));
         ASSERT_NE(right, nullptr);
         ASSERT_THAT(*right->value, IsListOfSize(2));
-        ASSERT_THAT(*right->value->listElems()[0], IsIntEq(23));
+        ASSERT_THAT(*right->value->listView()[0], IsIntEq(23));
-        ASSERT_THAT(*right->value->listElems()[1], IsIntEq(42));
+        ASSERT_THAT(*right->value->listView()[1], IsIntEq(42));
 
         auto wrong = v.attrs()->get(createSymbol("wrong"));
         ASSERT_NE(wrong, nullptr);
         ASSERT_EQ(wrong->value->type(), nList);
         ASSERT_EQ(wrong->value->listSize(), 3u);
         ASSERT_THAT(*wrong->value, IsListOfSize(3));
-        ASSERT_THAT(*wrong->value->listElems()[0], IsIntEq(1));
+        ASSERT_THAT(*wrong->value->listView()[0], IsIntEq(1));
-        ASSERT_THAT(*wrong->value->listElems()[1], IsIntEq(9));
+        ASSERT_THAT(*wrong->value->listView()[1], IsIntEq(9));
-        ASSERT_THAT(*wrong->value->listElems()[2], IsIntEq(3));
+        ASSERT_THAT(*wrong->value->listView()[2], IsIntEq(3));
     }
 
     TEST_F(PrimOpTest, concatMap) {
@@ -447,7 +452,8 @@ namespace nix {
         ASSERT_EQ(v.listSize(), 6u);
 
         const std::vector<int> numbers = { 1, 2, 0, 3, 4, 0 };
-        for (const auto [n, elem] : enumerate(v.listItems()))
+        auto listView = v.listView();
+        for (const auto [n, elem] : enumerate(listView))
             ASSERT_THAT(*elem, IsIntEq(numbers[n]));
     }
 
@@ -592,6 +598,16 @@ namespace nix {
         ASSERT_THAT(v, IsStringEq("n"));
     }
 
+    TEST_F(PrimOpTest, substringHugeStart){
+        auto v = eval("builtins.substring 4294967296 5 \"nixos\"");
+        ASSERT_THAT(v, IsStringEq(""));
+    }
+
+    TEST_F(PrimOpTest, substringHugeLength){
+        auto v = eval("builtins.substring 0 4294967296 \"nixos\"");
+        ASSERT_THAT(v, IsStringEq("nixos"));
+    }
+
     TEST_F(PrimOpTest, substringEmptyString){
         auto v = eval("builtins.substring 1 3 \"\"");
         ASSERT_THAT(v, IsStringEq(""));
@@ -656,8 +672,8 @@ namespace nix {
         auto v = eval("derivation");
         ASSERT_EQ(v.type(), nFunction);
         ASSERT_TRUE(v.isLambda());
-        ASSERT_NE(v.payload.lambda.fun, nullptr);
+        ASSERT_NE(v.lambda().fun, nullptr);
-        ASSERT_TRUE(v.payload.lambda.fun->hasFormals());
+        ASSERT_TRUE(v.lambda().fun->hasFormals());
     }
 
     TEST_F(PrimOpTest, currentTime) {
@@ -671,7 +687,8 @@ namespace nix {
         ASSERT_THAT(v, IsListOfSize(4));
 
         const std::vector<std::string_view> strings = { "1", "2", "3", "git" };
-        for (const auto [n, p] : enumerate(v.listItems()))
+        auto listView = v.listView();
+        for (const auto [n, p] : enumerate(listView))
             ASSERT_THAT(*p, IsStringEq(strings[n]));
     }
 
@@ -761,12 +778,12 @@ namespace nix {
         auto v = eval("builtins.split \"(a)b\" \"abc\"");
         ASSERT_THAT(v, IsListOfSize(3));
 
-        ASSERT_THAT(*v.listElems()[0], IsStringEq(""));
+        ASSERT_THAT(*v.listView()[0], IsStringEq(""));
 
-        ASSERT_THAT(*v.listElems()[1], IsListOfSize(1));
+        ASSERT_THAT(*v.listView()[1], IsListOfSize(1));
-        ASSERT_THAT(*v.listElems()[1]->listElems()[0], IsStringEq("a"));
+        ASSERT_THAT(*v.listView()[1]->listView()[0], IsStringEq("a"));
 
-        ASSERT_THAT(*v.listElems()[2], IsStringEq("c"));
+        ASSERT_THAT(*v.listView()[2], IsStringEq("c"));
     }
 
     TEST_F(PrimOpTest, split2) {
@@ -774,17 +791,17 @@ namespace nix {
         auto v = eval("builtins.split \"([ac])\" \"abc\"");
         ASSERT_THAT(v, IsListOfSize(5));
 
-        ASSERT_THAT(*v.listElems()[0], IsStringEq(""));
+        ASSERT_THAT(*v.listView()[0], IsStringEq(""));
 
-        ASSERT_THAT(*v.listElems()[1], IsListOfSize(1));
+        ASSERT_THAT(*v.listView()[1], IsListOfSize(1));
-        ASSERT_THAT(*v.listElems()[1]->listElems()[0], IsStringEq("a"));
+        ASSERT_THAT(*v.listView()[1]->listView()[0], IsStringEq("a"));
 
-        ASSERT_THAT(*v.listElems()[2], IsStringEq("b"));
+        ASSERT_THAT(*v.listView()[2], IsStringEq("b"));
 
-        ASSERT_THAT(*v.listElems()[3], IsListOfSize(1));
+        ASSERT_THAT(*v.listView()[3], IsListOfSize(1));
-        ASSERT_THAT(*v.listElems()[3]->listElems()[0], IsStringEq("c"));
+        ASSERT_THAT(*v.listView()[3]->listView()[0], IsStringEq("c"));
 
-        ASSERT_THAT(*v.listElems()[4], IsStringEq(""));
+        ASSERT_THAT(*v.listView()[4], IsStringEq(""));
     }
 
     TEST_F(PrimOpTest, split3) {
@@ -792,36 +809,36 @@ namespace nix {
         ASSERT_THAT(v, IsListOfSize(5));
 
         // First list element
-        ASSERT_THAT(*v.listElems()[0], IsStringEq(""));
+        ASSERT_THAT(*v.listView()[0], IsStringEq(""));
 
         // 2nd list element is a list [ "" null ]
-        ASSERT_THAT(*v.listElems()[1], IsListOfSize(2));
+        ASSERT_THAT(*v.listView()[1], IsListOfSize(2));
-        ASSERT_THAT(*v.listElems()[1]->listElems()[0], IsStringEq("a"));
+        ASSERT_THAT(*v.listView()[1]->listView()[0], IsStringEq("a"));
-        ASSERT_THAT(*v.listElems()[1]->listElems()[1], IsNull());
+        ASSERT_THAT(*v.listView()[1]->listView()[1], IsNull());
 
         // 3rd element
-        ASSERT_THAT(*v.listElems()[2], IsStringEq("b"));
+        ASSERT_THAT(*v.listView()[2], IsStringEq("b"));
 
         // 4th element is a list: [ null "c" ]
-        ASSERT_THAT(*v.listElems()[3], IsListOfSize(2));
+        ASSERT_THAT(*v.listView()[3], IsListOfSize(2));
-        ASSERT_THAT(*v.listElems()[3]->listElems()[0], IsNull());
+        ASSERT_THAT(*v.listView()[3]->listView()[0], IsNull());
-        ASSERT_THAT(*v.listElems()[3]->listElems()[1], IsStringEq("c"));
+        ASSERT_THAT(*v.listView()[3]->listView()[1], IsStringEq("c"));
 
         // 5th element is the empty string
-        ASSERT_THAT(*v.listElems()[4], IsStringEq(""));
+        ASSERT_THAT(*v.listView()[4], IsStringEq(""));
     }
 
     TEST_F(PrimOpTest, split4) {
         auto v = eval("builtins.split \"([[:upper:]]+)\" \" FOO \"");
         ASSERT_THAT(v, IsListOfSize(3));
-        auto first = v.listElems()[0];
+        auto first = v.listView()[0];
-        auto second = v.listElems()[1];
+        auto second = v.listView()[1];
-        auto third = v.listElems()[2];
+        auto third = v.listView()[2];
 
         ASSERT_THAT(*first, IsStringEq(" "));
 
         ASSERT_THAT(*second, IsListOfSize(1));
-        ASSERT_THAT(*second->listElems()[0], IsStringEq("FOO"));
+        ASSERT_THAT(*second->listView()[0], IsStringEq("FOO"));
 
         ASSERT_THAT(*third, IsStringEq(" "));
     }
@@ -839,14 +856,14 @@ namespace nix {
     TEST_F(PrimOpTest, match3) {
         auto v = eval("builtins.match \"a(b)(c)\" \"abc\"");
         ASSERT_THAT(v, IsListOfSize(2));
-        ASSERT_THAT(*v.listElems()[0], IsStringEq("b"));
+        ASSERT_THAT(*v.listView()[0], IsStringEq("b"));
-        ASSERT_THAT(*v.listElems()[1], IsStringEq("c"));
+        ASSERT_THAT(*v.listView()[1], IsStringEq("c"));
     }
 
     TEST_F(PrimOpTest, match4) {
         auto v = eval("builtins.match \"[[:space:]]+([[:upper:]]+)[[:space:]]+\" \"  FOO   \"");
         ASSERT_THAT(v, IsListOfSize(1));
-        ASSERT_THAT(*v.listElems()[0], IsStringEq("FOO"));
+        ASSERT_THAT(*v.listView()[0], IsStringEq("FOO"));
     }
 
     TEST_F(PrimOpTest, match5) {
@@ -863,7 +880,8 @@ namespace nix {
 
         // ensure that the list is sorted
         const std::vector<std::string_view> expected { "a", "x", "y", "z" };
-        for (const auto [n, elem] : enumerate(v.listItems()))
+        auto listView = v.listView();
+        for (const auto [n, elem] : enumerate(listView))
             ASSERT_THAT(*elem, IsStringEq(expected[n]));
     }
 

src/libexpr-tests/trivial.cc

@@ -143,7 +143,7 @@ namespace nix {
         // Usually Nix rejects duplicate keys in an attrset but it does allow
         // so if it is an attribute set that contains disjoint sets of keys.
         // The below is equivalent to `{a.b = 1; a.c = 2; }`.
-        // The attribute set `a` will be a Thunk at first as the attribuets
+        // The attribute set `a` will be a Thunk at first as the attributes
         // have to be merged (or otherwise computed) and that is done in a lazy
         // manner.
 

src/libexpr/attr-path.cc

@@ -95,7 +95,7 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
             if (*attrIndex >= v->listSize())
                 throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath);
 
-            v = v->listElems()[*attrIndex];
+            v = v->listView()[*attrIndex];
             pos = noPos;
         }
 

src/libexpr/eval-cache.cc

@@ -724,7 +724,7 @@ std::vector<std::string> AttrCursor::getListOfStrings()
 
     std::vector<std::string> res;
 
-    for (auto & elem : v.listItems())
+    for (auto elem : v.listView())
         res.push_back(std::string(root->state.forceStringNoCtx(*elem, noPos, "while evaluating an attribute for caching")));
 
     if (root->db)

src/libexpr/eval-gc.cc

@@ -4,6 +4,7 @@
 #include "nix/util/config-global.hh"
 #include "nix/util/serialise.hh"
 #include "nix/expr/eval-gc.hh"
+#include "nix/expr/value.hh"
 
 #include "expr-config-private.hh"
 
@@ -52,6 +53,13 @@ static inline void initGCReal()
 
     GC_INIT();
 
+    /* Register valid displacements in case we are using alignment niches
+       for storing the type information. This way tagged pointers are considered
+       to be valid, even when they are not aligned. */
+    if constexpr (detail::useBitPackedValueStorage<sizeof(void *)>)
+        for (std::size_t i = 1; i < sizeof(std::uintptr_t); ++i)
+            GC_register_displacement(i);
+
     GC_set_oom_fn(oomHandler);
 
     /* Set the initial heap size to something fairly big (25% of

src/libexpr/eval-profiler-settings.cc

@@ -0,0 +1,49 @@
+#include "nix/expr/eval-profiler-settings.hh"
+#include "nix/util/configuration.hh"
+#include "nix/util/logging.hh" /* Needs to be included before config-impl.hh */
+#include "nix/util/config-impl.hh"
+#include "nix/util/abstract-setting-to-json.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+template<>
+EvalProfilerMode BaseSetting<EvalProfilerMode>::parse(const std::string & str) const
+{
+    if (str == "disabled")
+        return EvalProfilerMode::disabled;
+    else if (str == "flamegraph")
+        return EvalProfilerMode::flamegraph;
+    else
+        throw UsageError("option '%s' has invalid value '%s'", name, str);
+}
+
+template<>
+struct BaseSetting<EvalProfilerMode>::trait
+{
+    static constexpr bool appendable = false;
+};
+
+template<>
+std::string BaseSetting<EvalProfilerMode>::to_string() const
+{
+    if (value == EvalProfilerMode::disabled)
+        return "disabled";
+    else if (value == EvalProfilerMode::flamegraph)
+        return "flamegraph";
+    else
+        unreachable();
+}
+
+NLOHMANN_JSON_SERIALIZE_ENUM(
+    EvalProfilerMode,
+    {
+        {EvalProfilerMode::disabled, "disabled"},
+        {EvalProfilerMode::flamegraph, "flamegraph"},
+    });
+
+/* Explicit instantiation of templates */
+template class BaseSetting<EvalProfilerMode>;
+
+}

src/libexpr/eval-profiler.cc

@@ -0,0 +1,355 @@
+#include "nix/expr/eval-profiler.hh"
+#include "nix/expr/nixexpr.hh"
+#include "nix/expr/eval.hh"
+#include "nix/util/lru-cache.hh"
+
+namespace nix {
+
+void EvalProfiler::preFunctionCallHook(EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos) {}
+
+void EvalProfiler::postFunctionCallHook(EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
+{
+}
+
+void MultiEvalProfiler::preFunctionCallHook(
+    EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
+{
+    for (auto & profiler : profilers) {
+        if (profiler->getNeededHooks().test(Hook::preFunctionCall))
+            profiler->preFunctionCallHook(state, v, args, pos);
+    }
+}
+
+void MultiEvalProfiler::postFunctionCallHook(
+    EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
+{
+    for (auto & profiler : profilers) {
+        if (profiler->getNeededHooks().test(Hook::postFunctionCall))
+            profiler->postFunctionCallHook(state, v, args, pos);
+    }
+}
+
+EvalProfiler::Hooks MultiEvalProfiler::getNeededHooksImpl() const
+{
+    Hooks hooks;
+    for (auto & p : profilers)
+        hooks |= p->getNeededHooks();
+    return hooks;
+}
+
+void MultiEvalProfiler::addProfiler(ref<EvalProfiler> profiler)
+{
+    profilers.push_back(profiler);
+    invalidateNeededHooks();
+}
+
+namespace {
+
+class PosCache : private LRUCache<PosIdx, Pos>
+{
+    const EvalState & state;
+
+public:
+    PosCache(const EvalState & state)
+        : LRUCache(524288) /* ~40MiB */
+        , state(state)
+    {
+    }
+
+    Pos lookup(PosIdx posIdx)
+    {
+        auto posOrNone = LRUCache::get(posIdx);
+        if (posOrNone)
+            return *posOrNone;
+
+        auto pos = state.positions[posIdx];
+        upsert(posIdx, pos);
+        return pos;
+    }
+};
+
+struct LambdaFrameInfo
+{
+    ExprLambda * expr;
+    /** Position where the lambda has been called. */
+    PosIdx callPos = noPos;
+    std::ostream & symbolize(const EvalState & state, std::ostream & os, PosCache & posCache) const;
+    auto operator<=>(const LambdaFrameInfo & rhs) const = default;
+};
+
+/** Primop call. */
+struct PrimOpFrameInfo
+{
+    const PrimOp * expr;
+    /** Position where the primop has been called. */
+    PosIdx callPos = noPos;
+    std::ostream & symbolize(const EvalState & state, std::ostream & os, PosCache & posCache) const;
+    auto operator<=>(const PrimOpFrameInfo & rhs) const = default;
+};
+
+/** Used for functor calls (attrset with __functor attr). */
+struct FunctorFrameInfo
+{
+    PosIdx pos;
+    std::ostream & symbolize(const EvalState & state, std::ostream & os, PosCache & posCache) const;
+    auto operator<=>(const FunctorFrameInfo & rhs) const = default;
+};
+
+struct DerivationStrictFrameInfo
+{
+    PosIdx callPos = noPos;
+    std::string drvName;
+    std::ostream & symbolize(const EvalState & state, std::ostream & os, PosCache & posCache) const;
+    auto operator<=>(const DerivationStrictFrameInfo & rhs) const = default;
+};
+
+/** Fallback frame info. */
+struct GenericFrameInfo
+{
+    PosIdx pos;
+    std::ostream & symbolize(const EvalState & state, std::ostream & os, PosCache & posCache) const;
+    auto operator<=>(const GenericFrameInfo & rhs) const = default;
+};
+
+using FrameInfo =
+    std::variant<LambdaFrameInfo, PrimOpFrameInfo, FunctorFrameInfo, DerivationStrictFrameInfo, GenericFrameInfo>;
+using FrameStack = std::vector<FrameInfo>;
+
+/**
+ * Stack sampling profiler.
+ */
+class SampleStack : public EvalProfiler
+{
+    /* How often stack profiles should be flushed to file. This avoids the need
+       to persist stack samples across the whole evaluation at the cost
+       of periodically flushing data to disk. */
+    static constexpr std::chrono::microseconds profileDumpInterval = std::chrono::milliseconds(2000);
+
+    Hooks getNeededHooksImpl() const override
+    {
+        return Hooks().set(preFunctionCall).set(postFunctionCall);
+    }
+
+    FrameInfo getPrimOpFrameInfo(const PrimOp & primOp, std::span<Value *> args, PosIdx pos);
+
+public:
+    SampleStack(EvalState & state, std::filesystem::path profileFile, std::chrono::nanoseconds period)
+        : state(state)
+        , sampleInterval(period)
+        , profileFd([&]() {
+            AutoCloseFD fd = toDescriptor(open(profileFile.string().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0660));
+            if (!fd)
+                throw SysError("opening file %s", profileFile);
+            return fd;
+        }())
+        , posCache(state)
+    {
+    }
+
+    [[gnu::noinline]] void
+    preFunctionCallHook(EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos) override;
+    [[gnu::noinline]] void
+    postFunctionCallHook(EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos) override;
+
+    void maybeSaveProfile(std::chrono::time_point<std::chrono::high_resolution_clock> now);
+    void saveProfile();
+    FrameInfo getFrameInfoFromValueAndPos(const Value & v, std::span<Value *> args, PosIdx pos);
+
+    SampleStack(SampleStack &&) = default;
+    SampleStack & operator=(SampleStack &&) = delete;
+    SampleStack(const SampleStack &) = delete;
+    SampleStack & operator=(const SampleStack &) = delete;
+    ~SampleStack();
+private:
+    /** Hold on to an instance of EvalState for symbolizing positions. */
+    EvalState & state;
+    std::chrono::nanoseconds sampleInterval;
+    AutoCloseFD profileFd;
+    FrameStack stack;
+    std::map<FrameStack, uint32_t> callCount;
+    std::chrono::time_point<std::chrono::high_resolution_clock> lastStackSample =
+        std::chrono::high_resolution_clock::now();
+    std::chrono::time_point<std::chrono::high_resolution_clock> lastDump = std::chrono::high_resolution_clock::now();
+    PosCache posCache;
+};
+
+FrameInfo SampleStack::getPrimOpFrameInfo(const PrimOp & primOp, std::span<Value *> args, PosIdx pos)
+{
+    auto derivationInfo = [&]() -> std::optional<FrameInfo> {
+        /* Here we rely a bit on the implementation details of libexpr/primops/derivation.nix
+           and derivationStrict primop. This is not ideal, but is necessary for
+           the usefulness of the profiler. This might actually affect the evaluation,
+           but the cost shouldn't be that high as to make the traces entirely inaccurate. */
+        if (primOp.name == "derivationStrict") {
+            try {
+                /* Error context strings don't actually matter, since we ignore all eval errors. */
+                state.forceAttrs(*args[0], pos, "");
+                auto attrs = args[0]->attrs();
+                auto nameAttr = state.getAttr(state.sName, attrs, "");
+                auto drvName = std::string(state.forceStringNoCtx(*nameAttr->value, pos, ""));
+                return DerivationStrictFrameInfo{.callPos = pos, .drvName = std::move(drvName)};
+            } catch (...) {
+                /* Ignore all errors, since those will be diagnosed by the evaluator itself. */
+            }
+        }
+
+        return std::nullopt;
+    }();
+
+    return derivationInfo.value_or(PrimOpFrameInfo{.expr = &primOp, .callPos = pos});
+}
+
+FrameInfo SampleStack::getFrameInfoFromValueAndPos(const Value & v, std::span<Value *> args, PosIdx pos)
+{
+    /* NOTE: No actual references to garbage collected values are not held in
+       the profiler. */
+    if (v.isLambda())
+        return LambdaFrameInfo{.expr = v.lambda().fun, .callPos = pos};
+    else if (v.isPrimOp()) {
+        return getPrimOpFrameInfo(*v.primOp(), args, pos);
+    } else if (v.isPrimOpApp())
+        /* Resolve primOp eagerly. Must not hold on to a reference to a Value. */
+        return PrimOpFrameInfo{.expr = v.primOpAppPrimOp(), .callPos = pos};
+    else if (state.isFunctor(v)) {
+        const auto functor = v.attrs()->get(state.sFunctor);
+        if (auto pos_ = posCache.lookup(pos); std::holds_alternative<std::monostate>(pos_.origin))
+            /* HACK: In case callsite position is unresolved. */
+            return FunctorFrameInfo{.pos = functor->pos};
+        return FunctorFrameInfo{.pos = pos};
+    } else
+        /* NOTE: Add a stack frame even for invalid cases (e.g. when calling a non-function). This is what
+         * trace-function-calls does. */
+        return GenericFrameInfo{.pos = pos};
+}
+
+[[gnu::noinline]] void
+SampleStack::preFunctionCallHook(EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
+{
+    stack.push_back(getFrameInfoFromValueAndPos(v, args, pos));
+
+    auto now = std::chrono::high_resolution_clock::now();
+
+    if (now - lastStackSample > sampleInterval) {
+        callCount[stack] += 1;
+        lastStackSample = now;
+    }
+
+    /* Do this in preFunctionCallHook because we might throw an exception, but
+       callFunction uses Finally, which doesn't play well with exceptions. */
+    maybeSaveProfile(now);
+}
+
+[[gnu::noinline]] void
+SampleStack::postFunctionCallHook(EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
+{
+    if (!stack.empty())
+        stack.pop_back();
+}
+
+std::ostream & LambdaFrameInfo::symbolize(const EvalState & state, std::ostream & os, PosCache & posCache) const
+{
+    if (auto pos = posCache.lookup(callPos); std::holds_alternative<std::monostate>(pos.origin))
+        /* HACK: To avoid dubious «none»:0 in the generated profile if the origin can't be resolved
+           resort to printing the lambda location instead of the callsite position. */
+        os << posCache.lookup(expr->getPos());
+    else
+        os << pos;
+    if (expr->name)
+        os << ":" << state.symbols[expr->name];
+    return os;
+}
+
+std::ostream & GenericFrameInfo::symbolize(const EvalState & state, std::ostream & os, PosCache & posCache) const
+{
+    os << posCache.lookup(pos);
+    return os;
+}
+
+std::ostream & FunctorFrameInfo::symbolize(const EvalState & state, std::ostream & os, PosCache & posCache) const
+{
+    os << posCache.lookup(pos) << ":functor";
+    return os;
+}
+
+std::ostream & PrimOpFrameInfo::symbolize(const EvalState & state, std::ostream & os, PosCache & posCache) const
+{
+    /* Sometimes callsite position can have an unresolved origin, which
+       leads to confusing «none»:0 locations in the profile. */
+    auto pos = posCache.lookup(callPos);
+    if (!std::holds_alternative<std::monostate>(pos.origin))
+        os << posCache.lookup(callPos) << ":";
+    os << *expr;
+    return os;
+}
+
+std::ostream &
+DerivationStrictFrameInfo::symbolize(const EvalState & state, std::ostream & os, PosCache & posCache) const
+{
+    /* Sometimes callsite position can have an unresolved origin, which
+       leads to confusing «none»:0 locations in the profile. */
+    auto pos = posCache.lookup(callPos);
+    if (!std::holds_alternative<std::monostate>(pos.origin))
+        os << posCache.lookup(callPos) << ":";
+    os << "primop derivationStrict:" << drvName;
+    return os;
+}
+
+void SampleStack::maybeSaveProfile(std::chrono::time_point<std::chrono::high_resolution_clock> now)
+{
+    if (now - lastDump >= profileDumpInterval)
+        saveProfile();
+    else
+        return;
+
+    /* Save the last dump timepoint. Do this after actually saving data to file
+       to not account for the time doing the flushing to disk. */
+    lastDump = std::chrono::high_resolution_clock::now();
+
+    /* Free up memory used for stack sampling. This might be very significant for
+       long-running evaluations, so we shouldn't hog too much memory. */
+    callCount.clear();
+}
+
+void SampleStack::saveProfile()
+{
+    auto os = std::ostringstream{};
+    for (auto & [stack, count] : callCount) {
+        auto first = true;
+        for (auto & pos : stack) {
+            if (first)
+                first = false;
+            else
+                os << ";";
+
+            std::visit([&](auto && info) { info.symbolize(state, os, posCache); }, pos);
+        }
+        os << " " << count;
+        writeLine(profileFd.get(), std::move(os).str());
+        /* Clear ostringstream. */
+        os.str("");
+        os.clear();
+    }
+}
+
+SampleStack::~SampleStack()
+{
+    /* Guard against cases when we are already unwinding the stack. */
+    try {
+        saveProfile();
+    } catch (...) {
+        ignoreExceptionInDestructor();
+    }
+}
+
+} // namespace
+
+ref<EvalProfiler> makeSampleStackProfiler(EvalState & state, std::filesystem::path profileFile, uint64_t frequency)
+{
+    /* 0 is a special value for sampling stack after each call. */
+    std::chrono::nanoseconds period = frequency == 0
+                                          ? std::chrono::nanoseconds{0}
+                                          : std::chrono::nanoseconds{std::nano::den / frequency / std::nano::num};
+    return make_ref<SampleStack>(state, profileFile, period);
+}
+
+}

src/libexpr/eval.cc

@@ -2,6 +2,7 @@
 #include "nix/expr/eval-settings.hh"
 #include "nix/expr/primops.hh"
 #include "nix/expr/print-options.hh"
+#include "nix/expr/symbol-table.hh"
 #include "nix/util/exit.hh"
 #include "nix/util/types.hh"
 #include "nix/util/util.hh"
@@ -90,20 +91,16 @@ std::string printValue(EvalState & state, Value & v)
     return out.str();
 }
 
+Value * Value::toPtr(SymbolStr str) noexcept
+{
+    return const_cast<Value *>(str.valuePtr());
+}
+
 void Value::print(EvalState & state, std::ostream & str, PrintOptions options)
 {
     printValue(state, str, *this, options);
 }
 
-const Value * getPrimOp(const Value &v) {
-    const Value * primOp = &v;
-    while (primOp->isPrimOpApp()) {
-        primOp = primOp->payload.primOpApp.left;
-    }
-    assert(primOp->isPrimOp());
-    return primOp;
-}
-
 std::string_view showType(ValueType type, bool withArticle)
 {
     #define WA(a, w) withArticle ? a " " w : w
@@ -129,12 +126,12 @@ std::string showType(const Value & v)
     // Allow selecting a subset of enum values
     #pragma GCC diagnostic push
     #pragma GCC diagnostic ignored "-Wswitch-enum"
-    switch (v.internalType) {
+    switch (v.getInternalType()) {
-        case tString: return v.payload.string.context ? "a string with context" : "a string";
+        case tString: return v.context() ? "a string with context" : "a string";
         case tPrimOp:
-            return fmt("the built-in function '%s'", std::string(v.payload.primOp->name));
+            return fmt("the built-in function '%s'", std::string(v.primOp()->name));
         case tPrimOpApp:
-            return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->payload.primOp->name));
+            return fmt("the partially applied built-in function '%s'", v.primOpAppPrimOp()->name);
         case tExternal: return v.external()->showType();
         case tThunk: return v.isBlackhole() ? "a black hole" : "a thunk";
         case tApp: return "a function application";
@@ -149,12 +146,10 @@ PosIdx Value::determinePos(const PosIdx pos) const
     // Allow selecting a subset of enum values
     #pragma GCC diagnostic push
     #pragma GCC diagnostic ignored "-Wswitch-enum"
-    if (this->pos != 0)
+    switch (getInternalType()) {
-        return PosIdx(this->pos);
-    switch (internalType) {
         case tAttrs: return attrs()->pos;
-        case tLambda: return payload.lambda.fun->pos;
+        case tLambda: return lambda().fun->pos;
-        case tApp: return payload.app.left->determinePos(pos);
+        case tApp: return app().left->determinePos(pos);
         default: return pos;
     }
     #pragma GCC diagnostic pop
@@ -163,13 +158,12 @@ PosIdx Value::determinePos(const PosIdx pos) const
 bool Value::isTrivial() const
 {
     return
-        internalType != tApp
+        !isa<tApp, tPrimOpApp>()
-        && internalType != tPrimOpApp
+        && (!isa<tThunk>()
-        && (internalType != tThunk
+            || (dynamic_cast<ExprAttrs *>(thunk().expr)
-            || (dynamic_cast<ExprAttrs *>(payload.thunk.expr)
+                && ((ExprAttrs *) thunk().expr)->dynamicAttrs.empty())
-                && ((ExprAttrs *) payload.thunk.expr)->dynamicAttrs.empty())
+            || dynamic_cast<ExprLambda *>(thunk().expr)
-            || dynamic_cast<ExprLambda *>(payload.thunk.expr)
+            || dynamic_cast<ExprList *>(thunk().expr));
-            || dynamic_cast<ExprList *>(payload.thunk.expr));
 }
 
 
@@ -215,6 +209,7 @@ EvalState::EvalState(
     , sRight(symbols.create("right"))
     , sWrong(symbols.create("wrong"))
     , sStructuredAttrs(symbols.create("__structuredAttrs"))
+    , sJson(symbols.create("__json"))
     , sAllowedReferences(symbols.create("allowedReferences"))
     , sAllowedRequisites(symbols.create("allowedRequisites"))
     , sDisallowedReferences(symbols.create("disallowedReferences"))
@@ -372,8 +367,20 @@ EvalState::EvalState(
     );
 
     createBaseEnv(settings);
-}
 
+    /* Register function call tracer. */
+    if (settings.traceFunctionCalls)
+        profiler.addProfiler(make_ref<FunctionCallTrace>());
+
+    switch (settings.evalProfilerMode) {
+    case EvalProfilerMode::flamegraph:
+        profiler.addProfiler(makeSampleStackProfiler(
+            *this, settings.evalProfileFile.get(), settings.evalProfilerFrequency));
+        break;
+    case EvalProfilerMode::disabled:
+        break;
+    }
+}
 
 EvalState::~EvalState()
 {
@@ -493,7 +500,7 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info)
         /* Install value the base environment. */
         staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
         baseEnv.values[baseEnvDispl++] = v;
-        getBuiltins().payload.attrs->push_back(Attr(symbols.create(name2), v));
+        const_cast<Bindings *>(getBuiltins().attrs())->push_back(Attr(symbols.create(name2), v));
     }
 }
 
@@ -515,13 +522,15 @@ std::ostream & operator<<(std::ostream & output, const PrimOp & primOp)
 
 const PrimOp * Value::primOpAppPrimOp() const
 {
-    Value * left = payload.primOpApp.left;
+    Value * left = primOpApp().left;
     while (left && !left->isPrimOp()) {
-        left = left->payload.primOpApp.left;
+        left = left->primOpApp().left;
     }
 
     if (!left)
         return nullptr;
+
+    assert(left->isPrimOp());
     return left->primOp();
 }
 
@@ -529,7 +538,7 @@ const PrimOp * Value::primOpAppPrimOp() const
 void Value::mkPrimOp(PrimOp * p)
 {
     p->check();
-    finishValue(tPrimOp, { .primOp = p });
+    setStorage(p);
 }
 
 
@@ -561,7 +570,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
     else {
         staticBaseEnv->vars.emplace_back(envName, baseEnvDispl);
         baseEnv.values[baseEnvDispl++] = v;
-        getBuiltins().payload.attrs->push_back(Attr(symbols.create(primOp.name), v));
+        const_cast<Bindings *>(getBuiltins().attrs())->push_back(Attr(symbols.create(primOp.name), v));
     }
 
     return v;
@@ -598,7 +607,7 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
             };
     }
     if (v.isLambda()) {
-        auto exprLambda = v.payload.lambda.fun;
+        auto exprLambda = v.lambda().fun;
 
         std::ostringstream s;
         std::string name;
@@ -645,7 +654,7 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
             Value & functor = *v.attrs()->find(sFunctor)->value;
             Value * vp[] = {&v};
             Value partiallyApplied;
-            // The first paramater is not user-provided, and may be
+            // The first parameter is not user-provided, and may be
             // handled by code that is opaque to the user, like lib.const = x: y: y;
             // So preferably we show docs that are relevant to the
             // "partially applied" function returned by e.g. `const`.
@@ -908,7 +917,7 @@ void Value::mkStringMove(const char * s, const NixStringContext & context)
 
 void Value::mkPath(const SourcePath & path)
 {
-    mkPath(&*path.accessor, makeImmutableString(path.path.abs()), noPos.get());
+    mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
 }
 
 
@@ -1535,9 +1544,14 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
 {
     auto _level = addCallDepth(pos);
 
-    auto trace = settings.traceFunctionCalls
+    auto neededHooks = profiler.getNeededHooks();
-        ? std::make_unique<FunctionCallTrace>(positions[pos])
+    if (neededHooks.test(EvalProfiler::preFunctionCall)) [[unlikely]]
-        : nullptr;
+        profiler.preFunctionCallHook(*this, fun, args, pos);
+
+    Finally traceExit_{[&](){
+        if (profiler.getNeededHooks().test(EvalProfiler::postFunctionCall)) [[unlikely]]
+            profiler.postFunctionCallHook(*this, fun, args, pos);
+    }};
 
     forceValue(fun, pos);
 
@@ -1559,13 +1573,13 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
 
         if (vCur.isLambda()) {
 
-            ExprLambda & lambda(*vCur.payload.lambda.fun);
+            ExprLambda & lambda(*vCur.lambda().fun);
 
             auto size =
                 (!lambda.arg ? 0 : 1) +
                 (lambda.hasFormals() ? lambda.formals->formals.size() : 0);
             Env & env2(allocEnv(size));
-            env2.up = vCur.payload.lambda.env;
+            env2.up = vCur.lambda().env;
 
             Displacement displ = 0;
 
@@ -1595,7 +1609,7 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
                                              symbols[i.name])
                                     .atPos(lambda.pos)
                                     .withTrace(pos, "from call site")
-                                    .withFrame(*fun.payload.lambda.env, lambda)
+                                    .withFrame(*fun.lambda().env, lambda)
                                     .debugThrow();
                         }
                         env2.values[displ++] = i.def->maybeThunk(*this, env2);
@@ -1622,7 +1636,7 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
                                 .atPos(lambda.pos)
                                 .withTrace(pos, "from call site")
                                 .withSuggestions(suggestions)
-                                .withFrame(*fun.payload.lambda.env, lambda)
+                                .withFrame(*fun.lambda().env, lambda)
                                 .debugThrow();
                         }
                     unreachable();
@@ -1694,7 +1708,7 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
             Value * primOp = &vCur;
             while (primOp->isPrimOpApp()) {
                 argsDone++;
-                primOp = primOp->payload.primOpApp.left;
+                primOp = primOp->primOpApp().left;
             }
             assert(primOp->isPrimOp());
             auto arity = primOp->primOp()->arity;
@@ -1710,8 +1724,8 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
 
                 Value * vArgs[maxPrimOpArity];
                 auto n = argsDone;
-                for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->payload.primOpApp.left)
+                for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp().left)
-                    vArgs[--n] = arg->payload.primOpApp.right;
+                    vArgs[--n] = arg->primOpApp().right;
 
                 for (size_t i = 0; i < argsLeft; ++i)
                     vArgs[argsDone + i] = args[i];
@@ -1817,14 +1831,14 @@ void EvalState::autoCallFunction(const Bindings & args, Value & fun, Value & res
         }
     }
 
-    if (!fun.isLambda() || !fun.payload.lambda.fun->hasFormals()) {
+    if (!fun.isLambda() || !fun.lambda().fun->hasFormals()) {
         res = fun;
         return;
     }
 
-    auto attrs = buildBindings(std::max(static_cast<uint32_t>(fun.payload.lambda.fun->formals->formals.size()), args.size()));
+    auto attrs = buildBindings(std::max(static_cast<uint32_t>(fun.lambda().fun->formals->formals.size()), args.size()));
 
-    if (fun.payload.lambda.fun->formals->ellipsis) {
+    if (fun.lambda().fun->formals->ellipsis) {
         // If the formals have an ellipsis (eg the function accepts extra args) pass
         // all available automatic arguments (which includes arguments specified on
         // the command line via --arg/--argstr)
@@ -1832,7 +1846,7 @@ void EvalState::autoCallFunction(const Bindings & args, Value & fun, Value & res
             attrs.insert(v);
     } else {
         // Otherwise, only pass the arguments that the function accepts
-        for (auto & i : fun.payload.lambda.fun->formals->formals) {
+        for (auto & i : fun.lambda().fun->formals->formals) {
             auto j = args.get(i.name);
             if (j) {
                 attrs.insert(*j);
@@ -1842,7 +1856,7 @@ Nix attempted to evaluate a function as a top level expression; in
 this case it must have its arguments supplied either by default
 values, or passed explicitly with '--arg' or '--argstr'. See
 https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbols[i.name])
-                    .atPos(i.pos).withFrame(*fun.payload.lambda.env, *fun.payload.lambda.fun).debugThrow();
+                    .atPos(i.pos).withFrame(*fun.lambda().env, *fun.lambda().fun).debugThrow();
             }
         }
     }
@@ -2000,9 +2014,10 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * const * lists, co
     auto list = buildList(len);
     auto out = list.elems;
     for (size_t n = 0, pos = 0; n < nrLists; ++n) {
-        auto l = lists[n]->listSize();
+        auto listView = lists[n]->listView();
+        auto l = listView.size();
         if (l)
-            memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
+            memcpy(out + pos, listView.data(), l * sizeof(Value *));
         pos += l;
     }
     v.mkList(list);
@@ -2155,7 +2170,7 @@ void EvalState::forceValueDeep(Value & v)
                 try {
                     // If the value is a thunk, we're evaling. Otherwise no trace necessary.
                     auto dts = debugRepl && i.value->isThunk()
-                        ? makeDebugTraceStacker(*this, *i.value->payload.thunk.expr, *i.value->payload.thunk.env, i.pos,
+                        ? makeDebugTraceStacker(*this, *i.value->thunk().expr, *i.value->thunk().env, i.pos,
                             "while evaluating the attribute '%1%'", symbols[i.name])
                         : nullptr;
 
@@ -2167,7 +2182,7 @@ void EvalState::forceValueDeep(Value & v)
         }
 
         else if (v.isList()) {
-            for (auto v2 : v.listItems())
+            for (auto v2 : v.listView())
                 recurse(*v2);
         }
     };
@@ -2235,8 +2250,18 @@ bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx
     return v.boolean();
 }
 
+Bindings::const_iterator EvalState::getAttr(Symbol attrSym, const Bindings * attrSet, std::string_view errorCtx)
+{
+    auto value = attrSet->find(attrSym);
+    if (value == attrSet->end()) {
+        error<TypeError>("attribute '%s' missing", symbols[attrSym])
+            .withTrace(noPos, errorCtx)
+            .debugThrow();
+    }
+    return value;
+}
 
-bool EvalState::isFunctor(Value & fun)
+bool EvalState::isFunctor(const Value & fun) const
 {
     return fun.type() == nAttrs && fun.attrs()->find(sFunctor) != fun.attrs()->end();
 }
@@ -2279,8 +2304,8 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string
 
 void copyContext(const Value & v, NixStringContext & context, const ExperimentalFeatureSettings & xpSettings)
 {
-    if (v.payload.string.context)
+    if (v.context())
-        for (const char * * p = v.payload.string.context; *p; ++p)
+        for (const char * * p = v.context(); *p; ++p)
             context.insert(NixStringContextElem::parse(*p, xpSettings));
 }
 
@@ -2356,7 +2381,7 @@ BackedStringView EvalState::coerceToString(
             !canonicalizePath && !copyToStore
             ? // FIXME: hack to preserve path literals that end in a
               // slash, as in /foo/${x}.
-              v.payload.path.path
+              v.pathStr()
             : copyToStore
             ? store->printStorePath(copyPathToStore(context, v.path(), v.determinePos(pos)))
             : ({
@@ -2409,7 +2434,8 @@ BackedStringView EvalState::coerceToString(
 
         if (v.isList()) {
             std::string result;
-            for (auto [n, v2] : enumerate(v.listItems())) {
+            auto listView = v.listView();
+            for (auto [n, v2] : enumerate(listView)) {
                 try {
                     result += *coerceToString(pos, *v2, context,
                             "while evaluating one element of the list",
@@ -2447,6 +2473,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
         ? *dstPathCached
         : [&]() {
             auto dstPath = fetchToStore(
+                fetchSettings,
                 *store,
                 path.resolveSymlinks(SymlinkResolution::Ancestors),
                 settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
@@ -2491,7 +2518,7 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext
         }
     }
 
-    /* Any other value should be coercable to a string, interpreted
+    /* Any other value should be coercible to a string, interpreted
        relative to the root filesystem. */
     auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
     if (path == "" || path[0] != '/')
@@ -2637,14 +2664,14 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st
         return;
 
     case nPath:
-        if (v1.payload.path.accessor != v2.payload.path.accessor) {
+        if (v1.pathAccessor() != v2.pathAccessor()) {
             error<AssertionError>(
                 "path '%s' is not equal to path '%s' because their accessors are different",
                 ValuePrinter(*this, v1, errorPrintOptions),
                 ValuePrinter(*this, v2, errorPrintOptions))
                 .debugThrow();
         }
-        if (strcmp(v1.payload.path.path, v2.payload.path.path) != 0) {
+        if (strcmp(v1.pathStr(), v2.pathStr()) != 0) {
             error<AssertionError>(
                 "path '%s' is not equal to path '%s'",
                 ValuePrinter(*this, v1, errorPrintOptions),
@@ -2668,7 +2695,7 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st
         }
         for (size_t n = 0; n < v1.listSize(); ++n) {
             try {
-                assertEqValues(*v1.listElems()[n], *v2.listElems()[n], pos, errorCtx);
+                assertEqValues(*v1.listView()[n], *v2.listView()[n], pos, errorCtx);
             } catch (Error & e) {
                 e.addTrace(positions[pos], "while comparing list element %d", n);
                 throw;
@@ -2811,8 +2838,8 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
         case nPath:
             return
                 // FIXME: compare accessors by their fingerprint.
-                v1.payload.path.accessor == v2.payload.path.accessor
+                v1.pathAccessor() == v2.pathAccessor()
-                && strcmp(v1.payload.path.path, v2.payload.path.path) == 0;
+                && strcmp(v1.pathStr(), v2.pathStr()) == 0;
 
         case nNull:
             return true;
@@ -2820,7 +2847,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
         case nList:
             if (v1.listSize() != v2.listSize()) return false;
             for (size_t n = 0; n < v1.listSize(); ++n)
-                if (!eqValues(*v1.listElems()[n], *v2.listElems()[n], pos, errorCtx)) return false;
+                if (!eqValues(*v1.listView()[n], *v2.listView()[n], pos, errorCtx)) return false;
             return true;
 
         case nAttrs: {
@@ -2867,7 +2894,7 @@ bool EvalState::fullGC() {
     GC_gcollect();
     // Check that it ran. We might replace this with a version that uses more
     // of the boehm API to get this reliably, at a maintenance cost.
-    // We use a 1K margin because technically this has a race condtion, but we
+    // We use a 1K margin because technically this has a race condition, but we
     // probably won't encounter it in practice, because the CLI isn't concurrent
     // like that.
     return GC_get_bytes_since_gc() < 1024;
@@ -3020,7 +3047,7 @@ void EvalState::printStatistics()
         // XXX: overrides earlier assignment
         topObj["symbols"] = json::array();
         auto &list = topObj["symbols"];
-        symbols.dump([&](const std::string & s) { list.emplace_back(s); });
+        symbols.dump([&](std::string_view s) { list.emplace_back(s); });
     }
     if (outPath == "-") {
         std::cerr << topObj.dump(2) << std::endl;
@@ -3159,7 +3186,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
                 store,
                 fetchSettings,
                 EvalSettings::resolvePseudoUrl(value));
-            auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
+            auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy);
             return finish(this->storePath(storePath));
         } catch (Error & e) {
             logWarning({

src/libexpr/function-trace.cc

@@ -3,16 +3,20 @@
 
 namespace nix {
 
-FunctionCallTrace::FunctionCallTrace(const Pos & pos) : pos(pos) {
+void FunctionCallTrace::preFunctionCallHook(
+    EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
+{
     auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
     auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
-    printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count());
+    printMsg(lvlInfo, "function-trace entered %1% at %2%", state.positions[pos], ns.count());
 }
 
-FunctionCallTrace::~FunctionCallTrace() {
+void FunctionCallTrace::postFunctionCallHook(
+    EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
+{
     auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
     auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
-    printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count());
+    printMsg(lvlInfo, "function-trace exited %1% at %2%", state.positions[pos], ns.count());
 }
 
 }

src/libexpr/get-drvs.cc

@@ -117,7 +117,7 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT
             state->forceList(*i->value, i->pos, "while evaluating the 'outputs' attribute of a derivation");
 
             /* For each output... */
-            for (auto elem : i->value->listItems()) {
+            for (auto elem : i->value->listView()) {
                 std::string output(state->forceStringNoCtx(*elem, i->pos, "while evaluating the name of an output of a derivation"));
 
                 if (withPaths) {
@@ -159,7 +159,7 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT
             /* ^ this shows during `nix-env -i` right under the bad derivation */
         if (!outTI->isList()) throw errMsg;
         Outputs result;
-        for (auto elem : outTI->listItems()) {
+        for (auto elem : outTI->listView()) {
             if (elem->type() != nString) throw errMsg;
             auto out = outputs.find(elem->c_str());
             if (out == outputs.end()) throw errMsg;
@@ -206,7 +206,7 @@ bool PackageInfo::checkMeta(Value & v)
 {
     state->forceValue(v, v.determinePos(noPos));
     if (v.type() == nList) {
-        for (auto elem : v.listItems())
+        for (auto elem : v.listView())
             if (!checkMeta(*elem)) return false;
         return true;
     }
@@ -400,7 +400,8 @@ static void getDerivations(EvalState & state, Value & vIn,
     }
 
     else if (v.type() == nList) {
-        for (auto [n, elem] : enumerate(v.listItems())) {
+        auto listView = v.listView();
+        for (auto [n, elem] : enumerate(listView)) {
             std::string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n));
             if (getDerivation(state, *elem, pathPrefix2, drvs, done, ignoreAssertionFailures))
                 getDerivations(state, *elem, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);

src/libexpr/include/nix/expr/eval-inline.hh

@@ -89,9 +89,9 @@ Env & EvalState::allocEnv(size_t size)
 void EvalState::forceValue(Value & v, const PosIdx pos)
 {
     if (v.isThunk()) {
-        Env * env = v.payload.thunk.env;
+        Env * env = v.thunk().env;
         assert(env || v.isBlackhole());
-        Expr * expr = v.payload.thunk.expr;
+        Expr * expr = v.thunk().expr;
         try {
             v.mkBlackhole();
             //checkInterrupt();
@@ -106,7 +106,7 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
         }
     }
     else if (v.isApp())
-        callFunction(*v.payload.app.left, *v.payload.app.right, v, pos);
+        callFunction(*v.app().left, *v.app().right, v, pos);
 }
 
 

src/libexpr/include/nix/expr/eval-profiler-settings.hh

@@ -0,0 +1,16 @@
+#pragma once
+///@file
+
+#include "nix/util/configuration.hh"
+
+namespace nix {
+
+enum struct EvalProfilerMode { disabled, flamegraph };
+
+template<>
+EvalProfilerMode BaseSetting<EvalProfilerMode>::parse(const std::string & str) const;
+
+template<>
+std::string BaseSetting<EvalProfilerMode>::to_string() const;
+
+}

src/libexpr/include/nix/expr/eval-profiler.hh

@@ -0,0 +1,114 @@
+#pragma once
+/**
+ * @file
+ *
+ * Evaluation profiler interface definitions and builtin implementations.
+ */
+
+#include "nix/util/ref.hh"
+
+#include <vector>
+#include <span>
+#include <bitset>
+#include <optional>
+#include <filesystem>
+
+namespace nix {
+
+class EvalState;
+class PosIdx;
+struct Value;
+
+class EvalProfiler
+{
+public:
+    enum Hook {
+        preFunctionCall,
+        postFunctionCall,
+    };
+
+    static constexpr std::size_t numHooks = Hook::postFunctionCall + 1;
+    using Hooks = std::bitset<numHooks>;
+
+private:
+    std::optional<Hooks> neededHooks;
+
+protected:
+    /** Invalidate the cached neededHooks. */
+    void invalidateNeededHooks()
+    {
+        neededHooks = std::nullopt;
+    }
+
+    /**
+     * Get which hooks need to be called.
+     *
+     * This is the actual implementation which has to be defined by subclasses.
+     * Public API goes through the needsHooks, which is a
+     * non-virtual interface (NVI) which caches the return value.
+     */
+    virtual Hooks getNeededHooksImpl() const
+    {
+        return Hooks{};
+    }
+
+public:
+    /**
+     * Hook called in the EvalState::callFunction preamble.
+     * Gets called only if (getNeededHooks().test(Hook::preFunctionCall)) is true.
+     *
+     * @param state Evaluator state.
+     * @param v Function being invoked.
+     * @param args Function arguments.
+     * @param pos Function position.
+     */
+    virtual void preFunctionCallHook(EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos);
+
+    /**
+     * Hook called on EvalState::callFunction exit.
+     * Gets called only if (getNeededHooks().test(Hook::postFunctionCall)) is true.
+     *
+     * @param state Evaluator state.
+     * @param v Function being invoked.
+     * @param args Function arguments.
+     * @param pos Function position.
+     */
+    virtual void postFunctionCallHook(EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos);
+
+    virtual ~EvalProfiler() = default;
+
+    /**
+     * Get which hooks need to be invoked for this EvalProfiler instance.
+     */
+    Hooks getNeededHooks()
+    {
+        if (neededHooks.has_value())
+            return *neededHooks;
+        return *(neededHooks = getNeededHooksImpl());
+    }
+};
+
+/**
+ * Profiler that invokes multiple profilers at once.
+ */
+class MultiEvalProfiler : public EvalProfiler
+{
+    std::vector<ref<EvalProfiler>> profilers;
+
+    [[gnu::noinline]] Hooks getNeededHooksImpl() const override;
+
+public:
+    MultiEvalProfiler() = default;
+
+    /** Register a profiler instance. */
+    void addProfiler(ref<EvalProfiler> profiler);
+
+    [[gnu::noinline]] void
+    preFunctionCallHook(EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos) override;
+    [[gnu::noinline]] void
+    postFunctionCallHook(EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos) override;
+};
+
+ref<EvalProfiler> makeSampleStackProfiler(EvalState & state, std::filesystem::path profileFile, uint64_t frequency);
+
+}

src/libexpr/include/nix/expr/eval-settings.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "nix/expr/eval-profiler-settings.hh"
 #include "nix/util/configuration.hh"
 #include "nix/util/source-path.hh"
 
@@ -12,7 +13,7 @@ struct PrimOp;
 struct EvalSettings : Config
 {
     /**
-     * Function used to interpet look path entries of a given scheme.
+     * Function used to interpret look path entries of a given scheme.
      *
      * The argument is the non-scheme part of the lookup path entry (see
      * `LookupPathHooks` below).
@@ -203,6 +204,29 @@ struct EvalSettings : Config
           `flamegraph.pl`.
         )"};
 
+    Setting<EvalProfilerMode> evalProfilerMode{this, EvalProfilerMode::disabled, "eval-profiler",
+        R"(
+          Enables evaluation profiling. The following modes are supported:
+
+          * `flamegraph` stack sampling profiler. Outputs folded format, one line per stack (suitable for `flamegraph.pl` and compatible tools).
+
+          Use [`eval-profile-file`](#conf-eval-profile-file) to specify where the profile is saved.
+
+          See [Using the `eval-profiler`](@docroot@/advanced-topics/eval-profiler.md).
+        )"};
+
+    Setting<Path> evalProfileFile{this, "nix.profile", "eval-profile-file",
+        R"(
+          Specifies the file where [evaluation profile](#conf-eval-profiler) is saved.
+        )"};
+
+    Setting<uint32_t> evalProfilerFrequency{this, 99, "eval-profiler-frequency",
+        R"(
+          Specifies the sampling rate in hertz for sampling evaluation profilers.
+          Use `0` to sample the stack after each function call.
+          See [`eval-profiler`](#conf-eval-profiler).
+        )"};
+
     Setting<bool> useEvalCache{this, true, "eval-cache",
         R"(
             Whether to use the flake evaluation cache.
@@ -212,7 +236,7 @@ struct EvalSettings : Config
 
     Setting<bool> ignoreExceptionsDuringTry{this, false, "ignore-try",
         R"(
-          If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in
+          If set to true, ignore exceptions inside 'tryEval' calls when evaluating Nix expressions in
           debug mode (using the --debugger flag). By default, the debugger pauses on all exceptions.
         )"};
 

src/libexpr/include/nix/expr/eval.hh

@@ -3,6 +3,7 @@
 
 #include "nix/expr/attr-set.hh"
 #include "nix/expr/eval-error.hh"
+#include "nix/expr/eval-profiler.hh"
 #include "nix/util/types.hh"
 #include "nix/expr/value.hh"
 #include "nix/expr/nixexpr.hh"
@@ -214,7 +215,7 @@ public:
     const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
-        sRight, sWrong, sStructuredAttrs,
+        sRight, sWrong, sStructuredAttrs, sJson,
         sAllowedReferences, sAllowedRequisites, sDisallowedReferences, sDisallowedRequisites,
         sMaxSize, sMaxClosureSize,
         sBuilder, sArgs,
@@ -552,6 +553,11 @@ public:
     std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx);
 
+    /**
+     * Get attribute from an attribute set and throw an error if it doesn't exist.
+     */
+    Bindings::const_iterator getAttr(Symbol attrSym, const Bindings * attrSet, std::string_view errorCtx);
+
     template<typename... Args>
     [[gnu::noinline]]
     void addErrorTrace(Error & e, const Args & ... formatArgs) const;
@@ -766,7 +772,7 @@ public:
      */
     void assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx);
 
-    bool isFunctor(Value & fun);
+    bool isFunctor(const Value & fun) const;
 
     void callFunction(Value & fun, std::span<Value *> args, Value & vRes, const PosIdx pos);
 
@@ -939,6 +945,9 @@ private:
     typedef std::map<ExprLambda *, size_t> FunctionCalls;
     FunctionCalls functionCalls;
 
+    /** Evaluation/call profiler. */
+    MultiEvalProfiler profiler;
+
     void incrFunctionCall(ExprLambda * fun);
 
     typedef std::map<PosIdx, size_t> AttrSelects;

src/libexpr/include/nix/expr/function-trace.hh

@@ -2,15 +2,24 @@
 ///@file
 
 #include "nix/expr/eval.hh"
-
+#include "nix/expr/eval-profiler.hh"
-#include <chrono>
 
 namespace nix {
 
-struct FunctionCallTrace
+class FunctionCallTrace : public EvalProfiler
 {
-    const Pos pos;
+    Hooks getNeededHooksImpl() const override
-    FunctionCallTrace(const Pos & pos);
+    {
-    ~FunctionCallTrace();
+        return Hooks().set(preFunctionCall).set(postFunctionCall);
+    }
+
+public:
+    FunctionCallTrace() = default;
+
+    [[gnu::noinline]] void
+    preFunctionCallHook(EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos) override;
+    [[gnu::noinline]] void
+    postFunctionCallHook(EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos) override;
 };
+
 }

src/libexpr/include/nix/expr/meson.build

@@ -14,6 +14,8 @@ headers = [config_pub_h] + files(
   'eval-error.hh',
   'eval-gc.hh',
   'eval-inline.hh',
+  'eval-profiler-settings.hh',
+  'eval-profiler.hh',
   'eval-settings.hh',
   'eval.hh',
   'function-trace.hh',

src/libexpr/include/nix/expr/nixexpr.hh

@@ -138,9 +138,9 @@ struct ExprPath : Expr
     ref<SourceAccessor> accessor;
     std::string s;
     Value v;
-    ExprPath(ref<SourceAccessor> accessor, std::string s, PosIdx pos) : accessor(accessor), s(std::move(s))
+    ExprPath(ref<SourceAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s))
     {
-        v.mkPath(&*accessor, this->s.c_str(), pos.get());
+        v.mkPath(&*accessor, this->s.c_str());
     }
     Value * maybeThunk(EvalState & state, Env & env) override;
     COMMON_METHODS
@@ -306,6 +306,9 @@ struct Formal
 struct Formals
 {
     typedef std::vector<Formal> Formals_;
+    /**
+     * @pre Sorted according to predicate (std::tie(a.name, a.pos) < std::tie(b.name, b.pos)).
+     */
     Formals_ formals;
     bool ellipsis;
 

src/libexpr/include/nix/expr/print-ambiguous.hh

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "nix/expr/value.hh"
+#include "nix/expr/symbol-table.hh"
 
 namespace nix {
 

src/libexpr/include/nix/expr/symbol-table.hh

@@ -1,51 +1,35 @@
 #pragma once
 ///@file
 
-#include <list>
+#include <memory_resource>
-#include <map>
+#include "nix/expr/value.hh"
-#include <unordered_map>
-
-#include "nix/util/types.hh"
 #include "nix/util/chunked-vector.hh"
 #include "nix/util/error.hh"
 
+#include <boost/version.hpp>
+#define USE_FLAT_SYMBOL_SET (BOOST_VERSION >= 108100)
+#if USE_FLAT_SYMBOL_SET
+#  include <boost/unordered/unordered_flat_set.hpp>
+#else
+#  include <boost/unordered/unordered_set.hpp>
+#endif
+
 namespace nix {
 
-/**
+class SymbolValue : protected Value
- * This class mainly exists to give us an operator<< for ostreams. We could also
- * return plain strings from SymbolTable, but then we'd have to wrap every
- * instance of a symbol that is fmt()ed, which is inconvenient and error-prone.
- */
-class SymbolStr
 {
+    friend class SymbolStr;
     friend class SymbolTable;
 
-private:
+    uint32_t size_;
-    const std::string * s;
+    uint32_t idx;
 
-    explicit SymbolStr(const std::string & symbol): s(&symbol) {}
+    SymbolValue() = default;
 
 public:
-    bool operator == (std::string_view s2) const
+    operator std::string_view() const noexcept
     {
-        return *s == s2;
+        return {c_str(), size_};
-    }
-
-    const char * c_str() const
-    {
-        return s->c_str();
-    }
-
-    operator const std::string_view () const
-    {
-        return *s;
-    }
-
-    friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol);
-
-    bool empty() const
-    {
-        return s->empty();
     }
 };
 
@@ -56,24 +40,161 @@ public:
  */
 class Symbol
 {
+    friend class SymbolStr;
     friend class SymbolTable;
 
 private:
     uint32_t id;
 
-    explicit Symbol(uint32_t id): id(id) {}
+    explicit Symbol(uint32_t id) noexcept : id(id) {}
 
 public:
-    Symbol() : id(0) {}
+    Symbol() noexcept : id(0) {}
 
-    explicit operator bool() const { return id > 0; }
+    [[gnu::always_inline]]
+    explicit operator bool() const noexcept { return id > 0; }
 
-    auto operator<=>(const Symbol other) const { return id <=> other.id; }
+    auto operator<=>(const Symbol other) const noexcept { return id <=> other.id; }
-    bool operator==(const Symbol other) const { return id == other.id; }
+    bool operator==(const Symbol other) const noexcept { return id == other.id; }
 
     friend class std::hash<Symbol>;
 };
 
+/**
+ * This class mainly exists to give us an operator<< for ostreams. We could also
+ * return plain strings from SymbolTable, but then we'd have to wrap every
+ * instance of a symbol that is fmt()ed, which is inconvenient and error-prone.
+ */
+class SymbolStr
+{
+    friend class SymbolTable;
+
+    constexpr static size_t chunkSize{8192};
+    using SymbolValueStore = ChunkedVector<SymbolValue, chunkSize>;
+
+    const SymbolValue * s;
+
+    struct Key
+    {
+        using HashType = boost::hash<std::string_view>;
+
+        SymbolValueStore & store;
+        std::string_view s;
+        std::size_t hash;
+        std::pmr::polymorphic_allocator<char> & alloc;
+
+        Key(SymbolValueStore & store, std::string_view s, std::pmr::polymorphic_allocator<char> & stringAlloc)
+            : store(store)
+            , s(s)
+            , hash(HashType{}(s))
+            , alloc(stringAlloc) {}
+    };
+
+public:
+    SymbolStr(const SymbolValue & s) noexcept : s(&s) {}
+
+    SymbolStr(const Key & key)
+    {
+        auto size = key.s.size();
+        if (size >= std::numeric_limits<uint32_t>::max()) {
+            throw Error("Size of symbol exceeds 4GiB and cannot be stored");
+        }
+        // for multi-threaded implementations: lock store and allocator here
+        const auto & [v, idx] = key.store.add(SymbolValue{});
+        if (size == 0) {
+            v.mkString("", nullptr);
+        } else {
+            auto s = key.alloc.allocate(size + 1);
+            memcpy(s, key.s.data(), size);
+            s[size] = '\0';
+            v.mkString(s, nullptr);
+        }
+        v.size_ = size;
+        v.idx = idx;
+        this->s = &v;
+    }
+
+    bool operator == (std::string_view s2) const noexcept
+    {
+        return *s == s2;
+    }
+
+    [[gnu::always_inline]]
+    const char * c_str() const noexcept
+    {
+        return s->c_str();
+    }
+
+    [[gnu::always_inline]]
+    operator std::string_view () const noexcept
+    {
+        return *s;
+    }
+
+    friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol);
+
+    [[gnu::always_inline]]
+    bool empty() const noexcept
+    {
+        return s->size_ == 0;
+    }
+
+    [[gnu::always_inline]]
+    size_t size() const noexcept
+    {
+        return s->size_;
+    }
+
+    [[gnu::always_inline]]
+    const Value * valuePtr() const noexcept
+    {
+        return s;
+    }
+
+    explicit operator Symbol() const noexcept
+    {
+        return Symbol{s->idx + 1};
+    }
+
+    struct Hash
+    {
+        using is_transparent = void;
+        using is_avalanching = std::true_type;
+
+        std::size_t operator()(SymbolStr str) const
+        {
+            return Key::HashType{}(*str.s);
+        }
+
+        std::size_t operator()(const Key & key) const noexcept
+        {
+            return key.hash;
+        }
+    };
+
+    struct Equal
+    {
+        using is_transparent = void;
+
+        bool operator()(SymbolStr a, SymbolStr b) const noexcept
+        {
+            // strings are unique, so that a pointer comparison is OK
+            return a.s == b.s;
+        }
+
+        bool operator()(SymbolStr a, const Key & b) const noexcept
+        {
+            return a == b.s;
+        }
+
+        [[gnu::always_inline]]
+        bool operator()(const Key & a, SymbolStr b) const noexcept
+        {
+            return operator()(b, a);
+        }
+    };
+};
+
 /**
  * Symbol table used by the parser and evaluator to represent and look
  * up identifiers and attributes efficiently.
@@ -82,29 +203,46 @@ class SymbolTable
 {
 private:
     /**
-     * Map from string view (backed by ChunkedVector) -> offset into the store.
+     * SymbolTable is an append only data structure.
+     * During its lifetime the monotonic buffer holds all strings and nodes, if the symbol set is node based.
+     */
+    std::pmr::monotonic_buffer_resource buffer;
+    std::pmr::polymorphic_allocator<char> stringAlloc{&buffer};
+    SymbolStr::SymbolValueStore store{16};
+
+    /**
+     * Transparent lookup of string view for a pointer to a ChunkedVector entry -> return offset into the store.
      * ChunkedVector references are never invalidated.
      */
-    std::unordered_map<std::string_view, uint32_t> symbols;
+#if USE_FLAT_SYMBOL_SET
-    ChunkedVector<std::string, 8192> store{16};
+    boost::unordered_flat_set<SymbolStr, SymbolStr::Hash, SymbolStr::Equal> symbols{SymbolStr::chunkSize};
+#else
+    using SymbolValueAlloc = std::pmr::polymorphic_allocator<SymbolStr>;
+    boost::unordered_set<SymbolStr, SymbolStr::Hash, SymbolStr::Equal, SymbolValueAlloc> symbols{SymbolStr::chunkSize, {&buffer}};
+#endif
 
 public:
 
     /**
      * Converts a string into a symbol.
      */
-    Symbol create(std::string_view s)
+    Symbol create(std::string_view s) {
-    {
         // Most symbols are looked up more than once, so we trade off insertion performance
         // for lookup performance.
         // FIXME: make this thread-safe.
-        auto it = symbols.find(s);
+        return [&]<typename T>(T && key) -> Symbol {
-        if (it != symbols.end())
+            if constexpr (requires { symbols.insert<T>(key); }) {
-            return Symbol(it->second + 1);
+                auto [it, _] = symbols.insert<T>(key);
+                return Symbol(*it);
+            } else {
+                auto it = symbols.find<T>(key);
+                if (it != symbols.end())
+                    return Symbol(*it);
 
-        const auto & [rawSym, idx] = store.add(s);
+                it = symbols.emplace(key).first;
-        symbols.emplace(rawSym, idx);
+                return Symbol(*it);
-        return Symbol(idx + 1);
+            }
+        }(SymbolStr::Key{store, s, stringAlloc});
     }
 
     std::vector<SymbolStr> resolve(const std::vector<Symbol> & symbols) const
@@ -118,12 +256,14 @@ public:
 
     SymbolStr operator[](Symbol s) const
     {
-        if (s.id == 0 || s.id > store.size())
+        uint32_t idx = s.id - uint32_t(1);
+        if (idx >= store.size())
             unreachable();
-        return SymbolStr(store[s.id - 1]);
+        return store[idx];
     }
 
-    size_t size() const
+    [[gnu::always_inline]]
+    size_t size() const noexcept
     {
         return store.size();
     }
@@ -147,3 +287,5 @@ struct std::hash<nix::Symbol>
         return std::hash<decltype(s.id)>{}(s.id);
     }
 };
+
+#undef USE_FLAT_SYMBOL_SET

src/libexpr/lexer-helpers.hh

@@ -2,7 +2,7 @@
 
 #include <cstddef>
 
-// inluding the generated headers twice leads to errors
+// including the generated headers twice leads to errors
 #ifndef BISON_HEADER
 #  include "lexer-tab.hh"
 #  include "parser-tab.hh"

src/libexpr/meson.build

@@ -140,6 +140,8 @@ sources = files(
   'eval-cache.cc',
   'eval-error.cc',
   'eval-gc.cc',
+  'eval-profiler-settings.cc',
+  'eval-profiler.cc',
   'eval-settings.cc',
   'eval.cc',
   'function-trace.cc',

src/libexpr/nixexpr.cc

@@ -606,7 +606,7 @@ void ExprLambda::setDocComment(DocComment docComment) {
 size_t SymbolTable::totalSize() const
 {
     size_t n = 0;
-    dump([&] (const std::string & s) { n += s.size(); });
+    dump([&] (SymbolStr s) { n += s.size(); });
     return n;
 }
 

src/libexpr/parser.y

@@ -374,8 +374,8 @@ path_start
            root filesystem accessor, rather than the accessor of the
            current Nix expression. */
         literal.front() == '/'
-        ? new ExprPath(state->rootFS, std::move(path), CUR_POS)
+        ? new ExprPath(state->rootFS, std::move(path))
-        : new ExprPath(state->basePath.accessor, std::move(path), CUR_POS);
+        : new ExprPath(state->basePath.accessor, std::move(path));
   }
   | HPATH {
     if (state->settings.pureEval) {
@@ -385,7 +385,7 @@ path_start
         );
     }
     Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path), CUR_POS);
+    $$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
   }
   ;
 

src/libexpr/paths.cc

@@ -24,7 +24,11 @@ StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites)
 {
     if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) {
         auto storePath = fetchToStore(
-            *store, SourcePath{ref(mount)}, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.name());
+            fetchSettings,
+            *store,
+            SourcePath{ref(mount)},
+            settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
+            path.name());
         assert(storePath.name() == path.name());
         if (rewrites)
             rewrites->emplace(path.hashPart(), storePath.hashPart());
@@ -57,13 +61,12 @@ std::string EvalState::computeBaseName(const SourcePath & path, PosIdx pos)
     if (path.accessor == rootFS) {
         if (auto storePath = store->maybeParseStorePath(path.path.abs())) {
             warn(
-                "Copying '%s' to the store again\n"
+                "Copying '%s' to the store again.\n"
                 "You can make Nix evaluate faster and copy fewer files by replacing `./.` with the `self` flake input, "
-                "or `builtins.path { path = ./.; name = \"source\"; }`\n\n"
+                "or `builtins.path { path = ./.; name = \"source\"; }`.\n",
-                "Location: %s\n",
+                path);
-                path,
+            return std::string(
-                positions[pos]);
+                fetchToStore(fetchSettings, *store, path, FetchMode::DryRun, storePath->name()).to_string());
-            return std::string(fetchToStore(*store, path, FetchMode::DryRun, storePath->name()).to_string());
         }
     }
     return std::string(path.baseName());
@@ -72,8 +75,9 @@ std::string EvalState::computeBaseName(const SourcePath & path, PosIdx pos)
 StorePath EvalState::mountInput(
     fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor, bool requireLockable)
 {
-    auto storePath = settings.lazyTrees ? StorePath::random(input.getName())
+    auto storePath = settings.lazyTrees
-                                        : fetchToStore(*store, accessor, FetchMode::Copy, input.getName());
+                         ? StorePath::random(input.getName())
+                         : fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
 
     allowPath(storePath); // FIXME: should just whitelist the entire virtual store
 
@@ -84,7 +88,7 @@ StorePath EvalState::mountInput(
             if (store->isValidPath(storePath))
                 _narHash = store->queryPathInfo(storePath)->narHash;
             else
-                _narHash = fetchToStore2(*store, accessor, FetchMode::DryRun, input.getName()).second;
+                _narHash = fetchToStore2(fetchSettings, *store, accessor, FetchMode::DryRun, input.getName()).second;
         }
         return _narHash;
     };

src/libexpr/primops.cc

@@ -14,6 +14,7 @@
 #include "nix/expr/value-to-xml.hh"
 #include "nix/expr/primops.hh"
 #include "nix/fetchers/fetch-to-store.hh"
+#include "nix/util/sort.hh"
 #include "nix/util/mounted-source-accessor.hh"
 
 #include <boost/container/small_vector.hpp>
@@ -421,7 +422,7 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu
 void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.exec");
-    auto elems = args[0]->listElems();
+    auto elems = args[0]->listView();
     auto count = args[0]->listSize();
     if (count == 0)
         state.error<EvalError>("at least one argument to 'exec' required").atPos(pos).debugThrow();
@@ -430,7 +431,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v)
             "while evaluating the first element of the argument passed to builtins.exec",
             false, false).toOwned();
     Strings commandArgs;
-    for (unsigned int i = 1; i < args[0]->listSize(); ++i) {
+    for (size_t i = 1; i < count; ++i) {
         commandArgs.push_back(
                 state.coerceToString(pos, *elems[i], context,
                         "while evaluating an element of the argument passed to builtins.exec",
@@ -658,7 +659,7 @@ struct CompareValues
                     // Note: we don't take the accessor into account
                     // since it's not obvious how to compare them in a
                     // reproducible way.
-                    return strcmp(v1->payload.path.path, v2->payload.path.path) < 0;
+                    return strcmp(v1->pathStr(), v2->pathStr()) < 0;
                 case nList:
                     // Lexicographic comparison
                     for (size_t i = 0;; i++) {
@@ -666,8 +667,8 @@ struct CompareValues
                             return false;
                         } else if (i == v1->listSize()) {
                             return true;
-                        } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i], pos, errorCtx)) {
+                        } else if (!state.eqValues(*v1->listView()[i], *v2->listView()[i], pos, errorCtx)) {
-                            return (*this)(v1->listElems()[i], v2->listElems()[i], "while comparing two list elements");
+                            return (*this)(v1->listView()[i], v2->listView()[i], "while comparing two list elements");
                         }
                     }
                 default:
@@ -685,31 +686,17 @@ struct CompareValues
 
 typedef std::list<Value *, gc_allocator<Value *>> ValueList;
 
-
-static Bindings::const_iterator getAttr(
-    EvalState & state,
-    Symbol attrSym,
-    const Bindings * attrSet,
-    std::string_view errorCtx)
-{
-    auto value = attrSet->find(attrSym);
-    if (value == attrSet->end()) {
-        state.error<TypeError>("attribute '%s' missing", state.symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow();
-    }
-    return value;
-}
-
 static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     state.forceAttrs(*args[0], noPos, "while evaluating the first argument passed to builtins.genericClosure");
 
     /* Get the start set. */
-    auto startSet = getAttr(state, state.sStartSet, args[0]->attrs(), "in the attrset passed as argument to builtins.genericClosure");
+    auto startSet = state.getAttr(state.sStartSet, args[0]->attrs(), "in the attrset passed as argument to builtins.genericClosure");
 
     state.forceList(*startSet->value, noPos, "while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure");
 
     ValueList workSet;
-    for (auto elem : startSet->value->listItems())
+    for (auto elem : startSet->value->listView())
         workSet.push_back(elem);
 
     if (startSet->value->listSize() == 0) {
@@ -718,7 +705,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a
     }
 
     /* Get the operator. */
-    auto op = getAttr(state, state.sOperator, args[0]->attrs(), "in the attrset passed as argument to builtins.genericClosure");
+    auto op = state.getAttr(state.sOperator, args[0]->attrs(), "in the attrset passed as argument to builtins.genericClosure");
     state.forceFunction(*op->value, noPos, "while evaluating the 'operator' attribute passed as argument to builtins.genericClosure");
 
     /* Construct the closure by applying the operator to elements of
@@ -735,7 +722,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a
 
         state.forceAttrs(*e, noPos, "while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure");
 
-        auto key = getAttr(state, state.sKey, e->attrs(), "in one of the attrsets generated by (or initially passed to) builtins.genericClosure");
+        auto key = state.getAttr(state.sKey, e->attrs(), "in one of the attrsets generated by (or initially passed to) builtins.genericClosure");
         state.forceValue(*key->value, noPos);
 
         if (!doneKeys.insert(key->value).second) continue;
@@ -747,7 +734,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a
         state.forceList(newElements, noPos, "while evaluating the return value of the `operator` passed to builtins.genericClosure");
 
         /* Add the values returned by the operator to the work set. */
-        for (auto elem : newElements.listItems()) {
+        for (auto elem : newElements.listView()) {
             state.forceValue(*elem, noPos); // "while evaluating one one of the elements returned by the `operator` passed to builtins.genericClosure");
             workSet.push_back(elem);
         }
@@ -919,7 +906,7 @@ static void prim_ceil(EvalState & state, const PosIdx pos, Value * * args, Value
         auto arg = args[0]->integer();
         auto res = v.integer();
         if (arg != res) {
-            state.error<EvalError>("Due to a bug (see https://github.com/NixOS/nix/issues/12899) a loss of precision occured in previous Nix versions because the NixInt argument %1% was rounded to %2%.\n\tFuture Nix versions might implement the correct behavior.", arg, res).atPos(pos).debugThrow();
+            state.error<EvalError>("Due to a bug (see https://github.com/NixOS/nix/issues/12899) a loss of precision occurred in previous Nix versions because the NixInt argument %1% was rounded to %2%.\n\tFuture Nix versions might implement the correct behavior.", arg, res).atPos(pos).debugThrow();
         }
     }
 }
@@ -960,7 +947,7 @@ static void prim_floor(EvalState & state, const PosIdx pos, Value * * args, Valu
         auto arg = args[0]->integer();
         auto res = v.integer();
         if (arg != res) {
-            state.error<EvalError>("Due to a bug (see https://github.com/NixOS/nix/issues/12899) a loss of precision occured in previous Nix versions because the NixInt argument %1% was rounded to %2%.\n\tFuture Nix versions might implement the correct behavior.", arg, res).atPos(pos).debugThrow();
+            state.error<EvalError>("Due to a bug (see https://github.com/NixOS/nix/issues/12899) a loss of precision occurred in previous Nix versions because the NixInt argument %1% was rounded to %2%.\n\tFuture Nix versions might implement the correct behavior.", arg, res).atPos(pos).debugThrow();
         }
     }
 }
@@ -994,7 +981,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va
     ReplExitStatus (* savedDebugRepl)(ref<EvalState> es, const ValMap & extraEnv) = nullptr;
     if (state.debugRepl && state.settings.ignoreExceptionsDuringTry)
     {
-        /* to prevent starting the repl from exceptions withing a tryEval, null it. */
+        /* to prevent starting the repl from exceptions within a tryEval, null it. */
         savedDebugRepl = state.debugRepl;
         state.debugRepl = nullptr;
     }
@@ -1200,7 +1187,7 @@ static void prim_second(EvalState & state, const PosIdx pos, Value * * args, Val
 
 static void derivationStrictInternal(
     EvalState & state,
-    const std::string & name,
+    std::string_view name,
     const Bindings * attrs,
     Value & v);
 
@@ -1218,9 +1205,9 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
     auto attrs = args[0]->attrs();
 
     /* Figure out the name first (for stack backtraces). */
-    auto nameAttr = getAttr(state, state.sName, attrs, "in the attrset passed as argument to builtins.derivationStrict");
+    auto nameAttr = state.getAttr(state.sName, attrs, "in the attrset passed as argument to builtins.derivationStrict");
 
-    std::string drvName;
+    std::string_view drvName;
     try {
         drvName = state.forceStringNoCtx(*nameAttr->value, pos, "while evaluating the `name` attribute passed to builtins.derivationStrict");
     } catch (Error & e) {
@@ -1279,7 +1266,7 @@ static void checkDerivationName(EvalState & state, std::string_view drvName)
 
 static void derivationStrictInternal(
     EvalState & state,
-    const std::string & drvName,
+    std::string_view drvName,
     const Bindings * attrs,
     Value & v)
 {
@@ -1387,7 +1374,7 @@ static void derivationStrictInternal(
                command-line arguments to the builder. */
             else if (i->name == state.sArgs) {
                 state.forceList(*i->value, pos, context_below);
-                for (auto elem : i->value->listItems()) {
+                for (auto elem : i->value->listView()) {
                     auto s = state.coerceToString(pos, *elem, context,
                             "while evaluating an element of the argument list",
                             true).toOwned();
@@ -1419,7 +1406,7 @@ static void derivationStrictInternal(
                         /* Require ‘outputs’ to be a list of strings. */
                         state.forceList(*i->value, pos, context_below);
                         Strings ss;
-                        for (auto elem : i->value->listItems())
+                        for (auto elem : i->value->listView())
                             ss.emplace_back(state.forceStringNoCtx(*elem, pos, context_below));
                         handleOutputs(ss);
                     }
@@ -1448,6 +1435,8 @@ static void derivationStrictInternal(
                     else if (i->name == state.sOutputHashMode) handleHashMode(s);
                     else if (i->name == state.sOutputs)
                         handleOutputs(tokenizeString<Strings>(s));
+                    else if (i->name == state.sJson)
+                        warn("In derivation '%s': setting structured attributes via '__json' is deprecated, and may be disallowed in future versions of Nix. Set '__structuredAttrs = true' instead.", drvName);
                 }
 
             }
@@ -1917,7 +1906,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
 
     LookupPath lookupPath;
 
-    for (auto v2 : args[0]->listItems()) {
+    for (auto v2 : args[0]->listView()) {
         state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.findFile");
 
         std::string prefix;
@@ -1925,7 +1914,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
         if (i != v2->attrs()->end())
             prefix = state.forceStringNoCtx(*i->value, pos, "while evaluating the `prefix` attribute of an element of the list passed to builtins.findFile");
 
-        i = getAttr(state, state.sPath, v2->attrs(), "in an element of the __nixPath");
+        i = state.getAttr(state.sPath, v2->attrs(), "in an element of the __nixPath");
 
         NixStringContext context;
         auto path = state.coerceToString(pos, *i->value, context,
@@ -1934,7 +1923,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
 
         try {
             auto rewrites = state.realiseContext(context);
-            path = rewriteStrings(path, rewrites);
+            path = rewriteStrings(std::move(path), rewrites);
         } catch (InvalidPathError & e) {
             state.error<EvalError>(
                 "cannot find '%1%', since path '%2%' is not valid",
@@ -1944,8 +1933,8 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
         }
 
         lookupPath.elements.emplace_back(LookupPath::Elem {
-            .prefix = LookupPath::Prefix { .s = prefix },
+            .prefix = LookupPath::Prefix { .s = std::move(prefix) },
-            .path = LookupPath::Path { .s = path },
+            .path = LookupPath::Path { .s = std::move(path) },
         });
     }
 
@@ -2218,7 +2207,7 @@ static RegisterPrimOp primop_outputOf({
       [input placeholder string](@docroot@/store/derivation/index.md#input-placeholder)
       if needed.
 
-      If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addresed), the output path is returned.
+      If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addressed), the output path is returned.
       But if the derivation is content-addressed or if the derivation is itself not-statically produced (i.e. is the output of another derivation), an input placeholder is returned instead.
 
       *`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be an input placeholder reference.
@@ -2410,7 +2399,7 @@ static RegisterPrimOp primop_fromJSON({
 static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     NixStringContext context;
-    std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile"));
+    auto name = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile");
     std::string contents(state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile"));
 
     StorePathSet refs;
@@ -2591,6 +2580,7 @@ static void addPath(
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
             // FIXME: make this lazy?
             auto dstPath = fetchToStore(
+                state.fetchSettings,
                 *state.store,
                 path.resolveSymlinks(),
                 settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
@@ -2636,7 +2626,7 @@ static RegisterPrimOp primop_filterSource({
       > the name of the input directory. Since `<hash>` depends on the
       > unfiltered directory, the name of the output directory
       > indirectly depends on files that are filtered out by the
-      > function. This triggers a rebuild even when a filtered-out
+      > function. This triggers a rebuild even when a filtered out
       > file is changed. Use `builtins.path` instead, which allows
       > specifying the name of the output directory.
 
@@ -2681,7 +2671,7 @@ static RegisterPrimOp primop_filterSource({
 static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     std::optional<SourcePath> path;
-    std::string name;
+    std::string_view name;
     Value * filterFun = nullptr;
     auto method = ContentAddressMethod::Raw::NixArchive;
     std::optional<Hash> expectedHash;
@@ -2769,7 +2759,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args,
     auto list = state.buildList(args[0]->attrs()->size());
 
     for (const auto & [n, i] : enumerate(*args[0]->attrs()))
-        (list[n] = state.allocValue())->mkString(state.symbols[i.name]);
+        list[n] = Value::toPtr(state.symbols[i.name]);
 
     std::sort(list.begin(), list.end(),
               [](Value * v1, Value * v2) { return strcmp(v1->c_str(), v2->c_str()) < 0; });
@@ -2827,8 +2817,7 @@ void prim_getAttr(EvalState & state, const PosIdx pos, Value * * args, Value & v
 {
     auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getAttr");
     state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.getAttr");
-    auto i = getAttr(
+    auto i = state.getAttr(
-        state,
         state.symbols.create(attr),
         args[1]->attrs(),
         "in the attribute set under consideration"
@@ -2875,7 +2864,7 @@ static RegisterPrimOp primop_unsafeGetAttrPos(PrimOp {
     .fun = prim_unsafeGetAttrPos,
 });
 
-// access to exact position information (ie, line and colum numbers) is deferred
+// access to exact position information (ie, line and column numbers) is deferred
 // due to the cost associated with calculating that information and how rarely
 // it is used in practice. this is achieved by creating thunks to otherwise
 // inaccessible primops that are not exposed as __op or under builtins to turn
@@ -2887,7 +2876,7 @@ static RegisterPrimOp primop_unsafeGetAttrPos(PrimOp {
 // but each type of thunk has an associated runtime cost in the current evaluator.
 // as with black holes this cost is too high to justify another thunk type to check
 // for in the very hot path that is forceValue.
-static struct LazyPosAcessors {
+static struct LazyPosAccessors {
     PrimOp primop_lineOfPos{
         .arity = 1,
         .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) {
@@ -2903,7 +2892,7 @@ static struct LazyPosAcessors {
 
     Value lineOfPos, columnOfPos;
 
-    LazyPosAcessors()
+    LazyPosAccessors()
     {
         lineOfPos.mkPrimOp(&primop_lineOfPos);
         columnOfPos.mkPrimOp(&primop_columnOfPos);
@@ -2969,7 +2958,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args
     // 64: large enough to fit the attributes of a derivation
     boost::container::small_vector<Attr, 64> names;
     names.reserve(args[1]->listSize());
-    for (auto elem : args[1]->listItems()) {
+    for (auto elem : args[1]->listView()) {
         state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs");
         names.emplace_back(state.symbols.create(elem->string_view()), nullptr);
     }
@@ -3011,25 +3000,48 @@ static void prim_listToAttrs(EvalState & state, const PosIdx pos, Value * * args
 {
     state.forceList(*args[0], pos, "while evaluating the argument passed to builtins.listToAttrs");
 
-    auto attrs = state.buildBindings(args[0]->listSize());
+    // Step 1. Sort the name-value attrsets in place using the memory we allocate for the result
+    auto listView = args[0]->listView();
+    size_t listSize = listView.size();
+    auto & bindings = *state.allocBindings(listSize);
+    using ElemPtr = decltype(&bindings[0].value);
 
-    std::set<Symbol> seen;
+    for (const auto & [n, v2] : enumerate(listView)) {
-
-    for (auto v2 : args[0]->listItems()) {
         state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.listToAttrs");
 
-        auto j = getAttr(state, state.sName, v2->attrs(), "in a {name=...; value=...;} pair");
+        auto j = state.getAttr(state.sName, v2->attrs(), "in a {name=...; value=...;} pair");
 
         auto name = state.forceStringNoCtx(*j->value, j->pos, "while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs");
-
         auto sym = state.symbols.create(name);
-        if (seen.insert(sym).second) {
+
-            auto j2 = getAttr(state, state.sValue, v2->attrs(), "in a {name=...; value=...;} pair");
+        // (ab)use Attr to store a Value * * instead of a Value *, so that we can stabilize the sort using the Value * *
-            attrs.insert(sym, j2->value, j2->pos);
+        bindings[n] = Attr(sym, std::bit_cast<Value *>(&v2));
-        }
     }
 
-    v.mkAttrs(attrs);
+    std::sort(&bindings[0], &bindings[listSize], [](const Attr & a, const Attr & b) {
+        // Note that .value is actually a Value * * that corresponds to the position in the list
+        return a < b || (!(a > b) && std::bit_cast<ElemPtr>(a.value) < std::bit_cast<ElemPtr>(b.value));
+    });
+
+    // Step 2. Unpack the bindings in place and skip name-value pairs with duplicate names
+    Symbol prev;
+    for (size_t n = 0; n < listSize; n++) {
+        auto attr = bindings[n];
+        if (prev == attr.name) {
+            continue;
+        }
+        // Note that .value is actually a Value * *; see earlier comments
+        Value * v2 = *std::bit_cast<ElemPtr>(attr.value);
+
+        auto j = state.getAttr(state.sValue, v2->attrs(), "in a {name=...; value=...;} pair");
+        prev = attr.name;
+        bindings.push_back({prev, j->value, j->pos});
+    }
+    // help GC and clear end of allocated array
+    for (size_t n = bindings.size(); n < listSize; n++) {
+        bindings[n] = Attr{};
+    }
+    v.mkAttrs(&bindings);
 }
 
 static RegisterPrimOp primop_listToAttrs({
@@ -3149,14 +3161,14 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V
     SmallValueVector<nonRecursiveStackReservation> res(args[1]->listSize());
     size_t found = 0;
 
-    for (auto v2 : args[1]->listItems()) {
+    for (auto v2 : args[1]->listView()) {
         state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs");
         if (auto i = v2->attrs()->get(attrName))
             res[found++] = i->value;
     }
 
     auto list = state.buildList(found);
-    for (unsigned int n = 0; n < found; ++n)
+    for (size_t n = 0; n < found; ++n)
         list[n] = res[n];
     v.mkList(list);
 }
@@ -3188,15 +3200,21 @@ static void prim_functionArgs(EvalState & state, const PosIdx pos, Value * * arg
     if (!args[0]->isLambda())
         state.error<TypeError>("'functionArgs' requires a function").atPos(pos).debugThrow();
 
-    if (!args[0]->payload.lambda.fun->hasFormals()) {
+    if (!args[0]->lambda().fun->hasFormals()) {
         v.mkAttrs(&state.emptyBindings);
         return;
     }
 
-    auto attrs = state.buildBindings(args[0]->payload.lambda.fun->formals->formals.size());
+    const auto &formals = args[0]->lambda().fun->formals->formals;
-    for (auto & i : args[0]->payload.lambda.fun->formals->formals)
+    auto attrs = state.buildBindings(formals.size());
+    for (auto & i : formals)
         attrs.insert(i.name, state.getBool(i.def), i.pos);
-    v.mkAttrs(attrs);
+    /* Optimization: avoid sorting bindings. `formals` must already be sorted according to
+       (std::tie(a.name, a.pos) < std::tie(b.name, b.pos)) predicate, so the following assertion
+       always holds:
+       assert(std::is_sorted(attrs.alreadySorted()->begin(), attrs.alreadySorted()->end()));
+       .*/
+    v.mkAttrs(attrs.alreadySorted());
 }
 
 static RegisterPrimOp primop_functionArgs({
@@ -3224,9 +3242,8 @@ static void prim_mapAttrs(EvalState & state, const PosIdx pos, Value * * args, V
     auto attrs = state.buildBindings(args[1]->attrs()->size());
 
     for (auto & i : *args[1]->attrs()) {
-        Value * vName = state.allocValue();
+        Value * vName = Value::toPtr(state.symbols[i.name]);
         Value * vFun2 = state.allocValue();
-        vName->mkString(state.symbols[i.name]);
         vFun2->mkApp(args[0], vName);
         attrs.alloc(i.name).mkApp(vFun2, i.value);
     }
@@ -3269,7 +3286,7 @@ static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value * * arg
 
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.zipAttrsWith");
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.zipAttrsWith");
-    const auto listItems = args[1]->listItems();
+    const auto listItems = args[1]->listView();
 
     for (auto & vElem : listItems) {
         state.forceAttrs(*vElem, noPos, "while evaluating a value of the list passed as second argument to builtins.zipAttrsWith");
@@ -3290,8 +3307,7 @@ static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value * * arg
     auto attrs = state.buildBindings(attrsSeen.size());
 
     for (auto & [sym, elem] : attrsSeen) {
-        auto name = state.allocValue();
+        auto name = Value::toPtr(state.symbols[sym]);
-        name->mkString(state.symbols[sym]);
         auto call1 = state.allocValue();
         call1->mkApp(args[0], name);
         auto call2 = state.allocValue();
@@ -3363,14 +3379,14 @@ static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Val
 {
     NixInt::Inner n = state.forceInt(*args[1], pos, "while evaluating the second argument passed to 'builtins.elemAt'").value;
     state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.elemAt'");
-    if (n < 0 || (unsigned int) n >= args[0]->listSize())
+    if (n < 0 || std::make_unsigned_t<NixInt::Inner>(n) >= args[0]->listSize())
         state.error<EvalError>(
             "'builtins.elemAt' called with index %d on a list of size %d",
             n,
             args[0]->listSize()
         ).atPos(pos).debugThrow();
-    state.forceValue(*args[0]->listElems()[n], pos);
+    state.forceValue(*args[0]->listView()[n], pos);
-    v = *args[0]->listElems()[n];
+    v = *args[0]->listView()[n];
 }
 
 static RegisterPrimOp primop_elemAt({
@@ -3391,8 +3407,8 @@ static void prim_head(EvalState & state, const PosIdx pos, Value * * args, Value
         state.error<EvalError>(
             "'builtins.head' called on an empty list"
         ).atPos(pos).debugThrow();
-    state.forceValue(*args[0]->listElems()[0], pos);
+    state.forceValue(*args[0]->listView()[0], pos);
-    v = *args[0]->listElems()[0];
+    v = *args[0]->listView()[0];
 }
 
 static RegisterPrimOp primop_head({
@@ -3417,7 +3433,7 @@ static void prim_tail(EvalState & state, const PosIdx pos, Value * * args, Value
 
     auto list = state.buildList(args[0]->listSize() - 1);
     for (const auto & [n, v] : enumerate(list))
-        v = args[0]->listElems()[n + 1];
+        v = args[0]->listView()[n + 1];
     v.mkList(list);
 }
 
@@ -3452,7 +3468,7 @@ static void prim_map(EvalState & state, const PosIdx pos, Value * * args, Value
     auto list = state.buildList(args[1]->listSize());
     for (const auto & [n, v] : enumerate(list))
         (v = state.allocValue())->mkApp(
-            args[0], args[1]->listElems()[n]);
+            args[0], args[1]->listView()[n]);
     v.mkList(list);
 }
 
@@ -3486,15 +3502,16 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val
 
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter");
 
-    SmallValueVector<nonRecursiveStackReservation> vs(args[1]->listSize());
+    auto len = args[1]->listSize();
+    SmallValueVector<nonRecursiveStackReservation> vs(len);
     size_t k = 0;
 
     bool same = true;
-    for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
+    for (size_t n = 0; n < len; ++n) {
         Value res;
-        state.callFunction(*args[0], *args[1]->listElems()[n], res, noPos);
+        state.callFunction(*args[0], *args[1]->listView()[n], res, noPos);
         if (state.forceBool(res, pos, "while evaluating the return value of the filtering function passed to builtins.filter"))
-            vs[k++] = args[1]->listElems()[n];
+            vs[k++] = args[1]->listView()[n];
         else
             same = false;
     }
@@ -3523,7 +3540,7 @@ static void prim_elem(EvalState & state, const PosIdx pos, Value * * args, Value
 {
     bool res = false;
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.elem");
-    for (auto elem : args[1]->listItems())
+    for (auto elem : args[1]->listView())
         if (state.eqValues(*args[0], *elem, pos, "while searching for the presence of the given element in the list")) {
             res = true;
             break;
@@ -3545,7 +3562,8 @@ static RegisterPrimOp primop_elem({
 static void prim_concatLists(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.concatLists");
-    state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos, "while evaluating a value of the list passed to builtins.concatLists");
+    auto listView = args[0]->listView();
+    state.concatLists(v, args[0]->listSize(), listView.data(), pos, "while evaluating a value of the list passed to builtins.concatLists");
 }
 
 static RegisterPrimOp primop_concatLists({
@@ -3583,7 +3601,8 @@ static void prim_foldlStrict(EvalState & state, const PosIdx pos, Value * * args
     if (args[2]->listSize()) {
         Value * vCur = args[1];
 
-        for (auto [n, elem] : enumerate(args[2]->listItems())) {
+        auto listView = args[2]->listView();
+        for (auto [n, elem] : enumerate(listView)) {
             Value * vs []{vCur, elem};
             vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
             state.callFunction(*args[0], vs, *vCur, pos);
@@ -3625,7 +3644,7 @@ static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * ar
         : "while evaluating the return value of the function passed to builtins.all";
 
     Value vTmp;
-    for (auto elem : args[1]->listItems()) {
+    for (auto elem : args[1]->listView()) {
         state.callFunction(*args[0], *elem, vTmp, pos);
         bool res = state.forceBool(vTmp, pos, errorCtx);
         if (res == any) {
@@ -3672,12 +3691,12 @@ static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Va
 {
     auto len_ = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList").value;
 
-    if (len_ < 0)
+    if (len_ < 0 || std::make_unsigned_t<NixInt::Inner>(len_) > std::numeric_limits<size_t>::max())
         state.error<EvalError>("cannot create list of size %1%", len_).atPos(pos).debugThrow();
 
     size_t len = size_t(len_);
 
-    // More strict than striclty (!) necessary, but acceptable
+    // More strict than strictly (!) necessary, but acceptable
     // as evaluating map without accessing any values makes little sense.
     state.forceFunction(*args[0], noPos, "while evaluating the first argument passed to builtins.genList");
 
@@ -3723,7 +3742,7 @@ static void prim_sort(EvalState & state, const PosIdx pos, Value * * args, Value
 
     auto list = state.buildList(len);
     for (const auto & [n, v] : enumerate(list))
-        state.forceValue(*(v = args[1]->listElems()[n]), pos);
+        state.forceValue(*(v = args[1]->listView()[n]), pos);
 
     auto comparator = [&](Value * a, Value * b) {
         /* Optimization: if the comparator is lessThan, bypass
@@ -3740,10 +3759,14 @@ static void prim_sort(EvalState & state, const PosIdx pos, Value * * args, Value
         return state.forceBool(vBool, pos, "while evaluating the return value of the sorting function passed to builtins.sort");
     };
 
-    /* FIXME: std::sort can segfault if the comparator is not a strict
+    /* NOTE: Using custom implementation because std::sort and std::stable_sort
-       weak ordering. What to do? std::stable_sort() seems more
+       are not resilient to comparators that violate strict weak ordering. Diagnosing
-       resilient, but no guarantees... */
+       incorrect implementations is a O(n^3) problem, so doing the checks is much more
-    std::stable_sort(list.begin(), list.end(), comparator);
+       expensive that doing the sorting. For this reason we choose to use sorting algorithms
+       that are can't be broken by invalid comprators. peeksort (mergesort)
+       doesn't misbehave when any of the strict weak order properties is
+       violated - output is always a reordering of the input. */
+    peeksort(list.begin(), list.end(), comparator);
 
     v.mkList(list);
 }
@@ -3765,6 +3788,32 @@ static RegisterPrimOp primop_sort({
 
       This is a stable sort: it preserves the relative order of elements
       deemed equal by the comparator.
+
+      *comparator* must impose a strict weak ordering on the set of values
+      in the *list*. This means that for any elements *a*, *b* and *c* from the
+      *list*, *comparator* must satisfy the following relations:
+
+        1. Transitivity
+
+        ```nix
+        comparator a b && comparator b c -> comparator a c
+        ```
+
+        1. Irreflexivity
+
+        ```nix
+        comparator a a == false
+        ```
+
+        1. Transitivity of equivalence
+
+        ```nix
+        let equiv = a: b: (!comparator a b && !comparator b a); in
+        equiv a b && equiv b c -> equiv a c
+        ```
+
+      If the *comparator* violates any of these properties, then `builtins.sort`
+      reorders elements in an unspecified manner.
     )",
     .fun = prim_sort,
 });
@@ -3778,8 +3827,8 @@ static void prim_partition(EvalState & state, const PosIdx pos, Value * * args,
 
     ValueVector right, wrong;
 
-    for (unsigned int n = 0; n < len; ++n) {
+    for (size_t n = 0; n < len; ++n) {
-        auto vElem = args[1]->listElems()[n];
+        auto vElem = args[1]->listView()[n];
         state.forceValue(*vElem, pos);
         Value res;
         state.callFunction(*args[0], *vElem, res, pos);
@@ -3836,7 +3885,7 @@ static void prim_groupBy(EvalState & state, const PosIdx pos, Value * * args, Va
 
     ValueVectorMap attrs;
 
-    for (auto vElem : args[1]->listItems()) {
+    for (auto vElem : args[1]->listView()) {
         Value res;
         state.callFunction(*args[0], *vElem, res, pos);
         auto name = state.forceStringNoCtx(res, pos, "while evaluating the return value of the grouping function passed to builtins.groupBy");
@@ -3891,8 +3940,8 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
     SmallTemporaryValueVector<conservativeStackReservation> lists(nrLists);
     size_t len = 0;
 
-    for (unsigned int n = 0; n < nrLists; ++n) {
+    for (size_t n = 0; n < nrLists; ++n) {
-        Value * vElem = args[1]->listElems()[n];
+        Value * vElem = args[1]->listView()[n];
         state.callFunction(*args[0], *vElem, lists[n], pos);
         state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to builtins.concatMap");
         len += lists[n].listSize();
@@ -3900,10 +3949,11 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
 
     auto list = state.buildList(len);
     auto out = list.elems;
-    for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
+    for (size_t n = 0, pos = 0; n < nrLists; ++n) {
-        auto l = lists[n].listSize();
+        auto listView = lists[n].listView();
+        auto l = listView.size();
         if (l)
-            memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *));
+            memcpy(out + pos, listView.data(), l * sizeof(Value *));
         pos += l;
     }
     v.mkList(list);
@@ -4165,22 +4215,20 @@ static RegisterPrimOp primop_toString({
    non-negative. */
 static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
+    using NixUInt = std::make_unsigned_t<NixInt::Inner>;
     NixInt::Inner start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring").value;
 
     if (start < 0)
         state.error<EvalError>("negative start position in 'substring'").atPos(pos).debugThrow();
 
-
     NixInt::Inner len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring").value;
 
     // Negative length may be idiomatically passed to builtins.substring to get
     // the tail of the string.
-    if (len < 0) {
+    auto _len = std::numeric_limits<std::string::size_type>::max();
-        len = std::numeric_limits<NixInt::Inner>::max();
-    }
 
     // Special-case on empty substring to avoid O(n) strlen
-    // This allows for the use of empty substrings to efficently capture string context
+    // This allows for the use of empty substrings to efficiently capture string context
     if (len == 0) {
         state.forceValue(*args[2], pos);
         if (args[2]->type() == nString) {
@@ -4189,10 +4237,14 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args,
         }
     }
 
+    if (len >= 0 && NixUInt(len) < _len) {
+        _len = len;
+    }
+
     NixStringContext context;
     auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring");
 
-    v.mkString((unsigned int) start >= s->size() ? "" : s->substr(start, len), context);
+    v.mkString(NixUInt(start) >= s->size() ? "" : s->substr(start, _len), context);
 }
 
 static RegisterPrimOp primop_substring({
@@ -4263,7 +4315,7 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args
     state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.convertHash");
     auto inputAttrs = args[0]->attrs();
 
-    auto iteratorHash = getAttr(state, state.symbols.create("hash"), inputAttrs, "while locating the attribute 'hash'");
+    auto iteratorHash = state.getAttr(state.symbols.create("hash"), inputAttrs, "while locating the attribute 'hash'");
     auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'");
 
     auto iteratorHashAlgo = inputAttrs->get(state.symbols.create("hashAlgo"));
@@ -4271,7 +4323,7 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args
     if (iteratorHashAlgo)
         ha = parseHashAlgo(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'"));
 
-    auto iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs(), "while locating the attribute 'toHashFormat'");
+    auto iteratorToHashFormat = state.getAttr(state.symbols.create("toHashFormat"), args[0]->attrs(), "while locating the attribute 'toHashFormat'");
     HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'"));
 
     v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI));
@@ -4496,7 +4548,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
             // Add a list for matched substrings.
             const size_t slen = match.size() - 1;
 
-            // Start at 1, beacause the first match is the whole string.
+            // Start at 1, because the first match is the whole string.
             auto list2 = state.buildList(slen);
             for (const auto & [si, v2] : enumerate(list2)) {
                 if (!match[si + 1].matched)
@@ -4577,7 +4629,7 @@ static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value * *
     res.reserve((args[1]->listSize() + 32) * sep.size());
     bool first = true;
 
-    for (auto elem : args[1]->listItems()) {
+    for (auto elem : args[1]->listView()) {
         if (first) first = false; else res += sep;
         res += *state.coerceToString(pos, *elem, context, "while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep");
     }
@@ -4605,13 +4657,13 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a
             "'from' and 'to' arguments passed to builtins.replaceStrings have different lengths"
         ).atPos(pos).debugThrow();
 
-    std::vector<std::string> from;
+    std::vector<std::string_view> from;
     from.reserve(args[0]->listSize());
-    for (auto elem : args[0]->listItems())
+    for (auto elem : args[0]->listView())
         from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings"));
 
-    std::unordered_map<size_t, std::string> cache;
+    std::unordered_map<size_t, std::string_view> cache;
-    auto to = args[1]->listItems();
+    auto to = args[1]->listView();
 
     NixStringContext context;
     auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings");
@@ -4864,7 +4916,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
           1683705525
           ```
 
-          The [store path](@docroot@/store/store-path.md) of a derivation depending on `currentTime` differs for each evaluation unless both evaluate `builtins.currentTime` in the same second.
+          The [store path](@docroot@/store/store-path.md) of a derivation depending on `currentTime` differs for each evaluation, unless both evaluate `builtins.currentTime` in the same second.
         )",
         .impureOnly = true,
     });
@@ -5040,7 +5092,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
-    getBuiltins().payload.attrs->sort();
+    const_cast<Bindings *>(getBuiltins().attrs())->sort();
 
     staticBaseEnv->sort();
 

src/libexpr/primops/context.cc

@@ -332,7 +332,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
                     name
                 ).atPos(i.pos).debugThrow();
             }
-            for (auto elem : attr->value->listItems()) {
+            for (auto elem : attr->value->listView()) {
                 auto outputName = state.forceStringNoCtx(*elem, attr->pos, "while evaluating an output name within a string context");
                 context.emplace(NixStringContextElem::Built {
                     .drvPath = makeConstantStorePathRef(namePath),

src/libexpr/primops/fetchClosure.cc

@@ -129,7 +129,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
 
         if (attrName == "fromPath") {
             NixStringContext context;
-            fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint());
+            fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); // FIXME: overflow
         }
 
         else if (attrName == "toPath") {

src/libexpr/primops/fetchTree.cc

@@ -303,7 +303,7 @@ static RegisterPrimOp primop_fetchTree({
       - `"tarball"`
 
         Download a tar archive and extract it into the Nix store.
-        This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
+        This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
 
         - `url` (String, required)
 
@@ -533,11 +533,12 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     auto storePath =
         unpack
         ? fetchToStore(
+            state.fetchSettings,
             *state.store,
             fetchers::downloadTarball(state.store, state.fetchSettings, *url),
             FetchMode::Copy,
             name)
-        : fetchers::downloadFile(state.store, *url, name).storePath;
+        : fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath;
 
     if (expectedHash) {
         auto hash = unpack

src/libexpr/primops/meson.build

@@ -1,6 +1,6 @@
 generated_headers += gen_header.process(
   'derivation.nix',
-  preserve_path_from: meson.project_source_root(),
+  preserve_path_from : meson.project_source_root(),
 )
 
 sources += files(

src/libexpr/print-ambiguous.cc

@@ -54,11 +54,13 @@ void printAmbiguous(
         break;
     }
     case nList:
-        if (seen && v.listSize() && !seen->insert(v.listElems()).second)
+        /* Use pointer to the Value instead of pointer to the elements, because
+           that would need to explicitly handle the case of SmallList. */
+        if (seen && v.listSize() && !seen->insert(&v).second)
             str << "«repeated»";
         else {
             str << "[ ";
-            for (auto v2 : v.listItems()) {
+            for (auto v2 : v.listView()) {
                 if (v2)
                     printAmbiguous(state, *v2, str, seen, depth - 1);
                 else

src/libexpr/print.cc

@@ -419,8 +419,8 @@ private:
         if (depth < options.maxDepth) {
             increaseIndent();
             output << "[";
-            auto listItems = v.listItems();
+            auto listItems = v.listView();
-            auto prettyPrint = shouldPrettyPrintList(listItems);
+            auto prettyPrint = shouldPrettyPrintList(listItems.span());
 
             size_t currentListItemsPrinted = 0;
 
@@ -457,13 +457,13 @@ private:
 
         if (v.isLambda()) {
             output << "lambda";
-            if (v.payload.lambda.fun) {
+            if (v.lambda().fun) {
-                if (v.payload.lambda.fun->name) {
+                if (v.lambda().fun->name) {
-                    output << " " << state.symbols[v.payload.lambda.fun->name];
+                    output << " " << state.symbols[v.lambda().fun->name];
                 }
 
                 std::ostringstream s;
-                s << state.positions[v.payload.lambda.fun->pos];
+                s << state.positions[v.lambda().fun->pos];
                 output << " @ " << filterANSIEscapes(toView(s));
             }
         } else if (v.isPrimOp()) {

src/libexpr/value-to-json.cc

@@ -74,7 +74,7 @@ json printValueAsJSON(EvalState & state, bool strict,
         case nList: {
             out = json::array();
             int i = 0;
-            for (auto elem : v.listItems()) {
+            for (auto elem : v.listView()) {
                 try {
                     out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore));
                 } catch (Error & e) {

src/libexpr/value-to-xml.cc

@@ -114,7 +114,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
         case nList: {
             XMLOpenElement _(doc, "list");
-            for (auto v2 : v.listItems())
+            for (auto v2 : v.listView())
                 printValueAsXML(state, strict, location, *v2, doc, context, drvsSeen, pos);
             break;
         }
@@ -126,18 +126,18 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
                 break;
             }
             XMLAttrs xmlAttrs;
-            if (location) posToXML(state, xmlAttrs, state.positions[v.payload.lambda.fun->pos]);
+            if (location) posToXML(state, xmlAttrs, state.positions[v.lambda().fun->pos]);
             XMLOpenElement _(doc, "function", xmlAttrs);
 
-            if (v.payload.lambda.fun->hasFormals()) {
+            if (v.lambda().fun->hasFormals()) {
                 XMLAttrs attrs;
-                if (v.payload.lambda.fun->arg) attrs["name"] = state.symbols[v.payload.lambda.fun->arg];
+                if (v.lambda().fun->arg) attrs["name"] = state.symbols[v.lambda().fun->arg];
-                if (v.payload.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1";
+                if (v.lambda().fun->formals->ellipsis) attrs["ellipsis"] = "1";
                 XMLOpenElement _(doc, "attrspat", attrs);
-                for (auto & i : v.payload.lambda.fun->formals->lexicographicOrder(state.symbols))
+                for (auto & i : v.lambda().fun->formals->lexicographicOrder(state.symbols))
                     doc.writeEmptyElement("attr", singletonAttrs("name", state.symbols[i.name]));
             } else
-                doc.writeEmptyElement("varpat", singletonAttrs("name", state.symbols[v.payload.lambda.fun->arg]));
+                doc.writeEmptyElement("varpat", singletonAttrs("name", state.symbols[v.lambda().fun->arg]));
 
             break;
         }

src/libfetchers/attrs.cc

@@ -89,9 +89,9 @@ bool getBoolAttr(const Attrs & attrs, const std::string & name)
     return *s;
 }
 
-std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
+StringMap attrsToQuery(const Attrs & attrs)
 {
-    std::map<std::string, std::string> query;
+    StringMap query;
     for (auto & attr : attrs) {
         if (auto v = std::get_if<uint64_t>(&attr.second)) {
             query.insert_or_assign(attr.first, fmt("%d", *v));

src/libfetchers/cache.cc

@@ -1,4 +1,5 @@
 #include "nix/fetchers/cache.hh"
+#include "nix/fetchers/fetch-settings.hh"
 #include "nix/util/users.hh"
 #include "nix/store/sqlite.hh"
 #include "nix/util/sync.hh"
@@ -163,10 +164,12 @@ struct CacheImpl : Cache
     }
 };
 
-ref<Cache> getCache()
+ref<Cache> Settings::getCache() const
 {
-    static auto cache = std::make_shared<CacheImpl>();
+    auto cache(_cache.lock());
-    return ref<Cache>(cache);
+    if (!*cache)
+        *cache = std::make_shared<CacheImpl>();
+    return ref<Cache>(*cache);
 }
 
 }

src/libfetchers/fetch-to-store.cc

@@ -1,5 +1,6 @@
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/fetchers/fetchers.hh"
+#include "nix/fetchers/fetch-settings.hh"
 
 namespace nix {
 
@@ -16,6 +17,7 @@ fetchers::Cache::Key makeSourcePathToHashCacheKey(
 }
 
 StorePath fetchToStore(
+    const fetchers::Settings & settings,
     Store & store,
     const SourcePath & path,
     FetchMode mode,
@@ -24,10 +26,11 @@ StorePath fetchToStore(
     PathFilter * filter,
     RepairFlag repair)
 {
-    return fetchToStore2(store, path, mode, name, method, filter, repair).first;
+    return fetchToStore2(settings, store, path, mode, name, method, filter, repair).first;
 }
 
 std::pair<StorePath, Hash> fetchToStore2(
+    const fetchers::Settings & settings,
     Store & store,
     const SourcePath & path,
     FetchMode mode,
@@ -45,7 +48,7 @@ std::pair<StorePath, Hash> fetchToStore2(
 
     if (fingerprint) {
         cacheKey = makeSourcePathToHashCacheKey(*fingerprint, method, subpath.abs());
-        if (auto res = fetchers::getCache()->lookup(*cacheKey)) {
+        if (auto res = settings.getCache()->lookup(*cacheKey)) {
             auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash"));
             auto storePath = store.makeFixedOutputPathFromCA(name,
                 ContentAddressWithReferences::fromParts(method, hash, {}));
@@ -96,7 +99,7 @@ std::pair<StorePath, Hash> fetchToStore2(
         });
 
     if (cacheKey)
-        fetchers::getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}});
+        settings.getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}});
 
     return {storePath, hash};
 }

src/libfetchers/fetchers.cc

@@ -135,7 +135,7 @@ ParsedURL Input::toURL() const
     return scheme->toURL(*this);
 }
 
-std::string Input::toURLString(const std::map<std::string, std::string> & extraQuery) const
+std::string Input::toURLString(const StringMap & extraQuery) const
 {
     auto url = toURL();
     for (auto & attr : extraQuery)
@@ -198,7 +198,7 @@ std::tuple<StorePath, ref<SourceAccessor>, Input> Input::fetchToStore(ref<Store>
     try {
         auto [accessor, result] = getAccessorUnchecked(store);
 
-        auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, result.getName());
+        auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
 
         auto narHash = store->queryPathInfo(storePath)->narHash;
         result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));

src/libfetchers/git-utils.cc

@@ -1,6 +1,7 @@
 #include "nix/fetchers/git-utils.hh"
 #include "nix/fetchers/git-lfs-fetch.hh"
 #include "nix/fetchers/cache.hh"
+#include "nix/fetchers/fetch-settings.hh"
 #include "nix/util/finally.hh"
 #include "nix/util/processes.hh"
 #include "nix/util/signals.hh"
@@ -321,8 +322,17 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
             for (size_t n = 0; n < git_commit_parentcount(commit->get()); ++n) {
                 git_commit * parent;
-                if (git_commit_parent(&parent, commit->get(), n))
+                if (git_commit_parent(&parent, commit->get(), n)) {
-                    throw Error("getting parent of Git commit '%s': %s", *git_commit_id(commit->get()), git_error_last()->message);
+                    throw Error(
+                        "Failed to retrieve the parent of Git commit '%s': %s. "
+                        "This may be due to an incomplete repository history. "
+                        "To resolve this, either enable the shallow parameter in your flake URL (?shallow=1) "
+                        "or add set the shallow parameter to true in builtins.fetchGit, "
+                        "or fetch the complete history for this branch.",
+                        *git_commit_id(commit->get()),
+                        git_error_last()->message
+                    );
+                }
                 todo.push(Commit(parent));
             }
         }
@@ -367,7 +377,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
             throw Error("iterating over .gitmodules: %s", git_error_last()->message);
 
-        std::map<std::string, std::string> entries;
+        StringMap entries;
 
         while (true) {
             git_config_entry * entry = nullptr;
@@ -586,7 +596,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         });
 
         /* Evaluate result through status code and checking if public
-           key fingerprints appear on stderr. This is neccessary
+           key fingerprints appear on stderr. This is necessary
            because the git command might also succeed due to the
            commit being signed by gpg keys that are present in the
            users key agent. */
@@ -610,18 +620,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
             throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output);
     }
 
-    Hash treeHashToNarHash(const Hash & treeHash) override
+    Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
     {
         auto accessor = getAccessor(treeHash, false, "");
 
         fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
 
-        if (auto res = fetchers::getCache()->lookup(cacheKey))
+        if (auto res = settings.getCache()->lookup(cacheKey))
             return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256);
 
         auto narHash = accessor->hashPath(CanonPath::root);
 
-        fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
+        settings.getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
 
         return narHash;
     }
@@ -655,28 +665,40 @@ ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create,
 
 struct GitSourceAccessor : SourceAccessor
 {
-    ref<GitRepoImpl> repo;
+    struct State
-    Object root;
+    {
-    std::optional<lfs::Fetch> lfsFetch = std::nullopt;
+        ref<GitRepoImpl> repo;
+        Object root;
+        std::optional<lfs::Fetch> lfsFetch = std::nullopt;
+    };
+
+    Sync<State> state_;
 
     GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs)
-        : repo(repo_)
+        : state_{
-        , root(peelToTreeOrBlob(lookupObject(*repo, hashToOID(rev)).get()))
+                State {
+                    .repo = repo_,
+                    .root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()),
+                    .lfsFetch = smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
+                }
+            }
     {
-        if (smudgeLfs)
-            lfsFetch = std::make_optional(lfs::Fetch(*repo, hashToOID(rev)));
     }
 
     std::string readBlob(const CanonPath & path, bool symlink)
     {
-        const auto blob = getBlob(path, symlink);
+        auto state(state_.lock());
 
-        if (lfsFetch) {
+        const auto blob = getBlob(*state, path, symlink);
-            if (lfsFetch->shouldFetch(path)) {
+
+        if (state->lfsFetch) {
+            if (state->lfsFetch->shouldFetch(path)) {
                 StringSink s;
                 try {
+                    // FIXME: do we need to hold the state lock while
+                    // doing this?
                     auto contents = std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
-                    lfsFetch->fetch(contents, path, s, [&s](uint64_t size){ s.s.reserve(size); });
+                    state->lfsFetch->fetch(contents, path, s, [&s](uint64_t size){ s.s.reserve(size); });
                 } catch (Error & e) {
                     e.addTrace({}, "while smudging git-lfs file '%s'", path);
                     throw;
@@ -695,15 +717,18 @@ struct GitSourceAccessor : SourceAccessor
 
     bool pathExists(const CanonPath & path) override
     {
-        return path.isRoot() ? true : (bool) lookup(path);
+        auto state(state_.lock());
+        return path.isRoot() ? true : (bool) lookup(*state, path);
     }
 
     std::optional<Stat> maybeLstat(const CanonPath & path) override
     {
-        if (path.isRoot())
+        auto state(state_.lock());
-            return Stat { .type = git_object_type(root.get()) == GIT_OBJECT_TREE ? tDirectory : tRegular };
 
-        auto entry = lookup(path);
+        if (path.isRoot())
+            return Stat { .type = git_object_type(state->root.get()) == GIT_OBJECT_TREE ? tDirectory : tRegular };
+
+        auto entry = lookup(*state, path);
         if (!entry)
             return std::nullopt;
 
@@ -731,6 +756,8 @@ struct GitSourceAccessor : SourceAccessor
 
     DirEntries readDirectory(const CanonPath & path) override
     {
+        auto state(state_.lock());
+
         return std::visit(overloaded {
             [&](Tree tree) {
                 DirEntries res;
@@ -748,7 +775,7 @@ struct GitSourceAccessor : SourceAccessor
             [&](Submodule) {
                 return DirEntries();
             }
-        }, getTree(path));
+        }, getTree(*state, path));
     }
 
     std::string readLink(const CanonPath & path) override
@@ -762,7 +789,9 @@ struct GitSourceAccessor : SourceAccessor
      */
     std::optional<Hash> getSubmoduleRev(const CanonPath & path)
     {
-        auto entry = lookup(path);
+        auto state(state_.lock());
+
+        auto entry = lookup(*state, path);
 
         if (!entry || git_tree_entry_type(entry) != GIT_OBJECT_COMMIT)
             return std::nullopt;
@@ -773,7 +802,7 @@ struct GitSourceAccessor : SourceAccessor
     std::unordered_map<CanonPath, TreeEntry> lookupCache;
 
     /* Recursively look up 'path' relative to the root. */
-    git_tree_entry * lookup(const CanonPath & path)
+    git_tree_entry * lookup(State & state, const CanonPath & path)
     {
         auto i = lookupCache.find(path);
         if (i != lookupCache.end()) return i->second.get();
@@ -783,7 +812,7 @@ struct GitSourceAccessor : SourceAccessor
 
         auto name = path.baseName().value();
 
-        auto parentTree = lookupTree(*parent);
+        auto parentTree = lookupTree(state, *parent);
         if (!parentTree) return nullptr;
 
         auto count = git_tree_entrycount(parentTree->get());
@@ -812,29 +841,29 @@ struct GitSourceAccessor : SourceAccessor
         return res;
     }
 
-    std::optional<Tree> lookupTree(const CanonPath & path)
+    std::optional<Tree> lookupTree(State & state, const CanonPath & path)
     {
         if (path.isRoot()) {
-            if (git_object_type(root.get()) == GIT_OBJECT_TREE)
+            if (git_object_type(state.root.get()) == GIT_OBJECT_TREE)
-                return dupObject<Tree>((git_tree *) &*root);
+                return dupObject<Tree>((git_tree *) &*state.root);
             else
                 return std::nullopt;
         }
 
-        auto entry = lookup(path);
+        auto entry = lookup(state, path);
         if (!entry || git_tree_entry_type(entry) != GIT_OBJECT_TREE)
             return std::nullopt;
 
         Tree tree;
-        if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry))
+        if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *state.repo, entry))
             throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message);
 
         return tree;
     }
 
-    git_tree_entry * need(const CanonPath & path)
+    git_tree_entry * need(State & state, const CanonPath & path)
     {
-        auto entry = lookup(path);
+        auto entry = lookup(state, path);
         if (!entry)
             throw Error("'%s' does not exist", showPath(path));
         return entry;
@@ -842,16 +871,16 @@ struct GitSourceAccessor : SourceAccessor
 
     struct Submodule { };
 
-    std::variant<Tree, Submodule> getTree(const CanonPath & path)
+    std::variant<Tree, Submodule> getTree(State & state, const CanonPath & path)
     {
         if (path.isRoot()) {
-            if (git_object_type(root.get()) == GIT_OBJECT_TREE)
+            if (git_object_type(state.root.get()) == GIT_OBJECT_TREE)
-                return dupObject<Tree>((git_tree *) &*root);
+                return dupObject<Tree>((git_tree *) &*state.root);
             else
-                throw Error("Git root object '%s' is not a directory", *git_object_id(root.get()));
+                throw Error("Git root object '%s' is not a directory", *git_object_id(state.root.get()));
         }
 
-        auto entry = need(path);
+        auto entry = need(state, path);
 
         if (git_tree_entry_type(entry) == GIT_OBJECT_COMMIT)
             return Submodule();
@@ -860,16 +889,16 @@ struct GitSourceAccessor : SourceAccessor
             throw Error("'%s' is not a directory", showPath(path));
 
         Tree tree;
-        if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry))
+        if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *state.repo, entry))
             throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message);
 
         return tree;
     }
 
-    Blob getBlob(const CanonPath & path, bool expectSymlink)
+    Blob getBlob(State & state, const CanonPath & path, bool expectSymlink)
     {
-        if (!expectSymlink && git_object_type(root.get()) == GIT_OBJECT_BLOB)
+        if (!expectSymlink && git_object_type(state.root.get()) == GIT_OBJECT_BLOB)
-            return dupObject<Blob>((git_blob *) &*root);
+            return dupObject<Blob>((git_blob *) &*state.root);
 
         auto notExpected = [&]()
         {
@@ -882,7 +911,7 @@ struct GitSourceAccessor : SourceAccessor
 
         if (path.isRoot()) notExpected();
 
-        auto entry = need(path);
+        auto entry = need(state, path);
 
         if (git_tree_entry_type(entry) != GIT_OBJECT_BLOB)
             notExpected();
@@ -897,7 +926,7 @@ struct GitSourceAccessor : SourceAccessor
         }
 
         Blob blob;
-        if (git_tree_entry_to_object((git_object * *) (git_blob * *) Setter(blob), *repo, entry))
+        if (git_tree_entry_to_object((git_object * *) (git_blob * *) Setter(blob), *state.repo, entry))
             throw Error("looking up file '%s': %s", showPath(path), git_error_last()->message);
 
         return blob;

src/libfetchers/git.cc

@@ -481,11 +481,11 @@ struct GitInputScheme : InputScheme
         return repoInfo;
     }
 
-    uint64_t getLastModified(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
+    uint64_t getLastModified(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
     {
         Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}};
 
-        auto cache = getCache();
+        auto cache = settings.getCache();
 
         if (auto res = cache->lookup(key))
             return getIntAttr(*res, "lastModified");
@@ -497,11 +497,11 @@ struct GitInputScheme : InputScheme
         return lastModified;
     }
 
-    uint64_t getRevCount(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
+    uint64_t getRevCount(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
     {
         Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}};
 
-        auto cache = getCache();
+        auto cache = settings.getCache();
 
         if (auto revCountAttrs = cache->lookup(key))
             return getIntAttr(*revCountAttrs, "revCount");
@@ -679,12 +679,12 @@ struct GitInputScheme : InputScheme
 
         Attrs infoAttrs({
             {"rev", rev.gitRev()},
-            {"lastModified", getLastModified(repoInfo, repoDir, rev)},
+            {"lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)},
         });
 
         if (!getShallowAttr(input))
             infoAttrs.insert_or_assign("revCount",
-                getRevCount(repoInfo, repoDir, rev));
+                getRevCount(*input.settings, repoInfo, repoDir, rev));
 
         printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
 
@@ -799,8 +799,10 @@ struct GitInputScheme : InputScheme
             auto rev = repoInfo.workdirInfo.headRev.value_or(nullRev);
 
             input.attrs.insert_or_assign("rev", rev.gitRev());
-            input.attrs.insert_or_assign("revCount",
+            if (!getShallowAttr(input)) {
-                rev == nullRev ? 0 : getRevCount(repoInfo, repoPath, rev));
+                input.attrs.insert_or_assign("revCount",
+                    rev == nullRev ? 0 : getRevCount(*input.settings, repoInfo, repoPath, rev));
+            }
 
             verifyCommit(input, repo);
         } else {
@@ -819,7 +821,7 @@ struct GitInputScheme : InputScheme
         input.attrs.insert_or_assign(
             "lastModified",
             repoInfo.workdirInfo.headRev
-            ? getLastModified(repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
+            ? getLastModified(*input.settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
             : 0);
 
         return {accessor, std::move(input)};

src/libfetchers/github.cc

@@ -175,7 +175,7 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
-    // Search for the longest possible match starting from the begining and ending at either the end or a path segment.
+    // Search for the longest possible match starting from the beginning and ending at either the end or a path segment.
     std::optional<std::string> getAccessToken(const fetchers::Settings & settings, const std::string & host, const std::string & url) const override
     {
         auto tokens = settings.accessTokens.get();
@@ -265,7 +265,7 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.erase("ref");
         input.attrs.insert_or_assign("rev", rev->gitRev());
 
-        auto cache = getCache();
+        auto cache = input.settings->getCache();
 
         Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}};
         Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}};
@@ -409,7 +409,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
         auto json = nlohmann::json::parse(
             readFile(
                 store->toRealPath(
-                    downloadFile(store, url, "source", headers).storePath)));
+                    downloadFile(store, *input.settings, url, "source", headers).storePath)));
 
         return RefInfo {
             .rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1),
@@ -483,7 +483,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         auto json = nlohmann::json::parse(
             readFile(
                 store->toRealPath(
-                    downloadFile(store, url, "source", headers).storePath)));
+                    downloadFile(store, *input.settings, url, "source", headers).storePath)));
 
         if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
           return RefInfo {
@@ -553,7 +553,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         std::string refUri;
         if (ref == "HEAD") {
             auto file = store->toRealPath(
-                downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath);
+                downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers).storePath);
             std::ifstream is(file);
             std::string line;
             getline(is, line);
@@ -569,7 +569,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         std::regex refRegex(refUri);
 
         auto file = store->toRealPath(
-            downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath);
+            downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers).storePath);
         std::ifstream is(file);
 
         std::string line;

src/libfetchers/include/nix/fetchers/attrs.hh

@@ -37,7 +37,7 @@ std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & na
 
 bool getBoolAttr(const Attrs & attrs, const std::string & name);
 
-std::map<std::string, std::string> attrsToQuery(const Attrs & attrs);
+StringMap attrsToQuery(const Attrs & attrs);
 
 Hash getRevAttr(const Attrs & attrs, const std::string & name);
 

src/libfetchers/include/nix/fetchers/cache.hh

@@ -92,6 +92,4 @@ struct Cache
         Store & store) = 0;
 };
 
-ref<Cache> getCache();
-
 }

src/libfetchers/include/nix/fetchers/fetch-settings.hh

@@ -3,6 +3,8 @@
 
 #include "nix/util/types.hh"
 #include "nix/util/configuration.hh"
+#include "nix/util/ref.hh"
+#include "nix/util/sync.hh"
 
 #include <map>
 #include <limits>
@@ -11,6 +13,8 @@
 
 namespace nix::fetchers {
 
+struct Cache;
+
 struct Settings : public Config
 {
     Settings();
@@ -106,6 +110,11 @@ struct Settings : public Config
 
           When empty, disables the global flake registry.
         )"};
+
+    ref<Cache> getCache() const;
+
+private:
+    mutable Sync<std::shared_ptr<Cache>> _cache;
 };
 
 }

src/libfetchers/include/nix/fetchers/fetch-to-store.hh

@@ -15,6 +15,7 @@ enum struct FetchMode { DryRun, Copy };
  * Copy the `path` to the Nix store.
  */
 StorePath fetchToStore(
+    const fetchers::Settings & settings,
     Store & store,
     const SourcePath & path,
     FetchMode mode,
@@ -24,6 +25,7 @@ StorePath fetchToStore(
     RepairFlag repair = NoRepair);
 
 std::pair<StorePath, Hash> fetchToStore2(
+    const fetchers::Settings & settings,
     Store & store,
     const SourcePath & path,
     FetchMode mode,

src/libfetchers/include/nix/fetchers/fetchers.hh

@@ -71,7 +71,7 @@ public:
 
     ParsedURL toURL() const;
 
-    std::string toURLString(const std::map<std::string, std::string> & extraQuery = {}) const;
+    std::string toURLString(const StringMap & extraQuery = {}) const;
 
     std::string to_string() const;
 

src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh

@@ -1,3 +1,6 @@
+#pragma once
+///@file
+
 #include "nix/util/canon-path.hh"
 #include "nix/util/serialise.hh"
 #include "nix/util/url.hh"

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -5,7 +5,7 @@
 
 namespace nix {
 
-namespace fetchers { struct PublicKey; }
+namespace fetchers { struct PublicKey; struct Settings; }
 
 /**
  * A sink that writes into a Git repository. Note that nothing may be written
@@ -115,7 +115,7 @@ struct GitRepo
      * Given a Git tree hash, compute the hash of its NAR
      * serialisation. This is memoised on-disk.
      */
-    virtual Hash treeHashToNarHash(const Hash & treeHash) = 0;
+    virtual Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) = 0;
 
     /**
      * If the specified Git object is a directory with a single entry

src/libfetchers/include/nix/fetchers/tarball.hh

@@ -26,6 +26,7 @@ struct DownloadFileResult
 
 DownloadFileResult downloadFile(
     ref<Store> store,
+    const Settings & settings,
     const std::string & url,
     const std::string & name,
     const Headers & headers = {});

src/libfetchers/mercurial.cc

@@ -253,13 +253,13 @@ struct MercurialInputScheme : InputScheme
         }};
 
         if (!input.getRev()) {
-            if (auto res = getCache()->lookupWithTTL(refToRevKey))
+            if (auto res = input.settings->getCache()->lookupWithTTL(refToRevKey))
                 input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev());
         }
 
         /* If we have a rev, check if we have a cached store path. */
         if (auto rev = input.getRev()) {
-            if (auto res = getCache()->lookupStorePath(revInfoKey(*rev), *store))
+            if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(*rev), *store))
                 return makeResult(res->value, res->storePath);
         }
 
@@ -309,7 +309,7 @@ struct MercurialInputScheme : InputScheme
 
         /* Now that we have the rev, check the cache again for a
            cached store path. */
-        if (auto res = getCache()->lookupStorePath(revInfoKey(rev), *store))
+        if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(rev), *store))
             return makeResult(res->value, res->storePath);
 
         Path tmpDir = createTempDir();
@@ -326,9 +326,9 @@ struct MercurialInputScheme : InputScheme
         });
 
         if (!origRev)
-            getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
+            input.settings->getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
 
-        getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
+        input.settings->getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
 
         return makeResult(infoAttrs, std::move(storePath));
     }

src/libfetchers/path.cc

@@ -4,6 +4,7 @@
 #include "nix/fetchers/store-path-accessor.hh"
 #include "nix/fetchers/cache.hh"
 #include "nix/fetchers/fetch-to-store.hh"
+#include "nix/fetchers/fetch-settings.hh"
 
 namespace nix::fetchers {
 
@@ -149,7 +150,7 @@ struct PathInputScheme : InputScheme
         // store, pre-create an entry in the fetcher cache.
         auto info = store->queryPathInfo(*storePath);
         accessor->fingerprint = fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
-        fetchers::getCache()->upsert(
+        input.settings->getCache()->upsert(
             makeSourcePathToHashCacheKey(*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
             {{"hash", info->narHash.to_string(HashFormat::SRI, true)}});
 

src/libfetchers/registry.cc

@@ -156,7 +156,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, re
         }
 
         if (!isAbsolute(path)) {
-            auto storePath = downloadFile(store, path, "flake-registry.json").storePath;
+            auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
                 store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
             path = store->toRealPath(storePath);